mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-16 13:21:30 +01:00
Make /eval-plan check judges, zeros, failure counts and decisions before results count (GST-2754) (#4384)
* Make /eval-plan check judges, zeros, failure counts and decisions before results count (GST-2754) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * Check zeros in every results table, apply Decisions and Scoring with stack: none, count only valid judge labels (GST-2754) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * Describe the /eval-plan result checks in the skills guide (GST-2754) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --------- Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
This commit is contained in:
1 parent
2c0e07886b
commit
fdf3f1e1b6
21 files changed
+643
-75
No files matched your search
@@ -20,7 +20,8 @@ const dirs: string[] = [];
|
||||
const tmp = () => { const d = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-no-gstack-arm-')); dirs.push(d); return d; };
|
||||
afterEach(() => { while (dirs.length) fs.rmSync(dirs.pop()!, { recursive: true, force: true }); });
|
||||
|
||||
const BASE = ['Bars', 'Decoys', 'Stop rules', 'Held-out exposure'].map(s => `## ${s}\n- filled in\n`).join('\n');
|
||||
const RESULT_SECTIONS = '## Decisions\nif the variant passes more items: adopt it\nif there is no difference: keep the default\n\n## Scoring\njudge: none\npaired: yes\ninterval: 95%\nmdd: 2 of 9 items\n';
|
||||
const BASE = ['Bars', 'Decoys', 'Stop rules', 'Held-out exposure'].map(s => `## ${s}\n- filled in\n`).join('\n') + '\n' + RESULT_SECTIONS;
|
||||
const ARMS = (...ids: string[]) => `\n## Arms\n${ids.map(id => `- ${id} default settings, $0.01 per item`).join('\n')}\n`;
|
||||
const LOADED = '\n## Comparison\nstack: loaded\n';
|
||||
const NO_GSTACK = (o: Partial<Record<'arms' | 'inputs' | 'prompt' | 'panel' | 'report' | 'masking' | 'install' | 'audit', string>> = {}) => {
|
||||
@@ -189,10 +190,10 @@ describe('gstack-eval-plan prereg --check', () => {
|
||||
return { code: r.status, out: r.stdout ?? '', err: r.stderr ?? '' };
|
||||
};
|
||||
|
||||
test('a loaded comparison without the arm: one PREREG line out of 6, PREREG_NO_GSTACK_ARM on stderr, exit 1', () => {
|
||||
test('a loaded comparison without the arm: one PREREG line out of 8, PREREG_NO_GSTACK_ARM on stderr, exit 1', () => {
|
||||
const r = run(BASE + VARIANTS + LOADED);
|
||||
expect(r.code).toBe(1);
|
||||
expect(r.out).toMatch(/^PREREG: incomplete present=6\/6 stack=loaded no_gstack_arm=missing file=/m);
|
||||
expect(r.out).toMatch(/^PREREG: incomplete present=8\/8 stack=loaded no_gstack_arm=missing file=/m);
|
||||
expect(r.err).toContain('missing section');
|
||||
expect(r.err).toContain('(PREREG_NO_GSTACK_ARM)');
|
||||
expect(r.err).not.toContain('(PREREG_INCOMPLETE)');
|
||||
@@ -216,9 +217,9 @@ describe('gstack-eval-plan prereg --check', () => {
|
||||
test('complete: PREREG: ok with stack=loaded no_gstack_arm=ok; an opt-out prints stack=none', () => {
|
||||
const ok = run(BASE + VARIANTS + LOADED + NO_GSTACK());
|
||||
expect(ok.code, ok.out + ok.err).toBe(0);
|
||||
expect(ok.out).toMatch(/^PREREG: ok present=6\/6 stack=loaded no_gstack_arm=ok file=/m);
|
||||
expect(ok.out).toMatch(/^PREREG: ok present=8\/8 stack=loaded no_gstack_arm=ok file=/m);
|
||||
const none = run(BASE + ARMS('claude-opus-5-5') + '\n## Comparison\nstack: none\nwhy: scorer calibration\n');
|
||||
expect(none.code, none.out + none.err).toBe(0);
|
||||
expect(none.out).toMatch(/^PREREG: ok present=6\/6 stack=none file=/m);
|
||||
expect(none.out).toMatch(/^PREREG: ok present=8\/8 stack=none file=/m);
|
||||
});
|
||||
});
|
||||
@@ -14,7 +14,7 @@ import { checkPrereg, parsePreregDiagnostics, preregTemplate } from '../lib/eval
|
||||
import { capyHost, type CapyHost } from './helpers/capy-host';
|
||||
|
||||
const BIN = path.resolve(import.meta.dir, '..', 'bin', 'gstack-eval-plan');
|
||||
const FILLED = ['Bars', 'Arms', 'Decoys', 'Stop rules', 'Held-out exposure'].map(s => `## ${s}\n- filled in\n`).join('\n') + '\n## Comparison\nstack: none\nwhy: one arm, no skill stack\n';
|
||||
const FILLED = ['Bars', 'Arms', 'Decoys', 'Stop rules', 'Held-out exposure'].map(s => `## ${s}\n- filled in\n`).join('\n') + '\n## Comparison\nstack: none\nwhy: one arm, no skill stack\n\n## Decisions\nif the bar is met: run the full set\nif the bar is missed: fix the scorer first\n\n## Scoring\njudge: none\n';
|
||||
const SUMMARIZER = [
|
||||
'touch ran.marker',
|
||||
'r=$(grep -c \'"kind":"refusal"\' "$1")',
|
||||
@@ -54,7 +54,7 @@ describe('prereg --check: the Diagnostics section', () => {
|
||||
const r = check(FILLED + diagnostics([['refusals', '2'], ['truncated', '1']]));
|
||||
expect(r.code).toBe(3);
|
||||
expect(r.ran).toBe(false);
|
||||
expect(r.out).toContain('PREREG: ok present=6/6 stack=none diagnostics=2 file=');
|
||||
expect(r.out).toContain('PREREG: ok present=8/8 stack=none diagnostics=2 file=');
|
||||
expect(r.out).toContain(`PREREG_COMMAND: summarizer="sh summarize.sh {fixture}" fixture=fixtures/synthetic records.jsonl cwd=${fs.realpathSync(repo)}`);
|
||||
expect(r.err).toContain('(REPO_COMMANDS_NOT_ALLOWED)');
|
||||
expect(r.out).not.toContain('PREREG_DIAGNOSTICS:');
|
||||
@@ -104,7 +104,7 @@ describe('prereg --check: the Diagnostics section', () => {
|
||||
const r = check(FILLED + diagnostics([['refusals', 'two']], { summarizer: 'sh summarize.sh fixtures/x.jsonl' }), '--allow-repo-commands');
|
||||
expect(r.code).toBe(1);
|
||||
expect(r.ran).toBe(false);
|
||||
expect(r.out).toContain('PREREG: incomplete present=6/6 stack=none diagnostics=invalid');
|
||||
expect(r.out).toContain('PREREG: incomplete present=8/8 stack=none diagnostics=invalid');
|
||||
expect(r.err).toContain('diagnostics: row "| refusals | two |"');
|
||||
expect(r.err).toContain('diagnostics: the summarizer command never names the fixture ({fixture})');
|
||||
expect(r.err).toContain('diagnostics: no | diagnostic | expected | rows');
|
||||
@@ -114,7 +114,7 @@ describe('prereg --check: the Diagnostics section', () => {
|
||||
test('without a Diagnostics section the check is unchanged; the template mentions the section without adding a heading', () => {
|
||||
const r = check(FILLED);
|
||||
expect(r.code).toBe(0);
|
||||
expect(r.out).toBe(`PREREG: ok present=6/6 stack=none file=${path.join(repo, 'evals', 'prereg.md')}\n`);
|
||||
expect(r.out).toBe(`PREREG: ok present=8/8 stack=none file=${path.join(repo, 'evals', 'prereg.md')}\n`);
|
||||
expect(parsePreregDiagnostics(FILLED)).toBeNull();
|
||||
expect(parsePreregDiagnostics(FILLED + '\n## Diagnostics\n<!-- none -->\n')).toBeNull();
|
||||
expect(preregTemplate()).toContain('## Diagnostics');
|
||||
|
||||
@@ -0,0 +1,180 @@
|
||||
/**
|
||||
* The result checks behind /eval-plan (docs/TESTING_INTERNALS.md#result-checks):
|
||||
* a preregistration names the decision each result changes and its judge and
|
||||
* statistics before anything is spent; a model judge's numbers count only after
|
||||
* its hand labels pass; a zero comes only from a counter shown to find a planted
|
||||
* case; a failures column sits beside its answer-only reading.
|
||||
*/
|
||||
import { afterEach, describe, expect, test } from 'bun:test';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import * as fs from 'node:fs';
|
||||
import * as os from 'node:os';
|
||||
import * as path from 'node:path';
|
||||
import { checkPrereg } from '../lib/eval-plan';
|
||||
import { checkDecisions, checkJudgeLabels, checkScoring, cohensKappa, modelFamily } from '../lib/eval-plan-checks';
|
||||
import { checkResults } from '../lib/eval-plan-report';
|
||||
|
||||
const BIN = path.resolve(import.meta.dir, '..', 'bin', 'gstack-eval-plan');
|
||||
const dirs: string[] = [];
|
||||
afterEach(() => { while (dirs.length) fs.rmSync(dirs.pop()!, { recursive: true, force: true }); });
|
||||
|
||||
const DECISIONS = '## Decisions\nif the skill arm passes more items: keep the skill\nif there is no difference: cut the skill\n';
|
||||
const SCORING = (judge = 'gpt-6-astra', extra = 'labels: 40\nagreement: 0.6\npaired: yes\ninterval: 95%\nmdd: 3 of 20 items\n') => `## Scoring\njudge: ${judge}\n${extra}`;
|
||||
const prereg = (o: { decisions?: string; scoring?: string; frozen?: string; diagnostics?: string } = {}) => [
|
||||
o.frozen ? `frozen: ${o.frozen}\n` : '',
|
||||
'## Bars\n- pass rate, higher is better\n',
|
||||
o.decisions ?? DECISIONS,
|
||||
'## Arms\n- claude-opus-5-5 the skill installed\n- claude-opus-5-5#plain no skill\n',
|
||||
'## Comparison\nstack: none\nwhy: the scorer calibration compares prompts, not skill stacks\n',
|
||||
o.scoring ?? SCORING(),
|
||||
...['Decoys', 'Stop rules', 'Held-out exposure'].map(s => `## ${s}\n- filled in\n`),
|
||||
o.diagnostics ?? '',
|
||||
].join('\n');
|
||||
const labels = (n: number, disagree = 0, cls = (i: number) => (i % 2 ? 'pass' : 'fail')) =>
|
||||
Array.from({ length: n }, (_, i) => JSON.stringify({ id: `item-${i}`, human: cls(i), judge: i < disagree ? (cls(i) === 'pass' ? 'fail' : 'pass') : cls(i) })).join('\n') + '\n';
|
||||
const table = (header: string, ...rows: string[]) => [header, header.replace(/[^|]+/g, '---'), ...rows].join('\n') + '\n';
|
||||
|
||||
describe('## Decisions', () => {
|
||||
test('a passing plan names at least two results with different actions', () => {
|
||||
expect(checkPrereg(prereg())).toMatchObject({ ok: true, missing: [], empty: [] });
|
||||
expect(checkDecisions(['if a: do x', 'if b: do y']).errors).toEqual([]);
|
||||
});
|
||||
|
||||
test('missing, one result, an empty action, or the same action for every result fails', () => {
|
||||
expect(checkPrereg(prereg({ decisions: '' })).missing).toContain('Decisions');
|
||||
expect(checkDecisions(['if it passes: ship it']).errors[0]).toContain('at least two');
|
||||
expect(checkDecisions(['if it passes: ship it', 'if it fails: tbd']).errors).toEqual(['if it fails: names no action']);
|
||||
expect(checkDecisions(['if it passes: ship it', 'if it fails: Ship it']).errors[0]).toContain('changes nothing');
|
||||
expect(checkPrereg(prereg({ decisions: '## Decisions\n- we will learn a lot\n' })).ok).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('## Scoring', () => {
|
||||
const arms = ['claude-opus-5-5', 'claude-opus-5-5#plain'];
|
||||
test('a judge from another family with 40 labels, kappa 0.6 and the paired statistics passes', () => {
|
||||
expect(checkScoring(SCORING().split('\n').slice(1), arms).errors).toEqual([]);
|
||||
});
|
||||
|
||||
test('a same-family judge, too few labels, a low kappa bar or missing statistics fail', () => {
|
||||
expect(checkScoring(SCORING('claude-fable-5-2').split('\n').slice(1), arms).errors.join()).toContain('same model family (anthropic)');
|
||||
expect(checkScoring(SCORING('gpt-6-astra', 'labels: 39\nagreement: 0.6\npaired: yes\ninterval: 95%\nmdd: 3\n').split('\n').slice(1), arms).errors[0]).toContain('at least 40');
|
||||
expect(checkScoring(SCORING('gpt-6-astra', 'labels: 40\nagreement: 0.5\npaired: yes\ninterval: 95%\nmdd: 3\n').split('\n').slice(1), arms).errors[0]).toContain('kappa bar');
|
||||
expect(checkScoring(['judge: none'], arms).errors).toEqual(['paired: yes (every comparison is reported on paired items)', 'interval: 95% (the interval every comparison is reported with)', 'mdd: the smallest difference this run can detect, stated before the run']);
|
||||
expect(checkScoring(['judge: none'], ['claude-opus-5-5']).errors).toEqual([]);
|
||||
expect(checkScoring(['labels: 40'], arms).errors[0]).toContain('no judge: line');
|
||||
});
|
||||
|
||||
test('model families: labs by name, provider prefixes and loadouts dropped, else the first word', () => {
|
||||
expect(['claude-opus-5-5', 'anthropic/claude-haiku-5-5', 'claude-opus-5-5#no-gstack'].map(modelFamily)).toEqual(['anthropic', 'anthropic', 'anthropic']);
|
||||
expect(['gpt-6.1-sol', 'openai/gpt-6-astra', 'o3'].map(modelFamily)).toEqual(['openai', 'openai', 'openai']);
|
||||
expect(['gemini-4-carbon', 'grok-4.7', 'deepseek/deepseek-v4-pro', 'particle-ai/particle-1'].map(modelFamily)).toEqual(['google', 'xai', 'deepseek', 'particle']);
|
||||
});
|
||||
|
||||
test('a preregistration frozen before the result checks keeps its rules; a later freeze date does not', () => {
|
||||
const old = prereg({ decisions: '', scoring: '', frozen: '2026-10-31' });
|
||||
expect(checkPrereg(old)).toMatchObject({ ok: true, grandfathered: true, frozen: '2026-10-31', missing: [] });
|
||||
expect(checkPrereg(prereg({ decisions: '', scoring: '', frozen: '2026-11-15' })).missing).toEqual(['Decisions', 'Scoring']);
|
||||
});
|
||||
});
|
||||
|
||||
describe('judge labels', () => {
|
||||
const scoring = { judge: 'gpt-6-astra', labels: 40, agreement: 0.6, errors: [] };
|
||||
test("Cohen's kappa: perfect, chance and undefined agreement", () => {
|
||||
expect(cohensKappa([['a', 'a'], ['b', 'b']])).toBe(1);
|
||||
expect(cohensKappa([['a', 'a'], ['a', 'b'], ['b', 'a'], ['b', 'b']])).toBe(0);
|
||||
expect(cohensKappa([['a', 'a'], ['a', 'a']])).toBeNull();
|
||||
});
|
||||
|
||||
test('40 labels at or above the bar pass; fewer labels, a low kappa, one class or a repeated id fail', () => {
|
||||
expect(checkJudgeLabels(scoring, labels(40, 4))).toMatchObject({ ok: true, n: 40, kappa: 0.8 });
|
||||
expect(checkJudgeLabels(scoring, labels(39)).problems).toEqual(['39 labelled outputs, needs 40']);
|
||||
expect(checkJudgeLabels(scoring, labels(40, 10)).problems).toEqual(['kappa 0.50 is below the bar 0.6']);
|
||||
expect(checkJudgeLabels(scoring, labels(40, 0, () => 'pass')).problems[0]).toContain('one class');
|
||||
expect(checkJudgeLabels(scoring, labels(40) + labels(1)).problems).toContain('id item-0 is labelled twice');
|
||||
expect(checkJudgeLabels({ ...scoring, labels: 60 }, labels(40)).problems).toEqual(['40 labelled outputs, needs 60']);
|
||||
});
|
||||
|
||||
test('a row whose id, human or judge value is null, an object or empty does not count toward the bar', () => {
|
||||
const bad = ['{"id": "x-1", "human": null, "judge": "pass"}', '{"id": "x-2", "human": "pass", "judge": {"v": 1}}', '{"id": {}, "human": "pass", "judge": "pass"}', '{"id": "x-4", "human": "", "judge": "pass"}'];
|
||||
const r = checkJudgeLabels(scoring, labels(39) + bad.join('\n') + '\n');
|
||||
expect(r).toMatchObject({ ok: false, n: 39 });
|
||||
expect(r.problems.filter(p => p.includes('each a string, number or boolean'))).toHaveLength(4);
|
||||
});
|
||||
});
|
||||
|
||||
describe('results --check: the result checks', () => {
|
||||
const ROWS = table('| arm | passed | answer-only failures | failures |', '| claude-opus-5-5 | 7/9 | 2 | 3 |', '| claude-opus-5-5#plain | 5/9 | 4 | 4 |');
|
||||
test('a judged result counts only with passing labels', () => {
|
||||
expect(checkResults(prereg(), ROWS).issues).toEqual([{ code: 'EVAL_JUDGE_UNCHECKED', message: 'the preregistration names judge gpt-6-astra; pass --labels <hand labels.jsonl>' }]);
|
||||
expect(checkResults(prereg(), ROWS, labels(40, 10)).issues[0]!.code).toBe('EVAL_JUDGE_UNCHECKED');
|
||||
expect(checkResults(prereg(), ROWS, labels(40))).toMatchObject({ ok: true, issues: [] });
|
||||
expect(checkResults(prereg({ scoring: SCORING('none') }), ROWS).ok).toBe(true);
|
||||
});
|
||||
|
||||
test('a zero in the arm table needs a Diagnostics counter shown to find a planted case', () => {
|
||||
const zero = table('| arm | skill opens |', '| claude-opus-5-5 | 4 |', '| claude-opus-5-5#plain | 0 |');
|
||||
const diag = (n: number) => `## Diagnostics\nsummarizer: sh count.sh {fixture}\nfixture: planted.jsonl\n\n| diagnostic | expected |\n|---|---|\n| skill opens | ${n} |\n`;
|
||||
expect(checkResults(prereg({ scoring: SCORING('none') }), zero).issues.map(i => i.code)).toEqual(['EVAL_REPORT_UNPLANTED_ZERO']);
|
||||
expect(checkResults(prereg({ scoring: SCORING('none'), diagnostics: diag(0) }), zero).issues.map(i => i.code)).toEqual(['EVAL_REPORT_UNPLANTED_ZERO']);
|
||||
expect(checkResults(prereg({ scoring: SCORING('none'), diagnostics: diag(5) }), zero).ok).toBe(true);
|
||||
for (const cell of ['0%', '0/9', '0.0']) expect(checkResults(prereg({ scoring: SCORING('none') }), zero.replace('| 0 |', `| ${cell} |`)).ok, cell).toBe(false);
|
||||
expect(checkResults(prereg({ scoring: SCORING('none') }), zero.replace('| 0 |', '| 0/6/3 |')).ok).toBe(true);
|
||||
});
|
||||
|
||||
test('a zero in any table counts, not only the table that holds the arms', () => {
|
||||
const arms = table('| arm | skill opens |', '| claude-opus-5-5 | 4 |', '| claude-opus-5-5#plain | 2 |');
|
||||
const other = table('| task | timeouts |', '| t1 | 0 |');
|
||||
expect(checkResults(prereg({ scoring: SCORING('none') }), arms + '\n' + other).issues).toEqual([{ code: 'EVAL_REPORT_UNPLANTED_ZERO', message: 'column "timeouts" reports a zero, and no Diagnostics counter by that name was shown to find a planted case' }]);
|
||||
});
|
||||
|
||||
test('with stack: none, Decisions and Scoring errors still refuse the results', () => {
|
||||
const r = checkResults(prereg({ scoring: SCORING('claude-fable-5-2') }), ROWS, labels(40));
|
||||
expect(r.ok).toBe(false);
|
||||
expect(r.issues.filter(i => i.code === 'PREREG_INCOMPLETE').map(i => i.message)).toEqual(['## Scoring: judge claude-fable-5-2 is the same model family (anthropic) as arm claude-opus-5-5; use a judge from another family', '## Scoring: judge claude-fable-5-2 is the same model family (anthropic) as arm claude-opus-5-5#plain; use a judge from another family']);
|
||||
expect(checkResults(prereg({ decisions: '## Decisions\n- we will learn a lot\n' }), ROWS, labels(40)).issues.map(i => i.code)).toContain('PREREG_INCOMPLETE');
|
||||
});
|
||||
|
||||
test('a failures column needs an answer-only column in the same table', () => {
|
||||
const bare = table('| arm | failures |', '| claude-opus-5-5 | 3 |', '| claude-opus-5-5#plain | 4 |');
|
||||
expect(checkResults(prereg({ scoring: SCORING('none') }), bare).issues).toEqual([{ code: 'EVAL_REPORT_NO_ANSWER_ONLY', message: 'column "failures" counts failures with no answer-only column beside it in the same table' }]);
|
||||
expect(checkResults(prereg({ scoring: SCORING('none') }), ROWS).ok).toBe(true);
|
||||
});
|
||||
|
||||
test('a grandfathered preregistration skips the result checks', () => {
|
||||
const bare = table('| arm | failures |', '| claude-opus-5-5 | 0 |', '| claude-opus-5-5#plain | 4 |');
|
||||
expect(checkResults(prereg({ decisions: '', scoring: '', frozen: '2026-10-31' }), bare).ok).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('gstack-eval-plan judge --check and results --labels', () => {
|
||||
const run = (args: string[], files: Record<string, string>) => {
|
||||
const d = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-result-checks-'));
|
||||
dirs.push(d);
|
||||
for (const [name, text] of Object.entries(files)) fs.writeFileSync(path.join(d, name), text);
|
||||
const r = spawnSync(BIN, args, { cwd: d, encoding: 'utf8', timeout: 60_000 });
|
||||
return { code: r.status, out: r.stdout ?? '', err: r.stderr ?? '' };
|
||||
};
|
||||
|
||||
test('judge --check prints JUDGE_AGREEMENT and exits 1 with EVAL_JUDGE_UNCHECKED below the bar', () => {
|
||||
const ok = run(['judge', '--check', 'prereg.md', '--labels', 'labels.jsonl'], { 'prereg.md': prereg(), 'labels.jsonl': labels(40, 4) });
|
||||
expect(ok.code, ok.err).toBe(0);
|
||||
expect(ok.out).toBe('JUDGE_AGREEMENT: ok judge=gpt-6-astra n=40/40 kappa=0.80 bar=0.6\n');
|
||||
const low = run(['judge', '--check', 'prereg.md', '--labels', 'labels.jsonl'], { 'prereg.md': prereg(), 'labels.jsonl': labels(20) });
|
||||
expect(low.code).toBe(1);
|
||||
expect(low.out).toBe('JUDGE_AGREEMENT: unchecked judge=gpt-6-astra n=20/40 kappa=1.00 bar=0.6\n');
|
||||
expect(low.err).toContain('(EVAL_JUDGE_UNCHECKED)');
|
||||
expect(run(['judge', '--check', 'prereg.md', '--labels', 'labels.jsonl'], { 'prereg.md': prereg({ scoring: SCORING('none') }), 'labels.jsonl': labels(40) }).code).toBe(2);
|
||||
});
|
||||
|
||||
test('results --check takes --labels before or after the results file and prints one code per kind', () => {
|
||||
const files = { 'prereg.md': prereg(), 'results.md': table('| arm | failures |', '| claude-opus-5-5 | 3 |', '| claude-opus-5-5#plain | 0 |'), 'labels.jsonl': labels(40) };
|
||||
const r = run(['results', '--check', 'prereg.md', '--labels', 'labels.jsonl', 'results.md'], files);
|
||||
expect(r.code).toBe(1);
|
||||
expect(r.out).toBe('RESULTS: incomplete no_gstack_rows=0/0 rows=0/0 file=results.md\nJUDGE_AGREEMENT: ok judge=gpt-6-astra n=40/40 kappa=1.00 bar=0.6\n');
|
||||
expect(r.err).toContain('(EVAL_REPORT_UNPLANTED_ZERO)');
|
||||
expect(r.err).toContain('(EVAL_REPORT_NO_ANSWER_ONLY)');
|
||||
expect(r.err).not.toContain('(EVAL_REPORT_NO_GSTACK_ROW)');
|
||||
const noLabels = run(['results', '--check', 'prereg.md', 'results.md'], files);
|
||||
expect(noLabels.err).toContain('(EVAL_JUDGE_UNCHECKED)');
|
||||
});
|
||||
});
|
||||
@@ -14,7 +14,8 @@ const BIN = path.resolve(import.meta.dir, '..', 'bin', 'gstack-eval-plan');
|
||||
const dirs: string[] = [];
|
||||
afterEach(() => { while (dirs.length) fs.rmSync(dirs.pop()!, { recursive: true, force: true }); });
|
||||
|
||||
const BASE = ['Bars', 'Decoys', 'Stop rules', 'Held-out exposure'].map(s => `## ${s}\n- filled in\n`).join('\n');
|
||||
const RESULT_SECTIONS = '## Decisions\nif the variant passes more items: adopt it\nif there is no difference: keep the default\n\n## Scoring\njudge: none\npaired: yes\ninterval: 95%\nmdd: 2 of 9 items\n';
|
||||
const BASE = ['Bars', 'Decoys', 'Stop rules', 'Held-out exposure'].map(s => `## ${s}\n- filled in\n`).join('\n') + '\n' + RESULT_SECTIONS;
|
||||
const PREREG = BASE + `
|
||||
## Arms
|
||||
- claude-opus-5-5#review-g default settings
|
||||
|
||||
@@ -327,6 +327,7 @@ describe('parity with the measured prototype', () => {
|
||||
const REVIEWED_SUPERSET: Record<string, string> = {
|
||||
setup: './setup, $ROOT/setup and join(..., "setup") exec edges (plan A2-2); setup is a full-gate file, so the closure stops there',
|
||||
'lib/bun-isolation.ts': 'GSTA-56: every bun launcher (script-invocation, restamp, qa-deadline, the hooks) imports it, so it reaches every consumer of those launchers',
|
||||
'lib/eval-plan-checks.ts': 'lib/eval-plan.ts and bin/gstack-eval-plan import it (the result checks), so it reaches every consumer of the eval planner',
|
||||
'lib/ship-receipt-docs.ts': 'bin/gstack-ship-receipt imports it (the docs audit record and doc caps), so it reaches every consumer of the receipt binary',
|
||||
'lib/ship-receipt-prose.ts': 'bin/gstack-ship-receipt imports it (read --check-prose), so it reaches every consumer of the receipt binary',
|
||||
'lib/default-base.ts': 'GST-511/GST-774: lib/ship-policy.ts (resolveBaseBranch), lib/gate-diff/git.ts, lib/pregate/context.ts, lib/ship-receipt-depth.ts and the review-plan, gate-diff, pregate, docs-candidate and diff-scope bins import or run it, so it reaches every consumer of those readers',
|
||||
|
||||
+2
@@ -2,6 +2,8 @@
|
||||
|
||||
Frozen on 2026-10-31 before the first paid run of this design: three with/without pairs per task, a no-gstack arm on the pinned model, a spend ledger, and a paired judge that decides only after calibration. The instrument gates nothing; its assertions cover harness mechanics only.
|
||||
|
||||
frozen: 2026-10-31
|
||||
|
||||
## Bars
|
||||
- Metric order per arm: functional checks pass rate (tasks with an oracle), then paired over_engineering (wins/ties/losses and mean over both orders), then the per-diff armJudge mean (the no-loss diagnostic), then net LOC and tokens.
|
||||
- Decider bar, paired-judge calibration (`arm-pair`): before the paid run, `bun run scripts/judge-calibration.ts run --config arm-pair --budget 8` on the 16 paired items in test/fixtures/judge-calibration/arm-pair/corpus.json. The bar is `pairCalibrationBar` (test/helpers/arm-pair-judge.ts) on the `new` panels: all 6 crossing pairs decided in both orders for the less over-engineered diff, so 6/6 correct, 0 reversed, 0 tied, 0 incomplete. Position bias on the 4 identical-diff pairs is reported, not gated.
|
||||
|
||||
Vendored
+2
@@ -8,6 +8,8 @@ seen. Instrument: test/skill-e2e-model-panel.test.ts with
|
||||
test/helpers/model-panel-harness.ts; free selftest:
|
||||
test/model-panel-selftest.test.ts.
|
||||
|
||||
frozen: 2026-10-31
|
||||
|
||||
## Bars
|
||||
- Pass rate per arm: scored (non-INFRA) cells whose hidden deterministic check passes, with a Wilson 95% interval. Descriptive only: the panel gates nothing and no arm "wins".
|
||||
- Publication floor: an arm needs PANEL_MIN_COMPLETED = 12 scored cells before any number about it is published; below that its row says INCOMPLETE.
|
||||
|
||||
@@ -149,6 +149,7 @@ export const IDENTIFIER_PLACEHOLDERS: Record<string, IdentifierGrammar> = {
|
||||
'<plan-path>': { grammar: PATH, what: 'bound plan file path' },
|
||||
'<prereg-path>': { grammar: PATH, what: 'preregistration file path' },
|
||||
'<results-path>': { grammar: PATH, what: 'results table file path' },
|
||||
'<labels-path>': { grammar: PATH, what: 'hand-label JSONL file path' },
|
||||
'<sketch-dir>': { grammar: PATH, what: 'sketch directory printed by an earlier block' },
|
||||
'<path-to-finalized.html>': { grammar: PATH, what: 'finalized HTML path' },
|
||||
'<path-to-page.html>': { grammar: PATH, what: 'saved page HTML path' },
|
||||
|
||||
@@ -57,7 +57,9 @@ describe('preregistration', () => {
|
||||
expect(check.empty.length).toBe(PREREG_SECTIONS.length);
|
||||
expect(check.ok).toBe(false);
|
||||
expect(PREREG_SECTIONS).toContain('Comparison');
|
||||
const filled = t.replace(/^(##\s+.+)$/gm, '$1\n- filled in').replace('## Comparison\n- filled in', '## Comparison\nstack: none\nwhy: one arm, no skill stack');
|
||||
const filled = t.replace(/^(##\s+.+)$/gm, '$1\n- filled in').replace('## Comparison\n- filled in', '## Comparison\nstack: none\nwhy: one arm, no skill stack')
|
||||
.replace('## Decisions\n- filled in', '## Decisions\nif the bar is met: run the full set\nif the bar is missed: stop and fix the scorer')
|
||||
.replace('## Scoring\n- filled in', '## Scoring\njudge: none');
|
||||
expect(checkPrereg(filled)).toMatchObject({ ok: true, missing: [], empty: [] });
|
||||
expect(checkPrereg('# nothing here\n').missing).toEqual([...PREREG_SECTIONS]);
|
||||
});
|
||||
|
||||
Reference in new issue
Block a user