gstack/test/eval-plan-results.test.ts

/**
 * GST-471: `gstack-eval-plan results --check`. A loaded preregistration's
 * results file must hold one markdown table with a filled row for every Arms
 * identity, so each no-gstack row sits beside its variants.
 */
import { afterEach, describe, expect, test } from 'bun:test';
import { spawnSync } from 'node:child_process';
import * as fs from 'node:fs';
import * as os from 'node:os';
import * as path from 'node:path';
import { checkResults } from '../lib/eval-plan-report';

const BIN = path.resolve(import.meta.dir, '..', 'bin', 'gstack-eval-plan');
const dirs: string[] = [];
afterEach(() => { while (dirs.length) fs.rmSync(dirs.pop()!, { recursive: true, force: true }); });

const BASE = ['Bars', 'Decoys', 'Stop rules', 'Held-out exposure'].map(s => `## ${s}\n- filled in\n`).join('\n');
const PREREG = BASE + `
## Arms
- claude-opus-5-5#review-g default settings
- claude-opus-5-5#no-gstack default settings

## Comparison
stack: loaded

## No-gstack arm
arms: claude-opus-5-5#no-gstack
inputs: the same rows
prompt: the task only
panel: rule, 3 trials, same judge
report: results.md, one row beside the variant
`;
const NONE = BASE + '\n## Arms\n- claude-opus-5-5 default\n\n## Comparison\nstack: none\nwhy: scorer calibration\n';
const table = (...rows: string[]) => ['| arm | passed | cost |', '|---|---|---|', ...rows].join('\n') + '\n';

describe('checkResults', () => {
  test('one table with a filled row per arm passes; backticks and the header row are handled', () => {
    const r = checkResults(PREREG, '# Results\n\n' + table('| `claude-opus-5-5#review-g` | 7/9 | $1.20 |', '| claude-opus-5-5#no-gstack | 5/9 | $0.80 |'));
    expect(r).toMatchObject({ ok: true, no_gstack: ['claude-opus-5-5#no-gstack'], missing: [] });
  });

  test('a missing no-gstack row fails naming it', () => {
    const r = checkResults(PREREG, table('| claude-opus-5-5#review-g | 7/9 | $1.20 |'));
    expect(r.ok).toBe(false);
    expect(r.problems).toEqual(['no filled row for claude-opus-5-5#no-gstack in the table that holds the other arms']);
  });

  test('placeholder cells do not count as a filled row', () => {
    for (const cell of ['', '-', '—', 'n/a', 'TBD']) {
      const r = checkResults(PREREG, table('| claude-opus-5-5#review-g | 7/9 | $1.20 |', `| claude-opus-5-5#no-gstack | ${cell} | ${cell} |`));
      expect(r.missing, JSON.stringify(cell)).toEqual(['claude-opus-5-5#no-gstack']);
    }
  });

  test('the no-gstack row must sit in the same table as its variants; prose mentions never count', () => {
    const split = table('| claude-opus-5-5#review-g | 7/9 | $1.20 |') + '\nThe claude-opus-5-5#no-gstack arm passed 5/9.\n\n' + table('| claude-opus-5-5#no-gstack | 5/9 | $0.80 |');
    expect(checkResults(PREREG, split).ok).toBe(false);
    expect(checkResults(PREREG, 'claude-opus-5-5#review-g 7/9, claude-opus-5-5#no-gstack 5/9\n').problems).toEqual(['no markdown table in the results file']);
  });

  test('identities match whole: a prefix model does not stand in for the arm', () => {
    const r = checkResults(PREREG, table('| claude-opus-5-5#review-g | 7/9 | $1.20 |', '| claude-opus-5#no-gstack | 5/9 | $0.80 |'));
    expect(r.missing).toEqual(['claude-opus-5-5#no-gstack']);
  });

  // Generated by /ship Step 7 coverage audit (PR #3708).
  // Value: protects=checkResults judges the table holding the most arms, so a complete table after a partial one passes and an aligned separator is no row; fails_when=the best-table reduce keeps the first table or alignment colons stop separator detection; why_new=existing cases use one table or two partial tables and plain --- separators; seam=none; no_claim=none beyond protects
  test('a complete table after a partial one passes; an aligned separator row is never an arm row', () => {
    const aligned = (...rows: string[]) => ['| arm | passed | cost |', '|:---|:---:|---:|', ...rows].join('\n') + '\n';
    const r = checkResults(PREREG, table('| claude-opus-5-5#review-g | 7/9 | $1.20 |') + '\nNotes.\n\n' + aligned('| claude-opus-5-5#review-g | 7/9 | $1.20 |', '| claude-opus-5-5#no-gstack | 5/9 | $0.80 |'));
    expect(r).toMatchObject({ ok: true, found: ['claude-opus-5-5#review-g', 'claude-opus-5-5#no-gstack'], missing: [] });
  });

  test('stack: none needs no row; a preregistration that fails prereg --check fails here too', () => {
    expect(checkResults(NONE, 'no table at all\n').ok).toBe(true);
    const broken = PREREG.replace(/## No-gstack arm[\s\S]*$/, '');
    const r = checkResults(broken, table('| claude-opus-5-5#review-g | 7/9 | $1.20 |', '| claude-opus-5-5#no-gstack | 5/9 | $0.80 |'));
    expect(r.problems).toEqual(['the preregistration does not pass prereg --check']);
  });
});

describe('gstack-eval-plan results --check', () => {
  const run = (prereg: string, results: string, extra: string[] = []) => {
    const d = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-results-check-'));
    dirs.push(d);
    fs.writeFileSync(path.join(d, 'prereg.md'), prereg);
    fs.writeFileSync(path.join(d, 'results.md'), results);
    const r = spawnSync(BIN, ['results', '--check', 'prereg.md', 'results.md', ...extra], { cwd: d, encoding: 'utf8', timeout: 60_000 });
    return { code: r.status, out: r.stdout ?? '', err: r.stderr ?? '' };
  };

  test('missing row: RESULTS: incomplete, EVAL_REPORT_NO_GSTACK_ROW, exit 1', () => {
    const r = run(PREREG, table('| claude-opus-5-5#review-g | 7/9 | $1.20 |'));
    expect(r.code).toBe(1);
    expect(r.out).toBe('RESULTS: incomplete no_gstack_rows=0/1 rows=1/2 file=results.md\n');
    expect(r.err).toContain('(EVAL_REPORT_NO_GSTACK_ROW)');
  });

  test('complete: RESULTS: ok, exit 0', () => {
    const r = run(PREREG, table('| claude-opus-5-5#review-g | 7/9 | $1.20 |', '| claude-opus-5-5#no-gstack | 5/9 | $0.80 |'));
    expect(r.code, r.err).toBe(0);
    expect(r.out).toBe('RESULTS: ok no_gstack_rows=1/1 rows=2/2 file=results.md\n');
  });

  // Generated by /ship Step 7 coverage audit (PR #3708).
  // Value: protects=--json prints one parseable object with the paths and the check fields, and keeps exit 1 plus the result code when a row is missing; fails_when=--json drops a field, prints the text line instead, or exits 0 on an incomplete file; why_new=the CLI tests cover only the text output; seam=none; no_claim=none beyond protects
  test('--json: one object with the paths and the check, exit 1 and the result code when a row is missing', () => {
    const r = run(PREREG, table('| claude-opus-5-5#review-g | 7/9 | $1.20 |'), ['--json']);
    expect(r.code).toBe(1);
    expect(JSON.parse(r.out)).toMatchObject({ prereg: 'prereg.md', results: 'results.md', ok: false, no_gstack: ['claude-opus-5-5#no-gstack'], found: ['claude-opus-5-5#review-g'], missing: ['claude-opus-5-5#no-gstack'] });
    expect(r.err).toContain('(EVAL_REPORT_NO_GSTACK_ROW)');
  });

  // Generated by /ship Step 7 coverage audit (PR #3708).
  // Value: protects=results with a missing results file or no --check is a usage error (exit 2) that checks nothing; fails_when=the existence guard is removed so a missing file throws or reads as an empty table; why_new=no test reaches the usage path of results; seam=none; no_claim=none beyond protects
  test('a missing results file or a missing --check is a usage error, exit 2, nothing on stdout', () => {
    const d = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-results-check-'));
    dirs.push(d);
    fs.writeFileSync(path.join(d, 'prereg.md'), PREREG);
    for (const args of [['results', '--check', 'prereg.md', 'nope.md'], ['results', 'prereg.md']]) {
      const r = spawnSync(BIN, args, { cwd: d, encoding: 'utf8', timeout: 60_000 });
      expect(r.status, args.join(' ')).toBe(2);
      expect(r.stdout).toBe('');
      expect(r.stderr).toContain('results needs --check <existing prereg> <existing results file>');
    }
  });
});