gstack/test/review-finalization-budget.test.ts

/** The real review registrations record late results and clean up before finalization, under the production zero-retry arguments. */
import { expect, test } from 'bun:test';
import * as fs from 'node:fs';
import * as os from 'node:os';
import * as path from 'node:path';
import { buildPaidShardArgs, retriesForFiles, resolvePaidShardTimeoutMs } from '../scripts/test-paid-shards';
const ROOT = path.resolve(import.meta.dir, '..');
const PAID_FILE = 'test/skill-e2e-review.test.ts';
const CASES = [
  ['review-sql-injection', 400, 20],
  ['review-enum-completeness', 300, 15],
  ['review-design-lite', 400, 35],
] as const;
const SYNTHETIC_REPORT = 'SQL injection. Returned enum status critical. Papyrus font family;14px font-size;outline focus;!important;purple gradient;generic hero copy;3-column feature grid;detector [low-contrast] x3.';
const DESIGN_CAPTURES = JSON.parse(fs.readFileSync(path.join(ROOT, 'test/fixtures/review-design-lite-reports-ci-36633323521.json'), 'utf8')) as {
  reports: Array<{ run: string; trial: string; scanRan: boolean; report: string }>;
};

function runRegistration(id: string, scenario: string, report: string) {
  const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'review-finalization-'));
  const script = path.join(dir, 'registration.test.ts');
  const facts = path.join(dir, 'events.jsonl');
  const evalDir = path.join(dir, 'eval');
  fs.writeFileSync(script, `
import { describe, expect, mock, test } from 'bun:test';
import * as fs from 'node:fs';
import * as path from 'node:path';
const root = ${JSON.stringify(ROOT)}, selected = ${JSON.stringify(id)};
let attempts = 0;
const event = value => fs.appendFileSync(${JSON.stringify(facts)}, JSON.stringify(value) + '\\n');
mock.module(path.join(root, 'test/helpers/eval-budgets.ts'), () => ({ JUDGE_MS: 300, CAPTURE_MS: 400 }));
mock.module(path.join(root, 'test/helpers/e2e-helpers.ts'), () => ({
  ROOT: root, browseBin: '', runId: 'free-review-finalization', evalsEnabled: true, selectedTests: [selected],
  describeIfSelected: (name, ids, body) => { if (ids.includes(selected)) describe(name, body); },
  testConcurrentIfSelected: (name, body, outerMs) => {
    if (name !== selected) return;
    event({ kind: 'registration', name, outerMs });
    test.concurrent(name, body, outerMs);
  },
  copyDirSync: () => {}, setupBrowseShims: () => {}, logCost: () => {}, createEvalCollector: () => null,
  recordE2E: (_collector, name, suite, result) => event({ kind: 'record', id: result.attemptId, exitReason: result.exitReason, name, suite }),
  finalizeEvalCollector: async () => event({ kind: 'finalized' }),
}));
mock.module(path.join(root, 'test/helpers/session-runner.ts'), () => ({
  SESSION_DRAIN_GRACE_MS: 50,
  runSkillTest: async opts => {
    const id = ++attempts;
    event({ kind: 'start', id, timeout: opts.timeout, maxTurns: opts.maxTurns, cwd: opts.workingDirectory });
    const timeout = ${JSON.stringify(scenario)} === 'timeout';
    // Use the caller's actual work budget; only the provider and budget
    // constants are scaled. The actual registered Bun outer deadline stays.
    await new Promise(resolve => setTimeout(resolve, timeout ? opts.timeout + 50 : 80));
    event({ kind: 'ready', id, fixtureExists: fs.existsSync(opts.workingDirectory) });
    if (!timeout) {
      const target = selected === 'review-enum-completeness'
        ? opts.prompt.match(/Write your review findings once to (\\S+)/)[1]
        : path.join(opts.workingDirectory, 'review-output.md');
      fs.writeFileSync(target, ${JSON.stringify(report)});
    }
    return { attemptId: id, exitReason: timeout ? 'timeout' : 'success', duration: opts.timeout,
      model: 'free-fixture-model', toolCalls: [], browseErrors: [], output: '', transcript: [],
      costEstimate: { estimatedCost: 0, estimatedTokens: 0, turnsUsed: 0 } };
  },
}));
await import(path.join(root, ${JSON.stringify(PAID_FILE)}));
`);
  const retries = retriesForFiles([PAID_FILE]);
  expect(retries).toBe(0);
  const child = Bun.spawnSync([process.execPath, ...buildPaidShardArgs([script], resolvePaidShardTimeoutMs([PAID_FILE]), 2, retries)], {
    cwd: ROOT, timeout: 15_000, stdout: 'pipe', stderr: 'pipe',
    env: { ...process.env, EVALS: '', EVALS_ALL: '', TMPDIR: dir, TMP: dir, TEMP: dir, GSTACK_EVAL_DIR: evalDir },
  });
  const violations = path.join(evalDir, 'contract-violations.jsonl');
  return { dir, facts, exitCode: child.exitCode, output: child.stdout.toString() + child.stderr.toString(),
    violations: fs.existsSync(violations) ? fs.readFileSync(violations, 'utf8') : '' };
}

for (const [id, workMs, maxTurns] of CASES) {
  test.each(['success', 'timeout'])(`${id} records late results before finalization: %s`, scenario => {
    const { dir, facts, exitCode, output } = runRegistration(id, scenario, SYNTHETIC_REPORT);
    try {
      expect(exitCode, output).toBe(scenario === 'success' ? 0 : 1);
      expect(output).not.toContain('Unhandled error between tests');
      const events = fs.readFileSync(facts, 'utf8').trim().split('\n').map(line => JSON.parse(line));
      const starts = events.filter(event => event.kind === 'start');
      const ready = events.filter(event => event.kind === 'ready');
      const records = events.filter(event => event.kind === 'record');
      expect(starts.map(event => event.id)).toEqual([1]);
      expect(starts.map(({ timeout, maxTurns }) => ({ timeout, maxTurns })))
        .toEqual([{ timeout: workMs, maxTurns }]);
      expect(ready).toEqual([{ kind: 'ready', id: 1, fixtureExists: true }]);
      expect(records.map(event => [event.id, event.exitReason]))
        .toEqual([[1, scenario === 'success' ? 'success' : 'timeout']]);
      expect(events.findIndex(event => event.kind === 'record' && event.id === 1))
        .toBeLessThan(events.findIndex(event => event.kind === 'finalized'));
      expect(events.filter(event => event.kind === 'finalized')).toHaveLength(1);
      expect(events.find(event => event.kind === 'registration')).toEqual({ kind: 'registration', name: id, outerMs: workMs + 50 + 5_000 });
      for (const attempt of starts) expect(fs.existsSync(attempt.cwd)).toBe(false);
    } finally { fs.rmSync(dir, { recursive: true, force: true }); }
  });
}

test.each(DESIGN_CAPTURES.reports.map(capture => [`${capture.run} ${capture.trial}`, capture] as const))(
  'review-design-lite credits detector rows in captured report %s only when the scan ran', (_label, capture) => {
    const { dir, exitCode, output, violations } = runRegistration('review-design-lite', 'success', capture.report);
    try {
      expect(exitCode, output).toBe(capture.scanRan ? 0 : 1);
      expect(violations.includes('the review omitted the mechanical detector rows'), output).toBe(!capture.scanRan);
    } finally { fs.rmSync(dir, { recursive: true, force: true }); }
  });