gstack/test/review-finalization-budget.test.ts

/** The real review registrations must finish capture cleanup before Bun retries. */
import { expect, test } from 'bun:test';
import * as fs from 'node:fs';
import * as os from 'node:os';
import * as path from 'node:path';
import { buildPaidShardArgs, retriesForFiles, resolvePaidShardTimeoutMs } from '../scripts/test-paid-shards';
const ROOT = path.resolve(import.meta.dir, '..');
const PAID_FILE = 'test/skill-e2e-review.test.ts';
const CASES = [
  ['review-sql-injection', 400, 20],
  ['review-enum-completeness', 300, 15],
  ['review-design-lite', 400, 35],
] as const;
for (const [id, workMs, maxTurns] of CASES) {
  test.each(['recover', 'both-timeout'])(`${id} records late results before retry or finalization: %s`, scenario => {
    const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'review-finalization-'));
    const script = path.join(dir, 'registration.test.ts');
    const facts = path.join(dir, 'events.jsonl');
    fs.writeFileSync(script, `
import { describe, expect, mock, test } from 'bun:test';
import * as fs from 'node:fs';
import * as path from 'node:path';
const root = ${JSON.stringify(ROOT)}, selected = ${JSON.stringify(id)};
let attempts = 0;
const event = value => fs.appendFileSync(${JSON.stringify(facts)}, JSON.stringify(value) + '\\n');
mock.module(path.join(root, 'test/helpers/eval-budgets.ts'), () => ({ JUDGE_MS: 300, CAPTURE_MS: 400 }));
mock.module(path.join(root, 'test/helpers/e2e-helpers.ts'), () => ({
  ROOT: root, browseBin: '', runId: 'free-review-finalization', evalsEnabled: true, selectedTests: [selected],
  describeIfSelected: (name, ids, body) => { if (ids.includes(selected)) describe(name, body); },
  testConcurrentIfSelected: (name, body, outerMs) => {
    if (name !== selected) return;
    event({ kind: 'registration', name, outerMs });
    test.concurrent(name, body, outerMs);
  },
  copyDirSync: () => {}, setupBrowseShims: () => {}, logCost: () => {}, createEvalCollector: () => null,
  recordE2E: (_collector, name, suite, result) => event({ kind: 'record', id: result.attemptId, exitReason: result.exitReason, name, suite }),
  finalizeEvalCollector: async () => event({ kind: 'finalized' }),
}));
mock.module(path.join(root, 'test/helpers/session-runner.ts'), () => ({
  SESSION_DRAIN_GRACE_MS: 50,
  runSkillTest: async opts => {
    const id = ++attempts;
    event({ kind: 'start', id, timeout: opts.timeout, maxTurns: opts.maxTurns, cwd: opts.workingDirectory });
    const timeout = id === 1 || ${JSON.stringify(scenario)} === 'both-timeout';
    // Use the caller's actual work budget; only the provider and budget
    // constants are scaled. The actual registered Bun outer deadline stays.
    await new Promise(resolve => setTimeout(resolve, timeout ? opts.timeout + 50 : 80));
    event({ kind: 'ready', id, fixtureExists: fs.existsSync(opts.workingDirectory) });
    if (!timeout) {
      const target = selected === 'review-enum-completeness'
        ? opts.prompt.match(/Write your review findings once to (\\S+)/)[1]
        : path.join(opts.workingDirectory, 'review-output.md');
      fs.writeFileSync(target,
        'SQL injection. Returned enum status critical. Papyrus font family;14px font-size;outline focus;!important;purple gradient;generic hero copy;3-column feature grid;impeccable detector [ai-color-palette].');
    }
    return { attemptId: id, exitReason: timeout ? 'timeout' : 'success', duration: opts.timeout,
      model: 'free-fixture-model', toolCalls: [], browseErrors: [], output: '', transcript: [],
      costEstimate: { estimatedCost: 0, estimatedTokens: 0, turnsUsed: 0 } };
  },
}));
await import(path.join(root, ${JSON.stringify(PAID_FILE)}));
`);
    try {
      const retries = retriesForFiles([PAID_FILE]);
      expect(retries).toBe(1);
      const child = Bun.spawnSync([process.execPath, ...buildPaidShardArgs([script], resolvePaidShardTimeoutMs([PAID_FILE]), 2, retries)], {
        cwd: ROOT, timeout: 15_000, stdout: 'pipe', stderr: 'pipe',
        env: { ...process.env, EVALS: '', EVALS_ALL: '', TMPDIR: dir, TMP: dir, TEMP: dir },
      });
      const output = child.stdout.toString() + child.stderr.toString();
      expect(child.exitCode, output).toBe(scenario === 'recover' ? 0 : 1);
      expect(output).not.toContain('Unhandled error between tests');
      const events = fs.readFileSync(facts, 'utf8').trim().split('\n').map(line => JSON.parse(line));
      const starts = events.filter(event => event.kind === 'start');
      const ready = events.filter(event => event.kind === 'ready');
      const records = events.filter(event => event.kind === 'record');
      expect(starts.map(event => event.id)).toEqual([1, 2]);
      expect(starts.map(({ timeout, maxTurns }) => ({ timeout, maxTurns })))
        .toEqual([{ timeout: workMs, maxTurns }, { timeout: workMs, maxTurns }]);
      expect(ready).toEqual([{ kind: 'ready', id: 1, fixtureExists: true }, { kind: 'ready', id: 2, fixtureExists: true }]);
      expect(records.map(event => [event.id, event.exitReason]))
        .toEqual([[1, 'timeout'], [2, scenario === 'recover' ? 'success' : 'timeout']]);
      expect(events.findIndex(event => event.kind === 'record' && event.id === 1))
        .toBeLessThan(events.findIndex(event => event.kind === 'start' && event.id === 2));
      expect(events.findIndex(event => event.kind === 'record' && event.id === 2))
        .toBeLessThan(events.findIndex(event => event.kind === 'finalized'));
      expect(events.filter(event => event.kind === 'finalized')).toHaveLength(1);
      expect(events.find(event => event.kind === 'registration')).toEqual({ kind: 'registration', name: id, outerMs: workMs + 50 + 5_000 });
      for (const attempt of starts) expect(fs.existsSync(attempt.cwd)).toBe(false);
    } finally { fs.rmSync(dir, { recursive: true, force: true }); }
  });
}