gstack/test/autoplan-eval-budget.test.ts

import { expect, test } from 'bun:test';
import * as fs from 'node:fs';
import * as os from 'node:os';
import * as path from 'node:path';
import {
  buildPaidShardArgs, buildRunManifest, parseCliOptions, parseRunManifest,
  planPaidShards, resolvePaidShardBudget, retriesForFiles, runPaidShard,
  verifySliceResults, type PaidRunManifest, type SliceResult,
} from '../scripts/test-paid-shards';
import { AUTOPLAN_CHAIN_BUDGET as budget, FINDING_RETRY_BUDGETS, assertPaidTestBudget, ALL_TIERS, PTY_LONG_MS } from './helpers/eval-budgets';

test('the one specified exception fits nested supervision and both unchanged retries', () => {
  for (const ms of [budget.workMs, budget.sessionMs, budget.testMs, budget.shardMs]) {
    expect(Number.isSafeInteger(ms) && ms > 0).toBe(true);
  }
  expect(budget.workMs).toBe(4 * PTY_LONG_MS);
  expect(budget.workMs).toBeLessThan(budget.sessionMs);
  expect(budget.sessionMs).toBeLessThan(budget.testMs);
  expect(budget.testMs * (retriesForFiles([budget.file]) + 1) + budget.shardReserveMs).toBe(budget.shardMs);
  expect(budget.shardMs + budget.ciReserveMs).toBe(budget.ciJobMs);
  expect(Math.max(...Object.values(ALL_TIERS))).toBe(PTY_LONG_MS);
  expect(() => assertPaidTestBudget(budget.file, budget.testMs)).not.toThrow();
  for (const [file, ms] of [[budget.file, budget.testMs + 1], ['test/other.test.ts', budget.testMs],
    [budget.file, Infinity], [budget.file, NaN], [budget.file, -1]] as const) {
    expect(() => assertPaidTestBudget(file, ms)).toThrow('Unregistered');
  }
});

test('only Autoplan receives the default exception and it cannot inflate a packed neighbor', () => {
  expect(resolvePaidShardBudget([budget.file])).toEqual({ timeoutMs: budget.shardMs, source: 'registered', policyId: budget.id });
  expect(resolvePaidShardBudget(['test/other.test.ts'])).toEqual({ timeoutMs: 1_800_000, source: 'default', policyId: null });
  expect(() => resolvePaidShardBudget([budget.file, 'test/other.test.ts'])).toThrow('own shard');
  const shards = planPaidShards(['test/a.test.ts', budget.file, 'test/z.test.ts'], { maxFilesPerShard: 3 });
  expect(shards.find(files => files.includes(budget.file))).toEqual([budget.file]);
  expect(shards.flat().sort()).toEqual(['test/a.test.ts', budget.file, 'test/z.test.ts'].sort());
  for (const value of [NaN, Infinity, -1, 0, 1.5, 2_147_483_648]) {
    expect(() => resolvePaidShardBudget([budget.file], value)).toThrow('timer-safe');
  }
});

test('CLI and environment distinguish user limits from the ordinary default', () => {
  const implicit = parseCliOptions([], {});
  expect(implicit.timeoutMs).toBe(1_800_000);
  expect(implicit.timeoutExplicit).toBe(false);
  for (const explicit of [parseCliOptions(['--timeout', '12'], {}), parseCliOptions([], { EVALS_SHARD_TIMEOUT_MS: '12000' })]) {
    expect(explicit.timeoutExplicit).toBe(true);
    expect(resolvePaidShardBudget([budget.file], explicit.timeoutMs).timeoutMs).toBe(12_000);
  }
  expect(buildPaidShardArgs([budget.file], budget.shardMs, 2, retriesForFiles([budget.file])))
    .toContain('--timeout=' + budget.shardMs);
  expect(retriesForFiles([budget.file])).toBe(1);
  expect(() => parseCliOptions(['--autoplan-slice'], {})).toThrow('--emit-plan');
});

function planned(): PaidRunManifest {
  return buildRunManifest({ tier: 'periodic', sliceCount: 7, dedicatedAutoplanSlice: true,
    evalsAll: true, env: { EVALS_ALL: '1' } });
}

function results(manifest: PaidRunManifest): SliceResult[] {
  return Array.from({ length: manifest.sliceCount }, (_, index) => ({ version: 1, tier: manifest.tier,
    sliceIndex: index + 1, sliceCount: manifest.sliceCount,
    outcomes: manifest.entries.filter(e => e.status === 'planned' && e.slice === index + 1).map(e => ({
      files: [e.file], status: 'passed', exitCode: 0, elapsedMs: 1, executedTests: FINDING_RETRY_BUDGETS.find(b => b.file === e.file)?.cases ?? 1, skippedTests: 0,
      ...(e.budget ? { budget: e.budget } : {}),
    })),
  }));
}

test('the seventh periodic slice isolates Autoplan and retains the full ordinary census', () => {
  const manifest = planned();
  const ordinary = buildRunManifest({ tier: 'periodic', sliceCount: 6, evalsAll: true, env: { EVALS_ALL: '1' } });
  expect(manifest.entries.map(e => e.file)).toEqual(ordinary.entries.map(e => e.file));
  expect(manifest.entries.filter(e => e.slice === 7).map(e => e.file)).toEqual([budget.file]);
  expect(manifest.entries.filter(e => e.file !== budget.file && e.status === 'planned').every(e => e.slice <= 6)).toBe(true);
  expect(parseRunManifest(JSON.stringify(manifest))).toEqual(manifest);
  expect(verifySliceResults(manifest, results(manifest))).toEqual({ ok: true, problems: [] });
  for (const mutate of [
    (m: PaidRunManifest) => { m.entries = m.entries.filter(e => e.file !== budget.file); },
    (m: PaidRunManifest) => { m.entries.push(m.entries.find(e => e.file === budget.file)!); },
    (m: PaidRunManifest) => { m.entries.find(e => e.file === budget.file)!.slice = 1; },
    (m: PaidRunManifest) => { delete m.entries.find(e => e.file === budget.file)!.budget; },
    (m: PaidRunManifest) => { m.entries.find(e => e.file === budget.file)!.budget!.timeoutMs = 999; },
  ]) {
    const invalid = structuredClone(manifest); mutate(invalid);
    expect(() => parseRunManifest(JSON.stringify(invalid))).toThrow();
    expect(verifySliceResults(invalid, results(manifest)).ok).toBe(false);
  }
  expect(verifySliceResults(manifest, results(manifest).slice(0, 6)).ok).toBe(false);
  const duplicate = results(manifest); duplicate[0]!.outcomes.push(duplicate[6]!.outcomes[0]!);
  expect(verifySliceResults(manifest, duplicate).ok).toBe(false);
  for (const change of [
    (o: SliceResult['outcomes'][number]) => { o.executedTests = 0; },
    (o: SliceResult['outcomes'][number]) => { o.skippedTests = 1; },
    (o: SliceResult['outcomes'][number]) => { o.exitCode = 1; },
    (o: SliceResult['outcomes'][number]) => { o.files = ['test/other.test.ts', budget.file]; },
  ]) { const bad = results(manifest); change(bad[6]!.outcomes[0]!); expect(verifySliceResults(manifest, bad).ok).toBe(false); }
  const reordered = structuredClone(manifest);
  const entry = reordered.entries.find(e => e.file === budget.file)!;
  entry.budget = { policyId: budget.id, source: 'registered', timeoutMs: budget.shardMs };
  expect(() => parseRunManifest(JSON.stringify(reordered))).not.toThrow();
  const wrongWall = results(manifest); delete wrongWall[6]!.outcomes[0]!.budget;
  expect(verifySliceResults(manifest, wrongWall).ok).toBe(false);
  const lower = results(manifest); lower[6]!.timeoutOverrideMs = 12000;
  lower[6]!.outcomes[0]!.budget = resolvePaidShardBudget([budget.file], 12000);
  expect(verifySliceResults(manifest, lower).ok).toBe(true);
});

test('a real fake subprocess records the chosen wall and obeys an explicit shorter deadline', async () => {
  const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'autoplan-wall-'));
  try {
    const common = { jobs: 1, log: () => {}, logDir: dir,
      env: { ...process.env, GSTACK_CLAUDE_CLI_VERSION: 'fixture-no-cli' } };
    const pass = await runPaidShard([budget.file], 1, 1, { ...common,
      commandFor: () => ({ command: process.execPath, args: ['-e', 'console.log(" 1 pass\\n 0 fail\\nRan 1 tests across 1 files. [1ms]")'] }) });
    expect(pass.status).toBe('passed');
    expect(pass.budget).toEqual(resolvePaidShardBudget([budget.file]));
    const start = Date.now();
    const stopped = await runPaidShard([budget.file], 1, 1, { ...common, timeoutMs: 150,
      commandFor: () => ({ command: process.execPath, args: ['-e', 'setInterval(()=>{},1000)'] }) });
    expect(stopped.status).toBe('timed-out');
    expect(stopped.budget).toEqual(resolvePaidShardBudget([budget.file], 150));
    expect(Date.now() - start).toBeLessThan(5000);
  } finally { fs.rmSync(dir, { recursive: true, force: true }); }
}, 10_000);

// Wiring is execution policy: a planner-only final slice would silently leave
// the long case unexecuted, or a smaller job cap would preempt both attempts.
test('periodic CI allocates and executes the dedicated eighth slice inside its existing cap', () => {
  const yaml = fs.readFileSync(path.resolve(import.meta.dir, '../.github/workflows/evals-periodic.yml'), 'utf8');
  expect(yaml).toMatch(/--emit-plan[^\n]+--slices 9 --autoplan-slice/);
  const slices = yaml.split('  eval-slices:')[1]!.split('\n  report:')[0]!;
  expect(slices).toContain('slice: [1, 2, 3, 4, 5, 6, 7, 8, 9]');
  expect(slices).toContain('max-parallel: 8');
  const jobMinutes = Number(slices.match(/timeout-minutes:\s*(\d+)/)?.[1]);
  expect(Number.isFinite(jobMinutes)).toBe(true);
  expect(jobMinutes * 60_000).toBeGreaterThanOrEqual(budget.ciJobMs);
  expect(slices).toContain('EVALS_JOBS: "2"');
  expect(slices).toContain('--plan /tmp/paid-plan/manifest.json --slice ${{ matrix.slice }}');
});