diff --git a/test/fixtures/shard-cli-parity/cases.ts b/test/fixtures/shard-cli-parity/cases.ts new file mode 100644 index 000000000..75b2a4db0 --- /dev/null +++ b/test/fixtures/shard-cli-parity/cases.ts @@ -0,0 +1,64 @@ +/** + * Argument vectors for the per-lane CLI parity check. results.json holds the + * parse results the pre-engine runners (base 96764e80) produced for them. + */ +export const FREE_CASES: string[][] = [ + [], + ['--dry-run', '--list', '--verbose'], + ['--windows-only', '--list'], + ['--record-durations'], + ['--quick'], + ['--shards', '4', '--shard', '2'], + ['--wall-timeout', '600'], + ['--ci-plan', 'plan.json', '--shards', '20'], + ['--ci-run', 'plan.json', '--shard', '1', '--result', 'result.json'], + ['--ci-verify', 'plan.json', '--results', 'results'], + ['--bogus'], + ['--list', '--bogus'], + ['--shards'], + ['--shard'], + ['--wall-timeout', '0'], + ['--wall-timeout'], + ['--ci-plan'], + ['--ci-plan', '--list'], + ['--ci-plan', 'a.json', '--ci-run', 'b.json'], + ['--ci-plan', 'a.json', '--quick'], + ['--ci-run', 'plan.json'], + ['--ci-verify', 'plan.json'], + ['--quick', '--shard', '1'], + ['--quick', '--windows-only'], +]; + +export const PAID_CASES: Array<{ argv: string[]; env?: Record }> = [ + { argv: [] }, + { argv: ['--list', '--tier', 'periodic', '--profile', 'full'] }, + { argv: ['--tier', 'gate', '--profile', 'pr', '--list'] }, + { argv: ['--timeout', '600', '--jobs', '2', '--files-per-shard', '3'] }, + { argv: ['--emit-plan', 'manifest.json', '--slices', '4', '--skip-judges'] }, + { argv: ['--plan', 'manifest.json', '--slice', '2'] }, + { argv: ['--report', 'reports', '--write-durations'] }, + { argv: [], env: { EVALS_TIER: 'periodic', EVALS_JOBS: '3', EVALS_CONCURRENCY: '5', EVALS_SHARD_TIMEOUT_MS: '1000', EVALS_PROFILE: 'full' } }, + { argv: ['--bogus'] }, + { argv: ['--list', '--bogus'] }, + { argv: ['--tier', 'e2e'] }, + { argv: ['--tier'] }, + { argv: ['--profile'] }, + { argv: ['--profile', 'fast'] }, + { argv: ['--timeout', '0'] }, + { argv: ['--jobs'] }, + { argv: ['--emit-plan'] }, + { argv: ['--plan'] }, + { argv: ['--report'] }, + { argv: ['--write-durations'] }, + { argv: ['--skip-judges'] }, + { argv: ['--profile', 'pr', '--tier', 'periodic'] }, + { argv: ['--profile', 'pr', '--files-per-shard', '2'] }, + { argv: [], env: { EVALS_TIER: 'e2e' } }, +]; + +export type ParseResult = { ok: unknown } | { error: string }; + +export function capture(parse: () => unknown): ParseResult { + try { return { ok: parse() }; } + catch (error) { return { error: error instanceof Error ? error.message : String(error) }; } +} diff --git a/test/fixtures/shard-cli-parity/results.json b/test/fixtures/shard-cli-parity/results.json new file mode 100644 index 000000000..f2d034188 --- /dev/null +++ b/test/fixtures/shard-cli-parity/results.json @@ -0,0 +1,762 @@ +{ + "recordedFrom": "96764e80 (pre-engine base runners)", + "free": [ + { + "argv": [], + "result": { + "ok": { + "dryRun": false, + "listOnly": false, + "recordDurations": false, + "windowsOnly": false, + "verbose": false, + "shardCount": 20, + "shardIndex": null, + "wallTimeoutMs": 360000, + "wallTimeoutExplicit": false, + "quick": false, + "ciPlan": null, + "ciRun": null, + "ciVerify": null, + "result": null, + "results": null + } + } + }, + { + "argv": [ + "--dry-run", + "--list", + "--verbose" + ], + "result": { + "ok": { + "dryRun": true, + "listOnly": true, + "recordDurations": false, + "windowsOnly": false, + "verbose": true, + "shardCount": 20, + "shardIndex": null, + "wallTimeoutMs": 360000, + "wallTimeoutExplicit": false, + "quick": false, + "ciPlan": null, + "ciRun": null, + "ciVerify": null, + "result": null, + "results": null + } + } + }, + { + "argv": [ + "--windows-only", + "--list" + ], + "result": { + "ok": { + "dryRun": false, + "listOnly": true, + "recordDurations": false, + "windowsOnly": true, + "verbose": false, + "shardCount": 20, + "shardIndex": null, + "wallTimeoutMs": 360000, + "wallTimeoutExplicit": false, + "quick": false, + "ciPlan": null, + "ciRun": null, + "ciVerify": null, + "result": null, + "results": null + } + } + }, + { + "argv": [ + "--record-durations" + ], + "result": { + "ok": { + "dryRun": false, + "listOnly": false, + "recordDurations": true, + "windowsOnly": false, + "verbose": false, + "shardCount": 20, + "shardIndex": null, + "wallTimeoutMs": 360000, + "wallTimeoutExplicit": false, + "quick": false, + "ciPlan": null, + "ciRun": null, + "ciVerify": null, + "result": null, + "results": null + } + } + }, + { + "argv": [ + "--quick" + ], + "result": { + "ok": { + "dryRun": false, + "listOnly": false, + "recordDurations": false, + "windowsOnly": false, + "verbose": false, + "shardCount": 20, + "shardIndex": null, + "wallTimeoutMs": 360000, + "wallTimeoutExplicit": false, + "quick": true, + "ciPlan": null, + "ciRun": null, + "ciVerify": null, + "result": null, + "results": null + } + } + }, + { + "argv": [ + "--shards", + "4", + "--shard", + "2" + ], + "result": { + "ok": { + "dryRun": false, + "listOnly": false, + "recordDurations": false, + "windowsOnly": false, + "verbose": false, + "shardCount": 4, + "shardIndex": 2, + "wallTimeoutMs": 360000, + "wallTimeoutExplicit": false, + "quick": false, + "ciPlan": null, + "ciRun": null, + "ciVerify": null, + "result": null, + "results": null + } + } + }, + { + "argv": [ + "--wall-timeout", + "600" + ], + "result": { + "ok": { + "dryRun": false, + "listOnly": false, + "recordDurations": false, + "windowsOnly": false, + "verbose": false, + "shardCount": 20, + "shardIndex": null, + "wallTimeoutMs": 600000, + "wallTimeoutExplicit": true, + "quick": false, + "ciPlan": null, + "ciRun": null, + "ciVerify": null, + "result": null, + "results": null + } + } + }, + { + "argv": [ + "--ci-plan", + "plan.json", + "--shards", + "20" + ], + "result": { + "ok": { + "dryRun": false, + "listOnly": false, + "recordDurations": false, + "windowsOnly": false, + "verbose": false, + "shardCount": 20, + "shardIndex": null, + "wallTimeoutMs": 360000, + "wallTimeoutExplicit": false, + "quick": false, + "ciPlan": "plan.json", + "ciRun": null, + "ciVerify": null, + "result": null, + "results": null + } + } + }, + { + "argv": [ + "--ci-run", + "plan.json", + "--shard", + "1", + "--result", + "result.json" + ], + "result": { + "ok": { + "dryRun": false, + "listOnly": false, + "recordDurations": false, + "windowsOnly": false, + "verbose": false, + "shardCount": 20, + "shardIndex": 1, + "wallTimeoutMs": 360000, + "wallTimeoutExplicit": false, + "quick": false, + "ciPlan": null, + "ciRun": "plan.json", + "ciVerify": null, + "result": "result.json", + "results": null + } + } + }, + { + "argv": [ + "--ci-verify", + "plan.json", + "--results", + "results" + ], + "result": { + "ok": { + "dryRun": false, + "listOnly": false, + "recordDurations": false, + "windowsOnly": false, + "verbose": false, + "shardCount": 20, + "shardIndex": null, + "wallTimeoutMs": 360000, + "wallTimeoutExplicit": false, + "quick": false, + "ciPlan": null, + "ciRun": null, + "ciVerify": "plan.json", + "result": null, + "results": "results" + } + } + }, + { + "argv": [ + "--bogus" + ], + "result": { + "error": "Unknown argument: --bogus" + } + }, + { + "argv": [ + "--list", + "--bogus" + ], + "result": { + "error": "Unknown argument: --bogus" + } + }, + { + "argv": [ + "--shards" + ], + "result": { + "error": "Missing value for --shards" + } + }, + { + "argv": [ + "--shard" + ], + "result": { + "error": "Missing value for --shard" + } + }, + { + "argv": [ + "--wall-timeout", + "0" + ], + "result": { + "error": "--wall-timeout needs a positive integer (seconds)" + } + }, + { + "argv": [ + "--wall-timeout" + ], + "result": { + "error": "--wall-timeout needs a positive integer (seconds)" + } + }, + { + "argv": [ + "--ci-plan" + ], + "result": { + "error": "Missing path for --ci-plan" + } + }, + { + "argv": [ + "--ci-plan", + "--list" + ], + "result": { + "error": "Missing path for --ci-plan" + } + }, + { + "argv": [ + "--ci-plan", + "a.json", + "--ci-run", + "b.json" + ], + "result": { + "error": "CI modes cannot be combined with other selection modes" + } + }, + { + "argv": [ + "--ci-plan", + "a.json", + "--quick" + ], + "result": { + "error": "CI modes cannot be combined with other selection modes" + } + }, + { + "argv": [ + "--ci-run", + "plan.json" + ], + "result": { + "error": "--ci-run requires --shard and --result" + } + }, + { + "argv": [ + "--ci-verify", + "plan.json" + ], + "result": { + "error": "--ci-verify requires --results" + } + }, + { + "argv": [ + "--quick", + "--shard", + "1" + ], + "result": { + "error": "--quick cannot change recording, Windows or shard selection" + } + }, + { + "argv": [ + "--quick", + "--windows-only" + ], + "result": { + "error": "--quick cannot change recording, Windows or shard selection" + } + } + ], + "paid": [ + { + "argv": [], + "result": { + "ok": { + "tier": "gate", + "profile": "full", + "profileExplicit": false, + "listOnly": false, + "skipJudges": false, + "timeoutExplicit": false, + "timeoutMs": 1800000, + "jobs": 8, + "withinShardConcurrency": 2, + "maxFilesPerShard": 1, + "emitPlanPath": null, + "slices": 1, + "planPath": null, + "sliceIndex": null, + "reportDir": null, + "writeDurations": false + } + } + }, + { + "argv": [ + "--list", + "--tier", + "periodic", + "--profile", + "full" + ], + "result": { + "ok": { + "tier": "periodic", + "profile": "full", + "profileExplicit": true, + "listOnly": true, + "skipJudges": false, + "timeoutExplicit": false, + "timeoutMs": 1800000, + "jobs": 8, + "withinShardConcurrency": 2, + "maxFilesPerShard": 1, + "emitPlanPath": null, + "slices": 1, + "planPath": null, + "sliceIndex": null, + "reportDir": null, + "writeDurations": false + } + } + }, + { + "argv": [ + "--tier", + "gate", + "--profile", + "pr", + "--list" + ], + "result": { + "ok": { + "tier": "gate", + "profile": "pr", + "profileExplicit": true, + "listOnly": true, + "skipJudges": false, + "timeoutExplicit": false, + "timeoutMs": 1800000, + "jobs": 8, + "withinShardConcurrency": 2, + "maxFilesPerShard": 1, + "emitPlanPath": null, + "slices": 1, + "planPath": null, + "sliceIndex": null, + "reportDir": null, + "writeDurations": false + } + } + }, + { + "argv": [ + "--timeout", + "600", + "--jobs", + "2", + "--files-per-shard", + "3" + ], + "result": { + "ok": { + "tier": "gate", + "profile": "full", + "profileExplicit": false, + "listOnly": false, + "skipJudges": false, + "timeoutExplicit": true, + "timeoutMs": 600000, + "jobs": 2, + "withinShardConcurrency": 2, + "maxFilesPerShard": 3, + "emitPlanPath": null, + "slices": 1, + "planPath": null, + "sliceIndex": null, + "reportDir": null, + "writeDurations": false + } + } + }, + { + "argv": [ + "--emit-plan", + "manifest.json", + "--slices", + "4", + "--skip-judges" + ], + "result": { + "ok": { + "tier": "gate", + "profile": "full", + "profileExplicit": false, + "listOnly": false, + "skipJudges": true, + "timeoutExplicit": false, + "timeoutMs": 1800000, + "jobs": 8, + "withinShardConcurrency": 2, + "maxFilesPerShard": 1, + "emitPlanPath": "manifest.json", + "slices": 4, + "planPath": null, + "sliceIndex": null, + "reportDir": null, + "writeDurations": false + } + } + }, + { + "argv": [ + "--plan", + "manifest.json", + "--slice", + "2" + ], + "result": { + "ok": { + "tier": "gate", + "profile": "full", + "profileExplicit": false, + "listOnly": false, + "skipJudges": false, + "timeoutExplicit": false, + "timeoutMs": 1800000, + "jobs": 8, + "withinShardConcurrency": 2, + "maxFilesPerShard": 1, + "emitPlanPath": null, + "slices": 1, + "planPath": "manifest.json", + "sliceIndex": 2, + "reportDir": null, + "writeDurations": false + } + } + }, + { + "argv": [ + "--report", + "reports", + "--write-durations" + ], + "result": { + "ok": { + "tier": "gate", + "profile": "full", + "profileExplicit": false, + "listOnly": false, + "skipJudges": false, + "timeoutExplicit": false, + "timeoutMs": 1800000, + "jobs": 8, + "withinShardConcurrency": 2, + "maxFilesPerShard": 1, + "emitPlanPath": null, + "slices": 1, + "planPath": null, + "sliceIndex": null, + "reportDir": "reports", + "writeDurations": true + } + } + }, + { + "argv": [], + "env": { + "EVALS_TIER": "periodic", + "EVALS_JOBS": "3", + "EVALS_CONCURRENCY": "5", + "EVALS_SHARD_TIMEOUT_MS": "1000", + "EVALS_PROFILE": "full" + }, + "result": { + "ok": { + "tier": "periodic", + "profile": "full", + "profileExplicit": true, + "listOnly": false, + "skipJudges": false, + "timeoutExplicit": true, + "timeoutMs": 1000, + "jobs": 3, + "withinShardConcurrency": 5, + "maxFilesPerShard": 1, + "emitPlanPath": null, + "slices": 1, + "planPath": null, + "sliceIndex": null, + "reportDir": null, + "writeDurations": false + } + } + }, + { + "argv": [ + "--bogus" + ], + "result": { + "error": "Unknown argument: --bogus" + } + }, + { + "argv": [ + "--list", + "--bogus" + ], + "result": { + "error": "Unknown argument: --bogus" + } + }, + { + "argv": [ + "--tier", + "e2e" + ], + "result": { + "error": "--tier must be gate or periodic. Received: e2e" + } + }, + { + "argv": [ + "--tier" + ], + "result": { + "error": "--tier must be gate or periodic. Received: undefined" + } + }, + { + "argv": [ + "--profile" + ], + "result": { + "error": "--profile needs pr or full" + } + }, + { + "argv": [ + "--profile", + "fast" + ], + "result": { + "error": "--profile must be pr or full. Received: fast" + } + }, + { + "argv": [ + "--timeout", + "0" + ], + "result": { + "error": "--timeout needs a positive integer. Received: 0" + } + }, + { + "argv": [ + "--jobs" + ], + "result": { + "error": "--jobs needs a positive integer. Received: undefined" + } + }, + { + "argv": [ + "--emit-plan" + ], + "result": { + "error": "--emit-plan needs a file path" + } + }, + { + "argv": [ + "--plan" + ], + "result": { + "error": "--plan needs a manifest path" + } + }, + { + "argv": [ + "--report" + ], + "result": { + "error": "--report needs a directory" + } + }, + { + "argv": [ + "--write-durations" + ], + "result": { + "error": "--write-durations requires --report" + } + }, + { + "argv": [ + "--skip-judges" + ], + "result": { + "error": "--skip-judges applies only to an emitted gate census plan" + } + }, + { + "argv": [ + "--profile", + "pr", + "--tier", + "periodic" + ], + "result": { + "error": "PR profile requires gate tier" + } + }, + { + "argv": [ + "--profile", + "pr", + "--files-per-shard", + "2" + ], + "result": { + "error": "PR profile requires one file per shard to preserve case accounting" + } + }, + { + "argv": [], + "env": { + "EVALS_TIER": "e2e" + }, + "result": { + "error": "EVALS_TIER must be gate or periodic. Received: e2e" + } + } + ], + "unknownArgumentCli": { + "free": { + "status": 1, + "stderr": "[test:free] Unknown argument: --bogus" + }, + "paid": { + "status": 1, + "stderr": "[test:paid] Unknown argument: --bogus" + } + } +} diff --git a/test/fixtures/shard-equivalence/expected.json b/test/fixtures/shard-equivalence/expected.json new file mode 100644 index 000000000..934099a62 --- /dev/null +++ b/test/fixtures/shard-equivalence/expected.json @@ -0,0 +1,127 @@ +{ + "recordedFrom": "96764e80 (pre-engine base) via test/fixtures/shard-equivalence/record.ts", + "free": { + "fixtures": { + "pass": { + "status": "passed", + "exitCode": 0, + "laneExit": 0 + }, + "fail": { + "status": "failed", + "exitCode": 1, + "laneExit": 1 + }, + "skip": { + "status": "passed", + "exitCode": 0, + "laneExit": 0 + }, + "wall-timeout": { + "status": "timed-out", + "exitCode": null, + "laneExit": 124 + }, + "module-load-error": { + "status": "failed", + "exitCode": 1, + "laneExit": 1 + }, + "zero-executed": { + "status": "passed", + "exitCode": 0, + "laneExit": 0 + }, + "unhandled-between-tests": { + "status": "failed", + "exitCode": 1, + "laneExit": 1 + } + }, + "laneExit": 124 + }, + "paid": { + "fixtures": { + "pass": { + "status": "passed", + "exitCode": 0, + "executedTests": 1, + "skippedTests": 0, + "laneExit": { + "selective": 0, + "evalsAll": 0 + } + }, + "fail": { + "status": "failed", + "exitCode": 1, + "executedTests": 1, + "skippedTests": 0, + "laneExit": { + "selective": 1, + "evalsAll": 1 + } + }, + "skip": { + "status": "passed", + "exitCode": 0, + "executedTests": 1, + "skippedTests": 1, + "laneExit": { + "selective": 0, + "evalsAll": 0 + } + }, + "wall-timeout": { + "status": "timed-out", + "exitCode": null, + "executedTests": null, + "skippedTests": null, + "laneExit": { + "selective": 1, + "evalsAll": 1 + } + }, + "module-load-error": { + "status": "failed", + "exitCode": 1, + "executedTests": 1, + "skippedTests": 0, + "laneExit": { + "selective": 1, + "evalsAll": 1 + } + }, + "zero-executed": { + "status": "passed", + "exitCode": 0, + "executedTests": 0, + "skippedTests": 0, + "laneExit": { + "selective": 0, + "evalsAll": 1 + } + }, + "unhandled-between-tests": { + "status": "failed", + "exitCode": 1, + "executedTests": 0, + "skippedTests": 0, + "laneExit": { + "selective": 1, + "evalsAll": 1 + } + } + }, + "laneExit": { + "selective": 1, + "evalsAll": 1 + } + }, + "realShard": { + "file": "test/strict-output.test.ts", + "status": "passed", + "filesRan": 1, + "sawTerminalSummary": true + } +} diff --git a/test/fixtures/shard-equivalence/fail.fixture.ts b/test/fixtures/shard-equivalence/fail.fixture.ts new file mode 100644 index 000000000..1b269d356 --- /dev/null +++ b/test/fixtures/shard-equivalence/fail.fixture.ts @@ -0,0 +1,3 @@ +import { expect, test } from 'bun:test'; + +test('fails', () => { expect(1 + 1).toBe(3); }); diff --git a/test/fixtures/shard-equivalence/module-load-error.fixture.ts b/test/fixtures/shard-equivalence/module-load-error.fixture.ts new file mode 100644 index 000000000..b4813f95f --- /dev/null +++ b/test/fixtures/shard-equivalence/module-load-error.fixture.ts @@ -0,0 +1,5 @@ +import { test } from 'bun:test'; + +throw new Error('module load failure fixture'); + +test('never registered', () => {}); diff --git a/test/fixtures/shard-equivalence/pass.fixture.ts b/test/fixtures/shard-equivalence/pass.fixture.ts new file mode 100644 index 000000000..e2400d773 --- /dev/null +++ b/test/fixtures/shard-equivalence/pass.fixture.ts @@ -0,0 +1,3 @@ +import { expect, test } from 'bun:test'; + +test('passes', () => { expect(1 + 1).toBe(2); }); diff --git a/test/fixtures/shard-equivalence/record.ts b/test/fixtures/shard-equivalence/record.ts new file mode 100644 index 000000000..77e1c8707 --- /dev/null +++ b/test/fixtures/shard-equivalence/record.ts @@ -0,0 +1,67 @@ +/** + * Runs the shard-equivalence fixture corpus through one checkout's free and + * paid runners and prints their classifications as one JSON line. + * + * bun test/fixtures/shard-equivalence/record.ts + * + * expected.json was recorded by running this against the pre-engine base + * commit (96764e80); test/shard-engine-equivalence.test.ts runs it against + * the current checkout and requires identical classifications and lane exits. + * Never imported by the test runner. + */ +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; + +export const FIXTURES = [ + 'pass', 'fail', 'skip', 'wall-timeout', 'module-load-error', 'zero-executed', 'unhandled-between-tests', +] as const; +export const REAL_SHARD = 'test/strict-output.test.ts'; +const WALL_MS = 3_000; + +const root = path.resolve(process.argv[2] ?? path.join(import.meta.dir, '../../..')); +const fixtureDir = import.meta.dir; +const scratch = fs.mkdtempSync(path.join(os.tmpdir(), 'shard-equivalence-')); +const free = await import(path.join(root, 'scripts/test-free-shards.ts')); +const paid = await import(path.join(root, 'scripts/test-paid-shards.ts')); +const freeExit = (status: string) => (status === 'passed' ? 0 : status === 'timed-out' ? 124 : 1); + +try { + const runFree = (file: string, index: number) => free.runFreeShard([file], index + 1, FIXTURES.length, { + rootDir: root, quiet: true, log: () => {}, wallTimeoutMs: WALL_MS, + logFilePath: path.join(scratch, `free-${index}.log`), + }); + const runPaid = (file: string, index: number) => paid.runPaidShard([file], index + 1, FIXTURES.length, { + rootDir: root, timeoutMs: WALL_MS, jobs: 2, logDir: scratch, log: () => {}, + env: { ...process.env, GSTACK_CLAUDE_CLI_VERSION: 'fixture', EVALS: '', EVALS_TIER: '', EVALS_ALL: '' }, + }); + const files = FIXTURES.map(name => path.join(fixtureDir, `${name}.fixture.ts`)); + const [freeOutcomes, paidOutcomes, real] = await Promise.all([ + Promise.all(files.map(runFree)), + Promise.all(files.map(runPaid)), + free.runFreeShard([REAL_SHARD], 1, 1, { + rootDir: root, quiet: true, log: () => {}, logFilePath: path.join(scratch, 'real.log'), + }), + ]); + const byName = (values: T[]) => Object.fromEntries(FIXTURES.map((name, index) => [name, values[index]])); + // Lane exit for a run of just that fixture, and for the whole corpus. + const paidExit = (outcomes: unknown[], evalsAll: boolean) => + paid.summaryExitCode(paid.summarize(paid.applyHollowShardGuard(outcomes, { evalsAll, warn: () => {} }))); + const record = { + free: { + fixtures: byName(freeOutcomes.map((o: any) => ({ status: o.status, exitCode: o.exitCode, laneExit: freeExit(o.status) }))), + laneExit: Math.max(...freeOutcomes.map((o: any) => freeExit(o.status))), + }, + paid: { + fixtures: byName(paidOutcomes.map((o: any) => ({ + status: o.status, exitCode: o.exitCode, executedTests: o.executedTests, skippedTests: o.skippedTests, + laneExit: { selective: paidExit([o], false), evalsAll: paidExit([o], true) }, + }))), + laneExit: { selective: paidExit(paidOutcomes, false), evalsAll: paidExit(paidOutcomes, true) }, + }, + realShard: { file: REAL_SHARD, status: real.status, ...real.summary }, + }; + console.log(`SHARD_EQUIVALENCE:${JSON.stringify(record)}`); +} finally { + fs.rmSync(scratch, { recursive: true, force: true }); +} diff --git a/test/fixtures/shard-equivalence/skip.fixture.ts b/test/fixtures/shard-equivalence/skip.fixture.ts new file mode 100644 index 000000000..c6531278e --- /dev/null +++ b/test/fixtures/shard-equivalence/skip.fixture.ts @@ -0,0 +1,3 @@ +import { test } from 'bun:test'; + +test.skip('skipped', () => {}); diff --git a/test/fixtures/shard-equivalence/unhandled-between-tests.fixture.ts b/test/fixtures/shard-equivalence/unhandled-between-tests.fixture.ts new file mode 100644 index 000000000..1ad5fa0c2 --- /dev/null +++ b/test/fixtures/shard-equivalence/unhandled-between-tests.fixture.ts @@ -0,0 +1,6 @@ +import { test } from 'bun:test'; + +test('first', () => {}); +// Rejects outside any test: bun prints "# Unhandled error between tests". +Promise.reject(new Error('unhandled between tests fixture')); +test('second', () => {}); diff --git a/test/fixtures/shard-equivalence/wall-timeout.fixture.ts b/test/fixtures/shard-equivalence/wall-timeout.fixture.ts new file mode 100644 index 000000000..6ed42c3c7 --- /dev/null +++ b/test/fixtures/shard-equivalence/wall-timeout.fixture.ts @@ -0,0 +1,7 @@ +import { test } from 'bun:test'; + +// Blocks the main thread, so no in-process timer can end it: only the +// runner's external wall-clock group kill can. +test('blocks past the wall deadline', () => { + Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 60_000); +}); diff --git a/test/fixtures/shard-equivalence/zero-executed.fixture.ts b/test/fixtures/shard-equivalence/zero-executed.fixture.ts new file mode 100644 index 000000000..7d1c1ce54 --- /dev/null +++ b/test/fixtures/shard-equivalence/zero-executed.fixture.ts @@ -0,0 +1,2 @@ +// Declares no tests: bun reports "Ran 0 tests across 1 file". +export const noTests = true; diff --git a/test/shard-cli-parity.test.ts b/test/shard-cli-parity.test.ts new file mode 100644 index 000000000..66246adf3 --- /dev/null +++ b/test/shard-cli-parity.test.ts @@ -0,0 +1,33 @@ +/** + * Per-lane CLI parity across the shard-engine refactor: each runner accepts + * the same flags with the same defaults and validation messages, and emits + * the same `Unknown argument:` error, as the pre-engine runners recorded in + * test/fixtures/shard-cli-parity/results.json (base 96764e80). + */ +import { describe, expect, test } from 'bun:test'; +import { spawnSync } from 'node:child_process'; +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import { parseCliOptions as parseFree } from '../scripts/test-free-shards'; +import { parseCliOptions as parsePaid } from '../scripts/test-paid-shards'; +import { FREE_CASES, PAID_CASES, capture } from './fixtures/shard-cli-parity/cases'; + +const ROOT = path.resolve(import.meta.dir, '..'); +const recorded = JSON.parse(fs.readFileSync(path.join(ROOT, 'test/fixtures/shard-cli-parity/results.json'), 'utf8')); + +describe('shard runner CLI parity with the pre-engine runners', () => { + test('free lane: same flags, defaults and validation errors', () => { + expect(FREE_CASES.map(argv => ({ argv, result: capture(() => parseFree(argv)) }))).toEqual(recorded.free); + }); + + test('paid lane: same flags, env-derived defaults and validation errors', () => { + expect(PAID_CASES.map(c => ({ ...c, result: capture(() => parsePaid(c.argv, c.env ?? {})) }))).toEqual(recorded.paid); + }); + + test.each(['free', 'paid'] as const)('%s CLI rejects an unknown flag with the same message and exit code', lane => { + const run = spawnSync(process.execPath, [path.join(ROOT, `scripts/test-${lane}-shards.ts`), '--bogus'], { + cwd: ROOT, encoding: 'utf8', timeout: 30_000, + }); + expect({ status: run.status, stderr: run.stderr.trim() }).toEqual(recorded.unknownArgumentCli[lane]); + }); +}); diff --git a/test/shard-engine-equivalence.test.ts b/test/shard-engine-equivalence.test.ts new file mode 100644 index 000000000..8788239c6 --- /dev/null +++ b/test/shard-engine-equivalence.test.ts @@ -0,0 +1,52 @@ +/** + * W2 runner equivalence: the free and paid lanes, now on the shared shard + * engine, classify a fixture corpus (pass, fail, skip, wall-timeout, + * module-load error, zero executed tests, unhandled error between tests) + * exactly as the pre-engine runners did. expected.json was recorded by the + * same recorder against base commit 96764e80; one real suite shard is run + * too. Re-record only for a deliberate, reviewed lane-policy change. + */ +import { describe, expect, test } from 'bun:test'; +import { spawnSync } from 'node:child_process'; +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import { collectFreeTestFiles } from '../scripts/test-free-shards'; + +const ROOT = path.resolve(import.meta.dir, '..'); +const CORPUS = path.join(ROOT, 'test/fixtures/shard-equivalence'); + +describe('shard engine runner equivalence (fixture corpus)', () => { + test('both lanes keep the base runners\' per-file classifications and lane exit codes', () => { + const run = spawnSync(process.execPath, [path.join(CORPUS, 'record.ts'), ROOT], { + cwd: ROOT, encoding: 'utf8', timeout: 90_000, + env: { ...process.env, EVALS: '', EVALS_TIER: '', EVALS_ALL: '' }, + }); + expect(run.error, run.stderr).toBeUndefined(); + expect(run.status, run.stderr).toBe(0); + const line = run.stdout.split('\n').find(entry => entry.startsWith('SHARD_EQUIVALENCE:')); + expect(line, run.stdout + run.stderr).toBeDefined(); + const actual = JSON.parse(line!.slice('SHARD_EQUIVALENCE:'.length)); + const expected = JSON.parse(fs.readFileSync(path.join(CORPUS, 'expected.json'), 'utf8')); + + expect(actual.free).toEqual(expected.free); + expect(actual.paid).toEqual(expected.paid); + // The real shard's test count moves as that file grows; its verdict and + // complete file accounting must not. + expect(actual.realShard).toMatchObject(expected.realShard); + expect(actual.realShard.testsRan).toBeGreaterThan(0); + }, 120_000); + + test('the corpus covers every outcome kind and stays outside the free census', () => { + const fixtures = fs.readdirSync(CORPUS).filter(name => name.endsWith('.fixture.ts')).sort(); + expect(fixtures).toEqual([ + 'fail.fixture.ts', 'module-load-error.fixture.ts', 'pass.fixture.ts', 'skip.fixture.ts', + 'unhandled-between-tests.fixture.ts', 'wall-timeout.fixture.ts', 'zero-executed.fixture.ts', + ]); + const expected = JSON.parse(fs.readFileSync(path.join(CORPUS, 'expected.json'), 'utf8')); + for (const lane of ['free', 'paid']) { + expect(Object.keys(expected[lane].fixtures).sort()).toEqual(fixtures.map(name => name.replace('.fixture.ts', ''))); + } + // A failing fixture inside the census would fail every full free run. + expect(collectFreeTestFiles(ROOT).filter(file => file.startsWith('test/fixtures/shard-equivalence/'))).toEqual([]); + }); +}); diff --git a/test/shard-engine.test.ts b/test/shard-engine.test.ts new file mode 100644 index 000000000..f625a9882 --- /dev/null +++ b/test/shard-engine.test.ts @@ -0,0 +1,273 @@ +/** + * Unit pins for the shared shard engine (scripts/lib/shard-engine.ts): the + * lane-exit consequences of failing, unhandled and module-load output, each + * lane's injected zero-execution and seed rules, whole-group kill on a wall + * timeout, the win32 path, log capture, sandbox isolation and the flag loop. + * run-shard-child and strict-output-* tests pin the moved primitives. + */ +import { afterEach, describe, expect, spyOn, test } from 'bun:test'; +import type { ChildProcess } from 'node:child_process'; +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import { + BunTestOutputClassifier, + createShardSandbox, + killProcessGroup, + nextShardLogPath, + openShardLog, + parseCliFlags, + readDurationSeed, + removeShardSandbox, + runShardChild, + strictShardStatus, + writeDurationSeed, + zeroExecutionVerdict, +} from '../scripts/lib/shard-engine'; +import { FREE_LANE_POLICY, exitCodeFor, runFreeShard } from '../scripts/test-free-shards'; +import { + PAID_LANE_POLICY, applyHollowShardGuard, runPaidShard, summarize, summaryExitCode, type ShardOutcome, +} from '../scripts/test-paid-shards'; + +const scratch: string[] = []; +const tempDir = (prefix: string) => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), prefix)); + scratch.push(dir); + return dir; +}; +afterEach(() => { for (const dir of scratch.splice(0)) fs.rmSync(dir, { recursive: true, force: true }); }); + +function summaryOf(output: string) { + const classifier = new BunTestOutputClassifier(); + classifier.write(output, 'stderr'); + return classifier.end(); +} + +const PASSING = 'test/a.test.ts:\n(pass) a [1.00ms]\n\n 1 pass\n 0 fail\nRan 1 test across 1 file. [2.00ms]\n'; +// Bun 1.3 output shapes observed for these outcomes (fixture corpus). +const FAILING = 'test/a.test.ts:\n(fail) a [1.00ms]\n\n 0 pass\n 1 fail\nRan 1 test across 1 file. [2.00ms]\n'; +const MODULE_LOAD = 'test/a.test.ts:\nerror: module load failure\n\n 0 pass\n 1 fail\n 1 error\nRan 1 test across 1 file. [2.00ms]\n'; +const UNHANDLED = 'test/a.test.ts:\n# Unhandled error between tests\nerror: boom\n\n 0 pass\n 0 fail\n 1 error\nRan 0 tests across 1 file. [2.00ms]\n'; + +describe('shard engine: verdicts and lane exits', () => { + const verdict = (output: string, exitCode: number | null, timedOut = false) => strictShardStatus({ + timedOut, exitCode, summary: summaryOf(output), expectedFiles: 1, evidenceComplete: true, + }); + const paidExit = (status: ShardOutcome['status']) => summaryExitCode(summarize([{ + shard: 1, files: ['test/a.test.ts'], status, exitCode: 0, elapsedMs: 1, groupPid: null, executedTests: 1, skippedTests: 0, + }])); + + test('a failing test file makes both lanes exit nonzero, even when bun exits 0', () => { + for (const exitCode of [0, 1]) { + const status = verdict(FAILING, exitCode); + expect(status).toBe('failed'); + expect(exitCodeFor(status)).toBe(1); + expect(paidExit(status)).toBe(1); + } + expect(exitCodeFor(verdict(PASSING, 0))).toBe(0); + expect(paidExit(verdict(PASSING, 0))).toBe(0); + }); + + test('unhandled between-test errors and module-load errors count as failures', () => { + expect(summaryOf(UNHANDLED).unhandledBetweenTests).toBe(1); + expect(verdict(UNHANDLED, 0)).toBe('failed'); + expect(summaryOf(MODULE_LOAD).failedTests).toBe(1); + expect(verdict(MODULE_LOAD, 0)).toBe('failed'); + expect(verdict(MODULE_LOAD, 1)).toBe('failed'); + }); + + test('missing summary, short file count or incomplete evidence never pass; a timeout is its own status', () => { + expect(verdict('(pass) a [1.00ms]\n', 0)).toBe('failed'); + expect(strictShardStatus({ timedOut: false, exitCode: 0, summary: summaryOf(PASSING), expectedFiles: 2, evidenceComplete: true })).toBe('failed'); + expect(strictShardStatus({ timedOut: false, exitCode: 0, summary: summaryOf(PASSING), expectedFiles: 1, evidenceComplete: false })).toBe('failed'); + expect(verdict(PASSING, 0, true)).toBe('timed-out'); + expect(exitCodeFor('timed-out')).toBe(124); + expect(paidExit('timed-out')).toBe(1); + }); +}); + +describe('shard engine: injected lane policy', () => { + const zero = (executedTests: number | null): ShardOutcome => ({ + shard: 1, files: ['test/a.test.ts'], status: 'passed', exitCode: 0, elapsedMs: 1, groupPid: null, executedTests, skippedTests: 0, + }); + + test('free lane: zero executed tests pass when every planned file was counted', () => { + for (const promisedAll of [false, true]) { + expect(zeroExecutionVerdict(0, FREE_LANE_POLICY, { promisedAll })).toBe('passed'); + } + const zeroTests = 'test/a.test.ts:\n\n 0 pass\n 0 fail\nRan 0 tests across 1 file. [2.00ms]\n'; + expect(strictShardStatus({ timedOut: false, exitCode: 0, summary: summaryOf(zeroTests), expectedFiles: 1, evidenceComplete: true })).toBe('passed'); + }); + + test('paid lane: zero executed tests warn under selection and are passed-empty under EVALS_ALL', () => { + expect(zeroExecutionVerdict(0, PAID_LANE_POLICY, { promisedAll: false })).toBe('passed-with-warning'); + expect(zeroExecutionVerdict(0, PAID_LANE_POLICY, { promisedAll: true })).toBe('passed-empty'); + expect(zeroExecutionVerdict(3, PAID_LANE_POLICY, { promisedAll: true })).toBe('passed'); + const warnings: string[] = []; + const selective = applyHollowShardGuard([zero(0)], { evalsAll: false, warn: line => warnings.push(line) }); + expect(selective[0].status).toBe('passed'); + expect(warnings).toHaveLength(1); + expect(summaryExitCode(summarize(selective))).toBe(0); + const all = applyHollowShardGuard([zero(0)], { evalsAll: true, warn: () => {} }); + expect(all[0].status).toBe('passed-empty'); + expect(summaryExitCode(summarize(all))).toBe(1); + expect(applyHollowShardGuard([zero(null)], { evalsAll: true, warn: () => {} })[0].status).toBe('passed'); + }); + + test('duration seeds: one reader, lane predicate decides (free >= 0, paid > 0)', () => { + const dir = tempDir('seed-'); + const file = path.join(dir, 'seed.json'); + fs.writeFileSync(file, JSON.stringify({ durations: { 'a.ts': 0, 'b.ts': 5, 'c.ts': -1, 'd.ts': 'x', 'e.ts': Infinity } })); + expect(readDurationSeed(file, FREE_LANE_POLICY.acceptsSeedDuration)).toEqual({ status: 'ok', durations: { 'a.ts': 0, 'b.ts': 5 } }); + expect(readDurationSeed(file, PAID_LANE_POLICY.acceptsSeedDuration)).toEqual({ status: 'ok', durations: { 'b.ts': 5 } }); + expect(readDurationSeed(path.join(dir, 'missing.json'), () => true).status).toBe('missing'); + for (const corrupt of ['{not json', 'null']) { + fs.writeFileSync(file, corrupt); + expect(readDurationSeed(file, () => true).status).toBe('corrupt'); + } + writeDurationSeed(file, { 'z.ts': 2, 'a.ts': 1 }); + const written = JSON.parse(fs.readFileSync(file, 'utf8')); + expect(written.version).toBe(1); + expect(Object.keys(written.durations)).toEqual(['a.ts', 'z.ts']); + expect(fs.readdirSync(dir)).toEqual(['seed.json']); + }); +}); + +describe('shard engine: process lifecycle', () => { + test.skipIf(process.platform === 'win32')('a timed-out shard\'s whole process group is killed and the lanes report it as not passed', async () => { + const dir = tempDir('group-kill-'); + const pidFile = path.join(dir, 'grandchild.pid'); + const spinner = `const { spawn } = require('node:child_process'); + const p = spawn(process.execPath, ['-e', 'setInterval(() => {}, 1000)'], { stdio: 'ignore' }); + require('node:fs').writeFileSync(${JSON.stringify(pidFile)}, String(p.pid)); + const end = Date.now() + 600000; while (Date.now() < end) {}`; + const alive = (pid: number) => { + try { + process.kill(pid, 0); + return !(process.platform === 'linux' && fs.readFileSync(`/proc/${pid}/stat`, 'utf8').split(') ')[1].startsWith('Z')); + } catch { return false; } + }; + const waitDead = async (pid: number) => { + const deadline = Date.now() + 2_000; + while (alive(pid) && Date.now() < deadline) await Bun.sleep(20); + return !alive(pid); + }; + const free = await runFreeShard(['spin.test.ts'], 1, 1, { + rootDir: dir, quiet: true, log: () => {}, wallTimeoutMs: 1_500, logFilePath: path.join(dir, 'free.log'), + commandFor: () => ({ command: process.execPath, args: ['-e', spinner] }), + }); + expect(free.status).toBe('timed-out'); + expect(exitCodeFor(free.status)).not.toBe(0); + expect(await waitDead(Number(fs.readFileSync(pidFile, 'utf8')))).toBe(true); + expect(await waitDead(free.groupPid!)).toBe(true); + + fs.rmSync(pidFile); + const paid = await runPaidShard(['spin.test.ts'], 1, 1, { + rootDir: dir, timeoutMs: 1_500, jobs: 2, logDir: dir, log: () => {}, + env: { ...process.env, GSTACK_CLAUDE_CLI_VERSION: 'fixture' }, + commandFor: () => ({ command: process.execPath, args: ['-e', spinner] }), + }); + expect(paid.status).toBe('timed-out'); + expect(summaryExitCode(summarize([paid]))).toBe(1); + expect(await waitDead(Number(fs.readFileSync(pidFile, 'utf8')))).toBe(true); + }, 30_000); + + test('win32 semantics: no process group and never a negative-pid kill', async () => { + const original = Object.getOwnPropertyDescriptor(process, 'platform')!; + const kills = spyOn(process, 'kill'); + Object.defineProperty(process, 'platform', { value: 'win32' }); + try { + const signalled: string[] = []; + killProcessGroup({ pid: 4242, kill: (signal: string) => { signalled.push(signal); return true; } } as unknown as ChildProcess, 'SIGKILL'); + expect(signalled).toEqual(['SIGKILL']); + const result = await runShardChild({ + command: process.execPath, args: ['-e', 'setInterval(() => {}, 1000)'], + cwd: process.cwd(), env: process.env, timeoutMs: 500, hookStreams: () => [], + }); + expect(result.timedOut).toBe(true); + expect(kills.mock.calls.some(([pid]) => typeof pid === 'number' && pid < 0)).toBe(false); + } finally { + Object.defineProperty(process, 'platform', original); + kills.mockRestore(); + } + }, 30_000); + + test('an attached companion settles after the final group kill', async () => { + let settledAfterExit = false; + let child: ChildProcess | undefined; + const result = await runShardChild({ + command: process.execPath, args: ['-e', 'console.log("done")'], + cwd: process.cwd(), env: process.env, timeoutMs: 10_000, + hookStreams: spawned => [new Promise(resolve => { spawned.stdout!.resume(); spawned.stdout!.once('end', resolve); })], + attach: spawned => { + child = spawned; + return { signal: () => {}, settle: async () => { settledAfterExit = spawned.exitCode !== null || spawned.signalCode !== null; } }; + }, + }); + expect(result.exitCode).toBe(0); + expect(child).toBeDefined(); + expect(settledAfterExit).toBe(true); + }, 30_000); +}); + +describe('shard engine: capture and isolation', () => { + test('log capture spools every chunk; a write error fails the log once, without throwing', async () => { + const dir = tempDir('shard-log-'); + const first = nextShardLogPath(dir, 'gstack-free-test'); + const second = nextShardLogPath(dir, 'gstack-free-test'); + expect(first).not.toBe(second); + expect(path.basename(first)).toMatch(new RegExp(`^gstack-free-test-.+-${process.pid}-\\d+\\.log$`)); + const log = openShardLog(first, '[test:unit]', 0o600); + log.write('one\n'); + log.write(Buffer.from('two\n')); + await new Promise(resolve => log.stream.end(() => resolve())); + expect(fs.readFileSync(first, 'utf8')).toBe('one\ntwo\n'); + if (process.platform !== 'win32') expect(fs.statSync(first).mode & 0o777).toBe(0o600); + expect(log.failed).toBe(false); + + const errors = spyOn(console, 'error').mockImplementation(() => {}); + try { + const broken = openShardLog(path.join(dir, 'missing', 'x.log'), '[test:unit]'); + await new Promise(resolve => broken.stream.once('error', resolve)); + broken.write('dropped'); + expect(broken.failed).toBe(true); + expect(errors.mock.calls.filter(([line]) => String(line).includes('could not write the full log'))).toHaveLength(1); + } finally { errors.mockRestore(); } + }); + + test('each shard gets its own tmp and Chromium profile, removed by the backstop', async () => { + const base = { KEEP: '1', TMPDIR: '/inherited', CHROMIUM_PROFILE: '/shared-profile' }; + const a = createShardSandbox('gstack-unit-shard-', base); + const b = createShardSandbox('gstack-unit-shard-', base, { realpath: true }); + try { + expect(a.stateDir).not.toBe(b.stateDir); + expect(b.stateDir).toBe(fs.realpathSync(b.stateDir)); + for (const box of [a, b]) { + expect(box.env.KEEP).toBe('1'); + expect(box.env.GSTACK_HOME).toBeUndefined(); + for (const key of ['TMPDIR', 'TEMP', 'TMP']) expect(box.env[key]).toBe(box.tmp); + expect(box.env.CHROMIUM_PROFILE).toBe(path.join(box.stateDir, 'chromium-profile')); + expect(fs.statSync(box.tmp).isDirectory()).toBe(true); + } + expect(base.TMPDIR).toBe('/inherited'); + fs.writeFileSync(path.join(a.tmp, 'left-behind'), 'x'); + await removeShardSandbox(a.stateDir); + expect(fs.existsSync(a.stateDir)).toBe(false); + await removeShardSandbox(a.stateDir); + } finally { + fs.rmSync(a.stateDir, { recursive: true, force: true }); + fs.rmSync(b.stateDir, { recursive: true, force: true }); + } + }); + + test('flag loop: declared handlers consume values; anything else is Unknown argument', () => { + const seen: Array<[string, string | undefined]> = []; + parseCliFlags(['--on', '--value', 'x', '--value'], { + '--on': () => { seen.push(['--on', undefined]); }, + '--value': next => { seen.push(['--value', next()]); }, + }); + expect(seen).toEqual([['--on', undefined], ['--value', 'x'], ['--value', undefined]]); + expect(() => parseCliFlags(['--nope'], {})).toThrow('Unknown argument: --nope'); + expect(() => parseCliFlags(['constructor'], {})).toThrow('Unknown argument: constructor'); + }); +});