mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-04 15:57:33 +01:00
test(shard-engine): engine unit tests, fixture-corpus equivalence, per-lane CLI parity
- test/shard-engine.test.ts: failing/unhandled/module-load output fails both
lanes, per-lane zero-execution and seed rules, whole-group kill on a wall
timeout (both lanes), mocked-win32 path with no negative-pid kill,
companion settle order, log capture, sandbox isolation, flag loop.
- test/shard-engine-equivalence.test.ts + test/fixtures/shard-equivalence:
seven outcome fixtures plus one real shard, run through both lanes and
compared with classifications recorded from the base runners (96764e80).
- test/shard-cli-parity.test.ts + test/fixtures/shard-cli-parity: flag set,
defaults, validation errors and the Unknown argument error per lane match
the base runners.
This commit is contained in:
garrytan committed
1 parent
cb203b2982
commit
eb44a747a9
14 files changed
+1407
No files matched your search
+64
@@ -0,0 +1,64 @@
|
||||
/**
|
||||
* Argument vectors for the per-lane CLI parity check. results.json holds the
|
||||
* parse results the pre-engine runners (base 96764e80) produced for them.
|
||||
*/
|
||||
export const FREE_CASES: string[][] = [
|
||||
[],
|
||||
['--dry-run', '--list', '--verbose'],
|
||||
['--windows-only', '--list'],
|
||||
['--record-durations'],
|
||||
['--quick'],
|
||||
['--shards', '4', '--shard', '2'],
|
||||
['--wall-timeout', '600'],
|
||||
['--ci-plan', 'plan.json', '--shards', '20'],
|
||||
['--ci-run', 'plan.json', '--shard', '1', '--result', 'result.json'],
|
||||
['--ci-verify', 'plan.json', '--results', 'results'],
|
||||
['--bogus'],
|
||||
['--list', '--bogus'],
|
||||
['--shards'],
|
||||
['--shard'],
|
||||
['--wall-timeout', '0'],
|
||||
['--wall-timeout'],
|
||||
['--ci-plan'],
|
||||
['--ci-plan', '--list'],
|
||||
['--ci-plan', 'a.json', '--ci-run', 'b.json'],
|
||||
['--ci-plan', 'a.json', '--quick'],
|
||||
['--ci-run', 'plan.json'],
|
||||
['--ci-verify', 'plan.json'],
|
||||
['--quick', '--shard', '1'],
|
||||
['--quick', '--windows-only'],
|
||||
];
|
||||
|
||||
export const PAID_CASES: Array<{ argv: string[]; env?: Record<string, string> }> = [
|
||||
{ argv: [] },
|
||||
{ argv: ['--list', '--tier', 'periodic', '--profile', 'full'] },
|
||||
{ argv: ['--tier', 'gate', '--profile', 'pr', '--list'] },
|
||||
{ argv: ['--timeout', '600', '--jobs', '2', '--files-per-shard', '3'] },
|
||||
{ argv: ['--emit-plan', 'manifest.json', '--slices', '4', '--skip-judges'] },
|
||||
{ argv: ['--plan', 'manifest.json', '--slice', '2'] },
|
||||
{ argv: ['--report', 'reports', '--write-durations'] },
|
||||
{ argv: [], env: { EVALS_TIER: 'periodic', EVALS_JOBS: '3', EVALS_CONCURRENCY: '5', EVALS_SHARD_TIMEOUT_MS: '1000', EVALS_PROFILE: 'full' } },
|
||||
{ argv: ['--bogus'] },
|
||||
{ argv: ['--list', '--bogus'] },
|
||||
{ argv: ['--tier', 'e2e'] },
|
||||
{ argv: ['--tier'] },
|
||||
{ argv: ['--profile'] },
|
||||
{ argv: ['--profile', 'fast'] },
|
||||
{ argv: ['--timeout', '0'] },
|
||||
{ argv: ['--jobs'] },
|
||||
{ argv: ['--emit-plan'] },
|
||||
{ argv: ['--plan'] },
|
||||
{ argv: ['--report'] },
|
||||
{ argv: ['--write-durations'] },
|
||||
{ argv: ['--skip-judges'] },
|
||||
{ argv: ['--profile', 'pr', '--tier', 'periodic'] },
|
||||
{ argv: ['--profile', 'pr', '--files-per-shard', '2'] },
|
||||
{ argv: [], env: { EVALS_TIER: 'e2e' } },
|
||||
];
|
||||
|
||||
export type ParseResult = { ok: unknown } | { error: string };
|
||||
|
||||
export function capture(parse: () => unknown): ParseResult {
|
||||
try { return { ok: parse() }; }
|
||||
catch (error) { return { error: error instanceof Error ? error.message : String(error) }; }
|
||||
}
|
||||
+762
@@ -0,0 +1,762 @@
|
||||
{
|
||||
"recordedFrom": "96764e80 (pre-engine base runners)",
|
||||
"free": [
|
||||
{
|
||||
"argv": [],
|
||||
"result": {
|
||||
"ok": {
|
||||
"dryRun": false,
|
||||
"listOnly": false,
|
||||
"recordDurations": false,
|
||||
"windowsOnly": false,
|
||||
"verbose": false,
|
||||
"shardCount": 20,
|
||||
"shardIndex": null,
|
||||
"wallTimeoutMs": 360000,
|
||||
"wallTimeoutExplicit": false,
|
||||
"quick": false,
|
||||
"ciPlan": null,
|
||||
"ciRun": null,
|
||||
"ciVerify": null,
|
||||
"result": null,
|
||||
"results": null
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"argv": [
|
||||
"--dry-run",
|
||||
"--list",
|
||||
"--verbose"
|
||||
],
|
||||
"result": {
|
||||
"ok": {
|
||||
"dryRun": true,
|
||||
"listOnly": true,
|
||||
"recordDurations": false,
|
||||
"windowsOnly": false,
|
||||
"verbose": true,
|
||||
"shardCount": 20,
|
||||
"shardIndex": null,
|
||||
"wallTimeoutMs": 360000,
|
||||
"wallTimeoutExplicit": false,
|
||||
"quick": false,
|
||||
"ciPlan": null,
|
||||
"ciRun": null,
|
||||
"ciVerify": null,
|
||||
"result": null,
|
||||
"results": null
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"argv": [
|
||||
"--windows-only",
|
||||
"--list"
|
||||
],
|
||||
"result": {
|
||||
"ok": {
|
||||
"dryRun": false,
|
||||
"listOnly": true,
|
||||
"recordDurations": false,
|
||||
"windowsOnly": true,
|
||||
"verbose": false,
|
||||
"shardCount": 20,
|
||||
"shardIndex": null,
|
||||
"wallTimeoutMs": 360000,
|
||||
"wallTimeoutExplicit": false,
|
||||
"quick": false,
|
||||
"ciPlan": null,
|
||||
"ciRun": null,
|
||||
"ciVerify": null,
|
||||
"result": null,
|
||||
"results": null
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"argv": [
|
||||
"--record-durations"
|
||||
],
|
||||
"result": {
|
||||
"ok": {
|
||||
"dryRun": false,
|
||||
"listOnly": false,
|
||||
"recordDurations": true,
|
||||
"windowsOnly": false,
|
||||
"verbose": false,
|
||||
"shardCount": 20,
|
||||
"shardIndex": null,
|
||||
"wallTimeoutMs": 360000,
|
||||
"wallTimeoutExplicit": false,
|
||||
"quick": false,
|
||||
"ciPlan": null,
|
||||
"ciRun": null,
|
||||
"ciVerify": null,
|
||||
"result": null,
|
||||
"results": null
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"argv": [
|
||||
"--quick"
|
||||
],
|
||||
"result": {
|
||||
"ok": {
|
||||
"dryRun": false,
|
||||
"listOnly": false,
|
||||
"recordDurations": false,
|
||||
"windowsOnly": false,
|
||||
"verbose": false,
|
||||
"shardCount": 20,
|
||||
"shardIndex": null,
|
||||
"wallTimeoutMs": 360000,
|
||||
"wallTimeoutExplicit": false,
|
||||
"quick": true,
|
||||
"ciPlan": null,
|
||||
"ciRun": null,
|
||||
"ciVerify": null,
|
||||
"result": null,
|
||||
"results": null
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"argv": [
|
||||
"--shards",
|
||||
"4",
|
||||
"--shard",
|
||||
"2"
|
||||
],
|
||||
"result": {
|
||||
"ok": {
|
||||
"dryRun": false,
|
||||
"listOnly": false,
|
||||
"recordDurations": false,
|
||||
"windowsOnly": false,
|
||||
"verbose": false,
|
||||
"shardCount": 4,
|
||||
"shardIndex": 2,
|
||||
"wallTimeoutMs": 360000,
|
||||
"wallTimeoutExplicit": false,
|
||||
"quick": false,
|
||||
"ciPlan": null,
|
||||
"ciRun": null,
|
||||
"ciVerify": null,
|
||||
"result": null,
|
||||
"results": null
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"argv": [
|
||||
"--wall-timeout",
|
||||
"600"
|
||||
],
|
||||
"result": {
|
||||
"ok": {
|
||||
"dryRun": false,
|
||||
"listOnly": false,
|
||||
"recordDurations": false,
|
||||
"windowsOnly": false,
|
||||
"verbose": false,
|
||||
"shardCount": 20,
|
||||
"shardIndex": null,
|
||||
"wallTimeoutMs": 600000,
|
||||
"wallTimeoutExplicit": true,
|
||||
"quick": false,
|
||||
"ciPlan": null,
|
||||
"ciRun": null,
|
||||
"ciVerify": null,
|
||||
"result": null,
|
||||
"results": null
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"argv": [
|
||||
"--ci-plan",
|
||||
"plan.json",
|
||||
"--shards",
|
||||
"20"
|
||||
],
|
||||
"result": {
|
||||
"ok": {
|
||||
"dryRun": false,
|
||||
"listOnly": false,
|
||||
"recordDurations": false,
|
||||
"windowsOnly": false,
|
||||
"verbose": false,
|
||||
"shardCount": 20,
|
||||
"shardIndex": null,
|
||||
"wallTimeoutMs": 360000,
|
||||
"wallTimeoutExplicit": false,
|
||||
"quick": false,
|
||||
"ciPlan": "plan.json",
|
||||
"ciRun": null,
|
||||
"ciVerify": null,
|
||||
"result": null,
|
||||
"results": null
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"argv": [
|
||||
"--ci-run",
|
||||
"plan.json",
|
||||
"--shard",
|
||||
"1",
|
||||
"--result",
|
||||
"result.json"
|
||||
],
|
||||
"result": {
|
||||
"ok": {
|
||||
"dryRun": false,
|
||||
"listOnly": false,
|
||||
"recordDurations": false,
|
||||
"windowsOnly": false,
|
||||
"verbose": false,
|
||||
"shardCount": 20,
|
||||
"shardIndex": 1,
|
||||
"wallTimeoutMs": 360000,
|
||||
"wallTimeoutExplicit": false,
|
||||
"quick": false,
|
||||
"ciPlan": null,
|
||||
"ciRun": "plan.json",
|
||||
"ciVerify": null,
|
||||
"result": "result.json",
|
||||
"results": null
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"argv": [
|
||||
"--ci-verify",
|
||||
"plan.json",
|
||||
"--results",
|
||||
"results"
|
||||
],
|
||||
"result": {
|
||||
"ok": {
|
||||
"dryRun": false,
|
||||
"listOnly": false,
|
||||
"recordDurations": false,
|
||||
"windowsOnly": false,
|
||||
"verbose": false,
|
||||
"shardCount": 20,
|
||||
"shardIndex": null,
|
||||
"wallTimeoutMs": 360000,
|
||||
"wallTimeoutExplicit": false,
|
||||
"quick": false,
|
||||
"ciPlan": null,
|
||||
"ciRun": null,
|
||||
"ciVerify": "plan.json",
|
||||
"result": null,
|
||||
"results": "results"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"argv": [
|
||||
"--bogus"
|
||||
],
|
||||
"result": {
|
||||
"error": "Unknown argument: --bogus"
|
||||
}
|
||||
},
|
||||
{
|
||||
"argv": [
|
||||
"--list",
|
||||
"--bogus"
|
||||
],
|
||||
"result": {
|
||||
"error": "Unknown argument: --bogus"
|
||||
}
|
||||
},
|
||||
{
|
||||
"argv": [
|
||||
"--shards"
|
||||
],
|
||||
"result": {
|
||||
"error": "Missing value for --shards"
|
||||
}
|
||||
},
|
||||
{
|
||||
"argv": [
|
||||
"--shard"
|
||||
],
|
||||
"result": {
|
||||
"error": "Missing value for --shard"
|
||||
}
|
||||
},
|
||||
{
|
||||
"argv": [
|
||||
"--wall-timeout",
|
||||
"0"
|
||||
],
|
||||
"result": {
|
||||
"error": "--wall-timeout needs a positive integer (seconds)"
|
||||
}
|
||||
},
|
||||
{
|
||||
"argv": [
|
||||
"--wall-timeout"
|
||||
],
|
||||
"result": {
|
||||
"error": "--wall-timeout needs a positive integer (seconds)"
|
||||
}
|
||||
},
|
||||
{
|
||||
"argv": [
|
||||
"--ci-plan"
|
||||
],
|
||||
"result": {
|
||||
"error": "Missing path for --ci-plan"
|
||||
}
|
||||
},
|
||||
{
|
||||
"argv": [
|
||||
"--ci-plan",
|
||||
"--list"
|
||||
],
|
||||
"result": {
|
||||
"error": "Missing path for --ci-plan"
|
||||
}
|
||||
},
|
||||
{
|
||||
"argv": [
|
||||
"--ci-plan",
|
||||
"a.json",
|
||||
"--ci-run",
|
||||
"b.json"
|
||||
],
|
||||
"result": {
|
||||
"error": "CI modes cannot be combined with other selection modes"
|
||||
}
|
||||
},
|
||||
{
|
||||
"argv": [
|
||||
"--ci-plan",
|
||||
"a.json",
|
||||
"--quick"
|
||||
],
|
||||
"result": {
|
||||
"error": "CI modes cannot be combined with other selection modes"
|
||||
}
|
||||
},
|
||||
{
|
||||
"argv": [
|
||||
"--ci-run",
|
||||
"plan.json"
|
||||
],
|
||||
"result": {
|
||||
"error": "--ci-run requires --shard and --result"
|
||||
}
|
||||
},
|
||||
{
|
||||
"argv": [
|
||||
"--ci-verify",
|
||||
"plan.json"
|
||||
],
|
||||
"result": {
|
||||
"error": "--ci-verify requires --results"
|
||||
}
|
||||
},
|
||||
{
|
||||
"argv": [
|
||||
"--quick",
|
||||
"--shard",
|
||||
"1"
|
||||
],
|
||||
"result": {
|
||||
"error": "--quick cannot change recording, Windows or shard selection"
|
||||
}
|
||||
},
|
||||
{
|
||||
"argv": [
|
||||
"--quick",
|
||||
"--windows-only"
|
||||
],
|
||||
"result": {
|
||||
"error": "--quick cannot change recording, Windows or shard selection"
|
||||
}
|
||||
}
|
||||
],
|
||||
"paid": [
|
||||
{
|
||||
"argv": [],
|
||||
"result": {
|
||||
"ok": {
|
||||
"tier": "gate",
|
||||
"profile": "full",
|
||||
"profileExplicit": false,
|
||||
"listOnly": false,
|
||||
"skipJudges": false,
|
||||
"timeoutExplicit": false,
|
||||
"timeoutMs": 1800000,
|
||||
"jobs": 8,
|
||||
"withinShardConcurrency": 2,
|
||||
"maxFilesPerShard": 1,
|
||||
"emitPlanPath": null,
|
||||
"slices": 1,
|
||||
"planPath": null,
|
||||
"sliceIndex": null,
|
||||
"reportDir": null,
|
||||
"writeDurations": false
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"argv": [
|
||||
"--list",
|
||||
"--tier",
|
||||
"periodic",
|
||||
"--profile",
|
||||
"full"
|
||||
],
|
||||
"result": {
|
||||
"ok": {
|
||||
"tier": "periodic",
|
||||
"profile": "full",
|
||||
"profileExplicit": true,
|
||||
"listOnly": true,
|
||||
"skipJudges": false,
|
||||
"timeoutExplicit": false,
|
||||
"timeoutMs": 1800000,
|
||||
"jobs": 8,
|
||||
"withinShardConcurrency": 2,
|
||||
"maxFilesPerShard": 1,
|
||||
"emitPlanPath": null,
|
||||
"slices": 1,
|
||||
"planPath": null,
|
||||
"sliceIndex": null,
|
||||
"reportDir": null,
|
||||
"writeDurations": false
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"argv": [
|
||||
"--tier",
|
||||
"gate",
|
||||
"--profile",
|
||||
"pr",
|
||||
"--list"
|
||||
],
|
||||
"result": {
|
||||
"ok": {
|
||||
"tier": "gate",
|
||||
"profile": "pr",
|
||||
"profileExplicit": true,
|
||||
"listOnly": true,
|
||||
"skipJudges": false,
|
||||
"timeoutExplicit": false,
|
||||
"timeoutMs": 1800000,
|
||||
"jobs": 8,
|
||||
"withinShardConcurrency": 2,
|
||||
"maxFilesPerShard": 1,
|
||||
"emitPlanPath": null,
|
||||
"slices": 1,
|
||||
"planPath": null,
|
||||
"sliceIndex": null,
|
||||
"reportDir": null,
|
||||
"writeDurations": false
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"argv": [
|
||||
"--timeout",
|
||||
"600",
|
||||
"--jobs",
|
||||
"2",
|
||||
"--files-per-shard",
|
||||
"3"
|
||||
],
|
||||
"result": {
|
||||
"ok": {
|
||||
"tier": "gate",
|
||||
"profile": "full",
|
||||
"profileExplicit": false,
|
||||
"listOnly": false,
|
||||
"skipJudges": false,
|
||||
"timeoutExplicit": true,
|
||||
"timeoutMs": 600000,
|
||||
"jobs": 2,
|
||||
"withinShardConcurrency": 2,
|
||||
"maxFilesPerShard": 3,
|
||||
"emitPlanPath": null,
|
||||
"slices": 1,
|
||||
"planPath": null,
|
||||
"sliceIndex": null,
|
||||
"reportDir": null,
|
||||
"writeDurations": false
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"argv": [
|
||||
"--emit-plan",
|
||||
"manifest.json",
|
||||
"--slices",
|
||||
"4",
|
||||
"--skip-judges"
|
||||
],
|
||||
"result": {
|
||||
"ok": {
|
||||
"tier": "gate",
|
||||
"profile": "full",
|
||||
"profileExplicit": false,
|
||||
"listOnly": false,
|
||||
"skipJudges": true,
|
||||
"timeoutExplicit": false,
|
||||
"timeoutMs": 1800000,
|
||||
"jobs": 8,
|
||||
"withinShardConcurrency": 2,
|
||||
"maxFilesPerShard": 1,
|
||||
"emitPlanPath": "manifest.json",
|
||||
"slices": 4,
|
||||
"planPath": null,
|
||||
"sliceIndex": null,
|
||||
"reportDir": null,
|
||||
"writeDurations": false
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"argv": [
|
||||
"--plan",
|
||||
"manifest.json",
|
||||
"--slice",
|
||||
"2"
|
||||
],
|
||||
"result": {
|
||||
"ok": {
|
||||
"tier": "gate",
|
||||
"profile": "full",
|
||||
"profileExplicit": false,
|
||||
"listOnly": false,
|
||||
"skipJudges": false,
|
||||
"timeoutExplicit": false,
|
||||
"timeoutMs": 1800000,
|
||||
"jobs": 8,
|
||||
"withinShardConcurrency": 2,
|
||||
"maxFilesPerShard": 1,
|
||||
"emitPlanPath": null,
|
||||
"slices": 1,
|
||||
"planPath": "manifest.json",
|
||||
"sliceIndex": 2,
|
||||
"reportDir": null,
|
||||
"writeDurations": false
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"argv": [
|
||||
"--report",
|
||||
"reports",
|
||||
"--write-durations"
|
||||
],
|
||||
"result": {
|
||||
"ok": {
|
||||
"tier": "gate",
|
||||
"profile": "full",
|
||||
"profileExplicit": false,
|
||||
"listOnly": false,
|
||||
"skipJudges": false,
|
||||
"timeoutExplicit": false,
|
||||
"timeoutMs": 1800000,
|
||||
"jobs": 8,
|
||||
"withinShardConcurrency": 2,
|
||||
"maxFilesPerShard": 1,
|
||||
"emitPlanPath": null,
|
||||
"slices": 1,
|
||||
"planPath": null,
|
||||
"sliceIndex": null,
|
||||
"reportDir": "reports",
|
||||
"writeDurations": true
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"argv": [],
|
||||
"env": {
|
||||
"EVALS_TIER": "periodic",
|
||||
"EVALS_JOBS": "3",
|
||||
"EVALS_CONCURRENCY": "5",
|
||||
"EVALS_SHARD_TIMEOUT_MS": "1000",
|
||||
"EVALS_PROFILE": "full"
|
||||
},
|
||||
"result": {
|
||||
"ok": {
|
||||
"tier": "periodic",
|
||||
"profile": "full",
|
||||
"profileExplicit": true,
|
||||
"listOnly": false,
|
||||
"skipJudges": false,
|
||||
"timeoutExplicit": true,
|
||||
"timeoutMs": 1000,
|
||||
"jobs": 3,
|
||||
"withinShardConcurrency": 5,
|
||||
"maxFilesPerShard": 1,
|
||||
"emitPlanPath": null,
|
||||
"slices": 1,
|
||||
"planPath": null,
|
||||
"sliceIndex": null,
|
||||
"reportDir": null,
|
||||
"writeDurations": false
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"argv": [
|
||||
"--bogus"
|
||||
],
|
||||
"result": {
|
||||
"error": "Unknown argument: --bogus"
|
||||
}
|
||||
},
|
||||
{
|
||||
"argv": [
|
||||
"--list",
|
||||
"--bogus"
|
||||
],
|
||||
"result": {
|
||||
"error": "Unknown argument: --bogus"
|
||||
}
|
||||
},
|
||||
{
|
||||
"argv": [
|
||||
"--tier",
|
||||
"e2e"
|
||||
],
|
||||
"result": {
|
||||
"error": "--tier must be gate or periodic. Received: e2e"
|
||||
}
|
||||
},
|
||||
{
|
||||
"argv": [
|
||||
"--tier"
|
||||
],
|
||||
"result": {
|
||||
"error": "--tier must be gate or periodic. Received: undefined"
|
||||
}
|
||||
},
|
||||
{
|
||||
"argv": [
|
||||
"--profile"
|
||||
],
|
||||
"result": {
|
||||
"error": "--profile needs pr or full"
|
||||
}
|
||||
},
|
||||
{
|
||||
"argv": [
|
||||
"--profile",
|
||||
"fast"
|
||||
],
|
||||
"result": {
|
||||
"error": "--profile must be pr or full. Received: fast"
|
||||
}
|
||||
},
|
||||
{
|
||||
"argv": [
|
||||
"--timeout",
|
||||
"0"
|
||||
],
|
||||
"result": {
|
||||
"error": "--timeout needs a positive integer. Received: 0"
|
||||
}
|
||||
},
|
||||
{
|
||||
"argv": [
|
||||
"--jobs"
|
||||
],
|
||||
"result": {
|
||||
"error": "--jobs needs a positive integer. Received: undefined"
|
||||
}
|
||||
},
|
||||
{
|
||||
"argv": [
|
||||
"--emit-plan"
|
||||
],
|
||||
"result": {
|
||||
"error": "--emit-plan needs a file path"
|
||||
}
|
||||
},
|
||||
{
|
||||
"argv": [
|
||||
"--plan"
|
||||
],
|
||||
"result": {
|
||||
"error": "--plan needs a manifest path"
|
||||
}
|
||||
},
|
||||
{
|
||||
"argv": [
|
||||
"--report"
|
||||
],
|
||||
"result": {
|
||||
"error": "--report needs a directory"
|
||||
}
|
||||
},
|
||||
{
|
||||
"argv": [
|
||||
"--write-durations"
|
||||
],
|
||||
"result": {
|
||||
"error": "--write-durations requires --report"
|
||||
}
|
||||
},
|
||||
{
|
||||
"argv": [
|
||||
"--skip-judges"
|
||||
],
|
||||
"result": {
|
||||
"error": "--skip-judges applies only to an emitted gate census plan"
|
||||
}
|
||||
},
|
||||
{
|
||||
"argv": [
|
||||
"--profile",
|
||||
"pr",
|
||||
"--tier",
|
||||
"periodic"
|
||||
],
|
||||
"result": {
|
||||
"error": "PR profile requires gate tier"
|
||||
}
|
||||
},
|
||||
{
|
||||
"argv": [
|
||||
"--profile",
|
||||
"pr",
|
||||
"--files-per-shard",
|
||||
"2"
|
||||
],
|
||||
"result": {
|
||||
"error": "PR profile requires one file per shard to preserve case accounting"
|
||||
}
|
||||
},
|
||||
{
|
||||
"argv": [],
|
||||
"env": {
|
||||
"EVALS_TIER": "e2e"
|
||||
},
|
||||
"result": {
|
||||
"error": "EVALS_TIER must be gate or periodic. Received: e2e"
|
||||
}
|
||||
}
|
||||
],
|
||||
"unknownArgumentCli": {
|
||||
"free": {
|
||||
"status": 1,
|
||||
"stderr": "[test:free] Unknown argument: --bogus"
|
||||
},
|
||||
"paid": {
|
||||
"status": 1,
|
||||
"stderr": "[test:paid] Unknown argument: --bogus"
|
||||
}
|
||||
}
|
||||
}
|
||||
+127
@@ -0,0 +1,127 @@
|
||||
{
|
||||
"recordedFrom": "96764e80 (pre-engine base) via test/fixtures/shard-equivalence/record.ts",
|
||||
"free": {
|
||||
"fixtures": {
|
||||
"pass": {
|
||||
"status": "passed",
|
||||
"exitCode": 0,
|
||||
"laneExit": 0
|
||||
},
|
||||
"fail": {
|
||||
"status": "failed",
|
||||
"exitCode": 1,
|
||||
"laneExit": 1
|
||||
},
|
||||
"skip": {
|
||||
"status": "passed",
|
||||
"exitCode": 0,
|
||||
"laneExit": 0
|
||||
},
|
||||
"wall-timeout": {
|
||||
"status": "timed-out",
|
||||
"exitCode": null,
|
||||
"laneExit": 124
|
||||
},
|
||||
"module-load-error": {
|
||||
"status": "failed",
|
||||
"exitCode": 1,
|
||||
"laneExit": 1
|
||||
},
|
||||
"zero-executed": {
|
||||
"status": "passed",
|
||||
"exitCode": 0,
|
||||
"laneExit": 0
|
||||
},
|
||||
"unhandled-between-tests": {
|
||||
"status": "failed",
|
||||
"exitCode": 1,
|
||||
"laneExit": 1
|
||||
}
|
||||
},
|
||||
"laneExit": 124
|
||||
},
|
||||
"paid": {
|
||||
"fixtures": {
|
||||
"pass": {
|
||||
"status": "passed",
|
||||
"exitCode": 0,
|
||||
"executedTests": 1,
|
||||
"skippedTests": 0,
|
||||
"laneExit": {
|
||||
"selective": 0,
|
||||
"evalsAll": 0
|
||||
}
|
||||
},
|
||||
"fail": {
|
||||
"status": "failed",
|
||||
"exitCode": 1,
|
||||
"executedTests": 1,
|
||||
"skippedTests": 0,
|
||||
"laneExit": {
|
||||
"selective": 1,
|
||||
"evalsAll": 1
|
||||
}
|
||||
},
|
||||
"skip": {
|
||||
"status": "passed",
|
||||
"exitCode": 0,
|
||||
"executedTests": 1,
|
||||
"skippedTests": 1,
|
||||
"laneExit": {
|
||||
"selective": 0,
|
||||
"evalsAll": 0
|
||||
}
|
||||
},
|
||||
"wall-timeout": {
|
||||
"status": "timed-out",
|
||||
"exitCode": null,
|
||||
"executedTests": null,
|
||||
"skippedTests": null,
|
||||
"laneExit": {
|
||||
"selective": 1,
|
||||
"evalsAll": 1
|
||||
}
|
||||
},
|
||||
"module-load-error": {
|
||||
"status": "failed",
|
||||
"exitCode": 1,
|
||||
"executedTests": 1,
|
||||
"skippedTests": 0,
|
||||
"laneExit": {
|
||||
"selective": 1,
|
||||
"evalsAll": 1
|
||||
}
|
||||
},
|
||||
"zero-executed": {
|
||||
"status": "passed",
|
||||
"exitCode": 0,
|
||||
"executedTests": 0,
|
||||
"skippedTests": 0,
|
||||
"laneExit": {
|
||||
"selective": 0,
|
||||
"evalsAll": 1
|
||||
}
|
||||
},
|
||||
"unhandled-between-tests": {
|
||||
"status": "failed",
|
||||
"exitCode": 1,
|
||||
"executedTests": 0,
|
||||
"skippedTests": 0,
|
||||
"laneExit": {
|
||||
"selective": 1,
|
||||
"evalsAll": 1
|
||||
}
|
||||
}
|
||||
},
|
||||
"laneExit": {
|
||||
"selective": 1,
|
||||
"evalsAll": 1
|
||||
}
|
||||
},
|
||||
"realShard": {
|
||||
"file": "test/strict-output.test.ts",
|
||||
"status": "passed",
|
||||
"filesRan": 1,
|
||||
"sawTerminalSummary": true
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
import { expect, test } from 'bun:test';
|
||||
|
||||
test('fails', () => { expect(1 + 1).toBe(3); });
|
||||
@@ -0,0 +1,5 @@
|
||||
import { test } from 'bun:test';
|
||||
|
||||
throw new Error('module load failure fixture');
|
||||
|
||||
test('never registered', () => {});
|
||||
@@ -0,0 +1,3 @@
|
||||
import { expect, test } from 'bun:test';
|
||||
|
||||
test('passes', () => { expect(1 + 1).toBe(2); });
|
||||
+67
@@ -0,0 +1,67 @@
|
||||
/**
|
||||
* Runs the shard-equivalence fixture corpus through one checkout's free and
|
||||
* paid runners and prints their classifications as one JSON line.
|
||||
*
|
||||
* bun test/fixtures/shard-equivalence/record.ts <checkout-root>
|
||||
*
|
||||
* expected.json was recorded by running this against the pre-engine base
|
||||
* commit (96764e80); test/shard-engine-equivalence.test.ts runs it against
|
||||
* the current checkout and requires identical classifications and lane exits.
|
||||
* Never imported by the test runner.
|
||||
*/
|
||||
import * as fs from 'node:fs';
|
||||
import * as os from 'node:os';
|
||||
import * as path from 'node:path';
|
||||
|
||||
export const FIXTURES = [
|
||||
'pass', 'fail', 'skip', 'wall-timeout', 'module-load-error', 'zero-executed', 'unhandled-between-tests',
|
||||
] as const;
|
||||
export const REAL_SHARD = 'test/strict-output.test.ts';
|
||||
const WALL_MS = 3_000;
|
||||
|
||||
const root = path.resolve(process.argv[2] ?? path.join(import.meta.dir, '../../..'));
|
||||
const fixtureDir = import.meta.dir;
|
||||
const scratch = fs.mkdtempSync(path.join(os.tmpdir(), 'shard-equivalence-'));
|
||||
const free = await import(path.join(root, 'scripts/test-free-shards.ts'));
|
||||
const paid = await import(path.join(root, 'scripts/test-paid-shards.ts'));
|
||||
const freeExit = (status: string) => (status === 'passed' ? 0 : status === 'timed-out' ? 124 : 1);
|
||||
|
||||
try {
|
||||
const runFree = (file: string, index: number) => free.runFreeShard([file], index + 1, FIXTURES.length, {
|
||||
rootDir: root, quiet: true, log: () => {}, wallTimeoutMs: WALL_MS,
|
||||
logFilePath: path.join(scratch, `free-${index}.log`),
|
||||
});
|
||||
const runPaid = (file: string, index: number) => paid.runPaidShard([file], index + 1, FIXTURES.length, {
|
||||
rootDir: root, timeoutMs: WALL_MS, jobs: 2, logDir: scratch, log: () => {},
|
||||
env: { ...process.env, GSTACK_CLAUDE_CLI_VERSION: 'fixture', EVALS: '', EVALS_TIER: '', EVALS_ALL: '' },
|
||||
});
|
||||
const files = FIXTURES.map(name => path.join(fixtureDir, `${name}.fixture.ts`));
|
||||
const [freeOutcomes, paidOutcomes, real] = await Promise.all([
|
||||
Promise.all(files.map(runFree)),
|
||||
Promise.all(files.map(runPaid)),
|
||||
free.runFreeShard([REAL_SHARD], 1, 1, {
|
||||
rootDir: root, quiet: true, log: () => {}, logFilePath: path.join(scratch, 'real.log'),
|
||||
}),
|
||||
]);
|
||||
const byName = <T>(values: T[]) => Object.fromEntries(FIXTURES.map((name, index) => [name, values[index]]));
|
||||
// Lane exit for a run of just that fixture, and for the whole corpus.
|
||||
const paidExit = (outcomes: unknown[], evalsAll: boolean) =>
|
||||
paid.summaryExitCode(paid.summarize(paid.applyHollowShardGuard(outcomes, { evalsAll, warn: () => {} })));
|
||||
const record = {
|
||||
free: {
|
||||
fixtures: byName(freeOutcomes.map((o: any) => ({ status: o.status, exitCode: o.exitCode, laneExit: freeExit(o.status) }))),
|
||||
laneExit: Math.max(...freeOutcomes.map((o: any) => freeExit(o.status))),
|
||||
},
|
||||
paid: {
|
||||
fixtures: byName(paidOutcomes.map((o: any) => ({
|
||||
status: o.status, exitCode: o.exitCode, executedTests: o.executedTests, skippedTests: o.skippedTests,
|
||||
laneExit: { selective: paidExit([o], false), evalsAll: paidExit([o], true) },
|
||||
}))),
|
||||
laneExit: { selective: paidExit(paidOutcomes, false), evalsAll: paidExit(paidOutcomes, true) },
|
||||
},
|
||||
realShard: { file: REAL_SHARD, status: real.status, ...real.summary },
|
||||
};
|
||||
console.log(`SHARD_EQUIVALENCE:${JSON.stringify(record)}`);
|
||||
} finally {
|
||||
fs.rmSync(scratch, { recursive: true, force: true });
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
import { test } from 'bun:test';
|
||||
|
||||
test.skip('skipped', () => {});
|
||||
@@ -0,0 +1,6 @@
|
||||
import { test } from 'bun:test';
|
||||
|
||||
test('first', () => {});
|
||||
// Rejects outside any test: bun prints "# Unhandled error between tests".
|
||||
Promise.reject(new Error('unhandled between tests fixture'));
|
||||
test('second', () => {});
|
||||
@@ -0,0 +1,7 @@
|
||||
import { test } from 'bun:test';
|
||||
|
||||
// Blocks the main thread, so no in-process timer can end it: only the
|
||||
// runner's external wall-clock group kill can.
|
||||
test('blocks past the wall deadline', () => {
|
||||
Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 60_000);
|
||||
});
|
||||
@@ -0,0 +1,2 @@
|
||||
// Declares no tests: bun reports "Ran 0 tests across 1 file".
|
||||
export const noTests = true;
|
||||
@@ -0,0 +1,33 @@
|
||||
/**
|
||||
* Per-lane CLI parity across the shard-engine refactor: each runner accepts
|
||||
* the same flags with the same defaults and validation messages, and emits
|
||||
* the same `Unknown argument:` error, as the pre-engine runners recorded in
|
||||
* test/fixtures/shard-cli-parity/results.json (base 96764e80).
|
||||
*/
|
||||
import { describe, expect, test } from 'bun:test';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import { parseCliOptions as parseFree } from '../scripts/test-free-shards';
|
||||
import { parseCliOptions as parsePaid } from '../scripts/test-paid-shards';
|
||||
import { FREE_CASES, PAID_CASES, capture } from './fixtures/shard-cli-parity/cases';
|
||||
|
||||
const ROOT = path.resolve(import.meta.dir, '..');
|
||||
const recorded = JSON.parse(fs.readFileSync(path.join(ROOT, 'test/fixtures/shard-cli-parity/results.json'), 'utf8'));
|
||||
|
||||
describe('shard runner CLI parity with the pre-engine runners', () => {
|
||||
test('free lane: same flags, defaults and validation errors', () => {
|
||||
expect(FREE_CASES.map(argv => ({ argv, result: capture(() => parseFree(argv)) }))).toEqual(recorded.free);
|
||||
});
|
||||
|
||||
test('paid lane: same flags, env-derived defaults and validation errors', () => {
|
||||
expect(PAID_CASES.map(c => ({ ...c, result: capture(() => parsePaid(c.argv, c.env ?? {})) }))).toEqual(recorded.paid);
|
||||
});
|
||||
|
||||
test.each(['free', 'paid'] as const)('%s CLI rejects an unknown flag with the same message and exit code', lane => {
|
||||
const run = spawnSync(process.execPath, [path.join(ROOT, `scripts/test-${lane}-shards.ts`), '--bogus'], {
|
||||
cwd: ROOT, encoding: 'utf8', timeout: 30_000,
|
||||
});
|
||||
expect({ status: run.status, stderr: run.stderr.trim() }).toEqual(recorded.unknownArgumentCli[lane]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,52 @@
|
||||
/**
|
||||
* W2 runner equivalence: the free and paid lanes, now on the shared shard
|
||||
* engine, classify a fixture corpus (pass, fail, skip, wall-timeout,
|
||||
* module-load error, zero executed tests, unhandled error between tests)
|
||||
* exactly as the pre-engine runners did. expected.json was recorded by the
|
||||
* same recorder against base commit 96764e80; one real suite shard is run
|
||||
* too. Re-record only for a deliberate, reviewed lane-policy change.
|
||||
*/
|
||||
import { describe, expect, test } from 'bun:test';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import { collectFreeTestFiles } from '../scripts/test-free-shards';
|
||||
|
||||
const ROOT = path.resolve(import.meta.dir, '..');
|
||||
const CORPUS = path.join(ROOT, 'test/fixtures/shard-equivalence');
|
||||
|
||||
describe('shard engine runner equivalence (fixture corpus)', () => {
|
||||
test('both lanes keep the base runners\' per-file classifications and lane exit codes', () => {
|
||||
const run = spawnSync(process.execPath, [path.join(CORPUS, 'record.ts'), ROOT], {
|
||||
cwd: ROOT, encoding: 'utf8', timeout: 90_000,
|
||||
env: { ...process.env, EVALS: '', EVALS_TIER: '', EVALS_ALL: '' },
|
||||
});
|
||||
expect(run.error, run.stderr).toBeUndefined();
|
||||
expect(run.status, run.stderr).toBe(0);
|
||||
const line = run.stdout.split('\n').find(entry => entry.startsWith('SHARD_EQUIVALENCE:'));
|
||||
expect(line, run.stdout + run.stderr).toBeDefined();
|
||||
const actual = JSON.parse(line!.slice('SHARD_EQUIVALENCE:'.length));
|
||||
const expected = JSON.parse(fs.readFileSync(path.join(CORPUS, 'expected.json'), 'utf8'));
|
||||
|
||||
expect(actual.free).toEqual(expected.free);
|
||||
expect(actual.paid).toEqual(expected.paid);
|
||||
// The real shard's test count moves as that file grows; its verdict and
|
||||
// complete file accounting must not.
|
||||
expect(actual.realShard).toMatchObject(expected.realShard);
|
||||
expect(actual.realShard.testsRan).toBeGreaterThan(0);
|
||||
}, 120_000);
|
||||
|
||||
test('the corpus covers every outcome kind and stays outside the free census', () => {
|
||||
const fixtures = fs.readdirSync(CORPUS).filter(name => name.endsWith('.fixture.ts')).sort();
|
||||
expect(fixtures).toEqual([
|
||||
'fail.fixture.ts', 'module-load-error.fixture.ts', 'pass.fixture.ts', 'skip.fixture.ts',
|
||||
'unhandled-between-tests.fixture.ts', 'wall-timeout.fixture.ts', 'zero-executed.fixture.ts',
|
||||
]);
|
||||
const expected = JSON.parse(fs.readFileSync(path.join(CORPUS, 'expected.json'), 'utf8'));
|
||||
for (const lane of ['free', 'paid']) {
|
||||
expect(Object.keys(expected[lane].fixtures).sort()).toEqual(fixtures.map(name => name.replace('.fixture.ts', '')));
|
||||
}
|
||||
// A failing fixture inside the census would fail every full free run.
|
||||
expect(collectFreeTestFiles(ROOT).filter(file => file.startsWith('test/fixtures/shard-equivalence/'))).toEqual([]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,273 @@
|
||||
/**
|
||||
* Unit pins for the shared shard engine (scripts/lib/shard-engine.ts): the
|
||||
* lane-exit consequences of failing, unhandled and module-load output, each
|
||||
* lane's injected zero-execution and seed rules, whole-group kill on a wall
|
||||
* timeout, the win32 path, log capture, sandbox isolation and the flag loop.
|
||||
* run-shard-child and strict-output-* tests pin the moved primitives.
|
||||
*/
|
||||
import { afterEach, describe, expect, spyOn, test } from 'bun:test';
|
||||
import type { ChildProcess } from 'node:child_process';
|
||||
import * as fs from 'node:fs';
|
||||
import * as os from 'node:os';
|
||||
import * as path from 'node:path';
|
||||
import {
|
||||
BunTestOutputClassifier,
|
||||
createShardSandbox,
|
||||
killProcessGroup,
|
||||
nextShardLogPath,
|
||||
openShardLog,
|
||||
parseCliFlags,
|
||||
readDurationSeed,
|
||||
removeShardSandbox,
|
||||
runShardChild,
|
||||
strictShardStatus,
|
||||
writeDurationSeed,
|
||||
zeroExecutionVerdict,
|
||||
} from '../scripts/lib/shard-engine';
|
||||
import { FREE_LANE_POLICY, exitCodeFor, runFreeShard } from '../scripts/test-free-shards';
|
||||
import {
|
||||
PAID_LANE_POLICY, applyHollowShardGuard, runPaidShard, summarize, summaryExitCode, type ShardOutcome,
|
||||
} from '../scripts/test-paid-shards';
|
||||
|
||||
const scratch: string[] = [];
|
||||
const tempDir = (prefix: string) => {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), prefix));
|
||||
scratch.push(dir);
|
||||
return dir;
|
||||
};
|
||||
afterEach(() => { for (const dir of scratch.splice(0)) fs.rmSync(dir, { recursive: true, force: true }); });
|
||||
|
||||
function summaryOf(output: string) {
|
||||
const classifier = new BunTestOutputClassifier();
|
||||
classifier.write(output, 'stderr');
|
||||
return classifier.end();
|
||||
}
|
||||
|
||||
const PASSING = 'test/a.test.ts:\n(pass) a [1.00ms]\n\n 1 pass\n 0 fail\nRan 1 test across 1 file. [2.00ms]\n';
|
||||
// Bun 1.3 output shapes observed for these outcomes (fixture corpus).
|
||||
const FAILING = 'test/a.test.ts:\n(fail) a [1.00ms]\n\n 0 pass\n 1 fail\nRan 1 test across 1 file. [2.00ms]\n';
|
||||
const MODULE_LOAD = 'test/a.test.ts:\nerror: module load failure\n\n 0 pass\n 1 fail\n 1 error\nRan 1 test across 1 file. [2.00ms]\n';
|
||||
const UNHANDLED = 'test/a.test.ts:\n# Unhandled error between tests\nerror: boom\n\n 0 pass\n 0 fail\n 1 error\nRan 0 tests across 1 file. [2.00ms]\n';
|
||||
|
||||
describe('shard engine: verdicts and lane exits', () => {
|
||||
const verdict = (output: string, exitCode: number | null, timedOut = false) => strictShardStatus({
|
||||
timedOut, exitCode, summary: summaryOf(output), expectedFiles: 1, evidenceComplete: true,
|
||||
});
|
||||
const paidExit = (status: ShardOutcome['status']) => summaryExitCode(summarize([{
|
||||
shard: 1, files: ['test/a.test.ts'], status, exitCode: 0, elapsedMs: 1, groupPid: null, executedTests: 1, skippedTests: 0,
|
||||
}]));
|
||||
|
||||
test('a failing test file makes both lanes exit nonzero, even when bun exits 0', () => {
|
||||
for (const exitCode of [0, 1]) {
|
||||
const status = verdict(FAILING, exitCode);
|
||||
expect(status).toBe('failed');
|
||||
expect(exitCodeFor(status)).toBe(1);
|
||||
expect(paidExit(status)).toBe(1);
|
||||
}
|
||||
expect(exitCodeFor(verdict(PASSING, 0))).toBe(0);
|
||||
expect(paidExit(verdict(PASSING, 0))).toBe(0);
|
||||
});
|
||||
|
||||
test('unhandled between-test errors and module-load errors count as failures', () => {
|
||||
expect(summaryOf(UNHANDLED).unhandledBetweenTests).toBe(1);
|
||||
expect(verdict(UNHANDLED, 0)).toBe('failed');
|
||||
expect(summaryOf(MODULE_LOAD).failedTests).toBe(1);
|
||||
expect(verdict(MODULE_LOAD, 0)).toBe('failed');
|
||||
expect(verdict(MODULE_LOAD, 1)).toBe('failed');
|
||||
});
|
||||
|
||||
test('missing summary, short file count or incomplete evidence never pass; a timeout is its own status', () => {
|
||||
expect(verdict('(pass) a [1.00ms]\n', 0)).toBe('failed');
|
||||
expect(strictShardStatus({ timedOut: false, exitCode: 0, summary: summaryOf(PASSING), expectedFiles: 2, evidenceComplete: true })).toBe('failed');
|
||||
expect(strictShardStatus({ timedOut: false, exitCode: 0, summary: summaryOf(PASSING), expectedFiles: 1, evidenceComplete: false })).toBe('failed');
|
||||
expect(verdict(PASSING, 0, true)).toBe('timed-out');
|
||||
expect(exitCodeFor('timed-out')).toBe(124);
|
||||
expect(paidExit('timed-out')).toBe(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe('shard engine: injected lane policy', () => {
|
||||
const zero = (executedTests: number | null): ShardOutcome => ({
|
||||
shard: 1, files: ['test/a.test.ts'], status: 'passed', exitCode: 0, elapsedMs: 1, groupPid: null, executedTests, skippedTests: 0,
|
||||
});
|
||||
|
||||
test('free lane: zero executed tests pass when every planned file was counted', () => {
|
||||
for (const promisedAll of [false, true]) {
|
||||
expect(zeroExecutionVerdict(0, FREE_LANE_POLICY, { promisedAll })).toBe('passed');
|
||||
}
|
||||
const zeroTests = 'test/a.test.ts:\n\n 0 pass\n 0 fail\nRan 0 tests across 1 file. [2.00ms]\n';
|
||||
expect(strictShardStatus({ timedOut: false, exitCode: 0, summary: summaryOf(zeroTests), expectedFiles: 1, evidenceComplete: true })).toBe('passed');
|
||||
});
|
||||
|
||||
test('paid lane: zero executed tests warn under selection and are passed-empty under EVALS_ALL', () => {
|
||||
expect(zeroExecutionVerdict(0, PAID_LANE_POLICY, { promisedAll: false })).toBe('passed-with-warning');
|
||||
expect(zeroExecutionVerdict(0, PAID_LANE_POLICY, { promisedAll: true })).toBe('passed-empty');
|
||||
expect(zeroExecutionVerdict(3, PAID_LANE_POLICY, { promisedAll: true })).toBe('passed');
|
||||
const warnings: string[] = [];
|
||||
const selective = applyHollowShardGuard([zero(0)], { evalsAll: false, warn: line => warnings.push(line) });
|
||||
expect(selective[0].status).toBe('passed');
|
||||
expect(warnings).toHaveLength(1);
|
||||
expect(summaryExitCode(summarize(selective))).toBe(0);
|
||||
const all = applyHollowShardGuard([zero(0)], { evalsAll: true, warn: () => {} });
|
||||
expect(all[0].status).toBe('passed-empty');
|
||||
expect(summaryExitCode(summarize(all))).toBe(1);
|
||||
expect(applyHollowShardGuard([zero(null)], { evalsAll: true, warn: () => {} })[0].status).toBe('passed');
|
||||
});
|
||||
|
||||
test('duration seeds: one reader, lane predicate decides (free >= 0, paid > 0)', () => {
|
||||
const dir = tempDir('seed-');
|
||||
const file = path.join(dir, 'seed.json');
|
||||
fs.writeFileSync(file, JSON.stringify({ durations: { 'a.ts': 0, 'b.ts': 5, 'c.ts': -1, 'd.ts': 'x', 'e.ts': Infinity } }));
|
||||
expect(readDurationSeed(file, FREE_LANE_POLICY.acceptsSeedDuration)).toEqual({ status: 'ok', durations: { 'a.ts': 0, 'b.ts': 5 } });
|
||||
expect(readDurationSeed(file, PAID_LANE_POLICY.acceptsSeedDuration)).toEqual({ status: 'ok', durations: { 'b.ts': 5 } });
|
||||
expect(readDurationSeed(path.join(dir, 'missing.json'), () => true).status).toBe('missing');
|
||||
for (const corrupt of ['{not json', 'null']) {
|
||||
fs.writeFileSync(file, corrupt);
|
||||
expect(readDurationSeed(file, () => true).status).toBe('corrupt');
|
||||
}
|
||||
writeDurationSeed(file, { 'z.ts': 2, 'a.ts': 1 });
|
||||
const written = JSON.parse(fs.readFileSync(file, 'utf8'));
|
||||
expect(written.version).toBe(1);
|
||||
expect(Object.keys(written.durations)).toEqual(['a.ts', 'z.ts']);
|
||||
expect(fs.readdirSync(dir)).toEqual(['seed.json']);
|
||||
});
|
||||
});
|
||||
|
||||
describe('shard engine: process lifecycle', () => {
|
||||
test.skipIf(process.platform === 'win32')('a timed-out shard\'s whole process group is killed and the lanes report it as not passed', async () => {
|
||||
const dir = tempDir('group-kill-');
|
||||
const pidFile = path.join(dir, 'grandchild.pid');
|
||||
const spinner = `const { spawn } = require('node:child_process');
|
||||
const p = spawn(process.execPath, ['-e', 'setInterval(() => {}, 1000)'], { stdio: 'ignore' });
|
||||
require('node:fs').writeFileSync(${JSON.stringify(pidFile)}, String(p.pid));
|
||||
const end = Date.now() + 600000; while (Date.now() < end) {}`;
|
||||
const alive = (pid: number) => {
|
||||
try {
|
||||
process.kill(pid, 0);
|
||||
return !(process.platform === 'linux' && fs.readFileSync(`/proc/${pid}/stat`, 'utf8').split(') ')[1].startsWith('Z'));
|
||||
} catch { return false; }
|
||||
};
|
||||
const waitDead = async (pid: number) => {
|
||||
const deadline = Date.now() + 2_000;
|
||||
while (alive(pid) && Date.now() < deadline) await Bun.sleep(20);
|
||||
return !alive(pid);
|
||||
};
|
||||
const free = await runFreeShard(['spin.test.ts'], 1, 1, {
|
||||
rootDir: dir, quiet: true, log: () => {}, wallTimeoutMs: 1_500, logFilePath: path.join(dir, 'free.log'),
|
||||
commandFor: () => ({ command: process.execPath, args: ['-e', spinner] }),
|
||||
});
|
||||
expect(free.status).toBe('timed-out');
|
||||
expect(exitCodeFor(free.status)).not.toBe(0);
|
||||
expect(await waitDead(Number(fs.readFileSync(pidFile, 'utf8')))).toBe(true);
|
||||
expect(await waitDead(free.groupPid!)).toBe(true);
|
||||
|
||||
fs.rmSync(pidFile);
|
||||
const paid = await runPaidShard(['spin.test.ts'], 1, 1, {
|
||||
rootDir: dir, timeoutMs: 1_500, jobs: 2, logDir: dir, log: () => {},
|
||||
env: { ...process.env, GSTACK_CLAUDE_CLI_VERSION: 'fixture' },
|
||||
commandFor: () => ({ command: process.execPath, args: ['-e', spinner] }),
|
||||
});
|
||||
expect(paid.status).toBe('timed-out');
|
||||
expect(summaryExitCode(summarize([paid]))).toBe(1);
|
||||
expect(await waitDead(Number(fs.readFileSync(pidFile, 'utf8')))).toBe(true);
|
||||
}, 30_000);
|
||||
|
||||
test('win32 semantics: no process group and never a negative-pid kill', async () => {
|
||||
const original = Object.getOwnPropertyDescriptor(process, 'platform')!;
|
||||
const kills = spyOn(process, 'kill');
|
||||
Object.defineProperty(process, 'platform', { value: 'win32' });
|
||||
try {
|
||||
const signalled: string[] = [];
|
||||
killProcessGroup({ pid: 4242, kill: (signal: string) => { signalled.push(signal); return true; } } as unknown as ChildProcess, 'SIGKILL');
|
||||
expect(signalled).toEqual(['SIGKILL']);
|
||||
const result = await runShardChild({
|
||||
command: process.execPath, args: ['-e', 'setInterval(() => {}, 1000)'],
|
||||
cwd: process.cwd(), env: process.env, timeoutMs: 500, hookStreams: () => [],
|
||||
});
|
||||
expect(result.timedOut).toBe(true);
|
||||
expect(kills.mock.calls.some(([pid]) => typeof pid === 'number' && pid < 0)).toBe(false);
|
||||
} finally {
|
||||
Object.defineProperty(process, 'platform', original);
|
||||
kills.mockRestore();
|
||||
}
|
||||
}, 30_000);
|
||||
|
||||
test('an attached companion settles after the final group kill', async () => {
|
||||
let settledAfterExit = false;
|
||||
let child: ChildProcess | undefined;
|
||||
const result = await runShardChild({
|
||||
command: process.execPath, args: ['-e', 'console.log("done")'],
|
||||
cwd: process.cwd(), env: process.env, timeoutMs: 10_000,
|
||||
hookStreams: spawned => [new Promise<void>(resolve => { spawned.stdout!.resume(); spawned.stdout!.once('end', resolve); })],
|
||||
attach: spawned => {
|
||||
child = spawned;
|
||||
return { signal: () => {}, settle: async () => { settledAfterExit = spawned.exitCode !== null || spawned.signalCode !== null; } };
|
||||
},
|
||||
});
|
||||
expect(result.exitCode).toBe(0);
|
||||
expect(child).toBeDefined();
|
||||
expect(settledAfterExit).toBe(true);
|
||||
}, 30_000);
|
||||
});
|
||||
|
||||
describe('shard engine: capture and isolation', () => {
|
||||
test('log capture spools every chunk; a write error fails the log once, without throwing', async () => {
|
||||
const dir = tempDir('shard-log-');
|
||||
const first = nextShardLogPath(dir, 'gstack-free-test');
|
||||
const second = nextShardLogPath(dir, 'gstack-free-test');
|
||||
expect(first).not.toBe(second);
|
||||
expect(path.basename(first)).toMatch(new RegExp(`^gstack-free-test-.+-${process.pid}-\\d+\\.log$`));
|
||||
const log = openShardLog(first, '[test:unit]', 0o600);
|
||||
log.write('one\n');
|
||||
log.write(Buffer.from('two\n'));
|
||||
await new Promise<void>(resolve => log.stream.end(() => resolve()));
|
||||
expect(fs.readFileSync(first, 'utf8')).toBe('one\ntwo\n');
|
||||
if (process.platform !== 'win32') expect(fs.statSync(first).mode & 0o777).toBe(0o600);
|
||||
expect(log.failed).toBe(false);
|
||||
|
||||
const errors = spyOn(console, 'error').mockImplementation(() => {});
|
||||
try {
|
||||
const broken = openShardLog(path.join(dir, 'missing', 'x.log'), '[test:unit]');
|
||||
await new Promise(resolve => broken.stream.once('error', resolve));
|
||||
broken.write('dropped');
|
||||
expect(broken.failed).toBe(true);
|
||||
expect(errors.mock.calls.filter(([line]) => String(line).includes('could not write the full log'))).toHaveLength(1);
|
||||
} finally { errors.mockRestore(); }
|
||||
});
|
||||
|
||||
test('each shard gets its own tmp and Chromium profile, removed by the backstop', async () => {
|
||||
const base = { KEEP: '1', TMPDIR: '/inherited', CHROMIUM_PROFILE: '/shared-profile' };
|
||||
const a = createShardSandbox('gstack-unit-shard-', base);
|
||||
const b = createShardSandbox('gstack-unit-shard-', base, { realpath: true });
|
||||
try {
|
||||
expect(a.stateDir).not.toBe(b.stateDir);
|
||||
expect(b.stateDir).toBe(fs.realpathSync(b.stateDir));
|
||||
for (const box of [a, b]) {
|
||||
expect(box.env.KEEP).toBe('1');
|
||||
expect(box.env.GSTACK_HOME).toBeUndefined();
|
||||
for (const key of ['TMPDIR', 'TEMP', 'TMP']) expect(box.env[key]).toBe(box.tmp);
|
||||
expect(box.env.CHROMIUM_PROFILE).toBe(path.join(box.stateDir, 'chromium-profile'));
|
||||
expect(fs.statSync(box.tmp).isDirectory()).toBe(true);
|
||||
}
|
||||
expect(base.TMPDIR).toBe('/inherited');
|
||||
fs.writeFileSync(path.join(a.tmp, 'left-behind'), 'x');
|
||||
await removeShardSandbox(a.stateDir);
|
||||
expect(fs.existsSync(a.stateDir)).toBe(false);
|
||||
await removeShardSandbox(a.stateDir);
|
||||
} finally {
|
||||
fs.rmSync(a.stateDir, { recursive: true, force: true });
|
||||
fs.rmSync(b.stateDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('flag loop: declared handlers consume values; anything else is Unknown argument', () => {
|
||||
const seen: Array<[string, string | undefined]> = [];
|
||||
parseCliFlags(['--on', '--value', 'x', '--value'], {
|
||||
'--on': () => { seen.push(['--on', undefined]); },
|
||||
'--value': next => { seen.push(['--value', next()]); },
|
||||
});
|
||||
expect(seen).toEqual([['--on', undefined], ['--value', 'x'], ['--value', undefined]]);
|
||||
expect(() => parseCliFlags(['--nope'], {})).toThrow('Unknown argument: --nope');
|
||||
expect(() => parseCliFlags(['constructor'], {})).toThrow('Unknown argument: constructor');
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user