Run large unit suites on Linux instead of failing with argument list too long (#4424) (#4426)

* Run large unit suites on Linux instead of failing with argument list too long

A lane passed every unit file in one bash -c argument; Linux caps one
argument at 128 KiB, so thousands of files failed with E2BIG. Split the
file list into byte-bounded chunks run in sequence and read back as one
run, and classify E2BIG as its own arg-limit cause.

Fixes #4424

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* fix: pre-landing review fixes

Put the failing fixture file last after sorting so the arg-limit test really checks a failure in the final chunk.

Review-Finding: p1/specialist-batched#1 reader=testing
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* docs: note the Windows chunk size for split lane file lists

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

---------

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
This commit is contained in:
Garry Tanandcapy-ai[bot] authored and GitHub committed 2026-11-17 06:09:13 +00:00
1 parent 5dcd76b4ea
commit 5b4fa9ea33
6 files changed
+139 -11

No files matched your search

+82
View File
@@ -0,0 +1,82 @@
/**
* #4424: on Linux a local lane with thousands of unit files never started. runOnce put every file of a run group into
* one command string and supervise started it as `bash -c <string>`; Linux caps one execve argument at MAX_ARG_STRLEN
* (128 KiB), so posix_spawn returned E2BIG, and the launch failure read as a missing binary ("install the named tool").
* The fixture selects 700 test files with 200-character names (about 150 KB of quoted paths) and a runner that reports
* one test per file argument, so the lane's count proves every file reached a runner.
*/
import { afterEach, describe, expect, test } from 'bun:test';
import { argBudget, argChunks } from '../lib/pregate/checks/lanes';
import { buildContext } from '../lib/pregate/context';
import { classifyEnvironment } from '../lib/pregate/environment';
import { runPregate } from '../lib/pregate/index';
import { BASE_PREGATE, cleanup, commit, makeFixtureRepo, type FixtureRepo } from './helpers/pregate-fixture';
const repos: FixtureRepo[] = [];
afterEach(() => { while (repos.length) cleanup(repos.pop()!); });
const FILES = 700;
const name = (i: number, tag = 'x') => `test/big/${tag.padEnd(186, 'x')}${String(i).padStart(4, '0')}.test.js`;
const RUNNER = [
'const files = process.argv.slice(2);',
"const failed = files.filter(f => f.includes('fail')).length;",
"console.log(`ℹ tests ${files.length}\\nℹ pass ${files.length - failed}\\nℹ fail ${failed}`);",
'process.exit(failed ? 1 : 0);',
].join('\n');
async function lanesOver(tests: string[]) {
const repo = makeFixtureRepo({
pregate: { ...BASE_PREGATE, runner: 'node scripts/count.js {files}', dependencies: { ...BASE_PREGATE.dependencies, 'src/big/**': tests } },
baseFiles: Object.fromEntries([...tests.map(t => [t, "require('node:test');\n"]), ['src/big/.keep', ''], ['scripts/count.js', `${RUNNER}\n`]]),
});
repos.push(repo);
commit(repo, { 'src/big/f.js': 'exports.n = 1;\n' }, 'touch the big tree');
const loaded = buildContext({ repoRoot: repo.work, base: 'main', policyFrom: 'base', allowRepoCommands: true, env: { ...process.env, ...repo.env }, progress: () => {}, cores: 64, testBackend: 'local' });
if (!('ctx' in loaded)) throw new Error('fixture context refused');
const file = await runPregate(loaded.ctx, { stage: 'tests', only: ['lanes'] });
return file.checks.find(c => c.id === 'lanes')!;
}
describe.skipIf(process.platform === 'win32')('a local lane whose file list is over the per-argument limit', () => {
// Value: protects=a unit group whose quoted file list is over Linux's 128 KiB per-argument cap still launches and every file
// is counted; fails_when=the whole list goes into one `bash -c` argument (E2BIG on Linux: PREGATE_ENVIRONMENT, nothing ran);
// why_new=#4424, gbrain's 5,567 unit files on gstack 1.99.5.0; seam=runPregate --only lanes on a fixture repo with a
// counting runner; no_claim=macOS has no per-argument cap, so there the old code passed too
test('every file runs and is counted, and a failure in the last chunk still fails the lane', async () => {
const tests = Array.from({ length: FILES }, (_, i) => name(i));
expect(tests.map(t => `'${t}'`).join(' ').length).toBeGreaterThan(128 * 1024);
const green = await lanesOver(tests);
expect(green.detail).not.toContain('E2BIG');
expect(green).toMatchObject({ status: 'pass', detail: `${FILES} tests ran in ${FILES} file(s), exit 0` });
const red = await lanesOver([...tests.slice(0, -1), name(FILES - 1, 'zfail')]);
expect(red).toMatchObject({ status: 'fail', code: 'PREGATE_LANE_FAILED', detail: `local lane exited 1 (1 failed of ${FILES} ran; ${FILES} files; pipefail)` });
}, 120_000);
});
describe('argChunks and argBudget', () => {
// Value: protects=the split keeps order, drops nothing and keeps each chunk's quoted text under the budget, with an oversized
// argument alone in its chunk, and the budget stays under Linux's per-argument cap and shrinks with the environment;
// fails_when=a chunk passes the budget, a file is lost or reordered, or the budget ignores the cap; why_new=#4424; seam=none
test('splits in order under the byte budget, which stays under the argument limits', () => {
const args = ['a'.repeat(10), 'b'.repeat(10), "c'd".repeat(4), 'e'.repeat(40), 'f'];
const chunks = argChunks(args, 30);
expect(chunks).toEqual([[args[0], args[1]], [args[2]], [args[3]], [args[4]]]);
expect(argChunks([], 30)).toEqual([]);
expect(argChunks(args, 1_000)).toEqual([args]);
const linux = argBudget('bun test {files}', {}, 'linux');
expect(linux).toBeLessThan(128 * 1024);
expect(argBudget('bun test {files}', { BIG: 'x'.repeat(900 * 1024) }, 'linux')).toBeLessThan(1024 * 1024 - 64 * 1024 - 900 * 1024);
expect(argBudget('bun test {files}', {}, 'win32')).toBeLessThan(32 * 1024);
});
});
describe('E2BIG is its own cause', () => {
// Value: protects=an argument list over the system limit is named as such, never as a missing tool; fails_when=E2BIG
// classifies as missing-binary or as nothing (the launch path then defaulted to missing-binary); why_new=#4424's
// misleading "install the named tool"; seam=none
test('classifies E2BIG and "argument list too long" as arg-limit', () => {
expect(classifyEnvironment("E2BIG: argument list too long, posix_spawn 'bash'", null)).toEqual({ kind: 'arg-limit', evidence: "E2BIG: argument list too long, posix_spawn 'bash'", proof: 'output' });
expect(classifyEnvironment('bash: /usr/bin/node: Argument list too long', 126)?.kind).toBe('arg-limit');
});
});