diff --git a/changelog.d/lanes-large-file-lists-on-linux.md b/changelog.d/lanes-large-file-lists-on-linux.md new file mode 100644 index 000000000..68f8401de --- /dev/null +++ b/changelog.d/lanes-large-file-lists-on-linux.md @@ -0,0 +1 @@ +- **/ship's test step runs a large unit suite on Linux.** A lane with thousands of unit files used to stop with `PREGATE: incomplete` and `E2BIG: argument list too long` before any test ran, because every file went into one command argument and Linux caps a single argument at 128 KiB. gstack now splits a big file list into runner processes of about 96 KB each, run one after another and counted as one run, so the result is the same as before: any failing chunk fails the lane. An `E2BIG` launch failure is now reported as its own cause (`arg-limit`) instead of telling you to install a missing tool. (#4424) diff --git a/docs/pregate.md b/docs/pregate.md index 892af1ecf..7abffb24b 100644 --- a/docs/pregate.md +++ b/docs/pregate.md @@ -428,7 +428,11 @@ Two stages with separate clocks, both printed: `workflow.yml/job`, unless `local` names that lane. `--explain` prints each test's provenance and every uncertain mapping (a planner-computed matrix without a `selection` manifest selects conservatively). Test files matching `serial_patterns` (default - `**/*.serial.test.*`) run one process per file after the unit files. The + `**/*.serial.test.*`) run one process per file after the unit files. A unit + group whose quoted file list would pass the system's argument limits (Linux + caps one argument at 128 KiB) runs as several runner processes in sequence, + each under about 96 KB of file arguments (24 KB on Windows), read back as one run: counts add up, + any failing chunk fails the lane, and they share the group's one `--timeout`. The runner and every guard run with the verified launch Bun first on PATH: the first of `$BUN_INSTALL/bin` and `~/.bun/bin` whose Bun satisfies gstack's and the project's `engines.bun` (the row prints `runtime: bun from `), @@ -1858,6 +1862,7 @@ failure the branch caused carries `cause: "diff"` (`PREGATE_LANE_FAILED`, | Kind | Evidence | |---|---| | `missing-binary` | exit 127, `command not found`, a spawn `ENOENT`, a runner tool `not on PATH` (dart, flutter, cargo, a package manager, `python3`, or `pytest` when python3 cannot import it), Playwright's browsers not installed | +| `arg-limit` | `E2BIG`, `Argument list too long`: a command line or environment over the system's limit. gstack splits a lane's file list to stay under it, so this is a gstack bug to report | | `network` | `ENOTFOUND`, `ECONNREFUSED`, `ECONNRESET`, `ETIMEDOUT`, `EAI_AGAIN`, `Could not resolve host`, `fetch failed`, a TLS handshake or certificate failure | | `quota` | `quota`, `429 Too Many Requests`, `rate limit`, `insufficient_quota` (a vCPU quota on the remote backend) | | `disk` | `ENOSPC`, `No space left on device`, `EDQUOT` | diff --git a/lib/pregate/checks/lanes.ts b/lib/pregate/checks/lanes.ts index 219e33735..075b1bdb2 100644 --- a/lib/pregate/checks/lanes.ts +++ b/lib/pregate/checks/lanes.ts @@ -365,18 +365,56 @@ interface LocalRun { exit: number | null; outcome: Supervised['outcome']; signal const quote = (f: string) => `'${f.replace(/'/g, `'\\''`)}'`; +/** + * Bytes of quoted file arguments one runner process may take. supervise passes the whole command as one `bash -c` + * argument: Linux caps a single execve argument at 128 KiB (MAX_ARG_STRLEN), every POSIX system caps argv plus the + * environment (ARG_MAX: 1 MiB on macOS, 2 MiB by default on Linux), and Windows caps a whole command line at 32,767 + * characters. The runner text itself and a margin come off the top. + */ +export function argBudget(runner: string, env: NodeJS.ProcessEnv, platform = process.platform): number { + const envBytes = Object.entries(env).reduce((n, [k, v]) => n + Buffer.byteLength(`${k}=${v ?? ''}`) + 9, 0); + const cap = platform === 'win32' ? 24 * 1024 : Math.min(96 * 1024, 1024 * 1024 - 64 * 1024 - envBytes); + return cap - Buffer.byteLength(runner) - 512; +} + +/** `args` split in order into chunks whose quoted, space-joined text fits `budget` bytes; an argument over it runs alone. */ +export function argChunks(args: readonly string[], budget: number): string[][] { + const chunks: string[][] = []; + let size = 0; + for (const a of args) { + const n = Buffer.byteLength(quote(a)) + 1; + if (!chunks.length || size + n > budget) { chunks.push([]); size = 0; } + chunks.at(-1)!.push(a); + size += n; + } + return chunks; +} + async function runOnce(ctx: PregateContext, env: NodeJS.ProcessEnv, g: RunGroup, files: string[], log: string, counter: Counter): Promise { // playwright test reads each file argument as a case-insensitive regex: escape it and anchor it to a path segment, so `a[1].spec.ts` runs that file, not `a1.spec.ts`. const args = /(? `(?:^|/)${f.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}$`) : testFileArgs(g.runner, files); - const joined = args.map(quote).join(' '); - const run = g.runner.replace('{files}', () => joined); - const command = g.cwd ? `(cd ${quote(g.cwd)} && ${run})` : run; - const shown = g.cwd ? files.map(f => `${g.cwd}/${f}`) : files; - const r = await supervise({ command: run, cwd: path.join(ctx.repoRoot, g.cwd), env: { ...env, CI: env.CI ?? '1' }, log, deadline: Date.now() + ctx.timeoutMs, stallMs: ctx.stallTimeoutMs, label: `lanes (${shown.length === 1 ? shown[0] : `${shown.length} files`})`, files, adapter: adapterFor(g.runner), counter, progress: ctx.progress }); - const output = readRange(log, r.start, r.end); - const summary = parseSummary(output, g.runner); - const noBrowsers = /(? joined); + const shown = g.cwd ? part.map(f => `${g.cwd}/${f}`) : part; + const r = await supervise({ command: run, cwd: path.join(ctx.repoRoot, g.cwd), env: runEnv, log, deadline, stallMs: ctx.stallTimeoutMs, label: `lanes (${shown.length === 1 ? shown[0] : `${shown.length} files`})`, files: part, adapter: adapterFor(g.runner), counter, progress: ctx.progress }); + const output = readRange(log, r.start, r.end); + const summary = parseSummary(output, g.runner); + const noBrowsers = /(? p.exit === null) ?? parts.find(p => p.exit !== 0) ?? parts.at(-1)!; + const sum = (k: 'ran' | 'failed') => (parts.some(p => p[k] === null) ? null : parts.reduce((n, p) => n + (p[k] as number), 0)); + return { ...(parts.every(p => p.noBrowsers) ? { noBrowsers: parts[0]!.noBrowsers } : {}), exit: bad.exit, outcome: bad.outcome, signal: bad.signal, observedStart: parts.some(p => p.observedStart), ran: sum('ran'), failed: sum('failed'), tail: bad.tail, runner: g.runner, command: parts.map(p => p.command).join(' && '), files: parts.flatMap(p => p.files), lastFile: bad.lastFile, ms: parts.reduce((n, p) => n + p.ms, 0), ...(bad.error ? { error: bad.error } : {}) }; } /** diff --git a/lib/pregate/environment.ts b/lib/pregate/environment.ts index da70ab24d..97c2988c8 100644 --- a/lib/pregate/environment.ts +++ b/lib/pregate/environment.ts @@ -23,6 +23,7 @@ const ANSI = /\x1b\[[0-9;?]*[A-Za-z]/g; const NOT_FOUND = /command not found|is not recognized as an internal or external command/; const RULES: Array<{ kind: EnvironmentKind; re: RegExp }> = [ { kind: 'missing-binary', re: new RegExp(`${NOT_FOUND.source}|No such file or directory: ['"]?[\\w./-]+|spawn \\S+ ENOENT|\\bENOENT\\b.*\\bspawn\\b|not on PATH`) }, + { kind: 'arg-limit', re: /\bE2BIG\b|[Aa]rgument list too long/ }, { kind: 'quota', re: /\bquota\b|\b429 Too Many Requests|status(?: code)?[:= ]+429\b|HTTP(?:\/[\d.]+)? 429\b|\brate.?limit(?:ed)?\b|insufficient_quota/i }, { kind: 'network', re: /\bE(?:NOTFOUND|CONNREFUSED|CONNRESET|TIMEDOUT|HOSTUNREACH|NETUNREACH|AI_AGAIN|PIPE)\b|getaddrinfo|Could not resolve host|Connection refused|Connection reset|Network is unreachable|fetch failed|TLS handshake|certificate verify failed|UNABLE_TO_GET_ISSUER_CERT|Temporary failure in name resolution/ }, { kind: 'disk', re: /\bENOSPC\b|No space left on device|\bEDQUOT\b|Disk quota exceeded/ }, @@ -72,6 +73,7 @@ export function classifyCommandEnvironment(command: string, output: string, exit const FIX: Record = { 'missing-binary': 'install the named tool on this machine (or put it on PATH), or run the lane remotely (--remote, or clear it with a run URL)', + 'arg-limit': 'the command line or environment passed to the runner is over the system\'s argument limit (E2BIG); gstack splits a lane\'s file list to stay under it, so report this as a gstack bug with this line, and meanwhile run the lane remotely or split the selection', network: 'check the network, proxy or registry this machine reaches, then rerun; or run the lane remotely', quota: 'wait for the quota or raise it, free the backend\'s capacity, then rerun', disk: 'free disk space (or move the state root) and rerun', diff --git a/lib/pregate/types.ts b/lib/pregate/types.ts index f0186a33e..8bbb5d103 100644 --- a/lib/pregate/types.ts +++ b/lib/pregate/types.ts @@ -105,7 +105,7 @@ export interface CheckResult extends PregateCheck { environment?: EnvironmentCause; } -export type EnvironmentKind = 'missing-binary' | 'network' | 'quota' | 'disk' | 'runtime' | 'tool'; +export type EnvironmentKind = 'missing-binary' | 'arg-limit' | 'network' | 'quota' | 'disk' | 'runtime' | 'tool'; /** How the cause was established: `launch` (the command could not start, its binary is off PATH, the runtime or a gstack tool refused), or `output` (a rule matched the command's text before any test was counted, which the diff's own output could have written). */ export type EnvironmentProof = 'launch' | 'output'; export interface EnvironmentCause { kind: EnvironmentKind; evidence: string; proof: EnvironmentProof } diff --git a/test/pregate-lanes-arg-limit.test.ts b/test/pregate-lanes-arg-limit.test.ts new file mode 100644 index 000000000..2ffb60b54 --- /dev/null +++ b/test/pregate-lanes-arg-limit.test.ts @@ -0,0 +1,82 @@ +/** + * #4424: on Linux a local lane with thousands of unit files never started. runOnce put every file of a run group into + * one command string and supervise started it as `bash -c `; Linux caps one execve argument at MAX_ARG_STRLEN + * (128 KiB), so posix_spawn returned E2BIG, and the launch failure read as a missing binary ("install the named tool"). + * The fixture selects 700 test files with 200-character names (about 150 KB of quoted paths) and a runner that reports + * one test per file argument, so the lane's count proves every file reached a runner. + */ +import { afterEach, describe, expect, test } from 'bun:test'; +import { argBudget, argChunks } from '../lib/pregate/checks/lanes'; +import { buildContext } from '../lib/pregate/context'; +import { classifyEnvironment } from '../lib/pregate/environment'; +import { runPregate } from '../lib/pregate/index'; +import { BASE_PREGATE, cleanup, commit, makeFixtureRepo, type FixtureRepo } from './helpers/pregate-fixture'; + +const repos: FixtureRepo[] = []; +afterEach(() => { while (repos.length) cleanup(repos.pop()!); }); + +const FILES = 700; +const name = (i: number, tag = 'x') => `test/big/${tag.padEnd(186, 'x')}${String(i).padStart(4, '0')}.test.js`; +const RUNNER = [ + 'const files = process.argv.slice(2);', + "const failed = files.filter(f => f.includes('fail')).length;", + "console.log(`ℹ tests ${files.length}\\nℹ pass ${files.length - failed}\\nℹ fail ${failed}`);", + 'process.exit(failed ? 1 : 0);', +].join('\n'); + +async function lanesOver(tests: string[]) { + const repo = makeFixtureRepo({ + pregate: { ...BASE_PREGATE, runner: 'node scripts/count.js {files}', dependencies: { ...BASE_PREGATE.dependencies, 'src/big/**': tests } }, + baseFiles: Object.fromEntries([...tests.map(t => [t, "require('node:test');\n"]), ['src/big/.keep', ''], ['scripts/count.js', `${RUNNER}\n`]]), + }); + repos.push(repo); + commit(repo, { 'src/big/f.js': 'exports.n = 1;\n' }, 'touch the big tree'); + const loaded = buildContext({ repoRoot: repo.work, base: 'main', policyFrom: 'base', allowRepoCommands: true, env: { ...process.env, ...repo.env }, progress: () => {}, cores: 64, testBackend: 'local' }); + if (!('ctx' in loaded)) throw new Error('fixture context refused'); + const file = await runPregate(loaded.ctx, { stage: 'tests', only: ['lanes'] }); + return file.checks.find(c => c.id === 'lanes')!; +} + +describe.skipIf(process.platform === 'win32')('a local lane whose file list is over the per-argument limit', () => { + // Value: protects=a unit group whose quoted file list is over Linux's 128 KiB per-argument cap still launches and every file + // is counted; fails_when=the whole list goes into one `bash -c` argument (E2BIG on Linux: PREGATE_ENVIRONMENT, nothing ran); + // why_new=#4424, gbrain's 5,567 unit files on gstack 1.99.5.0; seam=runPregate --only lanes on a fixture repo with a + // counting runner; no_claim=macOS has no per-argument cap, so there the old code passed too + test('every file runs and is counted, and a failure in the last chunk still fails the lane', async () => { + const tests = Array.from({ length: FILES }, (_, i) => name(i)); + expect(tests.map(t => `'${t}'`).join(' ').length).toBeGreaterThan(128 * 1024); + const green = await lanesOver(tests); + expect(green.detail).not.toContain('E2BIG'); + expect(green).toMatchObject({ status: 'pass', detail: `${FILES} tests ran in ${FILES} file(s), exit 0` }); + + const red = await lanesOver([...tests.slice(0, -1), name(FILES - 1, 'zfail')]); + expect(red).toMatchObject({ status: 'fail', code: 'PREGATE_LANE_FAILED', detail: `local lane exited 1 (1 failed of ${FILES} ran; ${FILES} files; pipefail)` }); + }, 120_000); +}); + +describe('argChunks and argBudget', () => { + // Value: protects=the split keeps order, drops nothing and keeps each chunk's quoted text under the budget, with an oversized + // argument alone in its chunk, and the budget stays under Linux's per-argument cap and shrinks with the environment; + // fails_when=a chunk passes the budget, a file is lost or reordered, or the budget ignores the cap; why_new=#4424; seam=none + test('splits in order under the byte budget, which stays under the argument limits', () => { + const args = ['a'.repeat(10), 'b'.repeat(10), "c'd".repeat(4), 'e'.repeat(40), 'f']; + const chunks = argChunks(args, 30); + expect(chunks).toEqual([[args[0], args[1]], [args[2]], [args[3]], [args[4]]]); + expect(argChunks([], 30)).toEqual([]); + expect(argChunks(args, 1_000)).toEqual([args]); + const linux = argBudget('bun test {files}', {}, 'linux'); + expect(linux).toBeLessThan(128 * 1024); + expect(argBudget('bun test {files}', { BIG: 'x'.repeat(900 * 1024) }, 'linux')).toBeLessThan(1024 * 1024 - 64 * 1024 - 900 * 1024); + expect(argBudget('bun test {files}', {}, 'win32')).toBeLessThan(32 * 1024); + }); +}); + +describe('E2BIG is its own cause', () => { + // Value: protects=an argument list over the system limit is named as such, never as a missing tool; fails_when=E2BIG + // classifies as missing-binary or as nothing (the launch path then defaulted to missing-binary); why_new=#4424's + // misleading "install the named tool"; seam=none + test('classifies E2BIG and "argument list too long" as arg-limit', () => { + expect(classifyEnvironment("E2BIG: argument list too long, posix_spawn 'bash'", null)).toEqual({ kind: 'arg-limit', evidence: "E2BIG: argument list too long, posix_spawn 'bash'", proof: 'output' }); + expect(classifyEnvironment('bash: /usr/bin/node: Argument list too long', 126)?.kind).toBe('arg-limit'); + }); +});