mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-17 13:44:32 +01:00
* Run large unit suites on Linux instead of failing with argument list too long A lane passed every unit file in one bash -c argument; Linux caps one argument at 128 KiB, so thousands of files failed with E2BIG. Split the file list into byte-bounded chunks run in sequence and read back as one run, and classify E2BIG as its own arg-limit cause. Fixes #4424 Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * fix: pre-landing review fixes Put the failing fixture file last after sorting so the arg-limit test really checks a failure in the final chunk. Review-Finding: p1/specialist-batched#1 reader=testing Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * docs: note the Windows chunk size for split lane file lists Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --------- Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
This commit is contained in:
1 parent
5dcd76b4ea
commit
5b4fa9ea33
6 files changed
+139
-11
No files matched your search
@@ -0,0 +1 @@
|
||||
- **/ship's test step runs a large unit suite on Linux.** A lane with thousands of unit files used to stop with `PREGATE: incomplete` and `E2BIG: argument list too long` before any test ran, because every file went into one command argument and Linux caps a single argument at 128 KiB. gstack now splits a big file list into runner processes of about 96 KB each, run one after another and counted as one run, so the result is the same as before: any failing chunk fails the lane. An `E2BIG` launch failure is now reported as its own cause (`arg-limit`) instead of telling you to install a missing tool. (#4424)
|
||||
+6
-1
@@ -428,7 +428,11 @@ Two stages with separate clocks, both printed:
|
||||
`workflow.yml/job`, unless `local` names that lane. `--explain` prints each test's provenance and every
|
||||
uncertain mapping (a planner-computed matrix without a `selection` manifest
|
||||
selects conservatively). Test files matching `serial_patterns` (default
|
||||
`**/*.serial.test.*`) run one process per file after the unit files. The
|
||||
`**/*.serial.test.*`) run one process per file after the unit files. A unit
|
||||
group whose quoted file list would pass the system's argument limits (Linux
|
||||
caps one argument at 128 KiB) runs as several runner processes in sequence,
|
||||
each under about 96 KB of file arguments (24 KB on Windows), read back as one run: counts add up,
|
||||
any failing chunk fails the lane, and they share the group's one `--timeout`. The
|
||||
runner and every guard run with the verified launch Bun first on PATH: the
|
||||
first of `$BUN_INSTALL/bin` and `~/.bun/bin` whose Bun satisfies gstack's and
|
||||
the project's `engines.bun` (the row prints `runtime: bun <v> from <dir>`),
|
||||
@@ -1858,6 +1862,7 @@ failure the branch caused carries `cause: "diff"` (`PREGATE_LANE_FAILED`,
|
||||
| Kind | Evidence |
|
||||
|---|---|
|
||||
| `missing-binary` | exit 127, `command not found`, a spawn `ENOENT`, a runner tool `not on PATH` (dart, flutter, cargo, a package manager, `python3`, or `pytest` when python3 cannot import it), Playwright's browsers not installed |
|
||||
| `arg-limit` | `E2BIG`, `Argument list too long`: a command line or environment over the system's limit. gstack splits a lane's file list to stay under it, so this is a gstack bug to report |
|
||||
| `network` | `ENOTFOUND`, `ECONNREFUSED`, `ECONNRESET`, `ETIMEDOUT`, `EAI_AGAIN`, `Could not resolve host`, `fetch failed`, a TLS handshake or certificate failure |
|
||||
| `quota` | `quota`, `429 Too Many Requests`, `rate limit`, `insufficient_quota` (a vCPU quota on the remote backend) |
|
||||
| `disk` | `ENOSPC`, `No space left on device`, `EDQUOT` |
|
||||
|
||||
@@ -365,18 +365,56 @@ interface LocalRun { exit: number | null; outcome: Supervised['outcome']; signal
|
||||
|
||||
const quote = (f: string) => `'${f.replace(/'/g, `'\\''`)}'`;
|
||||
|
||||
/**
|
||||
* Bytes of quoted file arguments one runner process may take. supervise passes the whole command as one `bash -c`
|
||||
* argument: Linux caps a single execve argument at 128 KiB (MAX_ARG_STRLEN), every POSIX system caps argv plus the
|
||||
* environment (ARG_MAX: 1 MiB on macOS, 2 MiB by default on Linux), and Windows caps a whole command line at 32,767
|
||||
* characters. The runner text itself and a margin come off the top.
|
||||
*/
|
||||
export function argBudget(runner: string, env: NodeJS.ProcessEnv, platform = process.platform): number {
|
||||
const envBytes = Object.entries(env).reduce((n, [k, v]) => n + Buffer.byteLength(`${k}=${v ?? ''}`) + 9, 0);
|
||||
const cap = platform === 'win32' ? 24 * 1024 : Math.min(96 * 1024, 1024 * 1024 - 64 * 1024 - envBytes);
|
||||
return cap - Buffer.byteLength(runner) - 512;
|
||||
}
|
||||
|
||||
/** `args` split in order into chunks whose quoted, space-joined text fits `budget` bytes; an argument over it runs alone. */
|
||||
export function argChunks(args: readonly string[], budget: number): string[][] {
|
||||
const chunks: string[][] = [];
|
||||
let size = 0;
|
||||
for (const a of args) {
|
||||
const n = Buffer.byteLength(quote(a)) + 1;
|
||||
if (!chunks.length || size + n > budget) { chunks.push([]); size = 0; }
|
||||
chunks.at(-1)!.push(a);
|
||||
size += n;
|
||||
}
|
||||
return chunks;
|
||||
}
|
||||
|
||||
async function runOnce(ctx: PregateContext, env: NodeJS.ProcessEnv, g: RunGroup, files: string[], log: string, counter: Counter): Promise<LocalRun> {
|
||||
// playwright test reads each file argument as a case-insensitive regex: escape it and anchor it to a path segment, so `a[1].spec.ts` runs that file, not `a1.spec.ts`.
|
||||
const args = /(?<![\w-])playwright\s+test(?![\w-])/.test(g.runner) ? files.map(f => `(?:^|/)${f.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}$`) : testFileArgs(g.runner, files);
|
||||
const joined = args.map(quote).join(' ');
|
||||
const run = g.runner.replace('{files}', () => joined);
|
||||
const command = g.cwd ? `(cd ${quote(g.cwd)} && ${run})` : run;
|
||||
const shown = g.cwd ? files.map(f => `${g.cwd}/${f}`) : files;
|
||||
const r = await supervise({ command: run, cwd: path.join(ctx.repoRoot, g.cwd), env: { ...env, CI: env.CI ?? '1' }, log, deadline: Date.now() + ctx.timeoutMs, stallMs: ctx.stallTimeoutMs, label: `lanes (${shown.length === 1 ? shown[0] : `${shown.length} files`})`, files, adapter: adapterFor(g.runner), counter, progress: ctx.progress });
|
||||
const output = readRange(log, r.start, r.end);
|
||||
const summary = parseSummary(output, g.runner);
|
||||
const noBrowsers = /(?<![\w-])playwright\s+test(?![\w-])/.test(g.runner) ? PLAYWRIGHT_NO_BROWSERS.exec(output)?.[0].trim().slice(0, 200) : undefined;
|
||||
return { ...(noBrowsers ? { noBrowsers } : {}), exit: r.exit, outcome: r.outcome, signal: r.signal, observedStart: r.observedStart, ...summary, tail: output.trim().split('\n').slice(-25), runner: g.runner, command, files: shown, lastFile: r.lastFile && g.cwd ? `${g.cwd}/${r.lastFile}` : r.lastFile, ms: r.ms, ...(r.error ? { error: r.error } : {}) };
|
||||
const runEnv = { ...env, CI: env.CI ?? '1' };
|
||||
// A file list over the argument limits (#4424: E2BIG on Linux) runs as chunks in sequence under the group's one deadline, read back as one run:
|
||||
// the first unfinished, else the first non-zero chunk decides; counts add up; a chunk that did not finish ends the group, as a killed runner would.
|
||||
const deadline = Date.now() + ctx.timeoutMs;
|
||||
const parts: LocalRun[] = [];
|
||||
let at = 0;
|
||||
for (const chunk of argChunks(args, argBudget(g.runner, runEnv))) {
|
||||
const part = files.slice(at, at += chunk.length);
|
||||
const joined = chunk.map(quote).join(' ');
|
||||
const run = g.runner.replace('{files}', () => joined);
|
||||
const shown = g.cwd ? part.map(f => `${g.cwd}/${f}`) : part;
|
||||
const r = await supervise({ command: run, cwd: path.join(ctx.repoRoot, g.cwd), env: runEnv, log, deadline, stallMs: ctx.stallTimeoutMs, label: `lanes (${shown.length === 1 ? shown[0] : `${shown.length} files`})`, files: part, adapter: adapterFor(g.runner), counter, progress: ctx.progress });
|
||||
const output = readRange(log, r.start, r.end);
|
||||
const summary = parseSummary(output, g.runner);
|
||||
const noBrowsers = /(?<![\w-])playwright\s+test(?![\w-])/.test(g.runner) ? PLAYWRIGHT_NO_BROWSERS.exec(output)?.[0].trim().slice(0, 200) : undefined;
|
||||
parts.push({ ...(noBrowsers ? { noBrowsers } : {}), exit: r.exit, outcome: r.outcome, signal: r.signal, observedStart: r.observedStart, ...summary, tail: output.trim().split('\n').slice(-25), runner: g.runner, command: g.cwd ? `(cd ${quote(g.cwd)} && ${run})` : run, files: shown, lastFile: r.lastFile && g.cwd ? `${g.cwd}/${r.lastFile}` : r.lastFile, ms: r.ms, ...(r.error ? { error: r.error } : {}) });
|
||||
if (r.exit === null) break;
|
||||
}
|
||||
if (parts.length === 1) return parts[0]!;
|
||||
const bad = parts.find(p => p.exit === null) ?? parts.find(p => p.exit !== 0) ?? parts.at(-1)!;
|
||||
const sum = (k: 'ran' | 'failed') => (parts.some(p => p[k] === null) ? null : parts.reduce((n, p) => n + (p[k] as number), 0));
|
||||
return { ...(parts.every(p => p.noBrowsers) ? { noBrowsers: parts[0]!.noBrowsers } : {}), exit: bad.exit, outcome: bad.outcome, signal: bad.signal, observedStart: parts.some(p => p.observedStart), ran: sum('ran'), failed: sum('failed'), tail: bad.tail, runner: g.runner, command: parts.map(p => p.command).join(' && '), files: parts.flatMap(p => p.files), lastFile: bad.lastFile, ms: parts.reduce((n, p) => n + p.ms, 0), ...(bad.error ? { error: bad.error } : {}) };
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -23,6 +23,7 @@ const ANSI = /\x1b\[[0-9;?]*[A-Za-z]/g;
|
||||
const NOT_FOUND = /command not found|is not recognized as an internal or external command/;
|
||||
const RULES: Array<{ kind: EnvironmentKind; re: RegExp }> = [
|
||||
{ kind: 'missing-binary', re: new RegExp(`${NOT_FOUND.source}|No such file or directory: ['"]?[\\w./-]+|spawn \\S+ ENOENT|\\bENOENT\\b.*\\bspawn\\b|not on PATH`) },
|
||||
{ kind: 'arg-limit', re: /\bE2BIG\b|[Aa]rgument list too long/ },
|
||||
{ kind: 'quota', re: /\bquota\b|\b429 Too Many Requests|status(?: code)?[:= ]+429\b|HTTP(?:\/[\d.]+)? 429\b|\brate.?limit(?:ed)?\b|insufficient_quota/i },
|
||||
{ kind: 'network', re: /\bE(?:NOTFOUND|CONNREFUSED|CONNRESET|TIMEDOUT|HOSTUNREACH|NETUNREACH|AI_AGAIN|PIPE)\b|getaddrinfo|Could not resolve host|Connection refused|Connection reset|Network is unreachable|fetch failed|TLS handshake|certificate verify failed|UNABLE_TO_GET_ISSUER_CERT|Temporary failure in name resolution/ },
|
||||
{ kind: 'disk', re: /\bENOSPC\b|No space left on device|\bEDQUOT\b|Disk quota exceeded/ },
|
||||
@@ -72,6 +73,7 @@ export function classifyCommandEnvironment(command: string, output: string, exit
|
||||
|
||||
const FIX: Record<EnvironmentKind, string> = {
|
||||
'missing-binary': 'install the named tool on this machine (or put it on PATH), or run the lane remotely (--remote, or clear it with a run URL)',
|
||||
'arg-limit': 'the command line or environment passed to the runner is over the system\'s argument limit (E2BIG); gstack splits a lane\'s file list to stay under it, so report this as a gstack bug with this line, and meanwhile run the lane remotely or split the selection',
|
||||
network: 'check the network, proxy or registry this machine reaches, then rerun; or run the lane remotely',
|
||||
quota: 'wait for the quota or raise it, free the backend\'s capacity, then rerun',
|
||||
disk: 'free disk space (or move the state root) and rerun',
|
||||
|
||||
@@ -105,7 +105,7 @@ export interface CheckResult extends PregateCheck {
|
||||
environment?: EnvironmentCause;
|
||||
}
|
||||
|
||||
export type EnvironmentKind = 'missing-binary' | 'network' | 'quota' | 'disk' | 'runtime' | 'tool';
|
||||
export type EnvironmentKind = 'missing-binary' | 'arg-limit' | 'network' | 'quota' | 'disk' | 'runtime' | 'tool';
|
||||
/** How the cause was established: `launch` (the command could not start, its binary is off PATH, the runtime or a gstack tool refused), or `output` (a rule matched the command's text before any test was counted, which the diff's own output could have written). */
|
||||
export type EnvironmentProof = 'launch' | 'output';
|
||||
export interface EnvironmentCause { kind: EnvironmentKind; evidence: string; proof: EnvironmentProof }
|
||||
|
||||
@@ -0,0 +1,82 @@
|
||||
/**
|
||||
* #4424: on Linux a local lane with thousands of unit files never started. runOnce put every file of a run group into
|
||||
* one command string and supervise started it as `bash -c <string>`; Linux caps one execve argument at MAX_ARG_STRLEN
|
||||
* (128 KiB), so posix_spawn returned E2BIG, and the launch failure read as a missing binary ("install the named tool").
|
||||
* The fixture selects 700 test files with 200-character names (about 150 KB of quoted paths) and a runner that reports
|
||||
* one test per file argument, so the lane's count proves every file reached a runner.
|
||||
*/
|
||||
import { afterEach, describe, expect, test } from 'bun:test';
|
||||
import { argBudget, argChunks } from '../lib/pregate/checks/lanes';
|
||||
import { buildContext } from '../lib/pregate/context';
|
||||
import { classifyEnvironment } from '../lib/pregate/environment';
|
||||
import { runPregate } from '../lib/pregate/index';
|
||||
import { BASE_PREGATE, cleanup, commit, makeFixtureRepo, type FixtureRepo } from './helpers/pregate-fixture';
|
||||
|
||||
const repos: FixtureRepo[] = [];
|
||||
afterEach(() => { while (repos.length) cleanup(repos.pop()!); });
|
||||
|
||||
const FILES = 700;
|
||||
const name = (i: number, tag = 'x') => `test/big/${tag.padEnd(186, 'x')}${String(i).padStart(4, '0')}.test.js`;
|
||||
const RUNNER = [
|
||||
'const files = process.argv.slice(2);',
|
||||
"const failed = files.filter(f => f.includes('fail')).length;",
|
||||
"console.log(`ℹ tests ${files.length}\\nℹ pass ${files.length - failed}\\nℹ fail ${failed}`);",
|
||||
'process.exit(failed ? 1 : 0);',
|
||||
].join('\n');
|
||||
|
||||
async function lanesOver(tests: string[]) {
|
||||
const repo = makeFixtureRepo({
|
||||
pregate: { ...BASE_PREGATE, runner: 'node scripts/count.js {files}', dependencies: { ...BASE_PREGATE.dependencies, 'src/big/**': tests } },
|
||||
baseFiles: Object.fromEntries([...tests.map(t => [t, "require('node:test');\n"]), ['src/big/.keep', ''], ['scripts/count.js', `${RUNNER}\n`]]),
|
||||
});
|
||||
repos.push(repo);
|
||||
commit(repo, { 'src/big/f.js': 'exports.n = 1;\n' }, 'touch the big tree');
|
||||
const loaded = buildContext({ repoRoot: repo.work, base: 'main', policyFrom: 'base', allowRepoCommands: true, env: { ...process.env, ...repo.env }, progress: () => {}, cores: 64, testBackend: 'local' });
|
||||
if (!('ctx' in loaded)) throw new Error('fixture context refused');
|
||||
const file = await runPregate(loaded.ctx, { stage: 'tests', only: ['lanes'] });
|
||||
return file.checks.find(c => c.id === 'lanes')!;
|
||||
}
|
||||
|
||||
describe.skipIf(process.platform === 'win32')('a local lane whose file list is over the per-argument limit', () => {
|
||||
// Value: protects=a unit group whose quoted file list is over Linux's 128 KiB per-argument cap still launches and every file
|
||||
// is counted; fails_when=the whole list goes into one `bash -c` argument (E2BIG on Linux: PREGATE_ENVIRONMENT, nothing ran);
|
||||
// why_new=#4424, gbrain's 5,567 unit files on gstack 1.99.5.0; seam=runPregate --only lanes on a fixture repo with a
|
||||
// counting runner; no_claim=macOS has no per-argument cap, so there the old code passed too
|
||||
test('every file runs and is counted, and a failure in the last chunk still fails the lane', async () => {
|
||||
const tests = Array.from({ length: FILES }, (_, i) => name(i));
|
||||
expect(tests.map(t => `'${t}'`).join(' ').length).toBeGreaterThan(128 * 1024);
|
||||
const green = await lanesOver(tests);
|
||||
expect(green.detail).not.toContain('E2BIG');
|
||||
expect(green).toMatchObject({ status: 'pass', detail: `${FILES} tests ran in ${FILES} file(s), exit 0` });
|
||||
|
||||
const red = await lanesOver([...tests.slice(0, -1), name(FILES - 1, 'zfail')]);
|
||||
expect(red).toMatchObject({ status: 'fail', code: 'PREGATE_LANE_FAILED', detail: `local lane exited 1 (1 failed of ${FILES} ran; ${FILES} files; pipefail)` });
|
||||
}, 120_000);
|
||||
});
|
||||
|
||||
describe('argChunks and argBudget', () => {
|
||||
// Value: protects=the split keeps order, drops nothing and keeps each chunk's quoted text under the budget, with an oversized
|
||||
// argument alone in its chunk, and the budget stays under Linux's per-argument cap and shrinks with the environment;
|
||||
// fails_when=a chunk passes the budget, a file is lost or reordered, or the budget ignores the cap; why_new=#4424; seam=none
|
||||
test('splits in order under the byte budget, which stays under the argument limits', () => {
|
||||
const args = ['a'.repeat(10), 'b'.repeat(10), "c'd".repeat(4), 'e'.repeat(40), 'f'];
|
||||
const chunks = argChunks(args, 30);
|
||||
expect(chunks).toEqual([[args[0], args[1]], [args[2]], [args[3]], [args[4]]]);
|
||||
expect(argChunks([], 30)).toEqual([]);
|
||||
expect(argChunks(args, 1_000)).toEqual([args]);
|
||||
const linux = argBudget('bun test {files}', {}, 'linux');
|
||||
expect(linux).toBeLessThan(128 * 1024);
|
||||
expect(argBudget('bun test {files}', { BIG: 'x'.repeat(900 * 1024) }, 'linux')).toBeLessThan(1024 * 1024 - 64 * 1024 - 900 * 1024);
|
||||
expect(argBudget('bun test {files}', {}, 'win32')).toBeLessThan(32 * 1024);
|
||||
});
|
||||
});
|
||||
|
||||
describe('E2BIG is its own cause', () => {
|
||||
// Value: protects=an argument list over the system limit is named as such, never as a missing tool; fails_when=E2BIG
|
||||
// classifies as missing-binary or as nothing (the launch path then defaulted to missing-binary); why_new=#4424's
|
||||
// misleading "install the named tool"; seam=none
|
||||
test('classifies E2BIG and "argument list too long" as arg-limit', () => {
|
||||
expect(classifyEnvironment("E2BIG: argument list too long, posix_spawn 'bash'", null)).toEqual({ kind: 'arg-limit', evidence: "E2BIG: argument list too long, posix_spawn 'bash'", proof: 'output' });
|
||||
expect(classifyEnvironment('bash: /usr/bin/node: Argument list too long', 126)?.kind).toBe('arg-limit');
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user