mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-18 06:04:33 +01:00
/ship's test step says when it hits its time limit, and long test file lists no longer overflow the remote command (GST-2958) (#4429)
* Say when /ship's test step hit its time limit, and keep long file lists off the remote test command line gstack-ship-preflight stopped gstack-pregate after a fixed hour and printed 'PREGATE: unreadable (exit 143)'. The whole step's budget is now --step-timeout <s> (default 3600), and a stop at it prints 'PREGATE: time limit hit' with the way out: push and let CI's full run carry the tests, or a larger budget. The test_backend command pregate prints for a small machine put every selected file into one argument, which Linux refuses past 128 KiB. The files now reach the VM in a setup script named by its content. Also adds an arg-limit troubleshooting entry and says the Windows job has eight shards. Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * Read the small-machine file list from the setup script in the hidden-folder path test The remote command now names a setup script instead of the files, so the test checks that the script lists the ./-prefixed path. Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * Keep a test path with a newline out of the remote file list A selected test path containing a newline, or equal to the list's end marker, could end the setup script's file list early and run the rest as shell on the test VM. Such paths now keep the quoted inline command. Review-Finding: p1/outside-voice#1 reader=outside-voice Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * Mark the preflight half of the Step 5 time limit to-do as done Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --------- Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
This commit is contained in:
1 parent
e8c8625bd6
commit
aeb74eb00b
11 files changed
+146
-20
No files matched your search
@@ -30,7 +30,7 @@
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import { baseInputs, sha256 } from '../context';
|
||||
import { baseInputs, defaultScratchRoot, sha256 } from '../context';
|
||||
import { PREGATE_CONFIG_PATH } from '../config';
|
||||
import { classifyCommandEnvironment, classifyEnvironment, environmentLine, environmentResult, launchCause } from '../environment';
|
||||
import { checkScratch } from '../run-scratch';
|
||||
@@ -498,12 +498,39 @@ const GSTACK_ROOT = path.resolve(import.meta.dir, '..', '..', '..');
|
||||
*/
|
||||
function remoteObligation(ctx: PregateContext, sel: Selection): RemoteObligation {
|
||||
const declared = ctx.config.remote_gate;
|
||||
const runner = detectRunner(ctx, sel.gateTests);
|
||||
const command = declared ?? `${GSTACK_ROOT}/scripts/ubicloud/ubi-runner.sh run --src ${quote(ctx.repoRoot)} -- ${quote(runner ? runner.replace('{files}', testFileArgs(runner, sel.gateTests).map(quote).join(' ')) : '<test runner> <selected files>')}`;
|
||||
const command = declared ?? backendCommand(ctx, sel.gateTests);
|
||||
const [workflow, job] = declared ? REMOTE_GATE_LANE.split('/') as [string, string] : ['test_backend', ctx.testBackend];
|
||||
return { lane: `${workflow}/${job}`, platform: 'remote', workflow, job, tests: sel.gateTests, receipt: null, command };
|
||||
}
|
||||
|
||||
/**
|
||||
* The test backend's command: gstack's Ubicloud runner, with the selected files in a setup script that writes them one per line
|
||||
* on the VM. The command reaches the VM as one argument and Linux caps one argument at 128 KiB (#4424), so the list never goes in
|
||||
* it. The script lives under the state root, named by its content, so the same selection prints the same command every run and a
|
||||
* same-tree clear still matches it; an unwritable state root keeps it in this run's scratch. A path the one-per-line list can't
|
||||
* hold (a newline, or the list's end marker) keeps the quoted inline form, so no path can end the list early and run as shell.
|
||||
*/
|
||||
function backendCommand(ctx: PregateContext, files: readonly string[]): string {
|
||||
const ubi = `${GSTACK_ROOT}/scripts/ubicloud/ubi-runner.sh run --src ${quote(ctx.repoRoot)}`;
|
||||
const runner = detectRunner(ctx, files);
|
||||
if (!runner) return `${ubi} -- ${quote('<test runner> <selected files>')}`;
|
||||
const args = testFileArgs(runner, files);
|
||||
if (args.some(f => f.includes('\n') || f === 'PREGATE_FILES')) return `${ubi} -- ${quote(runner.replace('{files}', () => args.map(quote).join(' ')))}`;
|
||||
const setup = `cat > ~/.pregate-files.txt <<'PREGATE_FILES'\n${args.map(f => `${f}\n`).join('')}PREGATE_FILES\n`;
|
||||
const name = `${sha256(setup).slice(0, 16)}.sh`;
|
||||
let file: string;
|
||||
try {
|
||||
const dir = path.join(path.dirname(defaultScratchRoot(ctx.repoRoot, ctx.env)), 'remote-files');
|
||||
fs.mkdirSync(dir, { recursive: true, mode: 0o700 });
|
||||
file = path.join(dir, name);
|
||||
fs.writeFileSync(file, setup);
|
||||
} catch {
|
||||
file = path.join(checkScratch(ctx.scratch, 'remote'), name);
|
||||
fs.writeFileSync(file, setup);
|
||||
}
|
||||
return `${ubi} --setup ${quote(file)} -- ${quote(`mapfile -t PREGATE_FILES < ~/.pregate-files.txt && ${runner.replace('{files}', () => '"${PREGATE_FILES[@]}"')}`)}`;
|
||||
}
|
||||
|
||||
export const REMOTE_GATE_LANE = 'pregate.json/remote_gate';
|
||||
/** How many leading lines of a remote gate's output are read for an environment cause: its preamble, before any test output could stream. */
|
||||
export const REMOTE_PREAMBLE_LINES = 40;
|
||||
|
||||
@@ -19,9 +19,10 @@ import { ghSpawnSync } from './gh-poll';
|
||||
import { extractVersion } from './version-source';
|
||||
import { BIN_DIR, runBashSync, runScriptSync, scriptInvocation } from './script-invocation';
|
||||
|
||||
export interface Run { status: number | null; stdout: string; stderr: string }
|
||||
/** `timedOut`: the call's own time limit stopped the process (spawnSync's ETIMEDOUT), whatever exit status it then gave. */
|
||||
export interface Run { status: number | null; stdout: string; stderr: string; timedOut?: boolean }
|
||||
const done = (r: { status: number | null; stdout?: string | Buffer | null; stderr?: string | Buffer | null; error?: Error }): Run =>
|
||||
({ status: r.status, stdout: String(r.stdout ?? ''), stderr: String(r.stderr ?? '') + (r.error ? r.error.message : '') });
|
||||
({ status: r.status, stdout: String(r.stdout ?? ''), stderr: String(r.stderr ?? '') + (r.error ? r.error.message : ''), ...((r.error as NodeJS.ErrnoException | undefined)?.code === 'ETIMEDOUT' ? { timedOut: true } : {}) });
|
||||
|
||||
export function git(cwd: string, args: string[], timeout = 120_000): Run {
|
||||
return done(spawnSync('git', args, { cwd, encoding: 'utf8', timeout, maxBuffer: 64 * 1024 * 1024, env: { ...process.env, GIT_TERMINAL_PROMPT: '0' } }));
|
||||
|
||||
+13
-3
@@ -30,6 +30,8 @@ export const PREFLIGHT_HELP = `gstack-ship-preflight — /ship Steps 1-3 and Ste
|
||||
(a branch that adds the lane for a file it touches); anything else is PREGATE_HEAD_POLICY_LOOSENS
|
||||
--timeout <s> Step 5's pregate --timeout for each runner process, 1 to 3600; unset, the base .gstack/pregate.json
|
||||
timeout_s applies, else 600 s, and a longer lane is PREGATE_INCOMPLETE
|
||||
--step-timeout <s> the whole Step 5 pregate's budget (every check and lane together), 1 to 86400, default 3600; at the
|
||||
limit it is stopped and prints PREGATE: time limit hit with the way out (push and let CI's full run carry it)
|
||||
gstack-ship-preflight --tests-only --base <base> --policy-from head --simulate [--allow-repo-commands] --lane <label>=<command>...
|
||||
a dry run that proves a new ship_lanes_from mapping: which lanes the branch's own map would carry from a verified
|
||||
CI run at this tree. Every line starts SIMULATION:; no lane runs, no evidence or ledger row is written and no
|
||||
@@ -195,8 +197,13 @@ function merge(cwd: string, base: string, r: Report): boolean {
|
||||
return true;
|
||||
}
|
||||
|
||||
async function tests(cwd: string, base: string, list: { label: string; command: string }[], policyFrom: string, timeout: string | undefined, run: { concurrency: number; line: string }, r: Report): Promise<void> {
|
||||
const pg = bin(cwd, 'gstack-pregate', ['--allow-repo-commands', '--base', base, '--policy-from', policyFrom, ...(timeout ? ['--timeout', timeout] : [])], 3_600_000);
|
||||
async function tests(cwd: string, base: string, list: { label: string; command: string }[], policyFrom: string, timeout: string | undefined, step: { seconds: number; source: string }, run: { concurrency: number; line: string }, r: Report): Promise<void> {
|
||||
const pg = bin(cwd, 'gstack-pregate', ['--allow-repo-commands', '--base', base, '--policy-from', policyFrom, ...(timeout ? ['--timeout', timeout] : [])], step.seconds * 1000);
|
||||
if (pg.timedOut) {
|
||||
r.add(`PREGATE: time limit hit (Step 5's whole pregate ran past its ${step.seconds} s budget, ${step.source}, and was stopped with exit ${pg.status ?? 'none'}; no verdict)`);
|
||||
r.stop(`the tests are longer than this step's ${step.seconds} s budget on this machine: push, let CI's full run at this tree carry them (gstack-ci-carry wait --base ${base}, then rerun --tests-only; gbrain docs/ci-event-parity.md#ship-lanes), or rerun --tests-only with --step-timeout <s> above ${step.seconds}`);
|
||||
return;
|
||||
}
|
||||
const refusal = pg.status === 3 ? /\((PREGATE_[A-Z_]+|POLICY_[A-Z_]+)[;)]/.exec(pg.stderr)?.[1] : undefined;
|
||||
const verdict = refusal ? `refused (${refusal})` : /^PREGATE: (.+?) tree=/m.exec(pg.stdout + pg.stderr)?.[1] ?? `unreadable (exit ${pg.status})`;
|
||||
r.add(`PREGATE: ${verdict}`);
|
||||
@@ -263,6 +270,9 @@ export async function preflight(argv: readonly string[], cwd: string): Promise<{
|
||||
if (!['base', 'head'].includes(policyFrom)) return { text: 'gstack-ship-preflight: --policy-from needs base|head\n', exit: 2 };
|
||||
const timeout = flag('--timeout');
|
||||
if (has('--timeout') && !(timeout && /^[1-9]\d*$/.test(timeout) && Number(timeout) <= 3600)) return { text: 'gstack-ship-preflight: --timeout needs whole seconds from 1 to 3600\n', exit: 2 };
|
||||
const stepTimeout = flag('--step-timeout');
|
||||
if (has('--step-timeout') && !(stepTimeout && /^[1-9]\d*$/.test(stepTimeout) && Number(stepTimeout) <= 86_400)) return { text: 'gstack-ship-preflight: --step-timeout needs whole seconds from 1 to 86400\n', exit: 2 };
|
||||
const step = stepTimeout ? { seconds: Number(stepTimeout), source: '--step-timeout' } : { seconds: 3600, source: 'the default; set --step-timeout' };
|
||||
if (!base || typeof list === 'string') return { text: `gstack-ship-preflight: ${typeof list === 'string' ? list : '--base <base> is required'}\n`, exit: 2 };
|
||||
const run = flag('--run');
|
||||
if (has('--run') && !(run && RUN_ID.test(run))) return { text: 'gstack-ship-preflight: --run needs a run id: letters, digits, dot, underscore or dash, at most 128\n', exit: 2 };
|
||||
@@ -273,7 +283,7 @@ export async function preflight(argv: readonly string[], cwd: string): Promise<{
|
||||
if (typeof concurrency === 'string') return { text: `gstack-ship-preflight: ${concurrency}\n`, exit: 2 };
|
||||
const r = new Report();
|
||||
if (has('--tests-only') || integrate(cwd, base, run, r)) {
|
||||
if (list.length) await tests(cwd, base, list, policyFrom, timeout, { concurrency, line: `LANES: ${concurrency === 1 ? 'serial' : 'parallel'} (${mode ? '--lanes' : `${os.availableParallelism()} cores`})` }, r);
|
||||
if (list.length) await tests(cwd, base, list, policyFrom, timeout, step, { concurrency, line: `LANES: ${concurrency === 1 ? 'serial' : 'parallel'} (${mode ? '--lanes' : `${os.availableParallelism()} cores`})` }, r);
|
||||
else if (r.exit === 0) r.next('Step 4: confirm the test framework, then rerun with --tests-only --lane <label>=<command> per suite (Step 5)');
|
||||
}
|
||||
return { text: r.render(), exit: r.exit };
|
||||
|
||||
Reference in new issue
Block a user