Files
gstack/test/pregate-remote.test.ts
T
Garry Tanandcapy-ai[bot] aeb74eb00b /ship's test step says when it hits its time limit, and long test file lists no longer overflow the remote command (GST-2958) (#4429)
* Say when /ship's test step hit its time limit, and keep long file lists off the remote test command line

gstack-ship-preflight stopped gstack-pregate after a fixed hour and printed
'PREGATE: unreadable (exit 143)'. The whole step's budget is now
--step-timeout <s> (default 3600), and a stop at it prints
'PREGATE: time limit hit' with the way out: push and let CI's full run
carry the tests, or a larger budget.

The test_backend command pregate prints for a small machine put every
selected file into one argument, which Linux refuses past 128 KiB. The
files now reach the VM in a setup script named by its content.

Also adds an arg-limit troubleshooting entry and says the Windows job has
eight shards.

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* Read the small-machine file list from the setup script in the hidden-folder path test

The remote command now names a setup script instead of the files, so the
test checks that the script lists the ./-prefixed path.

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* Keep a test path with a newline out of the remote file list

A selected test path containing a newline, or equal to the list's end marker, could end the setup script's file list early and run the rest as shell on the test VM. Such paths now keep the quoted inline command.

Review-Finding: p1/outside-voice#1 reader=outside-voice
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* Mark the preflight half of the Step 5 time limit to-do as done

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

---------

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
2026-11-17 13:13:19 +00:00

274 lines
20 KiB
TypeScript

/**
* Lane C4 (GBRA-95): on a machine under 8 cores, a remote test gate the base
* branch declares (`remote_gate`) or a remote `test_backend` records the
* local test tier as a requires-remote obligation with the exact command,
* instead of running thousands of files locally. `--remote` runs the
* declared gate now under a spend cap; the obligation clears only on the
* gate's `PREGATE_REMOTE: tree=<tested tree> tests=<n> failed=0` line.
*/
import { afterEach, describe, expect, test } from 'bun:test';
import * as fs from 'node:fs';
import * as path from 'node:path';
import { EXIT } from '../lib/headless-artifacts';
import { buildContext } from '../lib/pregate/context';
import { runPregate } from '../lib/pregate/index';
import { BASE_PREGATE, cleanup, commit, makeFixtureRepo, runBin, type FixtureRepo } from './helpers/pregate-fixture';
const repos: FixtureRepo[] = [];
afterEach(() => { while (repos.length) cleanup(repos.pop()!); });
const GATE = 'node scripts/remote-gate.js';
/** A fake remote gate: reports the tree it was handed (or GATE_TREE), GATE_TESTS / GATE_FAILED, and records its env. */
const GATE_SCRIPT = [
"const fs = require('node:fs');",
"fs.writeFileSync('../gate-env.json', JSON.stringify({ tree: process.env.PREGATE_TREE, cap: process.env.PREGATE_SPEND_CAP_USD, timeout: process.env.PREGATE_TIMEOUT_S, files: fs.readFileSync(process.env.PREGATE_FILES_FILE, 'utf8') }));",
"if (process.env.GATE_QUIET_MS) Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, Number(process.env.GATE_QUIET_MS));",
"if (process.env.GATE_SILENT) process.exit(0);",
"const spent = process.env.GATE_SPENT ? ` spent=${process.env.GATE_SPENT}` : '';",
"console.log(`PREGATE_REMOTE: tree=${process.env.GATE_TREE || process.env.PREGATE_TREE} tests=${process.env.GATE_TESTS ?? 3} failed=${process.env.GATE_FAILED ?? 0}${spent}`);",
"if (process.env.GATE_EXIT) process.exitCode = Number(process.env.GATE_EXIT);",
].join('\n') + '\n';
function gated(pregate: Record<string, unknown> = { ...BASE_PREGATE, remote_gate: GATE }): FixtureRepo {
const repo = makeFixtureRepo({ pregate });
repos.push(repo);
repo.git('checkout', '-q', 'main');
commit(repo, { 'scripts/remote-gate.js': GATE_SCRIPT }, 'remote gate');
repo.git('push', '-q', 'origin', 'main');
repo.git('checkout', '-q', 'feature');
repo.git('merge', '-q', '--ff-only', 'main');
commit(repo, { 'lib/util.js': "exports.add = (a, b) => a + b + 0;\n" });
return repo;
}
async function lanesOn(repo: FixtureRepo, o: { cores: number; testBackend: string }) {
const lines: string[] = [];
const loaded = buildContext({ repoRoot: repo.work, base: 'main', policyFrom: 'base', allowRepoCommands: true, env: { ...process.env, ...repo.env }, progress: l => lines.push(l), ...o });
if (!('ctx' in loaded)) throw new Error('fixture context refused');
const file = await runPregate(loaded.ctx, { stage: 'tests' });
return { file, lanes: file.checks.find(c => c.id === 'lanes')!, lines };
}
describe('small machines record the test tier for the remote gate (C4)', () => {
test('under 8 cores a declared remote_gate becomes a requires-remote obligation naming the command; nothing runs locally', async () => {
const repo = gated();
const { file, lanes, lines } = await lanesOn(repo, { cores: 4, testBackend: 'local' });
expect(lanes.status).toBe('requires-remote');
expect(lanes.detail).toContain('recorded as requires-remote pregate.json/remote_gate (4 cores, under 8)');
expect(file.requires_remote).toEqual([expect.objectContaining({ lane: 'pregate.json/remote_gate', platform: 'remote', command: GATE, tests: ['test/util.test.js'], receipt: null })]);
expect(file.verdict).toBe('requires-remote (1 uncleared)');
expect(fs.existsSync(path.join(repo.root, 'gate-env.json'))).toBe(false);
expect(lines.some(l => l.startsWith('PREGATE_TEST:'))).toBe(false);
expect(lines.at(-1)).toMatch(/^PREGATE: tests requires-remote \(1 uncleared\) in /);
});
test('a remote test_backend without remote_gate names gstack\'s Ubicloud runner with the selection; 8 cores or more, or local with no gate, runs locally', async () => {
const backend = gated(BASE_PREGATE);
const viaBackend = await lanesOn(backend, { cores: 4, testBackend: 'ubicloud' });
expect(viaBackend.lanes.status).toBe('requires-remote');
const obligation = viaBackend.file.requires_remote![0]!;
expect(obligation.lane).toBe('test_backend/ubicloud');
expect(obligation.command).toMatch(/scripts\/ubicloud\/ubi-runner\.sh run --src '.*\/work' --setup '(.*\/remote-files\/[0-9a-f]{16}\.sh)' -- 'mapfile -t PREGATE_FILES < ~\/\.pregate-files\.txt && node --test "\$\{PREGATE_FILES\[@\]\}"'$/);
expect(fs.readFileSync(/--setup '([^']+)'/.exec(obligation.command!)![1]!, 'utf8')).toBe("cat > ~/.pregate-files.txt <<'PREGATE_FILES'\ntest/util.test.js\nPREGATE_FILES\n");
expect((await lanesOn(backend, { cores: 4, testBackend: 'local' })).lanes.status).toBe('pass');
expect((await lanesOn(gated(), { cores: 16, testBackend: 'local' })).lanes.status).toBe('pass');
});
test.skipIf(process.platform === 'win32')('a selected test path the file list cannot hold (a newline) keeps the quoted inline command, so it cannot end the list and run as shell', async () => {
const backend = gated(BASE_PREGATE);
const odd = 'test/odd\nPREGATE_FILES\nname.test.js';
commit(backend, { [odd]: "require('node:test')('ok', () => {});\n" });
const command = (await lanesOn(backend, { cores: 4, testBackend: 'ubicloud' })).file.requires_remote![0]!.command!;
expect(command).not.toContain('--setup');
expect(command).toContain(odd);
});
test('`clear` on the remote_gate obligation needs the gate\'s PREGATE_REMOTE receipt for the tested tree; a run URL or a bad receipt leaves it uncleared', async () => {
const repo = gated();
const fileAt = path.join(repo.work, '.gstack/tmp/pregate.json');
const { file } = await lanesOn(repo, { cores: 4, testBackend: 'local' });
fs.mkdirSync(path.dirname(fileAt), { recursive: true });
fs.writeFileSync(fileAt, JSON.stringify(file));
const clear = (args: string[]) => runBin(repo, 'gstack-pregate', ['clear', '--remote', 'pregate.json/remote_gate', ...args]);
const receiptOf = (line: string) => { const f = path.join(repo.root, `receipt-${Math.random().toString(36).slice(2)}.log`); fs.writeFileSync(f, `gate log\n${line}\n`); return f; };
const url = clear(['--run-url', 'https://ci.example/run/1']);
expect(url.status).toBe(EXIT.usage);
expect(url.err).toContain('clears only with --receipt <file>');
for (const [line, why] of [
[`PREGATE_REMOTE: tree=${'deadbeef'.repeat(8)} tests=3 failed=0`, 'the remote gate tested tree deadbeefdead'],
[`PREGATE_REMOTE: tree=${file.tree} tests=0 failed=0`, 'the remote gate reported 0 tests'],
[`PREGATE_REMOTE: tree=${file.tree} tests=3 failed=1`, 'the remote gate reported 1 failed of 3'],
['all green, trust me', 'has no PREGATE_REMOTE: line'],
] as const) {
const r = clear(['--receipt', receiptOf(line)]);
expect(r.status).toBe(EXIT.fail);
expect(r.err).toContain(why);
expect(r.err).toContain('(PREGATE_RECEIPT_INVALID; https://github.com/garrytan/gstack/blob/main/docs/troubleshooting.md#pregate-receipt-invalid)');
expect(JSON.parse(fs.readFileSync(fileAt, 'utf8')).requires_remote[0].receipt).toBeNull();
}
const ok = clear(['--receipt', receiptOf(`PREGATE_REMOTE: tree=${file.tree} tests=3 failed=0`)]);
expect(ok.status).toBe(EXIT.ok);
expect(ok.out).toMatch(new RegExp(`^PREGATE_CLEARED: pregate\\.json/remote_gate receipt=remote-gate tree=${file.tree.slice(0, 12)} tests=3 receipt=\\S+ sha256=[0-9a-f]{12} verdict=requires-remote cleared$`, 'm'));
expect(ok.out).not.toContain('attested');
});
// GBRA-347 (gbrain#8730): a stage run's log prints the gate's line under the lanes check (` lanes: PREGATE_REMOTE: …`),
// and `clear --receipt` on that log refused "has no PREGATE_REMOTE: line".
// Value: protects=a gstack-pregate --remote log is a receipt clear accepts, validated like the bare line; fails_when=clear
// refuses the indented form, takes a line indented under another check, or takes the log of a run whose gate exited 1;
// seam=gstack-pregate clear --receipt
test('`clear --receipt` takes the stage run\'s own log, where the gate\'s line sits under lanes:', async () => {
const repo = gated();
const run = runBin(repo, 'gstack-pregate', ['--base', 'main', '--stage', 'tests', '--remote', '--spend-cap', '5', '--allow-repo-commands']);
expect(run.status, run.err).toBe(EXIT.ok);
const log = path.join(repo.root, 'stage-run.log');
fs.writeFileSync(log, run.out);
expect(run.out).toMatch(/^ +lanes: +PREGATE_REMOTE: tree=[0-9a-f]{40} tests=3 failed=0$/m);
expect(run.out).not.toMatch(/^PREGATE_REMOTE: /m);
const failed = runBin(repo, 'gstack-pregate', ['--base', 'main', '--stage', 'tests', '--remote', '--spend-cap', '5', '--allow-repo-commands'], { GATE_EXIT: '1' });
expect(failed.out).toMatch(/^ +lanes: +PREGATE_REMOTE: tree=[0-9a-f]{40} tests=3 failed=0$/m);
const failedLog = path.join(repo.root, 'failed-run.log');
fs.writeFileSync(failedLog, failed.out);
const fileAt = path.join(repo.work, '.gstack/tmp/pregate.json');
const { file } = await lanesOn(repo, { cores: 4, testBackend: 'local' });
fs.writeFileSync(fileAt, JSON.stringify(file));
const clear = (receipt: string) => runBin(repo, 'gstack-pregate', ['clear', '--remote', 'pregate.json/remote_gate', '--receipt', receipt]);
const other = path.join(repo.root, 'other-check.log');
fs.writeFileSync(other, ` guards: PREGATE_REMOTE: tree=${file.tree} tests=3 failed=0\n`);
expect(clear(other).err).toContain('has no PREGATE_REMOTE: line');
const refused = clear(failedLog);
expect(refused.status).not.toBe(EXIT.ok);
expect(refused.err).toContain('under a lanes check that is incomplete, not requires-remote');
const ok = clear(log);
expect(ok.status, ok.err).toBe(EXIT.ok);
expect(ok.out).toContain(`PREGATE_CLEARED: pregate.json/remote_gate receipt=remote-gate tree=${file.tree.slice(0, 12)} tests=3 `);
});
test('`clear` on a test_backend obligation records the run URL as attested, not validated, and says so', async () => {
const repo = gated(BASE_PREGATE);
const fileAt = path.join(repo.work, '.gstack/tmp/pregate.json');
const { file, lanes } = await lanesOn(repo, { cores: 4, testBackend: 'ubicloud' });
expect(lanes.detail).toContain('gstack-pregate clear --remote test_backend/ubicloud --run-url <url>');
fs.mkdirSync(path.dirname(fileAt), { recursive: true });
fs.writeFileSync(fileAt, JSON.stringify(file));
expect(runBin(repo, 'gstack-pregate', ['clear', '--remote', 'test_backend/ubicloud', '--receipt', fileAt]).status).toBe(EXIT.usage);
const r = runBin(repo, 'gstack-pregate', ['clear', '--remote', 'test_backend/ubicloud', '--run-url', 'https://ci.example/run/7']);
expect(r.status).toBe(EXIT.ok);
expect(r.out).toContain('PREGATE_CLEARED: test_backend/ubicloud receipt=attested:https://ci.example/run/7 verdict=requires-remote cleared (attested, not validated: test_backend/ubicloud has no machine-checkable receipt)');
expect(JSON.parse(fs.readFileSync(fileAt, 'utf8')).requires_remote[0].receipt).toBe('attested:https://ci.example/run/7');
});
test('config: stall_timeout_s and remote_gate are typed', async () => {
const { validateConfig } = await import('../lib/pregate/config');
const codes = (c: unknown) => validateConfig(c).map(e => `${e.code}:${e.path}`);
expect(codes({ stall_timeout_s: 0, remote_gate: GATE })).toEqual([]);
expect(codes({ stall_timeout_s: -1 })).toEqual(['PREGATE_CONFIG_INVALID:stall_timeout_s']);
expect(codes({ stall_timeout_s: 1.5 })).toEqual(['PREGATE_CONFIG_INVALID:stall_timeout_s']);
expect(codes({ remote_gate: 'a\nb' })).toEqual(['PREGATE_CONFIG_INVALID:remote_gate']);
});
});
describe('gstack-pregate --remote (C-c)', () => {
const remote = (repo: FixtureRepo, args: string[], env: Record<string, string> = {}) => runBin(repo, 'gstack-pregate', ['--base', 'main', '--stage', 'tests', ...args], env);
const lanes = (repo: FixtureRepo) => JSON.parse(fs.readFileSync(path.join(repo.work, '.gstack/tmp/pregate.json'), 'utf8')).checks.find((c: any) => c.id === 'lanes');
const pregateFile = (repo: FixtureRepo) => JSON.parse(fs.readFileSync(path.join(repo.work, '.gstack/tmp/pregate.json'), 'utf8'));
test('needs --allow-repo-commands and a spend cap', () => {
const repo = gated();
expect(remote(repo, ['--remote', '--spend-cap', '5']).status).toBe(EXIT.refused);
const noCap = remote(repo, ['--remote', '--allow-repo-commands']);
expect(noCap.status).toBe(EXIT.usage);
expect(noCap.err).toContain('pass --spend-cap <usd> (or set GSTACK_SPEND_CAP_USD)');
expect(remote(repo, ['--spend-cap', '5', '--allow-repo-commands']).status).toBe(EXIT.usage);
expect(fs.existsSync(path.join(repo.root, 'gate-env.json'))).toBe(false);
});
test('a gate reporting the tested tree, tests > 0 and failed = 0 clears the obligation and reports its spend', () => {
const repo = gated();
expect(runBin(repo, 'gstack-pregate', ['--base', 'main', '--stage', 'preflight', '--allow-repo-commands']).status).toBe(EXIT.ok);
const r = remote(repo, ['--remote', '--allow-repo-commands'], { GSTACK_SPEND_CAP_USD: '4', GATE_SPENT: '0.37' });
expect(r.status).toBe(EXIT.ok);
const file = pregateFile(repo);
expect(file.verdict).toBe('requires-remote cleared');
expect(file.requires_remote[0]).toMatchObject({ lane: 'pregate.json/remote_gate', receipt: `remote-gate tree=${file.tree.slice(0, 12)} tests=3` });
expect(lanes(repo).detail).toContain('spend: spent $0.37, reserved $0, unknown $0 (cap $4)');
expect(r.err).toContain('PREGATE_REMOTE_SPEND: spent $0.37, reserved $0, unknown $0 (cap $4)');
const env = JSON.parse(fs.readFileSync(path.join(repo.root, 'gate-env.json'), 'utf8'));
expect(env).toEqual({ tree: file.tree, cap: '4', timeout: '600', files: 'test/util.test.js\n' });
expect(runBin(repo, 'gstack-pregate', ['verify']).status).toBe(EXIT.ok);
});
// Value: protects=a head policy that only adds lanes can clear the remote obligation on a small machine, with the base copy's gate and the
// policy's timeout_s as the gate's deadline (PREGATE_TIMEOUT_S); a head policy that changes the gate is refused (GST-419, GST-420);
// fails_when=--remote refuses head mode, the gate deadline ignores timeout_s or --timeout, or a head remote_gate runs; why_new=head mode
// could never clear on under-8-core crew machines; seam=fake gate; no_claim=the Ubicloud VM
test('under --policy-from head the base gate runs with the head lanes and the timeout_s deadline; a head policy that changes the gate is refused', () => {
const policy = { ...BASE_PREGATE, remote_gate: GATE, timeout_s: 900 };
const repo = gated(policy);
const lane = { ...BASE_PREGATE.dependencies, 'lib/orphan.js': ['test/util.test.js'] };
commit(repo, { 'lib/orphan.js': 'exports.lonely = () => 2;\n', '.gstack/pregate.json': JSON.stringify({ ...policy, dependencies: lane }) + '\n' });
const env = () => JSON.parse(fs.readFileSync(path.join(repo.root, 'gate-env.json'), 'utf8'));
expect(remote(repo, ['--remote', '--allow-repo-commands', '--policy-from', 'head', '--spend-cap', '2']).status).toBe(EXIT.ok);
expect(env().timeout).toBe('900');
expect(env().files).toContain('test/util.test.js');
expect(remote(repo, ['--remote', '--allow-repo-commands', '--policy-from', 'head', '--spend-cap', '2', '--timeout', '1200']).status).toBe(EXIT.ok);
expect(env().timeout).toBe('1200');
commit(repo, { '.gstack/pregate.json': JSON.stringify({ ...policy, remote_gate: 'node scripts/other-gate.js', dependencies: lane }) + '\n' });
const changed = remote(repo, ['--remote', '--allow-repo-commands', '--policy-from', 'head', '--spend-cap', '2']);
expect(changed.status).toBe(EXIT.refused);
expect(changed.err).toContain('remote_gate differs from the trusted policy');
}, 60_000);
// Value: protects=a bootstrap head policy (no base copy, no user-owned file) cannot run its own remote_gate to clear its obligation;
// fails_when=--remote under head runs a branch-authored gate and verify passes with no test run; why_new=--remote now runs under
// head once the additive check passes, and a bootstrap has nothing to check against; seam=fake gate; no_claim=none
test('--remote under --policy-from head with no trusted policy is refused before any gate runs', () => {
const repo = makeFixtureRepo({ pregate: null });
repos.push(repo);
commit(repo, { 'scripts/remote-gate.js': GATE_SCRIPT, '.gstack/pregate.json': JSON.stringify({ ...BASE_PREGATE, remote_gate: GATE }) + '\n', 'lib/util.js': "exports.add = (a, b) => a + b + 0;\n" });
const r = remote(repo, ['--remote', '--allow-repo-commands', '--policy-from', 'head', '--spend-cap', '2'], { GSTACK_PROJECT_SLUG: 'remote-bootstrap' });
expect(r.status).toBe(EXIT.refused);
expect(r.err).toContain('the head file is a bootstrap and its remote_gate cannot clear its own obligation');
expect(fs.existsSync(path.join(repo.root, 'gate-env.json'))).toBe(false);
}, 60_000);
// Value: protects=a remote gate that waits quietly on CI is bounded by its deadline (--timeout, else timeout_s), not killed by the
// test-output stall timer; a gate past its deadline still stays uncleared (GST-431); fails_when=the stall timer kills a quiet gate
// inside its deadline, or a gate past --timeout clears; why_new=a CI-waiting gate died at 300 s on qm (CAPY-1); seam=fake gate;
// no_claim=the Ubicloud VM
test('a gate that is quiet past the stall timeout but inside its deadline clears; past --timeout it stays uncleared', () => {
const repo = gated();
const quiet = remote(repo, ['--remote', '--allow-repo-commands', '--spend-cap', '2', '--stall-timeout', '1', '--timeout', '60'], { GATE_QUIET_MS: '2500' });
expect(quiet.status).toBe(EXIT.ok);
expect(pregateFile(repo).verdict).toBe('requires-remote cleared');
const late = remote(repo, ['--remote', '--allow-repo-commands', '--spend-cap', '2', '--timeout', '2'], { GATE_QUIET_MS: '8000' });
expect(late.status).toBe(EXIT.fail);
expect(lanes(repo)).toMatchObject({ status: 'incomplete', code: 'PREGATE_INCOMPLETE' });
expect(lanes(repo).detail).toContain('the remote gate did not finish (timed out after 2 s, --timeout)');
expect(pregateFile(repo).requires_remote[0].receipt).toBeNull();
}, 60_000);
test('a wrong tree, zero tests, a failure, a missing line or a non-zero exit leaves the obligation uncleared and the check incomplete; unreported spend counts at the cap', () => {
const repo = gated();
const cases: Array<[Record<string, string>, string]> = [
[{ GATE_TREE: 'deadbeef'.repeat(8) }, 'the remote gate tested tree deadbeefdead'],
[{ GATE_TESTS: '0' }, 'the remote gate reported 0 tests'],
[{ GATE_FAILED: '1' }, 'the remote gate reported 1 failed of 3'],
[{ GATE_SILENT: '1' }, 'the remote gate exited 0 without a PREGATE_REMOTE: line'],
];
for (const [env, why] of cases) {
const r = remote(repo, ['--remote', '--allow-repo-commands', '--spend-cap', '2'], env);
expect(r.status).toBe(EXIT.fail);
const row = lanes(repo);
expect(row.status).toBe('incomplete');
expect(row.detail).toContain(why);
expect(row.detail).toContain('requires-remote pregate.json/remote_gate stays uncleared');
expect(pregateFile(repo).requires_remote[0].receipt).toBeNull();
}
expect(lanes(repo).lines).toContain('spend: spent $0, reserved $0, unknown $2 (no spent= reported; counted at the cap) (cap $2)');
const backendOnly = gated(BASE_PREGATE);
expect(remote(backendOnly, ['--remote', '--allow-repo-commands', '--spend-cap', '2']).status).toBe(EXIT.fail);
expect(lanes(backendOnly).detail).toContain('--remote runs only a declared remote gate');
});
});