mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-01 14:34:42 +01:00
fix(pregate): test stage never hangs silently; small machines record the remote gate (Lane C, GBRA-95)
C1 diagnosis (gbrain 92f0ea87, a core edit selecting 3,683 local files, 358 serial; Bun 1.4.2 and 1.4.3 identical): --timeout was applied per runner spawn (unit group plus one per serial file), so the check was unbounded, and spawnSync buffered all output until it returned: 360 s with 0 bytes on stdout and stderr. A backgrounded child holding stdout made spawnSync wait the full timeout and then report exit 0 (a silent wait, then green). Not the Bun 1.4.2 spawnSync hang. C2/C-b: test-tier commands (lanes runner, guards, the --remote gate) run via lib/pregate/supervise.ts: own process group (setsid), stdout/stderr appended to a log file (no pipes), async supervision; CheckDef.run may return a promise and runPregate/withFingerprint await it (a rejection is incomplete). Leftover group members are killed after exit; SIGINT/SIGTERM to pregate SIGKILLs running groups, writes no pregate.json, exits 130/143. Windows: taskkill /T /F (reparented or detached descendants can survive; documented). C3/C-a: --timeout bounds the whole check; groups past the deadline are not started. --stall-timeout <s> (default stall_timeout_s, else 300; 0 disables) kills a group whose log stops growing and records PREGATE_STALLED naming the last file started. Live stderr: PREGATE_TEST per file (bun test adapter; any runner for one-file runs), PREGATE_HEARTBEAT after 60 s without progress, PREGATE: tests <verdict>. --json stdout unchanged. C4/C-c: under 8 cores, a base-branch remote_gate or a non-local test_backend records the local tier as a requires-remote obligation with its command (pregate.json/remote_gate or test_backend/<backend>). --remote runs the declared remote_gate now (needs --allow-repo-commands, never --policy-from head) under --spend-cap/GSTACK_SPEND_CAP_USD; it clears only on PREGATE_REMOTE: tree=<tested tree> tests>0 failed=0 with exit 0, and reports spent/reserved/unknown. G1: new code PREGATE_STALLED has its result-codes row, troubleshooting anchor and the installer-form error line (CODE; url#anchor). Files (G2): lib/pregate/supervise.ts (new), lib/pregate/types.ts, lib/pregate/context.ts, lib/pregate/index.ts, lib/pregate/checks/lanes.ts, lib/pregate/checks/guards.ts, lib/pregate/config.ts, lib/headless-artifacts.ts, lib/result-codes.ts, bin/gstack-pregate, bin/gstack-doctor-components.sh (generated), docs/pregate.md, docs/troubleshooting.md, test/pregate-supervise.test.ts (new), test/pregate-remote.test.ts (new). Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
This commit is contained in:
garrytanandcapy-ai[bot] committed
1 parent
47d639abff
commit
d6f4ea9f12
15 files changed
+879
-60
No files matched your search
@@ -159,6 +159,7 @@ gstack_result_anchor() {
|
||||
PREGATE_NO_LANE) printf '%s' 'pregate-no-lane' ;;
|
||||
PREGATE_LANE_FAILED) printf '%s' 'pregate-lane-failed' ;;
|
||||
PREGATE_INCOMPLETE) printf '%s' 'pregate-incomplete' ;;
|
||||
PREGATE_STALLED) printf '%s' 'pregate-stalled' ;;
|
||||
PREGATE_STALE) printf '%s' 'pregate-stale' ;;
|
||||
PREGATE_REMOTE_UNCLEARED) printf '%s' 'pregate-remote-uncleared' ;;
|
||||
ISSUE_UNREADABLE) printf '%s' 'issue-unreadable' ;;
|
||||
|
||||
+47
-12
@@ -4,7 +4,7 @@
|
||||
*
|
||||
* gstack-pregate [run] [--stage preflight|tests|all] [--only <check,..>] [--base <ref>]
|
||||
* [--allow-repo-commands] [--policy-from base|head] [--out <pregate.json>]
|
||||
* [--timeout <s>] [--run <id>] [--explain] [--json]
|
||||
* [--timeout <s>] [--stall-timeout <s>] [--remote --spend-cap <usd>] [--run <id>] [--explain] [--json]
|
||||
* gstack-pregate verify [--out <pregate.json>] [--json] publication check: current tree, no fail/incomplete, obligations cleared
|
||||
* gstack-pregate clear --remote <workflow.yml/job> --run-url <url> | --bundle-lane <label> [--out <pregate.json>]
|
||||
* gstack-pregate lanes [--explain] the lane list and platform tiers read from .github/workflows
|
||||
@@ -32,7 +32,9 @@ import { buildContext } from '../lib/pregate/context';
|
||||
import { CHECKS, CHECK_IDS, PREGATE_OUT_REL, PREGATE_TIER2, clearObligation, readPregateFile, renderTable, runPregate, verifyPregate, writePregateFile } from '../lib/pregate/index';
|
||||
import { describeLane, discoverLanes } from '../lib/pregate/workflows';
|
||||
import { ensureTmpExcluded } from '../lib/restamp';
|
||||
import { resultLine, type ResultCodeName } from '../lib/result-codes';
|
||||
import { resultCodeSuffix, resultLine, type ResultCodeName } from '../lib/result-codes';
|
||||
import { DEFAULT_STALL_TIMEOUT_S } from '../lib/pregate/context';
|
||||
import { cancelActive } from '../lib/pregate/supervise';
|
||||
import { defaultGh, evaluateRegate, renderRegate } from '../lib/pregate/regate';
|
||||
import { GATE_AHEAD_REL } from '../lib/restamp';
|
||||
import { evaluateSealed, publishedCommits, rangeCommits, renderSealed } from '../lib/sealed-guard';
|
||||
@@ -59,19 +61,31 @@ const HELP = `gstack-pregate — cheap checks before anything paid or remote (do
|
||||
--allow-repo-commands run pin-file commands and the test runner (what /ship passes); without it those checks are incomplete
|
||||
--policy-from base|head base (default, reviewed) or head (unreviewed; needs --allow-repo-commands)
|
||||
--out <file> pregate.json path (default ${PREGATE_OUT_REL})
|
||||
--timeout <s> per-check timeout (default 600)
|
||||
--timeout <s> per-check timeout (default 600): it bounds the whole check, every runner process included
|
||||
--stall-timeout <s> kill a test command's process group after this long with no output and record the check
|
||||
incomplete (PREGATE_STALLED); default stall_timeout_s from the base .gstack/pregate.json,
|
||||
else ${DEFAULT_STALL_TIMEOUT_S}; 0 disables
|
||||
--remote run the base branch's remote_gate now instead of local tests (needs --allow-repo-commands and
|
||||
--policy-from base); it clears the requires-remote obligation only on its
|
||||
PREGATE_REMOTE: tree=<tested tree> tests=<n> failed=0 line with n > 0
|
||||
--spend-cap <usd> the cap --remote passes to the gate (default \$GSTACK_SPEND_CAP_USD); spend is reported as
|
||||
spent, reserved and unknown
|
||||
--explain print each selected test's provenance and each uncertain mapping
|
||||
--json JSON only on stdout
|
||||
|
||||
A failing check blocks (exit 1); incomplete never passes; requires-remote lanes are persisted
|
||||
obligations publication needs receipts for. Ends with GSTACK_RESULT: skill=pregate status=… run=<pregate.json>.
|
||||
obligations publication needs receipts for. Under 8 cores, a base-branch remote_gate or a remote
|
||||
test_backend (gstack-config) records the local test tier as a requires-remote obligation with the
|
||||
command to run instead of running it. Test progress goes to stderr live (PREGATE_TEST: per file
|
||||
for bun test, PREGATE_HEARTBEAT: after each 60 s without progress, then PREGATE: tests <verdict>);
|
||||
--json keeps stdout to the JSON document. Ends with GSTACK_RESULT: skill=pregate status=… run=<pregate.json>.
|
||||
${exitTableHelp()}`;
|
||||
|
||||
function flag(args: string[], name: string): string | undefined { const i = args.indexOf(name); return i >= 0 ? args[i + 1] : undefined; }
|
||||
|
||||
/** Flags each verb accepts: `[flags that take a value, bare flags]`. Anything else is usage (exit 2), never a silent default run. */
|
||||
const VERB_FLAGS: Record<string, [string[], string[]]> = {
|
||||
run: [['--stage', '--only', '--base', '--policy-from', '--out', '--timeout', '--run'], ['--allow-repo-commands', '--explain', '--json']],
|
||||
run: [['--stage', '--only', '--base', '--policy-from', '--out', '--timeout', '--stall-timeout', '--spend-cap', '--run'], ['--allow-repo-commands', '--explain', '--json', '--remote']],
|
||||
verify: [['--out'], ['--json']],
|
||||
clear: [['--remote', '--run-url', '--bundle-lane', '--out'], []],
|
||||
lanes: [[], ['--explain', '--json']],
|
||||
@@ -101,7 +115,7 @@ function repoRoot(): string | null {
|
||||
}
|
||||
function outPath(root: string, rest: string[]): string { const o = flag(rest, '--out'); return o ? path.resolve(o) : path.join(root, PREGATE_OUT_REL); }
|
||||
|
||||
function cmdRun(root: string, rest: string[], json: boolean): number {
|
||||
async function cmdRun(root: string, rest: string[], json: boolean): Promise<number> {
|
||||
const stage = (flag(rest, '--stage') ?? 'all') as 'preflight' | 'tests' | 'all';
|
||||
if (!['preflight', 'tests', 'all'].includes(stage)) { process.stderr.write('gstack-pregate: --stage needs preflight, tests or all\n'); return EXIT.usage; }
|
||||
const only = flag(rest, '--only')?.split(',').map(s => s.trim()).filter(Boolean);
|
||||
@@ -113,8 +127,18 @@ function cmdRun(root: string, rest: string[], json: boolean): number {
|
||||
if (policyFrom === 'head' && !allow) { process.stderr.write('gstack-pregate: --policy-from head needs --allow-repo-commands (the working-tree pin files are unreviewed)\n'); return EXIT.refused; }
|
||||
const timeoutMs = Number(flag(rest, '--timeout') ?? 600) * 1000;
|
||||
if (!Number.isFinite(timeoutMs) || timeoutMs <= 0) { process.stderr.write('gstack-pregate: --timeout needs a positive number of seconds\n'); return EXIT.usage; }
|
||||
const stallFlag = flag(rest, '--stall-timeout');
|
||||
const stallTimeoutMs = stallFlag === undefined ? undefined : Number(stallFlag) * 1000;
|
||||
if (stallTimeoutMs !== undefined && (!Number.isFinite(stallTimeoutMs) || stallTimeoutMs < 0)) { process.stderr.write('gstack-pregate: --stall-timeout needs a number of seconds (0 disables)\n'); return EXIT.usage; }
|
||||
let remote: { spendCapUsd: number } | null = null;
|
||||
if (rest.includes('--remote')) {
|
||||
if (!allow || policyFrom === 'head') { process.stderr.write('gstack-pregate: --remote runs the base branch\'s remote_gate and needs --allow-repo-commands with --policy-from base\n'); return EXIT.refused; }
|
||||
const cap = Number(flag(rest, '--spend-cap') ?? process.env.GSTACK_SPEND_CAP_USD ?? NaN);
|
||||
if (!Number.isFinite(cap) || cap <= 0) { process.stderr.write('gstack-pregate: --remote spends money; pass --spend-cap <usd> (or set GSTACK_SPEND_CAP_USD)\n'); return EXIT.usage; }
|
||||
remote = { spendCapUsd: cap };
|
||||
} else if (flag(rest, '--spend-cap') !== undefined) { process.stderr.write('gstack-pregate: --spend-cap applies only with --remote\n'); return EXIT.usage; }
|
||||
const base = resolveBaseBranch(root, flag(rest, '--base'));
|
||||
const loaded = buildContext({ repoRoot: root, base, policyFrom, allowRepoCommands: allow, timeoutMs, knownChecks: [...CHECK_IDS, ...PREGATE_TIER2] });
|
||||
const loaded = buildContext({ repoRoot: root, base, policyFrom, allowRepoCommands: allow, timeoutMs, stallTimeoutMs, remote, knownChecks: [...CHECK_IDS, ...PREGATE_TIER2] });
|
||||
if ('refused' in loaded) {
|
||||
const { code, message } = loaded.refused;
|
||||
process.stderr.write((code === 'POLICY_SOURCE_UNAVAILABLE' ? policyUnavailableLine('gstack-pregate', base, message).replace('.gstack/ship-policy.json', 'pin files') : resultLine('gstack-pregate', code as ResultCodeName, message)) + '\n');
|
||||
@@ -123,7 +147,14 @@ function cmdRun(root: string, rest: string[], json: boolean): number {
|
||||
}
|
||||
const ctx = loaded.ctx;
|
||||
ensureTmpExcluded(root);
|
||||
const file = runPregate(ctx, { stage, only, run: flag(rest, '--run') });
|
||||
for (const signal of ['SIGINT', 'SIGTERM'] as const) {
|
||||
process.once(signal, () => {
|
||||
const killed = cancelActive();
|
||||
process.stderr.write(`gstack-pregate: cancelled by ${signal}; killed ${killed} test process group(s); no pregate.json written\n`);
|
||||
process.exit(signal === 'SIGINT' ? 130 : 143);
|
||||
});
|
||||
}
|
||||
const file = await runPregate(ctx, { stage, only, run: flag(rest, '--run') });
|
||||
const out = outPath(root, rest);
|
||||
let written: string;
|
||||
try { written = writePregateFile(root, out, file); }
|
||||
@@ -134,7 +165,11 @@ function cmdRun(root: string, rest: string[], json: boolean): number {
|
||||
process.stdout.write(`pregate: policy ${ctx.configLabel}; registry ${ctx.registryLabel}; base ${ctx.baseRef}@${ctx.baseSha.slice(0, 12)}; ${ctx.touched.length} touched file(s); platform ${ctx.platform}\n`);
|
||||
process.stdout.write(renderTable(file));
|
||||
if (rest.includes('--explain')) { process.stdout.write('--- explain ---\n'); for (const line of ctx.explain) process.stdout.write(`${line}\n`); }
|
||||
for (const c of file.checks) if (c.status === 'fail' || c.status === 'incomplete') process.stderr.write(`gstack-pregate: ${c.id}: ${c.detail}; fix: ${c.fix ?? 'see the table'} (${c.code ?? 'PREGATE_INCOMPLETE'})\n`);
|
||||
for (const c of file.checks) {
|
||||
if (c.status !== 'fail' && c.status !== 'incomplete') continue;
|
||||
if (c.code === 'PREGATE_STALLED') process.stderr.write(`gstack-pregate: ${c.id} stalled — ${c.detail}; fix: ${c.fix} ${resultCodeSuffix(c.code)}\n`);
|
||||
else process.stderr.write(`gstack-pregate: ${c.id}: ${c.detail}; fix: ${c.fix ?? 'see the table'} (${c.code ?? 'PREGATE_INCOMPLETE'})\n`);
|
||||
}
|
||||
process.stdout.write(`pregate.json: ${written}\n`);
|
||||
process.stdout.write(formatResult('pregate', status, written) + '\n');
|
||||
return blocking ? EXIT.fail : EXIT.ok;
|
||||
@@ -221,7 +256,7 @@ function cmdRegate(root: string, rest: string[], json: boolean): number {
|
||||
return report.verdict === 'not-needed' ? EXIT.ok : EXIT.fail;
|
||||
}
|
||||
|
||||
function main(argv: string[]): number {
|
||||
async function main(argv: string[]): Promise<number> {
|
||||
const cmd = argv[0] && !argv[0].startsWith('--') ? argv[0] : argv.includes('--against') ? 'regate' : 'run';
|
||||
const rest = argv[0] && !argv[0].startsWith('--') ? argv.slice(1) : argv;
|
||||
const json = rest.includes('--json');
|
||||
@@ -230,7 +265,7 @@ function main(argv: string[]): number {
|
||||
if (bad) { process.stderr.write(`gstack-pregate: ${bad}; see gstack-pregate --help\n`); return EXIT.usage; }
|
||||
const root = repoRoot();
|
||||
if (!root) { process.stderr.write('gstack-pregate: not inside a git repository\n'); return EXIT.usage; }
|
||||
if (cmd === 'run') return cmdRun(root, rest, json);
|
||||
if (cmd === 'run') return await cmdRun(root, rest, json);
|
||||
if (cmd === 'verify') return cmdVerify(root, rest, json);
|
||||
if (cmd === 'clear') return cmdClear(root, rest);
|
||||
if (cmd === 'sealed') return cmdSealed(root, rest, json);
|
||||
@@ -262,4 +297,4 @@ function main(argv: string[]): number {
|
||||
return EXIT.usage;
|
||||
}
|
||||
|
||||
process.exitCode = main(process.argv.slice(2));
|
||||
process.exitCode = await main(process.argv.slice(2));
|
||||
+75
-1
@@ -66,6 +66,8 @@ the byte-pinned `/ship` renders in `test/fixtures/golden/`).
|
||||
| `serial_patterns` | test-file globs the `lanes` check runs one process per file, apart from the unit files (default `**/*.serial.test.*`) |
|
||||
| `sealed` | sealed or eval output that must never reach a public destination; read by the pre-push hook and `gstack-pregate sealed` (see [sealed output](#sealed)) |
|
||||
| `price_file` | repo-relative path of the paid gate's call-plan data file (`GSTACK_EVAL_CALLS v1`); `gstack-eval-plan price-gate` prices from it and never executes anything ([docs/eval-calls.md](eval-calls.md)) |
|
||||
| `stall_timeout_s` | seconds a test command may print nothing before its process group is killed and the check is `incomplete` (`PREGATE_STALLED`); default 300, `0` disables; `--stall-timeout` overrides (see [supervision](#supervision)) |
|
||||
| `remote_gate` | the repo's remote test gate command; under 8 cores the local test tier becomes a `requires-remote` obligation naming it, and `--remote` runs it (see [remote gate](#remote-gate)) |
|
||||
|
||||
Only the `origin/<base>` copy is honoured, so a downgrade is reviewed before it
|
||||
takes effect. `gstack-pregate validate` / `init` cover the working-tree file.
|
||||
@@ -73,7 +75,8 @@ takes effect. `gstack-pregate validate` / `init` cover the working-tree file.
|
||||
## `gstack-pregate`
|
||||
|
||||
```
|
||||
gstack-pregate [--stage preflight|tests|all] [--only <check,..>] [--allow-repo-commands] [--policy-from base|head] [--explain] [--json]
|
||||
gstack-pregate [--stage preflight|tests|all] [--only <check,..>] [--allow-repo-commands] [--policy-from base|head]
|
||||
[--timeout <s>] [--stall-timeout <s>] [--remote --spend-cap <usd>] [--explain] [--json]
|
||||
gstack-pregate verify | clear --remote <workflow.yml/job> --run-url <url> | lanes | validate | init
|
||||
```
|
||||
|
||||
@@ -123,6 +126,13 @@ Two stages with separate clocks, both printed:
|
||||
first of `$BUN_INSTALL/bin` and `~/.bun/bin` whose Bun satisfies gstack's and
|
||||
the project's `engines.bun` (the row prints `runtime: bun <v> from <dir>`),
|
||||
so a Capy host's older image Bun cannot flip the verdict.
|
||||
- Every test-tier command (the lanes runner, each guard, the `--remote` gate)
|
||||
runs supervised: see [supervision](#supervision). `--timeout` bounds the
|
||||
whole check (all runner processes together); a serial file that would start
|
||||
after the deadline is not started and the check is `incomplete`.
|
||||
- Under 8 cores, a base-branch `remote_gate` or a remote `test_backend`
|
||||
(`gstack-config set test_backend ubicloud`) records the local test tier as a
|
||||
`requires-remote` obligation instead of running it: see [remote gate](#remote-gate).
|
||||
|
||||
Every check records the input identity it ran against (the working-tree
|
||||
fingerprint before and after, the selection, the command hash); a tree that
|
||||
@@ -143,6 +153,70 @@ gstack lacks never gets a green it did not ask for). Tier 2 checks still to come
|
||||
(`ratchets`, `lint`, `links`, `hermetic`, `patches`, `integrator`) are named in
|
||||
`--only`'s help.
|
||||
|
||||
<a id="supervision"></a>
|
||||
## Supervision: progress, stalls, deadlines
|
||||
|
||||
GBRA-95 (gbrain, 4 cores): a core edit selected 3,683 local files, 358 of them
|
||||
serial; the test stage printed nothing for 90 minutes because each runner
|
||||
process had its own `--timeout` and its output was buffered until it returned.
|
||||
Now each test-tier command:
|
||||
|
||||
- runs in its own process group (`setsid`), with stdout and stderr appended to
|
||||
a log file, never a pipe, so a backgrounded grandchild that keeps the output
|
||||
open cannot hold the check; after the command exits, the group's leftover
|
||||
members are killed;
|
||||
- reports live on stderr (stdout stays the table, or one JSON document with
|
||||
`--json`):
|
||||
```
|
||||
PREGATE_TEST: <n>/<total> <file> start
|
||||
PREGATE_TEST: <n>/<total> <file> pass|fail <seconds>
|
||||
PREGATE_HEARTBEAT: <label> running <s>s; last file <file>; log <bytes> bytes, last growth <s>s ago (<log>)
|
||||
PREGATE: tests <verdict> in <s>s — lanes: <detail>
|
||||
```
|
||||
Per-file lines come from a progress adapter where the runner's reporter
|
||||
provides them: `bun test` (its `<file>:` headers) is the only adapter today;
|
||||
a one-file run (each serial file) reports its file from the launch for any
|
||||
runner. Other runners (node --test, vitest, jest, pytest) get heartbeats
|
||||
only. A heartbeat follows each 60 s with no progress line;
|
||||
- is killed as **stalled** when its log does not grow for the stall timeout
|
||||
(`--stall-timeout <s>`, else `stall_timeout_s`, else 300; `0` disables): the
|
||||
check is `incomplete` with `PREGATE_STALLED` and names the last file
|
||||
started; the next serial file still runs;
|
||||
- is killed as **timed out** at the check's `--timeout` deadline.
|
||||
|
||||
A kill is SIGTERM to the group, then SIGKILL after 5 s (a group that ignores
|
||||
SIGTERM still dies). SIGINT or SIGTERM to `gstack-pregate` itself SIGKILLs every
|
||||
running group, writes no `pregate.json` and exits 130 or 143. Windows has no
|
||||
process groups here: the kill is `taskkill /T /F` on the shell's tree, which
|
||||
misses a descendant that was reparented or started detached.
|
||||
|
||||
<a id="remote-gate"></a>
|
||||
## Small machines and the remote gate
|
||||
|
||||
Under 8 cores (`os.availableParallelism()`), when the base branch's
|
||||
`.gstack/pregate.json` declares `remote_gate` or `gstack-config get
|
||||
test_backend` is not `local`, the lanes check runs nothing locally: it records
|
||||
the selected local files as a `requires-remote` obligation with the command to
|
||||
run, `pregate.json/remote_gate` (the declared command) or
|
||||
`test_backend/<backend>` (gstack's `scripts/ubicloud/ubi-runner.sh run --src
|
||||
<repo> -- '<runner> <files>'`). Without either, the tier runs locally under
|
||||
the supervision above. Close the obligation one of two ways:
|
||||
|
||||
- `gstack-pregate clear --remote <lane> --run-url <url>` after running it; or
|
||||
- `gstack-pregate --stage tests --remote --spend-cap <usd> --allow-repo-commands`,
|
||||
which runs the declared `remote_gate` now (never under `--policy-from head`;
|
||||
a `test_backend` without `remote_gate` cannot be run this way). The gate runs
|
||||
in the repository root with `PREGATE_TREE` (the working-tree fingerprint
|
||||
being tested), `PREGATE_FILES_FILE` (the selected files, one per line) and
|
||||
`PREGATE_SPEND_CAP_USD`, and must print
|
||||
`PREGATE_REMOTE: tree=<tree> tests=<n> failed=<n> [spent=<usd>]`, reporting the
|
||||
tree it actually tested. The obligation clears only when that tree is
|
||||
`PREGATE_TREE`, `tests` > 0, `failed` = 0 and the gate exits 0; anything else
|
||||
leaves it uncleared and the check `incomplete`. The cap defaults to
|
||||
`$GSTACK_SPEND_CAP_USD`; there is no default spend. Spend prints as
|
||||
`spent, reserved, unknown`: a gate that prints no `spent=` is `unknown`,
|
||||
counted at the cap.
|
||||
|
||||
<a id="guards"></a>
|
||||
## Guards and the lane-done rule
|
||||
|
||||
|
||||
@@ -2381,6 +2381,21 @@ pass and blocks publication.
|
||||
**Fix.** Pass `--allow-repo-commands`, install the tool, raise `--timeout`, or
|
||||
keep the tree still, then rerun.
|
||||
|
||||
<a id="pregate-stalled"></a>
|
||||
### `PREGATE_STALLED`
|
||||
|
||||
**Meaning.** A test command the lanes check (or a guard, or the `--remote`
|
||||
gate) launched wrote nothing to its log for the stall timeout (default 300 s,
|
||||
`stall_timeout_s` in the base branch's `.gstack/pregate.json`, or
|
||||
`--stall-timeout`). pregate killed its process group and names the last test
|
||||
file started. The check is incomplete, never pass.
|
||||
|
||||
**Fix.** Open the printed log, find the hang in the named file (a blocked
|
||||
child, a server that never exits, a synchronous wait), fix it and rerun
|
||||
`gstack-pregate --stage tests`. A test that is legitimately quiet for longer
|
||||
needs a larger `--stall-timeout`; `0` disables the stall kill and leaves only
|
||||
`--timeout`.
|
||||
|
||||
<a id="pregate-stale"></a>
|
||||
### `PREGATE_STALE`
|
||||
|
||||
|
||||
@@ -66,7 +66,7 @@ export interface PregateCheck {
|
||||
// C-pregate (C8): stage, the input identity the check ran against, the fix clause and result code, and per-item lines.
|
||||
stage?: 'preflight' | 'tests'; inputs?: Record<string, string>; fix?: string; code?: string; lines?: string[];
|
||||
}
|
||||
export interface PregateRemote { lane: string; platform: string; workflow: string; job: string; tests: string[]; receipt: string | null; cleared_by?: string }
|
||||
export interface PregateRemote { lane: string; platform: string; workflow: string; job: string; tests: string[]; receipt: string | null; cleared_by?: string; command?: string }
|
||||
export interface PregateFile {
|
||||
schema_version: 1; tree: string; checks: PregateCheck[]; run?: string; stage1_ms?: number; stage2_ms?: number;
|
||||
// C-pregate (C8): head/base identity, the policy sources, persisted requires-remote obligations and the one-line verdict.
|
||||
@@ -185,7 +185,7 @@ export const ARTIFACT_SCHEMAS = {
|
||||
schema_version: { const: 1 }, run: str, tree: str, stage1_ms: num, stage2_ms: num,
|
||||
checks: { type: 'array', items: { type: 'object', required: ['id', 'status'], properties: { id: str, status: { enum: ['pass', 'fail', 'warn', 'incomplete', 'requires-remote', 'not_installed'] }, ms: num, detail: str, stage: { enum: ['preflight', 'tests'] }, inputs: { type: 'object' }, fix: str, code: str, lines: strList } } },
|
||||
head: str, base: str, generated_at: str, policy: str, registry: str, verdict: str,
|
||||
requires_remote: { type: 'array', items: { type: 'object', required: ['lane', 'platform', 'tests', 'receipt'], properties: { lane: str, platform: str, workflow: str, job: str, tests: strList, receipt: nullableStr, cleared_by: str } } },
|
||||
requires_remote: { type: 'array', items: { type: 'object', required: ['lane', 'platform', 'tests', 'receipt'], properties: { lane: str, platform: str, workflow: str, job: str, tests: strList, receipt: nullableStr, cleared_by: str, command: str } } },
|
||||
},
|
||||
},
|
||||
'ship-receipt': {
|
||||
|
||||
@@ -2,8 +2,9 @@
|
||||
* pregate check `guards` (tier 2, preflight; field wave CD1): the repo's own
|
||||
* mechanical gates a lane must pass before it reports done. Each
|
||||
* `.gstack/pregate.json` guard `{ id, when, command, fix?, timeout_s? }`
|
||||
* whose `when` globs match a touched file runs in its own process (bash,
|
||||
* under the verified launch Bun, with its own timeout). A failing guard
|
||||
* whose `when` globs match a touched file runs in its own process group
|
||||
* (lib/pregate/supervise.ts: bash, under the verified launch Bun, output to a
|
||||
* log file, with its own timeout and the stall timeout). A failing guard
|
||||
* fails pregate naming the guard and its fix; a timed-out one is
|
||||
* `incomplete`; without --allow-repo-commands the selected guards are listed
|
||||
* and the check is `incomplete`. Guards are never downgradable. Field
|
||||
@@ -11,7 +12,6 @@
|
||||
* passed both the isolation check and pregate; `test/cli-goldens.test.ts`
|
||||
* spawns the CLI as a subprocess, so no import graph selects it.
|
||||
*/
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import * as fs from 'node:fs';
|
||||
import * as os from 'node:os';
|
||||
import * as path from 'node:path';
|
||||
@@ -19,21 +19,21 @@ import { matchesAny } from '../../ship-policy';
|
||||
import type { Guard } from '../config';
|
||||
import { baseInputs, sha256 } from '../context';
|
||||
import { launchRuntime } from '../runtime';
|
||||
import { readRange, supervise } from '../supervise';
|
||||
import type { CheckDef, CheckResult, PregateContext } from '../types';
|
||||
|
||||
interface GuardRun { guard: Guard; exit: number | null; timedOut: boolean; tail: string[]; log: string }
|
||||
interface GuardRun { guard: Guard; exit: number | null; timedOut: boolean; stalled: boolean; tail: string[]; log: string }
|
||||
|
||||
function runGuard(ctx: PregateContext, guard: Guard, env: NodeJS.ProcessEnv, dir: string): GuardRun {
|
||||
async function runGuard(ctx: PregateContext, guard: Guard, env: NodeJS.ProcessEnv, dir: string): Promise<GuardRun> {
|
||||
const timeoutMs = guard.timeout_s ? guard.timeout_s * 1000 : ctx.timeoutMs;
|
||||
const r = spawnSync('bash', ['-c', `set -o pipefail; ${guard.command}`], { cwd: ctx.repoRoot, encoding: 'utf8', timeout: timeoutMs, killSignal: 'SIGKILL', maxBuffer: 64 * 1024 * 1024, env: { ...env, CI: env.CI ?? '1' } });
|
||||
const output = `${r.stdout ?? ''}\n${r.stderr ?? ''}`.trim();
|
||||
const log = path.join(dir, `${guard.id}.log`);
|
||||
fs.writeFileSync(log, `$ ${guard.command}\n${output}\n`);
|
||||
const timedOut = r.status === null;
|
||||
return { guard, exit: r.status, timedOut, tail: output ? output.split('\n').slice(-15) : [], log };
|
||||
fs.writeFileSync(log, '');
|
||||
const r = await supervise({ command: guard.command, cwd: ctx.repoRoot, env: { ...env, CI: env.CI ?? '1' }, log, deadline: Date.now() + timeoutMs, stallMs: ctx.stallTimeoutMs, label: `guard ${guard.id}`, progress: ctx.progress });
|
||||
const output = readRange(log, r.start, r.end).trim();
|
||||
return { guard, exit: r.exit, timedOut: r.outcome !== 'exited', stalled: r.outcome === 'stalled', tail: output ? output.split('\n').slice(-15) : [], log };
|
||||
}
|
||||
|
||||
function run(ctx: PregateContext): CheckResult {
|
||||
async function run(ctx: PregateContext): Promise<CheckResult> {
|
||||
const touched = ctx.touched.flatMap(t => (t.renamedFrom ? [t.path, t.renamedFrom] : [t.path]));
|
||||
const selected = ctx.config.guards.filter(g => touched.some(f => matchesAny(g.when, f)));
|
||||
const inputs = baseInputs(ctx, { guards: sha256(selected.map(g => `${g.id}\0${g.command}`).join('\n')) });
|
||||
@@ -46,10 +46,11 @@ function run(ctx: PregateContext): CheckResult {
|
||||
}
|
||||
const runtime = launchRuntime(ctx.repoRoot, ctx.env);
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-pregate-guards-'));
|
||||
const runs = selected.map(g => runGuard(ctx, g, runtime.env, dir));
|
||||
const runs: GuardRun[] = [];
|
||||
for (const g of selected) runs.push(await runGuard(ctx, g, runtime.env, dir));
|
||||
const lines = [`runtime: ${runtime.label}`];
|
||||
for (const r of runs) {
|
||||
lines.push(r.timedOut ? `GUARD: ${r.guard.id} timed out after ${r.guard.timeout_s ?? ctx.timeoutMs / 1000} s (log ${r.log})` : `GUARD: ${r.guard.id} exit=${r.exit} (log ${r.log})`);
|
||||
lines.push(r.stalled ? `GUARD: ${r.guard.id} stalled: no output for ${ctx.stallTimeoutMs / 1000} s; process group killed (log ${r.log})` : r.timedOut ? `GUARD: ${r.guard.id} timed out after ${r.guard.timeout_s ?? ctx.timeoutMs / 1000} s (log ${r.log})` : `GUARD: ${r.guard.id} exit=${r.exit} (log ${r.log})`);
|
||||
if (r.exit !== 0) lines.push(...r.tail.map(l => ` ${l}`));
|
||||
}
|
||||
const failed = runs.filter(r => !r.timedOut && r.exit !== 0);
|
||||
@@ -57,7 +58,7 @@ function run(ctx: PregateContext): CheckResult {
|
||||
const withRuns = { ...base, lines, inputs: { ...inputs, runtime: runtime.label } };
|
||||
const fixOf = (rs: GuardRun[]) => rs.map(r => `${r.guard.id}: ${r.guard.fix ?? `read ${r.log}, fix the failure, rerun gstack-pregate --only guards`}`).join('; ');
|
||||
if (failed.length) return { ...withRuns, status: 'fail', code: 'PREGATE_GUARD_FAILED', detail: `${failed.length} guard(s) failed: ${failed.map(r => r.guard.id).join(', ')}${timedOut.length ? `; timed out: ${timedOut.map(r => r.guard.id).join(', ')}` : ''}`, fix: fixOf(failed) };
|
||||
if (timedOut.length) return { ...withRuns, status: 'incomplete', code: 'PREGATE_INCOMPLETE', detail: `${timedOut.length} guard(s) timed out: ${timedOut.map(r => r.guard.id).join(', ')}`, fix: 'raise the guard\'s timeout_s or --timeout; a timed-out guard never passes' };
|
||||
if (timedOut.length) return { ...withRuns, status: 'incomplete', code: timedOut.some(r => r.stalled) ? 'PREGATE_STALLED' : 'PREGATE_INCOMPLETE', detail: `${timedOut.length} guard(s) did not finish: ${timedOut.map(r => `${r.guard.id} (${r.stalled ? 'stalled' : 'timed out'})`).join(', ')}`, fix: 'fix the hang, or raise the guard\'s timeout_s, --timeout or --stall-timeout; an unfinished guard never passes' };
|
||||
return { ...withRuns, detail: `${runs.length} guard(s) passed: ${runs.map(r => r.guard.id).join(', ')}` };
|
||||
}
|
||||
|
||||
|
||||
+101
-18
@@ -8,15 +8,15 @@
|
||||
* Incidents: p0-4-pipe-hidden-failure (`false | tail` read as green),
|
||||
* p0-4-windows-lane-unrun (a Windows-only change merged with its lane unrun).
|
||||
*/
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import * as fs from 'node:fs';
|
||||
import * as os from 'node:os';
|
||||
import * as path from 'node:path';
|
||||
import { baseInputs, sha256 } from '../context';
|
||||
import { selectLanes, type Selection } from '../lane-selection';
|
||||
import { launchRuntime } from '../runtime';
|
||||
import { adapterFor, readRange, supervise, type Counter, type Supervised } from '../supervise';
|
||||
import { matchesAny } from '../../ship-policy';
|
||||
import type { CheckDef, CheckResult, PregateContext } from '../types';
|
||||
import type { CheckDef, CheckResult, PregateContext, RemoteObligation } from '../types';
|
||||
|
||||
/** The test command with `{files}`; detected from the repo when the config does not name one. */
|
||||
export function detectRunner(ctx: PregateContext, files: readonly string[]): string | null {
|
||||
@@ -49,34 +49,56 @@ export function parseSummary(output: string): RunnerSummary {
|
||||
return { ran: null, failed: null };
|
||||
}
|
||||
|
||||
interface LocalRun { exit: number | null; ran: number | null; failed: number | null; log: string; tail: string[]; command: string }
|
||||
interface LocalRun { exit: number | null; outcome: Supervised['outcome'] | 'not-started'; ran: number | null; failed: number | null; tail: string[]; command: string; files: string[]; lastFile: string | null; ms: number }
|
||||
|
||||
function runOnce(ctx: PregateContext, env: NodeJS.ProcessEnv, runner: string, files: string[], log: string): LocalRun {
|
||||
const command = runner.replace('{files}', files.map(f => `'${f.replace(/'/g, `'\\''`)}'`).join(' '));
|
||||
const r = spawnSync('bash', ['-c', `set -o pipefail; ${command}`], { cwd: ctx.repoRoot, encoding: 'utf8', timeout: ctx.timeoutMs, maxBuffer: 256 * 1024 * 1024, env: { ...env, CI: env.CI ?? '1' } });
|
||||
const output = `${r.stdout ?? ''}\n${r.stderr ?? ''}`;
|
||||
fs.appendFileSync(log, `$ ${command}\n${output}\n`);
|
||||
const quote = (f: string) => `'${f.replace(/'/g, `'\\''`)}'`;
|
||||
|
||||
async function runOnce(ctx: PregateContext, env: NodeJS.ProcessEnv, runner: string, files: string[], log: string, deadline: number, counter: Counter): Promise<LocalRun> {
|
||||
const command = runner.replace('{files}', files.map(quote).join(' '));
|
||||
const r = await supervise({ command, cwd: ctx.repoRoot, env: { ...env, CI: env.CI ?? '1' }, log, deadline, stallMs: ctx.stallTimeoutMs, label: `lanes (${files.length === 1 ? files[0] : `${files.length} files`})`, files, adapter: adapterFor(runner), counter, progress: ctx.progress });
|
||||
const output = readRange(log, r.start, r.end);
|
||||
const summary = parseSummary(output);
|
||||
return { exit: r.status, ...summary, log, tail: output.trim().split('\n').slice(-25), command };
|
||||
return { exit: r.exit, outcome: r.outcome, ...summary, tail: output.trim().split('\n').slice(-25), command, files, lastFile: r.lastFile, ms: r.ms };
|
||||
}
|
||||
|
||||
/**
|
||||
* Unit files in one runner process, then each serial-pattern file in its
|
||||
* own (a serial file shares process-wide state: a module mock, an env
|
||||
* mutation). The first non-zero or timed-out run decides the exit; counts
|
||||
* add up only when every run reported a summary.
|
||||
* mutation). `--timeout` bounds the whole check: a group that would start
|
||||
* after the deadline is not started. A stalled group is killed and the next
|
||||
* one runs. The first unfinished, then the first non-zero run decides the
|
||||
* exit; counts add up only when every run reported a summary.
|
||||
*/
|
||||
function runLocal(ctx: PregateContext, env: NodeJS.ProcessEnv, runner: string, files: string[]): LocalRun & { serial: number } {
|
||||
async function runLocal(ctx: PregateContext, env: NodeJS.ProcessEnv, runner: string, files: string[]) {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-pregate-lanes-'));
|
||||
const log = path.join(dir, 'lanes.log');
|
||||
fs.writeFileSync(log, '');
|
||||
const serial = files.filter(f => matchesAny(ctx.config.serial_patterns, f));
|
||||
const unit = files.filter(f => !serial.includes(f));
|
||||
const runs = [...(unit.length ? [unit] : []), ...serial.map(f => [f])].map(group => runOnce(ctx, env, runner, group, log));
|
||||
const deadline = Date.now() + ctx.timeoutMs;
|
||||
const counter: Counter = { done: 0, total: files.length };
|
||||
const runs: LocalRun[] = [];
|
||||
for (const group of [...(unit.length ? [unit] : []), ...serial.map(f => [f])]) {
|
||||
if (Date.now() >= deadline) { runs.push({ exit: null, outcome: 'not-started', ran: null, failed: null, tail: [], command: '', files: group, lastFile: null, ms: 0 }); continue; }
|
||||
runs.push(await runOnce(ctx, env, runner, group, log, deadline, counter));
|
||||
}
|
||||
const bad = runs.find(r => r.exit === null) ?? runs.find(r => r.exit !== 0);
|
||||
const sum = (k: 'ran' | 'failed') => (runs.some(r => r[k] === null) ? null : runs.reduce((n, r) => n + (r[k] as number), 0));
|
||||
return { exit: bad ? bad.exit : 0, ran: sum('ran'), failed: sum('failed'), log, tail: (bad ?? runs[runs.length - 1]!).tail, command: runs.map(r => r.command).join(' && '), serial: serial.length };
|
||||
const started = runs.filter(r => r.outcome !== 'not-started');
|
||||
return { runs, bad, exit: bad ? bad.exit : 0, ran: sum('ran'), failed: sum('failed'), log, tail: (bad?.tail.length ? bad : started[started.length - 1] ?? runs[0]!).tail, command: started.map(r => r.command).join(' && '), serial: serial.length };
|
||||
}
|
||||
|
||||
/** Why the local lane did not finish: the stall, the deadline or a launch failure, with the last file started. */
|
||||
function unfinished(ctx: PregateContext, r: Awaited<ReturnType<typeof runLocal>>, total: number): Pick<CheckResult, 'code' | 'detail' | 'fix'> {
|
||||
const notStarted = r.runs.filter(x => x.outcome === 'not-started').flatMap(x => x.files).length;
|
||||
const stalled = r.runs.filter(x => x.outcome === 'stalled');
|
||||
const rest = `${notStarted ? `; ${notStarted} of ${total} file(s) not started` : ''}${r.failed ? `; ${r.failed} failed before that` : ''}`;
|
||||
if (stalled.length) {
|
||||
return { code: 'PREGATE_STALLED', detail: `no test output for ${ctx.stallTimeoutMs / 1000} s; process group killed; last file started: ${stalled.map(x => x.lastFile ?? `(none reported; ${x.files.length} file(s))`).join(', ')}${rest}`, fix: `read ${r.log} for the stalled file; fix the hang, or raise --stall-timeout (stall_timeout_s in .gstack/pregate.json; 0 disables) for a test that is quiet for longer` };
|
||||
}
|
||||
const why = r.runs.find(x => x.outcome === 'spawn-failed') ? 'the runner could not be launched' : `local lane timed out after ${ctx.timeoutMs} ms`;
|
||||
return { code: 'PREGATE_INCOMPLETE', detail: `${why} (${total} files${rest})`, fix: 'raise --timeout or split the selection; a timed-out lane never passes' };
|
||||
}
|
||||
function describe(sel: Selection): string[] {
|
||||
const lines = sel.noLane.map(f => `FAIL lanes — ${f} has no test lane`);
|
||||
for (const r of sel.remote) lines.push(`requires-remote ${r.lane} (${r.platform}): ${r.tests.length} test(s) — ${r.tests.slice(0, 8).join(' ')}${r.tests.length > 8 ? ' …' : ''}`);
|
||||
@@ -84,7 +106,62 @@ function describe(sel: Selection): string[] {
|
||||
return lines;
|
||||
}
|
||||
|
||||
function run(ctx: PregateContext): CheckResult {
|
||||
export const SMALL_MACHINE_CORES = 8;
|
||||
const GSTACK_ROOT = path.resolve(import.meta.dir, '..', '..', '..');
|
||||
|
||||
/**
|
||||
* The small-machine obligation: the repo's declared `remote_gate`, else the
|
||||
* configured test backend running this selection through gstack's Ubicloud
|
||||
* runner (that command prints no PREGATE_REMOTE line, so only `clear` with a
|
||||
* run URL discharges it).
|
||||
*/
|
||||
function remoteObligation(ctx: PregateContext, sel: Selection): RemoteObligation {
|
||||
const declared = ctx.config.remote_gate;
|
||||
const runner = detectRunner(ctx, sel.localTests);
|
||||
const command = declared ?? `${GSTACK_ROOT}/scripts/ubicloud/ubi-runner.sh run --src ${quote(ctx.repoRoot)} -- ${quote(runner ? runner.replace('{files}', sel.localTests.map(quote).join(' ')) : '<test runner> <selected files>')}`;
|
||||
const [workflow, job] = declared ? ['pregate.json', 'remote_gate'] : ['test_backend', ctx.testBackend];
|
||||
return { lane: `${workflow}/${job}`, platform: 'remote', workflow, job, tests: sel.localTests, receipt: null, command };
|
||||
}
|
||||
|
||||
/**
|
||||
* `--remote`: run the declared remote gate now, under the spend cap. The
|
||||
* gate gets PREGATE_TREE, PREGATE_FILES_FILE and PREGATE_SPEND_CAP_USD and
|
||||
* must print `PREGATE_REMOTE: tree=<tree> tests=<n> failed=<n> [spent=<usd>]`;
|
||||
* the obligation clears only when that tree is the tested tree, tests > 0,
|
||||
* failed = 0 and the gate exited 0. Anything else leaves it uncleared and
|
||||
* the check incomplete.
|
||||
*/
|
||||
async function runRemote(ctx: PregateContext, sel: Selection, base: CheckResult): Promise<CheckResult> {
|
||||
const lines = base.lines ?? [];
|
||||
if (!ctx.config.remote_gate) return { ...base, status: 'incomplete', code: 'PREGATE_INCOMPLETE', detail: `--remote runs only a declared remote gate; ${ctx.configLabel} has no remote_gate`, fix: 'declare remote_gate (a command that prints PREGATE_REMOTE: tree=… tests=… failed=…) in .gstack/pregate.json on the base branch, or run the tests locally without --remote' };
|
||||
const obligation = remoteObligation(ctx, sel);
|
||||
const cap = ctx.remote!.spendCapUsd;
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-pregate-remote-'));
|
||||
const log = path.join(dir, 'remote.log');
|
||||
const filesFile = path.join(dir, 'files.txt');
|
||||
fs.writeFileSync(log, '');
|
||||
fs.writeFileSync(filesFile, sel.localTests.join('\n') + '\n');
|
||||
const r = await supervise({ command: obligation.command!, cwd: ctx.repoRoot, env: { ...ctx.env, PREGATE_TREE: ctx.wtree, PREGATE_FILES_FILE: filesFile, PREGATE_SPEND_CAP_USD: String(cap) }, log, deadline: Date.now() + ctx.timeoutMs, stallMs: ctx.stallTimeoutMs, label: `remote gate ${obligation.lane}`, progress: ctx.progress });
|
||||
const output = readRange(log, r.start, r.end);
|
||||
const reported = [...output.matchAll(/^PREGATE_REMOTE: tree=(\S+) tests=(\d+) failed=(\d+)(?: spent=(\d+(?:\.\d+)?))?\s*$/gm)].pop();
|
||||
const spent = reported?.[4];
|
||||
const spend = `spend: spent ${spent ? `$${spent}` : '$0'}, reserved $0, unknown ${spent ? '$0' : `$${cap} (no spent= reported; counted at the cap)`} (cap $${cap})`;
|
||||
ctx.progress(`PREGATE_REMOTE_SPEND: ${spend.slice('spend: '.length)}`);
|
||||
const runLines = [...lines, `remote gate: ${obligation.command}`, `log: ${log}`, spend, ...output.trim().split('\n').slice(-15).map(l => ` ${l}`)];
|
||||
const inputs = { ...base.inputs, command: sha256(obligation.command!) };
|
||||
const why = r.outcome !== 'exited' ? `the remote gate did not finish (${r.outcome === 'stalled' ? `no output for ${ctx.stallTimeoutMs / 1000} s` : r.outcome})`
|
||||
: !reported ? `the remote gate exited ${r.exit} without a PREGATE_REMOTE: line`
|
||||
: reported[1] !== ctx.wtree ? `the remote gate tested tree ${reported[1]!.slice(0, 12)}, not ${ctx.wtree.slice(0, 12)}`
|
||||
: Number(reported[2]) === 0 ? 'the remote gate reported 0 tests'
|
||||
: Number(reported[3]) > 0 ? `the remote gate reported ${reported[3]} failed of ${reported[2]}`
|
||||
: r.exit !== 0 ? `the remote gate exited ${r.exit}`
|
||||
: null;
|
||||
if (why) return { ...base, inputs, lines: runLines, status: 'incomplete', remote: [...sel.remote, obligation], code: r.outcome === 'stalled' ? 'PREGATE_STALLED' : 'PREGATE_INCOMPLETE', detail: `${why}; requires-remote ${obligation.lane} stays uncleared`, fix: `read ${log}; the obligation clears only on PREGATE_REMOTE: tree=${ctx.wtree.slice(0, 12)}… tests>0 failed=0 with exit 0` };
|
||||
const cleared = { ...obligation, receipt: `remote-gate tree=${ctx.wtree.slice(0, 12)} tests=${reported![2]}`, cleared_by: `gstack-pregate --remote ${new Date().toISOString()}` };
|
||||
return { ...base, inputs, lines: runLines, status: 'requires-remote', remote: [...sel.remote, cleared], detail: `remote gate ${obligation.lane}: ${reported![2]} tests, 0 failed on tree ${ctx.wtree.slice(0, 12)}; ${spend}` };
|
||||
}
|
||||
|
||||
async function run(ctx: PregateContext): Promise<CheckResult> {
|
||||
const sel = selectLanes(ctx);
|
||||
ctx.explain.push(...sel.provenance);
|
||||
for (const [lane, set] of Object.entries(sel.laneSets)) ctx.explain.push(`lane ${lane}: ${set === '?' ? 'file set unknown' : `${set.length} files`}`);
|
||||
@@ -95,15 +172,21 @@ function run(ctx: PregateContext): CheckResult {
|
||||
return { ...base, status: 'fail', code: 'PREGATE_NO_LANE', detail: `${sel.noLane.length} touched file(s) with no test lane: ${sel.noLane.join(' ')}`, fix: 'add a test that names the file, or declare its lane (or []) under dependencies in .gstack/pregate.json' };
|
||||
}
|
||||
if (sel.tests.length === 0) return { ...base, detail: ctx.touched.length ? 'nothing touched selects a test' : 'no touched files' };
|
||||
if (ctx.remote) return runRemote(ctx, sel, base);
|
||||
const smallMachine = ctx.cores < SMALL_MACHINE_CORES && sel.localTests.length > 0 && (ctx.config.remote_gate || ctx.testBackend !== 'local');
|
||||
if (smallMachine) {
|
||||
const obligation = remoteObligation(ctx, sel);
|
||||
return { ...base, status: 'requires-remote', remote: [...sel.remote, obligation], lines: [...lines, `requires-remote ${obligation.lane}: ${ctx.cores} cores (under ${SMALL_MACHINE_CORES}); ${sel.localTests.length} local test file(s) recorded for the remote gate, not run here`, `run: ${obligation.command}`], detail: `${sel.localTests.length} local test file(s) recorded as requires-remote ${obligation.lane} (${ctx.cores} cores, under ${SMALL_MACHINE_CORES}); run it, then gstack-pregate clear --remote ${obligation.lane} --run-url <url>, or gstack-pregate --remote --spend-cap <usd> --allow-repo-commands` };
|
||||
}
|
||||
if (!ctx.allowRepoCommands) return { ...base, status: 'incomplete', code: 'REPO_COMMANDS_NOT_ALLOWED', detail: `${sel.localTests.length} local test file(s) selected but not run (pass --allow-repo-commands)`, fix: 'rerun with --allow-repo-commands (what /ship passes)' };
|
||||
if (sel.localTests.length === 0) return { ...base, status: sel.remote.length ? 'requires-remote' : 'pass', detail: `no local lane; ${sel.remote.length} requires-remote obligation(s)` };
|
||||
const runner = detectRunner(ctx, sel.localTests);
|
||||
if (!runner) return { ...base, status: 'incomplete', code: 'PREGATE_INCOMPLETE', detail: `no test runner detected for ${sel.localTests.length} selected file(s)`, fix: 'declare `runner` (with {files}) in .gstack/pregate.json' };
|
||||
const runtime = launchRuntime(ctx.repoRoot, ctx.env);
|
||||
const r = runLocal(ctx, runtime.env, runner, sel.localTests);
|
||||
const runLines = [...lines, `runtime: ${runtime.label}`, `ran: ${r.command.length > 200 ? `${r.command.slice(0, 200)}…` : r.command}`, `log: ${r.log}`, ...r.tail.map(l => ` ${l}`)];
|
||||
const r = await runLocal(ctx, runtime.env, runner, sel.localTests);
|
||||
const runLines = [...lines, `runtime: ${runtime.label}`, `progress: ${adapterFor(runner) ?? 'none (no progress adapter for this runner; serial files and heartbeats only)'}`, `ran: ${r.command.length > 200 ? `${r.command.slice(0, 200)}…` : r.command}`, `log: ${r.log}`, ...r.tail.map(l => ` ${l}`)];
|
||||
const withRun = { ...base, lines: runLines, inputs: { ...inputs, command: sha256(r.command), runtime: runtime.label } };
|
||||
if (r.exit === null) return { ...withRun, status: 'incomplete', code: 'PREGATE_INCOMPLETE', detail: `local lane timed out after ${ctx.timeoutMs} ms (${sel.localTests.length} files)`, fix: 'raise --timeout or split the selection; a timed-out lane never passes' };
|
||||
if (r.exit === null) return { ...withRun, status: 'incomplete', ...unfinished(ctx, r, sel.localTests.length) };
|
||||
if (r.exit !== 0) return { ...withRun, status: 'fail', code: 'PREGATE_LANE_FAILED', detail: `local lane exited ${r.exit} (${r.failed ?? '?'} failed of ${r.ran ?? '?'} ran; ${sel.localTests.length} files; pipefail)`, fix: `read ${r.log}, fix the in-branch failure, rerun gstack-pregate --stage tests` };
|
||||
if (r.ran === 0) return { ...withRun, status: 'fail', code: 'PREGATE_LANE_FAILED', detail: `ZERO-RUN: the runner reported 0 tests for ${sel.localTests.length} selected file(s)`, fix: 'a bad selector, a missing file or a skipped describe; exit 0 is not a pass' };
|
||||
const detail = `${r.ran ?? 'count unavailable'} tests ran in ${sel.localTests.length} file(s)${r.serial ? ` (${r.serial} serial)` : ''}, exit 0${sel.remote.length ? `; ${sel.remote.length} requires-remote obligation(s)` : ''}`;
|
||||
|
||||
+19
-3
@@ -22,6 +22,11 @@
|
||||
* the pre-push hook and `gstack-pregate sealed`, not by `run`)
|
||||
* price_file the call-plan data file `gstack-eval-plan price-gate` prices
|
||||
* a paid gate from (GSTACK_EVAL_CALLS v1; never executed)
|
||||
* stall_timeout_s seconds without test output before the lanes check kills
|
||||
* the runner's process group (default 300; 0 disables)
|
||||
* remote_gate the repo's remote test gate command: under 8 cores the test
|
||||
* tier is a requires-remote obligation naming it, and
|
||||
* `--remote` runs it (it prints `PREGATE_REMOTE: tree=… tests=… failed=…`)
|
||||
*/
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
@@ -45,14 +50,16 @@ export interface PregateConfig {
|
||||
guards: Guard[];
|
||||
serial_patterns: string[];
|
||||
price_file: string | null;
|
||||
stall_timeout_s: number | null;
|
||||
remote_gate: string | null;
|
||||
}
|
||||
export interface Guard { id: string; when: string[]; command: string; fix?: string; timeout_s?: number }
|
||||
export const DEFAULT_CONFIG: PregateConfig = { runner: null, dependencies: {}, selection: {}, warn: [], strays: { allow: [] }, guards: [], serial_patterns: DEFAULT_SERIAL_PATTERNS, price_file: null };
|
||||
export const DEFAULT_CONFIG: PregateConfig = { runner: null, dependencies: {}, selection: {}, warn: [], strays: { allow: [] }, guards: [], serial_patterns: DEFAULT_SERIAL_PATTERNS, price_file: null, stall_timeout_s: null, remote_gate: null };
|
||||
export interface ConfigError { code: ResultCodeName; path: string; message: string }
|
||||
|
||||
const MAX_LIST = 256;
|
||||
const MAX_COMMAND = 512;
|
||||
const KEYS = ['runner', 'dependencies', 'selection', 'warn', 'strays', 'guards', 'serial_patterns', 'price_file', 'sealed'];
|
||||
const KEYS = ['runner', 'dependencies', 'selection', 'warn', 'strays', 'guards', 'serial_patterns', 'price_file', 'sealed', 'stall_timeout_s', 'remote_gate'];
|
||||
const MAX_GUARDS = 32;
|
||||
const GUARD_KEYS = ['id', 'when', 'command', 'fix', 'timeout_s'];
|
||||
|
||||
@@ -104,6 +111,13 @@ export function validateConfig(raw: unknown, knownChecks: readonly string[] = []
|
||||
const why = typeof obj.price_file === 'string' ? containedRelative(obj.price_file) : 'must be a repo-relative path or null';
|
||||
if (why) errors.push({ code: 'PREGATE_CONFIG_INVALID', path: 'price_file', message: why });
|
||||
}
|
||||
if ('stall_timeout_s' in obj && obj.stall_timeout_s !== null && (!Number.isInteger(obj.stall_timeout_s) || (obj.stall_timeout_s as number) < 0 || (obj.stall_timeout_s as number) > 86_400)) {
|
||||
errors.push({ code: 'PREGATE_CONFIG_INVALID', path: 'stall_timeout_s', message: 'must be an integer number of seconds from 0 (disabled) to 86400, or null' });
|
||||
}
|
||||
if ('remote_gate' in obj && obj.remote_gate !== null) {
|
||||
const why = commandError(obj.remote_gate);
|
||||
if (why) errors.push({ code: 'PREGATE_CONFIG_INVALID', path: 'remote_gate', message: why });
|
||||
}
|
||||
if ('guards' in obj) guardErrors(obj.guards, errors);
|
||||
if ('sealed' in obj) for (const e of sealedErrors(obj.sealed)) errors.push({ code: 'PREGATE_CONFIG_INVALID', path: e.path, message: e.message });
|
||||
if ('serial_patterns' in obj) {
|
||||
@@ -162,7 +176,7 @@ export function parseConfig(text: string, knownChecks: readonly string[] = []):
|
||||
const errors = validateConfig(raw, knownChecks);
|
||||
if (errors.length) return { config: DEFAULT_CONFIG, errors };
|
||||
const o = raw as Partial<PregateConfig>;
|
||||
return { config: { $schema: o.$schema, runner: o.runner ?? null, dependencies: o.dependencies ?? {}, selection: o.selection ?? {}, warn: o.warn ?? [], strays: { allow: o.strays?.allow ?? [] }, guards: o.guards ?? [], serial_patterns: o.serial_patterns ?? DEFAULT_SERIAL_PATTERNS, price_file: o.price_file ?? null }, errors: [] };
|
||||
return { config: { $schema: o.$schema, runner: o.runner ?? null, dependencies: o.dependencies ?? {}, selection: o.selection ?? {}, warn: o.warn ?? [], strays: { allow: o.strays?.allow ?? [] }, guards: o.guards ?? [], serial_patterns: o.serial_patterns ?? DEFAULT_SERIAL_PATTERNS, price_file: o.price_file ?? null, stall_timeout_s: o.stall_timeout_s ?? null, remote_gate: o.remote_gate ?? null }, errors: [] };
|
||||
}
|
||||
|
||||
export type ConfigLoad = { config: PregateConfig; errors: ConfigError[]; present: boolean; label: string; kind: 'base' | 'head' | 'none' } | { unavailable: true; base: string; detail: string };
|
||||
@@ -214,5 +228,7 @@ export function explainConfig(): string {
|
||||
`serial_patterns: test-file globs the lanes check runs one process per file, apart from the unit files (default ${DEFAULT_SERIAL_PATTERNS.join(', ')})`,
|
||||
'sealed: { paths, transcripts, answer_markers, seed_patterns, local_paths, visibility }: sealed or eval output the pre-push hook and `gstack-pregate sealed` refuse to publish to a public destination (docs/pregate.md#sealed)',
|
||||
'price_file: repo-relative path of the GSTACK_EVAL_CALLS v1 call-plan file `gstack-eval-plan price-gate` prices a paid gate from; never executed (docs/eval-calls.md)',
|
||||
'stall_timeout_s: seconds without test output before the lanes check kills the runner\'s process group and records it incomplete (default 300; 0 disables; --stall-timeout overrides)',
|
||||
'remote_gate: the repo\'s remote test gate command; under 8 cores the test tier becomes a requires-remote obligation naming it, and gstack-pregate --remote runs it (it must print PREGATE_REMOTE: tree=<tree> tests=<n> failed=<n>) (docs/pregate.md#remote-gate)',
|
||||
].join('\n') + '\n';
|
||||
}
|
||||
+26
-6
@@ -9,6 +9,8 @@
|
||||
*/
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import { createHash } from 'node:crypto';
|
||||
import * as os from 'node:os';
|
||||
import * as path from 'node:path';
|
||||
import { wtreeFingerprint, isRegistryUnavailable, loadRegistryFromBase, loadRegistryFromHead, type Registry } from '../regen';
|
||||
import { isConfigUnavailable, loadConfigFromBase, loadConfigFromHead, type ConfigError, type PregateConfig } from './config';
|
||||
import type { CheckResult, PregateContext, TouchedFile } from './types';
|
||||
@@ -18,6 +20,17 @@ import type { RegistryError } from '../regen';
|
||||
export interface BuildContextOptions {
|
||||
repoRoot: string; base: string; policyFrom: 'base' | 'head'; allowRepoCommands: boolean;
|
||||
timeoutMs?: number; env?: NodeJS.ProcessEnv; platform?: NodeJS.Platform; knownChecks?: readonly string[];
|
||||
/** `--stall-timeout`; default the config's `stall_timeout_s`, else 300 s. */
|
||||
stallTimeoutMs?: number; progress?: (line: string) => void; cores?: number; testBackend?: string; remote?: { spendCapUsd: number } | null;
|
||||
}
|
||||
|
||||
export const DEFAULT_STALL_TIMEOUT_S = 300;
|
||||
const GSTACK_CONFIG = path.resolve(import.meta.dir, '..', '..', 'bin', 'gstack-config');
|
||||
|
||||
/** `gstack-config get test_backend` (local | ubicloud); `local` when unreadable. */
|
||||
export function readTestBackend(env: NodeJS.ProcessEnv): string {
|
||||
const r = spawnSync(GSTACK_CONFIG, ['get', 'test_backend'], { encoding: 'utf8', timeout: 15_000, env, stdio: ['ignore', 'pipe', 'ignore'] });
|
||||
return (r.status === 0 && (r.stdout ?? '').trim()) || 'local';
|
||||
}
|
||||
export type ContextLoad = { ctx: PregateContext } | { refused: { code: 'POLICY_SOURCE_UNAVAILABLE' | 'PREGATE_CONFIG_INVALID' | 'REGEN_REGISTRY_INVALID'; message: string; errors?: Array<ConfigError | RegistryError> } };
|
||||
|
||||
@@ -66,11 +79,17 @@ export function buildContext(o: BuildContextOptions): ContextLoad {
|
||||
const wtree = wtreeFingerprint(o.repoRoot);
|
||||
if (!wtree) return { refused: { code: 'POLICY_SOURCE_UNAVAILABLE', message: 'cannot fingerprint the working tree (gstack-wtree failed)' } };
|
||||
const platform = o.platform ?? process.platform;
|
||||
const env = o.env ?? process.env;
|
||||
const config = configLoad.config as PregateConfig;
|
||||
return {
|
||||
ctx: {
|
||||
repoRoot: o.repoRoot, base, baseRef, baseSha: baseSha.out, mergeBase, head, wtree, touched: touchedFiles(o.repoRoot, mergeBase),
|
||||
config: configLoad.config as PregateConfig, configLabel: configLoad.label, registry: registryLoad.registry as Registry, registryLabel: registryLoad.source.label,
|
||||
lanes: discoverLanes(o.repoRoot, platform), platform, allowRepoCommands: o.allowRepoCommands, timeoutMs: o.timeoutMs ?? 600_000, env: o.env ?? process.env, explain: [],
|
||||
config, configLabel: configLoad.label, registry: registryLoad.registry as Registry, registryLabel: registryLoad.source.label,
|
||||
lanes: discoverLanes(o.repoRoot, platform), platform, allowRepoCommands: o.allowRepoCommands, timeoutMs: o.timeoutMs ?? 600_000,
|
||||
stallTimeoutMs: o.stallTimeoutMs ?? (config.stall_timeout_s ?? DEFAULT_STALL_TIMEOUT_S) * 1000,
|
||||
progress: o.progress ?? (line => process.stderr.write(`${line}\n`)),
|
||||
cores: o.cores ?? os.availableParallelism(), testBackend: o.testBackend ?? readTestBackend(env), policyFrom: o.policyFrom, remote: o.remote ?? null,
|
||||
env, explain: [],
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -84,13 +103,14 @@ export function baseInputs(ctx: PregateContext, extra: Record<string, string> =
|
||||
|
||||
/**
|
||||
* Run one check under the before/after fingerprint (bin/gstack-evidence:394):
|
||||
* a tree that changed while the check ran, a thrown error or a timeout is
|
||||
* `incomplete`. Wall time is measured here so every row carries `ms`.
|
||||
* a tree that changed while the check ran, a thrown error or rejected promise
|
||||
* or a timeout is `incomplete`. Wall time is measured here so every row
|
||||
* carries `ms`.
|
||||
*/
|
||||
export function withFingerprint(ctx: PregateContext, id: string, stage: CheckResult['stage'], fn: () => CheckResult): CheckResult {
|
||||
export async function withFingerprint(ctx: PregateContext, id: string, stage: CheckResult['stage'], fn: () => CheckResult | Promise<CheckResult>): Promise<CheckResult> {
|
||||
const started = Date.now();
|
||||
let result: CheckResult;
|
||||
try { result = fn(); }
|
||||
try { result = await fn(); }
|
||||
catch (e: any) {
|
||||
result = { id, stage, status: 'incomplete', detail: `threw: ${e?.message ?? e}`, code: 'PREGATE_INCOMPLETE', fix: 'read the error and rerun the check', inputs: baseInputs(ctx) };
|
||||
}
|
||||
|
||||
@@ -40,16 +40,18 @@ export function applyDowngrades(ctx: PregateContext, result: CheckResult, def: C
|
||||
return { ...result, status: 'warn', detail: `${result.detail} (downgraded to warn by ${ctx.configLabel})` };
|
||||
}
|
||||
|
||||
export function runPregate(ctx: PregateContext, o: RunOptions): PregateFile {
|
||||
export async function runPregate(ctx: PregateContext, o: RunOptions): Promise<PregateFile> {
|
||||
const selected = CHECKS.filter(c => (o.stage === 'all' || c.stage === o.stage) && (!o.only || o.only.includes(c.id)));
|
||||
const checks: CheckResult[] = [];
|
||||
const clocks: Record<Stage, number> = { preflight: 0, tests: 0 };
|
||||
for (const def of selected) {
|
||||
const started = Date.now();
|
||||
const result = applyDowngrades(ctx, withFingerprint(ctx, def.id, def.stage, () => def.run(ctx)), def);
|
||||
const result = applyDowngrades(ctx, await withFingerprint(ctx, def.id, def.stage, () => def.run(ctx)), def);
|
||||
clocks[def.stage] += Date.now() - started;
|
||||
checks.push({ ...result, stage: def.stage });
|
||||
}
|
||||
const tests = checks.filter(c => c.stage === 'tests');
|
||||
if (tests.length) ctx.progress(`PREGATE: tests ${verdictOf({ schema_version: 1, tree: ctx.wtree, checks: tests, requires_remote: tests.flatMap(c => c.remote ?? []) })} in ${(clocks.tests / 1000).toFixed(1)}s — ${tests.map(c => `${c.id}: ${c.detail ?? c.status}`).join('; ')}`);
|
||||
const remote: PregateRemote[] = checks.flatMap(c => c.remote ?? []).map(r => ({ ...r }));
|
||||
const file: PregateFile = {
|
||||
schema_version: 1, tree: ctx.wtree, head: ctx.head, base: `${ctx.baseRef}@${ctx.baseSha.slice(0, 12)}`, generated_at: new Date().toISOString(),
|
||||
@@ -82,7 +84,10 @@ export function renderTable(file: PregateFile, opts: { lines?: boolean } = { lin
|
||||
file.stage1_ms !== undefined ? `stage 1 (mechanical preflight): ${file.stage1_ms} ms` : null,
|
||||
file.stage2_ms !== undefined ? `stage 2 (test execution): ${file.stage2_ms} ms` : null,
|
||||
].filter(Boolean);
|
||||
const remote = (file.requires_remote ?? []).map(r => `requires-remote: ${r.lane} (${r.platform}) ${r.tests.length} test(s) — ${r.receipt ? `cleared by ${r.receipt}` : 'uncleared; clear with gstack-pregate clear --remote ' + r.lane + ' --run-url <url>'}`);
|
||||
const remote = (file.requires_remote ?? []).map(r => {
|
||||
const run = r.command ? `run ${r.command.length > 200 ? `${r.command.slice(0, 200)}… (full command in pregate.json)` : r.command}, then ` : '';
|
||||
return `requires-remote: ${r.lane} (${r.platform}) ${r.tests.length} test(s) — ${r.receipt ? `cleared by ${r.receipt}` : `uncleared; ${run}clear with gstack-pregate clear --remote ${r.lane} --run-url <url>`}`;
|
||||
});
|
||||
return [
|
||||
`stage check status time detail`,
|
||||
...rows, ...body, ...clocks, ...remote,
|
||||
|
||||
@@ -0,0 +1,182 @@
|
||||
/**
|
||||
* pregate/supervise — how a test-tier command runs (Lane C, GBRA-95): its
|
||||
* own process group, stdout and stderr appended to a log file (no pipes, so
|
||||
* a backgrounded grandchild holding the output can never keep the check
|
||||
* waiting), supervised asynchronously. A stall is no log growth for
|
||||
* `stallMs`; the deadline is the check's `--timeout`. Either kills the whole
|
||||
* process group (SIGTERM, then SIGKILL after a grace period) and the run is
|
||||
* incomplete. After a normal exit the group's leftover members are killed.
|
||||
*
|
||||
* Progress: a runner with a progress adapter (`bun test`: its `<file>:`
|
||||
* headers) reports `PREGATE_TEST: <n>/<total> <file> start|pass|fail <s>`
|
||||
* per file; a run of one file reports it from the launch itself for any
|
||||
* runner. Without either, only heartbeats: a line after each `heartbeatMs`
|
||||
* with no progress line.
|
||||
*
|
||||
* Platforms: POSIX signals the negative pid (the group `detached` creates
|
||||
* with setsid). Windows has no process groups here: the kill is
|
||||
* `taskkill /T /F` on the shell's tree, which misses a descendant that was
|
||||
* reparented or started detached.
|
||||
*/
|
||||
import { spawn, spawnSync } from 'node:child_process';
|
||||
import * as fs from 'node:fs';
|
||||
|
||||
export type ProgressAdapter = 'bun';
|
||||
export const PROGRESS_ADAPTERS: Record<ProgressAdapter, RegExp> = { bun: /(?:^|[\s;&|(])bun\s+test\b/ };
|
||||
|
||||
export interface Counter { done: number; total: number }
|
||||
|
||||
export interface SuperviseOptions {
|
||||
command: string;
|
||||
cwd: string;
|
||||
env: NodeJS.ProcessEnv;
|
||||
log: string;
|
||||
/** Absolute epoch ms when the run is killed as timed out. */
|
||||
deadline: number;
|
||||
/** No log growth for this long kills the group; 0 disables. */
|
||||
stallMs: number;
|
||||
heartbeatMs?: number;
|
||||
graceMs?: number;
|
||||
pollMs?: number;
|
||||
label: string;
|
||||
files?: readonly string[];
|
||||
adapter?: ProgressAdapter | null;
|
||||
counter?: Counter;
|
||||
progress: (line: string) => void;
|
||||
}
|
||||
|
||||
export interface Supervised {
|
||||
outcome: 'exited' | 'timeout' | 'stalled' | 'spawn-failed';
|
||||
exit: number | null;
|
||||
signal: string | null;
|
||||
/** Byte range of this run's output in the log. */
|
||||
start: number;
|
||||
end: number;
|
||||
lastFile: string | null;
|
||||
error?: string;
|
||||
ms: number;
|
||||
}
|
||||
|
||||
const ACTIVE = new Set<number>();
|
||||
|
||||
/** SIGKILL every supervised process group still running (the CLI's SIGINT/SIGTERM handler, before it exits). */
|
||||
export function cancelActive(): number {
|
||||
const pids = [...ACTIVE];
|
||||
for (const pid of pids) killGroup(pid, 'SIGKILL');
|
||||
ACTIVE.clear();
|
||||
return pids.length;
|
||||
}
|
||||
|
||||
export function killGroup(pid: number, signal: NodeJS.Signals): void {
|
||||
if (process.platform === 'win32') {
|
||||
spawnSync('taskkill', ['/pid', String(pid), '/T', '/F'], { timeout: 15_000, windowsHide: true });
|
||||
return;
|
||||
}
|
||||
try { process.kill(-pid, signal); } catch { /* group already gone */ }
|
||||
}
|
||||
|
||||
/** The adapter that reads per-file progress from this runner's output, or null when it has none. */
|
||||
export function adapterFor(command: string): ProgressAdapter | null {
|
||||
for (const [name, re] of Object.entries(PROGRESS_ADAPTERS)) if (re.test(command)) return name as ProgressAdapter;
|
||||
return null;
|
||||
}
|
||||
|
||||
/** The last `bytes` of the log range as text. */
|
||||
export function readRange(log: string, start: number, end: number, bytes = 4 * 1024 * 1024): string {
|
||||
const from = Math.max(start, end - bytes);
|
||||
if (end <= from) return '';
|
||||
const fd = fs.openSync(log, 'r');
|
||||
try { const buf = Buffer.alloc(end - from); fs.readSync(fd, buf, 0, buf.length, from); return buf.toString('utf8'); }
|
||||
finally { fs.closeSync(fd); }
|
||||
}
|
||||
|
||||
export function supervise(o: SuperviseOptions): Promise<Supervised> {
|
||||
const heartbeatMs = o.heartbeatMs ?? 60_000;
|
||||
const graceMs = o.graceMs ?? 5_000;
|
||||
const pollMs = o.pollMs ?? 250;
|
||||
const counter = o.counter ?? { done: 0, total: o.files?.length ?? 0 };
|
||||
const files = new Set(o.files ?? []);
|
||||
const single = o.files?.length === 1 ? o.files[0]! : null;
|
||||
const adapter = single ? null : (o.adapter ?? null);
|
||||
const started = Date.now();
|
||||
fs.appendFileSync(o.log, `$ ${o.command}\n`);
|
||||
const start = fs.statSync(o.log).size;
|
||||
let size = start;
|
||||
let readAt = start;
|
||||
let partial = '';
|
||||
let lastGrowth = started;
|
||||
let lastProgress = started;
|
||||
let current: { file: string; at: number; failed: boolean } | null = null;
|
||||
let lastFile: string | null = null;
|
||||
const emit = (line: string) => { lastProgress = Date.now(); o.progress(line); };
|
||||
const startFile = (file: string) => { current = { file, at: Date.now(), failed: false }; lastFile = file; emit(`PREGATE_TEST: ${counter.done + 1}/${counter.total} ${file} start`); };
|
||||
const finishFile = (failed?: boolean) => {
|
||||
if (!current) return;
|
||||
counter.done += 1;
|
||||
emit(`PREGATE_TEST: ${counter.done}/${counter.total} ${current.file} ${(failed ?? current.failed) ? 'fail' : 'pass'} ${((Date.now() - current.at) / 1000).toFixed(1)}`);
|
||||
current = null;
|
||||
};
|
||||
const readNew = () => {
|
||||
if (!adapter || size <= readAt) return;
|
||||
const text = partial + readRange(o.log, readAt, size, size - readAt);
|
||||
readAt = size;
|
||||
const lines = text.split('\n');
|
||||
partial = lines.pop() ?? '';
|
||||
for (const line of lines) {
|
||||
const header = /^(\S.*):$/.exec(line);
|
||||
if (header && files.has(header[1]!)) { finishFile(); startFile(header[1]!); continue; }
|
||||
if (/^\s*\d+\s+pass$/.test(line) || /^Ran \d+ tests? across/.test(line) || /^\s*\d+\s+tests? failed:?$/.test(line)) { finishFile(); continue; }
|
||||
if (current && /^\(fail\)\s/.test(line)) current.failed = true;
|
||||
}
|
||||
};
|
||||
|
||||
return new Promise(resolve => {
|
||||
let fd: number;
|
||||
try { fd = fs.openSync(o.log, 'a'); }
|
||||
catch (e: any) { resolve({ outcome: 'spawn-failed', exit: null, signal: null, start, end: start, lastFile: null, error: e.message, ms: 0 }); return; }
|
||||
let child: ReturnType<typeof spawn>;
|
||||
try {
|
||||
child = spawn('bash', ['-c', `set -o pipefail; ${o.command}`], { cwd: o.cwd, env: o.env, stdio: ['ignore', fd, fd], detached: process.platform !== 'win32', windowsHide: true });
|
||||
} catch (e: any) {
|
||||
fs.closeSync(fd);
|
||||
resolve({ outcome: 'spawn-failed', exit: null, signal: null, start, end: start, lastFile: null, error: e.message, ms: Date.now() - started });
|
||||
return;
|
||||
}
|
||||
fs.closeSync(fd);
|
||||
if (single) startFile(single);
|
||||
let outcome: Supervised['outcome'] | null = null;
|
||||
let settled = false;
|
||||
let escalate: ReturnType<typeof setTimeout> | null = null;
|
||||
const stop = (why: Supervised['outcome']) => {
|
||||
if (outcome || !child.pid) return;
|
||||
outcome = why;
|
||||
killGroup(child.pid, 'SIGTERM');
|
||||
escalate = setTimeout(() => killGroup(child.pid!, 'SIGKILL'), graceMs);
|
||||
};
|
||||
const finish = (exit: number | null, signal: string | null, error?: string) => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
clearInterval(timer);
|
||||
if (escalate) clearTimeout(escalate);
|
||||
if (child.pid) { ACTIVE.delete(child.pid); if (process.platform !== 'win32') killGroup(child.pid, 'SIGKILL'); }
|
||||
try { size = fs.statSync(o.log).size; } catch { /* log removed */ }
|
||||
readNew();
|
||||
const done: Supervised['outcome'] = error ? 'spawn-failed' : (outcome ?? 'exited');
|
||||
if (current) finishFile(done !== 'exited' || exit !== 0 ? true : undefined);
|
||||
resolve({ outcome: done, exit: done === 'exited' ? exit : null, signal, start, end: size, lastFile, error, ms: Date.now() - started });
|
||||
};
|
||||
const timer = setInterval(() => {
|
||||
const now = Date.now();
|
||||
let grown = size;
|
||||
try { grown = fs.statSync(o.log).size; } catch { /* log removed */ }
|
||||
if (grown > size) { size = grown; lastGrowth = now; readNew(); }
|
||||
if (outcome) return;
|
||||
if (now >= o.deadline) { stop('timeout'); return; }
|
||||
if (o.stallMs > 0 && now - lastGrowth >= o.stallMs) { stop('stalled'); return; }
|
||||
if (now - lastProgress >= heartbeatMs) emit(`PREGATE_HEARTBEAT: ${o.label} running ${Math.round((now - started) / 1000)}s; last file ${lastFile ?? 'none reported'}; log ${size - start} bytes, last growth ${Math.round((now - lastGrowth) / 1000)}s ago (${o.log})`);
|
||||
}, pollMs);
|
||||
if (child.pid) ACTIVE.add(child.pid);
|
||||
child.on('error', e => finish(null, null, e.message));
|
||||
child.on('exit', (code, signal) => finish(code, signal));
|
||||
});
|
||||
}
|
||||
+13
-1
@@ -39,6 +39,16 @@ export interface PregateContext {
|
||||
platform: NodeJS.Platform;
|
||||
allowRepoCommands: boolean;
|
||||
timeoutMs: number;
|
||||
/** No output for this long kills a test-tier process group (`--stall-timeout`, `stall_timeout_s`); 0 disables. */
|
||||
stallTimeoutMs: number;
|
||||
/** Live stderr lines: per-file progress, heartbeats, the stage result. */
|
||||
progress: (line: string) => void;
|
||||
/** CPU count and `gstack-config test_backend`: under 8 cores a configured remote gate records an obligation instead of running locally. */
|
||||
cores: number;
|
||||
testBackend: string;
|
||||
policyFrom: 'base' | 'head';
|
||||
/** `--remote`: run the declared remote gate now, under this spend cap. */
|
||||
remote: { spendCapUsd: number } | null;
|
||||
env: NodeJS.ProcessEnv;
|
||||
/** `--explain`: checks append provenance lines here. */
|
||||
explain: string[];
|
||||
@@ -55,6 +65,8 @@ export interface RemoteObligation {
|
||||
/** What clears it: a run URL or an evidence-bundle lane label, recorded by `gstack-pregate clear`. */
|
||||
receipt: string | null;
|
||||
cleared_by?: string;
|
||||
/** The exact command that discharges the obligation (a small machine's remote gate). */
|
||||
command?: string;
|
||||
}
|
||||
|
||||
export interface CheckResult extends PregateCheck {
|
||||
@@ -78,5 +90,5 @@ export interface CheckDef {
|
||||
downgradable: boolean;
|
||||
/** True when the check executes commands the repo declares (pin files or its test runner). */
|
||||
needsRepoCommands: boolean;
|
||||
run(ctx: PregateContext): CheckResult;
|
||||
run(ctx: PregateContext): CheckResult | Promise<CheckResult>;
|
||||
}
|
||||
@@ -389,6 +389,11 @@ export const RESULT_CODES = {
|
||||
summary: 'a check timed out, its tool is unavailable, or the tree changed while it ran; the result is incomplete, never pass',
|
||||
fix: 'install the named tool or raise --timeout, keep the tree still during the run, then rerun the check',
|
||||
},
|
||||
PREGATE_STALLED: {
|
||||
anchor: 'pregate-stalled',
|
||||
summary: 'a test command printed nothing for the stall timeout, so its process group was killed; the check is incomplete, never pass',
|
||||
fix: 'read the lane log for the last file started, fix the hang, or raise `--stall-timeout` (`stall_timeout_s`; 0 disables) for a test that is quiet for longer, then rerun `gstack-pregate --stage tests`',
|
||||
},
|
||||
PREGATE_STALE: {
|
||||
anchor: 'pregate-stale',
|
||||
summary: 'pregate.json was recorded against a different tree than the one being published',
|
||||
|
||||
@@ -0,0 +1,148 @@
|
||||
/**
|
||||
* Lane C4 (GBRA-95): on a machine under 8 cores, a remote test gate the base
|
||||
* branch declares (`remote_gate`) or a remote `test_backend` records the
|
||||
* local test tier as a requires-remote obligation with the exact command,
|
||||
* instead of running thousands of files locally. `--remote` runs the
|
||||
* declared gate now under a spend cap; the obligation clears only on the
|
||||
* gate's `PREGATE_REMOTE: tree=<tested tree> tests=<n> failed=0` line.
|
||||
*/
|
||||
import { afterEach, describe, expect, test } from 'bun:test';
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import { EXIT } from '../lib/headless-artifacts';
|
||||
import { buildContext } from '../lib/pregate/context';
|
||||
import { runPregate } from '../lib/pregate/index';
|
||||
import { BASE_PREGATE, cleanup, commit, makeFixtureRepo, runBin, type FixtureRepo } from './helpers/pregate-fixture';
|
||||
|
||||
const repos: FixtureRepo[] = [];
|
||||
afterEach(() => { while (repos.length) cleanup(repos.pop()!); });
|
||||
|
||||
const GATE = 'node scripts/remote-gate.js';
|
||||
/** A fake remote gate: reports the tree it was handed (or GATE_TREE), GATE_TESTS / GATE_FAILED, and records its env. */
|
||||
const GATE_SCRIPT = [
|
||||
"const fs = require('node:fs');",
|
||||
"fs.writeFileSync('../gate-env.json', JSON.stringify({ tree: process.env.PREGATE_TREE, cap: process.env.PREGATE_SPEND_CAP_USD, files: fs.readFileSync(process.env.PREGATE_FILES_FILE, 'utf8') }));",
|
||||
"if (process.env.GATE_SILENT) process.exit(0);",
|
||||
"const spent = process.env.GATE_SPENT ? ` spent=${process.env.GATE_SPENT}` : '';",
|
||||
"console.log(`PREGATE_REMOTE: tree=${process.env.GATE_TREE || process.env.PREGATE_TREE} tests=${process.env.GATE_TESTS ?? 3} failed=${process.env.GATE_FAILED ?? 0}${spent}`);",
|
||||
].join('\n') + '\n';
|
||||
|
||||
function gated(pregate: Record<string, unknown> = { ...BASE_PREGATE, remote_gate: GATE }): FixtureRepo {
|
||||
const repo = makeFixtureRepo({ pregate });
|
||||
repos.push(repo);
|
||||
repo.git('checkout', '-q', 'main');
|
||||
commit(repo, { 'scripts/remote-gate.js': GATE_SCRIPT }, 'remote gate');
|
||||
repo.git('push', '-q', 'origin', 'main');
|
||||
repo.git('checkout', '-q', 'feature');
|
||||
repo.git('merge', '-q', '--ff-only', 'main');
|
||||
commit(repo, { 'lib/util.js': "exports.add = (a, b) => a + b + 0;\n" });
|
||||
return repo;
|
||||
}
|
||||
|
||||
async function lanesOn(repo: FixtureRepo, o: { cores: number; testBackend: string }) {
|
||||
const lines: string[] = [];
|
||||
const loaded = buildContext({ repoRoot: repo.work, base: 'main', policyFrom: 'base', allowRepoCommands: true, env: { ...process.env, ...repo.env }, progress: l => lines.push(l), ...o });
|
||||
if (!('ctx' in loaded)) throw new Error('fixture context refused');
|
||||
const file = await runPregate(loaded.ctx, { stage: 'tests' });
|
||||
return { file, lanes: file.checks.find(c => c.id === 'lanes')!, lines };
|
||||
}
|
||||
|
||||
describe('small machines record the test tier for the remote gate (C4)', () => {
|
||||
test('under 8 cores a declared remote_gate becomes a requires-remote obligation naming the command; nothing runs locally', async () => {
|
||||
const repo = gated();
|
||||
const { file, lanes, lines } = await lanesOn(repo, { cores: 4, testBackend: 'local' });
|
||||
expect(lanes.status).toBe('requires-remote');
|
||||
expect(lanes.detail).toContain('recorded as requires-remote pregate.json/remote_gate (4 cores, under 8)');
|
||||
expect(file.requires_remote).toEqual([expect.objectContaining({ lane: 'pregate.json/remote_gate', platform: 'remote', command: GATE, tests: ['test/util.test.js'], receipt: null })]);
|
||||
expect(file.verdict).toBe('requires-remote (1 uncleared)');
|
||||
expect(fs.existsSync(path.join(repo.root, 'gate-env.json'))).toBe(false);
|
||||
expect(lines.some(l => l.startsWith('PREGATE_TEST:'))).toBe(false);
|
||||
expect(lines.at(-1)).toMatch(/^PREGATE: tests requires-remote \(1 uncleared\) in /);
|
||||
});
|
||||
|
||||
test('a remote test_backend without remote_gate names gstack\'s Ubicloud runner with the selection; 8 cores or more, or local with no gate, runs locally', async () => {
|
||||
const backend = gated(BASE_PREGATE);
|
||||
const viaBackend = await lanesOn(backend, { cores: 4, testBackend: 'ubicloud' });
|
||||
expect(viaBackend.lanes.status).toBe('requires-remote');
|
||||
const obligation = viaBackend.file.requires_remote![0]!;
|
||||
expect(obligation.lane).toBe('test_backend/ubicloud');
|
||||
expect(obligation.command).toMatch(/scripts\/ubicloud\/ubi-runner\.sh run --src '.*\/work' -- 'node --test '\\''test\/util\.test\.js'\\'''$/);
|
||||
expect((await lanesOn(backend, { cores: 4, testBackend: 'local' })).lanes.status).toBe('pass');
|
||||
expect((await lanesOn(gated(), { cores: 16, testBackend: 'local' })).lanes.status).toBe('pass');
|
||||
});
|
||||
|
||||
test('`clear` discharges the recorded obligation like any requires-remote lane', () => {
|
||||
const repo = gated();
|
||||
const fileAt = path.join(repo.work, '.gstack/tmp/pregate.json');
|
||||
return lanesOn(repo, { cores: 4, testBackend: 'local' }).then(({ file }) => {
|
||||
fs.mkdirSync(path.dirname(fileAt), { recursive: true });
|
||||
fs.writeFileSync(fileAt, JSON.stringify(file));
|
||||
const cleared = runBin(repo, 'gstack-pregate', ['clear', '--remote', 'pregate.json/remote_gate', '--run-url', 'https://ci.example/run/1']);
|
||||
expect(cleared.status).toBe(EXIT.ok);
|
||||
expect(cleared.out).toContain('PREGATE_CLEARED: pregate.json/remote_gate receipt=https://ci.example/run/1 verdict=requires-remote cleared');
|
||||
});
|
||||
});
|
||||
|
||||
test('config: stall_timeout_s and remote_gate are typed', async () => {
|
||||
const { validateConfig } = await import('../lib/pregate/config');
|
||||
const codes = (c: unknown) => validateConfig(c).map(e => `${e.code}:${e.path}`);
|
||||
expect(codes({ stall_timeout_s: 0, remote_gate: GATE })).toEqual([]);
|
||||
expect(codes({ stall_timeout_s: -1 })).toEqual(['PREGATE_CONFIG_INVALID:stall_timeout_s']);
|
||||
expect(codes({ stall_timeout_s: 1.5 })).toEqual(['PREGATE_CONFIG_INVALID:stall_timeout_s']);
|
||||
expect(codes({ remote_gate: 'a\nb' })).toEqual(['PREGATE_CONFIG_INVALID:remote_gate']);
|
||||
});
|
||||
});
|
||||
|
||||
describe('gstack-pregate --remote (C-c)', () => {
|
||||
const remote = (repo: FixtureRepo, args: string[], env: Record<string, string> = {}) => runBin(repo, 'gstack-pregate', ['--base', 'main', '--stage', 'tests', ...args], env);
|
||||
const lanes = (repo: FixtureRepo) => JSON.parse(fs.readFileSync(path.join(repo.work, '.gstack/tmp/pregate.json'), 'utf8')).checks.find((c: any) => c.id === 'lanes');
|
||||
const pregateFile = (repo: FixtureRepo) => JSON.parse(fs.readFileSync(path.join(repo.work, '.gstack/tmp/pregate.json'), 'utf8'));
|
||||
|
||||
test('needs --allow-repo-commands, the base policy and a spend cap', () => {
|
||||
const repo = gated();
|
||||
expect(remote(repo, ['--remote', '--spend-cap', '5']).status).toBe(EXIT.refused);
|
||||
expect(remote(repo, ['--remote', '--spend-cap', '5', '--allow-repo-commands', '--policy-from', 'head']).status).toBe(EXIT.refused);
|
||||
const noCap = remote(repo, ['--remote', '--allow-repo-commands']);
|
||||
expect(noCap.status).toBe(EXIT.usage);
|
||||
expect(noCap.err).toContain('pass --spend-cap <usd> (or set GSTACK_SPEND_CAP_USD)');
|
||||
expect(remote(repo, ['--spend-cap', '5', '--allow-repo-commands']).status).toBe(EXIT.usage);
|
||||
expect(fs.existsSync(path.join(repo.root, 'gate-env.json'))).toBe(false);
|
||||
});
|
||||
|
||||
test('a gate reporting the tested tree, tests > 0 and failed = 0 clears the obligation and reports its spend', () => {
|
||||
const repo = gated();
|
||||
const r = remote(repo, ['--remote', '--allow-repo-commands'], { GSTACK_SPEND_CAP_USD: '4', GATE_SPENT: '0.37' });
|
||||
expect(r.status).toBe(EXIT.ok);
|
||||
const file = pregateFile(repo);
|
||||
expect(file.verdict).toBe('requires-remote cleared');
|
||||
expect(file.requires_remote[0]).toMatchObject({ lane: 'pregate.json/remote_gate', receipt: `remote-gate tree=${file.tree.slice(0, 12)} tests=3` });
|
||||
expect(lanes(repo).detail).toContain('spend: spent $0.37, reserved $0, unknown $0 (cap $4)');
|
||||
expect(r.err).toContain('PREGATE_REMOTE_SPEND: spent $0.37, reserved $0, unknown $0 (cap $4)');
|
||||
const env = JSON.parse(fs.readFileSync(path.join(repo.root, 'gate-env.json'), 'utf8'));
|
||||
expect(env).toEqual({ tree: file.tree, cap: '4', files: 'test/util.test.js\n' });
|
||||
expect(runBin(repo, 'gstack-pregate', ['verify']).status).toBe(EXIT.ok);
|
||||
});
|
||||
|
||||
test('a wrong tree, zero tests, a failure, a missing line or a non-zero exit leaves the obligation uncleared and the check incomplete; unreported spend counts at the cap', () => {
|
||||
const repo = gated();
|
||||
const cases: Array<[Record<string, string>, string]> = [
|
||||
[{ GATE_TREE: 'deadbeef'.repeat(8) }, 'the remote gate tested tree deadbeefdead'],
|
||||
[{ GATE_TESTS: '0' }, 'the remote gate reported 0 tests'],
|
||||
[{ GATE_FAILED: '1' }, 'the remote gate reported 1 failed of 3'],
|
||||
[{ GATE_SILENT: '1' }, 'the remote gate exited 0 without a PREGATE_REMOTE: line'],
|
||||
];
|
||||
for (const [env, why] of cases) {
|
||||
const r = remote(repo, ['--remote', '--allow-repo-commands', '--spend-cap', '2'], env);
|
||||
expect(r.status).toBe(EXIT.fail);
|
||||
const row = lanes(repo);
|
||||
expect(row.status).toBe('incomplete');
|
||||
expect(row.detail).toContain(why);
|
||||
expect(row.detail).toContain('requires-remote pregate.json/remote_gate stays uncleared');
|
||||
expect(pregateFile(repo).requires_remote[0].receipt).toBeNull();
|
||||
}
|
||||
expect(lanes(repo).lines).toContain('spend: spent $0, reserved $0, unknown $2 (no spent= reported; counted at the cap) (cap $2)');
|
||||
const backendOnly = gated(BASE_PREGATE);
|
||||
expect(remote(backendOnly, ['--remote', '--allow-repo-commands', '--spend-cap', '2']).status).toBe(EXIT.fail);
|
||||
expect(lanes(backendOnly).detail).toContain('--remote runs only a declared remote gate');
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,222 @@
|
||||
/**
|
||||
* Lane C (GBRA-95: `gstack-pregate run` sat silent for 90 minutes in its test
|
||||
* stage on gbrain). The test tier now runs through lib/pregate/supervise.ts:
|
||||
* its own process group, output to a log file instead of pipes, live
|
||||
* progress on stderr, a stall timeout on log growth and a per-check deadline.
|
||||
* Fixtures: a test that blocks forever, a child that backgrounds and holds
|
||||
* stdout, a command that ignores SIGTERM, a quiet file and a printing one.
|
||||
*/
|
||||
import { afterEach, describe, expect, test } from 'bun:test';
|
||||
import { spawn } from 'node:child_process';
|
||||
import * as fs from 'node:fs';
|
||||
import * as os from 'node:os';
|
||||
import * as path from 'node:path';
|
||||
import { EXIT } from '../lib/headless-artifacts';
|
||||
import { buildContext, withFingerprint } from '../lib/pregate/context';
|
||||
import { adapterFor, supervise, type SuperviseOptions } from '../lib/pregate/supervise';
|
||||
import { BASE_PREGATE, BIN, cleanup, commit, makeFixtureRepo, runBin, type FixtureRepo } from './helpers/pregate-fixture';
|
||||
|
||||
const dirs: string[] = [];
|
||||
const repos: FixtureRepo[] = [];
|
||||
afterEach(() => { while (dirs.length) fs.rmSync(dirs.pop()!, { recursive: true, force: true }); while (repos.length) cleanup(repos.pop()!); });
|
||||
|
||||
function scratch(): string { const d = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-supervise-')); dirs.push(d); return d; }
|
||||
function alive(pid: number): boolean { try { process.kill(pid, 0); return true; } catch { return false; } }
|
||||
async function goneSoon(pid: number, ms = 3000): Promise<boolean> {
|
||||
const until = Date.now() + ms;
|
||||
while (Date.now() < until) { if (!alive(pid)) return true; await Bun.sleep(50); }
|
||||
return !alive(pid);
|
||||
}
|
||||
async function pidFrom(file: string, ms = 5000): Promise<number> {
|
||||
const until = Date.now() + ms;
|
||||
while (Date.now() < until) { const t = fs.existsSync(file) ? fs.readFileSync(file, 'utf8').trim() : ''; if (t) return Number(t); await Bun.sleep(25); }
|
||||
throw new Error(`no pid in ${file}`);
|
||||
}
|
||||
|
||||
function run(dir: string, command: string, o: Partial<SuperviseOptions> = {}) {
|
||||
const lines: string[] = [];
|
||||
const log = path.join(dir, 'run.log');
|
||||
fs.writeFileSync(log, '');
|
||||
const started = Date.now();
|
||||
return supervise({ command, cwd: dir, env: process.env, log, deadline: Date.now() + 30_000, stallMs: 0, graceMs: 300, pollMs: 50, label: 'fixture', progress: l => lines.push(l), ...o })
|
||||
.then(r => ({ ...r, lines, wall: Date.now() - started, log: fs.readFileSync(log, 'utf8') }));
|
||||
}
|
||||
|
||||
describe('supervise: process group, log file, stall and deadline', () => {
|
||||
test('a backgrounded child holding stdout does not keep the run waiting, and is killed with the group after the exit', async () => {
|
||||
const dir = scratch();
|
||||
const r = await run(dir, `sleep 1000 & echo $! > ${dir}/bg.pid; echo done`);
|
||||
expect(r.outcome).toBe('exited');
|
||||
expect(r.exit).toBe(0);
|
||||
expect(r.wall).toBeLessThan(5000);
|
||||
expect(r.log).toContain('done');
|
||||
expect(await goneSoon(await pidFrom(`${dir}/bg.pid`))).toBe(true);
|
||||
});
|
||||
|
||||
test('a command that prints nothing for the stall timeout is killed as stalled; a long command that keeps printing is not', async () => {
|
||||
const dir = scratch();
|
||||
const quiet = await run(dir, `echo $$ > ${dir}/quiet.pid; echo starting; sleep 1000`, { stallMs: 700 });
|
||||
expect(quiet.outcome).toBe('stalled');
|
||||
expect(quiet.exit).toBeNull();
|
||||
expect(quiet.wall).toBeLessThan(5000);
|
||||
expect(await goneSoon(await pidFrom(`${dir}/quiet.pid`))).toBe(true);
|
||||
const printing = await run(dir, 'for i in 1 2 3 4 5 6 7 8; do echo tick $i; sleep 0.25; done', { stallMs: 700 });
|
||||
expect(printing.outcome).toBe('exited');
|
||||
expect(printing.exit).toBe(0);
|
||||
expect(printing.wall).toBeGreaterThan(1500);
|
||||
});
|
||||
|
||||
test('a group that ignores SIGTERM is SIGKILLed after the grace period', async () => {
|
||||
const dir = scratch();
|
||||
const r = await run(dir, `trap '' TERM; echo $$ > ${dir}/stubborn.pid; while :; do sleep 0.1; done`, { stallMs: 500, graceMs: 400 });
|
||||
expect(r.outcome).toBe('stalled');
|
||||
expect(r.wall).toBeLessThan(5000);
|
||||
expect(await goneSoon(await pidFrom(`${dir}/stubborn.pid`))).toBe(true);
|
||||
});
|
||||
|
||||
test('the deadline kills the group as timeout; a launch that cannot start is spawn-failed, never a pass', async () => {
|
||||
const dir = scratch();
|
||||
const slow = await run(dir, 'while :; do echo busy; sleep 0.1; done', { deadline: Date.now() + 600, stallMs: 0 });
|
||||
expect(slow.outcome).toBe('timeout');
|
||||
expect(slow.exit).toBeNull();
|
||||
const missing = await run(dir, 'true', { cwd: path.join(dir, 'does-not-exist') });
|
||||
expect(missing.outcome).toBe('spawn-failed');
|
||||
expect(missing.exit).toBeNull();
|
||||
});
|
||||
|
||||
test('heartbeats arrive after each quiet interval without progress lines', async () => {
|
||||
const dir = scratch();
|
||||
const r = await run(dir, 'sleep 1.2; echo late', { heartbeatMs: 300 });
|
||||
expect(r.outcome).toBe('exited');
|
||||
expect(r.lines.filter(l => l.startsWith('PREGATE_HEARTBEAT: fixture running ')).length).toBeGreaterThanOrEqual(2);
|
||||
expect(r.lines.some(l => l.startsWith('PREGATE_TEST:'))).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('supervise: per-file progress adapters', () => {
|
||||
const bunTest = (body: string) => `import { test, expect } from 'bun:test';\n${body}\n`;
|
||||
|
||||
test('bun test has an adapter: start and pass|fail per file from its headers; a runner without one reports nothing per file', async () => {
|
||||
expect(adapterFor('bun test {files}')).toBe('bun');
|
||||
expect(adapterFor('bun test --timeout=60000 {files}')).toBe('bun');
|
||||
expect(adapterFor('node --test {files}')).toBeNull();
|
||||
expect(adapterFor('npx vitest run {files}')).toBeNull();
|
||||
const dir = scratch();
|
||||
fs.writeFileSync(path.join(dir, 'a.test.ts'), bunTest("test('a', () => expect(1).toBe(1));"));
|
||||
fs.writeFileSync(path.join(dir, 'b.test.ts'), bunTest("test('b', () => expect(1).toBe(2));"));
|
||||
const files = ['a.test.ts', 'b.test.ts'];
|
||||
const r = await run(dir, `${process.execPath} test ${files.join(' ')}`, { files, adapter: 'bun' });
|
||||
expect(r.exit).toBe(1);
|
||||
const events = r.lines.filter(l => l.startsWith('PREGATE_TEST:')).map(l => l.replace(/ \d+\.\d$/, ''));
|
||||
expect(events).toEqual(['PREGATE_TEST: 1/2 a.test.ts start', 'PREGATE_TEST: 1/2 a.test.ts pass', 'PREGATE_TEST: 2/2 b.test.ts start', 'PREGATE_TEST: 2/2 b.test.ts fail']);
|
||||
const unsupported = await run(dir, 'echo "test a.test.ts ok"; echo "test b.test.ts ok"', { files, adapter: null });
|
||||
expect(unsupported.lines.some(l => l.startsWith('PREGATE_TEST:'))).toBe(false);
|
||||
});
|
||||
|
||||
test('a one-file run reports its file from the launch for any runner; a bun test that blocks forever stalls naming that file', async () => {
|
||||
const dir = scratch();
|
||||
const one = await run(dir, 'echo ok', { files: ['test/x.test.js'], adapter: null, counter: { done: 4, total: 9 } });
|
||||
expect(one.lines.map(l => l.replace(/ \d+\.\d$/, ''))).toEqual(['PREGATE_TEST: 5/9 test/x.test.js start', 'PREGATE_TEST: 5/9 test/x.test.js pass']);
|
||||
fs.writeFileSync(path.join(dir, 'ok.test.ts'), bunTest("test('ok', () => expect(1).toBe(1));"));
|
||||
fs.writeFileSync(path.join(dir, 'hang.test.ts'), bunTest("test('blocks', () => { Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0); });"));
|
||||
const files = ['ok.test.ts', 'hang.test.ts'];
|
||||
const hung = await run(dir, `${process.execPath} test ${files.join(' ')}`, { files, adapter: 'bun', stallMs: 1500 });
|
||||
expect(hung.outcome).toBe('stalled');
|
||||
expect(hung.lastFile).toBe('hang.test.ts');
|
||||
expect(hung.lines.some(l => /^PREGATE_TEST: [12]\/2 hang\.test\.ts start$/.test(l))).toBe(true);
|
||||
expect(hung.lines.some(l => /^PREGATE_TEST: [12]\/2 hang\.test\.ts fail /.test(l))).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('withFingerprint over async checks', () => {
|
||||
test('a rejected check promise is incomplete, never pass', async () => {
|
||||
const repo = makeFixtureRepo(); repos.push(repo);
|
||||
const loaded = buildContext({ repoRoot: repo.work, base: 'main', policyFrom: 'base', allowRepoCommands: false, env: { ...process.env, ...repo.env }, testBackend: 'local', cores: 16 });
|
||||
if (!('ctx' in loaded)) throw new Error('fixture context refused');
|
||||
const r = await withFingerprint(loaded.ctx, 'lanes', 'tests', async () => { throw new Error('runner vanished'); });
|
||||
expect(r).toMatchObject({ status: 'incomplete', code: 'PREGATE_INCOMPLETE', detail: 'threw: runner vanished' });
|
||||
});
|
||||
});
|
||||
|
||||
describe('gstack-pregate: the test stage never hangs silently', () => {
|
||||
const NODE_HANG = "const test = require('node:test');\ntest('blocks', () => { require('node:fs').writeFileSync(require('node:path').join(__dirname, '..', '..', 'hang.pid'), String(process.pid)); Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0); });\n";
|
||||
const NODE_BACKGROUND = "const test = require('node:test');\nconst { spawn } = require('node:child_process');\ntest('backgrounds a child holding stdout', () => { const c = spawn('sleep', ['1000'], { stdio: 'inherit' }); require('node:fs').writeFileSync(require('node:path').join(__dirname, '..', '..', 'bg.pid'), String(c.pid)); c.unref(); });\n";
|
||||
const pregate = (repo: FixtureRepo, args: string[]) => runBin(repo, 'gstack-pregate', ['--base', 'main', '--allow-repo-commands', '--stage', 'tests', ...args]);
|
||||
const lanes = (repo: FixtureRepo) => JSON.parse(fs.readFileSync(path.join(repo.work, '.gstack/tmp/pregate.json'), 'utf8')).checks.find((c: any) => c.id === 'lanes');
|
||||
|
||||
test('a test that blocks forever is killed at the stall timeout, named, and recorded incomplete; --json stdout stays one JSON document', () => {
|
||||
const repo = makeFixtureRepo(); repos.push(repo);
|
||||
commit(repo, { 'test/hang.test.js': NODE_HANG });
|
||||
const started = Date.now();
|
||||
const r = pregate(repo, ['--stall-timeout', '2', '--json']);
|
||||
expect(Date.now() - started).toBeLessThan(60_000);
|
||||
expect(r.status).toBe(EXIT.fail);
|
||||
const doc = JSON.parse(r.out);
|
||||
expect(doc.ok).toBe(false);
|
||||
const row = doc.checks.find((c: any) => c.id === 'lanes');
|
||||
expect(row).toMatchObject({ status: 'incomplete', code: 'PREGATE_STALLED' });
|
||||
expect(row.detail).toContain('last file started: test/hang.test.js');
|
||||
expect(r.err).toContain('PREGATE_TEST: 1/1 test/hang.test.js start');
|
||||
expect(r.err).toMatch(/^PREGATE: tests incomplete in \d+\.\ds — lanes: no test output for 2 s/m);
|
||||
expect(alive(Number(fs.readFileSync(path.join(repo.root, 'hang.pid'), 'utf8')))).toBe(false);
|
||||
}, 90_000);
|
||||
|
||||
test('the base branch stall_timeout_s applies without the flag', () => {
|
||||
const repo = makeFixtureRepo({ pregate: { ...BASE_PREGATE, stall_timeout_s: 2 } }); repos.push(repo);
|
||||
commit(repo, { 'test/hang.test.js': NODE_HANG });
|
||||
const r = pregate(repo, []);
|
||||
expect(r.status).toBe(EXIT.fail);
|
||||
expect(lanes(repo)).toMatchObject({ status: 'incomplete', code: 'PREGATE_STALLED' });
|
||||
expect(r.out).toMatch(/^tests {6}lanes {5}incomplete .*no test output for 2 s.*\(PREGATE_STALLED\)$/m);
|
||||
expect(r.err).toMatch(/^gstack-pregate: lanes stalled — .*; fix: .* \(PREGATE_STALLED; https:\/\/github\.com\/garrytan\/gstack\/blob\/main\/docs\/troubleshooting\.md#pregate-stalled\)$/m);
|
||||
}, 90_000);
|
||||
|
||||
test('a test whose child backgrounds and holds stdout passes promptly instead of waiting out --timeout', () => {
|
||||
const repo = makeFixtureRepo({ pregate: { ...BASE_PREGATE, runner: 'node {files}' } }); repos.push(repo);
|
||||
commit(repo, { 'test/bg.test.js': NODE_BACKGROUND });
|
||||
const started = Date.now();
|
||||
const r = pregate(repo, ['--timeout', '90']);
|
||||
expect(Date.now() - started).toBeLessThan(45_000);
|
||||
expect(r.status).toBe(EXIT.ok);
|
||||
expect(lanes(repo).status).toBe('pass');
|
||||
expect(alive(Number(fs.readFileSync(path.join(repo.root, 'bg.pid'), 'utf8')))).toBe(false);
|
||||
}, 90_000);
|
||||
|
||||
test('--timeout bounds the whole check: serial files past the deadline are not started and the lane is incomplete', () => {
|
||||
const repo = makeFixtureRepo(); repos.push(repo);
|
||||
const slow = "const test = require('node:test');\ntest('slow', async () => { await new Promise(r => setTimeout(r, 2500)); });\n";
|
||||
commit(repo, { 'test/a.serial.test.js': slow, 'test/b.serial.test.js': slow, 'test/c.serial.test.js': slow });
|
||||
const r = pregate(repo, ['--timeout', '4']);
|
||||
expect(r.status).toBe(EXIT.fail);
|
||||
const row = lanes(repo);
|
||||
expect(row).toMatchObject({ status: 'incomplete', code: 'PREGATE_INCOMPLETE' });
|
||||
expect(row.detail).toMatch(/local lane timed out after 4000 ms \(3 files; 1 of 3 file\(s\) not started\)/);
|
||||
}, 90_000);
|
||||
|
||||
test('a late tree mutation by a test\'s child is caught while the check runs, and a leftover child cannot mutate the tree after it', async () => {
|
||||
const during = makeFixtureRepo(); repos.push(during);
|
||||
commit(during, { 'test/mutate.test.js': "const test = require('node:test');\nconst { spawn } = require('node:child_process');\ntest('mutates soon', async () => { spawn('sh', ['-c', 'sleep 0.3; echo changed >> README.md'], { stdio: 'ignore' }); await new Promise(r => setTimeout(r, 1500)); });\n" });
|
||||
expect(pregate(during, []).status).toBe(EXIT.fail);
|
||||
expect(lanes(during).detail).toContain('working tree changed during the check');
|
||||
const after = makeFixtureRepo(); repos.push(after);
|
||||
commit(after, { 'test/leftover.test.js': "const test = require('node:test');\nconst { spawn } = require('node:child_process');\ntest('leaves a child', () => { spawn('sh', ['-c', 'sleep 2; echo changed >> README.md'], { stdio: 'ignore' }).unref(); });\n" });
|
||||
expect(pregate(after, []).status).toBe(EXIT.ok);
|
||||
await Bun.sleep(2500);
|
||||
expect(fs.readFileSync(path.join(after.work, 'README.md'), 'utf8')).toBe('# fixture\n');
|
||||
}, 90_000);
|
||||
|
||||
test('SIGTERM to pregate kills the running test process group, exits 143 and writes no pregate.json', async () => {
|
||||
const repo = makeFixtureRepo(); repos.push(repo);
|
||||
commit(repo, { 'test/hang.test.js': NODE_HANG });
|
||||
const child = spawn(BIN('gstack-pregate'), ['--base', 'main', '--allow-repo-commands', '--stage', 'tests', '--stall-timeout', '0'], { cwd: repo.work, env: { ...process.env, ...repo.env }, stdio: ['ignore', 'pipe', 'pipe'] });
|
||||
let err = '';
|
||||
child.stderr!.on('data', d => { err += d; });
|
||||
const exited = new Promise<number | null>(resolve => child.on('exit', code => resolve(code)));
|
||||
const testPid = await pidFrom(path.join(repo.root, 'hang.pid'), 30_000);
|
||||
child.kill('SIGTERM');
|
||||
expect(await exited).toBe(143);
|
||||
expect(err).toContain('gstack-pregate: cancelled by SIGTERM; killed 1 test process group(s); no pregate.json written');
|
||||
expect(await goneSoon(testPid)).toBe(true);
|
||||
expect(fs.existsSync(path.join(repo.work, '.gstack/tmp/pregate.json'))).toBe(false);
|
||||
}, 60_000);
|
||||
});
|
||||
Reference in new issue
Block a user