train S3: merge #3899 at f79edd28f5 v1.93.35.0

This commit is contained in:
garrytan committed 2000-01-01 00:00:00 +00:00
commit 910de9812e
13 files changed
+103 -24

No files matched your search

+2 -2
View File
@@ -20,7 +20,7 @@ import { baseInputs, sha256 } from '../context';
import { classifyCommandEnvironment, classifyEnvironment, environmentLine, environmentResult, launchCause } from '../environment';
import { launchRuntime } from '../runtime';
import { checkScratch } from '../run-scratch';
import { readRange, supervise } from '../supervise';
import { readRange, STALL_IDLE, supervise } from '../supervise';
import type { EnvironmentCause } from '../types';
import type { CheckDef, CheckResult, PregateContext } from '../types';
@@ -56,7 +56,7 @@ async function run(ctx: PregateContext): Promise<CheckResult> {
for (const g of selected) runs.push(await runGuard(ctx, g, runtime.env, dir));
const lines = [`runtime: ${runtime.label}`];
for (const r of runs) {
lines.push(r.stalled ? `GUARD: ${r.guard.id} stalled: no output for ${ctx.stallTimeoutMs / 1000} s; process group killed (log ${r.log})` : r.timedOut ? `GUARD: ${r.guard.id} timed out after ${r.guard.timeout_s ?? ctx.timeoutMs / 1000} s (log ${r.log})` : `GUARD: ${r.guard.id} exit=${r.exit} (log ${r.log})`);
lines.push(r.stalled ? `GUARD: ${r.guard.id} stalled: no output for ${ctx.stallTimeoutMs / 1000} s${STALL_IDLE}; process group killed (log ${r.log})` : r.timedOut ? `GUARD: ${r.guard.id} timed out after ${r.guard.timeout_s ?? ctx.timeoutMs / 1000} s (log ${r.log})` : `GUARD: ${r.guard.id} exit=${r.exit} (log ${r.log})`);
if (r.exit !== 0) lines.push(...r.tail.map(l => ` ${l}`));
if (r.environment) lines.push(` ${r.guard.id} ${environmentLine(r.environment)}`);
}
+2 -2
View File
@@ -39,7 +39,7 @@ import { isDeviceDart, listRepoFiles, readRepoFile } from '../import-graph';
import { selectLanes, type Selection } from '../lane-selection';
import { launchRuntime } from '../runtime';
import { cargoModel, type CargoModel } from '../rust';
import { adapterFor, readRange, supervise, type Counter, type Supervised } from '../supervise';
import { adapterFor, readRange, STALL_IDLE, supervise, type Counter, type Supervised } from '../supervise';
import { verifyFromRun } from '../../run-evidence';
import { matchesAny } from '../../ship-policy';
import type { CheckDef, CheckResult, PregateContext, RemoteObligation } from '../types';
@@ -401,7 +401,7 @@ function unfinished(ctx: PregateContext, r: Awaited<ReturnType<typeof runLocal>>
const timedOut = r.runs.filter(x => x.outcome === 'timeout');
const rest = r.failed ? `; ${r.failed} failed` : '';
if (stalled.length) {
return { code: 'PREGATE_STALLED', detail: `no test output for ${ctx.stallTimeoutMs / 1000} s; process group killed; last file started: ${stalled.map(x => x.lastFile ?? `(none reported; ${x.files.length} file(s))`).join(', ')}${rest}`, fix: `read ${r.log} for the stalled file; fix the hang, or raise --stall-timeout (stall_timeout_s in ${ctx.configFile}; 0 disables) for a test that is quiet for longer` };
return { code: 'PREGATE_STALLED', detail: `no test output for ${ctx.stallTimeoutMs / 1000} s${STALL_IDLE}; process group killed; last file started: ${stalled.map(x => x.lastFile ?? `(none reported; ${x.files.length} file(s))`).join(', ')}${rest}`, fix: `read ${r.log} for the stalled file; fix the hang, or raise --stall-timeout (stall_timeout_s in ${ctx.configFile}; 0 disables) for a test that waits quietly for longer` };
}
const launch = r.runs.find(x => x.outcome === 'spawn-failed');
if (launch) {
+4 -3
View File
@@ -29,8 +29,9 @@
* the pre-push hook and `gstack-pregate sealed`, not by `run`)
* price_file the call-plan data file `gstack-eval-plan price-gate` prices
* a paid gate from (GSTACK_EVAL_CALLS v1; never executed)
* stall_timeout_s seconds without test output before the lanes check kills
* the runner's process group (default 300; 0 disables)
* stall_timeout_s seconds without test output or CPU activity before the
* lanes check kills the runner's process group (default 300;
* 0 disables)
* timeout_s seconds each runner process may take (the unit-file group,
* each serial file, each guard without its own timeout_s, the
* remote gate) when `--timeout` is not passed (default 600)
@@ -352,7 +353,7 @@ export function explainConfig(): string {
`serial_patterns: test-file globs the lanes check runs one process per file, apart from the unit files (default ${DEFAULT_SERIAL_PATTERNS.join(', ')})`,
'sealed: { paths, transcripts, answer_markers, seed_patterns, local_paths, allow_paths, case_insensitive, label_keys, identifier_keys, visibility }: sealed or eval output the pre-push hook and `gstack-pregate sealed` refuse to publish to a public destination; allow_paths (base branch only) exempt publishable files from the path globs; label_keys turns on sealed.label_join, which refuses a published record that puts one of those fields next to an identifier (identifier_keys globs, or a hex value) (docs/pregate.md#sealed)',
'price_file: repo-relative path of the GSTACK_EVAL_CALLS v1 call-plan file `gstack-eval-plan price-gate` prices a paid gate from; never executed (docs/eval-calls.md)',
'stall_timeout_s: seconds without test output before the lanes check kills the runner\'s process group and records it incomplete (default 300; 0 disables; --stall-timeout overrides)',
'stall_timeout_s: seconds without test output or CPU activity before the lanes check kills the runner\'s process group and records it incomplete (default 300; 0 disables; --stall-timeout overrides)',
'timeout_s: seconds each runner process may take (the unit-file group, each serial file, each guard without its own timeout_s, and the remote gate\'s deadline) when --timeout is not passed; 1 to 3600, default 600',
'remote_gate: the repo\'s remote test gate command; under 8 cores the test tier becomes a requires-remote obligation naming it, and gstack-pregate --remote runs it (it must print PREGATE_REMOTE: tree=<tree> tests=<n> failed=<n>) (docs/pregate.md#remote-gate)',
'run_profiles: name → { workflow, events, jobs, artifact, max_age_hours?, os? }; the GitHub Actions runs gstack-pregate clear --run verifies as evidence on the identical tree (docs/pregate.md#run-evidence)',
+43 -4
View File
@@ -2,8 +2,10 @@
* pregate/supervise — how a test-tier command runs (Lane C, GBRA-95): its
* own process group, stdout and stderr appended to a log file (no pipes, so
* a backgrounded grandchild holding the output can never keep the check
* waiting), supervised asynchronously. A stall is no log growth for
* `stallMs`; the deadline is the check's `--timeout`. Either kills the whole
* waiting), supervised asynchronously. A stall is `stallMs` with no log growth
* and no CPU activity in the process group (GST-1095: a quiet parallel suite
* that is busy working is alive; a hang that keeps burning CPU is left to the
* deadline); the deadline is the check's `--timeout`. Either kills the whole
* process group (SIGTERM, then SIGKILL after a grace period) and the run is
* incomplete. After a normal exit the group's leftover members are killed.
*
@@ -34,7 +36,7 @@ export interface SuperviseOptions {
log: string;
/** Absolute epoch ms when the run is killed as timed out. */
deadline: number;
/** No log growth for this long kills the group; 0 disables. */
/** No log growth and no CPU activity in the group for this long kills it; 0 disables. */
stallMs: number;
heartbeatMs?: number;
graceMs?: number;
@@ -63,6 +65,11 @@ export interface Supervised {
}
const ACTIVE = new Set<number>();
const USER_HZ = 100;
const CPU_SAMPLE_MS = 1_000;
const CPU_BUSY_SHARE = 0.1;
/** What a stall means beyond silence, for the PREGATE_STALLED detail and the guard line; Windows reads no CPU time. */
export const STALL_IDLE = process.platform === 'win32' ? '' : ' and its process group used under a tenth of one CPU';
/** SIGKILL every supervised process group still running (the CLI's SIGINT/SIGTERM handler, before it exits). */
export function cancelActive(): number {
@@ -80,6 +87,31 @@ export function killGroup(pid: number, signal: NodeJS.Signals): void {
try { process.kill(-pid, signal); } catch { /* group already gone */ }
}
/** CPU time in ms used by the process group `pgid`, reaped children included where the platform reports them; null where it cannot be read (Windows). */
export function groupCpuMs(pgid: number): number | null {
if (process.platform === 'win32') return null;
if (fs.existsSync('/proc/self/stat')) {
let ticks = 0;
for (const name of fs.readdirSync('/proc')) {
if (!/^\d+$/.test(name)) continue;
let stat: string;
try { stat = fs.readFileSync(`/proc/${name}/stat`, 'utf8'); } catch { continue; }
const f = stat.slice(stat.lastIndexOf(')') + 2).split(' ');
if (Number(f[2]) === pgid) ticks += Number(f[11]) + Number(f[12]) + Number(f[13]) + Number(f[14]);
}
return ticks * 1000 / USER_HZ;
}
const ps = spawnSync('ps', ['-A', '-S', '-o', 'pgid=,time='], { encoding: 'utf8', timeout: 15_000 });
if (ps.status !== 0) return null;
let ms = 0;
for (const line of ps.stdout.split('\n')) {
const [id, time] = line.trim().split(/\s+/);
const t = time && /^(?:(\d+)-)?(?:(\d+):)?(\d+):(\d+(?:\.\d+)?)$/.exec(time);
if (Number(id) === pgid && t) ms += (((Number(t[1] ?? 0) * 24 + Number(t[2] ?? 0)) * 60 + Number(t[3])) * 60 + Number(t[4])) * 1000;
}
return ms;
}
/** The adapter that reads per-file progress from this runner's output, or null when it has none. */
export function adapterFor(command: string): ProgressAdapter | null {
for (const [name, re] of Object.entries(PROGRESS_ADAPTERS)) if (re.test(command)) return name as ProgressAdapter;
@@ -110,6 +142,8 @@ export function supervise(o: SuperviseOptions): Promise<Supervised> {
let readAt = start;
let partial = '';
let lastGrowth = started;
let lastBusy = started;
let cpu = { at: started, ms: 0 };
let lastProgress = started;
let current: { file: string; at: number; failed: boolean } | null = null;
let lastFile: string | null = null;
@@ -184,7 +218,12 @@ export function supervise(o: SuperviseOptions): Promise<Supervised> {
if (grown > size) { size = grown; lastGrowth = now; readNew(); }
if (outcome) return;
if (now >= o.deadline) { stop('timeout'); return; }
if (o.stallMs > 0 && now - lastGrowth >= o.stallMs) { stop('stalled'); return; }
if (o.stallMs > 0 && process.platform !== 'win32' && now - cpu.at >= CPU_SAMPLE_MS) {
const used = groupCpuMs(child.pid!);
if (used !== null && used - cpu.ms >= CPU_BUSY_SHARE * (now - cpu.at)) lastBusy = now;
cpu = { at: now, ms: used ?? cpu.ms };
}
if (o.stallMs > 0 && now - Math.max(lastGrowth, lastBusy) >= o.stallMs) { stop('stalled'); return; }
if (now - lastProgress < heartbeatMs) return;
let last: string | undefined;
if (o.lastLine) {
+1 -1
View File
@@ -54,7 +54,7 @@ export interface PregateContext {
timeoutMs: number;
/** Where timeoutMs came from: `--timeout`, `timeout_s from <policy>`, or the 600 s default. */
timeoutSource?: string;
/** No output for this long kills a test-tier process group (`--stall-timeout`, `stall_timeout_s`); 0 disables. */
/** No output and no CPU activity for this long kills a test-tier process group (`--stall-timeout`, `stall_timeout_s`); 0 disables. */
stallTimeoutMs: number;
/** Live stderr lines: per-file progress, heartbeats, the stage result. */
progress: (line: string) => void;
+2 -2
View File
@@ -410,8 +410,8 @@ export const RESULT_CODES = {
},
PREGATE_STALLED: {
anchor: 'pregate-stalled',
summary: 'a test command printed nothing for the stall timeout, so its process group was killed; the check is incomplete, never pass',
fix: 'read the lane log for the last file started, fix the hang, or raise `--stall-timeout` (`stall_timeout_s`; 0 disables) for a test that is quiet for longer, then rerun `gstack-pregate --stage tests`',
summary: 'a test command printed nothing for the stall timeout while its process group used under a tenth of one CPU (log growth alone on Windows), so the group was killed; the check is incomplete, never pass',
fix: 'read the lane log for the last file started, fix the hang, or raise `--stall-timeout` (`stall_timeout_s`; 0 disables) for a test that waits quietly for longer, then rerun `gstack-pregate --stage tests`',
},
PREGATE_RECEIPT_INVALID: {
anchor: 'pregate-receipt-invalid',