mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-15 12:57:22 +01:00
fix(pregate): stall wording says what was measured; a failed CPU read skips one sample; ps -S on BSDs (review pass 1)
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
This commit is contained in:
1 parent
bb19bfb13e
commit
bd5b954b27
6 files changed
+19
-13
No files matched your search
@@ -20,7 +20,7 @@ import { baseInputs, sha256 } from '../context';
|
||||
import { classifyCommandEnvironment, classifyEnvironment, environmentLine, environmentResult, launchCause } from '../environment';
|
||||
import { launchRuntime } from '../runtime';
|
||||
import { checkScratch } from '../run-scratch';
|
||||
import { readRange, supervise } from '../supervise';
|
||||
import { readRange, STALL_IDLE, supervise } from '../supervise';
|
||||
import type { EnvironmentCause } from '../types';
|
||||
import type { CheckDef, CheckResult, PregateContext } from '../types';
|
||||
|
||||
@@ -56,7 +56,7 @@ async function run(ctx: PregateContext): Promise<CheckResult> {
|
||||
for (const g of selected) runs.push(await runGuard(ctx, g, runtime.env, dir));
|
||||
const lines = [`runtime: ${runtime.label}`];
|
||||
for (const r of runs) {
|
||||
lines.push(r.stalled ? `GUARD: ${r.guard.id} stalled: no output for ${ctx.stallTimeoutMs / 1000} s and its process group used no CPU; process group killed (log ${r.log})` : r.timedOut ? `GUARD: ${r.guard.id} timed out after ${r.guard.timeout_s ?? ctx.timeoutMs / 1000} s (log ${r.log})` : `GUARD: ${r.guard.id} exit=${r.exit} (log ${r.log})`);
|
||||
lines.push(r.stalled ? `GUARD: ${r.guard.id} stalled: no output for ${ctx.stallTimeoutMs / 1000} s${STALL_IDLE}; process group killed (log ${r.log})` : r.timedOut ? `GUARD: ${r.guard.id} timed out after ${r.guard.timeout_s ?? ctx.timeoutMs / 1000} s (log ${r.log})` : `GUARD: ${r.guard.id} exit=${r.exit} (log ${r.log})`);
|
||||
if (r.exit !== 0) lines.push(...r.tail.map(l => ` ${l}`));
|
||||
if (r.environment) lines.push(` ${r.guard.id} ${environmentLine(r.environment)}`);
|
||||
}
|
||||
|
||||
@@ -39,7 +39,7 @@ import { isDeviceDart, listRepoFiles, readRepoFile } from '../import-graph';
|
||||
import { selectLanes, type Selection } from '../lane-selection';
|
||||
import { launchRuntime } from '../runtime';
|
||||
import { cargoModel, type CargoModel } from '../rust';
|
||||
import { adapterFor, readRange, supervise, type Counter, type Supervised } from '../supervise';
|
||||
import { adapterFor, readRange, STALL_IDLE, supervise, type Counter, type Supervised } from '../supervise';
|
||||
import { verifyFromRun } from '../../run-evidence';
|
||||
import { matchesAny } from '../../ship-policy';
|
||||
import type { CheckDef, CheckResult, PregateContext, RemoteObligation } from '../types';
|
||||
@@ -401,7 +401,7 @@ function unfinished(ctx: PregateContext, r: Awaited<ReturnType<typeof runLocal>>
|
||||
const timedOut = r.runs.filter(x => x.outcome === 'timeout');
|
||||
const rest = r.failed ? `; ${r.failed} failed` : '';
|
||||
if (stalled.length) {
|
||||
return { code: 'PREGATE_STALLED', detail: `no test output for ${ctx.stallTimeoutMs / 1000} s and its process group used no CPU; process group killed; last file started: ${stalled.map(x => x.lastFile ?? `(none reported; ${x.files.length} file(s))`).join(', ')}${rest}`, fix: `read ${r.log} for the stalled file; fix the hang, or raise --stall-timeout (stall_timeout_s in ${ctx.configFile}; 0 disables) for a test that waits quietly for longer` };
|
||||
return { code: 'PREGATE_STALLED', detail: `no test output for ${ctx.stallTimeoutMs / 1000} s${STALL_IDLE}; process group killed; last file started: ${stalled.map(x => x.lastFile ?? `(none reported; ${x.files.length} file(s))`).join(', ')}${rest}`, fix: `read ${r.log} for the stalled file; fix the hang, or raise --stall-timeout (stall_timeout_s in ${ctx.configFile}; 0 disables) for a test that waits quietly for longer` };
|
||||
}
|
||||
const launch = r.runs.find(x => x.outcome === 'spawn-failed');
|
||||
if (launch) {
|
||||
|
||||
@@ -68,6 +68,8 @@ const ACTIVE = new Set<number>();
|
||||
const USER_HZ = 100;
|
||||
const CPU_SAMPLE_MS = 1_000;
|
||||
const CPU_BUSY_SHARE = 0.1;
|
||||
/** What a stall means beyond silence, for the PREGATE_STALLED detail and the guard line; Windows reads no CPU time. */
|
||||
export const STALL_IDLE = process.platform === 'win32' ? '' : ' and its process group used under a tenth of one CPU';
|
||||
|
||||
/** SIGKILL every supervised process group still running (the CLI's SIGINT/SIGTERM handler, before it exits). */
|
||||
export function cancelActive(): number {
|
||||
@@ -99,7 +101,7 @@ export function groupCpuMs(pgid: number): number | null {
|
||||
}
|
||||
return ticks * 1000 / USER_HZ;
|
||||
}
|
||||
const ps = spawnSync('ps', ['-A', '-o', 'pgid=,time='], { encoding: 'utf8', timeout: 15_000 });
|
||||
const ps = spawnSync('ps', ['-A', '-S', '-o', 'pgid=,time='], { encoding: 'utf8', timeout: 15_000 });
|
||||
if (ps.status !== 0) return null;
|
||||
let ms = 0;
|
||||
for (const line of ps.stdout.split('\n')) {
|
||||
@@ -141,7 +143,7 @@ export function supervise(o: SuperviseOptions): Promise<Supervised> {
|
||||
let partial = '';
|
||||
let lastGrowth = started;
|
||||
let lastBusy = started;
|
||||
let cpu: { at: number; ms: number } | null = { at: started, ms: 0 };
|
||||
let cpu = { at: started, ms: 0 };
|
||||
let lastProgress = started;
|
||||
let current: { file: string; at: number; failed: boolean } | null = null;
|
||||
let lastFile: string | null = null;
|
||||
@@ -216,10 +218,12 @@ export function supervise(o: SuperviseOptions): Promise<Supervised> {
|
||||
if (grown > size) { size = grown; lastGrowth = now; readNew(); }
|
||||
if (outcome) return;
|
||||
if (now >= o.deadline) { stop('timeout'); return; }
|
||||
if (o.stallMs > 0 && cpu && now - cpu.at >= CPU_SAMPLE_MS) {
|
||||
if (o.stallMs > 0 && process.platform !== 'win32' && now - cpu.at >= CPU_SAMPLE_MS) {
|
||||
const used = groupCpuMs(child.pid!);
|
||||
if (used !== null && used - cpu.ms >= CPU_BUSY_SHARE * (now - cpu.at)) lastBusy = now;
|
||||
cpu = used === null ? null : { at: now, ms: used };
|
||||
if (used !== null) {
|
||||
if (used - cpu.ms >= CPU_BUSY_SHARE * (now - cpu.at)) lastBusy = now;
|
||||
cpu = { at: now, ms: used };
|
||||
}
|
||||
}
|
||||
if (o.stallMs > 0 && now - Math.max(lastGrowth, lastBusy) >= o.stallMs) { stop('stalled'); return; }
|
||||
if (now - lastProgress < heartbeatMs) return;
|
||||
|
||||
+1
-1
@@ -410,7 +410,7 @@ export const RESULT_CODES = {
|
||||
},
|
||||
PREGATE_STALLED: {
|
||||
anchor: 'pregate-stalled',
|
||||
summary: 'a test command printed nothing and used no CPU for the stall timeout, so its process group was killed; the check is incomplete, never pass',
|
||||
summary: 'a test command printed nothing for the stall timeout while its process group used under a tenth of one CPU (log growth alone on Windows), so the group was killed; the check is incomplete, never pass',
|
||||
fix: 'read the lane log for the last file started, fix the hang, or raise `--stall-timeout` (`stall_timeout_s`; 0 disables) for a test that waits quietly for longer, then rerun `gstack-pregate --stage tests`',
|
||||
},
|
||||
PREGATE_RECEIPT_INVALID: {
|
||||
|
||||
Reference in new issue
Block a user