Merge remote-tracking branch 'origin/capy/next-wave-fix-c' into capy/next-wave

lib/pregate/checks/lanes.ts

@@ -49,13 +49,13 @@ export function parseSummary(output: string): RunnerSummary {
   return { ran: null, failed: null };
 }
 
-interface LocalRun { exit: number | null; outcome: Supervised['outcome'] | 'not-started'; ran: number | null; failed: number | null; tail: string[]; command: string; files: string[]; lastFile: string | null; ms: number }
+interface LocalRun { exit: number | null; outcome: Supervised['outcome']; ran: number | null; failed: number | null; tail: string[]; command: string; files: string[]; lastFile: string | null; ms: number }
 
 const quote = (f: string) => `'${f.replace(/'/g, `'\\''`)}'`;
 
-async function runOnce(ctx: PregateContext, env: NodeJS.ProcessEnv, runner: string, files: string[], log: string, deadline: number, counter: Counter): Promise<LocalRun> {
+async function runOnce(ctx: PregateContext, env: NodeJS.ProcessEnv, runner: string, files: string[], log: string, counter: Counter): Promise<LocalRun> {
   const command = runner.replace('{files}', files.map(quote).join(' '));
-  const r = await supervise({ command, cwd: ctx.repoRoot, env: { ...env, CI: env.CI ?? '1' }, log, deadline, stallMs: ctx.stallTimeoutMs, label: `lanes (${files.length === 1 ? files[0] : `${files.length} files`})`, files, adapter: adapterFor(runner), counter, progress: ctx.progress });
+  const r = await supervise({ command, cwd: ctx.repoRoot, env: { ...env, CI: env.CI ?? '1' }, log, deadline: Date.now() + ctx.timeoutMs, stallMs: ctx.stallTimeoutMs, label: `lanes (${files.length === 1 ? files[0] : `${files.length} files`})`, files, adapter: adapterFor(runner), counter, progress: ctx.progress });
   const output = readRange(log, r.start, r.end);
   const summary = parseSummary(output);
   return { exit: r.exit, outcome: r.outcome, ...summary, tail: output.trim().split('\n').slice(-25), command, files, lastFile: r.lastFile, ms: r.ms };
@@ -64,8 +64,8 @@ async function runOnce(ctx: PregateContext, env: NodeJS.ProcessEnv, runner: stri
 /**
  * Unit files in one runner process, then each serial-pattern file in its
  * own (a serial file shares process-wide state: a module mock, an env
- * mutation). `--timeout` bounds the whole check: a group that would start
- * after the deadline is not started. A stalled group is killed and the next
+ * mutation). `--timeout` bounds each runner process; the stall timeout
+ * catches a silent one. A stalled or timed-out group is killed and the next
  * one runs. The first unfinished, then the first non-zero run decides the
  * exit; counts add up only when every run reported a summary.
  */
@@ -75,29 +75,26 @@ async function runLocal(ctx: PregateContext, env: NodeJS.ProcessEnv, runner: str
   fs.writeFileSync(log, '');
   const serial = files.filter(f => matchesAny(ctx.config.serial_patterns, f));
   const unit = files.filter(f => !serial.includes(f));
-  const deadline = Date.now() + ctx.timeoutMs;
   const counter: Counter = { done: 0, total: files.length };
   const runs: LocalRun[] = [];
   for (const group of [...(unit.length ? [unit] : []), ...serial.map(f => [f])]) {
-    if (Date.now() >= deadline) { runs.push({ exit: null, outcome: 'not-started', ran: null, failed: null, tail: [], command: '', files: group, lastFile: null, ms: 0 }); continue; }
-    runs.push(await runOnce(ctx, env, runner, group, log, deadline, counter));
+    runs.push(await runOnce(ctx, env, runner, group, log, counter));
   }
   const bad = runs.find(r => r.exit === null) ?? runs.find(r => r.exit !== 0);
   const sum = (k: 'ran' | 'failed') => (runs.some(r => r[k] === null) ? null : runs.reduce((n, r) => n + (r[k] as number), 0));
-  const started = runs.filter(r => r.outcome !== 'not-started');
-  return { runs, bad, exit: bad ? bad.exit : 0, ran: sum('ran'), failed: sum('failed'), log, tail: (bad?.tail.length ? bad : started[started.length - 1] ?? runs[0]!).tail, command: started.map(r => r.command).join(' && '), serial: serial.length };
+  return { runs, bad, exit: bad ? bad.exit : 0, ran: sum('ran'), failed: sum('failed'), log, tail: (bad ?? runs[runs.length - 1]!).tail, command: runs.map(r => r.command).join(' && '), serial: serial.length };
 }
 
-/** Why the local lane did not finish: the stall, the deadline or a launch failure, with the last file started. */
+/** Why the local lane did not finish: a stall, a runner process past --timeout or a launch failure, with the last file started. */
 function unfinished(ctx: PregateContext, r: Awaited<ReturnType<typeof runLocal>>, total: number): Pick<CheckResult, 'code' | 'detail' | 'fix'> {
-  const notStarted = r.runs.filter(x => x.outcome === 'not-started').flatMap(x => x.files).length;
   const stalled = r.runs.filter(x => x.outcome === 'stalled');
-  const rest = `${notStarted ? `; ${notStarted} of ${total} file(s) not started` : ''}${r.failed ? `; ${r.failed} failed before that` : ''}`;
+  const timedOut = r.runs.filter(x => x.outcome === 'timeout');
+  const rest = r.failed ? `; ${r.failed} failed` : '';
   if (stalled.length) {
     return { code: 'PREGATE_STALLED', detail: `no test output for ${ctx.stallTimeoutMs / 1000} s; process group killed; last file started: ${stalled.map(x => x.lastFile ?? `(none reported; ${x.files.length} file(s))`).join(', ')}${rest}`, fix: `read ${r.log} for the stalled file; fix the hang, or raise --stall-timeout (stall_timeout_s in .gstack/pregate.json; 0 disables) for a test that is quiet for longer` };
   }
-  const why = r.runs.find(x => x.outcome === 'spawn-failed') ? 'the runner could not be launched' : `local lane timed out after ${ctx.timeoutMs} ms`;
-  return { code: 'PREGATE_INCOMPLETE', detail: `${why} (${total} files${rest})`, fix: 'raise --timeout or split the selection; a timed-out lane never passes' };
+  if (!timedOut.length) return { code: 'PREGATE_INCOMPLETE', detail: `the runner could not be launched (${total} files${rest})`, fix: 'read the lane log, fix the runner command, then rerun gstack-pregate --stage tests' };
+  return { code: 'PREGATE_INCOMPLETE', detail: `a runner process timed out after ${ctx.timeoutMs} ms (--timeout applies to each runner process; ${total} files; last file started: ${timedOut.map(x => x.lastFile ?? `(none reported; ${x.files.length} file(s))`).join(', ')}${rest})`, fix: 'raise --timeout or split the selection; a timed-out lane never passes' };
 }
 
 function describe(sel: Selection): string[] {
@@ -120,10 +117,29 @@ function remoteObligation(ctx: PregateContext, sel: Selection): RemoteObligation
   const declared = ctx.config.remote_gate;
   const runner = detectRunner(ctx, sel.localTests);
   const command = declared ?? `${GSTACK_ROOT}/scripts/ubicloud/ubi-runner.sh run --src ${quote(ctx.repoRoot)} -- ${quote(runner ? runner.replace('{files}', sel.localTests.map(quote).join(' ')) : '<test runner> <selected files>')}`;
-  const [workflow, job] = declared ? ['pregate.json', 'remote_gate'] : ['test_backend', ctx.testBackend];
+  const [workflow, job] = declared ? REMOTE_GATE_LANE.split('/') as [string, string] : ['test_backend', ctx.testBackend];
   return { lane: `${workflow}/${job}`, platform: 'remote', workflow, job, tests: sel.localTests, receipt: null, command };
 }
 
+export const REMOTE_GATE_LANE = 'pregate.json/remote_gate';
+/** The `clear` command for an obligation: the remote gate's own output for remote_gate, a run URL otherwise. */
+export const clearHint = (lane: string) => `gstack-pregate clear --remote ${lane} ${lane === REMOTE_GATE_LANE ? '--receipt <gate output file>' : '--run-url <url>'}`;
+export interface RemoteReport { tree: string; tests: number; failed: number; spent?: string }
+
+/** The last `PREGATE_REMOTE: tree=<tree> tests=<n> failed=<n> [spent=<usd>]` line in a remote gate's output. */
+export function parseRemoteReport(output: string): RemoteReport | null {
+  const m = [...output.matchAll(/^PREGATE_REMOTE: tree=(\S+) tests=(\d+) failed=(\d+)(?: spent=(\d+(?:\.\d+)?))?\s*$/gm)].pop();
+  return m ? { tree: m[1]!, tests: Number(m[2]), failed: Number(m[3]), spent: m[4] } : null;
+}
+
+/** Why a remote gate report does not clear the obligation for `tree` (`--remote` and `clear --receipt`), or null when it does. */
+export function remoteReportProblem(report: RemoteReport, tree: string): string | null {
+  if (report.tree !== tree) return `the remote gate tested tree ${report.tree.slice(0, 12)}, not ${tree.slice(0, 12)}`;
+  if (report.tests === 0) return 'the remote gate reported 0 tests';
+  if (report.failed > 0) return `the remote gate reported ${report.failed} failed of ${report.tests}`;
+  return null;
+}
+
 /**
  * `--remote`: run the declared remote gate now, under the spend cap. The
  * gate gets PREGATE_TREE, PREGATE_FILES_FILE and PREGATE_SPEND_CAP_USD and
@@ -144,22 +160,18 @@ async function runRemote(ctx: PregateContext, sel: Selection, base: CheckResult)
   fs.writeFileSync(filesFile, sel.localTests.join('\n') + '\n');
   const r = await supervise({ command: obligation.command!, cwd: ctx.repoRoot, env: { ...ctx.env, PREGATE_TREE: ctx.wtree, PREGATE_FILES_FILE: filesFile, PREGATE_SPEND_CAP_USD: String(cap) }, log, deadline: Date.now() + ctx.timeoutMs, stallMs: ctx.stallTimeoutMs, label: `remote gate ${obligation.lane}`, progress: ctx.progress });
   const output = readRange(log, r.start, r.end);
-  const reported = [...output.matchAll(/^PREGATE_REMOTE: tree=(\S+) tests=(\d+) failed=(\d+)(?: spent=(\d+(?:\.\d+)?))?\s*$/gm)].pop();
-  const spent = reported?.[4];
+  const reported = parseRemoteReport(output);
+  const spent = reported?.spent;
   const spend = `spend: spent ${spent ? `$${spent}` : '$0'}, reserved $0, unknown ${spent ? '$0' : `$${cap} (no spent= reported; counted at the cap)`} (cap $${cap})`;
   ctx.progress(`PREGATE_REMOTE_SPEND: ${spend.slice('spend: '.length)}`);
   const runLines = [...lines, `remote gate: ${obligation.command}`, `log: ${log}`, spend, ...output.trim().split('\n').slice(-15).map(l => `  ${l}`)];
   const inputs = { ...base.inputs, command: sha256(obligation.command!) };
   const why = r.outcome !== 'exited' ? `the remote gate did not finish (${r.outcome === 'stalled' ? `no output for ${ctx.stallTimeoutMs / 1000} s` : r.outcome})`
     : !reported ? `the remote gate exited ${r.exit} without a PREGATE_REMOTE: line`
-    : reported[1] !== ctx.wtree ? `the remote gate tested tree ${reported[1]!.slice(0, 12)}, not ${ctx.wtree.slice(0, 12)}`
-    : Number(reported[2]) === 0 ? 'the remote gate reported 0 tests'
-    : Number(reported[3]) > 0 ? `the remote gate reported ${reported[3]} failed of ${reported[2]}`
-    : r.exit !== 0 ? `the remote gate exited ${r.exit}`
-    : null;
+    : remoteReportProblem(reported, ctx.wtree) ?? (r.exit !== 0 ? `the remote gate exited ${r.exit}` : null);
   if (why) return { ...base, inputs, lines: runLines, status: 'incomplete', remote: [...sel.remote, obligation], code: r.outcome === 'stalled' ? 'PREGATE_STALLED' : 'PREGATE_INCOMPLETE', detail: `${why}; requires-remote ${obligation.lane} stays uncleared`, fix: `read ${log}; the obligation clears only on PREGATE_REMOTE: tree=${ctx.wtree.slice(0, 12)}… tests>0 failed=0 with exit 0` };
-  const cleared = { ...obligation, receipt: `remote-gate tree=${ctx.wtree.slice(0, 12)} tests=${reported![2]}`, cleared_by: `gstack-pregate --remote ${new Date().toISOString()}` };
-  return { ...base, inputs, lines: runLines, status: 'requires-remote', remote: [...sel.remote, cleared], detail: `remote gate ${obligation.lane}: ${reported![2]} tests, 0 failed on tree ${ctx.wtree.slice(0, 12)}; ${spend}` };
+  const cleared = { ...obligation, receipt: `remote-gate tree=${ctx.wtree.slice(0, 12)} tests=${reported!.tests}`, cleared_by: `gstack-pregate --remote ${new Date().toISOString()}` };
+  return { ...base, inputs, lines: runLines, status: 'requires-remote', remote: [...sel.remote, cleared], detail: `remote gate ${obligation.lane}: ${reported!.tests} tests, 0 failed on tree ${ctx.wtree.slice(0, 12)}; ${spend}` };
 }
 
 async function run(ctx: PregateContext): Promise<CheckResult> {
@@ -177,7 +189,7 @@ async function run(ctx: PregateContext): Promise<CheckResult> {
   const smallMachine = ctx.cores < SMALL_MACHINE_CORES && sel.localTests.length > 0 && (ctx.config.remote_gate || ctx.testBackend !== 'local');
   if (smallMachine) {
     const obligation = remoteObligation(ctx, sel);
-    return { ...base, status: 'requires-remote', remote: [...sel.remote, obligation], lines: [...lines, `requires-remote ${obligation.lane}: ${ctx.cores} cores (under ${SMALL_MACHINE_CORES}); ${sel.localTests.length} local test file(s) recorded for the remote gate, not run here`, `run: ${obligation.command}`], detail: `${sel.localTests.length} local test file(s) recorded as requires-remote ${obligation.lane} (${ctx.cores} cores, under ${SMALL_MACHINE_CORES}); run it, then gstack-pregate clear --remote ${obligation.lane} --run-url <url>, or gstack-pregate --remote --spend-cap <usd> --allow-repo-commands` };
+    return { ...base, status: 'requires-remote', remote: [...sel.remote, obligation], lines: [...lines, `requires-remote ${obligation.lane}: ${ctx.cores} cores (under ${SMALL_MACHINE_CORES}); ${sel.localTests.length} local test file(s) recorded for the remote gate, not run here`, `run: ${obligation.command}`], detail: `${sel.localTests.length} local test file(s) recorded as requires-remote ${obligation.lane} (${ctx.cores} cores, under ${SMALL_MACHINE_CORES}); run it, then ${clearHint(obligation.lane)}, or gstack-pregate --remote --spend-cap <usd> --allow-repo-commands` };
   }
   if (!ctx.allowRepoCommands) return { ...base, status: 'incomplete', code: 'REPO_COMMANDS_NOT_ALLOWED', detail: `${sel.localTests.length} local test file(s) selected but not run (pass --allow-repo-commands)`, fix: 'rerun with --allow-repo-commands (what /ship passes)' };
   if (sel.localTests.length === 0) return { ...base, status: sel.remote.length ? 'requires-remote' : 'pass', detail: `no local lane; ${sel.remote.length} requires-remote obligation(s)` };

lib/pregate/index.ts

@@ -10,12 +10,13 @@
  * land: `guards` (field wave CD1); ratchets, lint, links, hermetic, patches
  * and integrator later.
  */
+import { createHash } from 'node:crypto';
 import * as fs from 'node:fs';
 import * as path from 'node:path';
 import { ARTIFACT_SCHEMAS, checkSchema, type PregateFile, type PregateRemote, type ValidationError } from '../headless-artifacts';
 import { wtreeFingerprint } from '../regen';
 import { guardsCheck } from './checks/guards';
-import { lanesCheck } from './checks/lanes';
+import { clearHint, lanesCheck, parseRemoteReport, REMOTE_GATE_LANE, remoteReportProblem } from './checks/lanes';
 import { literalsCheck } from './checks/literals';
 import { regenCheck } from './checks/regen';
 import { secretsCheck } from './checks/secrets';
@@ -86,7 +87,7 @@ export function renderTable(file: PregateFile, opts: { lines?: boolean } = { lin
   ].filter(Boolean);
   const remote = (file.requires_remote ?? []).map(r => {
     const run = r.command ? `run ${r.command.length > 200 ? `${r.command.slice(0, 200)}… (full command in pregate.json)` : r.command}, then ` : '';
-    return `requires-remote: ${r.lane} (${r.platform}) ${r.tests.length} test(s) — ${r.receipt ? `cleared by ${r.receipt}` : `uncleared; ${run}clear with gstack-pregate clear --remote ${r.lane} --run-url <url>`}`;
+    return `requires-remote: ${r.lane} (${r.platform}) ${r.tests.length} test(s) — ${r.receipt ? `cleared by ${r.receipt}` : `uncleared; ${run}clear with ${clearHint(r.lane)}`}`;
   });
   return [
     `stage      check     status            time  detail`,
@@ -134,6 +135,41 @@ export function verifyPregate(repoRoot: string, file: PregateFile, treeNow = wtr
   return { ok: true, message: `pregate ${file.verdict} is current for tree ${file.tree.slice(0, 12)}`, current, uncleared };
 }
 
+export interface ClearInput { runUrl?: string; bundleLane?: string; receiptPath?: string }
+export type ClearDecision = { receipt: string; attested: boolean } | { refused: 'usage' | 'PREGATE_RECEIPT_INVALID'; message: string };
+
+/**
+ * What `clear` may record for one obligation. A `pregate.json/remote_gate`
+ * obligation needs the gate's own report (`--receipt <file>` holding its
+ * PREGATE_REMOTE: line), validated as `--remote` validates it against the
+ * tested tree; a run URL is not checkable there. A `test_backend/<backend>`
+ * obligation has no machine-checkable receipt, so its run URL is recorded as
+ * attested. Platform lanes take a run URL or an evidence-bundle lane label.
+ */
+export function clearDecision(file: PregateFile, lane: string, input: ClearInput): ClearDecision {
+  const list = file.requires_remote ?? [];
+  const target = list.find(r => r.lane === lane);
+  if (!target) return { refused: 'usage', message: `no requires-remote obligation named ${lane}; obligations: ${list.map(r => r.lane).join(', ') || '(none)'}` };
+  if (target.lane === REMOTE_GATE_LANE) {
+    if (!input.receiptPath) return { refused: 'usage', message: `${lane} clears only with --receipt <file> holding the remote gate's PREGATE_REMOTE: tree=… tests=… failed=… line (a run URL is not checkable); or rerun gstack-pregate --remote` };
+    let text: string;
+    try { text = fs.readFileSync(input.receiptPath, 'utf8'); }
+    catch (e: any) { return { refused: 'PREGATE_RECEIPT_INVALID', message: `cannot read ${input.receiptPath}: ${e.message}` }; }
+    const report = parseRemoteReport(text);
+    const problem = report ? remoteReportProblem(report, file.tree) : `${input.receiptPath} has no PREGATE_REMOTE: line`;
+    if (problem) return { refused: 'PREGATE_RECEIPT_INVALID', message: `${problem}; ${lane} stays uncleared` };
+    return { receipt: `remote-gate tree=${file.tree.slice(0, 12)} tests=${report!.tests} receipt=${path.resolve(input.receiptPath)} sha256=${createHash('sha256').update(text).digest('hex').slice(0, 12)}`, attested: false };
+  }
+  if (input.receiptPath) return { refused: 'usage', message: `--receipt applies only to ${REMOTE_GATE_LANE}; clear ${lane} with --run-url <url>${target.workflow === 'test_backend' ? '' : ' or --bundle-lane <label>'}` };
+  if (target.workflow === 'test_backend') {
+    if (!input.runUrl) return { refused: 'usage', message: `${lane} clears with --run-url <url>` };
+    return { receipt: `attested:${input.runUrl}`, attested: true };
+  }
+  const receipt = input.runUrl ?? (input.bundleLane ? `bundle-lane:${input.bundleLane}` : undefined);
+  if (!receipt) return { refused: 'usage', message: `${lane} clears with --run-url <url> or --bundle-lane <label>` };
+  return { receipt, attested: false };
+}
+
 /** Record the receipt that clears one obligation (a run URL or an evidence-bundle lane label). */
 export function clearObligation(file: PregateFile, lane: string, receipt: string, by: string): PregateFile {
   const list = file.requires_remote ?? [];

lib/result-codes.ts

@@ -394,6 +394,11 @@ export const RESULT_CODES = {
     summary: 'a test command printed nothing for the stall timeout, so its process group was killed; the check is incomplete, never pass',
     fix: 'read the lane log for the last file started, fix the hang, or raise `--stall-timeout` (`stall_timeout_s`; 0 disables) for a test that is quiet for longer, then rerun `gstack-pregate --stage tests`',
   },
+  PREGATE_RECEIPT_INVALID: {
+    anchor: 'pregate-receipt-invalid',
+    summary: 'the receipt offered to `gstack-pregate clear` for the remote gate obligation does not show the tested tree passing',
+    fix: 'rerun the remote gate on this tree and pass its output with `gstack-pregate clear --remote pregate.json/remote_gate --receipt <file>`, or run `gstack-pregate --stage tests --remote --spend-cap <usd> --allow-repo-commands`',
+  },
   PREGATE_STALE: {
     anchor: 'pregate-stale',
     summary: 'pregate.json was recorded against a different tree than the one being published',