mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-10 18:50:18 +01:00
Merge remote-tracking branch 'origin/main' into gst-450-qa-parent
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
This commit is contained in:
garrytanandcapy-ai[bot] committed
commit
e432553433
136 files changed
+7124
-811
No files matched your search
@@ -0,0 +1,46 @@
|
||||
/**
|
||||
* eval-plan-report: the results check behind `gstack-eval-plan results --check`.
|
||||
* A preregistration that says `stack: loaded` names its no-gstack arms; its
|
||||
* results file must hold one markdown table with a filled row for every Arms
|
||||
* identity, so each no-gstack row sits beside its variants
|
||||
* (docs/TESTING_INTERNALS.md#no-gstack-arm). Reads text only; executes nothing.
|
||||
*/
|
||||
import { armIdentities, checkPrereg } from './eval-plan';
|
||||
|
||||
export interface ResultsCheck { ok: boolean; required: string[]; no_gstack: string[]; found: string[]; missing: string[]; problems: string[] }
|
||||
|
||||
const PLACEHOLDER = /^(?:|-+|—|–|n\/a|tbd|todo|\?)$/i;
|
||||
const cellsOf = (line: string) => line.trim().replace(/^\|/, '').replace(/\|$/, '').split('|').map(c => c.trim());
|
||||
const separator = (line: string | undefined) => !!line && /^\s*\|[\s:|-]+\|?\s*$/.test(line) && line.includes('-');
|
||||
|
||||
/** The filled-row identities of each markdown table: first cell, backticks stripped, header and separator rows skipped. */
|
||||
export function resultTables(text: string): Array<Set<string>> {
|
||||
const lines = text.split('\n');
|
||||
const tables: Array<Set<string>> = [];
|
||||
let current: Set<string> | null = null;
|
||||
lines.forEach((line, i) => {
|
||||
if (!/^\s*\|/.test(line)) { current = null; return; }
|
||||
if (!current) { current = new Set(); tables.push(current); }
|
||||
if (separator(line) || separator(lines[i + 1])) return;
|
||||
const [first, ...rest] = cellsOf(line);
|
||||
const id = (first ?? '').replace(/`/g, '').trim();
|
||||
if (id && rest.some(c => !PLACEHOLDER.test(c.replace(/`/g, '').trim()))) current.add(id);
|
||||
});
|
||||
return tables;
|
||||
}
|
||||
|
||||
export function checkResults(preregText: string, resultsText: string): ResultsCheck {
|
||||
const prereg = checkPrereg(preregText);
|
||||
if (prereg.comparison.stack === 'none') return { ok: true, required: [], no_gstack: [], found: [], missing: [], problems: [] };
|
||||
const required = [...new Set(armIdentities(preregText))];
|
||||
const noGstack = prereg.no_gstack.arms;
|
||||
const problems: string[] = [];
|
||||
if (!prereg.ok) problems.push('the preregistration does not pass prereg --check');
|
||||
const tables = resultTables(resultsText);
|
||||
const best = tables.reduce<Set<string>>((b, t) => (required.filter(id => t.has(id)).length > required.filter(id => b.has(id)).length ? t : b), new Set());
|
||||
const found = required.filter(id => best.has(id));
|
||||
const missing = required.filter(id => !best.has(id));
|
||||
if (!tables.length) problems.push('no markdown table in the results file');
|
||||
else for (const id of missing) problems.push(`no filled row for ${id} in the table that holds the other arms`);
|
||||
return { ok: problems.length === 0, required, no_gstack: noGstack, found, missing, problems };
|
||||
}
|
||||
@@ -17,6 +17,8 @@ export interface AllowlistFinding {
|
||||
id: string; category: string; severity: string; file: string; line?: number; col?: number;
|
||||
/** The added line the finding sits on (for `pattern`); absent when the scanner gave no position. */
|
||||
lineText?: string;
|
||||
/** `line` numbers the whole file (a Dart or Python file scanned whole, #3630), not the added lines. */
|
||||
fileLine?: boolean;
|
||||
}
|
||||
export interface Allowlisted<F extends AllowlistFinding = AllowlistFinding> { finding: F; index: number; entry: AllowlistEntry }
|
||||
|
||||
@@ -38,26 +40,34 @@ function patternHits(entry: AllowlistEntry, f: AllowlistFinding): boolean {
|
||||
return re.test(f.lineText!.slice(f.col! - 1));
|
||||
}
|
||||
|
||||
function matchesLine(pattern: string, line: string): boolean {
|
||||
try { return new RegExp(pattern).test(line); } catch { return false; }
|
||||
}
|
||||
|
||||
/**
|
||||
* Split findings into the ones an entry covers (with the entry) and the ones
|
||||
* kept; env.value and engine.* are always kept. `untested` lists kept findings
|
||||
* that a pattern entry named by id and path but could not test because the
|
||||
* scanner gave no position, so the table says why the entry was inert.
|
||||
* scanner gave no position, so the table says why the entry was inert;
|
||||
* `missed` lists kept findings that a pattern entry named by id and path
|
||||
* whose pattern matches the finding's line but not at its position (#3630:
|
||||
* the entry was written from the name, not the value).
|
||||
*/
|
||||
export function applyAllowlist<F extends AllowlistFinding>(findings: readonly F[], entries: readonly AllowlistEntry[]): { kept: F[]; allowlisted: Allowlisted<F>[]; untested: Allowlisted<F>[] } {
|
||||
export function applyAllowlist<F extends AllowlistFinding>(findings: readonly F[], entries: readonly AllowlistEntry[]): { kept: F[]; allowlisted: Allowlisted<F>[]; untested: Allowlisted<F>[]; missed: Allowlisted<F>[] } {
|
||||
const kept: F[] = [];
|
||||
const allowlisted: Allowlisted<F>[] = [];
|
||||
const untested: Allowlisted<F>[] = [];
|
||||
const missed: Allowlisted<F>[] = [];
|
||||
for (const f of findings) {
|
||||
const scoped = (e: AllowlistEntry) => names(e, f) && (!e.paths || matchesAny(e.paths, f.file));
|
||||
const index = nonAllowlistable(f.id) ? -1 : entries.findIndex(e => scoped(e) && patternHits(e, f));
|
||||
if (index >= 0) { allowlisted.push({ finding: f, index, entry: entries[index]! }); continue; }
|
||||
kept.push(f);
|
||||
if (nonAllowlistable(f.id) || positioned(f)) continue;
|
||||
const named = entries.findIndex(e => e.pattern !== undefined && scoped(e));
|
||||
if (named >= 0) untested.push({ finding: f, index: named, entry: entries[named]! });
|
||||
if (nonAllowlistable(f.id)) continue;
|
||||
const named = entries.findIndex(e => e.pattern !== undefined && scoped(e) && (!positioned(f) || matchesLine(e.pattern, f.lineText!)));
|
||||
if (named >= 0) (positioned(f) ? missed : untested).push({ finding: f, index: named, entry: entries[named]! });
|
||||
}
|
||||
return { kept, allowlisted, untested };
|
||||
return { kept, allowlisted, untested, missed };
|
||||
}
|
||||
|
||||
/** sha256 of the entries, for the check's `inputs.allowlist`. */
|
||||
@@ -68,8 +78,13 @@ export function describeUntested(a: Allowlisted): string {
|
||||
return `allowlist[${a.index}] ${a.entry.finding} not tested on ${a.finding.id} (${a.finding.file}): the scanner gave no position for its pattern; the finding is kept`;
|
||||
}
|
||||
|
||||
/** One table line per finding a pattern entry named but did not match: the pattern is tested from where the flagged value starts, never the name before it. */
|
||||
export function describeMissed(a: Allowlisted): string {
|
||||
return `allowlist[${a.index}] ${a.entry.finding} pattern did not match ${a.finding.id} (${a.finding.file}${a.finding.line ? `, ${a.finding.fileLine ? 'line' : 'added line'} ${a.finding.line}` : ''}, col ${a.finding.col}): a pattern is matched from where the flagged value starts, not from the name; the finding is kept`;
|
||||
}
|
||||
|
||||
/** One table line per allowlisted finding: what, where, and which entry (with its reason) covered it. */
|
||||
export function describeAllowlisted(a: Allowlisted): string {
|
||||
const where = a.finding.line ? ` (${a.finding.file}, added line ${a.finding.line})` : ` (${a.finding.file})`;
|
||||
const where = a.finding.line ? ` (${a.finding.file}, ${a.finding.fileLine ? 'line' : 'added line'} ${a.finding.line})` : ` (${a.finding.file})`;
|
||||
return `allowlisted ${a.finding.id}${where} by allowlist[${a.index}] ${a.entry.finding}: ${a.entry.reason}`;
|
||||
}
|
||||
@@ -11,7 +11,7 @@ import { spawnSync } from 'node:child_process';
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import { runScriptSync } from '../../script-invocation';
|
||||
import { allowlistDigest, applyAllowlist, describeAllowlisted, describeUntested, type AllowlistFinding } from '../allowlist';
|
||||
import { allowlistDigest, applyAllowlist, describeAllowlisted, describeMissed, describeUntested, type AllowlistFinding } from '../allowlist';
|
||||
import { baseInputs, sha256 } from '../context';
|
||||
import { environmentResult, launchCause } from '../environment';
|
||||
import { checkScratch } from '../run-scratch';
|
||||
@@ -33,24 +33,69 @@ export function parseAddedLines(diff: string): Map<string, string[]> {
|
||||
return files;
|
||||
}
|
||||
|
||||
/** Each file's added line numbers in the working-tree file, from the hunk headers' `+start`. */
|
||||
export function parseAddedLineNumbers(diff: string): Map<string, number[]> {
|
||||
const files = new Map<string, number[]>();
|
||||
let file = '';
|
||||
let next = 0;
|
||||
for (const line of diff.split('\n')) {
|
||||
if (line.startsWith('diff --')) { file = ''; next = 0; continue; }
|
||||
const hunk = /^@@ -\d+(?:,\d+)? \+(\d+)(?:,\d+)? @@/.exec(line);
|
||||
if (hunk) { next = Number(hunk[1]); continue; }
|
||||
if (!next) { if (line.startsWith('+++ ')) file = line.startsWith('+++ b/') ? line.slice(6) : ''; continue; }
|
||||
if (line.startsWith('+')) { if (!files.has(file)) files.set(file, []); files.get(file)!.push(next++); }
|
||||
else if (line.startsWith(' ')) next++;
|
||||
}
|
||||
return files;
|
||||
}
|
||||
|
||||
/** Files whose source-syntax exemption needs the whole file: a line added inside an existing `'''`/`"""` block is a literal (#3630). */
|
||||
const LEXICAL_CONTEXT_SOURCE = /\.(?:dart|pyi?)$/i;
|
||||
const WHOLE_FILE_MAX_BYTES = 4 * 1024 * 1024;
|
||||
|
||||
/**
|
||||
* Added lines from the merge-base to the working tree, plus every line of
|
||||
* untracked files, grouped by file so gstack-redact can use each file's type
|
||||
* (#3211). External diff and textconv drivers are off: either can empty or
|
||||
* rewrite the diff, and an empty diff reads as clean.
|
||||
* (#3211). A tracked Dart or Python file is passed whole with `keep`, its
|
||||
* added line numbers, so the scanner sees the strings an added line sits in
|
||||
* and only findings on those lines count (#3630); an untracked one keeps every
|
||||
* line. One too large, unreadable, not a regular file, or whose working copy
|
||||
* no longer holds the diff's added lines at their numbers
|
||||
* to read whole is scanned under a neutral name, without that exemption. External diff and textconv
|
||||
* drivers are off: either can empty or rewrite the diff, and an empty diff
|
||||
* reads as clean.
|
||||
*/
|
||||
export function addedLines(ctx: PregateContext): Array<{ file: string; text: string }> {
|
||||
export function addedLines(ctx: PregateContext): Array<{ file: string; text: string; keep?: number[] }> {
|
||||
const diff = spawnSync('git', ['diff', '--no-color', '--no-ext-diff', '--no-textconv', '--src-prefix=a/', '--dst-prefix=b/', '--unified=0', '--diff-filter=ACMR', ctx.mergeBase, '--'], { cwd: ctx.repoRoot, encoding: 'utf8', timeout: 60_000, maxBuffer: 256 * 1024 * 1024, env: ctx.env });
|
||||
const files = parseAddedLines(diff.stdout ?? '');
|
||||
const numbers = parseAddedLineNumbers(diff.stdout ?? '');
|
||||
const whole = new Map<string, { text: string; keep: number[] }>();
|
||||
for (const [file, keep] of numbers) {
|
||||
if (!LEXICAL_CONTEXT_SOURCE.test(file)) continue;
|
||||
try {
|
||||
const st = fs.lstatSync(path.join(ctx.repoRoot, file));
|
||||
if (!st.isFile() || st.size > WHOLE_FILE_MAX_BYTES) continue;
|
||||
const text = fs.readFileSync(path.join(ctx.repoRoot, file), 'utf8');
|
||||
const lines = text.split('\n');
|
||||
const added = files.get(file) ?? [];
|
||||
if (keep.length === added.length && keep.every((n, k) => lines[n - 1] === added[k])) whole.set(file, { text, keep });
|
||||
} catch { /* unreadable: its added lines are scanned alone */ }
|
||||
}
|
||||
for (const t of ctx.touched) {
|
||||
if (!t.uncommitted || t.status === 'D') continue;
|
||||
const tracked = spawnSync('git', ['ls-files', '--error-unmatch', '--', t.path], { cwd: ctx.repoRoot, encoding: 'utf8', timeout: 15_000 }).status === 0;
|
||||
if (tracked) continue;
|
||||
try { const st = fs.statSync(path.join(ctx.repoRoot, t.path)); if (st.isFile() && st.size <= 4 * 1024 * 1024) files.set(t.path, [...(files.get(t.path) ?? []), ...fs.readFileSync(path.join(ctx.repoRoot, t.path), 'utf8').split('\n')]); } catch { /* unreadable: skipped */ }
|
||||
try {
|
||||
const st = fs.statSync(path.join(ctx.repoRoot, t.path));
|
||||
if (!st.isFile() || st.size > WHOLE_FILE_MAX_BYTES) continue;
|
||||
const lines = fs.readFileSync(path.join(ctx.repoRoot, t.path), 'utf8').split('\n');
|
||||
if (LEXICAL_CONTEXT_SOURCE.test(t.path) && !files.has(t.path)) whole.set(t.path, { text: lines.join('\n'), keep: lines.map((_, n) => n + 1) });
|
||||
files.set(t.path, [...(files.get(t.path) ?? []), ...lines]);
|
||||
} catch { /* unreadable: skipped */ }
|
||||
}
|
||||
return [...files].map(([file, lines]) => ({ file, text: lines.join('\n') })).filter(p => p.text.trim());
|
||||
return [...files].map(([file, lines]) => whole.get(file) ? { file, ...whole.get(file)! } : { file, text: lines.join('\n') }).filter(p => p.text.trim());
|
||||
}
|
||||
type Finding = { id?: string; description?: string; severity?: string; category?: string; line?: number; col?: number; file: string };
|
||||
type Finding = { id?: string; description?: string; severity?: string; category?: string; line?: number; col?: number; file: string; fileLine?: boolean };
|
||||
type Scanned = Finding & AllowlistFinding;
|
||||
|
||||
function run(ctx: PregateContext): CheckResult {
|
||||
@@ -67,7 +112,7 @@ function run(ctx: PregateContext): CheckResult {
|
||||
let medium = false;
|
||||
let parsedAll = true;
|
||||
for (const [i, part] of parts.entries()) {
|
||||
const file = path.join(tmp, String(i), path.basename(part.file) || 'added.txt');
|
||||
const file = path.join(tmp, String(i), (!part.keep && LEXICAL_CONTEXT_SOURCE.test(part.file) ? '' : path.basename(part.file)) || 'added.txt');
|
||||
fs.mkdirSync(path.dirname(file), { mode: 0o700 });
|
||||
fs.writeFileSync(file, part.text, { mode: 0o600 });
|
||||
const r = runScriptSync(path.join(BIN_DIR, 'gstack-redact'), ['--from-file', file, '--json', '--max-bytes', String(64 * 1024 * 1024)], { cwd: ctx.repoRoot, encoding: 'utf8', timeout: Math.min(ctx.timeoutMs, 300_000), maxBuffer: 64 * 1024 * 1024, env: ctx.env });
|
||||
@@ -76,21 +121,21 @@ function run(ctx: PregateContext): CheckResult {
|
||||
high ||= r.status === 3;
|
||||
medium ||= r.status === 2;
|
||||
const lines = part.text.split('\n');
|
||||
try { findings.push(...(JSON.parse(r.stdout).findings ?? []).map((f: Omit<Finding, 'file'>) => ({ ...f, id: f.id ?? 'finding', category: f.category ?? '?', severity: f.severity ?? '?', file: part.file, lineText: f.line ? lines[f.line - 1] : undefined }))); } catch { parsedAll = false; }
|
||||
try { findings.push(...(JSON.parse(r.stdout).findings ?? []).filter((f: Omit<Finding, 'file'>) => !part.keep || f.id?.startsWith('engine.') || (f.line !== undefined && part.keep.includes(f.line))).map((f: Omit<Finding, 'file'>) => ({ ...f, id: f.id ?? 'finding', category: f.category ?? '?', severity: f.severity ?? '?', file: part.file, fileLine: !!part.keep, lineText: f.line ? lines[f.line - 1] : undefined }))); } catch { parsedAll = false; }
|
||||
}
|
||||
// The allowlist applies only to findings the scanner reported in full; an unparsed report keeps its exit-code verdict.
|
||||
const applied = parsedAll && ctx.config.allowlist.length ? applyAllowlist(findings, ctx.config.allowlist) : { kept: findings, allowlisted: [], untested: [] };
|
||||
const applied = parsedAll && ctx.config.allowlist.length ? applyAllowlist(findings, ctx.config.allowlist) : { kept: findings, allowlisted: [], untested: [], missed: [] };
|
||||
const kept = applied.kept;
|
||||
if (parsedAll) { high = kept.some(f => f.severity === 'HIGH'); medium = kept.some(f => f.severity === 'MEDIUM'); }
|
||||
const render = (f: Scanned) => `${f.severity} ${f.id} [${f.category}]: ${f.description ?? ''}${f.line ? ` (${f.file ? `${f.file}, ` : ''}added line ${f.line})` : ''}`;
|
||||
const allowLines = [...applied.allowlisted.map(describeAllowlisted), ...applied.untested.map(describeUntested)];
|
||||
const render = (f: Scanned) => `${f.severity} ${f.id} [${f.category}]: ${f.description ?? ''}${f.line ? ` (${f.file ? `${f.file}, ` : ''}${f.fileLine ? 'line' : 'added line'} ${f.line})` : ''}`;
|
||||
const allowLines = [...applied.allowlisted.map(describeAllowlisted), ...applied.untested.map(describeUntested), ...applied.missed.map(describeMissed)];
|
||||
const allowNote = applied.allowlisted.length ? `; ${applied.allowlisted.length} allowlisted (${[...new Set(applied.allowlisted.map(a => a.finding.id))].join(', ')})` : '';
|
||||
// HIGH and any MEDIUM credential fail; a MEDIUM hygiene/internal/pii/legal
|
||||
// hit in code (a `.local` property reads as a hostname) is listed as warn:
|
||||
// there is no user to answer gstack-redact's per-finding question here.
|
||||
const blocking = kept.filter(f => f.severity === 'HIGH' || (f.severity === 'MEDIUM' && f.category === 'secret'));
|
||||
const advisory = kept.filter(f => f.severity === 'MEDIUM' && f.category !== 'secret');
|
||||
const lineCount = parts.reduce((n, p) => n + p.text.split('\n').length, 0);
|
||||
const lineCount = parts.reduce((n, p) => n + (p.keep?.length ?? p.text.split('\n').length), 0);
|
||||
const fail = (lines: string[], detail: string): CheckResult => ({ ...base, status: 'fail', cause: 'diff', lines: [...lines, ...allowLines], code: 'PREGATE_SECRETS', detail: `${detail}${allowNote}`, fix: 'rotate and remove the credential from the added lines (history included); secrets cannot be downgraded (a benign finding is allowlisted by id, path and pattern in the policy, never env.value)' });
|
||||
if (!high && !medium) return { ...base, status: 'pass', ...(allowLines.length ? { lines: allowLines } : {}), detail: `${lineCount} added lines clean${kept.length ? ` (${kept.length} WARN)` : ''}${allowNote}` };
|
||||
if (blocking.length) return fail([...blocking, ...advisory].map(render), `${blocking.length} credential finding(s) in added lines (${blocking.map(f => f.id).join(', ')})`);
|
||||
|
||||
@@ -26,6 +26,7 @@
|
||||
import {
|
||||
PATTERNS,
|
||||
PATTERNS_BY_ID,
|
||||
clearStringSpanCache,
|
||||
isPlaceholderSpan,
|
||||
type RedactPattern,
|
||||
type Tier,
|
||||
@@ -472,6 +473,7 @@ function scanInternal(input: string, opts: ScanOptions, spans?: Map<Finding, Ori
|
||||
}
|
||||
}
|
||||
}
|
||||
clearStringSpanCache();
|
||||
|
||||
// Environment values, by exact value: a credential that is already in the
|
||||
// process environment needs no shape to be recognized.
|
||||
|
||||
+122
-14
@@ -121,7 +121,7 @@ export function shannonEntropy(s: string): number {
|
||||
// (apiKey, authToken, clientSecret, stripeApiKey) — cacheKey/sortKey/
|
||||
// monkey have no credential prefix and are rejected.
|
||||
const ENV_KV_NAME =
|
||||
/^[ \t]*(?:export[ \t]+)?["']?([A-Za-z0-9_.-]*?(?:KEY|TOKEN|SECRET|PASSWORD|PASSWD|CREDENTIALS?|DSN|AUTH|COOKIE|SESSION|PRIVATE))["']?[ \t]*[:=]/i;
|
||||
/^(?:[ +-]?\+|[ +-]?-(?! [^ \t]))?[ \t]*(?:export[ \t]+)?["']?([A-Za-z0-9_.-]*?(?:KEY|TOKEN|SECRET|PASSWORD|PASSWD|CREDENTIALS?|DSN|AUTH|COOKIE|SESSION|PRIVATE))["']?[ \t]*[:=]/i;
|
||||
const ENV_KV_SUFFIX =
|
||||
/(KEY|TOKEN|SECRET|PASSWORD|PASSWD|CREDENTIALS?|DSN|AUTH|COOKIE|SESSION|PRIVATE)$/i;
|
||||
const ENV_KV_CRED_PREFIX =
|
||||
@@ -656,7 +656,7 @@ const CALL_SHAPED_VALUE = /^[A-Za-z_$][\w$]*(?:\.[A-Za-z_$][\w$]*)*\(/;
|
||||
* A random key is mixed-case or one unbroken run, so it never takes this shape.
|
||||
*/
|
||||
const SCHEMA_FIELD_NAME =
|
||||
/^[ \t]*["']?(?:input|output|field|column|lookup|cache|sort|partition|range|index|row|group|join|property|metric|catalog|schema|translation|idempotency|dedup)[_.-]key["']?[ \t]*[:=]/;
|
||||
/^(?:[ +-]?\+|[ +-]?-(?! [^ \t]))?[ \t]*["']?(?:input|output|field|column|lookup|cache|sort|partition|range|index|row|group|join|property|metric|catalog|schema|translation|idempotency|dedup)[_.-]key["']?[ \t]*[:=]/;
|
||||
const SCHEMA_FIELD_VALUE = /^[a-z][a-z0-9]*(?:[_.-][a-z][a-z0-9]*)+$/;
|
||||
function isSchemaFieldIdentifier(span: string, match: RegExpExecArray): boolean {
|
||||
return SCHEMA_FIELD_NAME.test(match[0]) && SCHEMA_FIELD_VALUE.test(span);
|
||||
@@ -673,28 +673,127 @@ function isCallExpression(span: string, match: RegExpExecArray): boolean {
|
||||
/**
|
||||
* #3048: in a TypeScript/JSX file, an unquoted value after `:` is a type or an
|
||||
* expression, never a string literal (`session: SessionState,`), and a JSX
|
||||
* brace holding only names and property reads (`key={turn.requestId + turn.role}`)
|
||||
* is an expression. Decided by file context, not by how the value looks:
|
||||
* brace holding only names and property reads joined by + ?? || or &&
|
||||
* (`key={turn.requestId + turn.role}`, `key={item.id ?? index}`, #3605) is an
|
||||
* expression. Decided by file context, not by how the value looks:
|
||||
* `API_KEY=VelvetRiverOrbitSunset;` in a .env or YAML file is still a literal.
|
||||
* Without a known TS/JSX path nothing is exempt.
|
||||
* #3630 (#3418, #3419): Dart and Python (.py, .pyi) qualify on the same grammar
|
||||
* fact, no unquoted string literal, so `'password': request.password` and
|
||||
* `session: HttpSession,` are names. Python's `=` forms
|
||||
* (`password=request.password`) still fire: the separator must be `:`.
|
||||
* Without a known source path nothing is exempt.
|
||||
*/
|
||||
const TS_SOURCE = /\.(?:[cm]?tsx?|jsx)$/i;
|
||||
const CODE_SOURCE = /\.(?:[cm]?tsx?|jsx|dart|pyi?)$/i;
|
||||
const CODE_NAME_CHAIN = /^[A-Za-z_$][\w$]*(?:\??\.[A-Za-z_$][\w$]*)*(?:<[\w$.,\s<>[\]|]*>)?(?:\[\])*$/;
|
||||
|
||||
/**
|
||||
* #3418: a `password:` pair inside a string is text in an embedded config,
|
||||
* not a dict entry or annotation, so the name exemption must not reach it.
|
||||
* In Python and Dart (no regex literals, so every quote is a string's), one
|
||||
* linear pass per input (a scan validates many candidates against the same
|
||||
* text) records every string's content span: a `'''`/`"""` block, or a
|
||||
* `'`/`"` string that ends at its line unless a backslash continues it.
|
||||
* Comments are skipped (`#` in Python; `//` and nested `/* *\/` in Dart), so a
|
||||
* delimiter inside one opens nothing; a backslash escapes the next character, except in a Dart raw
|
||||
* string (`r'''…'''`). The exemption needs the
|
||||
* whole file to see a block's opener: pregate passes Dart and Python files
|
||||
* whole, and the pre-push hook, which scans added lines alone, passes them
|
||||
* without their path.
|
||||
*/
|
||||
const STRING_SPAN_SOURCE = /\.(?:dart|pyi?)$/i;
|
||||
let stringSpansCache: { text: string; lang: string; spans: Array<[number, number]> } | null = null;
|
||||
|
||||
/** Drops the cached text once a scan ends, so a long-lived process holds no scanned input. */
|
||||
export function clearStringSpanCache(): void {
|
||||
stringSpansCache = null;
|
||||
}
|
||||
|
||||
function stringSpans(text: string, lang: "py" | "dart"): Array<[number, number]> {
|
||||
if (stringSpansCache?.text === text && stringSpansCache.lang === lang) return stringSpansCache.spans;
|
||||
const spans: Array<[number, number]> = [];
|
||||
const n = text.length;
|
||||
for (let i = 0; i < n; ) {
|
||||
const c = text[i]!;
|
||||
if (lang === "py" ? c === "#" : text.startsWith("//", i)) {
|
||||
const eol = text.indexOf("\n", i);
|
||||
i = eol < 0 ? n : eol;
|
||||
continue;
|
||||
}
|
||||
if (lang === "dart" && text.startsWith("/*", i)) {
|
||||
let depth = 1;
|
||||
for (i += 2; i < n && depth > 0; ) {
|
||||
if (text.startsWith("/*", i)) { depth += 1; i += 2; }
|
||||
else if (text.startsWith("*/", i)) { depth -= 1; i += 2; }
|
||||
else i += 1;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if (c !== '"' && c !== "'") { i += 1; continue; }
|
||||
const close = text.startsWith(c.repeat(3), i) ? c.repeat(3) : c;
|
||||
const multiline = close.length === 3;
|
||||
const raw = lang === "dart" && text[i - 1] === "r" && !/[\w$]/.test(text[i - 2] ?? "");
|
||||
let j = i + close.length;
|
||||
while (j < n && !text.startsWith(close, j) && (multiline || text[j] !== "\n")) j += !raw && text[j] === "\\" ? 2 : 1;
|
||||
spans.push([i + close.length, Math.min(j, n)]);
|
||||
i = j + close.length;
|
||||
}
|
||||
stringSpansCache = { text, lang, spans };
|
||||
return spans;
|
||||
}
|
||||
|
||||
function insideString(text: string, offset: number, sourcePath: string): boolean {
|
||||
if (!STRING_SPAN_SOURCE.test(sourcePath)) return false;
|
||||
const spans = stringSpans(text, /\.dart$/i.test(sourcePath) ? "dart" : "py");
|
||||
let lo = 0;
|
||||
let hi = spans.length - 1;
|
||||
while (lo <= hi) {
|
||||
const mid = (lo + hi) >> 1;
|
||||
if (offset < spans[mid]![0]) hi = mid - 1;
|
||||
else if (offset >= spans[mid]![1]) lo = mid + 1;
|
||||
else return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/** The rest of a name's expression on its line, up to the comma, semicolon, bracket or comment that ends it; null when it continues on the next line. */
|
||||
function expressionTail(line: string, comment: string): string | null {
|
||||
let rest = line;
|
||||
let depth = 0;
|
||||
for (let i = 0; i < rest.length; i += 1) {
|
||||
const c = rest[i]!;
|
||||
if (rest.startsWith(comment, i)) { rest = rest.slice(0, i); break; }
|
||||
if (c === '"' || c === "'" || c === "`") {
|
||||
const close = rest.indexOf(c, i + 1);
|
||||
if (close < 0) return null;
|
||||
i = close;
|
||||
} else if ("([{".includes(c)) depth += 1;
|
||||
else if (")]}".includes(c)) { if (--depth < 0) return rest.slice(0, i); }
|
||||
else if ((c === "," || c === ";") && depth === 0) return rest.slice(0, i);
|
||||
}
|
||||
return depth > 0 || /(?:[=+\\]|\?\?|\|\||&&|\b(?:or|and))\s*$/.test(rest) ? null : rest;
|
||||
}
|
||||
|
||||
function isSourceExpression(span: string, match: RegExpExecArray, sourcePath: string | undefined): boolean {
|
||||
if (!sourcePath || !TS_SOURCE.test(sourcePath)) return false;
|
||||
const { start } = spanBounds(match);
|
||||
if (!sourcePath || !CODE_SOURCE.test(sourcePath)) return false;
|
||||
const { start, end } = spanBounds(match);
|
||||
const before = match.input[start - 1];
|
||||
if (before === '"' || before === "'" || before === "`") return false;
|
||||
if (span.startsWith("{")) {
|
||||
const line = match.input.slice(start + 1).split("\n", 1)[0];
|
||||
const close = line.indexOf("}");
|
||||
if (close < 0 || /["'`]/.test(line.slice(0, close))) return false;
|
||||
return line.slice(0, close).split("+").every(term => CODE_NAME_CHAIN.test(term.trim()));
|
||||
if (!line.slice(0, close).split(/\+|\?\?|\|\||&&/).every(term => CODE_NAME_CHAIN.test(term.trim()))) return false;
|
||||
} else {
|
||||
const head = match[0].slice(0, match[0].length - span.length).trimEnd();
|
||||
const separator = head.slice(-1);
|
||||
if (separator !== ":" || !CODE_NAME_CHAIN.test(span.replace(/[,;)=|]+$/, ""))) return false;
|
||||
const rest = (span.endsWith("=") ? "=" : "") + match.input.slice(end).split("\n", 1)[0];
|
||||
if (!/[,;]$/.test(span)) {
|
||||
const tail = expressionTail(rest, /\.pyi?$/i.test(sourcePath) ? "#" : "//");
|
||||
if (tail === null || carriesSecretLiteral(tail)) return false;
|
||||
}
|
||||
}
|
||||
const separator = match[0].slice(0, match[0].length - span.length).trimEnd().slice(-1);
|
||||
if (separator !== ":") return false;
|
||||
return CODE_NAME_CHAIN.test(span.replace(/[,;)=|]+$/, ""));
|
||||
return !insideString(match.input, start, sourcePath);
|
||||
}
|
||||
|
||||
export const PATTERNS: RedactPattern[] = [
|
||||
@@ -962,7 +1061,16 @@ export const PATTERNS: RedactPattern[] = [
|
||||
// (cacheKey:, sortKey:, monkey:) fires a MEDIUM confirm on entropic
|
||||
// values. The value must stay capture group 1 (the engine masks group 1),
|
||||
// so name-shape checking lives in validate, not in a second group.
|
||||
regex: /^[ \t]*(?:export[ \t]+)?["']?[A-Za-z0-9_.-]*(?:KEY|TOKEN|SECRET|PASSWORD|PASSWD|CREDENTIALS?|DSN|AUTH|COOKIE|SESSION|PRIVATE)["']?[ \t]*[:=][ \t]*["']?([^\s'"]{8,})["']?/i,
|
||||
// GST-543: an optional leading `+` or `-` is a unified-diff line marker
|
||||
// (`git diff | gstack-redact`, a diff fence in a PR body), and one more
|
||||
// column before it (`++`, ` +`, `+-`) is a combined (merge) diff. Without
|
||||
// them the line anchor stopped at the marker, so an added or removed credential
|
||||
// went unreported (`-` alone matched only as a name character, never
|
||||
// before indentation or `export`). A `-` followed by exactly one space
|
||||
// and text is a Markdown bullet or YAML list item, not a marker, so those
|
||||
// read as they did before. ENV_KV_NAME and SCHEMA_FIELD_NAME take the same
|
||||
// marker so the name-shape and schema exemptions hold on a diff line.
|
||||
regex: /^(?:[ +-]?\+|[ +-]?-(?! [^ \t]))?[ \t]*(?:export[ \t]+)?["']?[A-Za-z0-9_.-]*(?:KEY|TOKEN|SECRET|PASSWORD|PASSWD|CREDENTIALS?|DSN|AUTH|COOKIE|SESSION|PRIVATE)["']?[ \t]*[:=][ \t]*["']?([^\s'"]{8,})["']?/i,
|
||||
// Only fire on credential-shaped names with high-entropy values — kills
|
||||
// `FOO_KEY=changeme` and `cacheKey: <entropic-id>` FPs. #2912: a value
|
||||
// that is exactly an environment read (`os.environ["X"]`,
|
||||
@@ -971,7 +1079,7 @@ export const PATTERNS: RedactPattern[] = [
|
||||
// literal (`os.getenv("X", "<secret>")`). A literal appended to the read
|
||||
// itself (`process.env.X||"…"`) is not an exact read and still fires.
|
||||
// #2899: a function call assigned to the name is code, not a value (see
|
||||
// isCallExpression). #3048: so is a TS/JSX type or expression (isSourceExpression).
|
||||
// isCallExpression). #3048, #3630: so is a TS/JSX/Dart/Python type or expression (isSourceExpression).
|
||||
// #3208: so is a schema field identifier (isSchemaFieldIdentifier).
|
||||
validate: (span, match, opts) =>
|
||||
isCredentialShapedEnvName(match[0]) &&
|
||||
|
||||
@@ -25,6 +25,11 @@ export const EVAL_RESULT_CODES = {
|
||||
summary: 'the Comparison says stack: loaded and the No-gstack arm section is missing, lacks a key, or does not give every model its own <model>#no-gstack Arms line',
|
||||
fix: 'add `## No-gstack arm` with arms:, inputs:, prompt:, panel: and report:, and list each <model>#no-gstack arm under Arms; when no arm loads a skill stack, say stack: none with a why: line',
|
||||
},
|
||||
EVAL_REPORT_NO_GSTACK_ROW: {
|
||||
anchor: 'eval-report-no-gstack-row',
|
||||
summary: 'the results file has no single table with a filled row for every arm the preregistration lists, so a no-gstack row is missing or not beside its variants',
|
||||
fix: 'put one row per arm identity (each <model>#no-gstack beside its variants) in one markdown table of the results file, then rerun `gstack-eval-plan results --check`; publish no number before it says ok',
|
||||
},
|
||||
EVAL_ARM_DUPLICATE: {
|
||||
anchor: 'eval-arm-duplicate',
|
||||
summary: 'the pilot was given the same arm identity twice, so two arms would share one set of attempts',
|
||||
|
||||
@@ -137,6 +137,41 @@ export const RELEASE_RESULT_CODES = {
|
||||
summary: 'every slot of this train already merged on GitHub and main is a prefix another train verified, so the record is stale; comparing today’s main to its prefixes proves nothing and does not freeze the queue',
|
||||
fix: 'settle the record with train prune --stale --train <nonce>; no backstop is needed',
|
||||
},
|
||||
QUEUE_TRAIN_BASE_RED: {
|
||||
anchor: 'queue-train-base-red',
|
||||
summary: 'a conditional (base-red) PR or its fixer cannot ride this train: the train has no gates list, the fixer is not slot 1 or not in the train, two PRs declare FIXES_BASE_RED, the condition expired, or the fixer\'s own prefix is red',
|
||||
fix: 'seat the fixer (its READY declares FIXES_BASE_RED) at slot 1 of a train whose policy declares gates and build again; a red fixer prefix is the fixer lane\'s to repair, and an expired condition is reclassified by its lane',
|
||||
},
|
||||
QUEUE_TRAIN_SPEND_CAP: {
|
||||
anchor: 'queue-train-spend-cap',
|
||||
summary: 'a command gate\'s next Ubicloud run would pass the one spend cap that covers every run of the train, or UBI_OWNER is not set, so nothing more ran',
|
||||
fix: 'set UBI_OWNER to your thread code and train build --train <nonce> runs what is left; for the cap, raise the gate\'s spend_cap_usd (or lower its timeout_sec) on the base branch, then build a new train on the new main (the policy commit moves main past T0)',
|
||||
},
|
||||
QUEUE_TRAIN_LAND_METHOD: {
|
||||
anchor: 'queue-train-land-method',
|
||||
summary: 'train land must publish this slot through a tool-made update commit (a stamped train, strict branch protection, a conditional PR, or a merge that resolved release files), and the policy\'s merge method is rebase, which would replay commits instead of landing the gated tree',
|
||||
fix: 'set train.land.merge_method to squash or merge on the base branch (history merge-only allows merge only), then build a new train on the new main (the policy commit moves main off this train\'s prefixes)',
|
||||
},
|
||||
QUEUE_TRAIN_INIT_REFUSED: {
|
||||
anchor: 'queue-train-init-refused',
|
||||
summary: 'train init refuses this repository: its open PRs edit version files and the policy neither declares changelog_fragments nor stamps at merge, so every train merge would conflict on the version',
|
||||
fix: 'adopt changelog_fragments (recommended) or queue_mode stamp-at-merge on the base branch first, then run train init again',
|
||||
},
|
||||
QUEUE_TRAIN_DOCTOR_FINDINGS: {
|
||||
anchor: 'queue-train-doctor-findings',
|
||||
summary: 'train doctor found the live train block out of step with the repository: a gate workflow missing, without the nonce input or run-name, a required job it does not define, a global concurrency group, a push trigger on train-prefix/ refs, a workflow never green on the base, or a merge method GitHub refuses',
|
||||
fix: 'fix each FINDING line on the base branch (train init prints the workflow snippet), then run train doctor again',
|
||||
},
|
||||
QUEUE_BASE_RED_NO_TRAIN: {
|
||||
anchor: 'queue-base-red-no-train',
|
||||
summary: 'a conditional READY (ready --base-red) needs a train that gates every prefix, and the base policy has no train block with gates',
|
||||
fix: 'wait for the base to pass the cases and run /ship --resume (the normal READY follows), or turn on a train with gates on the base branch',
|
||||
},
|
||||
SHIP_BASE_RED_FIXER: {
|
||||
anchor: 'ship-base-red-fixer',
|
||||
summary: 'ready --base-red names no fixer PR, or the receipt\'s base-red condition no longer matches the base: a conditional READY needs an owner for its condition and a current classification',
|
||||
fix: 'pass --fixer <pr> naming the open PR whose merge fixes the cases; when the base moved, rerun gstack-base-red classify (and /ship) first',
|
||||
},
|
||||
QUEUE_TRAIN_FROZEN: {
|
||||
anchor: 'queue-train-frozen',
|
||||
summary: 'a train stopped partway through publication, so main holds a prefix no gate covered on its own; ordinary trains are refused until a repair lands',
|
||||
@@ -156,7 +191,7 @@ export const RELEASE_RESULT_CODES = {
|
||||
QUEUE_RECEIPT_CONDITIONAL: {
|
||||
anchor: 'queue-receipt-conditional',
|
||||
summary: 'the PR body’s /ship receipt is conditional: Step 5 failed on cases gstack-base-red proved are the base branch’s own, so /ship paused before READY',
|
||||
fix: 'wait for the base to pass those cases (gstack-base-red wait --pause <file>), then /ship --resume <record>, which re-runs Step 5 and rewrites the receipt without base_red; nothing READYs, trains or merges on a conditional receipt',
|
||||
fix: 'wait for the base to pass those cases (gstack-base-red wait --pause <file>), then /ship --resume <record>, which re-runs Step 5 and rewrites the receipt without base_red; only ready --base-red --fixer <pr> READYs a conditional receipt, and it rides only a train with gates whose slot 1 is its fixer',
|
||||
},
|
||||
BASE_RED_UNCONFIGURED: {
|
||||
anchor: 'base-red-unconfigured',
|
||||
|
||||
+1
-1
@@ -589,7 +589,7 @@ export const RESULT_CODES = {
|
||||
// Field wave lane A (items 1-4): gstack-upgrade, the Capy skill bootstrapper, install admission.
|
||||
UPGRADE_REFUSED: {
|
||||
anchor: 'upgrade-refused',
|
||||
summary: 'bin/gstack-upgrade changed nothing: the checkout is dirty, diverged, pinned (detached or off the remote default branch), not a gstack checkout, or the incoming release holds',
|
||||
summary: 'bin/gstack-upgrade changed nothing: the checkout is dirty, diverged, pinned (detached or off the remote default branch), not a gstack checkout, tracked by an enclosing project (#1022), or the incoming release holds',
|
||||
fix: 'resolve the named cause (commit or move the listed changes, check out the default branch), then re-run bin/gstack-upgrade; it never stashes or resets',
|
||||
},
|
||||
UPGRADE_BUSY: {
|
||||
|
||||
+4
-3
@@ -77,7 +77,7 @@ const STAGE_OF: Record<string, Stage> = { specialist: 'specialists', 'red-team':
|
||||
export function buildManifest(o: {
|
||||
pass: string; out: string; ledger: string; base: string; diffBase: string; root: string; checklistRoot: string;
|
||||
dispatches: Dispatch[]; next: string; planLines: string[]; learnings?: string | null; extras?: Extra[];
|
||||
targetedClasses?: readonly string[]; mergeBase?: string;
|
||||
targetedClasses?: readonly string[]; mergeBase?: string; rangeBase?: string;
|
||||
}): Manifest {
|
||||
const readers: PassReader[] = [];
|
||||
for (const d of o.dispatches) {
|
||||
@@ -88,7 +88,8 @@ export function buildManifest(o: {
|
||||
const reader = readerKey(role, name);
|
||||
const exclude = d.exclude.length ? diffCommand(d).slice('git diff <diff-base> '.length) : undefined;
|
||||
readers.push({ reader, stage, title: `ship ${o.pass} ${reader}`, state: 'planned', role, name, effort: d.effort, exclude, checklists: d.checklists,
|
||||
...(role === 'targeted' ? { classes: [...(o.targetedClasses ?? [])], diff_base: o.mergeBase ?? o.diffBase } : {}) });
|
||||
...(role === 'targeted' ? { classes: [...(o.targetedClasses ?? [])], diff_base: o.mergeBase ?? o.diffBase } : {}),
|
||||
...(role === 'red-team' && o.rangeBase ? { diff_base: o.rangeBase } : {}) });
|
||||
}
|
||||
for (const e of o.extras ?? []) {
|
||||
if (!/^[a-z0-9-]+$/.test(e.name)) throw new BriefError(`an audit reader's name is lowercase letters, digits and dashes: ${e.name}`);
|
||||
@@ -148,7 +149,7 @@ export function renderStage(m: Manifest, stage: Stage, t: SectionTexts, diffFor:
|
||||
continue;
|
||||
}
|
||||
const role = r.role!;
|
||||
const slots: Slots = { name: r.name, diff_base: m.diff_base, exclude: r.exclude, stack: m.stack, test_fw: m.test_fw, learnings: m.learnings ?? undefined,
|
||||
const slots: Slots = { name: r.name, diff_base: r.diff_base ?? m.diff_base, exclude: r.exclude, stack: m.stack, test_fw: m.test_fw, learnings: m.learnings ?? undefined,
|
||||
checklists: r.checklists?.map(n => path.join(m.checklist_root, 'review/specialists', `${n}.md`)).join(',') };
|
||||
if (role === 'specialist') slots.checklist = path.join(m.checklist_root, 'review/specialists', `${r.name}.md`);
|
||||
let findingsSha: string | undefined;
|
||||
|
||||
+126
-10
@@ -100,6 +100,44 @@ export interface TrainPolicy {
|
||||
screen_selection: 'gstack-paid' | 'none';
|
||||
/** Bisect admission price of one case trial that has no recorded cost. */
|
||||
trial_reservation_usd: number;
|
||||
/**
|
||||
* The repo's own gates (docs/ship-policy.md#run-a-merge-train): every entry must be GREEN on every prefix T0..Tk.
|
||||
* A block with gates declares only gates, land, max_size and fill_hours; without gates the block is the gstack-evals
|
||||
* kind above, and the six gstack-evals fields above are absent on a block with gates (read them only when gates is absent).
|
||||
*/
|
||||
gates?: TrainGate[];
|
||||
/** How `train land` merges a slot; absent means squash (gstack-evals trains only). */
|
||||
land?: TrainLand;
|
||||
}
|
||||
|
||||
/** One generic train gate: the repo's CI workflows dispatched by nonce on each prefix, or a command run on a Ubicloud VM. */
|
||||
export type TrainGate = WorkflowsGate | CommandGate;
|
||||
export interface WorkflowsGate {
|
||||
kind: 'workflows';
|
||||
/** Workflow files dispatched on each prefix ref; each accepts a `nonce` input and names it in run-name as `[eval-bg <nonce>]`. */
|
||||
workflows: string[];
|
||||
/** `<workflow file>/<job name as GitHub reports it>`: each must conclude success (skipped or missing is INCOMPLETE). */
|
||||
required_jobs: string[];
|
||||
/** Artifact with JUnit XML or a JSON test list; when set it must exist and report at least one executed test. */
|
||||
record?: string;
|
||||
/** Check runs `train land` waits on at the merged commit (default none). */
|
||||
post_merge_checks?: string[];
|
||||
}
|
||||
export interface CommandGate {
|
||||
kind: 'command';
|
||||
/** Run on a Ubicloud VM (scripts/ubicloud/ubi-runner.sh) from a clean checkout of the prefix head; exit 0 is GREEN. */
|
||||
command: string;
|
||||
timeout_sec: number;
|
||||
/** One cap over every prefix run of one train, at the VM's list price. */
|
||||
spend_cap_usd: number;
|
||||
/** The only local variables forwarded to the VM. */
|
||||
pass_env?: string[];
|
||||
}
|
||||
export type MergeMethod = 'squash' | 'merge' | 'rebase';
|
||||
export interface TrainLand {
|
||||
merge_method: MergeMethod;
|
||||
/** Checks `train land` requires green on the landing commit, for repos without changelog_fragments (whose required_checks are the one list). */
|
||||
required_checks?: string[];
|
||||
}
|
||||
|
||||
export const DEFAULT_POLICY: ShipPolicy = {
|
||||
@@ -154,7 +192,7 @@ export const POLICY_EXPLANATIONS: Record<PolicyKey, string> = {
|
||||
migrations: '{ dir, id }: the migration directory and a regex with one capture group for the number in a file name; `gstack-pregate regate` reports MIGRATION_COLLISION against the base branch and every open PR.',
|
||||
gate_command: 'The repo’s full pre-merge gate command. gstack-ship-queue gate-ahead and ready print it for the caller to run (in the background or in CI); gstack never executes it. null when the repo has none.',
|
||||
full_e2e: 'true: every merge needs a passing full-gate result (gstack-ship-queue gate-ahead --record, or the lane\'s --gate-exit 0 --gate-log on the gated head) bound to a tree equal to the merging one modulo stamps; ready, stamp and verify refuse without one (QUEUE_GATE_MISSING).',
|
||||
train: '{ gate_workflow, free_workflow, full_gate_check, record_artifact, max_size, fill_hours, screen_selection, trial_reservation_usd } or null: turns on merge trains in gstack-ship-queue (train build|status|verdict|bisect|rebuild|prune, ready --screen|--no-screen, verify --train). null keeps today\'s per-PR queue unchanged.',
|
||||
train: '{ gate_workflow, free_workflow, full_gate_check, record_artifact, max_size, fill_hours, screen_selection, trial_reservation_usd, land? } (the gstack-evals gate) or { gates, land, max_size, fill_hours } (any repo: gates lists { kind: workflows, workflows, required_jobs, record?, post_merge_checks? } and { kind: command, command, timeout_sec, spend_cap_usd, pass_env? }, each GREEN on every prefix; land is { merge_method: squash|merge|rebase, required_checks? }) or null: turns on merge trains in gstack-ship-queue (train build|status|verdict|land|watch|init|doctor|prune, ready --screen|--no-screen, verify --train). gates and land need min_gstack_version at the release that reads them (validate names it). null keeps today\'s per-PR queue unchanged.',
|
||||
docs: '{ caps: [{ path, max_bytes }] } or null: documentation files with a byte cap. /ship Step 14.5 runs `gstack-ship-receipt docs-caps` before the verification gate and prints one DOCS_CAP: line per cap; a file over its cap (or absent) blocks the ship until the doc is trimmed or the cap raised on the base branch. null declares none.',
|
||||
base_red: '{ setup, select, timeout_sec } or null: how /ship Step 5 proves a failure is the base branch\'s own (docs/ship-policy.md#base-red). gstack-base-red checks out the base head and the PR head in scratch worktrees, runs setup (dependency install, or null), then select with the failing test files in its {files} slot; only the same failures in both, in files the PR does not touch, are base-red, and /ship then pauses before READY instead of failing. Read from the base branch only. null keeps every failure in-branch.',
|
||||
min_gstack_version: 'The oldest gstack that may read this file. Any key newer than 1.91.98.0 set to a non-default value requires it at that key’s introducing version (validate prints each); a gstack older than it refuses the file (POLICY_GSTACK_TOO_OLD).',
|
||||
@@ -191,6 +229,8 @@ export const CHANGELOG_FRAGMENTS_KEY_VERSION = '1.91.336.0';
|
||||
/** `bump: never` is a value of an older key, so the per-key floor check misses it; compatErrors checks it by value. */
|
||||
export const BUMP_NEVER_KEY_VERSION = '1.91.353.0';
|
||||
export const BASE_RED_KEY_VERSION = '1.91.374.0';
|
||||
/** `train.gates` and `train.land` are values of the older train key, so compatErrors checks them by value, as for bump: never. */
|
||||
export const TRAIN_GATES_KEY_VERSION = '1.92.0.0';
|
||||
export const POLICY_KEY_INTRODUCED: Record<PolicyKey, string> = {
|
||||
bump: '1.91.73.0', queue_mode: '1.91.73.0', release_tool: '1.91.73.0', release_outputs: '1.91.73.0', stamp_paths: '1.91.73.0',
|
||||
changelog: '1.91.73.0', mirror: '1.91.73.0', history: '1.91.73.0', preregistration_shas: '1.91.73.0', stamp_sensitive_gates: '1.91.73.0',
|
||||
@@ -211,14 +251,14 @@ const newer = (a: string, b: string): boolean => cmpVersion(parseVersion(a)!, pa
|
||||
export function runningGstackVersion(): string {
|
||||
let own = '0.0.0.0';
|
||||
try { own = fs.readFileSync(path.resolve(import.meta.dir, '..', 'VERSION'), 'utf8').trim(); } catch {}
|
||||
const known = [...Object.values(POLICY_KEY_INTRODUCED), BUMP_NEVER_KEY_VERSION].reduce((a, b) => (newer(b, a) ? b : a));
|
||||
const known = [...Object.values(POLICY_KEY_INTRODUCED), BUMP_NEVER_KEY_VERSION, TRAIN_GATES_KEY_VERSION].reduce((a, b) => (newer(b, a) ? b : a));
|
||||
return parseVersion(own) && !newer(known, own) ? own : known;
|
||||
}
|
||||
|
||||
/** The newest introducing version among the keys a parsed policy sets: the gstack a crew needs to read it. */
|
||||
export function minimumReader(raw: Record<string, unknown>): string {
|
||||
return Object.keys(raw).filter((k): k is PolicyKey => POLICY_KEYS.includes(k as PolicyKey))
|
||||
.map(k => (k === 'bump' && raw.bump === 'never' ? BUMP_NEVER_KEY_VERSION : POLICY_KEY_INTRODUCED[k])).reduce((a, b) => (newer(b, a) ? b : a), POLICY_KEY_INTRODUCED.bump);
|
||||
.map(k => (k === 'bump' && raw.bump === 'never' ? BUMP_NEVER_KEY_VERSION : k === 'train' && genericTrainKeys(raw.train) ? TRAIN_GATES_KEY_VERSION : POLICY_KEY_INTRODUCED[k])).reduce((a, b) => (newer(b, a) ? b : a), POLICY_KEY_INTRODUCED.bump);
|
||||
}
|
||||
|
||||
export interface PolicyError { code: ResultCodeName; path: string; message: string }
|
||||
@@ -311,7 +351,7 @@ function migrationErrors(value: unknown, out: PolicyError[]): void {
|
||||
} catch (e: any) { out.push({ code: 'POLICY_INVALID', path: 'migrations.id', message: `not a valid regex: ${e.message}` }); }
|
||||
}
|
||||
|
||||
const TRAIN_FIELDS: Record<keyof TrainPolicy, (v: unknown) => string | null> = {
|
||||
const TRAIN_FIELDS: Record<Exclude<keyof TrainPolicy, 'gates' | 'land'>, (v: unknown) => string | null> = {
|
||||
gate_workflow: v => (typeof v === 'string' && /^[A-Za-z0-9_.-]{1,64}\.ya?ml$/.test(v) ? null : 'must be a workflow file name such as evals.yml'),
|
||||
free_workflow: v => (typeof v === 'string' && /^[A-Za-z0-9_.-]{1,64}\.ya?ml$/.test(v) ? null : 'must be a workflow file name such as free-tests.yml'),
|
||||
full_gate_check: v => (typeof v === 'string' && /^[A-Za-z0-9_./-]{1,64}$/.test(v) ? null : 'must be a check-run name ([A-Za-z0-9_./-]{1,64})'),
|
||||
@@ -322,15 +362,88 @@ const TRAIN_FIELDS: Record<keyof TrainPolicy, (v: unknown) => string | null> = {
|
||||
trial_reservation_usd: v => (typeof v === 'number' && v > 0 && v <= 100 ? null : 'must be dollars above 0 and at most 100'),
|
||||
};
|
||||
|
||||
function trainErrors(value: unknown, out: PolicyError[]): void {
|
||||
const WORKFLOW_FILE = /^[A-Za-z0-9_.-]{1,64}\.ya?ml$/;
|
||||
const CHECK_NAME = /^[^\n]{1,100}$/;
|
||||
const GENERIC_TRAIN_KEYS = ['gates', 'land', 'max_size', 'fill_hours'];
|
||||
|
||||
/** Whether a raw train block uses the generic keys (gates or land), which need TRAIN_GATES_KEY_VERSION. */
|
||||
export function genericTrainKeys(train: unknown): boolean {
|
||||
return !!train && typeof train === 'object' && !Array.isArray(train) && ('gates' in train || 'land' in train);
|
||||
}
|
||||
|
||||
function nameList(at: string, v: unknown, max: number, item: RegExp, out: PolicyError[], what: string): v is string[] {
|
||||
if (!Array.isArray(v) || !v.length || v.length > max || new Set(v).size !== v.length || v.some(x => typeof x !== 'string' || !item.test(x))) {
|
||||
out.push({ code: 'POLICY_INVALID', path: at, message: `${JSON.stringify(v)} must be 1-${max} distinct ${what}` });
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
function workflowsGateErrors(o: Record<string, unknown>, at: string, out: PolicyError[]): void {
|
||||
const wfs = nameList(`${at}.workflows`, o.workflows, 8, WORKFLOW_FILE, out, 'workflow file names such as ci.yml') ? o.workflows as string[] : [];
|
||||
if (nameList(`${at}.required_jobs`, o.required_jobs, 64, /^[A-Za-z0-9_.-]{1,64}\.ya?ml\/[^\n]{1,100}$/, out, '<workflow file>/<job name> entries')) {
|
||||
for (const job of o.required_jobs as string[]) {
|
||||
const wf = job.slice(0, job.indexOf('/'));
|
||||
if (wfs.length && !wfs.includes(wf)) out.push({ code: 'POLICY_INVALID', path: `${at}.required_jobs`, message: `${JSON.stringify(job)} names ${wf}, which is not in workflows` });
|
||||
}
|
||||
for (const wf of wfs) if (!(o.required_jobs as string[]).some(j => j.startsWith(`${wf}/`))) out.push({ code: 'POLICY_INVALID', path: `${at}.required_jobs`, message: `no required job of ${wf}; a dispatched workflow with no required job proves nothing` });
|
||||
}
|
||||
if ('record' in o && (typeof o.record !== 'string' || !/^[A-Za-z0-9_.-]{1,64}$/.test(o.record))) out.push({ code: 'POLICY_INVALID', path: `${at}.record`, message: 'must be an artifact name ([A-Za-z0-9_.-]{1,64})' });
|
||||
if ('post_merge_checks' in o) nameList(`${at}.post_merge_checks`, o.post_merge_checks, 20, CHECK_NAME, out, 'check names of 1-100 characters');
|
||||
}
|
||||
|
||||
function commandGateErrors(o: Record<string, unknown>, at: string, out: PolicyError[]): void {
|
||||
if (typeof o.command !== 'string' || !o.command.trim()) out.push({ code: 'POLICY_INVALID', path: `${at}.command`, message: 'must be a non-empty string' });
|
||||
else if (o.command.length > MAX_COMMAND || /[\0\r\n]/.test(o.command)) out.push({ code: 'POLICY_CONTAINMENT', path: `${at}.command`, message: `must be one line under ${MAX_COMMAND} characters` });
|
||||
if (!Number.isInteger(o.timeout_sec) || (o.timeout_sec as number) < 60 || (o.timeout_sec as number) > 21600) out.push({ code: 'POLICY_INVALID', path: `${at}.timeout_sec`, message: 'must be an integer from 60 to 21600' });
|
||||
if (typeof o.spend_cap_usd !== 'number' || !(o.spend_cap_usd > 0) || o.spend_cap_usd > 100) out.push({ code: 'POLICY_INVALID', path: `${at}.spend_cap_usd`, message: 'must be dollars above 0 and at most 100' });
|
||||
if (!('pass_env' in o)) return;
|
||||
const p = o.pass_env;
|
||||
if (!Array.isArray(p) || p.length > 16 || new Set(p).size !== p.length || p.some(n => typeof n !== 'string' || !/^[A-Za-z_][A-Za-z0-9_]{0,63}$/.test(n))) out.push({ code: 'POLICY_INVALID', path: `${at}.pass_env`, message: 'must be up to 16 distinct variable names' });
|
||||
else if (p.some(n => /^UBICLOUD_/.test(n))) out.push({ code: 'POLICY_CONTAINMENT', path: `${at}.pass_env`, message: 'the runner never forwards its own Ubicloud credentials to a VM' });
|
||||
}
|
||||
|
||||
function gateErrors(g: unknown, at: string, out: PolicyError[]): void {
|
||||
const o = g as Record<string, unknown> | null;
|
||||
if (!o || typeof o !== 'object' || Array.isArray(o)) { out.push({ code: 'POLICY_INVALID', path: at, message: 'must be { kind: workflows, ... } or { kind: command, ... }' }); return; }
|
||||
const allowed = o.kind === 'workflows' ? ['kind', 'workflows', 'required_jobs', 'record', 'post_merge_checks'] : o.kind === 'command' ? ['kind', 'command', 'timeout_sec', 'spend_cap_usd', 'pass_env'] : null;
|
||||
if (!allowed) { out.push({ code: 'POLICY_INVALID', path: `${at}.kind`, message: `${JSON.stringify(o.kind)} must be workflows | command` }); return; }
|
||||
for (const k of Object.keys(o)) if (!allowed.includes(k)) out.push({ code: 'POLICY_INVALID', path: `${at}.${k}`, message: `unknown key for a ${o.kind} gate` });
|
||||
if (o.kind === 'workflows') workflowsGateErrors(o, at, out); else commandGateErrors(o, at, out);
|
||||
}
|
||||
|
||||
function landErrors(v: unknown, obj: Record<string, unknown>, out: PolicyError[]): void {
|
||||
const l = v as Record<string, unknown> | null;
|
||||
if (!l || typeof l !== 'object' || Array.isArray(l)) { out.push({ code: 'POLICY_INVALID', path: 'train.land', message: 'must be { merge_method, required_checks? }' }); return; }
|
||||
for (const k of Object.keys(l)) if (k !== 'merge_method' && k !== 'required_checks') out.push({ code: 'POLICY_INVALID', path: `train.land.${k}`, message: 'unknown key' });
|
||||
if (!['squash', 'merge', 'rebase'].includes(l.merge_method as string)) out.push({ code: 'POLICY_INVALID', path: 'train.land.merge_method', message: `${JSON.stringify(l.merge_method)} must be squash | merge | rebase` });
|
||||
if (!('required_checks' in l)) return;
|
||||
if (obj.changelog_fragments != null) out.push({ code: 'POLICY_INVALID', path: 'train.land.required_checks', message: 'this policy declares changelog_fragments; its required_checks are the one list train land reads' });
|
||||
else nameList('train.land.required_checks', l.required_checks, 50, CHECK_NAME, out, 'check names of 1-100 characters');
|
||||
}
|
||||
|
||||
function trainErrors(value: unknown, obj: Record<string, unknown>, out: PolicyError[]): void {
|
||||
const t = value as Record<string, unknown> | null;
|
||||
if (!t || typeof t !== 'object' || Array.isArray(t)) { out.push({ code: 'POLICY_INVALID', path: 'train', message: `must be { ${Object.keys(TRAIN_FIELDS).join(', ')} } or null` }); return; }
|
||||
for (const k of Object.keys(t)) if (!(k in TRAIN_FIELDS)) out.push({ code: 'POLICY_INVALID', path: `train.${k}`, message: 'unknown key' });
|
||||
if (!t || typeof t !== 'object' || Array.isArray(t)) { out.push({ code: 'POLICY_INVALID', path: 'train', message: `must be { ${Object.keys(TRAIN_FIELDS).join(', ')} }, { ${GENERIC_TRAIN_KEYS.join(', ')} } or null` }); return; }
|
||||
const generic = 'gates' in t;
|
||||
const fields = generic ? GENERIC_TRAIN_KEYS : [...Object.keys(TRAIN_FIELDS), 'land'];
|
||||
for (const k of Object.keys(t)) {
|
||||
if (!fields.includes(k)) out.push({ code: 'POLICY_INVALID', path: `train.${k}`, message: generic && k in TRAIN_FIELDS ? 'belongs to the gstack-evals gate; a train with gates declares only gates, land, max_size and fill_hours' : 'unknown key' });
|
||||
}
|
||||
for (const k of generic ? GENERIC_TRAIN_KEYS : Object.keys(TRAIN_FIELDS)) if (!(k in t)) out.push({ code: 'POLICY_INVALID', path: `train.${k}`, message: `missing (every ${generic ? 'field of a train with gates' : 'train field'} is required)` });
|
||||
for (const [k, check] of Object.entries(TRAIN_FIELDS)) {
|
||||
if (!(k in t)) { out.push({ code: 'POLICY_INVALID', path: `train.${k}`, message: 'missing (every train field is required)' }); continue; }
|
||||
const why = check(t[k]);
|
||||
if (!(k in t) || (generic && k !== 'max_size' && k !== 'fill_hours')) continue;
|
||||
const size = t[k] as number;
|
||||
const why = generic && k === 'max_size' ? (Number.isInteger(size) && size >= 1 && size <= 5 ? null : 'must be an integer from 1 to 5 (every prefix is gated: at most six runs per gate)') : check(t[k]);
|
||||
if (why) out.push({ code: 'POLICY_INVALID', path: `train.${k}`, message: `${JSON.stringify(t[k])} ${why}` });
|
||||
}
|
||||
if (generic) {
|
||||
if (!Array.isArray(t.gates) || !t.gates.length || t.gates.length > 4) out.push({ code: 'POLICY_INVALID', path: 'train.gates', message: 'must be a list of 1-4 gates' });
|
||||
else t.gates.forEach((g, i) => gateErrors(g, `train.gates[${i}]`, out));
|
||||
}
|
||||
if ('land' in t) landErrors(t.land, obj, out);
|
||||
const method = (t.land as { merge_method?: unknown } | undefined)?.merge_method ?? 'squash';
|
||||
if ('land' in t && obj.history === 'merge-only' && method !== 'merge') out.push({ code: 'POLICY_INVALID', path: 'train.land.merge_method', message: `history merge-only allows only the merge method; train land would use ${String(method)}` });
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -379,7 +492,7 @@ export function validatePolicy(raw: unknown, opts: { gstackVersion?: string } =
|
||||
else if (t.length > MAX_COMMAND || /[\0\r\n]/.test(t)) errors.push({ code: 'POLICY_CONTAINMENT', path: 'gate_command', message: `must be one line under ${MAX_COMMAND} characters` });
|
||||
}
|
||||
if ('full_e2e' in obj && typeof obj.full_e2e !== 'boolean') errors.push({ code: 'POLICY_INVALID', path: 'full_e2e', message: 'must be true or false' });
|
||||
if ('train' in obj && obj.train !== null) trainErrors(obj.train, errors);
|
||||
if ('train' in obj && obj.train !== null) trainErrors(obj.train, obj, errors);
|
||||
if ('docs' in obj && obj.docs !== null) docsErrors(obj.docs, errors);
|
||||
if ('changelog_fragments' in obj && obj.changelog_fragments !== null) fragmentsErrors(obj.changelog_fragments, errors);
|
||||
if ('base_red' in obj && obj.base_red !== null) baseRedErrors(obj.base_red, errors);
|
||||
@@ -428,6 +541,9 @@ function compatErrors(obj: Record<string, unknown>, running: string, errors: Pol
|
||||
if (typeof min === 'string' && !newer(since, min)) continue;
|
||||
errors.push({ code: 'POLICY_INVALID', path: key, message: `introduced in gstack ${since}; set min_gstack_version to ${since} or later so an older gstack refuses this policy by name` });
|
||||
}
|
||||
if (genericTrainKeys(obj.train) && !(typeof min === 'string' && !newer(TRAIN_GATES_KEY_VERSION, min))) {
|
||||
errors.push({ code: 'POLICY_INVALID', path: 'train', message: `gates and land were introduced in gstack ${TRAIN_GATES_KEY_VERSION}; set min_gstack_version to ${TRAIN_GATES_KEY_VERSION} or later so an older gstack refuses this policy by name` });
|
||||
}
|
||||
if (obj.bump === 'never' && !(typeof min === 'string' && !newer(BUMP_NEVER_KEY_VERSION, min))) {
|
||||
errors.push({ code: 'POLICY_INVALID', path: 'bump', message: `never was introduced in gstack ${BUMP_NEVER_KEY_VERSION}; set min_gstack_version to ${BUMP_NEVER_KEY_VERSION} or later so an older gstack refuses this policy by name` });
|
||||
}
|
||||
|
||||
@@ -18,18 +18,24 @@ import { outFailed, outOk, outPending, type TrainOut } from './ship-queue-train'
|
||||
const Q = 'gstack-ship-queue';
|
||||
const sleep = (ms: number) => new Promise(resolve => setTimeout(resolve, ms));
|
||||
|
||||
export interface PostMergeOptions { sha: string; timeoutSec: number; pollMs: number }
|
||||
export interface PostMergeOptions { sha: string; timeoutSec: number; pollMs: number; checks?: string[] }
|
||||
|
||||
export async function verifyPostMerge(ctx: QueueContext, o: PostMergeOptions): Promise<TrainOut> {
|
||||
const deadline = Date.now() + o.timeoutSec * 1000;
|
||||
const at = o.sha.slice(0, 12);
|
||||
const again = `${Q} verify --post-merge ${o.sha} --timeout ${o.timeoutSec}`;
|
||||
const named = o.checks ?? [];
|
||||
const again = `${Q} verify --post-merge ${o.sha} --timeout ${o.timeoutSec}${named.length ? ` --checks ${JSON.stringify(named.join(','))}` : ''}`;
|
||||
for (;;) {
|
||||
const ci = ciAtHead(ctx, o.sha);
|
||||
const lines = [`POST_MERGE: ${ci.line.replace(/^(\S+) at ([0-9a-f]+)/, '$2 $1')}`, ...ci.checks.map(c => `CHECK: ${c.name} ${c.status === 'completed' ? c.conclusion ?? 'unknown' : c.status}`)];
|
||||
const settled = !ci.none && !/^(pending|unknown) /.test(ci.line) && !ci.checks.some(c => c.status !== 'completed');
|
||||
if (settled && ci.green) return outOk(lines, 'none', 'none');
|
||||
if (settled) {
|
||||
const runs = (n: string) => ci.checks.filter(c => c.name === n || c.name.startsWith(`${n} (`));
|
||||
const absent = named.filter(n => !runs(n).length);
|
||||
const unsuccessful = named.filter(n => runs(n).some(c => c.status === 'completed' && c.conclusion !== 'success'));
|
||||
if (settled && ci.green && unsuccessful.length) return outFailed({ code: 'QUEUE_POST_MERGE_RED', message: `the post-merge checks the train gate names did not succeed at ${at}: ${unsuccessful.join(', ')}` }, lines,
|
||||
`read the run, repair on a new PR or roll back; ${Q} train status names the freeze if the backstop filed one`, 'none');
|
||||
if (settled && ci.green && !absent.length) return outOk(lines, 'none', 'none');
|
||||
if (settled && !ci.green) {
|
||||
const red = ci.checks.filter(c => c.status === 'completed' && !['success', 'skipped', 'neutral'].includes(c.conclusion ?? '')).map(c => c.name);
|
||||
return outFailed({ code: 'QUEUE_POST_MERGE_RED', message: `main's push workflows at ${at} are red: ${red.join(', ') || 'a commit status (no check run failed)'}` }, lines,
|
||||
`read the red run, repair on a new PR or roll back; ${Q} train status names the freeze if the backstop filed one`, 'none');
|
||||
@@ -37,7 +43,7 @@ export async function verifyPostMerge(ctx: QueueContext, o: PostMergeOptions): P
|
||||
if (Date.now() >= deadline) {
|
||||
const pending = ci.checks.filter(c => c.status !== 'completed').map(c => c.name);
|
||||
const message = ci.line.startsWith('unknown') ? `the check runs at ${at} are unreadable (${ci.line.replace(/^unknown at [0-9a-f]+ /, '')})`
|
||||
: ci.none ? `no check run has attached to ${at} yet` : `main's push workflows at ${at} have not settled at the timeout: ${pending.join(', ') || 'a commit status'} still running`;
|
||||
: ci.none ? `no check run has attached to ${at} yet` : settled ? `the post-merge checks the train gate names never ran at ${at}: ${absent.join(', ')}` : `main's push workflows at ${at} have not settled at the timeout: ${pending.join(', ') || 'a commit status'} still running`;
|
||||
return outPending({ code: 'QUEUE_TRAIN_PENDING', message }, lines, again, 'none');
|
||||
}
|
||||
await sleep(o.pollMs);
|
||||
|
||||
@@ -23,6 +23,13 @@ export const TRAIN_HELP = ` train build --prs <n>@<sha>[,...] [--order <n>,...]
|
||||
base policy's restamp_command or release_tool
|
||||
--order: explicit slot order, every PR in --prs once (default: screened
|
||||
first, then READY order)
|
||||
train init [--command "<test cmd>" [--timeout <s>] [--spend-cap <usd>]]
|
||||
(any repository) read branch protection, the base's workflows and open PRs; print
|
||||
a train block with gates, the nonce snippet each gate workflow needs and the land
|
||||
mode; refuses when PRs bump the version without changelog_fragments or stamp-at-merge
|
||||
train doctor re-check the base policy's train block with gates against the repository: gate
|
||||
workflows, nonce input, run-name, required jobs, concurrency, push triggers on
|
||||
train-prefix/ refs, green on the base, merge method (writes nothing)
|
||||
train build --train <nonce> resume a train (push, free tests, dispatch) from its record
|
||||
merge pinned unstamped heads in slot order onto main, free tests on
|
||||
train/<nonce>, then one full gate through eval-bg's nonce
|
||||
@@ -30,14 +37,17 @@ export const TRAIN_HELP = ` train build --prs <n>@<sha>[,...] [--order <n>,...]
|
||||
unconfirmed dispatches by nonce
|
||||
train verdict --train <nonce> read the gate run and its record: GREEN, RED, INCOMPLETE or PENDING
|
||||
train land --train <nonce> --pr <n> [--no-wait] [--timeout <min>]
|
||||
(changelog_fragments trains) publish slot n's stamped prefix S_i: commit its tree on
|
||||
[pin, main], push under the pin's lease, wait for checks, squash, confirm main is S_i
|
||||
publish slot n's prefix (stamped S_i, else gated T_i): update mode commits its tree on
|
||||
[pin, main], pushes under the pin's lease, waits for checks and merges; direct mode
|
||||
(trains with gates, no strict protection, a green pin) runs gh pr merge
|
||||
--match-head-commit <pin>; then confirms main's tree is the prefix's
|
||||
train bisect --train <nonce> --spend-cap <usd> [--dry-run]
|
||||
red cases on every prefix T0..Tk, resumable and capped
|
||||
train rebuild --train <nonce> [--infra] the one rebuild an all-green bisect earns (or the one INFRA re-dispatch)
|
||||
train watch --train <nonce> [--timeout <s>]
|
||||
train watch --train <nonce> [--timeout <s>] [--land]
|
||||
resumable follow: runs the step the record asks for (build, status), waits for the
|
||||
pending run, reads the verdict, prints NEXT; rerun it after a restart (no daemon)
|
||||
pending run, reads the verdict, prints NEXT; rerun it after a restart (no daemon);
|
||||
--land (trains with gates) lands each landable slot in order
|
||||
train forget --train <nonce> | --receipt <pr>@<sha> [--dry-run]
|
||||
remove a settled train record (attempts, bisect) or a READY receipt from
|
||||
refs/notes/gstack-trains under an exact lease, then delete the local files
|
||||
@@ -46,6 +56,9 @@ export const TRAIN_HELP = ` train build --prs <n>@<sha>[,...] [--order <n>,...]
|
||||
settles merging or green trains whose every PR is merged or closed on GitHub, and
|
||||
deletes screen/<pr>-* branches of merged or closed PRs
|
||||
ready --pr <n> --screen [--broad] | --no-screen [--dry-run] [--dispatch-ci <wf.yml[,…]> [--timeout <s>]]
|
||||
[--base-red --fixer <pr> | --fixes-base-red <case,...>]
|
||||
(trains with gates) --base-red: a conditional READY for a paused /ship, seated
|
||||
only behind fixer <pr> at slot 1; --fixes-base-red: the fixer's READY
|
||||
(train policy) READY plus the screen choice, recorded as a receipt; a head whose
|
||||
screen run is RED is refused (QUEUE_SCREEN_RED). --dispatch-ci runs the named
|
||||
workflows on the branch first (a conflicting PR has no pull_request CI), waits for
|
||||
@@ -54,7 +67,7 @@ export const TRAIN_HELP = ` train build --prs <n>@<sha>[,...] [--order <n>,...]
|
||||
(train policy) the merge check against a green train record
|
||||
verify --pr <n> --screen (train policy) the PR head's screen verdict read from its run: SCREEN_VERDICT and
|
||||
FLAKY lines, recorded in the READY receipt
|
||||
verify --post-merge <sha> [--timeout <s>] after a merge: wait for every check run at the merged commit to settle;
|
||||
verify --post-merge <sha> [--timeout <s>] [--checks <a,b>] after a merge: wait for every check run at the merged commit to settle;
|
||||
POST_MERGE: green|red|pending plus one CHECK: line per workflow (resumable)
|
||||
Merge trains need a "train" block in the base revision's ship policy (docs/ship-policy.md#run-a-merge-train).`;
|
||||
|
||||
@@ -113,6 +126,15 @@ export async function trainMain(ctx: QueueContext, args: string[]): Promise<numb
|
||||
allowRepoCommands: rest.includes('--allow-repo-commands'),
|
||||
}));
|
||||
}
|
||||
if (sub === 'init') {
|
||||
const timeoutSec = positive(flag(rest, '--timeout'), 1800);
|
||||
const spendCap = flag(rest, '--spend-cap') === undefined ? undefined : positive(flag(rest, '--spend-cap'), NaN);
|
||||
if (Number.isNaN(timeoutSec) || (spendCap !== undefined && Number.isNaN(spendCap))) return usage('--timeout needs seconds and --spend-cap dollars');
|
||||
const command = flag(rest, '--command');
|
||||
if (rest.includes('--command') && !command?.trim()) return usage('--command needs the test command a command gate runs');
|
||||
return guarded('train init', async () => (await import('./ship-queue-train-init')).trainInit(ctx, { command, timeoutSec, spendCap }));
|
||||
}
|
||||
if (sub === 'doctor') return guarded('train doctor', async () => (await import('./ship-queue-train-init')).trainDoctor(ctx));
|
||||
if (sub === 'status') return guarded('train status', () => trainStatus(ctx, { train: nonce }));
|
||||
if (sub === 'prune') return guarded('train prune', () => pruneTrains(ctx, { dryRun, train: nonce, stale: rest.includes('--stale') }));
|
||||
if (sub === 'forget') {
|
||||
@@ -126,7 +148,7 @@ export async function trainMain(ctx: QueueContext, args: string[]): Promise<numb
|
||||
if (Number.isNaN(timeoutSec)) return usage('--timeout needs seconds');
|
||||
const freeTimeoutSec = positive(flag(rest, '--free-timeout'), 2700);
|
||||
if (Number.isNaN(freeTimeoutSec)) return usage('--free-timeout needs seconds');
|
||||
return guarded('train watch', () => trainWatch(ctx, { train: nonce, timeoutSec, pollMs: Number(process.env.GSTACK_SHIP_QUEUE_POLL_MS) || 30_000, freeTimeoutSec }));
|
||||
return guarded('train watch', () => trainWatch(ctx, { train: nonce, timeoutSec, pollMs: Number(process.env.GSTACK_SHIP_QUEUE_POLL_MS) || 30_000, freeTimeoutSec, land: rest.includes('--land') }));
|
||||
}
|
||||
if (sub === 'verdict') return guarded('train verdict', () => trainVerdict(ctx, { train: nonce }));
|
||||
if (sub === 'land') {
|
||||
@@ -155,7 +177,15 @@ export async function readyMain(ctx: QueueContext, args: string[], base: { pr: s
|
||||
if (dispatchCi !== undefined && (!workflows.length || workflows.some(w => !/^[\w.-]+\.ya?ml$/.test(w)))) return usage('--dispatch-ci names workflow files under .github/workflows, comma-separated (free-tests.yml,windows-free-tests.yml)');
|
||||
const timeoutSec = positive(flag(args, '--timeout'), 2700);
|
||||
if (!Number.isFinite(timeoutSec)) return usage('--timeout needs a positive number of seconds');
|
||||
return guarded('ready', () => trainReady(ctx, { ...base, screen: screen ? 'screen' : noScreen ? 'no-screen' : undefined, broad: args.includes('--broad'), dryRun: args.includes('--dry-run'),
|
||||
const fixer = flag(args, '--fixer');
|
||||
const fixes = flag(args, '--fixes-base-red');
|
||||
if (fixer !== undefined && !/^\d+$/.test(fixer)) return usage('--fixer needs the fixer PR number');
|
||||
if (fixer !== undefined && !args.includes('--base-red')) return usage('--fixer names the fixer of a --base-red READY');
|
||||
if (args.includes('--base-red') && fixes !== undefined) return usage('--base-red (the paused PR) and --fixes-base-red (its fixer) are different READYs; pass one');
|
||||
if ((args.includes('--base-red') || fixes !== undefined) && !noScreen) return usage('a base-red READY is --no-screen: the train gates every prefix');
|
||||
const fixesBaseRed = fixes?.split(',').map(s => s.trim()).filter(Boolean);
|
||||
if (fixes !== undefined && !fixesBaseRed?.length) return usage('--fixes-base-red names the gate cases the fixer repairs, comma-separated');
|
||||
return guarded('ready', () => trainReady(ctx, { ...base, baseRed: args.includes('--base-red') ? { fixer } : undefined, fixesBaseRed, screen: screen ? 'screen' : noScreen ? 'no-screen' : undefined, broad: args.includes('--broad'), dryRun: args.includes('--dry-run'),
|
||||
dispatchCi: workflows.length ? { workflows, timeoutSec, pollMs: Number(process.env.GSTACK_SHIP_QUEUE_POLL_MS) || 30_000 } : undefined }));
|
||||
}
|
||||
|
||||
@@ -164,7 +194,8 @@ export async function postMergeMain(ctx: QueueContext, args: string[]): Promise<
|
||||
if (!sha || !/^[0-9a-f]{40}$/.test(sha)) return usage('verify --post-merge needs the merged commit\'s full 40-hex SHA');
|
||||
const timeoutSec = positive(flag(args, '--timeout'), 2700);
|
||||
if (!Number.isFinite(timeoutSec)) return usage('--timeout needs a positive number of seconds');
|
||||
return guarded('verify', () => verifyPostMerge(ctx, { sha, timeoutSec, pollMs: Number(process.env.GSTACK_SHIP_QUEUE_POLL_MS) || 30_000 }));
|
||||
const checks = flag(args, '--checks')?.split(',').map(s => s.trim()).filter(Boolean);
|
||||
return guarded('verify', () => verifyPostMerge(ctx, { sha, timeoutSec, pollMs: Number(process.env.GSTACK_SHIP_QUEUE_POLL_MS) || 30_000, checks }));
|
||||
}
|
||||
|
||||
export async function verifyMain(ctx: QueueContext, args: string[], pr: string): Promise<number> {
|
||||
|
||||
@@ -0,0 +1,570 @@
|
||||
/**
|
||||
* Generic train gates (docs/ship-policy.md#run-a-merge-train; plan GSTA-157 PR 2): a train whose base policy
|
||||
* declares `train.gates` gates every prefix T0..Tk (or T0 and S1..Sk when the train stamps) on its own
|
||||
* `train-prefix/<nonce>/t<i>` ref, with every gate in the list:
|
||||
*
|
||||
* - `workflows`: each workflow is dispatched on the prefix ref with a `nonce` input, recorded before it is sent,
|
||||
* and only the workflow_dispatch run whose run-name carries `[eval-bg <nonce>]` is adopted, never a run found by
|
||||
* sha. GREEN needs that run at exactly the prefix head, every required job (`<workflow>/<job name>`) concluded
|
||||
* success, the run concluded success and, when the gate declares `record`, an artifact reporting at least one
|
||||
* executed test. A red run gets one `gh run rerun --failed` (CREW rule 16): a pass is GREEN with FLAKY lines,
|
||||
* a second red is real; both attempts' jobs are kept.
|
||||
* - `command`: a clean detached checkout of the prefix head runs on a Ubicloud VM through
|
||||
* scripts/ubicloud/ubi-runner.sh, forwarding only `pass_env`; the VM's `git write-tree` must equal the prefix
|
||||
* tree and exit 0 is GREEN. One spend cap covers every run of the train.
|
||||
*
|
||||
* The verdict is per prefix; the train is GREEN when every prefix is, and otherwise the first red prefix after a
|
||||
* green one names the culprit (classifyPrefixes). Slot 1's READY may declare FIXES_BASE_RED: T0 is then allowed
|
||||
* red on exactly those cases and nothing else.
|
||||
*/
|
||||
import { spawn } from 'node:child_process';
|
||||
import { randomBytes } from 'node:crypto';
|
||||
import * as fs from 'node:fs';
|
||||
import * as os from 'node:os';
|
||||
import * as path from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { awaitNonceRun, listDispatchRuns, makeNonce, runForNonce } from '../scripts/eval-bg';
|
||||
import { scriptInvocation } from './script-invocation';
|
||||
import type { ShipReceipt } from './headless-artifacts';
|
||||
import { extractReceipt, placeReceiptFirst } from './ship-receipt';
|
||||
import { bindShippedReceipt } from './ship-receipt-depth';
|
||||
import type { CommandGate, TrainGate, WorkflowsGate } from './ship-policy';
|
||||
import { git, type QueueFailure } from './ship-queue-gate';
|
||||
import { execFor, repoSlug } from './ship-queue-train-github';
|
||||
import {
|
||||
attemptsOf, listTrains, loadTrain, nowIso, queueRoot, trainDir, trainFile,
|
||||
type AttemptRecord, type GateOutcome, type GatedPrefix, type GenericVerdict, type TrainPrefix, type TrainRecord, type TrainSlot,
|
||||
} from './ship-queue-train-record';
|
||||
import { effect, outFailed, outOk, outPending, outRefused, prsArgs, save, type TrainEnv, type TrainOut } from './ship-queue-train';
|
||||
|
||||
const Q = 'gstack-ship-queue';
|
||||
const UBI_RUNNER = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..', 'scripts', 'ubicloud', 'ubi-runner.sh');
|
||||
/** Ubicloud list price per minute of the size command gates run on (scripts/ubicloud/pregate-remote.sh). */
|
||||
export const COMMAND_SIZE = 'standard-16';
|
||||
export const COMMAND_USD_PER_MIN = 0.00831;
|
||||
/** Create, upload and teardown minutes reserved beside a command's own timeout. */
|
||||
const COMMAND_OVERHEAD_MIN = 10;
|
||||
const FAILED = ['failure', 'timed_out', 'startup_failure'];
|
||||
|
||||
/** One dispatch unit: a workflow of a workflows gate, or a command gate (workflow 'command'). */
|
||||
export interface GateUnit { gate: number; workflow: string }
|
||||
export const gateUnits = (gates: readonly TrainGate[]): GateUnit[] =>
|
||||
gates.flatMap((g, i) => (g.kind === 'workflows' ? g.workflows.map(workflow => ({ gate: i, workflow })) : [{ gate: i, workflow: 'command' }]));
|
||||
|
||||
/** The commits a generic train gates, with their refs: T0, then T1..Tk, or S1..Sk when the train stamps. */
|
||||
export function gatedChain(nonce: string, prefixes: readonly TrainPrefix[], stamped?: ReadonlyArray<{ commit: string; tree: string }>): GatedPrefix[] {
|
||||
const chain = [prefixes[0]!, ...(stamped ?? prefixes.slice(1))];
|
||||
return chain.map((p, i) => ({ index: i, ref: `train-prefix/${nonce}/${stamped && i ? 's' : 't'}${i}`, commit: p.commit, tree: p.tree, outcome: null, red: [], detail: null }));
|
||||
}
|
||||
|
||||
/**
|
||||
* bindShippedReceipt for a train slot. A conditional receipt (its base_red block) binds only for a slot whose READY
|
||||
* carried the condition (`ready --base-red --fixer`): the owner-exception and review-depth checks then run on the
|
||||
* receipt without that block, so the condition is the READY's to carry and every other check still applies.
|
||||
*/
|
||||
export function bindTrainReceipt(root: string, baseRef: string, body: string, pr: string, accepted: readonly string[], conditional: boolean): ReturnType<typeof bindShippedReceipt> {
|
||||
const b = bindShippedReceipt(root, baseRef, body, pr, accepted);
|
||||
if (b.ok || b.code !== 'QUEUE_RECEIPT_CONDITIONAL' || !conditional) return b;
|
||||
const { base_red: _, ...rest } = extractReceipt(body).receipt!;
|
||||
return bindShippedReceipt(root, baseRef, placeReceiptFirst(body, rest as ShipReceipt), pr, accepted);
|
||||
}
|
||||
|
||||
/** Per-run price reservation of a command gate: its timeout plus overhead, at the list price. */
|
||||
export const commandReservation = (g: CommandGate) => Math.ceil((g.timeout_sec / 60 + COMMAND_OVERHEAD_MIN) * COMMAND_USD_PER_MIN * 100) / 100;
|
||||
|
||||
/**
|
||||
* A conditional slot rides only a generic train whose slot 1 is its fixer, and a fixer (FIXES_BASE_RED) rides only
|
||||
* at slot 1. Without an explicit order the fixer moves to slot 1; with one, a fixer elsewhere is refused.
|
||||
*/
|
||||
export function baseRedAdmission(slots: TrainSlot[], generic: boolean, explicit: boolean): { slots: TrainSlot[]; lines: string[] } | QueueFailure {
|
||||
const conditional = slots.filter(s => s.base_red);
|
||||
const fixers = slots.filter(s => s.fixes_base_red?.length);
|
||||
if (!conditional.length && !fixers.length) return { slots, lines: [] };
|
||||
const named = [...new Set([...conditional, ...fixers])].map(s => '#' + s.pr).join(', ');
|
||||
if (!generic) return { code: 'QUEUE_TRAIN_BASE_RED', message: `${named}: a base-red condition rides only a train with gates, which gates every prefix; a gstack-evals train gates only its tip` };
|
||||
if (fixers.length > 1) return { code: 'QUEUE_TRAIN_BASE_RED', message: `${fixers.map(s => '#' + s.pr).join(' and ')} each declare FIXES_BASE_RED; only slot 1 may, so one train carries one fixer` };
|
||||
const fixer = fixers[0];
|
||||
let order = slots;
|
||||
if (fixer && slots[0] !== fixer) {
|
||||
if (explicit) return { code: 'QUEUE_TRAIN_BASE_RED', message: `#${fixer.pr} declares FIXES_BASE_RED but --order seats it at slot ${slots.indexOf(fixer) + 1}; a fixer must be slot 1` };
|
||||
order = [fixer, ...slots.filter(s => s !== fixer)];
|
||||
}
|
||||
for (const s of conditional) {
|
||||
if (s === fixer) return { code: 'QUEUE_TRAIN_BASE_RED', message: `#${s.pr} is conditional and declares FIXES_BASE_RED; a fixer's own head must be green` };
|
||||
if (!fixer || s.base_red!.fixer !== fixer.pr) return { code: 'QUEUE_TRAIN_BASE_RED', message: `#${s.pr} is conditional on fixer #${s.base_red!.fixer}, which is not in this train at slot 1 with FIXES_BASE_RED` };
|
||||
if (!(Date.parse(s.base_red!.expires_at) >= Date.parse(nowIso()))) return { code: 'QUEUE_TRAIN_BASE_RED', message: `#${s.pr}'s base-red condition expired at ${s.base_red!.expires_at}; its lane classifies again` };
|
||||
}
|
||||
return { slots: order, lines: [`BASE_RED: fixer #${fixer!.pr} at slot 1 (FIXES_BASE_RED: ${fixer!.fixes_base_red!.join(', ')}); conditional ${conditional.map(s => '#' + s.pr).join(', ') || '(none)'}; every prefix is gated with nothing excluded`] };
|
||||
}
|
||||
|
||||
const sameSet = (a: readonly string[], b: readonly string[]) => a.length === b.length && [...a].sort().join('\n') === [...b].sort().join('\n');
|
||||
|
||||
/**
|
||||
* The train verdict over its prefix verdicts. T0 red is red on main, unless slot 1 declares FIXES_BASE_RED and
|
||||
* T0's red cases are exactly those. Then the first non-green prefix decides: incomplete stops the train; red with
|
||||
* any later prefix green is unattributed (a repair or a new order, never an unchanged rebuild); otherwise its slot
|
||||
* is the culprit and the slots before it may land.
|
||||
*/
|
||||
export function classifyPrefixes(p: ReadonlyArray<Pick<GatedPrefix, 'outcome' | 'red'>>, slots: ReadonlyArray<Pick<TrainSlot, 'pr' | 'fixes_base_red'>>): Omit<GenericVerdict, 'at'> | { kind: 'pending' } {
|
||||
if (p.some(x => !x.outcome || x.outcome === 'PENDING')) return { kind: 'pending' };
|
||||
const none = { landable: 0, culprit: null };
|
||||
const t0 = p[0]!;
|
||||
if (t0.outcome === 'INCOMPLETE') return { kind: 'incomplete', ...none };
|
||||
const fixes = slots[0]?.fixes_base_red ?? [];
|
||||
if (t0.outcome === 'RED' && !(fixes.length && sameSet(t0.red, fixes))) return { kind: 'red-on-main', ...none };
|
||||
const j = p.findIndex((x, i) => i > 0 && x.outcome !== 'GREEN');
|
||||
if (j < 0) return { kind: 'green', landable: slots.length, culprit: null };
|
||||
if (p[j]!.outcome === 'INCOMPLETE') return { kind: 'incomplete', ...none };
|
||||
if (p.slice(j + 1).some(x => x.outcome === 'GREEN')) return { kind: 'unattributed', ...none };
|
||||
return { kind: 'attributed', landable: j - 1, culprit: slots[j - 1]!.pr };
|
||||
}
|
||||
|
||||
export interface RunFacts { status: string; conclusion: string | null; head_sha: string; event: string; run_attempt: number }
|
||||
export interface JobFacts { name: string; conclusion: string | null }
|
||||
export interface RecordFacts { executed: number; failed: string[] }
|
||||
|
||||
/** One workflow run against its prefix: PENDING until completed, RED on a failed job (or failed record tests), GREEN only when nothing is missing. */
|
||||
export function classifyWorkflowRun(o: { run: RunFacts; head: string; workflow: string; jobs: JobFacts[]; required: string[]; record?: RecordFacts | null }): { outcome: GateOutcome; red: string[]; detail: string } {
|
||||
const { run } = o;
|
||||
if (run.status !== 'completed') return { outcome: 'PENDING', red: [], detail: `run is ${run.status}` };
|
||||
if (run.event !== 'workflow_dispatch') return { outcome: 'INCOMPLETE', red: [], detail: `the adopted run is a ${run.event} run, not the nonce dispatch` };
|
||||
if (run.head_sha !== o.head) return { outcome: 'INCOMPLETE', red: [], detail: `the run tested ${run.head_sha.slice(0, 12) || 'no commit'}, not the prefix head ${o.head.slice(0, 12)}` };
|
||||
const failedJobs = o.jobs.filter(j => FAILED.includes(j.conclusion ?? '')).map(j => `${o.workflow}/${j.name}`);
|
||||
const red = o.record?.failed.length ? o.record.failed.map(id => `${o.workflow}:${id}`) : failedJobs;
|
||||
if (red.length) return { outcome: 'RED', red: [...new Set(red)].sort(), detail: `${red.length} red: ${red.slice(0, 8).join(', ')}` };
|
||||
const gaps: string[] = [];
|
||||
for (const name of o.required) {
|
||||
const hits = o.jobs.filter(j => j.name === name);
|
||||
if (!hits.length) gaps.push(`required job ${o.workflow}/${name} is missing`);
|
||||
else if (hits.some(j => j.conclusion !== 'success')) gaps.push(`required job ${o.workflow}/${name} concluded ${hits.map(j => j.conclusion ?? 'unknown').join('/')}`);
|
||||
}
|
||||
if (o.record === null) gaps.push('the declared record artifact is missing or unreadable');
|
||||
else if (o.record && o.record.executed < 1) gaps.push('the record reports no executed test');
|
||||
if (run.conclusion !== 'success') gaps.push(`the run concluded ${run.conclusion ?? 'unknown'}`);
|
||||
if (gaps.length) return { outcome: 'INCOMPLETE', red: [], detail: gaps.join('; ') };
|
||||
return { outcome: 'GREEN', red: [], detail: `${o.required.length} required job(s) green${o.record ? `, ${o.record.executed} test(s) executed` : ''}` };
|
||||
}
|
||||
|
||||
/** JUnit XML (testcase elements) or a JSON test list ({ tests: [{ id, status }] }) to executed and failed test ids. */
|
||||
export function parseTestRecord(files: Array<{ name: string; text: string }>): RecordFacts | null {
|
||||
let executed = 0; const failed: string[] = []; let read = false;
|
||||
for (const f of files) {
|
||||
if (f.name.endsWith('.json')) {
|
||||
let tests: unknown;
|
||||
try { tests = (JSON.parse(f.text) as { tests?: unknown }).tests; } catch { continue; }
|
||||
if (!Array.isArray(tests)) continue;
|
||||
read = true;
|
||||
for (const t of tests as Array<{ id?: unknown; status?: unknown }>) {
|
||||
if (typeof t?.id !== 'string' || t.status === 'skip' || t.status === 'skipped') continue;
|
||||
executed++;
|
||||
if (t.status !== 'pass' && t.status !== 'passed') failed.push(t.id);
|
||||
}
|
||||
} else if (f.name.endsWith('.xml')) {
|
||||
read = true;
|
||||
for (const m of f.text.matchAll(/<testcase\b([^>]*?)(\/>|>([\s\S]*?)<\/testcase>)/g)) {
|
||||
const attr = (k: string) => new RegExp(`\\b${k}="([^"]*)"`).exec(m[1]!)?.[1] ?? '';
|
||||
const body = m[3] ?? '';
|
||||
if (/<skipped\b/.test(body)) continue;
|
||||
executed++;
|
||||
if (/<(failure|error)\b/.test(body)) failed.push([attr('classname'), attr('name')].filter(Boolean).join('.'));
|
||||
}
|
||||
}
|
||||
}
|
||||
return read ? { executed, failed: failed.sort() } : null;
|
||||
}
|
||||
|
||||
/** Whether a workflow's concurrency groups key on the ref, the run or the nonce; a global group (or one keyed only on github.head_ref, empty on a dispatch) serializes the prefix dispatches. */
|
||||
export function concurrencyKeyed(yaml: string): boolean {
|
||||
const groups = [...yaml.matchAll(/^\s*concurrency:[ \t]*(.*)$/gm)].map(m => {
|
||||
const inline = m[1]!.trim();
|
||||
if (inline && !inline.startsWith('#')) return inline;
|
||||
const after = yaml.slice(m.index! + m[0].length).split('\n').slice(1, 4).join('\n');
|
||||
return /^\s*group:\s*(.+)$/m.exec(after)?.[1] ?? '';
|
||||
});
|
||||
return groups.every(g => /github\.(ref|run_id)\b|inputs\.nonce/.test(g));
|
||||
}
|
||||
|
||||
const json = <T>(r: { ok: boolean; out: string }): T | null => { if (!r.ok) return null; try { return JSON.parse(r.out) as T; } catch { return null; } };
|
||||
|
||||
function readRun(env: TrainEnv, slug: string, id: number): RunFacts | null {
|
||||
const r = json<Partial<RunFacts>>(env.ctx.gh(['api', `repos/${slug}/actions/runs/${id}`]));
|
||||
return r && typeof r.status === 'string' ? { status: r.status, conclusion: r.conclusion ?? null, head_sha: r.head_sha ?? '', event: r.event ?? '', run_attempt: Number(r.run_attempt) || 1 } : null;
|
||||
}
|
||||
|
||||
function readJobs(env: TrainEnv, slug: string, id: number, attempt: number): JobFacts[] | string {
|
||||
const r = json<{ total_count?: number; jobs?: Array<{ name?: string; conclusion?: string | null }> }>(env.ctx.gh(['api', `repos/${slug}/actions/runs/${id}/attempts/${attempt}/jobs?per_page=100`]));
|
||||
if (!Array.isArray(r?.jobs)) return `the jobs of run ${id} attempt ${attempt} are unreadable`;
|
||||
if ((r.total_count ?? 0) > 100) return `run ${id} has ${r.total_count} jobs; a gate reads at most 100`;
|
||||
return r.jobs.map(j => ({ name: String(j.name ?? ''), conclusion: j.conclusion ?? null }));
|
||||
}
|
||||
|
||||
function readRecord(env: TrainEnv, slug: string, t: TrainRecord, id: number, name: string): RecordFacts | null {
|
||||
const dir = path.join(trainDir(t.nonce), 'scratch', `${id}-${name}`);
|
||||
fs.rmSync(dir, { recursive: true, force: true });
|
||||
fs.mkdirSync(dir, { recursive: true });
|
||||
try {
|
||||
if (!env.ctx.gh(['run', 'download', String(id), '--repo', slug, '-n', name, '-D', dir]).ok) return null;
|
||||
const files = (fs.readdirSync(dir, { recursive: true }) as string[]).filter(f => /\.(xml|json)$/.test(f) && fs.statSync(path.join(dir, f)).isFile());
|
||||
return parseTestRecord(files.map(f => ({ name: f, text: fs.readFileSync(path.join(dir, f), 'utf8') })));
|
||||
} catch { return null; } finally { fs.rmSync(dir, { recursive: true, force: true }); }
|
||||
}
|
||||
|
||||
/** The newest attempt of one unit on one prefix. */
|
||||
export const unitAttempt = (all: AttemptRecord[], u: GateUnit, index: number) =>
|
||||
all.filter(a => a.gate === u.gate && a.workflow === u.workflow && a.prefixes[0] === index).at(-1);
|
||||
|
||||
function newAttempt(t: TrainRecord, u: GateUnit, p: GatedPrefix, nonce: string, inputs: Record<string, string>): AttemptRecord {
|
||||
const now = nowIso();
|
||||
return {
|
||||
schema: 1, kind: 'train-attempt', train: t.nonce, id: `a${t.attempts.filter(id => id.startsWith('a')).length + 1}`, purpose: 'gate', nonce, workflow: u.workflow, ref: p.ref,
|
||||
inputs, state: 'dispatching', run: null, outcome: null, detail: null, red_cases: [], panels: [], infra_eligible: false, cost_usd: null, prefixes: [p.index],
|
||||
gate: u.gate, created_at: now, updated_at: now,
|
||||
};
|
||||
}
|
||||
|
||||
/** A green run of the same unit at main's exact sha in another train's record: T0 needs no new run. */
|
||||
function reusedT0(env: TrainEnv, t: TrainRecord, u: GateUnit, gate: TrainGate): AttemptRecord | null {
|
||||
const same = (a: AttemptRecord) => a.workflow === u.workflow && a.outcome === 'GREEN' && a.prefixes[0] === 0 && !a.reused
|
||||
&& (gate.kind === 'workflows' || a.inputs.command === gate.command);
|
||||
for (const other of listTrains(env.repo.keys)) {
|
||||
if (other.nonce === t.nonce || other.gated?.[0]?.commit !== t.base.sha) continue;
|
||||
const hit = attemptsOf(other).find(same);
|
||||
if (hit) return { ...newAttempt(t, u, t.gated![0]!, hit.nonce, hit.inputs), state: 'completed', run: hit.run, outcome: 'GREEN', detail: `reused ${other.nonce}/${hit.id} (green at main ${t.base.sha.slice(0, 12)})`, reused: `${other.nonce}/${hit.id}` };
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/** Push the prefix refs (one atomic push); the record says so before any dispatch. */
|
||||
function pushPrefixes(env: TrainEnv, t: TrainRecord): string | null {
|
||||
if (t.state !== 'built') return null;
|
||||
const p = git(env.ctx.repoRoot, ['push', '-q', '--atomic', env.remote, ...t.gated!.map(x => `${x.commit}:refs/heads/${x.ref}`)]);
|
||||
if (!p.ok) { t.error = `prefix push failed: ${p.err.slice(0, 160)}`; save(env, t); return t.error; }
|
||||
t.state = 'pushed';
|
||||
effect(t, `prefix refs pushed: ${t.gated!.map(x => x.ref).join(', ')}`);
|
||||
save(env, t);
|
||||
return null;
|
||||
}
|
||||
|
||||
/** Dispatch every workflow unit that has no live attempt on a prefix; a workflow with a global concurrency group gets one prefix at a time. A unit is sent at most twice per prefix: two sends no run adopted refuse. */
|
||||
async function dispatchWorkflows(env: TrainEnv, slug: string, t: TrainRecord, units: GateUnit[], lines: string[]): Promise<QueueFailure | null> {
|
||||
const exec = execFor(env.ctx);
|
||||
for (const u of units.filter(x => x.workflow !== 'command')) {
|
||||
const yaml = git(env.ctx.repoRoot, ['show', `${env.remote}/${env.ctx.base}:.github/workflows/${u.workflow}`]);
|
||||
const serial = yaml.ok && !concurrencyKeyed(yaml.out);
|
||||
if (serial) lines.push(`SERIAL: ${u.workflow} has a concurrency group not keyed by ref; its prefixes run one at a time`);
|
||||
for (const p of t.gated!) {
|
||||
const all = attemptsOf(t);
|
||||
const prior = unitAttempt(all, u, p.index);
|
||||
if (prior && prior.state !== 'absent') continue;
|
||||
if (all.filter(a => a.gate === u.gate && a.workflow === u.workflow && a.prefixes[0] === p.index && a.state === 'absent').length >= 2) return { code: 'QUEUE_TRAIN_DISPATCH_UNKNOWN', message: `${u.workflow} on T${p.index} was sent twice under nonce ${prior!.nonce} and no run carrying it appeared; it is not sent a third time (check that its run-name names the nonce: train doctor)` };
|
||||
if (serial && t.gated!.some(q => { const a = unitAttempt(all, u, q.index); return !!a && a.state !== 'absent' && a.state !== 'completed'; })) break;
|
||||
const reuse = p.index === 0 ? reusedT0(env, t, u, env.train.gates![u.gate]!) : null;
|
||||
const nonce = prior?.nonce ?? makeNonce('gate', p.commit, t.base.sha, randomBytes(3).toString('hex'), { purpose: 'train' });
|
||||
const a = reuse ?? newAttempt(t, u, p, nonce, { nonce });
|
||||
t.attempts.push(a.id);
|
||||
save(env, t, a);
|
||||
if (reuse) { lines.push(`REUSED: T0 ${u.workflow} ${a.detail}`); continue; }
|
||||
const sent = env.ctx.gh(['workflow', 'run', u.workflow, '--repo', slug, '--ref', p.ref, '-f', `nonce=${a.nonce}`]).ok;
|
||||
const found = sent ? await awaitNonceRun(u.workflow, a.nonce, exec, slug) : null;
|
||||
a.state = found ? 'dispatched' : 'unknown';
|
||||
a.run = found ? { id: found.databaseId, url: found.url } : null;
|
||||
effect(t, `gate ${u.gate} ${u.workflow} on T${p.index}: ${found ? `run ${found.databaseId}` : 'dispatch unconfirmed'} (nonce ${a.nonce})`);
|
||||
save(env, t, a);
|
||||
lines.push(`DISPATCH: ${u.workflow} on ${p.ref} ${found ? `run ${found.databaseId}` : 'unconfirmed'} nonce ${a.nonce}`);
|
||||
if (serial) break;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/** Dollars reserved by one command gate's runs in this train (an interrupted run stays counted); each gate has its own cap. */
|
||||
function commandSpend(t: TrainRecord, gate: number): number {
|
||||
return Math.round(attemptsOf(t).filter(a => a.workflow === 'command' && a.gate === gate && !a.reused).reduce((s, a) => s + (a.spend_usd ?? 0), 0) * 100) / 100;
|
||||
}
|
||||
|
||||
const pidAlive = (pid: unknown): boolean => {
|
||||
if (typeof pid !== 'number') return false;
|
||||
try { process.kill(pid, 0); return true; } catch (e) { return (e as NodeJS.ErrnoException).code === 'EPERM'; }
|
||||
};
|
||||
|
||||
/** Whether a command attempt's process may still be running: its owner is alive on this host, or on another host within the run's time. */
|
||||
function commandLive(a: AttemptRecord, g: TrainGate | undefined): boolean {
|
||||
if (a.workflow !== 'command' || a.state !== 'dispatching' || !a.owner) return false;
|
||||
if (a.owner.host !== os.hostname()) return g?.kind === 'command' && Date.now() - Date.parse(a.updated_at) < (g.timeout_sec + COMMAND_OVERHEAD_MIN * 60) * 2000;
|
||||
return pidAlive(a.owner.pid);
|
||||
}
|
||||
|
||||
/** One build or verdict at a time writes a generic train's attempts: a lock file naming its owner, taken over once that owner is gone (another host's after six hours). */
|
||||
function trainLease(t: TrainRecord): { release: () => void } | QueueFailure {
|
||||
const file = path.join(trainDir(t.nonce), 'gate.lock');
|
||||
fs.mkdirSync(path.dirname(file), { recursive: true });
|
||||
for (let i = 0; i < 2; i++) {
|
||||
try {
|
||||
fs.writeFileSync(file, JSON.stringify({ pid: process.pid, host: os.hostname() }), { flag: 'wx' });
|
||||
return { release: () => fs.rmSync(file, { force: true }) };
|
||||
} catch (e) { if ((e as NodeJS.ErrnoException).code !== 'EEXIST') throw e; }
|
||||
let o: { pid?: unknown; host?: unknown } | null = null;
|
||||
try { o = JSON.parse(fs.readFileSync(file, 'utf8')); } catch { o = null; }
|
||||
const age = Date.now() - (fs.statSync(file, { throwIfNoEntry: false })?.mtimeMs ?? 0);
|
||||
const held = !o ? age < 10_000 : o.host === os.hostname() ? pidAlive(o.pid) : age < 6 * 3600_000;
|
||||
if (held) return { code: 'QUEUE_TRAIN_PENDING', message: `another train build or verdict (process ${String(o?.pid ?? '?')} on ${String(o?.host ?? '?')}) holds ${file}; nothing read or sent` };
|
||||
fs.rmSync(file, { force: true });
|
||||
}
|
||||
return { code: 'QUEUE_TRAIN_PENDING', message: `${file} changed hands twice while this process waited; nothing read or sent` };
|
||||
}
|
||||
|
||||
/** Run one command gate on a prefix through ubi-runner; resolves with the outcome. */
|
||||
async function runCommand(env: TrainEnv, t: TrainRecord, a: AttemptRecord, g: CommandGate, p: GatedPrefix): Promise<{ outcome: GateOutcome; detail: string; tree: string | null }> {
|
||||
const wt = path.join(queueRoot(), 'gw', `${a.nonce.slice(-6)}-${p.index}-${a.id}`);
|
||||
fs.rmSync(wt, { recursive: true, force: true });
|
||||
fs.mkdirSync(path.dirname(wt), { recursive: true });
|
||||
if (!git(env.ctx.repoRoot, ['worktree', 'add', '--detach', '-q', wt, p.commit]).ok) return { outcome: 'INCOMPLETE', detail: `cannot check out T${p.index}`, tree: null };
|
||||
try {
|
||||
const dirty = git(wt, ['status', '--porcelain', '--untracked-files=all']);
|
||||
if (!dirty.ok || dirty.out) return { outcome: 'INCOMPLETE', detail: 'the checkout of the prefix is not clean', tree: null };
|
||||
const remote = `git add -A >/dev/null 2>&1 && printf 'GSTACK_GATE_TREE=%s\n' "$(git write-tree)" && ( ${g.command} )`;
|
||||
const inv = scriptInvocation(process.env.GSTACK_TRAIN_UBI_RUNNER || UBI_RUNNER, ['run', '--src', wt, ...(g.pass_env ?? []).flatMap(n => ['--pass', n]), '--', remote]);
|
||||
if (!inv.ok) return { outcome: 'INCOMPLETE', detail: inv.error, tree: null };
|
||||
const childEnv = { ...process.env, UBI_SIZE: COMMAND_SIZE, UBI_RUN_TIMEOUT: String(g.timeout_sec), UBI_TTL_HOURS: String(Math.ceil(g.timeout_sec / 3600) + 1) };
|
||||
const child = spawn(inv.cmd, inv.argv, { cwd: wt, env: childEnv, timeout: (g.timeout_sec + COMMAND_OVERHEAD_MIN * 60) * 1000 });
|
||||
let out = '';
|
||||
let tree = null as string | null;
|
||||
const keep = (b: Buffer) => { const all = out + b.toString('utf8'); tree ??= /^GSTACK_GATE_TREE=([0-9a-f]{40,64})$/m.exec(all)?.[1] ?? null; out = all.slice(-1_000_000); };
|
||||
child.stdout.on('data', keep);
|
||||
child.stderr.on('data', keep);
|
||||
const code = await new Promise<number | null>(resolve => { child.on('close', c => resolve(c)); child.on('error', () => resolve(null)); });
|
||||
const log = path.join(trainDir(t.nonce), 'attempts', `${a.id}.log`);
|
||||
fs.mkdirSync(path.dirname(log), { recursive: true });
|
||||
fs.writeFileSync(log, out, { mode: 0o600 });
|
||||
const left = [...out.matchAll(/UBICLOUD: left=(\S+)/g)].at(-1)?.[1];
|
||||
if (!tree) return { outcome: 'INCOMPLETE', detail: `the VM reported no tree (exit ${code ?? 'none'}; log ${log})`, tree };
|
||||
if (tree !== p.tree) return { outcome: 'INCOMPLETE', detail: `the VM tested tree ${tree.slice(0, 12)}, not the prefix tree ${p.tree.slice(0, 12)}`, tree };
|
||||
if (left !== undefined && left !== '0') return { outcome: 'INCOMPLETE', detail: `ubi-runner reports left=${left}; the run is not counted`, tree };
|
||||
if (code === 0) return { outcome: 'GREEN', detail: `exit 0 on tree ${tree.slice(0, 12)}`, tree };
|
||||
if (code === 124 || code === null) return { outcome: 'INCOMPLETE', detail: `the command ${code === 124 ? 'timed out' : 'did not finish'} (log ${log})`, tree };
|
||||
return { outcome: 'RED', detail: `exit ${code} (log ${log})`, tree };
|
||||
} finally {
|
||||
git(env.ctx.repoRoot, ['worktree', 'remove', '--force', wt]);
|
||||
fs.rmSync(wt, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
/** Run one command attempt; a red run gets one rerun while the cap allows: a pass is GREEN and FLAKY, a second red is real. */
|
||||
async function settleCommand(env: TrainEnv, t: TrainRecord, a: AttemptRecord, g: CommandGate, p: GatedPrefix, lines: string[]): Promise<void> {
|
||||
const price = commandReservation(g);
|
||||
let r = await runCommand(env, t, a, g, p);
|
||||
if (r.outcome === 'RED' && commandSpend(t, a.gate!) + price <= g.spend_cap_usd) {
|
||||
a.spend_usd = (a.spend_usd ?? 0) + price;
|
||||
save(env, t, a);
|
||||
const again = await runCommand(env, t, a, g, p);
|
||||
if (again.outcome === 'GREEN') { a.flaky = [`command:${a.gate}`]; r = { ...again, detail: `${again.detail}; FLAKY: the first run was red (${r.detail})` }; }
|
||||
else r = again.outcome === 'RED' ? { ...again, detail: `red twice: ${again.detail}` } : again;
|
||||
}
|
||||
Object.assign(a, { state: 'completed', outcome: r.outcome, detail: r.detail, tree: r.tree, red_cases: r.outcome === 'RED' ? [`command:${a.gate}`] : [] });
|
||||
effect(t, `gate ${a.gate} command on T${p.index}: ${r.outcome}`);
|
||||
save(env, t, a);
|
||||
lines.push(`COMMAND: gate ${a.gate} on T${p.index} ${r.outcome} (${r.detail})`);
|
||||
}
|
||||
|
||||
/** Run every command unit that has no finished attempt, all prefixes in parallel, under the train's one spend cap. */
|
||||
async function runCommands(env: TrainEnv, t: TrainRecord, units: GateUnit[], lines: string[]): Promise<QueueFailure | null> {
|
||||
const commands = units.filter(u => u.workflow === 'command');
|
||||
if (!commands.length) return null;
|
||||
if (!process.env.UBI_OWNER) return { code: 'QUEUE_TRAIN_SPEND_CAP', message: 'a command gate runs on a Ubicloud VM, and UBI_OWNER is not set (every VM names its owner); nothing ran' };
|
||||
const jobs: Array<Promise<void>> = [];
|
||||
for (const u of commands) {
|
||||
const g = env.train.gates![u.gate] as CommandGate;
|
||||
const price = commandReservation(g);
|
||||
for (const p of t.gated!) {
|
||||
if (unitAttempt(attemptsOf(t), u, p.index)?.state === 'completed') continue;
|
||||
const reuse = p.index === 0 ? reusedT0(env, t, u, g) : null;
|
||||
if (reuse) { t.attempts.push(reuse.id); save(env, t, reuse); lines.push(`REUSED: T0 command ${reuse.detail}`); continue; }
|
||||
if (commandSpend(t, u.gate) + price > g.spend_cap_usd) {
|
||||
await Promise.all(jobs);
|
||||
return { code: 'QUEUE_TRAIN_SPEND_CAP', message: `gate ${u.gate}'s run on T${p.index} reserves $${price.toFixed(2)}; with $${commandSpend(t, u.gate).toFixed(2)} reserved, this train would pass the gate's cap $${g.spend_cap_usd.toFixed(2)}` };
|
||||
}
|
||||
const a = newAttempt(t, u, p, makeNonce('gate', p.commit, t.base.sha, randomBytes(3).toString('hex'), { purpose: 'train' }), { command: g.command });
|
||||
a.spend_usd = price;
|
||||
a.owner = { pid: process.pid, host: os.hostname() };
|
||||
t.attempts.push(a.id);
|
||||
save(env, t, a);
|
||||
jobs.push(settleCommand(env, t, a, g, p, lines));
|
||||
}
|
||||
}
|
||||
await Promise.all(jobs);
|
||||
return null;
|
||||
}
|
||||
|
||||
/** `train build` for a generic train: push the prefix refs, dispatch every workflow unit, run every command unit. */
|
||||
export async function advanceGeneric(env: TrainEnv, t: TrainRecord, lines: string[], describe: (t: TrainRecord) => string[]): Promise<TrainOut> {
|
||||
const run = trainFile(t.nonce);
|
||||
const slug = repoSlug(env.ctx);
|
||||
if (!slug) return outRefused({ code: 'QUEUE_PROOF_MISSING', message: 'gh repo view could not name the repository' }, lines, `${Q} train build --train ${t.nonce}`, run);
|
||||
const lease = trainLease(t);
|
||||
if ('code' in lease) return outPending(lease, lines, `${Q} train build --train ${t.nonce} (once that process finishes)`, run);
|
||||
try {
|
||||
Object.assign(t, loadTrain(t.nonce) ?? {});
|
||||
const pushed = pushPrefixes(env, t);
|
||||
if (pushed) return outRefused({ code: 'QUEUE_PROOF_MISSING', message: pushed }, lines, `${Q} train build --train ${t.nonce}`, run);
|
||||
const units = gateUnits(env.train.gates!);
|
||||
if (t.verdict) return outOk([...lines, ...describe(t), `VERDICT: ${t.verdict.kind.toUpperCase()} (settled; nothing dispatched)`], genericNext(t), run);
|
||||
const live = attemptsOf(t).find(a => commandLive(a, env.train.gates![a.gate ?? -1]));
|
||||
if (live) return outPending({ code: 'QUEUE_TRAIN_PENDING', message: `command run ${live.id} is still running under process ${live.owner!.pid} on ${live.owner!.host}` }, lines, `${Q} train verdict --train ${t.nonce} (once that run finishes)`, run);
|
||||
reconcile(env, slug, t);
|
||||
const unsent = await dispatchWorkflows(env, slug, t, units, lines);
|
||||
const capped = unsent ? null : await runCommands(env, t, units, lines);
|
||||
t.state = 'gate-pending';
|
||||
save(env, t);
|
||||
if (unsent) return outRefused(unsent, [...lines, ...describe(t)], `run ${Q} train doctor and fix the workflow on the base branch, then build a new train on the new main: ${Q} train build ${prsArgs(t.slots, t.explicit_order)}`, run);
|
||||
if (capped) return outRefused(capped, [...lines, ...describe(t)], `raise the gate's spend_cap_usd (or lower its timeout_sec) on the base branch, then build a new train on the new main: ${Q} train build ${prsArgs(t.slots, t.explicit_order)}`, run);
|
||||
return outOk([...lines, ...describe(t), `GATES: ${units.length} unit(s) on ${t.gated!.length} prefix(es); record: ${run}`], `${Q} train verdict --train ${t.nonce}`, run);
|
||||
} finally { lease.release(); }
|
||||
}
|
||||
|
||||
/** Settle unconfirmed workflow dispatches by nonce (found, or absent so the next build re-sends the same nonce); an interrupted command run is absent. */
|
||||
function reconcile(env: TrainEnv, slug: string, t: TrainRecord): void {
|
||||
for (const a of attemptsOf(t)) {
|
||||
if (a.state !== 'dispatching' && a.state !== 'unknown') continue;
|
||||
if (a.workflow === 'command') {
|
||||
if (commandLive(a, env.train.gates?.[a.gate ?? -1])) continue;
|
||||
a.state = 'absent'; effect(t, `command run ${a.id} was interrupted; it runs again`); save(env, t, a); continue;
|
||||
}
|
||||
const runs = listDispatchRuns(a.workflow, execFor(env.ctx), slug, 100);
|
||||
if (!runs) continue;
|
||||
const hit = runForNonce(runs, a.nonce);
|
||||
a.state = hit ? 'dispatched' : 'absent';
|
||||
a.run = hit ? { id: hit.databaseId, url: hit.url } : null;
|
||||
effect(t, `reconciled ${a.id} (nonce ${a.nonce}): ${hit ? `run ${hit.databaseId}` : 'absent'}`);
|
||||
save(env, t, a);
|
||||
}
|
||||
}
|
||||
|
||||
/** Read one workflow attempt's run; a first red asks for one rerun of the failed jobs and waits for it. */
|
||||
function readWorkflowAttempt(env: TrainEnv, slug: string, t: TrainRecord, a: AttemptRecord): void {
|
||||
const gate = env.train.gates![a.gate!] as WorkflowsGate;
|
||||
const run = readRun(env, slug, a.run!.id);
|
||||
if (!run) { a.detail = `run ${a.run!.id} is unreadable`; return; }
|
||||
if (run.status !== 'completed' || (a.rerun === 'requested' && run.run_attempt < 2)) {
|
||||
Object.assign(a, { outcome: 'PENDING', detail: `run ${a.run!.id} attempt ${run.run_attempt} is ${run.status}${a.rerun === 'requested' ? ' (waiting for the rerun)' : ''}` });
|
||||
return;
|
||||
}
|
||||
const jobs = readJobs(env, slug, a.run!.id, run.run_attempt);
|
||||
if (typeof jobs === 'string') { Object.assign(a, { outcome: 'INCOMPLETE', state: 'completed', detail: jobs }); return; }
|
||||
const earlier = (a.jobs ?? []).filter(j => j.attempt < run.run_attempt);
|
||||
const merged = new Map(earlier.filter(j => j.attempt === run.run_attempt - 1).map(j => [j.name, j.conclusion] as const));
|
||||
for (const j of jobs) merged.set(j.name, j.conclusion);
|
||||
a.jobs = [...earlier, ...jobs.map(j => ({ attempt: run.run_attempt, ...j }))];
|
||||
a.run_attempt = run.run_attempt;
|
||||
const required = gate.required_jobs.filter(r => r.startsWith(`${a.workflow}/`)).map(r => r.slice(a.workflow.length + 1));
|
||||
const record = gate.record ? readRecord(env, slug, t, a.run!.id, gate.record) : undefined;
|
||||
const c = classifyWorkflowRun({ run, head: t.gated![a.prefixes[0]!]!.commit, workflow: a.workflow, jobs: [...merged].map(([name, conclusion]) => ({ name, conclusion })), required, record });
|
||||
if (c.outcome === 'RED' && !a.rerun) {
|
||||
const sent = env.ctx.gh(['run', 'rerun', String(a.run!.id), '--failed', '--repo', slug]).ok;
|
||||
a.rerun = sent ? 'requested' : 'refused';
|
||||
a.red_cases = c.red;
|
||||
if (sent) { Object.assign(a, { outcome: 'PENDING', detail: `red on ${c.red.join(', ')}; one rerun of the failed jobs requested (CREW rule 16)` }); effect(t, `rerun requested for run ${a.run!.id}`); return; }
|
||||
}
|
||||
const firstRed = a.rerun === 'requested' ? a.red_cases : [];
|
||||
Object.assign(a, { state: 'completed', outcome: c.outcome, red_cases: c.red, detail: c.detail });
|
||||
if (c.outcome === 'GREEN' && firstRed.length) { a.flaky = firstRed; a.detail = `${c.detail}; FLAKY: ${firstRed.join(', ')} red on attempt 1, green on the rerun`; }
|
||||
else if (c.outcome === 'RED' && a.rerun === 'requested') a.detail = `red on the rerun too: ${c.detail}`;
|
||||
}
|
||||
|
||||
/** Per prefix: PENDING while any unit is, then INCOMPLETE, then RED (red cases united), else GREEN. */
|
||||
function aggregate(env: TrainEnv, t: TrainRecord): void {
|
||||
const all = attemptsOf(t);
|
||||
const units = gateUnits(env.train.gates!);
|
||||
for (const p of t.gated!) {
|
||||
const attempts = units.map(u => unitAttempt(all, u, p.index));
|
||||
const outs = attempts.map(a => (!a || a.state === 'absent' ? 'PENDING' : a.outcome ?? 'PENDING'));
|
||||
p.outcome = outs.includes('PENDING') ? 'PENDING' : outs.includes('INCOMPLETE') ? 'INCOMPLETE' : outs.includes('RED') ? 'RED' : 'GREEN';
|
||||
p.red = p.outcome === 'RED' ? [...new Set(attempts.flatMap(a => (a?.outcome === 'RED' ? a.red_cases : [])))].sort() : [];
|
||||
p.detail = attempts.map((a, i) => `${units[i]!.workflow}: ${!a || a.state === 'absent' ? 'not dispatched' : a.detail ?? a.state}`).join('; ');
|
||||
}
|
||||
}
|
||||
|
||||
const VERDICT_STATE = { green: 'green', attributed: 'attributed', 'red-on-main': 'red-on-main', unattributed: 'unattributed', incomplete: 'incomplete' } as const;
|
||||
|
||||
/** The coded failure for a settled, non-green verdict. */
|
||||
function verdictFailure(t: TrainRecord, v: GenericVerdict): QueueFailure {
|
||||
if (v.kind === 'attributed') {
|
||||
return t.slots[0]?.fixes_base_red?.length && v.culprit === t.slots[0]!.pr && t.slots.some(s => s.base_red)
|
||||
? { code: 'QUEUE_TRAIN_BASE_RED', message: `fixer #${v.culprit}'s prefix T1 is red (${t.gated![1]!.red.join(', ')}); its conditional PRs cannot ride without it` }
|
||||
: { code: 'QUEUE_TRAIN_RED', message: `T${v.landable + 1} is the first red prefix after a green one: #${v.culprit} is held${v.landable ? `; slots 1..${v.landable} may land` : ''}` };
|
||||
}
|
||||
if (v.kind === 'red-on-main') {
|
||||
const fixes = t.slots[0]?.fixes_base_red;
|
||||
return { code: 'QUEUE_TRAIN_RED_ON_MAIN', message: `T0 (main ${t.base.sha.slice(0, 12)}) is red on ${t.gated![0]!.red.join(', ') || 'its gate'}${fixes?.length ? `, not exactly slot 1's FIXES_BASE_RED (${fixes.join(', ')})` : ''}` };
|
||||
}
|
||||
if (v.kind === 'unattributed') return { code: 'QUEUE_TRAIN_UNATTRIBUTED', message: 'a red prefix is followed by a green one, so no slot is the culprit; a generic train gets no unchanged rebuild' };
|
||||
return { code: 'QUEUE_TRAIN_INCOMPLETE', message: `a prefix gate is incomplete: ${t.gated!.find(p => p.outcome === 'INCOMPLETE')?.detail ?? 'see the PREFIX lines'}` };
|
||||
}
|
||||
|
||||
/** `train verdict` for a generic train: read every run, settle reruns, then classify the prefixes. */
|
||||
export function genericVerdict(env: TrainEnv, t: TrainRecord): TrainOut {
|
||||
const run = trainFile(t.nonce);
|
||||
const slug = repoSlug(env.ctx);
|
||||
if (!slug) return outPending({ code: 'QUEUE_PROOF_MISSING', message: 'gh repo view could not name the repository' }, [], `${Q} train verdict --train ${t.nonce}`, run);
|
||||
if (!t.verdict) {
|
||||
const lease = trainLease(t);
|
||||
if ('code' in lease) return outPending(lease, [], `${Q} train verdict --train ${t.nonce} (once that process finishes)`, run);
|
||||
try {
|
||||
Object.assign(t, loadTrain(t.nonce) ?? {});
|
||||
const live = t.verdict ? null : attemptsOf(t).find(a => commandLive(a, env.train.gates![a.gate ?? -1]));
|
||||
if (live) return outPending({ code: 'QUEUE_TRAIN_PENDING', message: `command run ${live.id} is still running under process ${live.owner!.pid} on ${live.owner!.host}; nothing read or saved` }, [], `${Q} train verdict --train ${t.nonce} (once that run finishes)`, run);
|
||||
if (!t.verdict) {
|
||||
reconcile(env, slug, t);
|
||||
for (const a of attemptsOf(t)) {
|
||||
if (a.workflow === 'command' || a.state !== 'dispatched' || !a.run) continue;
|
||||
readWorkflowAttempt(env, slug, t, a);
|
||||
save(env, t, a);
|
||||
}
|
||||
aggregate(env, t);
|
||||
const c = classifyPrefixes(t.gated!, t.slots);
|
||||
if (c.kind !== 'pending') { t.verdict = { ...c, at: nowIso() }; t.state = VERDICT_STATE[c.kind]; effect(t, `verdict: ${c.kind}${c.culprit ? ` (culprit #${c.culprit})` : ''}`); }
|
||||
save(env, t);
|
||||
}
|
||||
} finally { lease.release(); }
|
||||
}
|
||||
const lines = [
|
||||
...t.gated!.map(p => `PREFIX: T${p.index} ${p.outcome ?? 'PENDING'} ${p.commit.slice(0, 12)} ${p.ref}${p.red.length ? ` red=${p.red.join(',')}` : ''}${p.detail ? ` (${p.detail})` : ''}`),
|
||||
...attemptsOf(t).filter(a => a.flaky?.length).map(a => `FLAKY: T${a.prefixes[0]} ${a.workflow} ${a.flaky!.join(', ')} (run ${a.run?.id ?? '-'}; flake recorded on ${a.id})`),
|
||||
];
|
||||
const v = t.verdict;
|
||||
if (!v) {
|
||||
const waiting = t.gated!.some(p => /not dispatched/.test(p.detail ?? ''));
|
||||
return outPending({ code: 'QUEUE_TRAIN_PENDING', message: `prefix gates still running${waiting ? ' or waiting for their serial turn' : ''}` }, lines, waiting ? `${Q} train build --train ${t.nonce}` : `${Q} train verdict --train ${t.nonce}`, run);
|
||||
}
|
||||
lines.push(`VERDICT: ${v.kind.toUpperCase()} train ${t.nonce} landable=${v.landable}/${t.slots.length}${v.culprit ? ` culprit=#${v.culprit}` : ''}`);
|
||||
return v.kind === 'green' ? outOk(lines, genericNext(t), run) : outFailed(verdictFailure(t, v), lines, genericNext(t), run);
|
||||
}
|
||||
|
||||
/** NEXT for a generic train. */
|
||||
export function genericNext(t: TrainRecord): string {
|
||||
const n = t.nonce;
|
||||
const merged = (pr: string) => t.merged.some(m => m.pr === pr);
|
||||
if (t.state === 'built' || t.state === 'pushed') return `${Q} train build --train ${n}`;
|
||||
if (!t.verdict) return `${Q} train verdict --train ${n}`;
|
||||
const v = t.verdict;
|
||||
const next = t.slots.slice(0, v.landable).find(s => !merged(s.pr));
|
||||
if (next) return `${Q} train land --train ${n} --pr ${next.pr}`;
|
||||
const rest = t.slots.filter(s => !merged(s.pr) && s.pr !== v.culprit);
|
||||
if (v.kind === 'green' || v.kind === 'attributed') return rest.length ? `${Q} train build ${prsArgs(rest, t.explicit_order)}` : `${Q} train prune --train ${n}`;
|
||||
if (v.kind === 'red-on-main') return `fix forward on main first, then ${Q} train build ${prsArgs(t.slots, t.explicit_order)}`;
|
||||
if (v.kind === 'unattributed') return `repair the red cases, or seat the PRs in another order (never this train unchanged): ${Q} train build --prs ${t.slots.map(s => `${s.pr}@${s.head}`).join(',')} --order <new order>`;
|
||||
return `diagnose the incomplete prefix run, then ${Q} train build ${prsArgs(t.slots, t.explicit_order)} (a new train)`;
|
||||
}
|
||||
|
||||
/** Freeze backstop for a generic train: every workflow unit dispatched on the base branch by nonce; the first run is recorded. */
|
||||
export async function genericBackstop(env: TrainEnv, mainSha: string): Promise<{ nonce: string; run: number | null; state: string }> {
|
||||
const slug = repoSlug(env.ctx);
|
||||
const nonce = makeNonce('gate', mainSha, mainSha, randomBytes(3).toString('hex'), { purpose: 'backstop' });
|
||||
const units = gateUnits(env.train.gates!).filter(u => u.workflow !== 'command');
|
||||
let first: number | null = null;
|
||||
for (const u of units) {
|
||||
if (!slug || !env.ctx.gh(['workflow', 'run', u.workflow, '--repo', slug, '--ref', env.ctx.base, '-f', `nonce=${nonce}`]).ok) continue;
|
||||
const found = await awaitNonceRun(u.workflow, nonce, execFor(env.ctx), slug);
|
||||
first ??= found?.databaseId ?? null;
|
||||
}
|
||||
return { nonce, run: first, state: first ? 'dispatched' : units.length ? 'unconfirmed' : 'not dispatched (command gates only: run the gate command on main)' };
|
||||
}
|
||||
@@ -0,0 +1,235 @@
|
||||
/**
|
||||
* `train init` and `train doctor` (docs/ship-policy.md#run-a-merge-train; plan GSTA-157 PR 2, item 12). Both only
|
||||
* read: branch protection (the strict flag and the required checks), the base branch's .github/workflows/* and the
|
||||
* open PRs. `init` prints a `train` block with gates for the base policy, the `nonce` input, run-name and
|
||||
* concurrency snippet each gate workflow needs, and the land mode slots will use; it recommends changelog_fragments
|
||||
* when open PRs bump the version, and refuses (QUEUE_TRAIN_INIT_REFUSED) only when they do and the policy neither
|
||||
* declares fragments nor stamps at merge. `doctor` re-checks a live block: a gate workflow missing, without the
|
||||
* nonce input or run-name, a required job it does not define, a concurrency group not keyed by ref, a push trigger
|
||||
* that matches train-prefix/ refs, a workflow never green on the base, and a merge method GitHub or history refuses.
|
||||
* Neither writes anything.
|
||||
*/
|
||||
import { loadPolicyFromBase, isUnavailable, TRAIN_GATES_KEY_VERSION, globToRegExp, type ShipPolicy } from './ship-policy';
|
||||
import type { QueueContext } from './ship-queue';
|
||||
import { git } from './ship-queue-gate';
|
||||
import { versionAtRef } from './restamp';
|
||||
import { commandReservation, concurrencyKeyed } from './ship-queue-train-gate';
|
||||
import { repoSlug } from './ship-queue-train-github';
|
||||
import { UNPROTECTED } from './ship-queue-train-land';
|
||||
import { fetchBase, outFailed, outOk, outRefused, type TrainOut } from './ship-queue-train';
|
||||
|
||||
const Q = 'gstack-ship-queue';
|
||||
const MAX_SIZE = 5;
|
||||
const unquote = (s: string) => s.trim().replace(/\s+#.*$/, '').replace(/^(['"])(.*)\1$/, '$2');
|
||||
|
||||
export interface WorkflowFacts {
|
||||
file: string; name: string; jobs: Array<{ id: string; name: string; matrix: boolean }>; nonce: boolean; runName: boolean; keyed: boolean; pushMatchesPrefix: boolean;
|
||||
}
|
||||
|
||||
/** The top-level block of a workflow under `key:` (its indented lines), or null when absent. */
|
||||
function block(yaml: string, key: string, indent = ''): string[] | null {
|
||||
const lines = yaml.split('\n');
|
||||
const at = lines.findIndex(l => new RegExp(`^${indent}${key}:\\s*(#.*)?$`).test(l));
|
||||
if (at < 0) return null;
|
||||
const out: string[] = [];
|
||||
for (const l of lines.slice(at + 1)) {
|
||||
if (l.trim() && !l.startsWith(indent + ' ') && !l.trim().startsWith('#')) break;
|
||||
out.push(l);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Whether GitHub's branch filter (`*` inside a segment, `**` across) matches a train-prefix ref. */
|
||||
function filterMatches(patterns: string[]): boolean {
|
||||
const ref = 'train-prefix/gate.train-0123456789ab-0123456789ab-abcdef/t1';
|
||||
return patterns.some(p => { try { return globToRegExp(p).test(ref); } catch { return true; } });
|
||||
}
|
||||
|
||||
/** Whether a push trigger runs on train-prefix/ refs: a bare push, or a branches filter that matches them and no branches-ignore that excludes them. */
|
||||
export function pushMatchesPrefix(yaml: string): boolean {
|
||||
const on = /^(?:on|"on"|'on'):[ \t]*(.*)$/m.exec(yaml);
|
||||
if (!on) return false;
|
||||
const inline = unquote(on[1]!);
|
||||
if (inline) return /\bpush\b/.test(inline);
|
||||
const push = block(yaml.slice(on.index!), 'push', ' ');
|
||||
if (push === null) return false;
|
||||
const list = (key: string): string[] | null => {
|
||||
const at = push.findIndex(l => new RegExp(`^\\s+${key}:`).test(l));
|
||||
if (at < 0) return null;
|
||||
const flow = /\[(.*)\]/.exec(push[at]!);
|
||||
if (flow) return flow[1]!.split(',').map(unquote).filter(Boolean);
|
||||
const out: string[] = [];
|
||||
for (const l of push.slice(at + 1)) { const m = /^\s+-\s*(.+)$/.exec(l); if (!m) break; out.push(unquote(m[1]!)); }
|
||||
return out;
|
||||
};
|
||||
const branches = list('branches');
|
||||
const ignore = list('branches-ignore');
|
||||
if (branches) return filterMatches(branches);
|
||||
if (ignore) return !filterMatches(ignore);
|
||||
return !list('tags');
|
||||
}
|
||||
|
||||
/** What train doctor reads from one workflow file (line-based; the snippets init prints are the supported shape). */
|
||||
export function workflowFacts(file: string, yaml: string): WorkflowFacts {
|
||||
const jobsBlock = block(yaml, 'jobs') ?? [];
|
||||
const jobs: WorkflowFacts['jobs'] = [];
|
||||
for (const [i, l] of jobsBlock.entries()) {
|
||||
const m = /^ ([A-Za-z0-9_-]+):\s*(#.*)?$/.exec(l);
|
||||
if (!m) continue;
|
||||
const body: string[] = [];
|
||||
for (const b of jobsBlock.slice(i + 1)) { if (/^ \S/.test(b)) break; body.push(b); }
|
||||
const name = body.map(b => /^ name:\s*(.+)$/.exec(b)?.[1]).find(Boolean);
|
||||
jobs.push({ id: m[1]!, name: name ? unquote(name) : m[1]!, matrix: body.some(b => /^\s+matrix:/.test(b)) });
|
||||
}
|
||||
const runName = /^run-name:\s*(.+)$/m.exec(yaml)?.[1] ?? '';
|
||||
return {
|
||||
file, name: unquote(/^name:\s*(.+)$/m.exec(yaml)?.[1] ?? file), jobs,
|
||||
nonce: /workflow_dispatch:[\s\S]*?inputs:[\s\S]*?^\s+nonce:/m.test(yaml), runName: /inputs\.nonce/.test(runName) && /eval-bg/.test(runName),
|
||||
keyed: concurrencyKeyed(yaml), pushMatchesPrefix: pushMatchesPrefix(yaml),
|
||||
};
|
||||
}
|
||||
|
||||
/** Whether a required job name is defined: a job's name, or a matrix job's name followed by its matrix values. */
|
||||
const defines = (w: WorkflowFacts, job: string) => w.jobs.some(j => j.name === job || (j.matrix && job.startsWith(`${j.name} (`)));
|
||||
|
||||
export const NONCE_SNIPPET = [
|
||||
'on:',
|
||||
' workflow_dispatch:',
|
||||
' inputs:',
|
||||
" nonce: { description: 'gstack merge-train nonce', required: false, type: string }",
|
||||
"run-name: ${{ inputs.nonce && format('{0} [eval-bg {1}]', github.workflow, inputs.nonce) || github.workflow }}",
|
||||
'concurrency:',
|
||||
" group: ${{ github.workflow }}-${{ github.ref }}",
|
||||
].join('\n');
|
||||
|
||||
interface Repo { ctx: QueueContext; policy: ShipPolicy; base: string; slug: string | null; workflows: WorkflowFacts[]; strict: boolean | null; required: string[] }
|
||||
|
||||
function readRepo(ctx: QueueContext): Repo | TrainOut {
|
||||
const fetchErr = fetchBase(ctx);
|
||||
if (fetchErr) return outRefused({ code: 'POLICY_SOURCE_UNAVAILABLE', message: fetchErr }, [], 'none', 'none');
|
||||
const loaded = loadPolicyFromBase(ctx.repoRoot, ctx.base);
|
||||
if (isUnavailable(loaded)) return outRefused({ code: 'POLICY_SOURCE_UNAVAILABLE', message: loaded.detail }, [], 'none', 'none');
|
||||
const ref = `${ctx.remote ?? 'origin'}/${ctx.base}`;
|
||||
const files = git(ctx.repoRoot, ['ls-tree', '--name-only', `${ref}:.github/workflows/`]).out.split('\n').filter(f => /\.ya?ml$/.test(f));
|
||||
const workflows = files.map(f => workflowFacts(f, git(ctx.repoRoot, ['show', `${ref}:.github/workflows/${f}`]).out));
|
||||
const prot = ctx.gh(['api', `repos/{owner}/{repo}/branches/${ctx.base}/protection/required_status_checks`]);
|
||||
let strict: boolean | null = null; let required: string[] = [];
|
||||
if (prot.ok) {
|
||||
try {
|
||||
const p = JSON.parse(prot.out) as { strict?: boolean; contexts?: string[]; checks?: Array<{ context?: string }> };
|
||||
strict = p.strict === true;
|
||||
required = [...new Set([...(p.contexts ?? []), ...(p.checks ?? []).map(c => c.context ?? '')].filter(Boolean))];
|
||||
} catch { strict = null; }
|
||||
} else if (UNPROTECTED.test(`${prot.err ?? ''} ${prot.out}`)) strict = false;
|
||||
return { ctx, policy: loaded.policy, base: ctx.base, slug: repoSlug(ctx), workflows, strict, required };
|
||||
}
|
||||
|
||||
/** Open PRs whose head bumps the version against their merge base (VERSION or the manifest's version), by number. */
|
||||
function versionBumpingPrs(r: Repo): number[] | string {
|
||||
const list = r.ctx.gh(['pr', 'list', '--state', 'open', '--base', r.base, '--limit', '100', '--json', 'number,headRefOid']);
|
||||
let prs: Array<{ number: number; headRefOid: string }>;
|
||||
try { prs = list.ok ? JSON.parse(list.out) : null; } catch { prs = null as never; }
|
||||
if (!Array.isArray(prs)) return `gh pr list could not be read (${(list.err ?? list.out).trim().slice(0, 120)})`;
|
||||
if (!prs.length) return [];
|
||||
const remote = r.ctx.remote ?? 'origin';
|
||||
const f = git(r.ctx.repoRoot, ['fetch', '--no-tags', '-q', remote, ...prs.map(p => `+refs/pull/${p.number}/head:refs/gstack/init/pr-${p.number}`)]);
|
||||
if (!f.ok) return `git fetch of the open PR heads failed: ${f.err.slice(0, 120)}`;
|
||||
const version = (rev: string) => { const v = versionAtRef(r.ctx.repoRoot, rev); return 'version' in v ? v.version : null; };
|
||||
return prs.filter(p => {
|
||||
const head = `refs/gstack/init/pr-${p.number}`;
|
||||
const fork = git(r.ctx.repoRoot, ['merge-base', head, `${remote}/${r.base}`]).out;
|
||||
return !!fork && version(head) !== version(fork);
|
||||
}).map(p => p.number);
|
||||
}
|
||||
|
||||
/** The version-file rule both commands apply: recommend fragments when PRs bump the version; refuse only without fragments and without stamp-at-merge. */
|
||||
function versionRule(r: Repo, lines: string[]): TrainOut | null {
|
||||
const bumping = versionBumpingPrs(r);
|
||||
if (typeof bumping === 'string') return outRefused({ code: 'QUEUE_PROOF_MISSING', message: bumping }, lines, `${Q} train init (once gh answers)`, 'none');
|
||||
const fragments = !!r.policy.changelog_fragments;
|
||||
const stamps = r.policy.queue_mode === 'stamp-at-merge';
|
||||
lines.push(`VERSION_PRS: ${bumping.length ? bumping.map(n => '#' + n).join(', ') + ' bump the version in the PR' : 'no open PR bumps the version'}; changelog_fragments ${fragments ? 'declared' : 'absent'}; queue_mode ${r.policy.queue_mode}`);
|
||||
if (bumping.length && !fragments) lines.push('RECOMMEND: changelog_fragments (docs/ship-policy.md#changelog-fragments): each slot is then stamped at merge, so PRs stop editing version files');
|
||||
if (bumping.length && !fragments && !stamps) return outRefused({ code: 'QUEUE_TRAIN_INIT_REFUSED', message: `${bumping.length} open PR(s) bump the version and the policy neither declares changelog_fragments nor stamps at merge` }, lines, 'add changelog_fragments to the base policy (recommended) or set queue_mode stamp-at-merge, then train init again', 'none');
|
||||
return null;
|
||||
}
|
||||
|
||||
/** Warnings about one gate workflow, shared by init (for the workflows it proposes) and doctor. */
|
||||
function workflowFindings(w: WorkflowFacts): string[] {
|
||||
return [
|
||||
...(w.nonce ? [] : [`${w.file}: no workflow_dispatch input named nonce (add the snippet)`]),
|
||||
...(w.runName ? [] : [`${w.file}: run-name does not name [eval-bg <inputs.nonce>], so a dispatch cannot be adopted by its nonce`]),
|
||||
...(w.keyed ? [] : [`${w.file}: a concurrency group not keyed by ref; the train dispatches its prefixes one at a time`]),
|
||||
...(w.pushMatchesPrefix ? [`${w.file}: a push trigger matches train-prefix/ refs, so every prefix push starts an extra run (add branches-ignore: ['train-prefix/**'])`] : []),
|
||||
];
|
||||
}
|
||||
|
||||
/** `train init`: print the block, the snippets and the land mode; refuse version-bumping PRs without fragments or stamp-at-merge. */
|
||||
export function trainInit(ctx: QueueContext, o: { command?: string; timeoutSec?: number; spendCap?: number }): TrainOut {
|
||||
const r = readRepo(ctx);
|
||||
if ('exit' in r) return r;
|
||||
const lines: string[] = [`INIT: ${r.slug ?? 'repository'} base ${r.base}; ${r.workflows.length} workflow file(s); protection ${r.strict === null ? 'unreadable' : r.strict ? 'strict (up to date required)' : 'not strict'}; required checks ${r.required.join(', ') || '(none)'}`];
|
||||
const refused = versionRule(r, lines);
|
||||
if (refused) return refused;
|
||||
const mapped = r.required.map(check => ({ check, w: r.workflows.find(w => defines(w, check)) }));
|
||||
for (const m of mapped.filter(x => !x.w)) lines.push(`UNMAPPED: required check "${m.check}" is no job of a workflow on ${r.base} (a status context or an app check); land still requires it green`);
|
||||
const byWorkflow = new Map<string, string[]>();
|
||||
for (const m of mapped.filter(x => x.w)) byWorkflow.set(m.w!.file, [...(byWorkflow.get(m.w!.file) ?? []), `${m.w!.file}/${m.check}`]);
|
||||
const gates: unknown[] = [];
|
||||
if (byWorkflow.size) gates.push({ kind: 'workflows', workflows: [...byWorkflow.keys()], required_jobs: [...byWorkflow.values()].flat() });
|
||||
const command = { kind: 'command' as const, command: o.command ?? '', timeout_sec: o.timeoutSec ?? 1800, spend_cap_usd: 0 };
|
||||
command.spend_cap_usd = o.spendCap ?? Math.min(100, Math.ceil((MAX_SIZE + 2) * commandReservation(command)));
|
||||
if (o.command) gates.push(command);
|
||||
if (!gates.length) lines.push(`GATES: no required check maps to a workflow job; name the gate yourself (--command "<test command>", or a workflows gate listing <workflow>/<job>)`);
|
||||
const method = r.policy.history === 'merge-only' ? 'merge' : 'squash';
|
||||
const land = { merge_method: method, ...(!r.policy.changelog_fragments && r.required.length ? { required_checks: r.required } : {}) };
|
||||
const block = { train: { gates, land, max_size: MAX_SIZE, fill_hours: 3 }, min_gstack_version: TRAIN_GATES_KEY_VERSION };
|
||||
if (o.command && o.spendCap !== undefined && o.spendCap < (MAX_SIZE + 2) * commandReservation(command)) lines.push(`WARN: --spend-cap $${o.spendCap} is below $${((MAX_SIZE + 2) * commandReservation(command)).toFixed(2)}, what ${MAX_SIZE + 1} prefixes and one rerun reserve; a full train would stop at QUEUE_TRAIN_SPEND_CAP`);
|
||||
lines.push('POLICY: add to .gstack/ship-policy.json on the base branch (merge with the keys it has):', ...JSON.stringify(block, null, 2).split('\n'));
|
||||
for (const file of byWorkflow.keys()) {
|
||||
lines.push(`WORKFLOW: ${file} needs, beside its own triggers:`, ...NONCE_SNIPPET.split('\n').map(l => ' ' + l));
|
||||
lines.push(...workflowFindings(r.workflows.find(w => w.file === file)!).map(f => `WARN: ${f}`));
|
||||
}
|
||||
lines.push(r.strict === false
|
||||
? 'LAND_MODE: direct (gh pr merge --match-head-commit <pin>) when the pin is green and merges onto main as its gated prefix; update (a tool-made commit of the gated prefix on [pin, main], checked, then merged) for a stamped train, a conditional PR, a merge that resolves release files, or a pin whose own checks are red'
|
||||
: `LAND_MODE: update for every slot (${r.strict ? 'branch protection requires branches to be up to date' : 'branch protection could not be read'}): a tool-made commit of the gated prefix on [pin, main], checked, then merged`);
|
||||
lines.push('WRITES: nothing; commit the block on the base branch through a PR, then run train doctor');
|
||||
return outOk(lines, `${Q} train doctor (after the block lands on ${r.base})`, 'none');
|
||||
}
|
||||
|
||||
/** `train doctor`: re-check the base policy's live block against the repository. */
|
||||
export function trainDoctor(ctx: QueueContext): TrainOut {
|
||||
const r = readRepo(ctx);
|
||||
if ('exit' in r) return r;
|
||||
const train = r.policy.train;
|
||||
if (!train?.gates) return outRefused({ code: 'QUEUE_TRAIN_OFF', message: `the base policy on ${r.base} declares no train block with gates` }, [], `${Q} train init`, 'none');
|
||||
const lines = [`DOCTOR: ${r.slug ?? 'repository'} base ${r.base}; ${train.gates.length} gate(s); land ${train.land?.merge_method ?? 'squash'}`];
|
||||
const refused = versionRule(r, lines);
|
||||
if (refused) return refused;
|
||||
const findings: string[] = [];
|
||||
for (const g of train.gates) {
|
||||
if (g.kind !== 'workflows') {
|
||||
lines.push(`GATE: command ${JSON.stringify(g.command)} timeout ${g.timeout_sec}s cap $${g.spend_cap_usd}`);
|
||||
const full = (train.max_size + 1) * commandReservation(g);
|
||||
if (g.spend_cap_usd < full) findings.push(`command gate cap $${g.spend_cap_usd} is below $${full.toFixed(2)}, what ${train.max_size + 1} prefixes reserve at timeout ${g.timeout_sec}s; a full train stops at QUEUE_TRAIN_SPEND_CAP`);
|
||||
continue;
|
||||
}
|
||||
for (const file of g.workflows) {
|
||||
const w = r.workflows.find(x => x.file === file);
|
||||
if (!w) { findings.push(`${file}: not in .github/workflows on ${r.base}`); continue; }
|
||||
findings.push(...workflowFindings(w));
|
||||
for (const job of g.required_jobs.filter(j => j.startsWith(`${file}/`)).map(j => j.slice(file.length + 1))) if (!defines(w, job)) findings.push(`${file}: required job "${job}" is no job name GitHub would report`);
|
||||
const green = r.slug ? r.ctx.gh(['run', 'list', '--repo', r.slug, '--workflow', file, '--branch', r.base, '--status', 'success', '--limit', '1', '--json', 'databaseId']) : null;
|
||||
if (!green?.ok || green.out.trim() === '[]') findings.push(`${file}: ${green?.ok ? `never green on ${r.base}` : 'its runs on the base could not be read'}`);
|
||||
}
|
||||
}
|
||||
const repo = r.slug ? r.ctx.gh(['api', `repos/${r.slug}`]) : null;
|
||||
const allowed = (() => { try { const j = JSON.parse(repo?.out ?? '') as Record<string, unknown>; return { squash: j.allow_squash_merge, merge: j.allow_merge_commit, rebase: j.allow_rebase_merge }; } catch { return null; } })();
|
||||
const method = train.land?.merge_method ?? 'squash';
|
||||
if (allowed && allowed[method] === false) findings.push(`train.land.merge_method is ${method}, which GitHub has turned off for this repository`);
|
||||
if (method === 'rebase') findings.push('train.land.merge_method rebase: slots that must land through an update commit (strict protection, stamped or conditional) refuse with QUEUE_TRAIN_LAND_METHOD');
|
||||
lines.push(`LAND_MODE: ${r.strict === false ? 'direct when the pin is green and merges onto main as its prefix, else update' : `update for every slot (${r.strict ? 'strict protection' : 'protection unreadable'})`}`);
|
||||
lines.push(...findings.map(f => `FINDING: ${f}`));
|
||||
if (findings.length) return outFailed({ code: 'QUEUE_TRAIN_DOCTOR_FINDINGS', message: `${findings.length} finding(s) in the live train block` }, lines, 'fix each FINDING on the base branch, then train doctor again', 'none');
|
||||
return outOk([...lines, 'DOCTOR: no findings'], `${Q} train build --prs <n>@<sha>,...`, 'none');
|
||||
}
|
||||
+241
-104
@@ -1,22 +1,33 @@
|
||||
/**
|
||||
* `train land --train <nonce> --pr <n>` (docs/ship-policy.md#train-land, GSTA-154 PR B): the merge step for
|
||||
* one slot of a stamped train (changelog_fragments). The train built S_1..S_k and gated S_k; slot i
|
||||
* publishes S_i exactly. With main's tree equal to S_(i-1)'s (slot 1: the train base), land commits S_i's
|
||||
* tree with parents [pin_i, main], labels the PR evals-no-debounce (a tool-made head gets no later push,
|
||||
* so the quiet window protects nothing), pushes under --force-with-lease=<branch>:<pin>, waits for the
|
||||
* PR's checks on that commit (CI runs live at the slot until GSTA-156's reuse is on main), merges with
|
||||
* --match-head-commit and confirms main's tree is S_i. Any other main is the freeze path
|
||||
* (QUEUE_TRAIN_TREE_MISMATCH), as for verify. Journaled beside the train record; a rerun resumes.
|
||||
* `train land --train <nonce> --pr <n>` (docs/ship-policy.md#train-land): the merge step for one slot of a train
|
||||
* whose slots publish their gated prefix exactly. A stamped train (changelog_fragments, GSTA-154 PR B; or a train with
|
||||
* gates under stamp-at-merge) publishes S_i; an unstamped train with gates (GSTA-157 PR 2) publishes T_i. With main's
|
||||
* tree equal to the prefix before the slot (slot 1: the train base), land picks one of two modes and records it, with
|
||||
* the update sha when there is one, in the train record and the notes before its first remote effect:
|
||||
*
|
||||
* - `update` (today's path): commit the prefix tree with parents [pin, main], push it to the PR branch under
|
||||
* --force-with-lease=<branch>:<pin>, wait for the PR's checks on that commit, merge with --match-head-commit and
|
||||
* the policy's merge method (train.land.merge_method, default squash; rebase is refused, since it would replay
|
||||
* commits instead of landing the gated tree). Forced for a stamped train, strict branch protection (or protection
|
||||
* that cannot be read), a conditional PR, a merge that needed release-file resolution, and a pin whose own checks
|
||||
* are not green (they ran against an older base).
|
||||
* - `direct`: `gh pr merge --<method> --match-head-commit <pin>` once the PR's required checks are green at the pin
|
||||
* and merging the pin onto main gives the gated tree exactly.
|
||||
*
|
||||
* Either way main's tree must be the prefix after the merge; any other main is the freeze path
|
||||
* (QUEUE_TRAIN_TREE_MISMATCH), as for verify. Fork heads are refused. Journaled beside the train record; a rerun
|
||||
* resumes: open at the pin, open at the update commit, or merged (confirmed through main's tree).
|
||||
*/
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import { ciAtHead, type QueueContext } from './ship-queue';
|
||||
import { git, writeJsonAtomic } from './ship-queue-gate';
|
||||
import { waitChecks } from './ship-queue-land';
|
||||
import { bindShippedReceipt } from './ship-receipt-depth';
|
||||
import { checkRun, prInfo } from './ship-queue-train-github';
|
||||
import { activeFreeze, attemptsOf, trainFile } from './ship-queue-train-record';
|
||||
import { fetchBase, fetchTrainBranch, findTrain, frozenLines, nextFor, outFailed, outOk, outPending, outRefused, trainEnv, type TrainOut } from './ship-queue-train';
|
||||
import { verifyPostMerge } from './ship-queue-post-merge';
|
||||
import { bindTrainReceipt } from './ship-queue-train-gate';
|
||||
import { checkRun, prInfo, type PrInfo } from './ship-queue-train-github';
|
||||
import { activeFreeze, attemptsOf, nowIso, trainFile, type LandingRecord, type TrainRecord } from './ship-queue-train-record';
|
||||
import { fetchBase, fetchTrainBranch, findTrain, frozenLines, nextFor, outFailed, outPending, outRefused, save, trainEnv, type TrainEnv, type TrainOut } from './ship-queue-train';
|
||||
import { confirmMerged, stopped } from './ship-queue-train-verify';
|
||||
|
||||
const Q = 'gstack-ship-queue';
|
||||
@@ -24,6 +35,190 @@ const STAMP_IDENTITY = { GIT_AUTHOR_NAME: 'gstack-ship-queue', GIT_AUTHOR_EMAIL:
|
||||
GIT_COMMITTER_EMAIL: 'ship-queue@gstack.local', GIT_AUTHOR_DATE: '2000-01-01T00:00:00Z', GIT_COMMITTER_DATE: '2000-01-01T00:00:00Z' };
|
||||
const isSha = (s: string) => /^[0-9a-f]{40,64}$/.test(s);
|
||||
interface SlotJournal { schema_version: 1; pr: string; pin: string; stamp_commit: string; base: string }
|
||||
interface Link { index: number; commit: string; tree: string; pin: string }
|
||||
interface Slot { env: TrainEnv; t: TrainRecord; i: number; pin: string; target: Link & { version?: string }; pred: Link; run: string; rerun: string; lines: string[] }
|
||||
|
||||
/** The checks train land requires on the landing commit: changelog_fragments.required_checks, else train.land.required_checks. */
|
||||
export const landRequiredChecks = (env: TrainEnv): string[] => env.policy.policy.changelog_fragments?.required_checks ?? env.train.land?.required_checks ?? [];
|
||||
|
||||
/** Required checks missing or not green in a ciAtHead read. */
|
||||
function missingChecks(ci: ReturnType<typeof ciAtHead>, required: string[]): string[] {
|
||||
return required.filter(name => {
|
||||
const runs = ci.checks.filter(c => c.name === name || c.name.startsWith(`${name} (`));
|
||||
const contexts = ci.contexts.filter(c => c.name === name);
|
||||
return !(runs.length || contexts.length) || runs.some(c => c.status !== 'completed' || c.conclusion !== 'success') || contexts.some(c => c.state !== 'success');
|
||||
});
|
||||
}
|
||||
|
||||
/** GitHub's answers that mean no required-checks protection; a bare 404 or 403 (a token that cannot read protection) is unreadable, never unprotected. */
|
||||
export const UNPROTECTED = /Branch not protected|Required status checks not enabled/i;
|
||||
|
||||
/** Branch protection's "require branches to be up to date": true, false (unprotected or not strict), or null when unreadable. */
|
||||
function strictProtection(ctx: QueueContext): boolean | null {
|
||||
const r = ctx.gh(['api', `repos/{owner}/{repo}/branches/${ctx.base}/protection/required_status_checks`]);
|
||||
if (!r.ok) return UNPROTECTED.test(`${r.err ?? ''} ${r.out}`) ? false : null;
|
||||
try { return (JSON.parse(r.out) as { strict?: unknown }).strict === true; } catch { return null; }
|
||||
}
|
||||
|
||||
/** The gate a slot needs before it lands: a generic train's verdict seats it, else the gstack-evals green record and check run. */
|
||||
function gateRefusal(ctx: QueueContext, env: TrainEnv, t: TrainRecord, i: number, run: string): TrainOut | null {
|
||||
if (t.gated) {
|
||||
if (t.verdict && i < t.verdict.landable) return null;
|
||||
return outRefused({ code: 'QUEUE_GATE_MISSING', message: t.verdict ? `train ${t.nonce}'s verdict (${t.verdict.kind}) lands slots 1..${t.verdict.landable}; #${t.slots[i]!.pr} is slot ${i + 1}` : `train ${t.nonce} has no verdict yet (state ${t.state})` }, [], nextFor(t), run);
|
||||
}
|
||||
const gate = attemptsOf(t).filter(a => a.purpose !== 'bisect').at(-1);
|
||||
if (gate?.outcome !== 'GREEN' || !['green', 'merging', 'merged'].includes(t.state)) return outRefused({ code: 'QUEUE_GATE_MISSING', message: `train ${t.nonce} has no green record (state ${t.state}, last gate ${gate?.outcome ?? 'none'})` }, [], nextFor(t), run);
|
||||
const check = checkRun(ctx, t.head, env.train.full_gate_check);
|
||||
if (check?.conclusion !== 'success') return outRefused({ code: 'QUEUE_GATE_MISSING', message: `no successful ${env.train.full_gate_check} check run on the train head ${t.head.slice(0, 12)}` }, [], `${Q} train verdict --train ${t.nonce}`, run);
|
||||
return null;
|
||||
}
|
||||
|
||||
/** T0 and the published prefixes as the first-parent chain of the gated head, each slot's pin its second parent; null when the record disagrees. */
|
||||
function landChain(root: string, t: TrainRecord): Link[] | null {
|
||||
const k = t.slots.length;
|
||||
const chain = Array.from({ length: k + 1 }, (_, n) => {
|
||||
const commit = git(root, ['rev-parse', '--verify', '-q', `${t.head}~${k - n}^{commit}`]).out;
|
||||
return { index: n, commit, tree: isSha(commit) ? git(root, ['rev-parse', `${commit}^{tree}`]).out : '', pin: n && isSha(commit) ? git(root, ['rev-parse', '--verify', '-q', `${commit}^2`]).out : '' };
|
||||
});
|
||||
const recorded = t.stamped ? [t.prefixes[0]!, ...t.stamped] : t.prefixes;
|
||||
if (t.stamped && t.stamped.length !== k) return null;
|
||||
if (t.gated && (t.gated.length !== k + 1 || t.gated.some((g, n) => g.commit !== recorded[n]?.commit || g.tree !== recorded[n]?.tree))) return null;
|
||||
return chain.some((c, n) => !isSha(c.commit) || c.commit !== recorded[n]?.commit || c.tree !== recorded[n]?.tree || (n > 0 && c.pin !== t.slots[n - 1]!.head)) ? null : chain;
|
||||
}
|
||||
|
||||
function mainNow(ctx: QueueContext, env: TrainEnv): { sha: string; tree: string } | { error: string } {
|
||||
const error = fetchBase(ctx);
|
||||
if (error) return { error };
|
||||
const sha = git(ctx.repoRoot, ['rev-parse', `${env.remote}/${ctx.base}`]).out;
|
||||
return { sha, tree: git(ctx.repoRoot, ['rev-parse', `${sha}^{tree}`]).out };
|
||||
}
|
||||
|
||||
const label = (s: Slot) => (s.t.stamped ? `S${s.target.index}` : `T${s.target.index}`);
|
||||
|
||||
/** Record the slot's landing (mode, pin, update commit, base, expected tree) in the train record and the notes, before any remote effect. */
|
||||
function recordLanding(s: Slot, mode: LandingRecord['mode'], update: string | null, base: string): void {
|
||||
s.t.landing = { ...s.t.landing, [s.t.slots[s.i]!.pr]: { slot: s.i + 1, mode, method: s.env.train.land?.merge_method ?? 'squash', pin: s.pin, update, base, expected_tree: s.target.tree, at: nowIso() } };
|
||||
if (s.t.state !== 'merged') s.t.state = 'merging';
|
||||
save(s.env, s.t);
|
||||
}
|
||||
|
||||
/**
|
||||
* Which mode lands this slot, and why. `wait`: the pin's checks are still running, so whether the pin itself can be
|
||||
* merged is not known yet.
|
||||
*/
|
||||
function chooseMode(ctx: QueueContext, s: Slot, mainSha: string, conditional: boolean): { mode: 'direct' | 'update' | 'wait'; why: string } {
|
||||
if (s.t.stamped) return { mode: 'update', why: `the slot publishes the stamped ${label(s)}` };
|
||||
if (conditional) return { mode: 'update', why: 'a conditional (base-red) PR lands only through a commit whose own checks run on the gated tree' };
|
||||
if ((s.env.train.land?.merge_method ?? 'squash') === 'rebase') return { mode: 'update', why: 'a rebase replays the pin\'s commits, and the replayed tree is not the one the train gated' };
|
||||
const strict = strictProtection(ctx);
|
||||
if (strict !== false) return { mode: 'update', why: strict ? 'branch protection requires branches to be up to date' : 'branch protection could not be read, so the up-to-date path is taken' };
|
||||
const m = git(ctx.repoRoot, ['merge-tree', '--write-tree', mainSha, s.pin]);
|
||||
if (!m.ok || m.out.split('\n')[0] !== s.target.tree) return { mode: 'update', why: `merging the pin onto main does not give ${label(s)} exactly (the train resolved release files)` };
|
||||
const ci = ciAtHead(ctx, s.pin);
|
||||
const missing = missingChecks(ci, landRequiredChecks(s.env));
|
||||
if (ci.green && !missing.length) return { mode: 'direct', why: `the pin's checks are green (${ci.line}) and it merges onto main as ${label(s)} exactly` };
|
||||
if (ci.line.startsWith('pending') && !missing.some(n => !ci.checks.some(c => c.name === n || c.name.startsWith(`${n} (`)))) return { mode: 'wait', why: `the pin's checks are still running (${ci.line})` };
|
||||
return { mode: 'update', why: `the pin's own checks are not green (${ci.line}${missing.length ? `; missing ${missing.join(', ')}` : ''}); they ran against an older base` };
|
||||
}
|
||||
|
||||
/** Merge with the policy's method at an exact head, then confirm main's tree is the slot's prefix. */
|
||||
async function mergeAt(ctx: QueueContext, s: Slot, head: string, base: string): Promise<TrainOut> {
|
||||
const pr = s.t.slots[s.i]!.pr;
|
||||
const method = s.env.train.land?.merge_method ?? 'squash';
|
||||
let main = mainNow(ctx, s.env);
|
||||
if ('error' in main) return outPending({ code: 'QUEUE_PROOF_MISSING', message: main.error }, s.lines, s.rerun, s.run);
|
||||
if (main.sha !== base) return stopped(s.env, s.t, 'QUEUE_TRAIN_TREE_MISMATCH', `main moved to ${main.sha.slice(0, 12)} before the merge (the slot was prepared on ${base.slice(0, 12)})`, main.sha, s.lines, true);
|
||||
const target = prInfo(ctx, pr);
|
||||
if ('error' in target) return outPending({ code: 'QUEUE_PROOF_MISSING', message: target.error }, s.lines, s.rerun, s.run);
|
||||
if (target.baseRefName && target.baseRefName !== ctx.base) return retargeted(s, target.baseRefName, ctx.base);
|
||||
const m = ctx.gh(['pr', 'merge', pr, `--${method}`, '--match-head-commit', head]);
|
||||
if (!m.ok) return outFailed({ code: 'QUEUE_NOT_READY', message: `gh pr merge refused: ${(m.err ?? m.out).trim().slice(0, 200)}` }, s.lines, s.rerun, s.run);
|
||||
main = mainNow(ctx, s.env);
|
||||
if ('error' in main) return outPending({ code: 'QUEUE_PROOF_MISSING', message: `merged; ${main.error}` }, s.lines, s.rerun, s.run);
|
||||
if (main.sha === base) return outPending({ code: 'QUEUE_NOT_READY', message: `merged; main still reads ${base.slice(0, 12)}, so the merge is not visible yet` }, s.lines, s.rerun, s.run);
|
||||
if (main.tree !== s.target.tree) return stopped(s.env, s.t, 'QUEUE_TRAIN_TREE_MISMATCH', `after the merge main ${main.sha.slice(0, 12)} is not ${label(s)} exactly`, main.sha, s.lines, true);
|
||||
return confirmed(s, main.sha, `MERGED: #${pr} as ${main.sha} (${method}); main's tree is ${label(s)} exactly`);
|
||||
}
|
||||
|
||||
/** The check names every workflows gate asks to see green on main after each land. */
|
||||
export const postMergeChecks = (env: TrainEnv): string[] => (env.train.gates ?? []).flatMap(g => (g.kind === 'workflows' ? g.post_merge_checks ?? [] : []));
|
||||
|
||||
/** confirmMerged, with the post-merge checks a workflows gate declares in the NEXT: train land refuses the next slot until they are green. */
|
||||
function confirmed(s: Slot, mainSha: string, line: string): TrainOut {
|
||||
const out = confirmMerged(s.env, s.t, s.t.slots[s.i]!.pr, mainSha, s.target.index, [...s.lines, line]);
|
||||
const post = postMergeChecks(s.env);
|
||||
if (!post.length || out.exit !== 0) return out;
|
||||
return { ...out, lines: [...out.lines, `POST_MERGE_CHECKS: ${post.join(', ')} at ${mainSha.slice(0, 12)}`], next: `${Q} verify --post-merge ${mainSha} --checks ${JSON.stringify(post.join(','))} && ${out.next}` };
|
||||
}
|
||||
|
||||
/** `direct`: merge the pin itself once its required checks are green there. */
|
||||
async function landDirect(ctx: QueueContext, s: Slot, info: PrInfo, base: string, why: string): Promise<TrainOut> {
|
||||
const pr = s.t.slots[s.i]!.pr;
|
||||
if (info.headRefOid !== s.pin) return stopped(s.env, s.t, 'QUEUE_TRAIN_MOVED', `#${pr}'s head is ${info.headRefOid.slice(0, 12)}, not its pin ${s.pin.slice(0, 12)}`, base, s.lines, true);
|
||||
const shipped = bindTrainReceipt(ctx.repoRoot, `refs/remotes/${s.env.remote}/${ctx.base}`, info.body, pr, [s.pin], false);
|
||||
s.lines.push(shipped.line, ...(shipped.depthLine ? [shipped.depthLine] : []), `MODE: direct (${why})`);
|
||||
if (!shipped.ok) return outRefused({ code: shipped.code, message: shipped.message }, s.lines, `#${pr}'s lane runs /ship at its pin and READYs again`, s.run);
|
||||
recordLanding(s, 'direct', null, base);
|
||||
return mergeAt(ctx, s, s.pin, base);
|
||||
}
|
||||
|
||||
/** `update`: today's path, the prefix tree committed on [pin, main], pushed under the pin's lease, checked, then merged. */
|
||||
async function landUpdate(ctx: QueueContext, s: Slot, info: PrInfo, base: string, why: string, o: { noWait?: boolean; timeoutMin?: number }): Promise<TrainOut> {
|
||||
const { env, t, pin } = s;
|
||||
const pr = t.slots[s.i]!.pr;
|
||||
const root = ctx.repoRoot;
|
||||
if ((env.train.land?.merge_method ?? 'squash') === 'rebase') return outRefused({ code: 'QUEUE_TRAIN_LAND_METHOD', message: `#${pr} must land through an update commit (${why}), and train.land.merge_method is rebase, which would replay commits instead of landing ${label(s)}` }, s.lines, 'none', s.run);
|
||||
s.lines.push(`MODE: update (${why})`);
|
||||
const what = t.stamped ? 'stamp' : 'update';
|
||||
const file = path.join(path.dirname(s.run), `land-${pr}.json`);
|
||||
let j: SlotJournal | null = null;
|
||||
try { j = JSON.parse(fs.readFileSync(file, 'utf8')) as SlotJournal; } catch { j = null; }
|
||||
if (!j || j.base !== base || info.headRefOid !== j.stamp_commit) {
|
||||
const message = t.stamped ? `v${s.target.version} stamp: train ${t.nonce} S${s.target.index} (#${pr})` : `train ${t.nonce} T${s.target.index}: land #${pr} as the gated tree`;
|
||||
const commit = git(root, ['commit-tree', '--no-gpg-sign', s.target.tree, '-p', pin, '-p', base, '-m', message], { ...process.env, ...STAMP_IDENTITY }).out;
|
||||
if (!isSha(commit)) return outFailed({ code: 'QUEUE_LAND_CONFLICT', message: `cannot create the ${what} commit for tree ${s.target.tree.slice(0, 12)}; nothing pushed` }, s.lines, `${Q} train status --train ${t.nonce}`, s.run);
|
||||
if (![pin, commit, j?.stamp_commit].includes(info.headRefOid)) return stopped(env, t, 'QUEUE_TRAIN_MOVED', `#${pr}'s head is ${info.headRefOid.slice(0, 12)}, not its pin ${pin.slice(0, 12)}`, base, s.lines, true);
|
||||
const shipped = bindTrainReceipt(root, `refs/remotes/${env.remote}/${ctx.base}`, info.body, pr, [pin], !!t.slots[s.i]!.base_red);
|
||||
s.lines.push(shipped.line, ...(shipped.depthLine ? [shipped.depthLine] : []));
|
||||
if (!shipped.ok) return outRefused({ code: shipped.code, message: shipped.message }, s.lines, `#${pr}'s lane runs /ship at its pin and READYs again`, s.run);
|
||||
if (!t.gated) {
|
||||
const added = ctx.gh(['pr', 'edit', pr, '--add-label', 'evals-no-debounce']);
|
||||
s.lines.push(`LABEL: evals-no-debounce ${added.ok ? 'added' : `not added (${(added.err ?? added.out).trim().slice(0, 80)}); the 10-minute quiet window applies`}`);
|
||||
}
|
||||
j = { schema_version: 1, pr, pin, stamp_commit: commit, base };
|
||||
writeJsonAtomic(file, j);
|
||||
recordLanding(s, 'update', commit, base);
|
||||
const pushed = info.headRefOid !== commit;
|
||||
if (pushed) {
|
||||
const push = git(root, ['push', '-q', `--force-with-lease=refs/heads/${info.headRefName}:${info.headRefOid}`, env.remote, `${commit}:refs/heads/${info.headRefName}`]);
|
||||
if (!push.ok) {
|
||||
const remote = git(root, ['ls-remote', env.remote, `refs/heads/${info.headRefName}`]);
|
||||
const now = remote.ok ? remote.out.split(/\s/)[0] ?? '' : '';
|
||||
if (remote.ok && now !== info.headRefOid && now !== commit) return stopped(env, t, 'QUEUE_TRAIN_MOVED', `#${pr}'s branch moved off ${info.headRefOid.slice(0, 12)} before the ${what} push; train land never overwrites it`, base, s.lines, true);
|
||||
if (!remote.ok || now !== commit) return outFailed({ code: 'QUEUE_NOT_READY', message: `the ${what} push failed and ${info.headRefName} still reads ${remote.ok ? info.headRefOid.slice(0, 12) : 'unknown'}: ${push.err.split('\n')[0]}` }, s.lines, s.rerun, s.run);
|
||||
}
|
||||
}
|
||||
s.lines.push(`${pushed ? 'PUSHED' : 'RESUMED'}: ${commit} (tree ${label(s)}, parents [pin, main ${base.slice(0, 12)}]) ${pushed ? 'to' : 'is already the head of'} ${info.headRefName}`);
|
||||
if (o.noWait && pushed) return outPending({ code: 'QUEUE_NOT_READY', message: `checks run on the ${what} commit ${commit.slice(0, 12)}` }, s.lines, `gstack-pr-wait ${pr} --until green && ${s.rerun}`, s.run);
|
||||
if (!o.noWait) waitChecks(ctx, commit, o.timeoutMin ?? 60);
|
||||
}
|
||||
const head = prInfo(ctx, pr);
|
||||
if ('error' in head) return outPending({ code: 'QUEUE_PROOF_MISSING', message: head.error }, s.lines, s.rerun, s.run);
|
||||
if (head.headRefOid !== j.stamp_commit) return stopped(env, t, 'QUEUE_TRAIN_MOVED', `#${pr}'s head is not the ${what} commit ${j.stamp_commit.slice(0, 12)}`, base, s.lines, true);
|
||||
if (head.baseRefName && head.baseRefName !== ctx.base) return retargeted(s, head.baseRefName, ctx.base);
|
||||
const ci = ciAtHead(ctx, j.stamp_commit);
|
||||
s.lines.push(`CI: ${ci.line}`);
|
||||
const missing = missingChecks(ci, landRequiredChecks(env));
|
||||
if (missing.length) {
|
||||
s.lines.push(`REQUIRED_CHECKS: not green at ${j.stamp_commit.slice(0, 12)}: ${missing.join(', ')}`);
|
||||
const fix = `wait for ${missing.join(', ')} to pass on ${j.stamp_commit.slice(0, 12)} (dispatch the workflow on ${head.headRefName} if it never ran), then ${s.rerun}`;
|
||||
return outPending({ code: 'QUEUE_NOT_READY', message: `required checks missing or not green on the ${what} commit ${j.stamp_commit.slice(0, 12)}: ${missing.join(', ')}`, fix }, s.lines, fix, s.run);
|
||||
}
|
||||
if (!ci.green) return outPending({ code: 'QUEUE_NOT_READY', message: `checks on the ${what} commit ${j.stamp_commit.slice(0, 12)} are not green` }, s.lines, `gstack-pr-wait ${pr} --until green && ${s.rerun}`, s.run);
|
||||
return mergeAt(ctx, s, j.stamp_commit, j.base);
|
||||
}
|
||||
|
||||
const retargeted = (s: Slot, b: string, base: string) => outRefused({ code: 'QUEUE_NOT_READY', message: `#${s.t.slots[s.i]!.pr} targets ${b}, not ${base}; train land merges only PRs on the base the train built` }, s.lines, `retarget #${s.t.slots[s.i]!.pr} to ${base}, then ${s.rerun}`, s.run);
|
||||
|
||||
export async function trainLand(ctx: QueueContext, o: { train: string; pr: string; noWait?: boolean; timeoutMin?: number }): Promise<TrainOut> {
|
||||
const fetchErr = fetchBase(ctx);
|
||||
@@ -35,104 +230,46 @@ export async function trainLand(ctx: QueueContext, o: { train: string; pr: strin
|
||||
const run = trainFile(t.nonce);
|
||||
const i = t.slots.findIndex(s => s.pr === o.pr);
|
||||
if (i < 0) return outRefused({ code: 'QUEUE_GATE_MISSING', message: `train ${t.nonce} does not include #${o.pr}` }, [], nextFor(t), run);
|
||||
if (!t.stamped) return outRefused({ code: 'QUEUE_GATE_MISSING', message: `train ${t.nonce} was built without stamped prefixes; merge it with the slot stamp and ${Q} verify` }, [], `${Q} verify --pr ${o.pr} --train ${t.nonce}`, run);
|
||||
const gate = attemptsOf(t).filter(a => a.purpose !== 'bisect').at(-1);
|
||||
if (gate?.outcome !== 'GREEN' || !['green', 'merging', 'merged'].includes(t.state)) return outRefused({ code: 'QUEUE_GATE_MISSING', message: `train ${t.nonce} has no green record (state ${t.state}, last gate ${gate?.outcome ?? 'none'})` }, [], nextFor(t), run);
|
||||
const check = checkRun(ctx, t.head, env.train.full_gate_check);
|
||||
if (check?.conclusion !== 'success') return outRefused({ code: 'QUEUE_GATE_MISSING', message: `no successful ${env.train.full_gate_check} check run on the train head ${t.head.slice(0, 12)}` }, [], `${Q} train verdict --train ${t.nonce}`, run);
|
||||
if (!t.stamped && !t.gated) return outRefused({ code: 'QUEUE_GATE_MISSING', message: `train ${t.nonce} was built without stamped prefixes; merge it with the slot stamp and ${Q} verify` }, [], `${Q} verify --pr ${o.pr} --train ${t.nonce}`, run);
|
||||
const gate = gateRefusal(ctx, env, t, i, run);
|
||||
if (gate) return gate;
|
||||
if (!fetchTrainBranch(env, t)) return outRefused({ code: 'QUEUE_PROOF_MISSING', message: `the train's prefix commits are not here and ${t.branch} could not be fetched` }, [], `${Q} train status --train ${t.nonce}`, run);
|
||||
const freeze = activeFreeze(env.repo.keys);
|
||||
if (freeze && freeze.repair?.train !== t.nonce) return outRefused({ code: 'QUEUE_TRAIN_FROZEN', message: `the queue is frozen (${freeze.reason}); only the admitted repair train merges` }, frozenLines(freeze), `${Q} train build --repair --prs <repair-pr>@<sha>`, run);
|
||||
|
||||
const root = ctx.repoRoot;
|
||||
const chain = landChain(ctx.repoRoot, t);
|
||||
const k = t.slots.length;
|
||||
const chain = Array.from({ length: k + 1 }, (_, n) => {
|
||||
const commit = git(root, ['rev-parse', '--verify', '-q', `${t.head}~${k - n}^{commit}`]).out;
|
||||
return { index: n, commit, tree: isSha(commit) ? git(root, ['rev-parse', `${commit}^{tree}`]).out : '', pin: n && isSha(commit) ? git(root, ['rev-parse', '--verify', '-q', `${commit}^2`]).out : '' };
|
||||
});
|
||||
const recorded = [t.prefixes[0], ...t.stamped];
|
||||
if (t.stamped.length !== k || chain.some((c, n) => !isSha(c.commit) || c.commit !== recorded[n]?.commit || c.tree !== recorded[n]?.tree || (n > 0 && c.pin !== t.slots[n - 1]!.head))) return outRefused({ code: 'QUEUE_PROOF_MISSING', message: `the record's T0 and S1..S${k} (with each slot's pin as second parent) are not the first-parent chain of the gated head ${t.head.slice(0, 12)}` }, [], `${Q} train status --train ${t.nonce}`, run);
|
||||
const s = { ...chain[i + 1]!, version: t.stamped[i]!.version };
|
||||
const pred = chain[i]!;
|
||||
if (!chain) return outRefused({ code: 'QUEUE_PROOF_MISSING', message: `the record's T0 and ${t.stamped ? 'S' : 'T'}1..${t.stamped ? 'S' : 'T'}${k} (with each slot's pin as second parent) are not the first-parent chain of the gated head ${t.head.slice(0, 12)}` }, [], `${Q} train status --train ${t.nonce}`, run);
|
||||
const pin = t.slots[i]!.head;
|
||||
const mainNow = (): { sha: string; tree: string } | { error: string } => {
|
||||
const error = fetchBase(ctx);
|
||||
if (error) return { error };
|
||||
const sha = git(root, ['rev-parse', `${env.remote}/${ctx.base}`]).out;
|
||||
return { sha, tree: git(root, ['rev-parse', `${sha}^{tree}`]).out };
|
||||
};
|
||||
const rerun = `${Q} train land --train ${t.nonce} --pr ${o.pr}`;
|
||||
const lines = [`TRAIN_LAND: ${t.nonce} slot ${i + 1}/${t.slots.length} #${o.pr} pin ${pin} -> S${s.index} v${s.version} tree ${s.tree}`];
|
||||
let main = mainNow();
|
||||
if ('error' in main) return outRefused({ code: 'QUEUE_PROOF_MISSING', message: main.error }, lines, rerun, run);
|
||||
if (main.tree === s.tree) return confirmMerged(env, t, o.pr, main.sha, s.index, [...lines, `main: ${main.sha} is S${s.index} exactly`]);
|
||||
const mainTree = main.tree;
|
||||
const earlier = chain.slice(0, i).find(p => p.tree === mainTree);
|
||||
if (earlier) return outRefused({ code: 'QUEUE_TRAIN_TREE_MISMATCH', message: `main is ${earlier.index ? `S${earlier.index}` : 'the train base'}; land #${t.slots[earlier.index]!.pr} (slot ${earlier.index + 1}) first` }, lines, `${Q} train land --train ${t.nonce} --pr ${t.slots[earlier.index]!.pr}`, run);
|
||||
const later = chain.slice(i + 2).find(p => p.tree === mainTree);
|
||||
if (later) return confirmMerged(env, t, o.pr, main.sha, s.index, [...lines, `main: ${main.sha} is S${later.index}, past S${s.index}`]);
|
||||
if (main.tree !== pred.tree) return stopped(env, t, 'QUEUE_TRAIN_TREE_MISMATCH', `main ${main.sha.slice(0, 12)} is neither S${pred.index} nor S${s.index} exactly`, main.sha, lines, true);
|
||||
|
||||
const file = path.join(path.dirname(run), `land-${o.pr}.json`);
|
||||
let j: SlotJournal | null = null;
|
||||
try { j = JSON.parse(fs.readFileSync(file, 'utf8')) as SlotJournal; } catch { j = null; }
|
||||
const s: Slot = { env, t, i, pin, target: { ...chain[i + 1]!, version: t.stamped?.[i]?.version }, pred: chain[i]!, run, rerun, lines: [] };
|
||||
s.lines.push(`TRAIN_LAND: ${t.nonce} slot ${i + 1}/${k} #${o.pr} pin ${pin} -> ${label(s)}${s.target.version ? ` v${s.target.version}` : ''} tree ${s.target.tree}`);
|
||||
const main = mainNow(ctx, env);
|
||||
if ('error' in main) return outRefused({ code: 'QUEUE_PROOF_MISSING', message: main.error }, s.lines, rerun, run);
|
||||
if (main.tree === s.target.tree) return confirmed(s, main.sha, `main: ${main.sha} is ${label(s)} exactly`);
|
||||
const earlier = chain.slice(0, i).find(p => p.tree === main.tree);
|
||||
if (earlier) return outRefused({ code: 'QUEUE_TRAIN_TREE_MISMATCH', message: `main is ${earlier.index ? (t.stamped ? 'S' : 'T') + earlier.index : 'the train base'}; land #${t.slots[earlier.index]!.pr} (slot ${earlier.index + 1}) first` }, s.lines, `${Q} train land --train ${t.nonce} --pr ${t.slots[earlier.index]!.pr}`, run);
|
||||
const later = chain.slice(i + 2).find(p => p.tree === main.tree);
|
||||
if (later) return confirmed(s, main.sha, `main: ${main.sha} is ${t.stamped ? 'S' : 'T'}${later.index}, past ${label(s)}`);
|
||||
if (main.tree !== s.pred.tree) return stopped(env, t, 'QUEUE_TRAIN_TREE_MISMATCH', `main ${main.sha.slice(0, 12)} is neither ${s.pred.index ? (t.stamped ? 'S' : 'T') + s.pred.index : 'T0'} nor ${label(s)} exactly`, main.sha, s.lines, true);
|
||||
const info = prInfo(ctx, o.pr);
|
||||
if ('error' in info) return outRefused({ code: 'QUEUE_PROOF_MISSING', message: info.error }, lines, rerun, run);
|
||||
if (info.isCrossRepository) return outRefused({ code: 'QUEUE_TRAIN_UNTRUSTED', message: `#${o.pr} is a fork head; train land pushes only crew branches` }, lines, 'none', run);
|
||||
const retargeted = (b?: string) => b && b !== ctx.base ? outRefused({ code: 'QUEUE_NOT_READY', message: `#${o.pr} targets ${b}, not ${ctx.base}; train land merges only PRs on the base the train stamped` }, lines, `retarget #${o.pr} to ${ctx.base}, then ${rerun}`, run) : null;
|
||||
const wrongBase = retargeted(info.baseRefName);
|
||||
if (wrongBase) return wrongBase;
|
||||
if (!j || j.base !== main.sha || info.headRefOid !== j.stamp_commit) {
|
||||
const commit = git(root, ['commit-tree', '--no-gpg-sign', s.tree, '-p', pin, '-p', main.sha, '-m', `v${s.version} stamp: train ${t.nonce} S${s.index} (#${o.pr})`], { ...process.env, ...STAMP_IDENTITY }).out;
|
||||
if (!isSha(commit)) return outFailed({ code: 'QUEUE_LAND_CONFLICT', message: `cannot create the stamp commit for tree ${s.tree.slice(0, 12)}; nothing pushed` }, lines, `${Q} train status --train ${t.nonce}`, run);
|
||||
if (![pin, commit, j?.stamp_commit].includes(info.headRefOid)) return stopped(env, t, 'QUEUE_TRAIN_MOVED', `#${o.pr}'s head is ${info.headRefOid.slice(0, 12)}, not its pin ${pin.slice(0, 12)}`, main.sha, lines, true);
|
||||
const shipped = bindShippedReceipt(root, `refs/remotes/${env.remote}/${ctx.base}`, info.body, o.pr, [pin]);
|
||||
lines.push(shipped.line, ...(shipped.depthLine ? [shipped.depthLine] : []));
|
||||
if (!shipped.ok) return outRefused({ code: shipped.code, message: shipped.message }, lines, `#${o.pr}'s lane runs /ship at its pin and READYs again`, run);
|
||||
const label = ctx.gh(['pr', 'edit', o.pr, '--add-label', 'evals-no-debounce']);
|
||||
lines.push(`LABEL: evals-no-debounce ${label.ok ? 'added' : `not added (${(label.err ?? label.out).trim().slice(0, 80)}); the 10-minute quiet window applies`}`);
|
||||
j = { schema_version: 1, pr: o.pr, pin, stamp_commit: commit, base: main.sha };
|
||||
writeJsonAtomic(file, j);
|
||||
const pushed = info.headRefOid !== commit;
|
||||
if (pushed) {
|
||||
const push = git(root, ['push', '-q', `--force-with-lease=refs/heads/${info.headRefName}:${info.headRefOid}`, env.remote, `${commit}:refs/heads/${info.headRefName}`]);
|
||||
if (!push.ok) {
|
||||
const remote = git(root, ['ls-remote', env.remote, `refs/heads/${info.headRefName}`]);
|
||||
const now = remote.ok ? remote.out.split(/\s/)[0] ?? '' : '';
|
||||
if (remote.ok && now !== info.headRefOid && now !== commit) return stopped(env, t, 'QUEUE_TRAIN_MOVED', `#${o.pr}'s branch moved off ${info.headRefOid.slice(0, 12)} before the stamp push; train land never overwrites it`, main.sha, lines, true);
|
||||
if (!remote.ok || now !== commit) return outFailed({ code: 'QUEUE_NOT_READY', message: `the stamp push failed and ${info.headRefName} still reads ${remote.ok ? info.headRefOid.slice(0, 12) : 'unknown'}: ${push.err.split('\n')[0]}` }, lines, rerun, run);
|
||||
}
|
||||
}
|
||||
lines.push(`${pushed ? 'PUSHED' : 'RESUMED'}: ${commit} (tree S${s.index}, parents [pin, main ${main.sha.slice(0, 12)}]) ${pushed ? 'to' : 'is already the head of'} ${info.headRefName}`);
|
||||
const next = `gstack-pr-wait ${o.pr} --until green && ${Q} train land --train ${t.nonce} --pr ${o.pr}`;
|
||||
if (o.noWait && pushed) return outPending({ code: 'QUEUE_NOT_READY', message: `checks run on the stamp commit ${commit.slice(0, 12)}` }, lines, next, run);
|
||||
if (!o.noWait) waitChecks(ctx, commit, o.timeoutMin ?? 60);
|
||||
if ('error' in info) return outRefused({ code: 'QUEUE_PROOF_MISSING', message: info.error }, s.lines, rerun, run);
|
||||
if (info.isCrossRepository) return outRefused({ code: 'QUEUE_TRAIN_UNTRUSTED', message: `#${o.pr} is a fork head; train land pushes and merges only crew branches` }, s.lines, 'none', run);
|
||||
if (info.baseRefName && info.baseRefName !== ctx.base) return retargeted(s, info.baseRefName, ctx.base);
|
||||
const post = postMergeChecks(env);
|
||||
const prev = i > 0 ? t.merged.find(m => m.pr === t.slots[i - 1]!.pr) : undefined;
|
||||
if (post.length && prev) {
|
||||
const checked = await verifyPostMerge(ctx, { sha: prev.sha, checks: post, timeoutSec: 0, pollMs: 0 });
|
||||
s.lines.push(`POST_MERGE_CHECKS: #${t.slots[i - 1]!.pr} at ${prev.sha.slice(0, 12)} (${checked.failure?.code ?? 'green'})`, ...checked.lines);
|
||||
if (checked.status !== 'complete') return { ...checked, lines: s.lines, next: checked.failure?.code === 'QUEUE_TRAIN_PENDING' ? `${checked.next} && ${rerun}` : checked.next, run };
|
||||
}
|
||||
const head = prInfo(ctx, o.pr);
|
||||
if ('error' in head) return outPending({ code: 'QUEUE_PROOF_MISSING', message: head.error }, lines, rerun, run);
|
||||
if (head.headRefOid !== j.stamp_commit) return stopped(env, t, 'QUEUE_TRAIN_MOVED', `#${o.pr}'s head is not the stamp commit ${j.stamp_commit.slice(0, 12)}`, main.sha, lines, true);
|
||||
const movedBase = retargeted(head.baseRefName);
|
||||
if (movedBase) return movedBase;
|
||||
const ci = ciAtHead(ctx, j.stamp_commit);
|
||||
lines.push(`CI: ${ci.line}`);
|
||||
const required = env.policy.policy.changelog_fragments?.required_checks ?? [];
|
||||
const missing = required.filter(name => {
|
||||
const runs = ci.checks.filter(c => c.name === name || c.name.startsWith(`${name} (`));
|
||||
const contexts = ci.contexts.filter(c => c.name === name);
|
||||
return !(runs.length || contexts.length) || runs.some(c => c.status !== 'completed' || c.conclusion !== 'success') || contexts.some(c => c.state !== 'success');
|
||||
});
|
||||
if (missing.length) lines.push(`REQUIRED_CHECKS: not green at ${j.stamp_commit.slice(0, 12)}: ${missing.join(', ')}`);
|
||||
const waitFix = `wait for ${missing.join(', ')} to pass on ${j.stamp_commit.slice(0, 12)} (dispatch the workflow on ${head.headRefName} if it never ran), then ${rerun}`;
|
||||
if (missing.length) return outPending({ code: 'QUEUE_NOT_READY', message: `required checks missing or not green on the stamp commit ${j.stamp_commit.slice(0, 12)}: ${missing.join(', ')}`, fix: waitFix }, lines, waitFix, run);
|
||||
if (!ci.green) return outPending({ code: 'QUEUE_NOT_READY', message: `checks on the stamp commit ${j.stamp_commit.slice(0, 12)} are not green` }, lines, `gstack-pr-wait ${o.pr} --until green && ${Q} train land --train ${t.nonce} --pr ${o.pr}`, run);
|
||||
main = mainNow();
|
||||
if ('error' in main) return outPending({ code: 'QUEUE_PROOF_MISSING', message: main.error }, lines, rerun, run);
|
||||
if (main.sha !== j.base) return stopped(env, t, 'QUEUE_TRAIN_TREE_MISMATCH', `main moved to ${main.sha.slice(0, 12)} during the slot's checks (stamped on ${j.base.slice(0, 12)})`, main.sha, lines, true);
|
||||
const m = ctx.gh(['pr', 'merge', o.pr, '--squash', '--match-head-commit', j.stamp_commit]);
|
||||
if (!m.ok) return outFailed({ code: 'QUEUE_NOT_READY', message: `gh pr merge refused: ${(m.err ?? m.out).trim().slice(0, 200)}` }, lines, rerun, run);
|
||||
main = mainNow();
|
||||
if ('error' in main) return outPending({ code: 'QUEUE_PROOF_MISSING', message: `merged; ${main.error}` }, lines, rerun, run);
|
||||
if (main.sha === j.base) return outPending({ code: 'QUEUE_NOT_READY', message: `merged; main still reads ${j.base.slice(0, 12)}, so the merge is not visible yet` }, lines, rerun, run);
|
||||
if (main.tree !== s.tree) return stopped(env, t, 'QUEUE_TRAIN_TREE_MISMATCH', `after the merge main ${main.sha.slice(0, 12)} is not S${s.index} exactly`, main.sha, lines, true);
|
||||
return confirmMerged(env, t, o.pr, main.sha, s.index, [...lines, `MERGED: #${o.pr} as ${main.sha}; main's tree is S${s.index} exactly`]);
|
||||
const landing = t.landing?.[o.pr];
|
||||
const conditional = !!t.slots[i]!.base_red;
|
||||
let mode = landing?.mode === 'update' || fs.existsSync(path.join(path.dirname(run), `land-${o.pr}.json`)) ? { mode: 'update' as const, why: 'resuming the recorded update' } : chooseMode(ctx, s, main.sha, conditional);
|
||||
if (mode.mode === 'wait') {
|
||||
if (o.noWait) return outPending({ code: 'QUEUE_NOT_READY', message: mode.why }, s.lines, `gstack-pr-wait ${o.pr} --until green && ${rerun}`, run);
|
||||
waitChecks(ctx, pin, o.timeoutMin ?? 60);
|
||||
mode = chooseMode(ctx, s, main.sha, conditional);
|
||||
if (mode.mode === 'wait') return outPending({ code: 'QUEUE_NOT_READY', message: mode.why }, s.lines, `gstack-pr-wait ${o.pr} --until green && ${rerun}`, run);
|
||||
}
|
||||
return mode.mode === 'direct' ? landDirect(ctx, s, info, main.sha, mode.why) : landUpdate(ctx, s, info, main.sha, mode.why, o);
|
||||
}
|
||||
@@ -26,7 +26,9 @@ import { classifyTrainHead } from './version-gate-train';
|
||||
import { git } from './ship-queue-gate';
|
||||
import { dispatchCi } from './ship-queue-dispatch-ci';
|
||||
import { recordVerdict, screenVerdict, verdictLines, type ScreenOutcome } from './ship-queue-screen';
|
||||
import { execFor, repoSlug } from './ship-queue-train-github';
|
||||
import { execFor, prInfo, repoSlug } from './ship-queue-train-github';
|
||||
import { bindTrainReceipt } from './ship-queue-train-gate';
|
||||
import { extractReceipt } from './ship-receipt';
|
||||
import { loadReady, nowIso, persist, readyPath, rebuildFromNotes, type ReadyReceipt, type ScreenChoice } from './ship-queue-train-record';
|
||||
import { baseOverlap, fetchBase, mergeCommit, outOk, outPending, outRefused, trainEnv, type TrainEnv, type TrainOut } from './ship-queue-train';
|
||||
|
||||
@@ -36,6 +38,10 @@ export interface ReadyOptions {
|
||||
pr: string; pregate?: string; gateExit?: number; gateLog?: string; screen?: 'screen' | 'no-screen'; broad: boolean; dryRun: boolean;
|
||||
/** `--dispatch-ci`: run these workflows on the PR branch, wait, and clear the matching requires-remote pregate lanes before reading the head. */
|
||||
dispatchCi?: { workflows: string[]; timeoutSec: number; pollMs: number };
|
||||
/** `--base-red --fixer <n>`: a conditional READY on a paused receipt, owned by the fixer PR (trains with gates only). */
|
||||
baseRed?: { fixer?: string };
|
||||
/** `--fixes-base-red <case,...>`: the fixer's READY names the gate cases T0 may be red on. */
|
||||
fixesBaseRed?: string[];
|
||||
}
|
||||
export interface ScreenSelection { lines: string[]; recommend: 'screen' | 'no-screen' | null }
|
||||
|
||||
@@ -90,7 +96,7 @@ export async function gstackPaidSelection(repoRoot: string, baseSha: string, cha
|
||||
|
||||
async function screenSelection(env: TrainEnv, head: string, baseSha: string): Promise<ScreenSelection> {
|
||||
const price = 'SCREEN: price pending the T0 cost replay (lib/gate-cost.ts): cases without cost data are unknown; a red train holds its PRs about an hour';
|
||||
if (env.train.screen_selection === 'none') return { recommend: null, lines: ['SCREEN: selection not computed (train.screen_selection is none); no recommendation, the lane decides', price] };
|
||||
if (env.train.gates || env.train.screen_selection === 'none') return { recommend: null, lines: ['SCREEN: selection not computed (train.screen_selection is none); no recommendation, the lane decides', price] };
|
||||
const changed = git(env.ctx.repoRoot, ['diff', '--name-only', `${baseSha}...${head}`]).out.split('\n').filter(Boolean);
|
||||
const hasTemplate = (dir: string) => git(env.ctx.repoRoot, ['cat-file', '-e', `${head}:${dir}/SKILL.md.tmpl`]).ok;
|
||||
const s = await gstackPaidSelection(env.ctx.repoRoot, baseSha, changed, hasTemplate);
|
||||
@@ -168,11 +174,48 @@ async function dispatchScreen(env: TrainEnv, o: ReadyOptions, slug: string | nul
|
||||
|
||||
const screenLine = (c: ScreenChoice) => `screen: ${c.choice}${c.policy ? ` policy=${c.policy} nonce=${c.nonce} run=${c.run ?? '-'}${c.verdict ? ` verdict=${c.verdict.outcome}` : ''}` : ''}`;
|
||||
|
||||
/**
|
||||
* `ready --base-red --fixer <n>` (plan GSTA-157 item 5): the head's receipt is conditional (its base_red block), so
|
||||
* Step 5, CI and the pregate did not pass on the broken base and the per-PR READY checks are not asked; instead the
|
||||
* condition must be current (classified on today's base, not expired) and owned by a fixer PR. The READY receipt
|
||||
* records it; train build seats the PR only behind its fixer at slot 1 of a train with gates, every prefix is gated
|
||||
* with nothing excluded, and train land merges it only through an update commit whose own checks are green.
|
||||
*/
|
||||
function conditionalReady(env: TrainEnv, o: ReadyOptions, lines: string[]): TrainOut {
|
||||
const { ctx } = env;
|
||||
const head = git(ctx.repoRoot, ['rev-parse', 'HEAD']).out;
|
||||
const baseSha = git(ctx.repoRoot, ['rev-parse', `${env.remote}/${ctx.base}`]).out;
|
||||
const info = prInfo(ctx, o.pr);
|
||||
if ('error' in info) return outRefused({ code: 'QUEUE_PROOF_MISSING', message: info.error }, lines, 'none', 'none');
|
||||
if (info.headRefOid !== head) return outRefused({ code: 'QUEUE_NOT_READY', message: `HEAD ${head.slice(0, 12)} is not PR #${o.pr}'s pushed head ${info.headRefOid.slice(0, 12)}` }, lines, 'push, then ready again', 'none');
|
||||
const red = extractReceipt(info.body).receipt?.base_red;
|
||||
const again = `gstack-base-red classify on the current base, /ship, then ${Q} ready --pr ${o.pr} --no-screen --base-red --fixer ${o.baseRed!.fixer}`;
|
||||
if (!red) return outRefused({ code: 'SHIP_BASE_RED_FIXER', message: `#${o.pr}'s receipt carries no base_red condition; ready --base-red is only for a paused /ship` }, lines, `${Q} ready --pr ${o.pr} --no-screen`, 'none');
|
||||
if (red.base !== baseSha) return outRefused({ code: 'SHIP_BASE_RED_FIXER', message: `the condition was classified at ${red.base.slice(0, 12)} and ${ctx.base} is now ${baseSha.slice(0, 12)}; a base move reclassifies, never trusts` }, lines, again, 'none');
|
||||
if (!(Date.parse(red.expires_at) >= Date.parse(nowIso()))) return outRefused({ code: 'BASE_RED_PAUSE_EXPIRED', message: `the condition expired at ${red.expires_at}` }, lines, again, 'none');
|
||||
const shipped = bindTrainReceipt(ctx.repoRoot, `refs/remotes/${env.remote}/${ctx.base}`, info.body, o.pr, [head], true);
|
||||
lines.push(shipped.line, ...(shipped.depthLine ? [shipped.depthLine] : []));
|
||||
if (!shipped.ok) return outRefused({ code: shipped.code, message: shipped.message }, lines, `run /ship on ${head.slice(0, 12)}, then ready again`, 'none');
|
||||
const now = nowIso();
|
||||
const base_red = { cases: red.cases, fixer: o.baseRed!.fixer!, base: red.base, expires_at: red.expires_at };
|
||||
const receipt: ReadyReceipt = { schema: 1, kind: 'ready', repo_key: env.repo.key, pr: o.pr, head, base_sha: baseSha, ready_at: now, screen: { choice: 'no-screen', policy: null, nonce: null, run: null, merge_sha: null }, updated_at: now, base_red };
|
||||
lines.unshift(`READY #${o.pr} ${head} CONDITIONAL`, 'screen: no-screen');
|
||||
lines.push(`BASE_RED: ${red.cases.join(', ')} base=${red.base.slice(0, 12)} fixer=#${base_red.fixer} expires=${red.expires_at} (CI and the pregate ran on the broken base; the train gates every prefix and lands this PR only through an update commit with green checks)`);
|
||||
if (o.dryRun) return outOk([...lines, 'DRY-RUN: no READY receipt written'], `${Q} ready --pr ${o.pr} --no-screen --base-red --fixer ${base_red.fixer}`, 'none');
|
||||
const p = persist(ctx.repoRoot, [receipt], { remote: env.remote });
|
||||
lines.push(`READY_RECEIPT: ${readyPath(env.repo.key, o.pr, head)} (${p.detail})`);
|
||||
return outOk(lines, `send the READY block; the coordinator builds the train with fixer #${base_red.fixer} at slot 1: ${Q} train build --prs ${base_red.fixer}@<sha>,${o.pr}@${head}`, readyPath(env.repo.key, o.pr, head));
|
||||
}
|
||||
|
||||
export async function trainReady(ctx: QueueContext, o: ReadyOptions): Promise<TrainOut> {
|
||||
const fetchErr = fetchBase(ctx);
|
||||
const env = trainEnv(ctx);
|
||||
if ('code' in env) return outRefused(env, [], 'none', 'none');
|
||||
if ('code' in env) return outRefused(o.baseRed || o.fixesBaseRed ? { code: 'QUEUE_BASE_RED_NO_TRAIN', message: `${env.message}; a base-red READY needs a train with gates` } : env, [], 'none', 'none');
|
||||
if (fetchErr) return outRefused({ code: 'POLICY_SOURCE_UNAVAILABLE', message: fetchErr }, [], 'none', 'none');
|
||||
if ((o.baseRed || o.fixesBaseRed) && !env.train.gates) return outRefused({ code: 'QUEUE_BASE_RED_NO_TRAIN', message: 'the base policy\'s train is the gstack-evals kind, which gates only its tip; a base-red READY needs a train with gates' }, [], 'none', 'none');
|
||||
if (o.baseRed && !o.baseRed.fixer) return outRefused({ code: 'SHIP_BASE_RED_FIXER', message: 'ready --base-red names no fixer: a conditional READY needs the PR whose merge fixes its cases' }, [], `${Q} ready --pr ${o.pr} --no-screen --base-red --fixer <pr>`, 'none');
|
||||
if (env.train.gates && o.screen === 'screen') return outRefused({ code: 'READY_SCREEN_CHOICE_REQUIRED', message: 'a train with gates has no paid screen; every prefix is gated at build' }, [], `${Q} ready --pr ${o.pr} --no-screen`, 'none');
|
||||
if (o.baseRed) return conditionalReady(env, o, []);
|
||||
const ciLines: string[] = [];
|
||||
if (o.dispatchCi && !o.dryRun) {
|
||||
const d = await dispatchCi(ctx, { pr: o.pr, pregate: o.pregate, ...o.dispatchCi });
|
||||
@@ -235,7 +278,8 @@ export async function trainReady(ctx: QueueContext, o: ReadyOptions): Promise<Tr
|
||||
}
|
||||
lines.splice(lines.findIndex(l => l.startsWith('READY ')) + 1, 0, screenLine(choice));
|
||||
const now = nowIso();
|
||||
const receipt: ReadyReceipt = { schema: 1, kind: 'ready', repo_key: env.repo.key, pr: o.pr, head, base_sha: baseSha, ready_at: prior?.ready_at ?? now, screen: choice, updated_at: now };
|
||||
const receipt: ReadyReceipt = { schema: 1, kind: 'ready', repo_key: env.repo.key, pr: o.pr, head, base_sha: baseSha, ready_at: prior?.ready_at ?? now, screen: choice, updated_at: now, ...(o.fixesBaseRed ? { fixes_base_red: o.fixesBaseRed } : {}) };
|
||||
if (o.fixesBaseRed) lines.push(`FIXES_BASE_RED: ${o.fixesBaseRed.join(', ')} (T0 may be red on exactly these gate cases while this PR rides at slot 1)`);
|
||||
if (o.dryRun) return outOk([...lines, 'DRY-RUN: no READY receipt written, nothing dispatched'], `${Q} ready --pr ${o.pr} --${o.screen}${o.broad ? ' --broad' : ''}`, 'none');
|
||||
const p = persist(ctx.repoRoot, [receipt], { remote: env.remote });
|
||||
lines.push(`READY_RECEIPT: ${readyPath(env.repo.key, o.pr, head)} (${p.detail})`);
|
||||
|
||||
@@ -32,7 +32,21 @@ export interface ScreenChoice {
|
||||
branch?: string;
|
||||
verdict?: ScreenVerdictRecord;
|
||||
}
|
||||
export interface TrainSlot { pr: string; head: string; ready_at: string; screen: ScreenChoice }
|
||||
/** A conditional READY (`ready --base-red --fixer <n>`): the receipt's base-red cases and the PR whose merge fixes them. */
|
||||
export interface BaseRedCondition { cases: string[]; fixer: string; base: string; expires_at: string }
|
||||
export interface TrainSlot {
|
||||
pr: string; head: string; ready_at: string; screen: ScreenChoice;
|
||||
/** Conditional slots ride only a generic train whose slot 1 is their fixer. */
|
||||
base_red?: BaseRedCondition;
|
||||
/** The fixer's READY (`ready --fixes-base-red`): the gate cases T0 may be red on, and only those. */
|
||||
fixes_base_red?: string[];
|
||||
}
|
||||
/** One prefix a generic train gates (lib/ship-queue-train-gate.ts), on its own train-prefix/<nonce>/t<i> (or s<i>) ref. */
|
||||
export interface GatedPrefix { index: number; ref: string; commit: string; tree: string; outcome: GateOutcome | null; red: string[]; detail: string | null }
|
||||
/** A generic train's verdict over its prefixes: slots 1..landable may land; culprit is the held slot's PR. */
|
||||
export interface GenericVerdict { kind: 'green' | 'attributed' | 'red-on-main' | 'unattributed' | 'incomplete'; landable: number; culprit: string | null; at: string }
|
||||
/** Written to the record and the notes before train land's first remote effect for a slot. */
|
||||
export interface LandingRecord { slot: number; mode: 'direct' | 'update'; method: 'squash' | 'merge' | 'rebase'; pin: string; update: string | null; base: string; expected_tree: string; at: string }
|
||||
export interface TrainPrefix { index: number; commit: string; tree: string; pr: string | null }
|
||||
/** A stamped prefix S_i (changelog_fragments trains, lib/ship-queue-train-stamp.ts): what slot i publishes, exactly. */
|
||||
export interface StampedPrefix { index: number; commit: string; tree: string; pr: string; version: string; date: string }
|
||||
@@ -44,6 +58,10 @@ export interface TrainRecord {
|
||||
merged: Array<{ pr: string; sha: string; tree: string; at: string }>; superseded_by: string | null;
|
||||
/** S_1..S_k when the base policy declares changelog_fragments; head and tree are then S_k's. Absent on plain trains. */
|
||||
stamped?: StampedPrefix[];
|
||||
/** Generic trains (a policy train block with gates): every prefix gated; absent on gstack-evals trains, which keep today's rules. */
|
||||
gated?: GatedPrefix[];
|
||||
verdict?: GenericVerdict;
|
||||
landing?: Record<string, LandingRecord>;
|
||||
effects: Array<{ at: string; what: string }>; error: string | null; created_at: string; updated_at: string;
|
||||
}
|
||||
export type AttemptPurpose = 'gate' | 'rebuild' | 'infra-redispatch' | 'bisect';
|
||||
@@ -58,6 +76,11 @@ export interface AttemptRecord {
|
||||
inputs: Record<string, string>; state: 'dispatching' | 'unknown' | 'absent' | 'dispatched' | 'completed'; run: { id: number; url: string } | null;
|
||||
outcome: GateOutcome | null; detail: string | null; red_cases: string[]; panels: CasePanel[]; infra_eligible: boolean;
|
||||
cost_usd: number | null; prefixes: number[]; created_at: string; updated_at: string;
|
||||
/** Generic trains: the policy gate index, the run attempt read, both attempts' jobs, the rerun and its FLAKY cases, a reused T0 run. */
|
||||
gate?: number; run_attempt?: number; rerun?: 'requested' | 'refused'; jobs?: Array<{ attempt: number; name: string; conclusion: string | null }>;
|
||||
flaky?: string[]; reused?: string; tree?: string | null; spend_usd?: number;
|
||||
/** A command gate's running process: reconcile marks the attempt absent only once this owner is gone. */
|
||||
owner?: { pid: number; host: string };
|
||||
}
|
||||
export type PrefixResult = 'red' | 'green';
|
||||
export interface BisectRecord {
|
||||
@@ -75,6 +98,7 @@ export interface FreezeRecord {
|
||||
}
|
||||
export interface ReadyReceipt {
|
||||
schema: 1; kind: 'ready'; repo_key: string; pr: string; head: string; base_sha: string; ready_at: string; screen: ScreenChoice; updated_at: string;
|
||||
base_red?: BaseRedCondition; fixes_base_red?: string[];
|
||||
}
|
||||
export type AnyRecord = TrainRecord | AttemptRecord | BisectRecord | FreezeRecord | ReadyReceipt;
|
||||
|
||||
|
||||
@@ -26,6 +26,7 @@ import {
|
||||
dispatchAttempt, effect, findTrain, gateDispatch, nextFor, outFailed, outOk, outPending, outRefused, prsArgs, save, trainEnv,
|
||||
type TrainEnv, type TrainOut,
|
||||
} from './ship-queue-train';
|
||||
import { genericVerdict } from './ship-queue-train-gate';
|
||||
|
||||
/** EVAL_POLICY.infraRedispatch (test/helpers/periodic-exclude-data.ts): one re-dispatch of an INFRA/INCOMPLETE-only census. */
|
||||
export const INFRA_REDISPATCH = 1;
|
||||
@@ -110,6 +111,7 @@ export function trainVerdict(ctx: QueueContext, o: { train: string }): TrainOut
|
||||
const t = findTrain(env, o.train);
|
||||
if (!t) return outRefused({ code: 'QUEUE_PROOF_MISSING', message: `no train record ${o.train} here or in refs/notes/gstack-trains` }, [], `${Q} train status`, 'none');
|
||||
const run = trainFile(t.nonce);
|
||||
if (t.gated) return env.train.gates ? genericVerdict(env, t) : outRefused({ code: 'QUEUE_TRAIN_OFF', message: `train ${t.nonce} has gated prefixes, but the base policy's train block no longer declares gates` }, [], 'none', run);
|
||||
const a = gateAttempts(t).at(-1);
|
||||
if (!a) return outRefused({ code: 'QUEUE_PROOF_MISSING', message: `train ${t.nonce} has no gate dispatch yet` }, [], nextFor(t), run);
|
||||
if (a.state === 'dispatching' || a.state === 'unknown') return unknownDispatch(t, a);
|
||||
@@ -248,6 +250,7 @@ export async function trainBisect(ctx: QueueContext, o: { train: string; spendCa
|
||||
const t = findTrain(env, o.train);
|
||||
if (!t) return outRefused({ code: 'QUEUE_PROOF_MISSING', message: `no train record ${o.train} here or in refs/notes/gstack-trains` }, [], `${Q} train status`, 'none');
|
||||
const run = trainFile(t.nonce);
|
||||
if (t.gated) return outRefused({ code: 'QUEUE_TRAIN_REBUILD_REFUSED', message: `train ${t.nonce} gated every prefix; its verdict already names the culprit, so there is nothing to bisect` }, [], nextFor(t), run);
|
||||
const red = gateAttempts(t).at(-1);
|
||||
if (!red || red.outcome !== 'RED') return outRefused({ code: 'QUEUE_PROOF_MISSING', message: `bisect needs a red verdict; the train's last gate is ${red?.outcome ?? 'missing'}` }, [], nextFor(t), run);
|
||||
if (red.purpose === 'rebuild') return outRefused({ code: 'QUEUE_PROOF_MISSING', message: 'a rebuild\'s second red is not bisected again' }, [], nextFor(t), run);
|
||||
@@ -315,6 +318,7 @@ export async function trainRebuild(ctx: QueueContext, o: { train: string; infra:
|
||||
const t = findTrain(env, o.train);
|
||||
if (!t) return outRefused({ code: 'QUEUE_PROOF_MISSING', message: `no train record ${o.train} here or in refs/notes/gstack-trains` }, [], `${Q} train status`, 'none');
|
||||
const run = trainFile(t.nonce);
|
||||
if (t.gated) return outRefused({ code: 'QUEUE_TRAIN_REBUILD_REFUSED', message: `train ${t.nonce} gated every prefix; a generic train gets no unchanged rebuild (repair the red cases or change the order)` }, [], nextFor(t), run);
|
||||
const gates = gateAttempts(t);
|
||||
const last = gates.at(-1);
|
||||
const repair = `route ${last?.red_cases.join(',') || 'the red cases'} to a repair lane (GStack Minder); the train is rebuilt only once the repair merges: ${Q} train build ${prsArgs(t.slots, t.explicit_order)}`;
|
||||
|
||||
@@ -25,6 +25,7 @@ import {
|
||||
} from './ship-queue-train';
|
||||
import { mergeChangelogs, ownEntryAboveMain, parseChangelog } from './changelog-check';
|
||||
import { buildReceipt } from './tree-receipt';
|
||||
import { genericBackstop } from './ship-queue-train-gate';
|
||||
|
||||
const Q = 'gstack-ship-queue';
|
||||
|
||||
@@ -121,7 +122,7 @@ function published(env: TrainEnv, t: TrainRecord, i: number, main: string): { sa
|
||||
* never seats a PR that may already be in.
|
||||
*/
|
||||
/** A record entry that counts as merged without a GitHub read: plain trains only; a stamped train counts only what GitHub reports MERGED (#3603). */
|
||||
const recordedMerged = (t: TrainRecord, pr: string): boolean => !t.stamped && t.merged.some(m => m.pr === pr);
|
||||
const recordedMerged = (t: TrainRecord, pr: string): boolean => !t.stamped && !t.gated && t.merged.some(m => m.pr === pr);
|
||||
|
||||
async function freezeQueue(env: TrainEnv, t: TrainRecord, reason: string, mainSha: string, states: ReadonlyMap<string, string>): Promise<FreezeRecord> {
|
||||
const now = nowIso();
|
||||
@@ -142,6 +143,12 @@ async function freezeQueue(env: TrainEnv, t: TrainRecord, reason: string, mainSh
|
||||
persist(env.ctx.repoRoot, [f], { remote: env.remote });
|
||||
effect(t, `queue frozen: ${reason}`);
|
||||
save(env, t);
|
||||
if (t.gated && env.train.gates) {
|
||||
f.backstop = await genericBackstop(env, mainSha);
|
||||
f.updated_at = nowIso();
|
||||
persist(env.ctx.repoRoot, [f], { remote: env.remote });
|
||||
return f;
|
||||
}
|
||||
const slug = repoSlug(env.ctx);
|
||||
const nonce = makeNonce('gate', mainSha, mainSha, randomBytes(3).toString('hex'), { purpose: 'backstop' });
|
||||
const dispatch = { workflow: env.train.gate_workflow, inputs: { [DISPATCH_INPUTS.evalsAll]: 'true', [DISPATCH_INPUTS.expectedSha]: mainSha, [DISPATCH_INPUTS.nonce]: nonce } };
|
||||
@@ -203,6 +210,7 @@ export async function verifyTrain(ctx: QueueContext, o: { pr: string; train: str
|
||||
const i = t.slots.findIndex(s => s.pr === o.pr);
|
||||
if (i < 0) return outRefused({ code: 'QUEUE_GATE_MISSING', message: `train ${t.nonce} does not include #${o.pr}` }, [], nextFor(t), run);
|
||||
if (t.stamped) return outRefused({ code: 'QUEUE_GATE_MISSING', message: `train ${t.nonce} has stamped prefixes; its slots publish S_i exactly through train land, never a hand stamp checked modulo stamps` }, [], `${Q} train land --train ${t.nonce} --pr ${o.pr}`, run);
|
||||
if (t.gated) return outRefused({ code: 'QUEUE_GATE_MISSING', message: `train ${t.nonce} gates every prefix; its slots publish their gated prefix exactly through train land, never a merge checked modulo stamps` }, [], `${Q} train land --train ${t.nonce} --pr ${o.pr}`, run);
|
||||
const gate = attemptsOf(t).filter(a => a.purpose !== 'bisect').at(-1);
|
||||
if (gate?.outcome !== 'GREEN' || !['green', 'merging', 'merged'].includes(t.state)) {
|
||||
return outRefused({ code: 'QUEUE_GATE_MISSING', message: `train ${t.nonce} has no green record (state ${t.state}, last gate ${gate?.outcome ?? 'none'})` }, [], nextFor(t), run);
|
||||
@@ -238,11 +246,12 @@ export async function verifyTrain(ctx: QueueContext, o: { pr: string; train: str
|
||||
|
||||
export function confirmMerged(env: TrainEnv, t: TrainRecord, pr: string, main: string, index: number, lines: string[]): TrainOut {
|
||||
const run = trainFile(t.nonce);
|
||||
const state = t.stamped ? prState(env.ctx, pr) : 'MERGED';
|
||||
if (state !== 'MERGED') return outRefused({ code: 'QUEUE_PROOF_MISSING', message: `main's tree is S${index} of train ${t.nonce}, but GitHub reports #${pr} as ${state}, not MERGED; only a merge GitHub reports counts` }, lines, `gh pr view ${pr} --json state,headRefOid; rerun once GitHub reports #${pr} MERGED`, run);
|
||||
const state = t.stamped || t.gated ? prState(env.ctx, pr) : 'MERGED';
|
||||
if (state !== 'MERGED') return outRefused({ code: 'QUEUE_PROOF_MISSING', message: `main's tree is ${t.stamped ? 'S' : 'T'}${index} of train ${t.nonce}, but GitHub reports #${pr} as ${state}, not MERGED; only a merge GitHub reports counts` }, lines, `gh pr view ${pr} --json state,headRefOid; rerun once GitHub reports #${pr} MERGED`, run);
|
||||
if (!t.merged.some(m => m.pr === pr)) t.merged.push({ pr, sha: main, tree: git(env.ctx.repoRoot, ['rev-parse', `${main}^{tree}`]).out, at: nowIso() });
|
||||
const merged = t.slots.filter(s => s.pr === pr || recordedMerged(t, s.pr) || (t.merged.some(m => m.pr === s.pr) && prState(env.ctx, s.pr) === 'MERGED'));
|
||||
t.state = merged.length === t.slots.length ? 'merged' : 'merging';
|
||||
const landable = t.verdict?.landable ?? t.slots.length;
|
||||
t.state = merged.length >= landable ? (landable === t.slots.length ? 'merged' : 'attributed') : 'merging';
|
||||
effect(t, `verified #${pr} merged as T${index}`);
|
||||
save(env, t);
|
||||
lines.push(`VERIFY_TRAIN: #${pr} merged; main's tree is T${index} of train ${t.nonce} (${merged.length}/${t.slots.length} merged)`);
|
||||
|
||||
@@ -8,17 +8,23 @@
|
||||
* polling gh, then reads the verdict, and prints the last step's NEXT. Run it
|
||||
* again after an interruption and it continues from the record. It never
|
||||
* dispatches anything the record would not dispatch on its own.
|
||||
*
|
||||
* A generic train (gates in the policy; GSTA-157 PR 2) loops build, wait, verdict until the verdict settles, and with
|
||||
* --land lands each landable slot in order through train land, stopping at the first refusal with its code and NEXT.
|
||||
* It never merges without --land and never merges main into a PR.
|
||||
*/
|
||||
import type { QueueContext } from './ship-queue';
|
||||
import { buildTrain, findTrain, nextFor, outOk, outPending, outRefused, trainEnv, trainStatus, type TrainOut } from './ship-queue-train';
|
||||
import { repoSlug, runView } from './ship-queue-train-github';
|
||||
import { attemptsOf, trainFile, type TrainRecord } from './ship-queue-train-record';
|
||||
import type { TrainEnv } from './ship-queue-train';
|
||||
import { trainVerdict } from './ship-queue-train-verdict';
|
||||
import { verifyPostMerge } from './ship-queue-post-merge';
|
||||
|
||||
const Q = 'gstack-ship-queue';
|
||||
const sleep = (ms: number) => new Promise(resolve => setTimeout(resolve, ms));
|
||||
|
||||
export interface WatchOptions { train: string; timeoutSec: number; pollMs: number; freeTimeoutSec: number }
|
||||
export interface WatchOptions { train: string; timeoutSec: number; pollMs: number; freeTimeoutSec: number; land?: boolean }
|
||||
|
||||
/** Wait for the train's current gate run to complete; null when it did, else why the wait stopped. */
|
||||
async function awaitGate(ctx: QueueContext, t: TrainRecord, deadline: number, pollMs: number): Promise<string | null> {
|
||||
@@ -34,6 +40,59 @@ async function awaitGate(ctx: QueueContext, t: TrainRecord, deadline: number, po
|
||||
}
|
||||
}
|
||||
|
||||
/** Wait until every dispatched prefix run of a generic train completes; null when they did, else why the wait stopped. */
|
||||
async function awaitPrefixRuns(ctx: QueueContext, t: TrainRecord, deadline: number, pollMs: number): Promise<string | null> {
|
||||
const slug = repoSlug(ctx);
|
||||
if (!slug) return 'gh repo view could not name the repository';
|
||||
for (;;) {
|
||||
const open = attemptsOf(t).filter(a => a.state === 'dispatched' && a.run && a.outcome !== 'GREEN' && a.outcome !== 'RED' && a.outcome !== 'INCOMPLETE');
|
||||
const running = open.filter(a => runView(ctx, slug, a.run!.id)?.status !== 'completed');
|
||||
if (!running.length) return null;
|
||||
if (Date.now() >= deadline) return `${running.length} prefix run(s) still running at the watch timeout (${running.map(a => a.run!.id).join(', ')})`;
|
||||
await sleep(pollMs);
|
||||
}
|
||||
}
|
||||
|
||||
async function watchGeneric(ctx: QueueContext, env: TrainEnv, start: TrainRecord, o: WatchOptions, lines: string[], again: string, deadline: number): Promise<TrainOut> {
|
||||
let t = start;
|
||||
const run = trainFile(t.nonce);
|
||||
const reload = () => { t = findTrain(env, o.train) ?? t; };
|
||||
let verdict: TrainOut | null = null;
|
||||
while (!t.verdict) {
|
||||
if (t.state === 'built' || t.state === 'pushed' || verdict?.next.includes('train build --train')) {
|
||||
const b = await buildTrain(ctx, { resume: t.nonce, dryRun: false, urgent: false, repair: false, freeTimeoutSec: o.freeTimeoutSec, pollMs: o.pollMs });
|
||||
reload();
|
||||
lines.push(`WATCH: train build --train ${t.nonce} (${b.failure?.code ?? b.status})`);
|
||||
if (b.status !== 'complete') return { ...b, lines: [...lines, ...b.lines] };
|
||||
}
|
||||
const why = await awaitPrefixRuns(ctx, t, deadline, o.pollMs);
|
||||
if (why) return outPending({ code: 'QUEUE_TRAIN_PENDING', message: why }, lines, again, run);
|
||||
verdict = trainVerdict(ctx, { train: t.nonce });
|
||||
reload();
|
||||
if (!t.verdict && Date.now() >= deadline) return { ...verdict, lines: [...lines, ...verdict.lines], next: again };
|
||||
if (!t.verdict) await sleep(o.pollMs);
|
||||
}
|
||||
verdict ??= trainVerdict(ctx, { train: t.nonce });
|
||||
lines.push(...verdict.lines);
|
||||
if (!o.land || !t.verdict.landable) return { ...verdict, lines };
|
||||
const { postMergeChecks, trainLand } = await import('./ship-queue-train-land');
|
||||
const post = postMergeChecks(env);
|
||||
for (const s of t.slots.slice(0, t.verdict.landable)) {
|
||||
if (t.merged.some(m => m.pr === s.pr)) continue;
|
||||
const remaining = Math.max(1, Math.floor((deadline - Date.now()) / 60_000));
|
||||
const landed = await trainLand(ctx, { train: t.nonce, pr: s.pr, timeoutMin: remaining });
|
||||
lines.push(`WATCH: train land --train ${t.nonce} --pr ${s.pr} (${landed.failure?.code ?? landed.status})`, ...landed.lines);
|
||||
if (landed.status !== 'complete') return { ...landed, lines };
|
||||
reload();
|
||||
const sha = t.merged.find(m => m.pr === s.pr)?.sha;
|
||||
if (!post.length || !sha) continue;
|
||||
const checked = await verifyPostMerge(ctx, { sha, checks: post, timeoutSec: Math.max(1, Math.floor((deadline - Date.now()) / 1000)), pollMs: o.pollMs });
|
||||
lines.push(`WATCH: verify --post-merge ${sha} (${checked.failure?.code ?? checked.status})`, ...checked.lines);
|
||||
if (checked.status !== 'complete') return { ...checked, lines, next: checked.failure?.code === 'QUEUE_TRAIN_PENDING' ? `${checked.next} && ${Q} train watch --train ${t.nonce} --land` : checked.next };
|
||||
}
|
||||
return outOk(lines, nextFor(t), run);
|
||||
}
|
||||
|
||||
export async function trainWatch(ctx: QueueContext, o: WatchOptions): Promise<TrainOut> {
|
||||
const env = trainEnv(ctx);
|
||||
if ('code' in env) return outRefused(env, [], 'none', 'none');
|
||||
@@ -45,6 +104,7 @@ export async function trainWatch(ctx: QueueContext, o: WatchOptions): Promise<Tr
|
||||
const again = `${Q} train watch --train ${t.nonce} --timeout ${o.timeoutSec}`;
|
||||
const step = (label: string, out: TrainOut): TrainOut => ({ ...out, lines: [...lines, `WATCH: ${label}`, ...out.lines] });
|
||||
const reload = () => { t = findTrain(env, o.train) ?? t; };
|
||||
if (t.gated) return watchGeneric(ctx, env, t, o, lines, again, deadline);
|
||||
if (t.state === 'dispatch-unknown') {
|
||||
const s = trainStatus(ctx, { train: t.nonce });
|
||||
lines.push(`WATCH: train status settled the unconfirmed dispatch`, ...s.lines.filter(l => l.startsWith('ATTEMPT:')));
|
||||
|
||||
+27
-14
@@ -23,11 +23,11 @@ import { versionAtRef, versionFiles } from './restamp';
|
||||
import type { LoadedPolicy, ShipPolicy, TrainPolicy } from './ship-policy';
|
||||
import { loadBasePolicy, type QueueContext } from './ship-queue';
|
||||
import { mergeManifest } from './ship-queue-land';
|
||||
import { bindShippedReceipt } from './ship-receipt-depth';
|
||||
import { git, sha256, withIndex, type QueueFailure } from './ship-queue-gate';
|
||||
import { recordVerdict, screenVerdict, verdictLines } from './ship-queue-screen';
|
||||
import { stackTestLine, stackTests } from './ship-queue-stack-tests';
|
||||
import { stampChain } from './ship-queue-train-stamp';
|
||||
import { advanceGeneric, baseRedAdmission, bindTrainReceipt, gatedChain, genericNext } from './ship-queue-train-gate';
|
||||
import { asRun, execFor, newestBranchRun, prInfo, remoteBranchSha, repoSlug, runView, checkRun } from './ship-queue-train-github';
|
||||
import {
|
||||
TERMINAL_STATES, activeFreeze, attemptsOf, listTrains, loadBisect, loadReady, loadTrain, nowIso, persist, rebuildFromNotes, repoIdentity, trainFile,
|
||||
@@ -273,6 +273,7 @@ export function buildChain(repoRoot: string, baseSha: string, slots: TrainSlot[]
|
||||
|
||||
/** NEXT for a train in its current state. */
|
||||
export function nextFor(t: TrainRecord): string {
|
||||
if (t.gated) return genericNext(t);
|
||||
const n = t.nonce;
|
||||
const unmerged = t.slots.filter(s => !t.merged.some(m => m.pr === s.pr));
|
||||
switch (t.state) {
|
||||
@@ -363,10 +364,10 @@ function admitSlots(env: TrainEnv, pins: Array<{ pr: string; sha: string }>, dry
|
||||
if (!info.headRefOid.startsWith(p.sha) || fetched !== info.headRefOid) {
|
||||
return outRefused({ code: 'QUEUE_TRAIN_MOVED', message: `PR #${p.pr} moved: pinned ${p.sha.slice(0, 12)}, head is now ${info.headRefOid.slice(0, 12)}` }, [], `ask #${p.pr}'s lane for READY on ${info.headRefOid.slice(0, 12)}, then ${Q} train build --prs <pins>`, 'none');
|
||||
}
|
||||
const shipped = bindShippedReceipt(ctx.repoRoot, `refs/remotes/${env.remote}/${ctx.base}`, info.body, p.pr, [info.headRefOid]);
|
||||
const ready = loadReady(env.repo.keys, p.pr, info.headRefOid);
|
||||
const shipped = bindTrainReceipt(ctx.repoRoot, `refs/remotes/${env.remote}/${ctx.base}`, info.body, p.pr, [info.headRefOid], !!ready?.base_red);
|
||||
if (!shipped.ok) return outRefused({ code: shipped.code, message: shipped.message }, [shipped.line, ...(shipped.depthLine ? [shipped.depthLine] : [])], `#${p.pr}'s lane runs /ship, then ${Q} ready --pr ${p.pr} --screen|--no-screen; then ${Q} train build --prs <pins>`, 'none');
|
||||
lines.push(`${shipped.targetedLine} pr=${p.pr}`);
|
||||
const ready = loadReady(env.repo.keys, p.pr, info.headRefOid);
|
||||
if (!ready) return outRefused({ code: 'QUEUE_NOT_READY', message: `no READY receipt for #${p.pr} at ${info.headRefOid.slice(0, 12)} (local state or ${'refs/notes/gstack-trains'})` }, [], `the lane runs ${Q} ready --pr ${p.pr} --screen|--no-screen`, 'none');
|
||||
let screen = ready.screen;
|
||||
if (screen.choice === 'screen' && screen.run && slug === undefined) slug = repoSlug(ctx);
|
||||
@@ -380,7 +381,7 @@ function admitSlots(env: TrainEnv, pins: Array<{ pr: string; sha: string }>, dry
|
||||
else if (read?.kind === 'unknown') lines.push(`SCREEN: #${p.pr} ${read.detail}; counted as screened, unverified`);
|
||||
else if (read?.kind === 'green') lines.push(`SCREEN: #${p.pr} run ${screen.run} GREEN${read.flaky.length ? ` (flaky: ${read.flaky.join(', ')})` : ''}`);
|
||||
else if (read?.kind === 'pending') lines.push(`SCREEN: #${p.pr} run ${screen.run} still running; counted as screened`);
|
||||
slots.push({ pr: p.pr, head: info.headRefOid, ready_at: ready.ready_at, screen });
|
||||
slots.push({ pr: p.pr, head: info.headRefOid, ready_at: ready.ready_at, screen, ...(ready.base_red ? { base_red: ready.base_red } : {}), ...(ready.fixes_base_red ? { fixes_base_red: ready.fixes_base_red } : {}) });
|
||||
}
|
||||
return { slots, lines };
|
||||
}
|
||||
@@ -398,9 +399,12 @@ async function newTrain(env: TrainEnv, o: BuildOptions): Promise<{ t: TrainRecor
|
||||
if (explicit && 'error' in explicit) return outRefused({ code: 'QUEUE_PROOF_MISSING', message: explicit.error }, [], 'none', 'none');
|
||||
const admitted = admitSlots(env, pins, o.dryRun);
|
||||
if ('exit' in admitted) return admitted;
|
||||
const order = orderSlots(admitted.slots, o.repair ? 1 : env.train.max_size, explicit);
|
||||
const ordered = orderSlots(admitted.slots, o.repair ? 1 : env.train.max_size, explicit);
|
||||
const admission = baseRedAdmission(ordered.train, !!env.train.gates, !!explicit);
|
||||
if ('code' in admission) return outRefused(admission, admitted.lines, `seat the fixer at slot 1 of a train with gates, or drop the conditional PR: ${Q} train build --prs <pins>`, 'none');
|
||||
const order = { train: admission.slots, held: ordered.held };
|
||||
const lines = [
|
||||
...admitted.lines,
|
||||
...admitted.lines, ...admission.lines,
|
||||
`ORDER: ${order.train.map((s, i) => `${i + 1}:#${s.pr}(${s.screen.choice === 'screen' ? 'screened' : 'unscreened'})`).join(' ')}${explicit ? ' (explicit --order)' : ''}`,
|
||||
...(order.held.length ? [`HELD: ${order.held.map(s => '#' + s.pr).join(',')} (over the train size ${env.train.max_size})`] : []),
|
||||
];
|
||||
@@ -423,18 +427,20 @@ async function newTrain(env: TrainEnv, o: BuildOptions): Promise<{ t: TrainRecor
|
||||
lines.push(`MERGE_CHECK: T1..T${order.train.length} merge clean onto ${env.ctx.base}@${baseSha.slice(0, 12)} (dry run in object storage; nothing pushed)`);
|
||||
const due = fillDue(order.train, env.train, nowIso(), o.urgent || o.repair);
|
||||
if (due && !o.dryRun) return outPending({ code: 'QUEUE_TRAIN_WAITING', message: `${order.train.length} of ${env.train.max_size} PRs; the oldest READY dispatches at ${due}` }, lines, `${Q} train build ${prsArgs(order.train, !!explicit)} (after ${due}, or with more PRs, or --urgent for one)`, 'none');
|
||||
const live = listTrains(env.repo.keys).find(t => t.key === key && !TERMINAL_STATES.includes(t.state) && t.state !== 'red' && t.state !== 'unattributed');
|
||||
const live = listTrains(env.repo.keys).find(t => t.key === key && !TERMINAL_STATES.includes(t.state) && t.state !== 'red' && t.state !== 'unattributed' && !(t.gated && t.state === 'incomplete'));
|
||||
if (live) return { t: Object.assign(live, { held: order.held.map(s => s.pr) }), lines };
|
||||
const policy = env.policy.policy;
|
||||
const stamped = policy.changelog_fragments ? stampChain({ repoRoot: env.ctx.repoRoot, base: env.ctx.base, policy, policyLabel: env.policy.source.label, baseSha, slots: order.train, plain: chain.prefixes, allowRepoCommands: !!o.allowRepoCommands, now: new Date(nowIso()) }) : null;
|
||||
const stamps = !!policy.changelog_fragments || (!!env.train.gates && policy.queue_mode === 'stamp-at-merge');
|
||||
const stamped = stamps ? stampChain({ repoRoot: env.ctx.repoRoot, base: env.ctx.base, policy, policyLabel: env.policy.source.label, baseSha, slots: order.train, plain: chain.prefixes, allowRepoCommands: !!o.allowRepoCommands, now: new Date(nowIso()) }) : null;
|
||||
if (stamped && 'code' in stamped) return outFailed(stamped, lines, `${Q} train build ${prsArgs(order.train, !!explicit)} --allow-repo-commands (after the named slot is fixed or dropped)`, 'none');
|
||||
if (stamped) lines.push(...stamped.lines);
|
||||
const head = stamped ? stamped.stamped.at(-1)! : chain.prefixes.at(-1)!;
|
||||
const nonce = makeNonce('gate', head.commit, baseSha, randomBytes(3).toString('hex'), { purpose: 'train' });
|
||||
const now = nowIso();
|
||||
const gated = env.train.gates ? gatedChain(nonce, chain.prefixes, stamped?.stamped) : undefined;
|
||||
return { lines, t: {
|
||||
schema: 1, kind: 'train', nonce, key, repo_key: env.repo.key, repo: env.repo.url, base: { ref: env.ctx.base, sha: baseSha }, policy: env.policy.source.label,
|
||||
slots: order.train, held: order.held.map(s => s.pr), prefixes: chain.prefixes, head: head.commit, tree: head.tree, branch: `train/${nonce}`,
|
||||
slots: order.train, held: order.held.map(s => s.pr), prefixes: chain.prefixes, head: head.commit, tree: head.tree, branch: gated ? gated.at(-1)!.ref : `train/${nonce}`, ...(gated ? { gated } : {}),
|
||||
urgent: o.urgent, repair: o.repair, ...(explicit ? { explicit_order: true } : {}), ...(stamped ? { stamped: stamped.stamped } : {}), state: 'built', free: null, attempts: [], merged: [], superseded_by: null, effects: [], error: null,
|
||||
created_at: now, updated_at: now,
|
||||
} };
|
||||
@@ -453,7 +459,8 @@ function describe(t: TrainRecord): string[] {
|
||||
`TRAIN: ${t.nonce} state=${t.state} prs=${t.slots.map(s => '#' + s.pr).join(',')} head=${t.head} tree=${t.tree}`,
|
||||
`base: origin/${t.base.ref}@${t.base.sha}`,
|
||||
...t.prefixes.map(p => `PREFIX: T${p.index} ${p.commit} tree=${p.tree}${p.pr ? ` (+#${p.pr})` : ' (main)'}`),
|
||||
...t.slots.map((s, i) => `SLOT: ${i + 1} #${s.pr} head=${s.head} ${s.screen.choice === 'screen' ? `screened (${s.screen.policy ?? 'narrow'}${s.screen.run ? `, run ${s.screen.run}` : ''})` : 'unscreened'} ready_at=${s.ready_at}`),
|
||||
...t.slots.map((s, i) => `SLOT: ${i + 1} #${s.pr} head=${s.head} ${s.screen.choice === 'screen' ? `screened (${s.screen.policy ?? 'narrow'}${s.screen.run ? `, run ${s.screen.run}` : ''})` : 'unscreened'} ready_at=${s.ready_at}${s.base_red ? ` conditional(fixer #${s.base_red.fixer})` : ''}${s.fixes_base_red ? ` FIXES_BASE_RED: ${s.fixes_base_red.join(',')}` : ''}`),
|
||||
...(t.gated ?? []).map(p => `GATED: T${p.index} ${p.ref} ${p.commit} tree=${p.tree}`),
|
||||
];
|
||||
}
|
||||
|
||||
@@ -501,6 +508,7 @@ async function freePhase(env: TrainEnv, slug: string, t: TrainRecord, o: BuildOp
|
||||
|
||||
/** Push the branch, gate it with the free tests, then dispatch the paid gate once. */
|
||||
async function advance(env: TrainEnv, t: TrainRecord, o: BuildOptions, lines: string[]): Promise<TrainOut> {
|
||||
if (t.gated) return env.train.gates ? advanceGeneric(env, t, lines, describe) : outRefused({ code: 'QUEUE_TRAIN_OFF', message: `train ${t.nonce} has gated prefixes, but the base policy's train block no longer declares gates` }, lines, 'none', trainFile(t.nonce));
|
||||
const run = trainFile(t.nonce);
|
||||
const slug = repoSlug(env.ctx);
|
||||
if (!slug) return outRefused({ code: 'QUEUE_PROOF_MISSING', message: 'gh repo view could not name the repository' }, lines, `${Q} train build --train ${t.nonce}`, run);
|
||||
@@ -538,7 +546,7 @@ export async function buildTrain(ctx: QueueContext, o: BuildOptions): Promise<Tr
|
||||
if (o.resume) {
|
||||
t = findTrain(env, o.resume) ?? outRefused({ code: 'QUEUE_PROOF_MISSING', message: `no train record ${o.resume} here or in refs/notes/gstack-trains` }, [], `${Q} train status`, 'none');
|
||||
if ('kind' in t && TERMINAL_STATES.includes(t.state)) return outRefused({ code: 'QUEUE_PROOF_MISSING', message: `train ${t.nonce} is ${t.state}; it never dispatches again` }, describe(t), nextFor(t), trainFile(t.nonce));
|
||||
if ('kind' in t && t.state !== 'free-pending' && t.state !== 'dispatch-absent' && t.state !== 'dispatch-unknown' && t.state !== 'built' && t.state !== 'pushed' && t.state !== 'free-green') {
|
||||
if ('kind' in t && t.state !== 'free-pending' && t.state !== 'dispatch-absent' && t.state !== 'dispatch-unknown' && t.state !== 'built' && t.state !== 'pushed' && t.state !== 'free-green' && !(t.gated && t.state === 'gate-pending')) {
|
||||
return outOk(describe(t), nextFor(t), trainFile(t.nonce));
|
||||
}
|
||||
} else {
|
||||
@@ -548,7 +556,8 @@ export async function buildTrain(ctx: QueueContext, o: BuildOptions): Promise<Tr
|
||||
}
|
||||
if (!('kind' in t)) return t;
|
||||
if (o.dryRun) {
|
||||
const dry = [...lines, ...describe(t), `ESTIMATE: one ${env.train.gate_workflow} full gate (evals_all) on T${t.prefixes.length - 1} plus ${env.train.free_workflow}; dollar figure pending the T0 cost replay (lib/gate-cost.ts)`];
|
||||
const dry = [...lines, ...describe(t), env.train.gates ? `ESTIMATE: ${env.train.gates.length} gate(s) on each of ${t.prefixes.length} prefixes (T0..T${t.prefixes.length - 1}, every one gated)`
|
||||
: `ESTIMATE: one ${env.train.gate_workflow} full gate (evals_all) on T${t.prefixes.length - 1} plus ${env.train.free_workflow}; dollar figure pending the T0 cost replay (lib/gate-cost.ts)`];
|
||||
if (o.stackTests) {
|
||||
const results = stackTests(env.ctx.repoRoot, t.prefixes[0]!.commit, t.slots, t.prefixes, o.stackTests === true ? undefined : o.stackTests);
|
||||
dry.push(...results.map(stackTestLine));
|
||||
@@ -598,7 +607,7 @@ export function trainStatus(ctx: QueueContext, o: { train?: string }): TrainOut
|
||||
for (const t of trains) {
|
||||
lines.push(...describe(t));
|
||||
for (const a of attemptsOf(t)) {
|
||||
if (slug && (a.state === 'dispatching' || a.state === 'unknown')) {
|
||||
if (slug && a.workflow !== 'command' && (a.state === 'dispatching' || a.state === 'unknown')) {
|
||||
const runs = listDispatchRuns(a.workflow, exec, slug, 100);
|
||||
const hit = runs ? runForNonce(runs, a.nonce) : null;
|
||||
if (runs) {
|
||||
@@ -612,7 +621,7 @@ export function trainStatus(ctx: QueueContext, o: { train?: string }): TrainOut
|
||||
const view = slug && a.run && !a.outcome ? runView(ctx, slug, a.run.id) : null;
|
||||
lines.push(`ATTEMPT: ${a.id} ${a.purpose} state=${a.state} nonce=${a.nonce} run=${a.run?.id ?? '-'}${view ? ` ${view.status}/${view.conclusion ?? '-'}` : ''} outcome=${a.outcome ?? '-'}`);
|
||||
}
|
||||
if (slug && (t.state === 'green' || t.state === 'merging')) {
|
||||
if (slug && !t.gated && (t.state === 'green' || t.state === 'merging')) {
|
||||
const c = checkRun(ctx, t.head, env.train.full_gate_check);
|
||||
lines.push(`CHECK: ${env.train.full_gate_check} on ${t.head.slice(0, 12)}: ${c ? `${c.status}/${c.conclusion ?? '-'}` : 'missing'}`);
|
||||
}
|
||||
@@ -666,6 +675,7 @@ export function pruneTrains(ctx: QueueContext, o: { dryRun: boolean; train?: str
|
||||
if (s.stale) {
|
||||
lines.push(`STALE: ${t.nonce} (${t.state}): ${s.detail}${o.dryRun ? '; would mark stale' : ''}`);
|
||||
if (!o.dryRun) { t.state = 'stale'; effect(t, `marked stale by train prune --stale: ${s.detail}`); save(env, t); }
|
||||
for (const p of (t.gated ?? []).filter(g => g.ref !== t.branch)) remove(p.ref, o.dryRun ? 'stale' : t.state);
|
||||
remove(t.branch, o.dryRun ? 'stale' : t.state, () => { effect(t, `branch deleted: ${t.branch}`); save(env, t); });
|
||||
continue;
|
||||
}
|
||||
@@ -673,6 +683,9 @@ export function pruneTrains(ctx: QueueContext, o: { dryRun: boolean; train?: str
|
||||
continue;
|
||||
}
|
||||
if (!TERMINAL_STATES.includes(t.state)) { lines.push(`KEEP: ${t.branch} (${t.state})`); continue; }
|
||||
const waiting = t.gated && t.verdict ? t.slots.slice(0, t.verdict.landable).filter(s => !t.merged.some(m => m.pr === s.pr)) : [];
|
||||
if (waiting.length) { lines.push(`KEEP: ${t.branch} (${t.state}; ${waiting.map(s => '#' + s.pr).join(', ')} landable, not merged)`); continue; }
|
||||
for (const p of (t.gated ?? []).filter(g => g.ref !== t.branch)) remove(p.ref, t.state);
|
||||
remove(t.branch, t.state, () => { effect(t, `branch deleted: ${t.branch}`); save(env, t); });
|
||||
}
|
||||
if (o.stale && !o.train) {
|
||||
|
||||
Reference in new issue
Block a user