Merge remote-tracking branch 'origin/main' into gsta258-s1-envrc-redact

This commit is contained in:
garrytan committed 2026-11-03 04:22:21 +00:00
commit 735ddb2654
39 files changed
+1516 -71

No files matched your search

+132
View File
@@ -0,0 +1,132 @@
/**
* ci-carry: CI is the one full test run (GSTA-258, R65 item 1). When a verified
* CI run already passed at the exact tree /ship or pregate is about to test,
* they use it instead of running the suite again.
*
* `findCarry` looks up the newest verified run of each named run profile (the
* base branch's `.gstack/pregate.json` `run_profiles`) for HEAD and accepts it
* only when every rule holds:
*
* the working tree is HEAD's committed tree (gstack-wtree), before and after
* the lookup, so admitted dirty or untracked work never carries;
* each run passes verifyRun (same-repository head, the profile's workflow,
* event and jobs, the newest attempt, no newer red or running run on HEAD,
* max age, the run's own tree.json naming HEAD's tree);
* every run executed tests;
* with an inventory command, the runs' tree.json files cover every file it
* prints (a lane carries only what CI provably ran);
* every run still stands after the inventory ran (runChanged).
*
* Anything else returns a refusal with a reason code, and the caller runs the
* suite as before. Security and advisory scans and the sealed guard are never
* carried: callers carry test lanes only.
*
* waitForRun polls the newest run of one workflow on HEAD until it completes,
* bounded by a timeout, and returns green, red with the failing jobs, still
* running, or none.
*/
import { runChanged, runReceiptFields, uncoveredFiles, verifyRun, type ApiRun, type RunDeps, type RunEvidence, type RunProfile, type RunSource } from './run-evidence';
export type CarryRefusal = 'dirty_tree' | 'no_profile' | 'no_run' | 'no_verified_run' | 'tree_mismatch' | 'no_tests'
| 'inventory_failed' | 'inventory_empty' | 'inventory_uncovered' | 'run_changed' | 'tree_changed' | 'api_error';
export type CarryRefused = { ok: false; reason: CarryRefusal; detail: string };
export type CarryVerdict = { ok: true; runs: RunEvidence[]; tests: number; receipt: string } | CarryRefused;
/** How many of HEAD's runs per workflow are tried, newest first. */
const CANDIDATES = 10;
const no = (reason: CarryRefusal, detail: string): CarryRefused => ({ ok: false, reason, detail });
const newestFirst = (a: ApiRun, b: ApiRun) => (b.created_at ?? '').localeCompare(a.created_at ?? '') || b.id - a.id;
/** The newest run of `profile` on `head` that verifies; a refusal lists each candidate's reason, newest first. */
function newestVerified(src: RunSource, name: string, profile: RunProfile, head: string): { ok: true; evidence: RunEvidence } | CarryRefused {
const listed = src.deps.runsForHead(profile.workflow, head).filter(r => profile.events.includes(r.event)).sort(newestFirst).slice(0, CANDIDATES);
if (!listed.length) return no('no_run', `no ${profile.workflow} run (${profile.events.join(', ')}) on ${head.slice(0, 12)} yet`);
const refusals: string[] = [];
for (const run of listed) {
const v = verifyRun(src.deps, { repo: src.repo, runRef: String(run.id), profileName: name, profile, vetoHeads: [head], now: src.now });
if (v.ok) return v;
refusals.push(`run ${run.id}: ${v.reason}`);
}
return no('no_verified_run', `${name}: ${refusals.join('; ')}`);
}
/**
* The CI evidence for HEAD's tree under `profiles` (one verified run each), or why there is none. `wtree` reads the
* working-tree fingerprint (before the lookup and after it); `inventory` lists the files the runs must cover.
*/
export function findCarry(src: RunSource, o: { profiles: string[]; head: string; tree: string; wtree: () => string | null; inventory?: () => string[] | { error: string } }): CarryVerdict {
const before = o.wtree();
if (before !== o.tree) return no('dirty_tree', `the working tree (${before ? before.slice(0, 12) : 'fingerprint unavailable'}) is not HEAD's committed tree ${o.tree.slice(0, 12)}; only a clean, committed tree carries`);
if (!o.profiles.length) return no('no_profile', 'no run profile is named');
const runs: RunEvidence[] = [];
try {
for (const name of o.profiles) {
const profile = src.profiles[name];
if (!profile) return no('no_profile', `${name} is not a run_profiles name on the base branch`);
const v = newestVerified(src, name, profile, o.head);
if (!v.ok) return v;
runs.push(v.evidence);
}
const other = runs.find(e => e.tree !== o.tree);
if (other) return no('tree_mismatch', `run ${other.run_id} tested tree ${other.tree.slice(0, 12)}, not ${o.tree.slice(0, 12)}`);
const empty = runs.find(e => !e.files.length || e.tests === 0);
if (empty) return no('no_tests', `run ${empty.run_id}'s tree.json ${empty.files.length ? 'records 0 tests' : 'lists no test files'}`);
if (o.inventory) {
const listed = o.inventory();
if ('error' in listed) return no('inventory_failed', listed.error);
if (!listed.length) return no('inventory_empty', 'the inventory command printed no files');
const missing = uncoveredFiles(listed, runs);
if (missing.length) return no('inventory_uncovered', `${missing.length} of ${listed.length} inventory file(s) are in no run's tree.json: ${missing.slice(0, 5).join(', ')}${missing.length > 5 ? ', …' : ''}`);
}
for (const e of runs) {
const changed = runChanged(src.deps, e.run_id, e.run_attempt);
if (changed) return no('run_changed', changed);
}
} catch (e) {
return no('api_error', `reading runs failed: ${String((e as Error)?.message ?? e).slice(0, 300)}`);
}
const after = o.wtree();
if (after !== o.tree) return no('tree_changed', `the working tree changed while the runs were verified (now ${after ? after.slice(0, 12) : 'unreadable'})`);
const tests = runs.reduce((n, e) => n + (e.tests ?? e.files.length), 0);
return { ok: true, runs, tests, receipt: `ci-carry tree=${o.tree.slice(0, 12)} tests=${tests} ${runReceiptFields(runs, src.now)}` };
}
export type WaitVerdict = { state: 'green'; run: ApiRun } | { state: 'red'; run: ApiRun; failing: string[] } | { state: 'running'; run: ApiRun } | { state: 'none'; reason: string };
const PASSING = ['success', 'skipped', 'neutral'];
/** Poll the newest `workflow` run on `head` (among `events`) until it completes or `timeoutMs` passes. */
export async function waitForRun(deps: RunDeps, o: { workflow: string; events: string[]; head: string; timeoutMs: number; intervalMs: number; sleep: (ms: number) => Promise<void>; clock: () => number }): Promise<WaitVerdict> {
const end = o.clock() + o.timeoutMs;
for (;;) {
const newest = deps.runsForHead(o.workflow, o.head).filter(r => o.events.includes(r.event)).sort(newestFirst)[0];
if (newest?.status === 'completed') {
if (newest.conclusion === 'success') return { state: 'green', run: newest };
const failing = deps.jobs(newest.id, newest.run_attempt).filter(j => !PASSING.includes(j.conclusion ?? '')).map(j => j.name);
return { state: 'red', run: newest, failing };
}
if (o.clock() >= end) return newest ? { state: 'running', run: newest } : { state: 'none', reason: `no ${o.workflow} run on ${o.head.slice(0, 12)} after ${Math.round(o.timeoutMs / 1000)} s` };
await o.sleep(Math.max(1, Math.min(o.intervalMs, end - o.clock())));
}
}
/** `ship_lanes_from` (base branch only): exact lane command → the profiles whose runs carry it and the command listing its test files. */
export interface ShipLaneMapping { profiles: string[]; inventory: string }
/** `<path>: <message>` errors for `ship_lanes_from`; empty when valid. Shared with lib/pregate/config.ts. */
export function shipLanesErrors(raw: unknown, profileNames: readonly string[]): Array<{ path: string; message: string }> {
if (raw === undefined) return [];
if (!raw || typeof raw !== 'object' || Array.isArray(raw)) return [{ path: 'ship_lanes_from', message: 'must be an object of exact lane command → { profiles: [run_profiles names], inventory: command }' }];
const errors: Array<{ path: string; message: string }> = [];
const oneLine = (s: unknown) => typeof s === 'string' && !!s.trim() && s.length <= 512 && !/[\0\r\n]/.test(s);
for (const [command, m] of Object.entries(raw as Record<string, unknown>)) {
const at = `ship_lanes_from[${JSON.stringify(command)}]`;
if (!oneLine(command)) errors.push({ path: at, message: 'the key must be the exact lane command: one line under 512 characters' });
if (!m || typeof m !== 'object' || Array.isArray(m)) { errors.push({ path: at, message: 'must be { profiles, inventory }' }); continue; }
const o = m as Record<string, unknown>;
for (const k of Object.keys(o)) if (k !== 'profiles' && k !== 'inventory') errors.push({ path: `${at}.${k}`, message: 'unknown key; valid keys: profiles, inventory' });
if (!Array.isArray(o.profiles) || !o.profiles.length || o.profiles.some(p => typeof p !== 'string')) errors.push({ path: `${at}.profiles`, message: 'must be a non-empty array of run_profiles names' });
else for (const p of o.profiles as string[]) if (!profileNames.includes(p)) errors.push({ path: `${at}.profiles`, message: `${JSON.stringify(p)} is not a run_profiles name` });
if (!oneLine(o.inventory)) errors.push({ path: `${at}.inventory`, message: 'must be one command line under 512 characters that prints the lane\'s test files, one per line' });
}
return errors;
}
+3
View File
@@ -97,6 +97,8 @@ export interface PregateFile {
export interface ShipReceipt {
schema_version: 1; head: string; base: string; pr?: number; version?: string; gated_tree?: string;
gate?: Record<string, unknown>; ci?: Record<string, unknown>; spend_usd?: number | string; predecessor?: string;
/** GSTA-258: the lanes CI carried at this head's tree, one entry per run (gstack-evidence carried). */
ci_carried?: Array<{ lane: string; run: number; attempt: number; tree: string }>;
session_kind?: string; artifacts_consumed?: string; run?: string;
// C-stamp (C3/C5/C7): the tree receipt's verdicts and the policy provenance, all optional.
thread?: string; tree?: string; gate_reuse?: string; policy?: string; queue_mode?: string; preregistration?: string; history?: string;
@@ -239,6 +241,7 @@ export const ARTIFACT_SCHEMAS = {
properties: {
schema_version: { const: 1 }, pr: { type: 'integer' }, head: str, version: str, base: str, gated_tree: str,
gate: { type: 'object' }, ci: { type: 'object' }, spend_usd: { type: ['number', 'string'] }, predecessor: str,
ci_carried: { type: 'array', items: { type: 'object', required: ['lane', 'run', 'attempt', 'tree'], properties: { lane: str, run: { type: 'integer', minimum: 1 }, attempt: { type: 'integer', minimum: 1 }, tree: str } } },
session_kind: str, artifacts_consumed: { ...str, description: '<n>/<m>' }, run: str,
thread: str, tree: { enum: ['same-modulo-stamps', 'changed', 'not-compared'] }, gate_reuse: str, policy: str, queue_mode: str, preregistration: str, history: str,
pregate: { ...str, description: 'pass|warn|requires-remote cleared … @<tree> from pregate.json, current for this head' }, requires_remote: strList,
+12 -4
View File
@@ -40,7 +40,7 @@ import { selectLanes, type Selection } from '../lane-selection';
import { launchRuntime } from '../runtime';
import { cargoModel, type CargoModel } from '../rust';
import { adapterFor, readRange, STALL_IDLE, supervise, type Counter, type Supervised } from '../supervise';
import { verifyFromRun } from '../../run-evidence';
import { verifyFromRun, type RunProfile } from '../../run-evidence';
import { matchesAny } from '../../ship-policy';
import type { CheckDef, CheckResult, PregateContext, RemoteObligation } from '../types';
@@ -533,7 +533,10 @@ async function runSelected(ctx: PregateContext, sel: Selection, base: CheckResul
const smallMachine = ctx.cores < SMALL_MACHINE_CORES && sel.localTests.length > 0 && (ctx.config.remote_gate || ctx.testBackend !== 'local');
if (smallMachine) {
const obligation = remoteObligation(ctx, sel);
return { ...base, status: 'requires-remote', remote: [...sel.remote, obligation], lines: [...lines, `requires-remote ${obligation.lane}: ${ctx.cores} cores (under ${SMALL_MACHINE_CORES}); ${sel.localTests.length} local test file(s) recorded for the remote gate, not run here`, `run: ${obligation.command}`], detail: `${sel.localTests.length} local test file(s) recorded as requires-remote ${obligation.lane} (${ctx.cores} cores, under ${SMALL_MACHINE_CORES}); run it, then ${clearHint(obligation.lane)}, or gstack-pregate --remote --spend-cap <usd> --allow-repo-commands` };
const route = obligation.lane === REMOTE_GATE_LANE && ctx.config.remote_gate_runs && ctx.ciCarry !== false
? `a verified CI run at this tree clears it: push, then gstack-ci-carry wait --base ${ctx.base} and rerun gstack-pregate --allow-repo-commands (a pull_request run of a branch behind its base tested the merge, another tree: dispatch the workflow on the branch); or opt in to the VM: gstack-pregate --remote --spend-cap <usd> --allow-repo-commands`
: `run it, then ${clearHint(obligation.lane)}, or gstack-pregate --remote --spend-cap <usd> --allow-repo-commands`;
return { ...base, status: 'requires-remote', remote: [...sel.remote, obligation], lines: [...lines, `requires-remote ${obligation.lane}: ${ctx.cores} cores (under ${SMALL_MACHINE_CORES}); ${sel.localTests.length} local test file(s) recorded for the remote gate, not run here`, `run: ${obligation.command}`], detail: `${sel.localTests.length} local test file(s) recorded as requires-remote ${obligation.lane} (${ctx.cores} cores, under ${SMALL_MACHINE_CORES}); ${route}` };
}
if (!ctx.allowRepoCommands) return { ...base, status: 'incomplete', code: 'REPO_COMMANDS_NOT_ALLOWED', detail: `${sel.localTests.length} local test file(s) selected but not run (pass --allow-repo-commands)`, fix: 'rerun with --allow-repo-commands (what /ship passes)' };
if (sel.localTests.length === 0) return { ...base, status: sel.remote.length ? 'requires-remote' : 'pass', detail: `no local lane; ${sel.remote.length} requires-remote obligation(s)` };
@@ -624,6 +627,12 @@ async function run(ctx: PregateContext): Promise<CheckResult> {
return mode === 'no-policy' ? withStarter(ctx, sel, result) : withNoLaneWarning(ctx, sel, result);
}
/** The base profiles a run may verify under here: this machine's OS, and a workflow that holds a lane this machine runs. */
export function fromRunProfiles(ctx: PregateContext, profiles: Record<string, RunProfile>): string[] {
const os = ({ linux: 'Linux', darwin: 'macOS', win32: 'Windows' } as Record<string, string>)[ctx.platform];
return Object.entries(profiles).filter(([, p]) => p.os === os && ctx.lanes.some(l => l.tier !== 'platform' && l.id.startsWith(`${p.workflow}/`))).map(([n]) => n);
}
/**
* `--from-run`: verify the run against the base branch's profiles for this machine's OS whose workflow holds a lane this
* machine runs (not a platform lane), on exactly this tree; the selected local test files its tree.json lists are the
@@ -631,8 +640,7 @@ async function run(ctx: PregateContext): Promise<CheckResult> {
*/
function fromRun(ctx: PregateContext, sel: Selection): { line: string; passed: string[]; receipt?: string } {
const { ref, check } = ctx.fromRun!;
const os = ({ linux: 'Linux', darwin: 'macOS', win32: 'Windows' } as Record<string, string>)[ctx.platform];
const names = 'error' in check ? [] : Object.entries(check.profiles).filter(([, p]) => p.os === os && ctx.lanes.some(l => l.tier !== 'platform' && l.id.startsWith(`${p.workflow}/`))).map(([n]) => n);
const names = 'error' in check ? [] : fromRunProfiles(ctx, check.profiles);
const v = 'error' in check ? { ok: false as const, reason: 'profiles_unavailable', detail: check.error } : verifyFromRun(check, ref, names, ctx.wtree, ctx.head);
if (!v.ok) return { line: `REUSE: refused reason=${v.reason} — ${v.detail}; every selected test runs here`, passed: [] };
const ran = new Set(v.evidence.files);
+12 -4
View File
@@ -46,6 +46,9 @@
* tree, the command listing the gate's test files they must
* cover, and the command naming the selected files the gate
* itself never runs (gstack: paid files)
* ship_lanes_from exact /ship lane command → `{ profiles, inventory }`: the
* run profiles whose verified runs at HEAD's tree carry that
* lane, and the command listing its test files (lib/ci-carry.ts)
* allowlist `{ finding, paths?, pattern?, reason }` entries the `secrets`
* check accepts as benign: a gstack-redact finding id or a
* non-secret category, scoped by path globs and/or a regex
@@ -67,6 +70,7 @@ import * as fs from 'node:fs';
import * as path from 'node:path';
import { spawnSync } from 'node:child_process';
import { createHash } from 'node:crypto';
import { shipLanesErrors, type ShipLaneMapping } from '../ci-carry';
import type { ResultCodeName } from '../result-codes';
import { runProfileErrors, type RemoteGateRuns, type RunProfile } from '../run-evidence';
import { sealedErrors } from '../sealed-guard';
@@ -102,15 +106,16 @@ export interface PregateConfig {
allowlist: AllowlistEntry[];
run_profiles: Record<string, RunProfile>;
remote_gate_runs: RemoteGateRuns | null;
ship_lanes_from: Record<string, ShipLaneMapping>;
}
export interface Guard { id: string; when: string[]; command: string; fix?: string; timeout_s?: number }
export interface AllowlistEntry { finding: string; paths?: string[]; pattern?: string; reason: string }
export const DEFAULT_CONFIG: PregateConfig = { runner: null, dependencies: {}, empty_lanes: [], selection: {}, warn: [], strays: { allow: [] }, local: [], guards: [], serial_patterns: DEFAULT_SERIAL_PATTERNS, price_file: null, stall_timeout_s: null, timeout_s: null, remote_gate: null, allowlist: [], run_profiles: {}, remote_gate_runs: null };
export const DEFAULT_CONFIG: PregateConfig = { runner: null, dependencies: {}, empty_lanes: [], selection: {}, warn: [], strays: { allow: [] }, local: [], guards: [], serial_patterns: DEFAULT_SERIAL_PATTERNS, price_file: null, stall_timeout_s: null, timeout_s: null, remote_gate: null, allowlist: [], run_profiles: {}, remote_gate_runs: null, ship_lanes_from: {} };
export interface ConfigError { code: ResultCodeName; path: string; message: string }
export const MAX_LIST = 256;
const MAX_COMMAND = 512;
const KEYS = ['runner', 'dependencies', 'empty_lanes', 'selection', 'warn', 'strays', 'local', 'guards', 'serial_patterns', 'price_file', 'sealed', 'stall_timeout_s', 'timeout_s', 'remote_gate', 'allowlist', 'run_profiles', 'remote_gate_runs'];
const KEYS = ['runner', 'dependencies', 'empty_lanes', 'selection', 'warn', 'strays', 'local', 'guards', 'serial_patterns', 'price_file', 'sealed', 'stall_timeout_s', 'timeout_s', 'remote_gate', 'allowlist', 'run_profiles', 'remote_gate_runs', 'ship_lanes_from'];
const LANE_ID = /^[A-Za-z0-9_.-]+\.ya?ml\/[A-Za-z0-9_-]+$/;
const MAX_GUARDS = 32;
const GUARD_KEYS = ['id', 'when', 'command', 'fix', 'timeout_s'];
@@ -202,6 +207,8 @@ export function validateConfig(raw: unknown, knownChecks: readonly string[] = []
if (why) errors.push({ code: 'PREGATE_CONFIG_INVALID', path: 'remote_gate', message: why });
}
for (const e of runProfileErrors(obj.run_profiles, obj.remote_gate_runs)) errors.push({ code: 'PREGATE_CONFIG_INVALID', path: e.path, message: e.message });
const profileNames = obj.run_profiles && typeof obj.run_profiles === 'object' ? Object.keys(obj.run_profiles) : [];
for (const e of shipLanesErrors(obj.ship_lanes_from, profileNames)) errors.push({ code: 'PREGATE_CONFIG_INVALID', path: e.path, message: e.message });
if ('guards' in obj) guardErrors(obj.guards, errors);
if ('allowlist' in obj) allowlistErrors(obj.allowlist, errors);
if ('sealed' in obj) for (const e of sealedErrors(obj.sealed)) errors.push({ code: 'PREGATE_CONFIG_INVALID', path: e.path, message: e.message });
@@ -273,7 +280,7 @@ export function parseConfig(text: string, knownChecks: readonly string[] = []):
const errors = validateConfig(raw, knownChecks);
if (errors.length) return { config: DEFAULT_CONFIG, errors, declaresLanes: false };
const o = raw as Partial<PregateConfig>;
return { declaresLanes: LANE_KEYS.some(k => k in o), config: { $schema: o.$schema, runner: o.runner ?? null, dependencies: o.dependencies ?? {}, empty_lanes: o.empty_lanes ?? [], selection: o.selection ?? {}, warn: o.warn ?? [], strays: { allow: o.strays?.allow ?? [] }, local: o.local ?? [], guards: o.guards ?? [], serial_patterns: o.serial_patterns ?? DEFAULT_SERIAL_PATTERNS, price_file: o.price_file ?? null, stall_timeout_s: o.stall_timeout_s ?? null, timeout_s: o.timeout_s ?? null, remote_gate: o.remote_gate ?? null, allowlist: o.allowlist ?? [], run_profiles: o.run_profiles ?? {}, remote_gate_runs: o.remote_gate_runs ?? null }, errors: [] };
return { declaresLanes: LANE_KEYS.some(k => k in o), config: { $schema: o.$schema, runner: o.runner ?? null, dependencies: o.dependencies ?? {}, empty_lanes: o.empty_lanes ?? [], selection: o.selection ?? {}, warn: o.warn ?? [], strays: { allow: o.strays?.allow ?? [] }, local: o.local ?? [], guards: o.guards ?? [], serial_patterns: o.serial_patterns ?? DEFAULT_SERIAL_PATTERNS, price_file: o.price_file ?? null, stall_timeout_s: o.stall_timeout_s ?? null, timeout_s: o.timeout_s ?? null, remote_gate: o.remote_gate ?? null, allowlist: o.allowlist ?? [], run_profiles: o.run_profiles ?? {}, remote_gate_runs: o.remote_gate_runs ?? null, ship_lanes_from: o.ship_lanes_from ?? {} }, errors: [] };
}
export type ConfigLoad = { config: PregateConfig; errors: ConfigError[]; present: boolean; label: string; kind: 'base' | 'head' | 'user' | 'none'; /** sha256 of the user file's bytes (kind 'user' only). */ digest?: string; /** The policy has a LANE_KEYS key; absent under kind 'none'. */ declaresLanes?: boolean } | { unavailable: true; base: string; detail: string };
@@ -329,7 +336,7 @@ export function loadConfigFromUser(file: string, knownChecks: readonly string[]
// `sealed` (the pre-push guard), `price_file` (gstack-eval-plan) and the run profiles (`clear --run`) read only the git copies, so a non-null value in the user file would look active and do nothing.
let raw: unknown = null;
try { raw = JSON.parse(text); } catch { /* parseConfig reported it */ }
const ignored = raw && typeof raw === 'object' && !Array.isArray(raw) ? ['sealed', 'price_file', 'run_profiles', 'remote_gate_runs'].filter(k => { const v = (raw as Record<string, unknown>)[k]; return v != null && !(k === 'run_profiles' && typeof v === 'object' && Object.keys(v).length === 0); }) : [];
const ignored = raw && typeof raw === 'object' && !Array.isArray(raw) ? ['sealed', 'price_file', 'run_profiles', 'remote_gate_runs', 'ship_lanes_from'].filter(k => { const v = (raw as Record<string, unknown>)[k]; return v != null && !(k === 'run_profiles' && typeof v === 'object' && Object.keys(v).length === 0); }) : [];
// parseConfig names the repo copy for a JSON error; this file is not it.
const errors = [...load.errors.map(e => e.path === PREGATE_CONFIG_PATH ? { ...e, path: file } : e), ...ignored.map(k => ({ code: 'PREGATE_CONFIG_INVALID' as const, path: k, message: `the user-owned policy covers gstack-pregate run only; ${k} is read from ${PREGATE_CONFIG_PATH} on the base branch` }))];
return { ...load, errors, present: true, label: `user ${file}`, kind: 'user', digest: createHash('sha256').update(text).digest('hex') };
@@ -358,6 +365,7 @@ export function explainConfig(): string {
'remote_gate: the repo\'s remote test gate command; under 8 cores the test tier becomes a requires-remote obligation naming it, and gstack-pregate --remote runs it (it must print PREGATE_REMOTE: tree=<tree> tests=<n> failed=<n>) (docs/pregate.md#remote-gate)',
'run_profiles: name → { workflow, events, jobs, artifact, max_age_hours?, os? }; the GitHub Actions runs gstack-pregate clear --run verifies as evidence on the identical tree (docs/pregate.md#run-evidence)',
'remote_gate_runs: { profiles, inventory, skipped? }; the run profiles that together clear the remote gate obligation, the command listing the gate\'s test files they must cover, and an optional command printing which selected files (PREGATE_FILES_FILE) the gate itself does not run, which then need no run (docs/pregate.md#run-evidence)',
'ship_lanes_from: exact /ship lane command → { profiles, inventory }; a verified run of each profile at HEAD\'s clean tree whose tree.json covers the inventory carries the lane instead of running it again (docs/pregate.md#ci-carry)',
`allowlist: [{ finding, paths?, pattern?, reason }]: benign findings the secrets check accepts and names as allowlisted. finding is a gstack-redact id (env.kv, google.api_key, pii.phone.e164, …) or a category (${ALLOWLISTABLE_CATEGORIES.join(', ')}); paths are globs; pattern is a regex matched at the finding's position in the added line; one of the two is required, reason always. Never ${NON_ALLOWLISTABLE.join(' or ')} or engine.* (docs/pregate.md#allowlist)`,
'no .gstack/pregate.json on the base: <state root>/projects/<slug>/pregate.json is read instead (state root: GSTACK_STATE_ROOT, else GSTACK_HOME, else GSTACK_STATE_DIR, else a gstack plugin data dir, else ~/.gstack, per docs/state-root.md; run --explain prints the resolved path; outside the repo, so a PR cannot reach it; a path inside the checkout is ignored)',
].join('\n') + '\n';
+57 -1
View File
@@ -10,6 +10,7 @@
* land: `guards` (field wave CD1); ratchets, lint, links, hermetic, patches
* and integrator later.
*/
import { spawnSync } from 'node:child_process';
import { createHash } from 'node:crypto';
import * as fs from 'node:fs';
import * as path from 'node:path';
@@ -17,7 +18,8 @@ import { ARTIFACT_SCHEMAS, checkSchema, type PregateFile, type PregateReceiptSub
import { wtreeFingerprint, type WtreeResult } from '../script-invocation';
import { resolveStateRoot } from '../state-root';
import { guardsCheck } from './checks/guards';
import { clearHint, lanesCheck, parseRemoteReport, REMOTE_GATE_LANE, remoteReportProblem, treeMismatch } from './checks/lanes';
import { findCarry } from '../ci-carry';
import { clearHint, fromRunProfiles, lanesCheck, parseRemoteReport, REMOTE_GATE_LANE, remoteReportProblem, treeMismatch } from './checks/lanes';
import { literalsCheck } from './checks/literals';
import { regenCheck } from './checks/regen';
import { secretsCheck } from './checks/secrets';
@@ -26,6 +28,7 @@ import { timingCheck } from './checks/timing';
import { NON_DOWNGRADABLE, PREGATE_CONFIG_PATH, userPolicyAbsent } from './config';
import { sha256, withFingerprint } from './context';
import { baseBranchOf, baseMismatchFix, describeSubstitution, substituteStampOnlyTree } from './remote-receipt';
import { buildRunCheck } from './run-check';
import { resolveBaseBranch } from '../ship-policy';
import { parseRunReceipt, parseRunRef, runChanged, runReceiptFields, uncoveredFiles, verifyRun, type ApiJob, type ApiRun, type RemoteGateRuns, type RunDeps, type RunEvidence, type RunProfile } from '../run-evidence';
import type { CheckDef, CheckResult, PregateContext, Stage } from './types';
@@ -47,8 +50,40 @@ export function applyDowngrades(ctx: PregateContext, result: CheckResult, def: C
return { ...result, status: 'warn', detail: `${result.detail} (downgraded to warn by ${ctx.configLabel})` };
}
/**
* The tests stage's automatic `--from-run` (GSTA-258): when the base branch declares run profiles and no run was named,
* the newest verified CI run at HEAD's clean tree under a profile for this OS becomes `ctx.fromRun`, so its tree.json
* files count as passed and only the rest run (on a small machine, an all-covered selection records no remote gate
* obligation). Returns the CI_CARRY line, or null when the base declares no profiles. Security scans, guards and the
* preflight stage are never carried.
*/
export function autoCarry(ctx: PregateContext): string | null {
if (ctx.fromRun || ctx.ciCarry === false || !Object.keys(ctx.config.run_profiles).length) return null;
const none = (why: string) => `CI_CARRY: none (${why.length > 300 ? `${why.slice(0, 300)}…` : why})`;
const tree = git(ctx.repoRoot, ['rev-parse', 'HEAD^{tree}']);
if (!tree || ctx.wtree !== tree) return none(`dirty_tree: the working tree ${ctx.wtree.slice(0, 12)} is not HEAD's committed tree; only a clean, committed tree carries`);
const check = buildRunCheck(ctx.repoRoot, null, ctx.allowRepoCommands, ctx.base);
if ('error' in check) return none(`profiles_unavailable: ${check.error}`);
const names = fromRunProfiles(ctx, check.profiles);
if (!names.length) return none('no_profile: no base run profile for this OS covers a lane this machine runs');
const wtree = () => { const fp = wtreeFingerprint({ cwd: ctx.repoRoot }); return 'tree' in fp ? fp.tree : null; };
const refusals: string[] = [];
for (const name of names) {
const v = findCarry(check, { profiles: [name], head: ctx.head, tree, wtree });
if (v.ok) { ctx.fromRun = { ref: String(v.runs[0]!.run_id), check }; return `CI_CARRY: accepted ${v.receipt}`; }
refusals.push(`${v.reason}: ${v.detail}`);
}
return none(refusals.join(' | '));
}
const git = (cwd: string, args: string[]): string | null => {
const r = spawnSync('git', args, { cwd, encoding: 'utf8', timeout: 30_000 });
return r.status === 0 ? r.stdout.trim() || null : null;
};
export async function runPregate(ctx: PregateContext, o: RunOptions): Promise<PregateFile> {
const selected = CHECKS.filter(c => (o.stage === 'all' || c.stage === o.stage) && (!o.only || o.only.includes(c.id)));
if (selected.some(c => c.id === 'lanes')) { const line = autoCarry(ctx); if (line) ctx.progress(line); }
const checks: CheckResult[] = [];
const clocks: Record<Stage, number> = { preflight: 0, tests: 0 };
for (const def of selected) {
@@ -68,9 +103,30 @@ export async function runPregate(ctx: PregateContext, o: RunOptions): Promise<Pr
requires_remote: remote,
};
file.verdict = verdictOf(file);
const gateLine = carryGate(ctx, file);
if (gateLine) ctx.progress(gateLine);
return file;
}
/**
* After a verified run carried part of the tests stage, that run clears the remote gate obligation under `clear --run`'s
* own checks (remote_gate_runs profiles, inventory coverage, skipped files). The obligation is checked as the carried
* files plus the rest: without that, a remainder of only gate-skipped files (gstack's paid evals) is refused paid_only,
* and the gate itself reports tests=0 for it, so nothing could ever clear it.
*/
function carryGate(ctx: PregateContext, file: PregateFile): string | null {
const gate = file.requires_remote?.find(r => r.lane === REMOTE_GATE_LANE && !r.receipt);
const carried = file.checks.find(c => c.id === 'lanes')?.from_run?.files;
if (!gate || !carried?.length || !ctx.fromRun || !('inventory' in ctx.fromRun.check)) return null;
const check = ctx.fromRun.check as RunCheck;
if (!check.gateRuns) return null;
const view: PregateFile = { ...file, requires_remote: file.requires_remote!.map(r => (r === gate ? { ...r, tests: [...new Set([...carried, ...r.tests])] } : r)) };
const d = clearByRuns(view, REMOTE_GATE_LANE, [ctx.fromRun.ref], check, ctx.repoRoot);
if (!('receipt' in d)) return `CI_CARRY: ${REMOTE_GATE_LANE} stays uncleared (${d.message})`;
clearObligation(file, REMOTE_GATE_LANE, d.receipt, 'ci-carry', d.substitution);
return `CI_CARRY: cleared ${REMOTE_GATE_LANE} ${d.receipt}`;
}
/** `pass`, `warn`, `requires-remote`, `incomplete` or `fail`: the worst status across checks, with uncleared obligations counted. */
export function verdictOf(file: PregateFile): string {
const statuses = new Set(file.checks.map(c => c.status));
+2
View File
@@ -64,6 +64,8 @@ export interface PregateContext {
policyFrom: 'base' | 'head';
/** `--from-run`: a Free Tests run whose tree.json files count as passed for this tree (lanes, lib/run-evidence.ts verifyFromRun), with the base branch's run check or why it could not be built. */
fromRun?: { ref: string; check: RunSource | { error: string } };
/** `--no-ci-carry`: false skips the tests stage's automatic --from-run (lib/ci-carry.ts findCarry at HEAD's clean tree). */
ciCarry?: boolean;
/** `--remote`: run the declared remote gate now, under this spend cap. */
remote: { spendCapUsd: number } | null;
env: NodeJS.ProcessEnv;
+15 -1
View File
@@ -661,13 +661,27 @@ const SCHEMA_FIELD_VALUE = /^[a-z][a-z0-9]*(?:[_.-][a-z][a-z0-9]*)+$/;
function isSchemaFieldIdentifier(span: string, match: RegExpExecArray): boolean {
return SCHEMA_FIELD_NAME.test(match[0]) && SCHEMA_FIELD_VALUE.test(span);
}
/**
* GST-1064: the first argument of a lookup call (`get`, `getenv`, `read`,
* `load`, `loads`, `open`, `pop`, `fetch`) that is a lowercase file name
* (`files.get("key.json")`) or a snake_case key of two or three words
* (`resp.json().get("access_token")`) names what the call reads. Eight distinct
* letters already reach 3.0 bits, so entropy alone cannot tell it from a
* secret. Any other callee (`make_password("orange_monkey_pass")`), a grouping
* paren, a later argument or a fallback (`os.getenv("X", "orange_monkey_pass")`)
* is a value and is still checked, and so is a first argument with a digit, an
* uppercase letter, kebab words or four or more snake_case words.
*/
const LOOKUP_ARGUMENT =
/(?<![\w$])(get|getenv|read|load|loads|open|pop|fetch)\([ \t]*(["'])(?:[a-z]+(?:_[a-z]+){1,2}|[a-z]+(?:[_-][a-z]+)*\.(?:json|ya?ml|toml|ini|cfg|conf|env|pem|crt|cer|key|txt|csv|xml|cookie))\2/g;
function isCallExpression(span: string, match: RegExpExecArray): boolean {
if (!CALL_SHAPED_VALUE.test(span)) return false;
const { start } = spanBounds(match);
if (match.input[start - 1] === '"' || match.input[start - 1] === "'") return false;
const call = match.input.slice(start + span.indexOf("(")).split("\n", 1)[0];
if (!call.includes(")") && call.trim() !== "(") return false;
return !carriesSecretLiteral(call);
const line = match.input.slice(start).split("\n", 1)[0];
return !carriesSecretLiteral(line.replace(LOOKUP_ARGUMENT, "$1("));
}
/**
+5 -2
View File
@@ -55,9 +55,11 @@ export const SECURITY_LINE = [
/\b(?:includes|startsWith|endsWith|indexOf)\s*\(\s*['"`]\.\.[/\\]{0,2}['"`]|[!=]==?\s*['"`]\.\.['"`]|['"`]\.\.['"`]\s*[!=]==?|\\\.\\\./,
/(?<![.\w])eval\s*\(/, /(?:^|[;&|])\s*eval\s+["'$]/, /\bnew\s+Function\s*\(/,
/\b(?:spawn|spawnSync|execFile|execFileSync)\s*\(\s*['"](?:\/bin\/)?(?:ba|z)?sh['"]/, /\bBun\.spawn(?:Sync)?\s*\(\s*\[\s*['"](?:\/bin\/)?(?:ba|z)?sh['"]/,
/\brmSync\s*\((?:[^)]{0,500}recursive\s*:\s*true|[^)]{501})/, /\brm\s+-(?=[a-z]*r)(?=[a-z]*f)[a-z]*[rf]\b/,
/\brmSync\s*\((?:.{0,500}?recursive\s*:\s*true|[^)]{501})/, /\brm\s*\((?:.{0,500}?recursive\s*:\s*true|[^)]{501})/, /\brm\s+-(?=[a-z]*r)(?=[a-z]*f)[a-z]*[rf]\b/,
/\b(?:innerHTML|outerHTML)\s*\+?=/, /\bdangerouslySetInnerHTML\b/, /\bv-html\b/, /\{@html\b/, /\binsertAdjacentHTML\s*\(/, /\bdocument\.write\s*\(/,
];
/** A recursive delete split across lines (a formatter puts `recursive: true` on its own line), read over a file's changed lines joined. */
export const SECURITY_JOINED = [/\brm(?:Sync)?\s*\((?:[\s\S]{0,500}?recursive\s*:\s*true|[^)]{501})/];
export const SVG_PATH = /\.svg$/i;
/** SVG text that can run code, read decoded and again with whitespace removed (java&#9;script: is javascript:); a file's changed lines are read joined, so a split attribute still counts. */
export const SVG_ACTIVE = [/<(?:[\w.-]+:)?(?:script|foreignObject)\b/i, /(?:^|[\s"'/])on[a-z]+\s*=/i, /(?:java|vb)script:|data:text\/html/i];
@@ -81,7 +83,7 @@ const namesMoney = (t: readonly string[], words: readonly string[]) => t.some(x
export const RISK_RULES = createHash('sha256').update(JSON.stringify({
RISK_CLASSES, MONEY_PATH_TOKENS, AUTH_PATH_TOKENS, AUTH_PATH_PREFIXES, KEY_NAME, SECURITY_PATH_TOKENS, DATA_PATH_TOKENS,
MONEY_LINE_TOKENS, MONEY_PAIRS,
patterns: Object.entries({ MONEY_LINE, SECRET_LINE, SECRET_PATH: [SECRET_PATH], SECURITY_LINE, DATA_LINE, SVG_PATH: [SVG_PATH], SVG_ACTIVE }).map(([k, v]) => [k, v.map(r => '/' + r.source + '/' + r.flags)]),
patterns: Object.entries({ MONEY_LINE, SECRET_LINE, SECRET_PATH: [SECRET_PATH], SECURITY_LINE, SECURITY_JOINED, DATA_LINE, SVG_PATH: [SVG_PATH], SVG_ACTIVE }).map(([k, v]) => [k, v.map(r => '/' + r.source + '/' + r.flags)]),
})).digest('hex').slice(0, 16);
export const tokens = (p: string) => p.replace(/([a-z0-9])([A-Z])/g, '$1 $2').toLowerCase().split(/[^a-z0-9]+/).filter(Boolean);
@@ -113,6 +115,7 @@ export function matchRisk(i: RiskInput): Map<RiskClass, string> {
if (!why.has('security') && SECURITY_LINE.some(r => r.test(line))) hit('security', 'security line: ' + p);
if (!why.has('data') && DATA_LINE.some(r => r.test(line))) hit('data', 'data line: ' + p);
}
if (!why.has('security') && [i.added?.[p], i.removed?.[p]].some(l => l && SECURITY_JOINED.some(r => r.test(l.join('\n'))))) hit('security', 'security lines: ' + p);
}
return new Map(RISK_CLASSES.filter(c => why.has(c)).map(c => [c, why.get(c)!]));
}
+46 -9
View File
@@ -10,6 +10,7 @@ import * as fs from 'node:fs';
import * as os from 'node:os';
import * as path from 'node:path';
import { fragmentSlug } from './changelog-fragments';
import { isConfigUnavailable, loadConfigFromBase } from './pregate/config';
import { parseUnifiedDiff, versionAtRef } from './restamp';
import { branchSides, stampOnlyLine } from './stamp-only';
import { matchesAny } from './ship-policy';
@@ -170,26 +171,62 @@ export function laneConcurrency(mode: string | undefined, cpus = os.availablePar
return cpus <= 4 ? 1 : Infinity;
}
/** Run each lane through gstack-evidence run, at most `concurrency` at a time, and read its recorded line; results keep the lanes' order. */
export async function runLanes(cwd: string, list: { label: string; command: string }[], concurrency = Infinity): Promise<{ label: string; exit: number | null; testsRan: string; log: string }[]> {
const out: { label: string; exit: number | null; testsRan: string; log: string }[] = [];
export interface LaneResult { label: string; exit: number | null; testsRan: string; log: string; carried?: string; reused?: string; carryNone?: string }
/** The base branch's `ship_lanes_from` entry for each lane whose command is a key, byte for byte (the ledger binds its sha256). */
export function mappedLanes(cwd: string, base: string, list: { label: string; command: string }[]): Map<string, { profiles: string[]; inventory: string }> {
const loaded = loadConfigFromBase(cwd, base);
const map = new Map<string, { profiles: string[]; inventory: string }>();
if (isConfigUnavailable(loaded) || loaded.errors.length) return map;
const from = loaded.config.ship_lanes_from;
for (const l of list) if (Object.prototype.hasOwnProperty.call(from, l.command)) map.set(l.label, from[l.command]!);
return map;
}
/**
* A mapped lane carried from CI at HEAD's clean tree: gstack-evidence record-ci finds and verifies the run (lib/ci-carry.ts)
* and records the CI-sourced ledger row; or why it was not (`carryNone`), so the caller runs it.
*/
export function carryLane(cwd: string, base: string, l: { label: string; command: string }): LaneResult {
const rec = bin(cwd, 'gstack-evidence', ['record-ci', '--label', l.label, '--base', base, '--', l.command], 600_000);
const got = / tests_ran=(\d+) log=- source=ci runs=(\S+)/.exec(rec.stderr);
if (rec.status === 0 && got) return { label: l.label, exit: 0, testsRan: got[1]!, log: '-', carried: got[2]! };
return { label: l.label, exit: null, testsRan: 'unknown', log: '-', carryNone: /^CARRY_NONE: (.+)$/m.exec(rec.stderr)?.[1] ?? `record_failed: ${firstLine(rec.stderr)}` };
}
/**
* Run each lane, at most `concurrency` at a time; results keep the lanes' order. With `base`, a lane the base branch maps in
* `ship_lanes_from` is carried from a verified CI run at this exact tree when one exists. Every other lane runs through
* gstack-evidence run --reuse (a pass at this tree, command and runtime is reused; `reuse: false` forces a run).
*/
export async function runLanes(cwd: string, list: { label: string; command: string }[], concurrency = Infinity, opts: { base?: string; reuse?: boolean } = {}): Promise<LaneResult[]> {
const out: LaneResult[] = [];
const mapped = opts.base ? mappedLanes(cwd, opts.base, list) : new Map<string, { profiles: string[]; inventory: string }>();
let next = 0;
const worker = async () => {
while (next < list.length) {
const i = next++;
const l = list[i]!;
const r = await binAsync(cwd, 'gstack-evidence', ['run', '--label', l.label, '--', l.command]);
const m = /gstack-evidence: recorded label=\S+ exit=(\S+) tests_ran=(\S+) log=(\S+)/.exec(r.stderr);
out[i] = { label: l.label, exit: r.status, testsRan: m?.[2] ?? 'unknown', log: m?.[3] ?? '-' };
const m = mapped.get(l.label);
const carry = m ? carryLane(cwd, opts.base!, l) : null;
if (carry?.carried) { out[i] = carry; continue; }
const r = await binAsync(cwd, 'gstack-evidence', ['run', ...(opts.reuse === false ? [] : ['--reuse']), '--label', l.label, '--', l.command]);
const rec = /gstack-evidence: recorded label=\S+ exit=(\S+) tests_ran=(\S+) log=(\S+)/.exec(r.stderr);
const reused = /gstack-evidence: reused label=\S+ tree=\S+ from=(\S+)/.exec(r.stderr)?.[1];
out[i] = { label: l.label, exit: r.status, testsRan: rec?.[2] ?? 'unknown', log: rec?.[3] ?? '-', ...(reused ? { reused } : {}), ...(carry?.carryNone ? { carryNone: carry.carryNone } : {}) };
}
};
await Promise.all(Array.from({ length: Math.max(1, Math.min(concurrency, list.length)) }, worker));
return out;
}
/** Step 5's lane verdict: a lane passes on its runner count, never exit 0 alone. */
export function laneLine(l: { label: string; exit: number | null; testsRan: string; log: string }): { line: string; ok: boolean } {
/** Step 5's lane verdict: a lane passes on its runner count, never exit 0 alone; a carried lane names its CI runs. */
export function laneLine(l: LaneResult): { line: string; ok: boolean } {
if (l.carried) return { line: `LANE: ${l.label} carried run=${l.carried} tests_ran=${l.testsRan}`, ok: true };
const zero = l.exit === 0 && l.testsRan === '0';
const state = l.exit !== 0 ? 'FAIL' : zero ? 'ZERO-RUN' : l.testsRan === 'unknown' ? 'pass (count unavailable)' : 'pass';
return { line: `LANE: ${l.label} ${state} exit=${l.exit} tests_ran=${l.testsRan} log=${l.log}`, ok: l.exit === 0 && !zero };
return { line: `LANE: ${l.label} ${state} exit=${l.exit} tests_ran=${l.testsRan} log=${l.log}${l.reused ? ` reused=${l.reused}` : ''}`, ok: l.exit === 0 && !zero };
}
/** The CI_CARRY line for a mapped lane that ran here instead. */
export const carryNoneLine = (l: LaneResult) => (l.carryNone ? `CI_CARRY: none label=${l.label} (${l.carryNone.length > 220 ? `${l.carryNone.slice(0, 220)}…` : l.carryNone})` : null);
+8 -5
View File
@@ -10,7 +10,7 @@ import * as os from 'node:os';
import * as path from 'node:path';
import { classifyDiff } from './review-depth';
import { audit } from './ship-audit';
import { bin, firstLine, flagReader, gh, git, hostedOrigin, laneConcurrency, laneLine, lanes, Report, runLanes, shipPolicy, short } from './ship-bin-common';
import { bin, carryNoneLine, firstLine, flagReader, gh, git, hostedOrigin, laneConcurrency, laneLine, lanes, Report, runLanes, shipPolicy, short } from './ship-bin-common';
export const PREFLIGHT_HELP = `gstack-ship-preflight — /ship Steps 1-3 and Step 5's pregate and lanes in one call (ship/SKILL.md)
@@ -23,8 +23,10 @@ export const PREFLIGHT_HELP = `gstack-ship-preflight — /ship Steps 1-3 and Ste
(CONTEXT, TODOS, COVERAGE_MAP, Coverage, PLAN_*, READER: coverage=needed|none plan=needed|none)
Prints at most 30 lines: BRANCH, STATUS, DIFF, PR/CARRY, APPLE, SCOPE, MERGE, DISTRIBUTION, POLICY,
RUNTIME_LINES, PREGATE (with up to three PREGATE_DETAIL lines when it does not pass), LANES, one LANE line per lane,
then NEXT: lines. STOP: names what the agent must
RUNTIME_LINES, PREGATE (with up to three PREGATE_DETAIL lines when it does not pass), LANES, one LANE line per lane
(a lane the base maps in .gstack/pregate.json ship_lanes_from is carried from a verified CI run at this exact clean tree:
LANE: <label> carried run=<id>/<attempt> tests_ran=<n>, else CI_CARRY: none (<reason>) and it runs; any other lane that
passed at this tree, command and runtime is reused: reused=<original time>), then NEXT: lines. STOP: names what the agent must
decide or repair; nothing after it ran. Exit codes: 0 ok · 2 usage · 3 STOP`;
const DIST = /(^|\/)(cmd\/[^/]+\/main\.go|bin\/[^/]+|Cargo\.toml|setup\.py|package\.json)$/;
@@ -151,8 +153,9 @@ async function tests(cwd: string, base: string, list: { label: string; command:
}
if (/uncleared/.test(verdict)) r.next('pregate requires-remote: gstack-pregate clear, or --remote --spend-cap <usd> under a user cap, before Step 18');
r.add(run.line);
const results = (await runLanes(cwd, list, run.concurrency)).map(laneLine);
for (const l of results) r.add(l.line);
const ran = await runLanes(cwd, list, run.concurrency, { base });
const results = ran.map(laneLine);
for (const [i, l] of results.entries()) { const none = carryNoneLine(ran[i]!); if (none) r.add(none); r.add(l.line); }
const bad = results.filter(l => !l.ok);
if (bad.length) { r.stop(`${bad.length} lane(s) failed or ran zero tests: Step 5 triage (read each log; a free-suite shard failure reruns its file list per measure.md); in-branch failures stop the ship`); return; }
r.next('Step 6: select eval suites for prompt-related changes with the project\'s selector, then Step 7');
+2 -1
View File
@@ -16,7 +16,7 @@ import { stampOnlyLine, type BranchSide } from './stamp-only';
export const RECEIPT_FENCE = 'gstack-ship-receipt';
const KEY_ORDER: Array<keyof ShipReceipt | 'docs'> = [
'schema_version', 'pr', 'head', 'version', 'base', 'gated_tree', 'predecessor', 'tree', 'gate_reuse', 'gate', 'ci', 'spend_usd',
'schema_version', 'pr', 'head', 'version', 'base', 'gated_tree', 'predecessor', 'tree', 'gate_reuse', 'gate', 'ci', 'ci_carried', 'spend_usd',
'session_kind', 'artifacts_consumed', 'run', 'thread', 'policy', 'queue_mode', 'preregistration', 'history',
'head_tree', 'product_patch_id', 'gstack_version', 'review_depth', 'pregate', 'requires_remote', 'issues', 'approval', 'docs', 'review_loop',
'review_loop_exception', 'review_targeted', 'review_exclude', 'carry', 'base_red',
@@ -131,6 +131,7 @@ export function receiptSummaryLine(r: ShipReceipt): string {
if (loop) parts.push(`passes=${loop.passes} blocking_found=${loop.blocking_found} deferred=${loop.deferred}`
+ (loop.fix_checks ? ` fix_checks=${loop.fix_checks}` : '') + (loop.rounds !== undefined ? ` rounds=${loop.rounds}` : '') + (loop.dispatches ? ` dispatches=${Object.values(loop.dispatches).reduce((a, c) => a + c, 0)}` : ''));
if (r.carry && typeof r.carry.from === 'string') parts.push(`carried_from=${r.carry.from.slice(0, 12)}`);
if (Array.isArray(r.ci_carried) && r.ci_carried.length) parts.push(`ci_carried=${r.ci_carried.map(c => `${c.lane}:${c.run}/${c.attempt}`).join(',')}`);
if (r.base_red && typeof r.base_red.base === 'string') parts.push(`base_red=${r.base_red.base.slice(0, 12)}`);
return `SHIP_RECEIPT: ${oneLine(parts.join(' '))}`;
}
+91 -5
View File
@@ -19,7 +19,9 @@ import { parseReviewLoopException, type ReviewLoopException } from './ship-recei
import { parseReceiptDocs, type ReceiptDocs } from './ship-receipt-docs';
import { resolveStateRoot } from './state-root';
import { bumpVersion, cmpVersion, fmtVersion, parseVersion, versionWidth, type Bump } from './version-source';
import { bin, evidenceExempt, firstLine, flagReader, fragmentPath, gh, git, hostedOrigin, laneLine, lanes, releasePaths, Report, runLanes, sh, shipPolicy, short, stampedFiles } from './ship-bin-common';
import { waitForRun } from './ci-carry';
import { buildRunCheck } from './pregate/run-check';
import { bin, carryLane, carryNoneLine, evidenceExempt, firstLine, flagReader, fragmentPath, gh, git, hostedOrigin, laneLine, lanes, mappedLanes, releasePaths, Report, runLanes, sh, shipPolicy, short, stampedFiles } from './ship-bin-common';
export const RELEASE_HELP = `gstack-ship-release — /ship Steps 12 and 15-17 (ship/SKILL.md)
@@ -33,8 +35,15 @@ export const RELEASE_HELP = `gstack-ship-release — /ship Steps 12 and 15-17 (s
each lane's receipt run once if stale) and the pregate on the stamped tree; --reviewed-base (the merge base
saved with the reviewed tree) routes paths only the base moved as base=, and an adjacent base edit as context=;
--review-loop-exception (the record write takes) routes paths the branch touched at fix_check.head (narrowed to fix_check.files when listed) and changed since as accepted=
push --branch <branch-name> [--gate-open <n>] [--gate-open-approved]
Step 17 after its guard block: the saved-branch check, the open-gate check, the sealed guard, the idempotency check, git push -u
A lane the base branch maps in .gstack/pregate.json ship_lanes_from is carried from a verified CI run at this exact
tree, else deferred (EVIDENCE: DEFERRED): the pregate then runs its preflight stage here and its tests stage in accept
push --branch <branch-name> [--gate-open <n>] [--gate-open-approved] [--early [--base <base>]]
Step 17 after its guard block: the saved-branch check, the open-gate check, the sealed guard, the idempotency check, git push -u;
--early (the Phase 2 push before review, so CI runs beside it) also runs pregate's secrets scan on the commits first
accept --base <base> --lane <label>=<command>... [--wait <s>]
after push: final-tree CI acceptance. Waits up to --wait seconds (default 170) for the CI run of each mapped profile at
the pushed HEAD, carries each mapped lane from that run (a clean tree equal to the run's), reuses or runs the rest,
checks every lane FRESH and reruns the pregate on the final tree (its tests stage carries the same run)
At most 30 lines each; STOP: names what the agent must decide or repair, and nothing after it ran.
Exit codes: 0 ok · 2 usage · 3 STOP`;
@@ -366,10 +375,19 @@ export async function finish(argv: readonly string[], cwd: string): Promise<Repo
if (!list.length && !has('--no-lanes-approved')) return r.stop('no test lanes: require Step 5\'s explicit untested-scope approval (rerun with --no-lanes-approved) or run Steps 5-15');
const allow = evidenceExempt(release).join(',');
const check = (l: { label: string; command: string }) => bin(cwd, 'gstack-evidence', ['check', '--label', l.label, '--expect-cmd', l.command, '--max-age', '24', '--allow-paths', allow], 60_000).stdout;
const mapped = mappedLanes(cwd, base, list);
const deferred: string[] = [];
for (const l of list) {
let e = check(l);
if (!/EVIDENCE: FRESH/.test(e)) {
const ran = laneLine((await runLanes(cwd, [l]))[0]!);
const m = mapped.get(l.label);
const res = m ? carryLane(cwd, base, l) : (await runLanes(cwd, [l], Infinity, { base }))[0]!;
if (m && !res.carried) {
r.add(`EVIDENCE: DEFERRED label=${l.label} (${short([res.carryNone ?? 'not carried'], 1).slice(0, 160)}): CI at the pushed head carries it in accept`);
deferred.push(l.label);
continue;
}
const ran = laneLine(res);
r.add(ran.line);
if (!ran.ok) return r.stop(`lane ${l.label} failed or ran zero tests: STOP publication; run Steps 5-15 from Step 5's triage, then Step 16 stage 1`);
e = check(l);
@@ -378,6 +396,15 @@ export async function finish(argv: readonly string[], cwd: string): Promise<Repo
r.add(line.replace(/ log=\S+/, ''));
if (!/EVIDENCE: FRESH/.test(line)) return r.stop(`lane ${l.label} is not FRESH after one rerun: Step 16 stage 4 recovery`);
}
const laneArgs = list.map(l => `--lane '${l.label}=${l.command.replace(/'/g, `'\\''`)}'`).join(' ');
if (deferred.length) {
const out = path.resolve(cwd, git(cwd, ['rev-parse', '--git-path', 'gstack/pregate-finish-preflight.json']).stdout.trim() || '.git/gstack/pregate-finish-preflight.json');
const pg = bin(cwd, 'gstack-pregate', ['--allow-repo-commands', '--base', base, '--stage', 'preflight', '--out', out], 3_600_000);
const verdict = /^PREGATE: (.+?) tree=/m.exec(pg.stdout + pg.stderr)?.[1] ?? `unreadable (exit ${pg.status})`;
r.add(`PREGATE: preflight ${verdict} on the stamped tree (the tests stage runs in accept, carried from CI)`);
if (!/^(pass|warn)/.test(verdict)) return r.stop(`pregate preflight ${verdict} on the stamped tree: fix it, then rerun finish`);
return r.next(`Step 16 stage 5, then Step 17's guard block and gstack-ship-release push; then gstack-ship-release accept --base ${base} ${laneArgs} carries ${deferred.join(', ')} from CI at the pushed head${loop ? '; Step 18 records the same exception with gstack-ship-receipt write --review-loop-exception' : ''}`);
}
const v = bin(cwd, 'gstack-pregate', ['verify'], 60_000);
if (v.status === 0) r.add('PREGATE: verify ok on the stamped tree');
else {
@@ -389,6 +416,57 @@ export async function finish(argv: readonly string[], cwd: string): Promise<Repo
return r.next(`Step 16 stage 5: paste these results and the two-line honesty inventory, then run Step 17's guard block and gstack-ship-release push${loop ? '; Step 18 records the same exception with gstack-ship-receipt write --review-loop-exception' : ''}`);
}
/**
* Final-tree CI acceptance (design fix A1): after push, each mapped lane is carried only from a green run at the pushed
* HEAD's exact, clean tree; the rest are reused or run; every lane must be FRESH, and the pregate reruns on that tree.
*/
export async function accept(argv: readonly string[], cwd: string): Promise<Report> {
const { flag } = flagReader(argv);
const base = flag('--base')!;
const list = lanes(argv) as { label: string; command: string }[];
const r = new Report();
const branch = git(cwd, ['branch', '--show-current']).stdout.trim();
if (!branch) return r.stop('detached HEAD: accept needs the feature branch');
const head = git(cwd, ['rev-parse', 'HEAD']).stdout.trim();
const remote = git(cwd, ['ls-remote', '--heads', 'origin', `refs/heads/${branch}`], 120_000);
if (remote.status !== 0) return r.stop(`cannot read origin/${branch} (${firstLine(remote.stderr)}): restore access, then rerun accept`);
if (remote.stdout.split(/\s/)[0] !== head) return r.stop(`HEAD ${head.slice(0, 12)} is not origin/${branch}: run gstack-ship-release push first; accept carries only a run at the pushed head`);
const mapped = mappedLanes(cwd, base, list);
if (!mapped.size) return r.stop(`no --lane command is a key of origin/${base}'s .gstack/pregate.json ship_lanes_from: nothing comes from CI, so finish runs every lane here`);
const runCheck = buildRunCheck(cwd, null, true, base);
if ('error' in runCheck) return r.stop(`run profiles unavailable (${runCheck.error}): carry needs them; fix it, or run the lanes here with gstack-ship-preflight --tests-only`);
const waitS = Number(flag('--wait') ?? '170');
if (!Number.isFinite(waitS) || waitS < 0 || waitS > 3600) return r.stop(`--wait ${flag('--wait')} is not 0-3600 seconds`);
for (const name of [...new Set([...mapped.values()].flatMap(m => m.profiles))]) {
const p = runCheck.profiles[name];
if (!p) return r.stop(`ship_lanes_from names ${name}, which is not a run_profiles entry on origin/${base}`);
const w = await waitForRun(runCheck.deps, { workflow: p.workflow, events: p.events, head, timeoutMs: waitS * 1000, intervalMs: 15_000, sleep: ms => new Promise(res => setTimeout(res, ms)), clock: Date.now });
r.add(`CI_WAIT: ${w.state} ${name} ${'run' in w ? `run=${w.run.id}/${w.run.run_attempt}` : w.reason}`);
if (w.state === 'running') return r.stop(`${p.workflow} run ${w.run.id} at ${head.slice(0, 12)} is still running: rerun accept (or gstack-ci-carry wait --base ${base} in the background, then accept)`);
if (w.state === 'red') return r.stop(`${p.workflow} run ${w.run.id} failed (${short(w.failing, 4) || 'no failing job named'}): Step 5 triage on its logs; fix, commit, push, then accept again`);
if (w.state === 'none') return r.stop(`${w.reason}: push the branch (a pull_request or workflow_dispatch run must exist at HEAD), then accept again`);
}
for (const l of list) {
const m = mapped.get(l.label);
const res = m ? carryLane(cwd, base, l) : (await runLanes(cwd, [l], Infinity, { base }))[0]!;
if (m && !res.carried) { r.add(carryNoneLine(res)!); return r.stop(`lane ${l.label} was not carried from the green run (${short([res.carryNone ?? ''], 1).slice(0, 120)}): fix the cause, or run it here with gstack-ship-preflight --tests-only`); }
const ran = laneLine(res);
r.add(ran.line);
if (!ran.ok) return r.stop(`lane ${l.label} failed or ran zero tests: Step 5 triage, then push and accept again`);
const e = bin(cwd, 'gstack-evidence', ['check', '--label', l.label, '--expect-cmd', l.command, '--max-age', '24'], 60_000).stdout;
const line = /^EVIDENCE: .*$/m.exec(e)?.[0] ?? `EVIDENCE: unreadable label=${l.label}`;
r.add(line.replace(/ log=\S+/, ''));
if (!/EVIDENCE: FRESH/.test(line)) return r.stop(`lane ${l.label} is not FRESH at the pushed head: Step 16 stage 4 recovery`);
}
const pg = bin(cwd, 'gstack-pregate', ['--allow-repo-commands', '--base', base], 3_600_000);
const verdict = /^PREGATE: (.+?) tree=/m.exec(pg.stdout + pg.stderr)?.[1] ?? `unreadable (exit ${pg.status})`;
const carried = /^CI_CARRY: .*$/m.exec(pg.stdout + pg.stderr)?.[0];
if (carried) r.add(carried.length > 240 ? `${carried.slice(0, 240)}…` : carried);
r.add(`PREGATE: ${verdict} (final tree)`);
if (!/^(pass|warn|requires-remote cleared)/.test(verdict)) return r.stop(`pregate ${verdict} on the final tree: fix it, then rerun accept`);
return r.next('Step 18: the receipt records ci_carried from the ledger');
}
export function push(argv: readonly string[], cwd: string): Report {
const { flag, has } = flagReader(argv);
const r = new Report();
@@ -411,6 +489,13 @@ export function push(argv: readonly string[], cwd: string): Report {
r.add(`SEALED: ${sealed.status === 0 ? 'clear' : sealed.status === 1 ? 'refused' : `not checked (exit ${sealed.status})`}`);
if (sealed.status === 1) return r.stop(`sealed output would be published: ${firstLine(sealed.stderr)}; remove it from the commits, then repeat Step 16`);
if (sealed.status !== 0) return r.stop(`the sealed-output guard did not run (exit ${sealed.status}: ${firstLine(sealed.stderr || sealed.stdout)}); repair it, then rerun push`);
if (has('--early')) {
const out = path.resolve(cwd, git(cwd, ['rev-parse', '--git-path', 'gstack/pregate-early-push.json']).stdout.trim() || '.git/gstack/pregate-early-push.json');
const scan = bin(cwd, 'gstack-pregate', ['--stage', 'preflight', '--only', 'secrets', '--out', out, ...(flag('--base') ? ['--base', flag('--base')!] : [])], 600_000);
const verdict = /^PREGATE: (.+?) tree=/m.exec(scan.stdout + scan.stderr)?.[1] ?? `unreadable (exit ${scan.status})`;
r.add(`SECRETS: ${verdict} (early push)`);
if (scan.status !== 0 || !/^(pass|warn)/.test(verdict)) return r.stop(`the secrets scan did not pass before the early push (${firstLine(scan.stderr) || verdict}): remove the finding from the commits, then rerun push --early`);
}
const local = git(cwd, ['rev-parse', 'HEAD']).stdout.trim();
const remote = git(cwd, ['ls-remote', '--heads', 'origin', `refs/heads/${branch}`], 120_000);
if (remote.status !== 0) return r.stop(`cannot verify the remote branch (${firstLine(remote.stderr)}): restore access before pushing`);
@@ -431,11 +516,12 @@ export async function release(argv: readonly string[], cwd: string): Promise<{ t
const usage = (m: string) => ({ text: `gstack-ship-release: ${m}\n`, exit: 2 });
if (cmd !== 'push' && !flag('--base')) return usage(`${cmd} needs --base <base>`);
if (cmd === 'push' && !flag('--branch')) return usage('push needs --branch <the branch Step 1 saved>');
if (cmd === 'accept' && !(lanes(rest) as unknown[]).length) return usage('accept needs at least one --lane <label>=<command> (the Step 5 lanes)');
if (typeof lanes(rest) === 'string') return usage(lanes(rest) as string);
const answer = flag('--answer');
if (has('--answer') && (!answer?.trim() || answer.startsWith('--') || /[\r\n]/.test(answer) || answer.length > 200)) return usage('--answer needs the owner\'s words: one non-empty line of at most 200 characters');
if (cmd === 'finish' && (!flag('--reviewed-tree') || !flag('--docs-post'))) return usage('finish needs --reviewed-tree <the tree saved before Step 12> and --docs-post <Step 14.5\'s post-audit snapshot>');
const r = cmd === 'version' ? version(rest, cwd) : cmd === 'finish' ? await finish(rest, cwd) : cmd === 'push' ? push(rest, cwd) : null;
const r = cmd === 'version' ? version(rest, cwd) : cmd === 'finish' ? await finish(rest, cwd) : cmd === 'push' ? push(rest, cwd) : cmd === 'accept' ? await accept(rest, cwd) : null;
if (!r) return usage(`unknown command ${cmd}; run gstack-ship-release --help`);
return { text: r.render(), exit: r.exit };
}