mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-14 20:37:14 +01:00
Merge commit '0c84f4ee14c32734313e68b700417e0748ad1d55' into capy/gst620-sealed-identifiers
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> # Conflicts: # test/helpers/free-fixtures-data.ts
This commit is contained in:
commit
c0d1482b05
568 files changed
+36850
-8414
No files matched your search
@@ -17,6 +17,8 @@ export interface AllowlistFinding {
|
||||
id: string; category: string; severity: string; file: string; line?: number; col?: number;
|
||||
/** The added line the finding sits on (for `pattern`); absent when the scanner gave no position. */
|
||||
lineText?: string;
|
||||
/** `line` numbers the whole file (a Dart or Python file scanned whole, #3630), not the added lines. */
|
||||
fileLine?: boolean;
|
||||
}
|
||||
export interface Allowlisted<F extends AllowlistFinding = AllowlistFinding> { finding: F; index: number; entry: AllowlistEntry }
|
||||
|
||||
@@ -38,26 +40,34 @@ function patternHits(entry: AllowlistEntry, f: AllowlistFinding): boolean {
|
||||
return re.test(f.lineText!.slice(f.col! - 1));
|
||||
}
|
||||
|
||||
function matchesLine(pattern: string, line: string): boolean {
|
||||
try { return new RegExp(pattern).test(line); } catch { return false; }
|
||||
}
|
||||
|
||||
/**
|
||||
* Split findings into the ones an entry covers (with the entry) and the ones
|
||||
* kept; env.value and engine.* are always kept. `untested` lists kept findings
|
||||
* that a pattern entry named by id and path but could not test because the
|
||||
* scanner gave no position, so the table says why the entry was inert.
|
||||
* scanner gave no position, so the table says why the entry was inert;
|
||||
* `missed` lists kept findings that a pattern entry named by id and path
|
||||
* whose pattern matches the finding's line but not at its position (#3630:
|
||||
* the entry was written from the name, not the value).
|
||||
*/
|
||||
export function applyAllowlist<F extends AllowlistFinding>(findings: readonly F[], entries: readonly AllowlistEntry[]): { kept: F[]; allowlisted: Allowlisted<F>[]; untested: Allowlisted<F>[] } {
|
||||
export function applyAllowlist<F extends AllowlistFinding>(findings: readonly F[], entries: readonly AllowlistEntry[]): { kept: F[]; allowlisted: Allowlisted<F>[]; untested: Allowlisted<F>[]; missed: Allowlisted<F>[] } {
|
||||
const kept: F[] = [];
|
||||
const allowlisted: Allowlisted<F>[] = [];
|
||||
const untested: Allowlisted<F>[] = [];
|
||||
const missed: Allowlisted<F>[] = [];
|
||||
for (const f of findings) {
|
||||
const scoped = (e: AllowlistEntry) => names(e, f) && (!e.paths || matchesAny(e.paths, f.file));
|
||||
const index = nonAllowlistable(f.id) ? -1 : entries.findIndex(e => scoped(e) && patternHits(e, f));
|
||||
if (index >= 0) { allowlisted.push({ finding: f, index, entry: entries[index]! }); continue; }
|
||||
kept.push(f);
|
||||
if (nonAllowlistable(f.id) || positioned(f)) continue;
|
||||
const named = entries.findIndex(e => e.pattern !== undefined && scoped(e));
|
||||
if (named >= 0) untested.push({ finding: f, index: named, entry: entries[named]! });
|
||||
if (nonAllowlistable(f.id)) continue;
|
||||
const named = entries.findIndex(e => e.pattern !== undefined && scoped(e) && (!positioned(f) || matchesLine(e.pattern, f.lineText!)));
|
||||
if (named >= 0) (positioned(f) ? missed : untested).push({ finding: f, index: named, entry: entries[named]! });
|
||||
}
|
||||
return { kept, allowlisted, untested };
|
||||
return { kept, allowlisted, untested, missed };
|
||||
}
|
||||
|
||||
/** sha256 of the entries, for the check's `inputs.allowlist`. */
|
||||
@@ -68,8 +78,13 @@ export function describeUntested(a: Allowlisted): string {
|
||||
return `allowlist[${a.index}] ${a.entry.finding} not tested on ${a.finding.id} (${a.finding.file}): the scanner gave no position for its pattern; the finding is kept`;
|
||||
}
|
||||
|
||||
/** One table line per finding a pattern entry named but did not match: the pattern is tested from where the flagged value starts, never the name before it. */
|
||||
export function describeMissed(a: Allowlisted): string {
|
||||
return `allowlist[${a.index}] ${a.entry.finding} pattern did not match ${a.finding.id} (${a.finding.file}${a.finding.line ? `, ${a.finding.fileLine ? 'line' : 'added line'} ${a.finding.line}` : ''}, col ${a.finding.col}): a pattern is matched from where the flagged value starts, not from the name; the finding is kept`;
|
||||
}
|
||||
|
||||
/** One table line per allowlisted finding: what, where, and which entry (with its reason) covered it. */
|
||||
export function describeAllowlisted(a: Allowlisted): string {
|
||||
const where = a.finding.line ? ` (${a.finding.file}, added line ${a.finding.line})` : ` (${a.finding.file})`;
|
||||
const where = a.finding.line ? ` (${a.finding.file}, ${a.finding.fileLine ? 'line' : 'added line'} ${a.finding.line})` : ` (${a.finding.file})`;
|
||||
return `allowlisted ${a.finding.id}${where} by allowlist[${a.index}] ${a.entry.finding}: ${a.entry.reason}`;
|
||||
}
|
||||
+65
-13
@@ -4,10 +4,12 @@
|
||||
* `set -o pipefail` with the exit code and the runner's own summary judged
|
||||
* (zero tests ran is a failure, exit 0 or not); platform lanes this machine
|
||||
* cannot run become persisted `requires-remote` obligations. A touched file
|
||||
* with no lane is `FAIL lanes — <file> has no test lane` once any policy
|
||||
* exists; with no policy anywhere (no base `.gstack/pregate.json`, no
|
||||
* user-owned file) it is the `warn` PREGATE_NO_POLICY with a starter printed
|
||||
* (lib/pregate/starter.ts), never a pass either way.
|
||||
* with no lane is `FAIL lanes — <file> has no test lane` once the policy
|
||||
* declares lanes (a `dependencies`, `empty_lanes`, `selection` or `local` key); with no policy
|
||||
* anywhere, or one that declares neither (a timeout-only file), it is the
|
||||
* `warn` PREGATE_NO_POLICY with a starter printed (lib/pregate/starter.ts),
|
||||
* never a pass either way. `warn: ["lanes"]` downgrades only that condition:
|
||||
* the selected tests still run and a red lane stays `fail` (GST-775).
|
||||
* Markdown no test depends on is the docs/no-tests empty lane instead
|
||||
* (lib/pregate/docs-lane.ts): it does not fail, and unmapped code still does.
|
||||
* A release manifest whose diff against the merge base changes only version
|
||||
@@ -19,6 +21,9 @@
|
||||
* and a deleted test runs its owning suite or names the files that lost their
|
||||
* last test (issue #3529). Any other removal owes what an edit owes.
|
||||
* A test that reads the file still selects that test.
|
||||
* Under `--from-run <run>` the selected local test files a verified Free Tests
|
||||
* run on this exact tree lists count as passed (recorded as `from_run` with the
|
||||
* run receipt) and only the rest run; a refused run runs everything.
|
||||
* Incidents: p0-4-pipe-hidden-failure (`false | tail` read as green),
|
||||
* p0-4-windows-lane-unrun (a Windows-only change merged with its lane unrun).
|
||||
*/
|
||||
@@ -34,6 +39,7 @@ import { selectLanes, type Selection } from '../lane-selection';
|
||||
import { launchRuntime } from '../runtime';
|
||||
import { cargoModel, type CargoModel } from '../rust';
|
||||
import { adapterFor, readRange, supervise, type Counter, type Supervised } from '../supervise';
|
||||
import { verifyFromRun } from '../../run-evidence';
|
||||
import { matchesAny } from '../../ship-policy';
|
||||
import type { CheckDef, CheckResult, PregateContext, RemoteObligation } from '../types';
|
||||
|
||||
@@ -347,8 +353,15 @@ function unfinished(ctx: PregateContext, r: Awaited<ReturnType<typeof runLocal>>
|
||||
return { code: 'PREGATE_INCOMPLETE', detail: `a runner process timed out after ${ctx.timeoutMs} ms (${ctx.timeoutSource ?? '--timeout'} applies to each runner process; ${total} files; last file started: ${timedOut.map(x => x.lastFile ?? `(none reported; ${x.files.length} file(s))`).join(', ')}${rest})`, fix: 'raise --timeout or split the selection; a timed-out lane never passes' };
|
||||
}
|
||||
|
||||
function describe(sel: Selection, noPolicy = false): string[] {
|
||||
const lines = noPolicy ? [`no policy: ${sel.noLane.join(' ')} have no test lane (a warning under no policy; the starter below maps them)`] : sel.noLane.map(f => `FAIL lanes — ${f} has no test lane`);
|
||||
/** How a touched file with no lane is judged: the no-policy warning, the `warn: ["lanes"]` downgrade, or the hard PREGATE_NO_LANE. */
|
||||
type NoLaneMode = 'no-policy' | 'warned' | 'strict';
|
||||
|
||||
function describe(ctx: PregateContext, sel: Selection, mode: NoLaneMode): string[] {
|
||||
const lines = !sel.noLane.length ? []
|
||||
: mode === 'warned' ? sel.noLane.map(f => `WARN lanes — ${f} has no test lane (warn: ["lanes"] in ${ctx.configLabel}; the selected tests are still judged)`)
|
||||
: mode === 'strict' ? sel.noLane.map(f => `FAIL lanes — ${f} has no test lane`)
|
||||
: ctx.configKind === 'none' ? [`no policy: ${sel.noLane.join(' ')} have no test lane (a warning under no policy; the starter below maps them)`]
|
||||
: [`no lane policy: ${sel.noLane.join(' ')} have no test lane (a warning while ${ctx.configLabel} declares no dependencies, empty_lanes, selection or local; the starter below maps them)`];
|
||||
for (const r of sel.removed) lines.push(`removed (no lane owed): ${r.file} — ${r.note}`);
|
||||
if (sel.emptyLane.length) lines.push(`docs/no-tests (empty lane): ${sel.emptyLane.join(' ')}`);
|
||||
if (sel.inert.length) lines.push(`no-tests (assets and tool config): ${sel.inert.join(' ')}`);
|
||||
@@ -500,9 +513,25 @@ async function runSelected(ctx: PregateContext, sel: Selection, base: CheckResul
|
||||
*/
|
||||
function withStarter(ctx: PregateContext, sel: Selection, result: CheckResult): CheckResult {
|
||||
const lines = [...(result.lines ?? []), ...starterPolicy(ctx, sel.noLane, detectRunner(ctx, sel.localTests.length ? sel.localTests : sel.tests))];
|
||||
const noted = `${sel.noLane.length} touched file(s) with no test lane under no policy (starter printed)`;
|
||||
const noted = `${sel.noLane.length} touched file(s) with no test lane under ${ctx.configKind === 'none' ? 'no policy' : 'a policy that declares no lanes'} (starter printed)`;
|
||||
if (result.status !== 'pass') return { ...result, lines, detail: `${result.detail}; ${noted}`, ...(result.status === 'requires-remote' && !result.code ? { code: 'PREGATE_NO_POLICY' } : {}) };
|
||||
return { ...result, status: 'warn', code: 'PREGATE_NO_POLICY', lines, detail: `${result.detail}; ${noted}`, fix: `review the starter above, write it with gstack-pregate init, edit each TODO, then commit ${PREGATE_CONFIG_PATH} on the base branch; from then on an unmapped file is PREGATE_NO_LANE` };
|
||||
const fix = ctx.configKind === 'none'
|
||||
? `review the starter above, write it with gstack-pregate init, edit each TODO, then commit ${PREGATE_CONFIG_PATH} on the base branch; from then on an unmapped file is PREGATE_NO_LANE`
|
||||
: `review the starter above and add its dependencies to ${ctx.configFile}${ctx.configKind === 'user' ? '' : ' on the base branch'}, editing each TODO; from then on an unmapped file is PREGATE_NO_LANE`;
|
||||
return { ...result, status: 'warn', code: 'PREGATE_NO_POLICY', lines, detail: `${result.detail}; ${noted}`, fix };
|
||||
}
|
||||
|
||||
/**
|
||||
* `warn: ["lanes"]` under a policy that declares lanes: the selected tests ran
|
||||
* and their verdict stands (a red lane is still `fail`, an unfinished one
|
||||
* `incomplete`); only a green run becomes `warn` PREGATE_NO_LANE naming the
|
||||
* files with no lane. Before GST-775 the downgraded failure returned before
|
||||
* any test ran, so a red test beside an unmapped file passed.
|
||||
*/
|
||||
function withNoLaneWarning(ctx: PregateContext, sel: Selection, result: CheckResult): CheckResult {
|
||||
const noted = `${sel.noLane.length} touched file(s) with no test lane: ${sel.noLane.join(' ')} (downgraded to warn by ${ctx.configLabel})`;
|
||||
if (result.status !== 'pass') return { ...result, detail: `${result.detail}; ${noted}`, ...(result.status === 'requires-remote' && !result.code ? { code: 'PREGATE_NO_LANE' } : {}) };
|
||||
return { ...result, status: 'warn', code: 'PREGATE_NO_LANE', detail: `${result.detail}; ${noted}`, fix: noLaneFix(ctx, sel.noLane) };
|
||||
}
|
||||
|
||||
async function run(ctx: PregateContext): Promise<CheckResult> {
|
||||
@@ -510,16 +539,39 @@ async function run(ctx: PregateContext): Promise<CheckResult> {
|
||||
ctx.explain.push(...sel.provenance);
|
||||
for (const [lane, set] of Object.entries(sel.laneSets)) ctx.explain.push(`lane ${lane}: ${set === '?' ? 'file set unknown' : `${set.length} files`}`);
|
||||
const inputs = baseInputs(ctx, { selection: sha256(sel.tests.join('\n')), localTests: sha256(sel.localTests.join('\n')), remote: sha256(sel.remote.map(r => `${r.lane}:${r.tests.join(',')}`).join('\n')) });
|
||||
const noPolicy = sel.noLane.length > 0 && ctx.configKind === 'none';
|
||||
const base: CheckResult = { id: 'lanes', stage: 'tests', status: 'pass', inputs, lines: describe(sel, noPolicy), remote: sel.remote };
|
||||
const declared = ctx.configKind !== 'none' && ctx.policyDeclaresLanes !== false;
|
||||
const mode: NoLaneMode = !declared ? 'no-policy' : ctx.config.warn.includes('lanes') ? 'warned' : 'strict';
|
||||
const base: CheckResult = { id: 'lanes', stage: 'tests', status: 'pass', inputs, lines: describe(ctx, sel, mode), remote: sel.remote };
|
||||
const lost = new Set(sel.removed.flatMap(r => r.lostCoverage)).size;
|
||||
const removedNote = sel.removed.length ? `${sel.removed.length} removed path(s)${lost ? `; ${lost} subject(s) lost their last test` : ''}` : '';
|
||||
const withRemoved = (r: CheckResult): CheckResult => (removedNote ? { ...r, detail: r.detail ? `${r.detail}; ${removedNote}` : removedNote } : r);
|
||||
if (sel.noLane.length && !noPolicy) {
|
||||
if (sel.noLane.length && mode === 'strict') {
|
||||
return withRemoved({ ...base, status: 'fail', cause: 'diff', code: 'PREGATE_NO_LANE', detail: `${sel.noLane.length} touched file(s) with no test lane: ${sel.noLane.join(' ')}`, fix: noLaneFix(ctx, sel.noLane) });
|
||||
}
|
||||
const result = withRemoved(await runSelected(ctx, sel, base));
|
||||
return noPolicy ? withStarter(ctx, sel, result) : result;
|
||||
const reuse = ctx.fromRun ? fromRun(ctx, sel) : null;
|
||||
if (reuse) ctx.progress(reuse.line);
|
||||
const rest = reuse?.passed.length ? { ...sel, localTests: sel.localTests.filter(t => !reuse.passed.includes(t)) } : sel;
|
||||
const ran = await runSelected(ctx, rest, base);
|
||||
const reused = !reuse ? ran : { ...ran, lines: [reuse.line, ...(ran.lines ?? [])], ...(reuse.receipt ? { from_run: { receipt: reuse.receipt, files: reuse.passed }, detail: `${reuse.passed.length} of ${sel.localTests.length} local test file(s) passed in the --from-run run; ${ran.detail ?? ran.status}` } : {}) };
|
||||
const result = withRemoved(reused);
|
||||
if (!sel.noLane.length) return result;
|
||||
return mode === 'no-policy' ? withStarter(ctx, sel, result) : withNoLaneWarning(ctx, sel, result);
|
||||
}
|
||||
|
||||
/**
|
||||
* `--from-run`: verify the run against the base branch's profiles for this machine's OS whose workflow holds a lane this
|
||||
* machine runs (not a platform lane), on exactly this tree; the selected local test files its tree.json lists are the
|
||||
* ones it already passed. Profiles are matched by workflow and OS, never by name, so none has to be named after a lane.
|
||||
*/
|
||||
function fromRun(ctx: PregateContext, sel: Selection): { line: string; passed: string[]; receipt?: string } {
|
||||
const { ref, check } = ctx.fromRun!;
|
||||
const os = ({ linux: 'Linux', darwin: 'macOS', win32: 'Windows' } as Record<string, string>)[ctx.platform];
|
||||
const names = 'error' in check ? [] : Object.entries(check.profiles).filter(([, p]) => p.os === os && ctx.lanes.some(l => l.tier !== 'platform' && l.id.startsWith(`${p.workflow}/`))).map(([n]) => n);
|
||||
const v = 'error' in check ? { ok: false as const, reason: 'profiles_unavailable', detail: check.error } : verifyFromRun(check, ref, names, ctx.wtree, ctx.head);
|
||||
if (!v.ok) return { line: `REUSE: refused reason=${v.reason} — ${v.detail}; every selected test runs here`, passed: [] };
|
||||
const ran = new Set(v.evidence.files);
|
||||
const passed = sel.localTests.filter(t => ran.has(t));
|
||||
return { line: `REUSE: accepted ${passed.length} of ${sel.localTests.length} selected local test file(s) from ${v.receipt}; ${sel.localTests.length - passed.length} still run`, passed, receipt: v.receipt };
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -61,8 +61,12 @@ function unquote(v: string): string { return /^['"`]/.test(v) ? v.slice(1, -1) :
|
||||
|
||||
/**
|
||||
* Test files that still carry the old literal beside the constant's name or
|
||||
* its module. The module counts as its full file name (`install.sh`) or as a
|
||||
* path fragment (`/retry.js`, `/retry'`): a bare stem such as `install` or `retry` is an
|
||||
* its module. The module counts as its full file name (`install.sh`, not
|
||||
* `install.sh.tmpl`) or as a path fragment (`/retry.js`, `/retry'`) whose stem is
|
||||
* followed by the module's own extension (for a JS/TS module also any JS/TS
|
||||
* extension or `.d.ts`), a sentence-ending `.`, or anything but a word
|
||||
* character, `.` or `-`, so a sibling `/retry.txt`, `/retry-v2.ts` or
|
||||
* `/retry.ts.snap`, or `lib/install.ts` for `install.sh`, is not the module (GST-820): a bare stem such as `install` or `retry` is an
|
||||
* English word in half the test tree, and a one-character old value (`2`)
|
||||
* sits in every test, so that value needs the constant's name (a wrapper
|
||||
* version 2 → 3 flagged 575 consumers through the stem alone).
|
||||
@@ -83,7 +87,11 @@ export function consumersOf(repoRoot: string, testFiles: readonly string[], chan
|
||||
const newRowRe = row(change.new);
|
||||
const base = path.basename(change.file);
|
||||
const stem = base.replace(/\.[^.]+$/, '');
|
||||
const moduleRe = new RegExp(`(?:(?:^|[/\\\\])${escape(stem)}|(?<![\\w-])${escape(base)})(?![\\w-])`, 'm');
|
||||
const own = base.slice(stem.length + 1);
|
||||
const jsLike = /^[cm]?[jt]sx?$/.test(own);
|
||||
const exts = (jsLike ? [...new Set([own, 'ts', 'tsx', 'js', 'jsx', 'mjs', 'cjs', 'mts', 'cts'])] : [own]).map(escape).join('|');
|
||||
const dts = jsLike ? '(?:d\\.)?' : '';
|
||||
const moduleRe = new RegExp(`(?:^|[/\\\\])${escape(stem)}(?=[^\\w.-]|$|\\.${dts}(?:${exts})(?![\\w.-])|\\.(?!\\w))|(?<![\\w-])${escape(base)}(?![\\w-]|\\.\\w)`, 'm');
|
||||
const nameRe = new RegExp(`\\b${escape(change.name)}\\b`);
|
||||
const nameRequired = value.length < 2;
|
||||
const hits: string[] = [];
|
||||
|
||||
@@ -11,7 +11,7 @@ import { spawnSync } from 'node:child_process';
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import { runScriptSync } from '../../script-invocation';
|
||||
import { allowlistDigest, applyAllowlist, describeAllowlisted, describeUntested, type AllowlistFinding } from '../allowlist';
|
||||
import { allowlistDigest, applyAllowlist, describeAllowlisted, describeMissed, describeUntested, type AllowlistFinding } from '../allowlist';
|
||||
import { baseInputs, sha256 } from '../context';
|
||||
import { environmentResult, launchCause } from '../environment';
|
||||
import { checkScratch } from '../run-scratch';
|
||||
@@ -33,24 +33,89 @@ export function parseAddedLines(diff: string): Map<string, string[]> {
|
||||
return files;
|
||||
}
|
||||
|
||||
/** Each file's added line numbers in the working-tree file, from the hunk headers' `+start`. */
|
||||
export function parseAddedLineNumbers(diff: string): Map<string, number[]> {
|
||||
const files = new Map<string, number[]>();
|
||||
let file = '';
|
||||
let next = 0;
|
||||
for (const line of diff.split('\n')) {
|
||||
if (line.startsWith('diff --')) { file = ''; next = 0; continue; }
|
||||
const hunk = /^@@ -\d+(?:,\d+)? \+(\d+)(?:,\d+)? @@/.exec(line);
|
||||
if (hunk) { next = Number(hunk[1]); continue; }
|
||||
if (!next) { if (line.startsWith('+++ ')) file = line.startsWith('+++ b/') ? line.slice(6) : ''; continue; }
|
||||
if (line.startsWith('+')) { if (!files.has(file)) files.set(file, []); files.get(file)!.push(next++); }
|
||||
else if (line.startsWith(' ')) next++;
|
||||
}
|
||||
return files;
|
||||
}
|
||||
|
||||
/** Each file's whole postimage from a diff with full context: one hunk starting at line 1, its context and added lines; a file with any other shape is left out. */
|
||||
export function parsePostimages(diff: string): Map<string, string[]> {
|
||||
const files = new Map<string, string[]>();
|
||||
let file = '';
|
||||
let lines: string[] | null = null;
|
||||
for (const line of diff.split('\n')) {
|
||||
if (line.startsWith('diff --')) { file = ''; lines = null; continue; }
|
||||
if (line.startsWith('@@')) {
|
||||
const first: boolean = lines === null && file !== '' && /^@@ -\d+(?:,\d+)? \+1(?:,\d+)? @@/.test(line);
|
||||
files.delete(file);
|
||||
lines = first ? [] : null;
|
||||
if (lines) files.set(file, lines); else file = '';
|
||||
continue;
|
||||
}
|
||||
if (lines === null) { if (line.startsWith('+++ ')) file = line.startsWith('+++ b/') ? line.slice(6) : ''; continue; }
|
||||
if (line.startsWith('+') || line.startsWith(' ')) lines.push(line.slice(1));
|
||||
}
|
||||
return files;
|
||||
}
|
||||
|
||||
/** Files whose source-syntax exemption needs the whole file: a line added inside an existing `'''`/`"""` block is a literal (#3630). */
|
||||
const LEXICAL_CONTEXT_SOURCE = /\.(?:dart|pyi?)$/i;
|
||||
const WHOLE_FILE_MAX_BYTES = 4 * 1024 * 1024;
|
||||
|
||||
/**
|
||||
* Added lines from the merge-base to the working tree, plus every line of
|
||||
* untracked files, grouped by file so gstack-redact can use each file's type
|
||||
* (#3211). External diff and textconv drivers are off: either can empty or
|
||||
* rewrite the diff, and an empty diff reads as clean.
|
||||
* (#3211). A tracked Dart or Python file is passed whole with `keep`, its
|
||||
* added line numbers, so the scanner sees the strings an added line sits in
|
||||
* and only findings on those lines count (#3630). The whole text is the
|
||||
* postimage git diffed (a full-context diff), never the working copy, which
|
||||
* assume-unchanged can hide from git (GST-738); an untracked file keeps every
|
||||
* line. One too large, or whose full diff does not parse as one hunk holding
|
||||
* its added lines, is scanned under a neutral name, without that exemption. External diff and textconv
|
||||
* drivers are off: either can empty or rewrite the diff, and an empty diff
|
||||
* reads as clean.
|
||||
*/
|
||||
export function addedLines(ctx: PregateContext): Array<{ file: string; text: string }> {
|
||||
export function addedLines(ctx: PregateContext): Array<{ file: string; text: string; keep?: number[] }> {
|
||||
const diff = spawnSync('git', ['diff', '--no-color', '--no-ext-diff', '--no-textconv', '--src-prefix=a/', '--dst-prefix=b/', '--unified=0', '--diff-filter=ACMR', ctx.mergeBase, '--'], { cwd: ctx.repoRoot, encoding: 'utf8', timeout: 60_000, maxBuffer: 256 * 1024 * 1024, env: ctx.env });
|
||||
const files = parseAddedLines(diff.stdout ?? '');
|
||||
const numbers = parseAddedLineNumbers(diff.stdout ?? '');
|
||||
const whole = new Map<string, { text: string; keep: number[] }>();
|
||||
const sources = [...numbers.keys()].filter(file => LEXICAL_CONTEXT_SOURCE.test(file));
|
||||
if (sources.length) {
|
||||
const full = spawnSync('git', ['-c', 'diff.suppressBlankEmpty=false', 'diff', '--no-color', '--no-ext-diff', '--no-textconv', '--src-prefix=a/', '--dst-prefix=b/', '--unified=100000000', '--diff-filter=ACMR', ctx.mergeBase, '--', ...sources.map(file => `:(literal)${file}`)], { cwd: ctx.repoRoot, encoding: 'utf8', timeout: 60_000, maxBuffer: 256 * 1024 * 1024, env: ctx.env });
|
||||
for (const [file, post] of parsePostimages(full.status === 0 ? full.stdout ?? '' : '')) {
|
||||
const keep = numbers.get(file) ?? [];
|
||||
const added = files.get(file) ?? [];
|
||||
const text = post.join('\n');
|
||||
if (text.length <= WHOLE_FILE_MAX_BYTES && keep.length === added.length && keep.every((n, k) => post[n - 1] === added[k])) whole.set(file, { text, keep });
|
||||
}
|
||||
}
|
||||
for (const t of ctx.touched) {
|
||||
if (!t.uncommitted || t.status === 'D') continue;
|
||||
const tracked = spawnSync('git', ['ls-files', '--error-unmatch', '--', t.path], { cwd: ctx.repoRoot, encoding: 'utf8', timeout: 15_000 }).status === 0;
|
||||
if (tracked) continue;
|
||||
try { const st = fs.statSync(path.join(ctx.repoRoot, t.path)); if (st.isFile() && st.size <= 4 * 1024 * 1024) files.set(t.path, [...(files.get(t.path) ?? []), ...fs.readFileSync(path.join(ctx.repoRoot, t.path), 'utf8').split('\n')]); } catch { /* unreadable: skipped */ }
|
||||
try {
|
||||
const st = fs.statSync(path.join(ctx.repoRoot, t.path));
|
||||
if (!st.isFile() || st.size > WHOLE_FILE_MAX_BYTES) continue;
|
||||
const lines = fs.readFileSync(path.join(ctx.repoRoot, t.path), 'utf8').split('\n');
|
||||
if (LEXICAL_CONTEXT_SOURCE.test(t.path) && !files.has(t.path)) whole.set(t.path, { text: lines.join('\n'), keep: lines.map((_, n) => n + 1) });
|
||||
files.set(t.path, [...(files.get(t.path) ?? []), ...lines]);
|
||||
} catch { /* unreadable: skipped */ }
|
||||
}
|
||||
return [...files].map(([file, lines]) => ({ file, text: lines.join('\n') })).filter(p => p.text.trim());
|
||||
return [...files].map(([file, lines]) => whole.get(file) ? { file, ...whole.get(file)! } : { file, text: lines.join('\n') }).filter(p => p.text.trim());
|
||||
}
|
||||
type Finding = { id?: string; description?: string; severity?: string; category?: string; line?: number; col?: number; file: string };
|
||||
type Finding = { id?: string; description?: string; severity?: string; category?: string; line?: number; col?: number; file: string; fileLine?: boolean };
|
||||
type Scanned = Finding & AllowlistFinding;
|
||||
|
||||
function run(ctx: PregateContext): CheckResult {
|
||||
@@ -67,7 +132,7 @@ function run(ctx: PregateContext): CheckResult {
|
||||
let medium = false;
|
||||
let parsedAll = true;
|
||||
for (const [i, part] of parts.entries()) {
|
||||
const file = path.join(tmp, String(i), path.basename(part.file) || 'added.txt');
|
||||
const file = path.join(tmp, String(i), (!part.keep && LEXICAL_CONTEXT_SOURCE.test(part.file) ? '' : path.basename(part.file)) || 'added.txt');
|
||||
fs.mkdirSync(path.dirname(file), { mode: 0o700 });
|
||||
fs.writeFileSync(file, part.text, { mode: 0o600 });
|
||||
const r = runScriptSync(path.join(BIN_DIR, 'gstack-redact'), ['--from-file', file, '--json', '--max-bytes', String(64 * 1024 * 1024)], { cwd: ctx.repoRoot, encoding: 'utf8', timeout: Math.min(ctx.timeoutMs, 300_000), maxBuffer: 64 * 1024 * 1024, env: ctx.env });
|
||||
@@ -76,21 +141,21 @@ function run(ctx: PregateContext): CheckResult {
|
||||
high ||= r.status === 3;
|
||||
medium ||= r.status === 2;
|
||||
const lines = part.text.split('\n');
|
||||
try { findings.push(...(JSON.parse(r.stdout).findings ?? []).map((f: Omit<Finding, 'file'>) => ({ ...f, id: f.id ?? 'finding', category: f.category ?? '?', severity: f.severity ?? '?', file: part.file, lineText: f.line ? lines[f.line - 1] : undefined }))); } catch { parsedAll = false; }
|
||||
try { findings.push(...(JSON.parse(r.stdout).findings ?? []).filter((f: Omit<Finding, 'file'>) => !part.keep || f.id?.startsWith('engine.') || (f.line !== undefined && part.keep.includes(f.line))).map((f: Omit<Finding, 'file'>) => ({ ...f, id: f.id ?? 'finding', category: f.category ?? '?', severity: f.severity ?? '?', file: part.file, fileLine: !!part.keep, lineText: f.line ? lines[f.line - 1] : undefined }))); } catch { parsedAll = false; }
|
||||
}
|
||||
// The allowlist applies only to findings the scanner reported in full; an unparsed report keeps its exit-code verdict.
|
||||
const applied = parsedAll && ctx.config.allowlist.length ? applyAllowlist(findings, ctx.config.allowlist) : { kept: findings, allowlisted: [], untested: [] };
|
||||
const applied = parsedAll && ctx.config.allowlist.length ? applyAllowlist(findings, ctx.config.allowlist) : { kept: findings, allowlisted: [], untested: [], missed: [] };
|
||||
const kept = applied.kept;
|
||||
if (parsedAll) { high = kept.some(f => f.severity === 'HIGH'); medium = kept.some(f => f.severity === 'MEDIUM'); }
|
||||
const render = (f: Scanned) => `${f.severity} ${f.id} [${f.category}]: ${f.description ?? ''}${f.line ? ` (${f.file ? `${f.file}, ` : ''}added line ${f.line})` : ''}`;
|
||||
const allowLines = [...applied.allowlisted.map(describeAllowlisted), ...applied.untested.map(describeUntested)];
|
||||
const render = (f: Scanned) => `${f.severity} ${f.id} [${f.category}]: ${f.description ?? ''}${f.line ? ` (${f.file ? `${f.file}, ` : ''}${f.fileLine ? 'line' : 'added line'} ${f.line})` : ''}`;
|
||||
const allowLines = [...applied.allowlisted.map(describeAllowlisted), ...applied.untested.map(describeUntested), ...applied.missed.map(describeMissed)];
|
||||
const allowNote = applied.allowlisted.length ? `; ${applied.allowlisted.length} allowlisted (${[...new Set(applied.allowlisted.map(a => a.finding.id))].join(', ')})` : '';
|
||||
// HIGH and any MEDIUM credential fail; a MEDIUM hygiene/internal/pii/legal
|
||||
// hit in code (a `.local` property reads as a hostname) is listed as warn:
|
||||
// there is no user to answer gstack-redact's per-finding question here.
|
||||
const blocking = kept.filter(f => f.severity === 'HIGH' || (f.severity === 'MEDIUM' && f.category === 'secret'));
|
||||
const advisory = kept.filter(f => f.severity === 'MEDIUM' && f.category !== 'secret');
|
||||
const lineCount = parts.reduce((n, p) => n + p.text.split('\n').length, 0);
|
||||
const lineCount = parts.reduce((n, p) => n + (p.keep?.length ?? p.text.split('\n').length), 0);
|
||||
const fail = (lines: string[], detail: string): CheckResult => ({ ...base, status: 'fail', cause: 'diff', lines: [...lines, ...allowLines], code: 'PREGATE_SECRETS', detail: `${detail}${allowNote}`, fix: 'rotate and remove the credential from the added lines (history included); secrets cannot be downgraded (a benign finding is allowlisted by id, path and pattern in the policy, never env.value)' });
|
||||
if (!high && !medium) return { ...base, status: 'pass', ...(allowLines.length ? { lines: allowLines } : {}), detail: `${lineCount} added lines clean${kept.length ? ` (${kept.length} WARN)` : ''}${allowNote}` };
|
||||
if (blocking.length) return fail([...blocking, ...advisory].map(render), `${blocking.length} credential finding(s) in added lines (${blocking.map(f => f.id).join(', ')})`);
|
||||
|
||||
+18
-8
@@ -13,8 +13,9 @@
|
||||
* depends on it; source code and test trees never do)
|
||||
* selection `<workflow.yml>/<job>` → the runner-owned command that lists
|
||||
* the lane's test files when the matrix is planner-computed
|
||||
* warn check ids downgraded to warn (never `secrets`; `lanes` keeps
|
||||
* its required-lane and requires-remote verdicts)
|
||||
* warn check ids downgraded to warn (never `secrets`; `lanes`
|
||||
* downgrades only a touched file with no lane, after the
|
||||
* selected tests ran: a red lane stays `fail`)
|
||||
* strays.allow root files or new top-level directories that are intentional
|
||||
* local `<workflow.yml>/<job>` lanes a local run satisfies (a
|
||||
* platform-agnostic toolchain, or a repo whose CI is off):
|
||||
@@ -49,6 +50,11 @@
|
||||
* `env.value` (a configured credential's real value) and never
|
||||
* the whole `secret` category
|
||||
*
|
||||
* A policy declares lanes when it has a `dependencies`, `empty_lanes`,
|
||||
* `selection` or `local` key (LANE_KEYS), even an empty one; only then is a touched file with no lane
|
||||
* the hard PREGATE_NO_LANE. A policy without either (a timeout-only file)
|
||||
* keeps the no-policy lane verdicts (lib/pregate/checks/lanes.ts).
|
||||
*
|
||||
* A repo that keeps `.gstack/` out of git can hold the same file at
|
||||
* `<state root>/projects/<slug>/pregate.json` (lib/state-root.ts), honoured
|
||||
* only when the base carries none. It must sit outside the repo, so a PR
|
||||
@@ -254,17 +260,20 @@ export function validateConfig(raw: unknown, knownChecks: readonly string[] = []
|
||||
return errors;
|
||||
}
|
||||
|
||||
export function parseConfig(text: string, knownChecks: readonly string[] = []): { config: PregateConfig; errors: ConfigError[] } {
|
||||
/** The keys that declare lanes: with either present, a touched file with no lane is PREGATE_NO_LANE. */
|
||||
export const LANE_KEYS = ['dependencies', 'empty_lanes', 'selection', 'local'] as const;
|
||||
|
||||
export function parseConfig(text: string, knownChecks: readonly string[] = []): { config: PregateConfig; errors: ConfigError[]; declaresLanes: boolean } {
|
||||
let raw: unknown;
|
||||
try { raw = JSON.parse(text); }
|
||||
catch (e: any) { return { config: DEFAULT_CONFIG, errors: [{ code: 'PREGATE_CONFIG_INVALID', path: PREGATE_CONFIG_PATH, message: `not valid JSON: ${e.message}` }] }; }
|
||||
catch (e: any) { return { config: DEFAULT_CONFIG, errors: [{ code: 'PREGATE_CONFIG_INVALID', path: PREGATE_CONFIG_PATH, message: `not valid JSON: ${e.message}` }], declaresLanes: false }; }
|
||||
const errors = validateConfig(raw, knownChecks);
|
||||
if (errors.length) return { config: DEFAULT_CONFIG, errors };
|
||||
if (errors.length) return { config: DEFAULT_CONFIG, errors, declaresLanes: false };
|
||||
const o = raw as Partial<PregateConfig>;
|
||||
return { config: { $schema: o.$schema, runner: o.runner ?? null, dependencies: o.dependencies ?? {}, empty_lanes: o.empty_lanes ?? [], selection: o.selection ?? {}, warn: o.warn ?? [], strays: { allow: o.strays?.allow ?? [] }, local: o.local ?? [], guards: o.guards ?? [], serial_patterns: o.serial_patterns ?? DEFAULT_SERIAL_PATTERNS, price_file: o.price_file ?? null, stall_timeout_s: o.stall_timeout_s ?? null, timeout_s: o.timeout_s ?? null, remote_gate: o.remote_gate ?? null, allowlist: o.allowlist ?? [], run_profiles: o.run_profiles ?? {}, remote_gate_runs: o.remote_gate_runs ?? null }, errors: [] };
|
||||
return { declaresLanes: LANE_KEYS.some(k => k in o), config: { $schema: o.$schema, runner: o.runner ?? null, dependencies: o.dependencies ?? {}, empty_lanes: o.empty_lanes ?? [], selection: o.selection ?? {}, warn: o.warn ?? [], strays: { allow: o.strays?.allow ?? [] }, local: o.local ?? [], guards: o.guards ?? [], serial_patterns: o.serial_patterns ?? DEFAULT_SERIAL_PATTERNS, price_file: o.price_file ?? null, stall_timeout_s: o.stall_timeout_s ?? null, timeout_s: o.timeout_s ?? null, remote_gate: o.remote_gate ?? null, allowlist: o.allowlist ?? [], run_profiles: o.run_profiles ?? {}, remote_gate_runs: o.remote_gate_runs ?? null }, errors: [] };
|
||||
}
|
||||
|
||||
export type ConfigLoad = { config: PregateConfig; errors: ConfigError[]; present: boolean; label: string; kind: 'base' | 'head' | 'user' | 'none'; /** sha256 of the user file's bytes (kind 'user' only). */ digest?: string } | { unavailable: true; base: string; detail: string };
|
||||
export type ConfigLoad = { config: PregateConfig; errors: ConfigError[]; present: boolean; label: string; kind: 'base' | 'head' | 'user' | 'none'; /** sha256 of the user file's bytes (kind 'user' only). */ digest?: string; /** The policy has a LANE_KEYS key; absent under kind 'none'. */ declaresLanes?: boolean } | { unavailable: true; base: string; detail: string };
|
||||
|
||||
function git(cwd: string, args: string[]): { ok: boolean; out: string } {
|
||||
const r = spawnSync('git', args, { cwd, encoding: 'utf8', timeout: 30_000 });
|
||||
@@ -333,7 +342,8 @@ export function explainConfig(): string {
|
||||
'dependencies: glob → test files the import graph cannot see (subprocess targets, runtime reads, generated inputs); [] declares no lane needed',
|
||||
'empty_lanes: extra globs treated as docs/no-tests when nothing else maps them. Built-in, with or without this file: any Markdown file the import graph maps to no test (a quoted path, a join() of path segments, an import, or a dependencies entry), plus README/CHANGELOG/CONTRIBUTING/LICENSE/AGENTS/CLAUDE and licence text under any name; assets (images, fonts, media) and allowlisted tool config take their own empty lane. Source code and files under a test directory never qualify. A test that reads the file still selects that test',
|
||||
'selection: <workflow.yml>/<job> → the runner-owned command that lists that lane’s test files when its matrix is planner-computed',
|
||||
`warn: check ids downgraded to warn; never ${NON_DOWNGRADABLE.join(', ')}; lanes keeps its required-lane and requires-remote verdicts`,
|
||||
`warn: check ids downgraded to warn; never ${NON_DOWNGRADABLE.join(', ')}; lanes downgrades only a touched file with no lane, after the selected tests ran (a red lane stays fail)`,
|
||||
`lane requirements: a policy with any of ${LANE_KEYS.join(', ')} (even empty) makes a touched file with no lane PREGATE_NO_LANE; a policy with none of them (timeout_s only, say) keeps the no-policy warning (PREGATE_NO_POLICY) and starter`,
|
||||
'strays.allow: root files or new top-level directories that are intentional',
|
||||
'local: <workflow.yml>/<job> lanes a local run satisfies (platform-agnostic toolchain, or CI off); their tests run here, not requires-remote',
|
||||
'guards: [{ id, when: [globs], command, fix?, timeout_s? }]: each guard whose globs match a touched file runs in its own process (--allow-repo-commands); a failure fails pregate with the guard id and fix; never downgradable',
|
||||
|
||||
@@ -177,13 +177,13 @@ export function loadedDotenvFiles(cwd: string, execArgv: readonly string[]): str
|
||||
|
||||
/** `<state root>/projects/<slug>/pregate.json` for the repository at `root`, from the caller's environment. */
|
||||
export function defaultUserConfigPath(root: string): string {
|
||||
return path.join(resolveStateRoot(), 'projects', slugFromEnvironment(undefined, root), 'pregate.json');
|
||||
return path.join(resolveStateRoot(), 'projects', slugFromEnvironment(undefined, root, { scope: 'repository' }), 'pregate.json');
|
||||
}
|
||||
|
||||
/** `<state root>/projects/<slug>/pregate/runs`: where every run's scratch directory lives (lib/pregate/run-scratch.ts), from the caller's environment. */
|
||||
export function defaultScratchRoot(root: string, env: NodeJS.ProcessEnv = process.env): string {
|
||||
const stateRoot = resolveStateRoot(env);
|
||||
return path.join(stateRoot, 'projects', slugFromEnvironment(stateRoot, root), 'pregate', 'runs');
|
||||
return path.join(stateRoot, 'projects', slugFromEnvironment(stateRoot, root, { scope: 'repository' }), 'pregate', 'runs');
|
||||
}
|
||||
|
||||
/** This run's scratch directory, created under the state root; an unwritable state root falls back to a private directory under os.tmpdir() (still one per run). */
|
||||
@@ -247,7 +247,7 @@ export function buildContext(o: BuildContextOptions): ContextLoad {
|
||||
return {
|
||||
ctx: {
|
||||
repoRoot: o.repoRoot, base, baseRef, baseSha: baseSha.out, mergeBase, head, wtree, touched: touchedFiles(o.repoRoot, mergeBase),
|
||||
config, configLabel: configLoad.label, configKind: configLoad.kind, scratch, configFile: configLoad.kind === 'user' && o.userConfigPath ? o.userConfigPath : PREGATE_CONFIG_PATH, ...(configLoad.digest ? { configDigest: configLoad.digest } : {}), ...(absentUserPolicy ? { absentUserPolicy } : {}), registry: registryLoad.registry as Registry, registryLabel: registryLoad.source.label,
|
||||
config, configLabel: configLoad.label, configKind: configLoad.kind, policyDeclaresLanes: configLoad.declaresLanes !== false, scratch, configFile: configLoad.kind === 'user' && o.userConfigPath ? o.userConfigPath : PREGATE_CONFIG_PATH, ...(configLoad.digest ? { configDigest: configLoad.digest } : {}), ...(absentUserPolicy ? { absentUserPolicy } : {}), registry: registryLoad.registry as Registry, registryLabel: registryLoad.source.label,
|
||||
lanes, platform, allowRepoCommands: o.allowRepoCommands, timeoutMs: o.timeoutMs ?? (config.timeout_s ?? 600) * 1000, timeoutSource: o.timeoutMs !== undefined ? '--timeout' : config.timeout_s ? `timeout_s from ${configLoad.label}` : 'the 600 s default',
|
||||
stallTimeoutMs: o.stallTimeoutMs ?? (config.stall_timeout_s ?? DEFAULT_STALL_TIMEOUT_S) * 1000,
|
||||
progress: o.progress ?? (line => process.stderr.write(`${line}\n`)),
|
||||
|
||||
@@ -66,7 +66,7 @@ export function headPolicy(repoRoot: string, trusted: Loaded, mergeBase: string,
|
||||
for (const [glob, tests] of Object.entries(adds)) dependencies[glob] = [...new Set([...(dependencies[glob] ?? []), ...tests])];
|
||||
const n = Object.keys(adds).length;
|
||||
return {
|
||||
load: { ...trusted, config: { ...trusted.config, dependencies }, kind: 'head', label: `head (unreviewed) ${PREGATE_CONFIG_PATH}: adds ${n} lane${n === 1 ? '' : 's'} over ${trusted.label}` },
|
||||
load: { ...trusted, config: { ...trusted.config, dependencies }, declaresLanes: trusted.declaresLanes === true || head.declaresLanes === true, kind: 'head', label: `head (unreviewed) ${PREGATE_CONFIG_PATH}: adds ${n} lane${n === 1 ? '' : 's'} over ${trusted.label}` },
|
||||
explain: Object.entries(adds).map(([glob, tests]) => `head policy adds: dependencies[${JSON.stringify(glob)}] → ${tests.join(', ')}`),
|
||||
};
|
||||
}
|
||||
+18
-4
@@ -27,7 +27,7 @@ import { NON_DOWNGRADABLE, PREGATE_CONFIG_PATH, userPolicyAbsent } from './confi
|
||||
import { sha256, withFingerprint } from './context';
|
||||
import { baseBranchOf, baseMismatchFix, describeSubstitution, substituteStampOnlyTree } from './remote-receipt';
|
||||
import { resolveBaseBranch } from '../ship-policy';
|
||||
import { parseRunReceipt, parseRunRef, runChanged, runReceiptFields, uncoveredFiles, verifyRun, type RemoteGateRuns, type RunDeps, type RunEvidence, type RunProfile } from '../run-evidence';
|
||||
import { parseRunReceipt, parseRunRef, runChanged, runReceiptFields, uncoveredFiles, verifyRun, type ApiRun, type RemoteGateRuns, type RunDeps, type RunEvidence, type RunProfile } from '../run-evidence';
|
||||
import type { CheckDef, CheckResult, PregateContext, Stage } from './types';
|
||||
|
||||
export const CHECKS: readonly CheckDef[] = [regenCheck, secretsCheck, straysCheck, literalsCheck, timingCheck, guardsCheck, lanesCheck];
|
||||
@@ -37,12 +37,12 @@ export const PREGATE_TIER2 = ['ratchets', 'lint', 'links', 'hermetic', 'patches'
|
||||
|
||||
export interface RunOptions { stage: Stage | 'all'; only?: string[]; run?: string }
|
||||
|
||||
/** Apply the repo's `warn` downgrades: never `secrets`, and never a lane that could run here and failed. */
|
||||
/** Apply the repo's `warn` downgrades: never `secrets`. `lanes` applies its own (a touched file with no lane, after the selected tests ran), so nothing it fails is downgraded here. */
|
||||
export function applyDowngrades(ctx: PregateContext, result: CheckResult, def: CheckDef): CheckResult {
|
||||
if (result.status !== 'fail') return result;
|
||||
if (!def.downgradable || (NON_DOWNGRADABLE as readonly string[]).includes(def.id)) return result;
|
||||
if (!ctx.config.warn.includes(def.id)) return result;
|
||||
if (def.id === 'lanes' && result.code === 'PREGATE_LANE_FAILED') return { ...result, detail: `${result.detail} (required lane; warn downgrade does not apply)` };
|
||||
if (def.id === 'lanes') return result.code === 'PREGATE_LANE_FAILED' ? { ...result, detail: `${result.detail} (required lane; warn downgrade does not apply)` } : result;
|
||||
return { ...result, status: 'warn', detail: `${result.detail} (downgraded to warn by ${ctx.configLabel})` };
|
||||
}
|
||||
|
||||
@@ -217,7 +217,10 @@ export type ClearDecision = { receipt: string; attested: boolean; substitution?:
|
||||
* tree and the substitution recorded; when any differing path is not a stamp,
|
||||
* the tree mismatch refuses as before. A `test_backend/<backend>` obligation
|
||||
* has no machine-checkable receipt, so its run URL is recorded as attested.
|
||||
* Platform lanes take a run URL or an evidence-bundle lane label.
|
||||
* Platform lanes take a run URL or an evidence-bundle lane label. When the run
|
||||
* evidence is readable, a run URL must name a run of this repository whose
|
||||
* workflow file is the lane's own; a run of another workflow, an unreadable
|
||||
* run or a URL that is not an Actions run refuses (GSTA-203).
|
||||
*
|
||||
* `--run` (and `--run-url` on a lane the base branch profiles) goes to
|
||||
* clearByRuns, which accepts only verified Actions runs; a profiled lane
|
||||
@@ -253,6 +256,17 @@ export function clearDecision(file: PregateFile, lane: string, input: ClearInput
|
||||
if (!input.runUrl) return { refused: 'usage', message: `${lane} clears with --run-url <url>` };
|
||||
return { receipt: `attested:${input.runUrl}`, attested: true };
|
||||
}
|
||||
if (input.runUrl && input.evidence) {
|
||||
const repo = input.evidence.repo;
|
||||
const invalid = (why: string, fix: string): ClearDecision => ({ refused: 'PREGATE_RECEIPT_INVALID', message: `REUSE: refused reason=${why}; ${lane} stays uncleared`, fix });
|
||||
const id = parseRunRef(input.runUrl.replace(/[?#].*$/, ''), repo);
|
||||
if (id === null) return invalid(`run_ref_invalid — ${JSON.stringify(input.runUrl)} is not a run id or a run URL on ${repo}`, `pass the run's numeric id or https://github.com/${repo}/actions/runs/<id>`);
|
||||
let run: ApiRun | null;
|
||||
try { run = input.evidence.deps.run(id); }
|
||||
catch (e: any) { return invalid(`api_error — reading run ${id} failed: ${String(e?.message ?? e).slice(0, 300)}`, `check gh auth status and that gh can read ${repo}'s Actions runs, then rerun the same clear`); }
|
||||
const want = `.github/workflows/${target.workflow}`;
|
||||
if (run?.path !== want) return invalid(`workflow_mismatch — run ${id} attempt ${run?.run_attempt ?? '?'} ran ${run?.path ?? 'an unknown workflow'}, not ${want}`, `clear ${lane} with a run of its own workflow: gh workflow run ${target.workflow} --ref <branch>, then gstack-pregate clear --remote ${lane} --run-url <that run's URL>`);
|
||||
}
|
||||
const receipt = input.runUrl ?? (input.bundleLane ? `bundle-lane:${input.bundleLane}` : undefined);
|
||||
if (!receipt) return { refused: 'usage', message: `${lane} clears with --run-url <url> or --bundle-lane <label>` };
|
||||
return { receipt, attested: false };
|
||||
|
||||
@@ -165,7 +165,7 @@ function removedPath(ctx: PregateContext, graph: Graph, gone: ReadonlySet<string
|
||||
const uncovered = namedBy(ctx, file).filter(f => !gone.has(f)).filter(n => {
|
||||
const m = testsFor(graph, n, ctx.config.dependencies);
|
||||
for (const t of m.tests) namerSuite.add(t);
|
||||
return !m.tests.length;
|
||||
return !m.tests.length && !m.declaredNone;
|
||||
});
|
||||
if (uncovered.length) return { refused: `removed, but still named by ${shown(uncovered)}, which no test covers` };
|
||||
const namerNote = namerSuite.size ? `files naming it run: ${shown([...namerSuite].sort())}` : '';
|
||||
@@ -211,6 +211,11 @@ export function selectLanes(ctx: PregateContext): Selection {
|
||||
const stampOnly: string[] = [];
|
||||
const seeds = new Map<string, string>();
|
||||
const touchedByPath = new Map(touched.map(t => [t.path, t]));
|
||||
const waive = (file: string, r: { removed: Removed; suite: string[] }) => {
|
||||
removed.push(r.removed);
|
||||
for (const t of r.suite) tests.add(t);
|
||||
provenance.push(`${file}: removed (no lane owed) — ${r.removed.note}`);
|
||||
};
|
||||
for (const t of touched) { seeds.set(t.path, t.status === 'D' ? 'deleted' : 'touched'); if (t.renamedFrom) seeds.set(t.renamedFrom, `renamed to ${t.path}`); }
|
||||
for (const e of ctx.registry.entries) {
|
||||
if (!touched.some(t => matchesAny(e.inputs, t.path) || matchesAny(e.outputs, t.path))) continue;
|
||||
@@ -221,8 +226,11 @@ export function selectLanes(ctx: PregateContext): Selection {
|
||||
for (const t of m.tests) tests.add(t);
|
||||
provenance.push(...m.provenance.map(p => `${p} [${why}]`));
|
||||
if (m.tests.length || why.startsWith('generated')) continue;
|
||||
// An explicit `[]` already cleared the file. Label it docs/no-tests only when it is that class.
|
||||
// An explicit `[]` already cleared the file. A removed test still runs its owning suite; label the rest docs/no-tests only when it is that class.
|
||||
if (m.declaredNone) {
|
||||
const r = gone.includes(file) ? removedPath(ctx, graph, goneSet, file) : null;
|
||||
if (r && 'removed' in r) { waive(file, r); continue; }
|
||||
if (r) provenance.push(`${file}: ${r.refused}; cleared by its [] declaration`);
|
||||
if (isDocsEmptyLane(file, ctx.config.empty_lanes ?? [])) emptyLane.push(file);
|
||||
continue;
|
||||
}
|
||||
@@ -259,12 +267,7 @@ export function selectLanes(ctx: PregateContext): Selection {
|
||||
}
|
||||
if (gone.includes(file)) {
|
||||
const r = removedPath(ctx, graph, goneSet, file);
|
||||
if ('removed' in r) {
|
||||
removed.push(r.removed);
|
||||
for (const t of r.suite) tests.add(t);
|
||||
provenance.push(`${file}: removed (no lane owed) — ${r.removed.note}`);
|
||||
continue;
|
||||
}
|
||||
if ('removed' in r) { waive(file, r); continue; }
|
||||
provenance.push(`${file}: ${r.refused}; no lane until declared`);
|
||||
}
|
||||
noLane.push(file);
|
||||
|
||||
@@ -0,0 +1,40 @@
|
||||
/**
|
||||
* The run-evidence inputs one consumer needs (GSTA-156): the base branch's run
|
||||
* profiles, this repository on GitHub, the verifier's gh- and git-backed
|
||||
* dependencies, and the remote gate's inventory command (a pin-file command,
|
||||
* so it runs only with --allow-repo-commands). Shared by `gstack-pregate
|
||||
* clear` and `gstack-pregate --from-run`, so both read profiles from the
|
||||
* base and nowhere else.
|
||||
*/
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import { ghRunDeps, runInventory } from '../run-evidence';
|
||||
import { wtreeFingerprint } from '../script-invocation';
|
||||
import { resolveBaseBranch } from '../ship-policy';
|
||||
import { normalizeRemoteUrl } from '../ship-queue-train-record';
|
||||
import { REMOTE_GATE_LANE } from './checks/lanes';
|
||||
import { PREGATE_CONFIG_PATH, isConfigUnavailable, loadConfigFromBase } from './config';
|
||||
import type { RunCheck } from './index';
|
||||
|
||||
/**
|
||||
* `profiled` says whether `lane` may have a run profile: an unreadable base counts as profiled, so a lane is
|
||||
* recorded unverified only once the base shows it has none.
|
||||
*/
|
||||
export function buildRunCheck(root: string, lane: string | null, allowRepoCommands: boolean, base = resolveBaseBranch(root)): RunCheck | { error: string; profiled?: boolean } {
|
||||
const loaded = loadConfigFromBase(root, base);
|
||||
if (isConfigUnavailable(loaded)) return { error: `the base branch's ${PREGATE_CONFIG_PATH} is unreadable (${loaded.detail}); run profiles come only from the base`, profiled: true };
|
||||
if (loaded.errors.length) return { error: `the base branch's ${PREGATE_CONFIG_PATH} is invalid: ${loaded.errors.map(e => `${e.path}: ${e.message}`).join('; ')}`, profiled: true };
|
||||
const origin = spawnSync('git', ['config', '--get', 'remote.origin.url'], { cwd: root, encoding: 'utf8', timeout: 30_000 });
|
||||
const repo = /^https:\/\/github\.com\/(.+)$/.exec(normalizeRemoteUrl(origin.status === 0 ? origin.stdout.trim() : ''))?.[1];
|
||||
if (!repo) return { error: 'origin is not a github.com repository; run evidence is read from GitHub Actions', profiled: lane === null ? false : lane === REMOTE_GATE_LANE ? !!loaded.config.remote_gate_runs : !!loaded.config.run_profiles[lane] };
|
||||
return {
|
||||
repo, deps: ghRunDeps(repo, root), profiles: loaded.config.run_profiles, gateRuns: loaded.config.remote_gate_runs, now: new Date(),
|
||||
inventory: (command, tree) => {
|
||||
if (!allowRepoCommands) return { error: 'the inventory command is a pin-file command; pass --allow-repo-commands' };
|
||||
const at = () => { const fp = wtreeFingerprint({ cwd: root }); return 'tree' in fp ? fp.tree : null; };
|
||||
if (at() !== tree) return { error: `the working tree is not pregate.json's tree ${tree.slice(0, 12)}; the inventory must list that tree's gate files` };
|
||||
const listed = runInventory(command, root);
|
||||
if (at() !== tree) return { error: `the working tree changed while ${command} ran` };
|
||||
return listed;
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -2,7 +2,8 @@
|
||||
* pregate/starter — the policy a repository with none would start from
|
||||
* (Lane D, GSTA-112; FOUN #292, harness-evals #81/#82). When no
|
||||
* `.gstack/pregate.json` exists on the base branch and no user-owned file is
|
||||
* present, the lanes check does not fail a touched file with no lane
|
||||
* present, or the active policy declares no lanes (no `dependencies` or
|
||||
* `empty_lanes`, `selection` or `local` key, GST-775), the lanes check does not fail a touched file with no lane
|
||||
* (PREGATE_NO_LANE); it warns (PREGATE_NO_POLICY) and prints this starter:
|
||||
* the detected runner, a `dependencies` entry for each unmapped touched file
|
||||
* (the tests nearest it, else the tests sharing its stem; never `[]`, which
|
||||
@@ -29,7 +30,7 @@ export function nearestTestsFor(tests: readonly string[], file: string): string[
|
||||
}
|
||||
|
||||
/** The starter for `noLane` (touched files with no lane) under `runner` (detected by the lanes check; null when none). */
|
||||
export function starterPolicy(ctx: Pick<PregateContext, 'repoRoot' | 'lanes' | 'config' | 'baseRef' | 'absentUserPolicy'>, noLane: readonly string[], runner: string | null): string[] {
|
||||
export function starterPolicy(ctx: Pick<PregateContext, 'repoRoot' | 'lanes' | 'config' | 'baseRef' | 'absentUserPolicy' | 'configKind' | 'configLabel' | 'configFile'>, noLane: readonly string[], runner: string | null): string[] {
|
||||
const tests = listRepoFiles(ctx.repoRoot).filter(isTestFile).sort();
|
||||
const dependencies: Record<string, string[]> = {};
|
||||
const todo: string[] = [];
|
||||
@@ -43,7 +44,9 @@ export function starterPolicy(ctx: Pick<PregateContext, 'repoRoot' | 'lanes' | '
|
||||
const dirs = [...dirCounts].sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0])).slice(0, MAX_DIRS).map(([d]) => d);
|
||||
const lanes = ctx.lanes.filter(l => l.runsTests && !l.manual).map(l => `${l.id} (${l.platform}${l.matrixExpression && l.testFiles.length === 0 && !ctx.config.selection[l.id] ? ', matrix computed; declare selection' : ''})`);
|
||||
return [
|
||||
`PREGATE_STARTER: no ${PREGATE_CONFIG_PATH} on ${ctx.baseRef}${ctx.absentUserPolicy ? ` and no user-owned policy at ${ctx.absentUserPolicy}` : ''}; a starter drafted from this repository follows. Review it, write it with gstack-pregate init, edit each TODO, then commit it on the base branch (from then on an unmapped file is PREGATE_NO_LANE)`,
|
||||
ctx.configKind === 'none'
|
||||
? `PREGATE_STARTER: no ${PREGATE_CONFIG_PATH} on ${ctx.baseRef}${ctx.absentUserPolicy ? ` and no user-owned policy at ${ctx.absentUserPolicy}` : ''}; a starter drafted from this repository follows. Review it, write it with gstack-pregate init, edit each TODO, then commit it on the base branch (from then on an unmapped file is PREGATE_NO_LANE)`
|
||||
: `PREGATE_STARTER: ${ctx.configLabel} declares no lanes (no dependencies, empty_lanes, selection or local key); a starter drafted from this repository follows. Review it, add its dependencies to ${ctx.configFile}${ctx.configKind === 'user' ? '' : ' on the base branch'} and edit each TODO (from then on an unmapped file is PREGATE_NO_LANE)`,
|
||||
`PREGATE_STARTER: runner ${runner ?? 'null (none detected; set runner with {files})'}`,
|
||||
`PREGATE_STARTER: test directories: ${dirs.join(', ') || '(no test files found)'}${dirCounts.size > MAX_DIRS ? ` (+${dirCounts.size - MAX_DIRS} more)` : ''}`,
|
||||
`PREGATE_STARTER: CI lanes: ${lanes.join(', ') || '(no test job in .github/workflows)'}`,
|
||||
|
||||
@@ -4,6 +4,7 @@
|
||||
*/
|
||||
import type { PregateCheck } from '../headless-artifacts';
|
||||
import type { Registry } from '../regen';
|
||||
import type { RunSource } from '../run-evidence';
|
||||
import type { PregateConfig } from './config';
|
||||
import type { Lane } from './workflows';
|
||||
|
||||
@@ -35,6 +36,8 @@ export interface PregateContext {
|
||||
configLabel: string;
|
||||
/** Where the policy came from: the base branch, the unreviewed head, the user-owned file, or nowhere (defaults). */
|
||||
configKind: 'base' | 'head' | 'user' | 'none';
|
||||
/** The active policy has a `dependencies`, `empty_lanes`, `selection` or `local` key (config LANE_KEYS); false keeps the no-policy lane verdicts. Absent counts as declared. */
|
||||
policyDeclaresLanes?: boolean;
|
||||
/** This run's scratch directory under the state root (lib/pregate/run-scratch.ts): logs and scan files, never os.tmpdir(). */
|
||||
scratch: string;
|
||||
/** The policy file to edit: the user-owned path when it is the active policy, else `.gstack/pregate.json`. */
|
||||
@@ -59,6 +62,8 @@ export interface PregateContext {
|
||||
cores: number;
|
||||
testBackend: string;
|
||||
policyFrom: 'base' | 'head';
|
||||
/** `--from-run`: a Free Tests run whose tree.json files count as passed for this tree (lanes, lib/run-evidence.ts verifyFromRun), with the base branch's run check or why it could not be built. */
|
||||
fromRun?: { ref: string; check: RunSource | { error: string } };
|
||||
/** `--remote`: run the declared remote gate now, under this spend cap. */
|
||||
remote: { spendCapUsd: number } | null;
|
||||
env: NodeJS.ProcessEnv;
|
||||
|
||||
Reference in new issue
Block a user