mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-20 23:23:26 +01:00
train S1: merge #3883 at 60d7ae8bc1 v1.93.18.0
This commit is contained in:
commit
97308cebba
11 files changed
+244
-26
No files matched your search
@@ -40,9 +40,11 @@
|
||||
* run_profiles name → `{ workflow, events, jobs, artifact, max_age_hours?, os? }`:
|
||||
* the GitHub Actions runs `clear --run` accepts as evidence
|
||||
* (lib/run-evidence.ts); a profile named after a lane clears it
|
||||
* remote_gate_runs `{ profiles, inventory }`: the run profiles that together
|
||||
* clear the remote gate obligation on the identical tree, and
|
||||
* the command listing the gate's test files they must cover
|
||||
* remote_gate_runs `{ profiles, inventory, skipped? }`: the run profiles that
|
||||
* together clear the remote gate obligation on the identical
|
||||
* tree, the command listing the gate's test files they must
|
||||
* cover, and the command naming the selected files the gate
|
||||
* itself never runs (gstack: paid files)
|
||||
* allowlist `{ finding, paths?, pattern?, reason }` entries the `secrets`
|
||||
* check accepts as benign: a gstack-redact finding id or a
|
||||
* non-secret category, scoped by path globs and/or a regex
|
||||
@@ -354,7 +356,7 @@ export function explainConfig(): string {
|
||||
'timeout_s: seconds each runner process may take (the unit-file group, each serial file, each guard without its own timeout_s, and the remote gate\'s deadline) when --timeout is not passed; 1 to 3600, default 600',
|
||||
'remote_gate: the repo\'s remote test gate command; under 8 cores the test tier becomes a requires-remote obligation naming it, and gstack-pregate --remote runs it (it must print PREGATE_REMOTE: tree=<tree> tests=<n> failed=<n>) (docs/pregate.md#remote-gate)',
|
||||
'run_profiles: name → { workflow, events, jobs, artifact, max_age_hours?, os? }; the GitHub Actions runs gstack-pregate clear --run verifies as evidence on the identical tree (docs/pregate.md#run-evidence)',
|
||||
'remote_gate_runs: { profiles, inventory }; the run profiles that together clear the remote gate obligation, and the command listing the gate\'s test files they must cover (docs/pregate.md#run-evidence)',
|
||||
'remote_gate_runs: { profiles, inventory, skipped? }; the run profiles that together clear the remote gate obligation, the command listing the gate\'s test files they must cover, and an optional command printing which selected files (PREGATE_FILES_FILE) the gate itself does not run, which then need no run (docs/pregate.md#run-evidence)',
|
||||
`allowlist: [{ finding, paths?, pattern?, reason }]: benign findings the secrets check accepts and names as allowlisted. finding is a gstack-redact id (env.kv, google.api_key, pii.phone.e164, …) or a category (${ALLOWLISTABLE_CATEGORIES.join(', ')}); paths are globs; pattern is a regex matched at the finding's position in the added line; one of the two is required, reason always. Never ${NON_ALLOWLISTABLE.join(' or ')} or engine.* (docs/pregate.md#allowlist)`,
|
||||
'no .gstack/pregate.json on the base: <state root>/projects/<slug>/pregate.json is read instead (state root: GSTACK_STATE_ROOT, else GSTACK_HOME, else GSTACK_STATE_DIR, else a gstack plugin data dir, else ~/.gstack, per docs/state-root.md; run --explain prints the resolved path; outside the repo, so a PR cannot reach it; a path inside the checkout is ignored)',
|
||||
].join('\n') + '\n';
|
||||
|
||||
+13
-3
@@ -205,7 +205,7 @@ function userPolicyChange(file: PregateFile): string | null {
|
||||
|
||||
export interface ClearInput { runUrl?: string; bundleLane?: string; receiptPath?: string; runs?: string[]; evidence?: RunCheck }
|
||||
/** What a run clear needs: the base branch's run profiles, the verifier's dependencies, the clock, and the remote gate's inventory. */
|
||||
export interface RunCheck { repo: string; deps: RunDeps; profiles: Record<string, RunProfile>; gateRuns: RemoteGateRuns | null; now: Date; inventory: (command: string, tree: string) => string[] | { error: string } }
|
||||
export interface RunCheck { repo: string; deps: RunDeps; profiles: Record<string, RunProfile>; gateRuns: RemoteGateRuns | null; now: Date; inventory: (command: string, tree: string, selected?: string[]) => string[] | { error: string } }
|
||||
export type ClearDecision = { receipt: string; attested: boolean; substitution?: PregateReceiptSubstitution } | { refused: 'usage' | 'PREGATE_RECEIPT_INVALID'; message: string; fix?: string };
|
||||
|
||||
/**
|
||||
@@ -312,7 +312,8 @@ function treeSubstitution(file: PregateFile, gatedTree: string, lane: string, re
|
||||
/**
|
||||
* `clear --run`: GitHub Actions runs as the obligation's evidence (lib/run-evidence.ts). The remote gate needs one
|
||||
* verified run per `remote_gate_runs` profile, all on one tree, whose tree.json file lists together cover every file
|
||||
* the gate's inventory command prints; any other lane needs one verified run of the profile named after it. The run
|
||||
* the gate's inventory command prints and every selected file except those its skipped command says the gate itself
|
||||
* does not run (never an inventory file); any other lane needs one verified run of the profile named after it. The run
|
||||
* tree must be pregate.json's tree or a stamp-only predecessor of it, exactly as a gate report's tree must.
|
||||
*/
|
||||
function clearByRuns(file: PregateFile, lane: string, refs: string[], check: RunCheck | undefined, repoRoot: string): ClearDecision {
|
||||
@@ -359,10 +360,19 @@ function clearByRuns(file: PregateFile, lane: string, refs: string[], check: Run
|
||||
const idle = evidence.filter(e => (e.tests ?? e.files.length) === 0);
|
||||
if (idle.length) return { refused: 'PREGATE_RECEIPT_INVALID', message: `REUSE: refused reason=no_tests — ${idle.map(e => `run ${e.run_id} (${e.profile})`).join(', ')} records no tests; ${lane} stays uncleared` };
|
||||
const required = new Set(file.requires_remote?.find(r => r.lane === lane)?.tests ?? []);
|
||||
let skippedCount = 0;
|
||||
if (isGate) {
|
||||
const listed = check.inventory(check.gateRuns!.inventory, file.tree);
|
||||
if ('error' in listed) return { refused: 'PREGATE_RECEIPT_INVALID', message: `REUSE: refused reason=inventory_failed — the remote gate's inventory command failed: ${listed.error}; ${lane} stays uncleared` };
|
||||
if (!listed.length) return { refused: 'PREGATE_RECEIPT_INVALID', message: `REUSE: refused reason=inventory_empty — the remote gate's inventory command listed no test files; ${lane} stays uncleared` };
|
||||
const skipCommand = check.gateRuns!.skipped;
|
||||
if (skipCommand && required.size) {
|
||||
const skipped = check.inventory(skipCommand, file.tree, [...required]);
|
||||
if ('error' in skipped) return { refused: 'PREGATE_RECEIPT_INVALID', message: `REUSE: refused reason=skipped_failed — the remote gate's skipped command failed: ${skipped.error}; ${lane} stays uncleared` };
|
||||
const inventory = new Set(listed);
|
||||
for (const f of skipped) if (!inventory.has(f) && required.delete(f)) skippedCount++;
|
||||
if (!required.size) return { refused: 'PREGATE_RECEIPT_INVALID', message: `REUSE: refused reason=paid_only — the remote gate skips every selected test file (${skippedCount}), so no selected test ran; like the gate's own tests=0 report, that never clears; ${lane} stays uncleared` };
|
||||
}
|
||||
for (const f of listed) required.add(f);
|
||||
}
|
||||
const missing = uncoveredFiles([...required], evidence);
|
||||
@@ -375,7 +385,7 @@ function clearByRuns(file: PregateFile, lane: string, refs: string[], check: Run
|
||||
}
|
||||
const tests = evidence.reduce((n, e) => n + (e.tests ?? e.files.length), 0);
|
||||
const accepted = sub.substitution ? ` accepted-for=${file.tree.slice(0, 12)} stamp-only=${sub.substitution.files.length}` : '';
|
||||
return { receipt: `${isGate ? 'remote-gate' : 'run'} tree=${trees[0]!.slice(0, 12)} tests=${tests}${accepted} ${runReceiptFields(evidence, check.now)}`, attested: false, substitution: sub.substitution };
|
||||
return { receipt: `${isGate ? 'remote-gate' : 'run'} tree=${trees[0]!.slice(0, 12)} tests=${tests}${skippedCount ? ` skipped=${skippedCount}` : ''}${accepted} ${runReceiptFields(evidence, check.now)}`, attested: false, substitution: sub.substitution };
|
||||
}
|
||||
|
||||
/** Record the receipt that clears one obligation (a run URL, an evidence-bundle lane label, or a remote gate report, with the stamp-only substitution when one was accepted). */
|
||||
|
||||
@@ -1,12 +1,16 @@
|
||||
/**
|
||||
* The run-evidence inputs one consumer needs (GSTA-156): the base branch's run
|
||||
* profiles, this repository on GitHub, the verifier's gh- and git-backed
|
||||
* dependencies, and the remote gate's inventory command (a pin-file command,
|
||||
* so it runs only with --allow-repo-commands). Shared by `gstack-pregate
|
||||
* dependencies, and the remote gate's inventory and skipped commands (pin-file
|
||||
* commands, so they run only with --allow-repo-commands; skipped reads the
|
||||
* selected files from PREGATE_FILES_FILE). Shared by `gstack-pregate
|
||||
* clear` and `gstack-pregate --from-run`, so both read profiles from the
|
||||
* base and nowhere else.
|
||||
*/
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import * as fs from 'node:fs';
|
||||
import * as os from 'node:os';
|
||||
import * as path from 'node:path';
|
||||
import { ghRunDeps, runInventory } from '../run-evidence';
|
||||
import { wtreeFingerprint } from '../script-invocation';
|
||||
import { resolveBaseBranch } from '../ship-policy';
|
||||
@@ -28,13 +32,19 @@ export function buildRunCheck(root: string, lane: string | null, allowRepoComman
|
||||
if (!repo) return { error: 'origin is not a github.com repository; run evidence is read from GitHub Actions', profiled: lane === null ? false : lane === REMOTE_GATE_LANE ? !!loaded.config.remote_gate_runs : !!loaded.config.run_profiles[lane] };
|
||||
return {
|
||||
repo, deps: ghRunDeps(repo, root), profiles: loaded.config.run_profiles, gateRuns: loaded.config.remote_gate_runs, now: new Date(),
|
||||
inventory: (command, tree) => {
|
||||
inventory: (command, tree, selected) => {
|
||||
if (!allowRepoCommands) return { error: 'the inventory command is a pin-file command; pass --allow-repo-commands' };
|
||||
const at = () => { const fp = wtreeFingerprint({ cwd: root }); return 'tree' in fp ? fp.tree : null; };
|
||||
if (at() !== tree) return { error: `the working tree is not pregate.json's tree ${tree.slice(0, 12)}; the inventory must list that tree's gate files` };
|
||||
const listed = runInventory(command, root);
|
||||
if (at() !== tree) return { error: `the working tree changed while ${command} ran` };
|
||||
return listed;
|
||||
const dir = selected && fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-pregate-skipped-'));
|
||||
try {
|
||||
if (dir) fs.writeFileSync(path.join(dir, 'selected.txt'), selected!.map(f => `${f}\n`).join(''));
|
||||
const listed = runInventory(command, root, dir ? { PREGATE_FILES_FILE: path.join(dir, 'selected.txt') } : undefined);
|
||||
if (at() !== tree) return { error: `the working tree changed while ${command} ran` };
|
||||
return listed;
|
||||
} finally {
|
||||
if (dir) fs.rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
},
|
||||
};
|
||||
}
|
||||
+7
-6
@@ -43,7 +43,7 @@ import { sha256Hex, writeReceipt } from './egress-receipt';
|
||||
import { runBashSync } from './script-invocation';
|
||||
|
||||
export interface RunProfile { workflow: string; events: string[]; jobs: string[]; artifact: string; max_age_hours?: number; os?: string }
|
||||
export interface RemoteGateRuns { profiles: string[]; inventory: string }
|
||||
export interface RemoteGateRuns { profiles: string[]; inventory: string; skipped?: string }
|
||||
export interface ApiRun {
|
||||
id: number; run_attempt: number; status: string; conclusion: string | null; event: string; head_sha: string;
|
||||
path?: string; created_at?: string; run_started_at?: string; updated_at?: string;
|
||||
@@ -108,15 +108,16 @@ export function runProfileErrors(profiles: unknown, gateRuns: unknown): Array<{
|
||||
}
|
||||
if (gateRuns !== undefined && gateRuns !== null) {
|
||||
const g = gateRuns as Record<string, unknown>;
|
||||
if (!g || typeof g !== 'object' || Array.isArray(g)) errors.push({ path: 'remote_gate_runs', message: 'must be { profiles: [names], inventory: command }' });
|
||||
if (!g || typeof g !== 'object' || Array.isArray(g)) errors.push({ path: 'remote_gate_runs', message: 'must be { profiles: [names], inventory: command, skipped?: command }' });
|
||||
else {
|
||||
for (const k of Object.keys(g)) if (k !== 'profiles' && k !== 'inventory') errors.push({ path: `remote_gate_runs.${k}`, message: 'unknown key; valid keys: profiles, inventory' });
|
||||
for (const k of Object.keys(g)) if (k !== 'profiles' && k !== 'inventory' && k !== 'skipped') errors.push({ path: `remote_gate_runs.${k}`, message: 'unknown key; valid keys: profiles, inventory, skipped' });
|
||||
if (!Array.isArray(g.profiles) || g.profiles.length === 0 || g.profiles.some(p => typeof p !== 'string')) errors.push({ path: 'remote_gate_runs.profiles', message: 'must be a non-empty array of run_profiles names' });
|
||||
else {
|
||||
for (const p of g.profiles as string[]) if (!names.has(p)) errors.push({ path: 'remote_gate_runs.profiles', message: `${JSON.stringify(p)} is not a run_profiles name` });
|
||||
if (new Set(g.profiles).size !== g.profiles.length) errors.push({ path: 'remote_gate_runs.profiles', message: 'names each profile once' });
|
||||
}
|
||||
if (typeof g.inventory !== 'string' || !g.inventory.trim() || g.inventory.length > 512 || /[\0\r\n]/.test(g.inventory)) errors.push({ path: 'remote_gate_runs.inventory', message: 'must be one command line under 512 characters that prints the remote gate\'s test files, one per line' });
|
||||
if ('skipped' in g && (typeof g.skipped !== 'string' || !g.skipped.trim() || g.skipped.length > 512 || /[\0\r\n]/.test(g.skipped))) errors.push({ path: 'remote_gate_runs.skipped', message: 'must be one command line under 512 characters that prints which selected test files (PREGATE_FILES_FILE) the remote gate itself does not run, one per line' });
|
||||
}
|
||||
}
|
||||
return errors;
|
||||
@@ -201,9 +202,9 @@ export function runChanged(deps: RunDeps, id: number, attempt: number): string |
|
||||
return again.run_attempt === attempt && again.status === 'completed' && again.conclusion === 'success' ? null : `run ${id} moved to attempt ${again.run_attempt} (${again.status === 'completed' ? again.conclusion : again.status}) while it was being verified`;
|
||||
}
|
||||
|
||||
/** The remote gate's inventory: `command`'s non-empty output lines under pipefail, so a pipeline whose producer fails refuses instead of listing part of the gate. */
|
||||
export function runInventory(command: string, cwd: string): string[] | { error: string } {
|
||||
const r = runBashSync(`set -o pipefail\n${command}`, { cwd, encoding: 'utf8', timeout: 300_000, maxBuffer: 64 * 1024 * 1024 });
|
||||
/** The remote gate's inventory (or skipped list, with PREGATE_FILES_FILE in `env`): `command`'s non-empty output lines under pipefail, so a pipeline whose producer fails refuses instead of listing part of the gate. */
|
||||
export function runInventory(command: string, cwd: string, env?: Record<string, string>): string[] | { error: string } {
|
||||
const r = runBashSync(`set -o pipefail\n${command}`, { cwd, encoding: 'utf8', timeout: 300_000, maxBuffer: 64 * 1024 * 1024, ...(env && { env: { ...process.env, ...env } }) });
|
||||
if (r.status !== 0 || r.error) return { error: `${command} exited ${r.status ?? r.error}: ${(r.stderr ?? '').trim().slice(0, 200)}` };
|
||||
return [...new Set(r.stdout.split('\n').map(l => l.trim()).filter(Boolean))];
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user