train S1: merge #3852 at 61803d99bc v1.93.33.0

This commit is contained in:
garrytan committed 2000-01-01 00:00:00 +00:00
commit 68bef52f1e
119 files changed
+3616 -493

No files matched your search

+111
View File
@@ -0,0 +1,111 @@
/**
* default-base — the base branch a command uses when none is given (GST-511), and the
* refusal every base-relative step shares when its base is the branch itself (GST-774).
* A remote's HEAD counts only when it names a branch other than the checked-out one, its
* upstream, or HEAD's own commit: a clone made by a PR flow can point origin/HEAD at the PR
* branch, and that base diffs the branch against itself, an empty diff every gate passes.
* Past it: the PR's base from gh, then the remote's live HEAD (`git ls-remote --symref`,
* which keeps a default branch such as develop or trunk), then `<remote>/main`, then
* `<remote>/master`. Both network reads use a 5 s timeout with prompts off and ignore any
* failure; gh names no base for a PR from another repository. bin/gstack-diff-scope reads
* both through this file's command line.
*/
import { spawnSync } from 'node:child_process';
import { resultLine } from './result-codes';
export interface DefaultBase {
/** The base branch name, or null when nothing resolves. */
branch: string | null;
/** The remote whose HEAD named `branch`; unset for the PR base, main and master. */
remote?: string;
/** The first remote HEAD skipped because it names the checked-out branch. */
skipped?: { remote: string; branch: string };
}
/** gh's jq for the PR's base branch; a PR from another repository names a branch origin may not hold, so it names none. */
const PR_BASE = 'if .isCrossRepository then "" else .baseRefName end';
function read(cmd: string, args: string[], cwd: string, timeout: number): string {
const r = spawnSync(cmd, args, {
cwd, encoding: 'utf8', timeout, stdio: ['ignore', 'pipe', 'pipe'],
env: { ...process.env, GIT_TERMINAL_PROMPT: '0', GH_PROMPT_DISABLED: '1' },
});
return r.status === 0 ? (r.stdout ?? '').trim() : '';
}
const commit = (cwd: string, ref: string) => read('git', ['rev-parse', '--verify', '--quiet', '--end-of-options', `${ref}^{commit}`], cwd, 30_000);
const ancestor = (cwd: string, a: string, b: string) => spawnSync('git', ['merge-base', '--is-ancestor', a, b], { cwd, timeout: 30_000, stdio: 'ignore' }).status === 0;
export function defaultBaseBranch(cwd: string, remotes: readonly string[] = ['origin']): DefaultBase {
const current = read('git', ['symbolic-ref', '--quiet', 'HEAD'], cwd, 30_000).replace(/^refs\/heads\//, '');
const upstream = current ? read('git', ['rev-parse', '--abbrev-ref', '--symbolic-full-name', '@{upstream}'], cwd, 30_000) : '';
const headSha = commit(cwd, 'HEAD');
let skipped: DefaultBase['skipped'];
for (const remote of remotes) {
const prefix = `refs/remotes/${remote}/`;
const target = read('git', ['symbolic-ref', '--quiet', `${prefix}HEAD`], cwd, 30_000);
if (!target.startsWith(prefix)) continue;
const branch = target.slice(prefix.length);
if (branch !== current && upstream !== `${remote}/${branch}` && commit(cwd, target) !== headSha) return { branch, remote };
skipped ??= { remote, branch };
}
const pr = current ? read('gh', ['pr', 'view', '--json', 'baseRefName,isCrossRepository', '-q', PR_BASE], cwd, 5_000).split('\n')[0] : '';
if (pr) return { branch: pr, skipped };
if (skipped) {
const live = /^ref: refs\/heads\/(\S+)\tHEAD$/m.exec(read('git', ['ls-remote', '--symref', skipped.remote, 'HEAD'], cwd, 5_000))?.[1];
if (live) return { branch: live, remote: skipped.remote, skipped };
}
for (const name of ['main', 'master']) {
if (remotes.some((remote) => commit(cwd, `refs/remotes/${remote}/${name}`))) return { branch: name, skipped };
}
return { branch: null, skipped };
}
/** Why there is no default base: a remote HEAD named the checked-out branch and nothing past it resolved. */
export function noDefaultBaseLine({ remote, branch }: { remote: string; branch: string }): string {
return `${remote}/HEAD names the checked-out branch ${branch}, and no PR base, ${remote}/main or ${remote}/master resolves; pass the base explicitly or run git remote set-head ${remote} <base>`;
}
/**
* Why `base` cannot be the base of a comparison with `head` (BASE_CONTAINS_HEAD), or null. A
* base that is the head or already contains it compares the branch with itself, so the step
* would report no changes. A pull-request head (`head` other than HEAD) is refused whenever the
* base contains it. The checked-out branch is refused only when its real base (defaultBaseBranch)
* is another branch that lacks HEAD, so the branch has commits this base hides; on the base branch
* itself, on a detached HEAD, on a fresh branch with no commits past its base, or with no real base
* resolvable, it is null and the step keeps today's result.
*/
export function baseContainsHead(cwd: string, base: string, head = 'HEAD'): string | null {
if (!ancestor(cwd, head, base)) return null;
const baseSha = commit(cwd, base);
const headSha = commit(cwd, head);
if (!baseSha || !headSha) return null;
const what = `${base} (${baseSha.slice(0, 12)}) ${baseSha === headSha ? 'is' : 'already contains'}`;
if (head !== 'HEAD') return `${what} the head ${headSha.slice(0, 12)}, which then has no changes of its own`;
const branch = read('git', ['symbolic-ref', '--quiet', '--short', 'HEAD'], cwd, 30_000);
if (!branch) return null;
const real = defaultBaseBranch(cwd);
if (!real.branch || real.branch === branch) return null;
const realRef = `${real.remote ?? 'origin'}/${real.branch}`;
const realSha = commit(cwd, `refs/remotes/${realRef}`);
if (!realSha || ancestor(cwd, headSha, realSha)) return null;
return `${what} HEAD ${headSha.slice(0, 12)} on ${branch}, whose commits ${realRef} lacks; the real base is ${real.branch}`;
}
if (import.meta.main) {
const [verb, ...args] = process.argv.slice(2);
const cwd = process.cwd();
if (verb === 'default') {
const found = defaultBaseBranch(cwd);
if (found.branch) console.log(found.branch);
else if (found.skipped) console.log(noDefaultBaseLine(found.skipped));
process.exitCode = found.branch ? 0 : found.skipped ? 3 : 1;
} else if (verb === 'check' && args[0] && args.length <= 3) {
const refused = baseContainsHead(cwd, args[0], args[1] || 'HEAD');
if (refused) console.log(resultLine(args[2] || 'gstack', 'BASE_CONTAINS_HEAD', refused));
process.exitCode = refused ? 1 : 0;
} else {
console.error('usage: default-base.ts default | check <base> [<head>] [<context>]');
process.exitCode = 2;
}
}
+20 -16
View File
@@ -13,6 +13,7 @@
import { spawnSync } from 'node:child_process';
import * as fs from 'node:fs';
import * as path from 'node:path';
import { defaultBaseBranch, type DefaultBase } from '../default-base';
import type { CommitInfo, FileDiff, FileStatus, Hunk, Unsupported } from './types';
export const EMPTY_TREE = '4b825dc642cb6eb9a060e54bf8d69288fbee4904';
@@ -63,20 +64,19 @@ function remoteNames(cwd: string): string[] {
}
/**
* The remote's HEAD target → `main` → `master` (the gstack-diff-scope chain),
* as a ref name; the remotes are walked in resolveBase's order, so the default
* and the fix line come from the same remote.
* The default base (lib/default-base.ts, the gstack-diff-scope chain) as a ref
* name: a remote's HEAD target as `<remote>/<branch>` unless it names the
* checked-out branch, the PR's base, `main`, `master`; the remotes are walked
* in resolveBase's order, so the default and the fix line come from the same
* remote. With no remote naming one, a local main or master, else `main`.
* `skipped` when a remote HEAD named the checked-out branch and nothing past it
* resolved: the caller fails closed instead of diffing the branch against itself.
*/
export function defaultBase(cwd: string): string {
const remotes = remoteNames(cwd);
for (const remote of remotes) {
const head = gitText(['symbolic-ref', '-q', `refs/remotes/${remote}/HEAD`], cwd);
if (head) return head.replace(/^refs\/remotes\//, '');
}
for (const name of ['main', 'master']) {
if (resolvesToCommit(name, cwd) || remotes.some((remote) => resolvesToCommit(`${remote}/${name}`, cwd))) return name;
}
return 'main';
export function defaultBase(cwd: string): { ref: string } | { skipped: NonNullable<DefaultBase['skipped']> } {
const found = defaultBaseBranch(cwd, remoteNames(cwd));
if (found.branch) return { ref: found.remote ? `${found.remote}/${found.branch}` : found.branch };
if (found.skipped) return { skipped: found.skipped };
return { ref: ['main', 'master'].find((name) => resolvesToCommit(name, cwd)) ?? 'main' };
}
export function isShallow(cwd: string): boolean {
@@ -89,7 +89,8 @@ export function isShallow(cwd: string): boolean {
* (`refs/remotes/origin/main` for `main`), the base /ship merges against and
* the one review-collect's collectLive and review-dispatch read: a stale local
* `main` behind it would scan upstream commits the branch only inherited
* (#3650, GST-883), so a local branch that differs is reported as `differs`.
* (#3650, GST-883), so a local branch that differs says how: `behind` it,
* `ahead` of it (commits the branch would land unscanned) or `diverged` (GST-462).
* Without a related origin twin the related local branch wins over any other
* remote, so a fork's branch cannot replace it; it is `fallback` when origin
* lacks the branch or shares no history with HEAD (named as `unrelatedRemote`),
@@ -102,7 +103,7 @@ export function isShallow(cwd: string): boolean {
* cannot stand in for them, and a remote-qualified name (`upstream/main`) is
* taken as given.
*/
export function resolveBase(ref: string, cwd: string, head?: string): { ref: string; sha: string; local?: 'missing' | 'unrelated' | 'differs' | 'fallback'; unrelatedRemote?: string } | null {
export function resolveBase(ref: string, cwd: string, head?: string): { ref: string; sha: string; local?: 'missing' | 'unrelated' | 'behind' | 'ahead' | 'diverged' | 'fallback'; unrelatedRemote?: string } | null {
const related = (sha: string) => !head || mergeBase(sha, head, cwd) !== null;
const direct = () => resolvesToCommit(ref, cwd);
if (!isBranchName(ref)) { const sha = direct(); return sha ? { ref, sha } : null; }
@@ -115,7 +116,10 @@ export function resolveBase(ref: string, cwd: string, head?: string): { ref: str
const sha = resolvesToCommit(`refs/remotes/` + remote + '/' + ref, cwd);
if (!sha || !related(sha)) return null;
const tracking = remote + '/' + ref;
return sha === local ? { ref: tracking, sha } : { ref: tracking, sha, local: !local ? 'missing' as const : localRelated ? 'differs' as const : 'unrelated' as const };
if (sha === local) return { ref: tracking, sha };
if (!local || !localRelated) return { ref: tracking, sha, local: !local ? 'missing' as const : 'unrelated' as const };
const fork = mergeBase(local, sha, cwd);
return { ref: tracking, sha, local: fork === local ? 'behind' as const : fork === sha ? 'ahead' as const : 'diverged' as const };
};
const hasOrigin = remotes.includes('origin');
const origin = hasOrigin ? twin('origin') : null;
+7 -4
View File
@@ -24,6 +24,7 @@
import * as fs from "fs";
import * as path from "path";
import { resolveStateRoot, readConfigKey, type StateRootEnv } from "./state-root";
import { botLists, botOwning } from "./pr-wait";
import { remoteSlug } from "./remote-identity";
export interface EarlyEnv {
@@ -78,14 +79,14 @@ function readOr(file: string): string {
try { return fs.readFileSync(file, "utf8"); } catch { return ""; }
}
/** A Greptile comment on any PR of this repo in the last 90 days (conversation or review comments). */
/** A comment by Greptile's bot account (gstack-pr-wait's login rule, Bot accounts only) on any PR of this repo in the last 90 days. */
function recentComment(env: EarlyEnv): boolean {
const since = new Date(env.now() - COMMENT_WINDOW_DAYS * 86_400_000).toISOString();
for (const kind of ["issues", "pulls"]) {
const r = env.run("gh", ["api", `repos/{owner}/{repo}/${kind}/comments?sort=created&direction=desc&per_page=100&since=${since}`,
"--jq", "[.[] | {login: .user.login, created: .created_at}]"]);
const rows = r.status === 0 ? json<Array<{ login: string; created: string }>>(r.stdout) : undefined;
if (rows?.some(c => isGreptile(c.login) && c.created >= since)) return true;
"--jq", "[.[] | {login: .user.login, type: .user.type, created: .created_at}]"]);
const rows = r.status === 0 ? json<Array<{ login: string; type?: string; created: string }>>(r.stdout) : undefined;
if (rows?.some(c => c.type === "Bot" && botOwning(c.login, ["greptile"]) && c.created >= since)) return true;
}
return false;
}
@@ -129,6 +130,8 @@ function detect(env: EarlyEnv): number {
env.out(`Greptile early review is off for you. Turn it back on: ${config(env)} set ship_greptile_early true\n`);
return 0;
}
const lists = botLists(env.vars);
if (![...lists.gating, ...lists.advisory].includes("greptile")) return off("greptile is not in pr_wait_gating_bots or pr_wait_advisory_bots");
const root = gitRoot(env);
if (!root) return off("not in a git repository");
if (env.run("gh", ["auth", "status"]).status !== 0) return off("gh is missing or not logged in");
+3
View File
@@ -121,6 +121,8 @@ export interface ShipReceipt {
review_targeted?: { classes: string[]; rules: string; brief_sha: string; product_patch_id: string; status: 'done' | 'failed' | 'not-run'; findings: number; raised_by: string };
/** Step 5 failures gstack-base-red proved are the base's own (lib/base-red.ts): a conditional receipt the queue refuses until /ship --resume rewrites it without this block. */
base_red?: { cases: string[]; files: string[]; base: string; lane: string; expires_at: string };
/** The base policy's review_exclude_paths at the reviewed head (lib/review-exclude.ts); absent when the base declares none. Carry and the queue compare `digest`. */
review_exclude?: { source: string; patterns: string[]; applied: string[]; excluded: number; excluded_bytes: number; kept: number; gate_tagged: string[]; patterns_sha256: string; list_sha256: string; digest: string };
}
export const RESULT_STATUSES: readonly ResultStatus[] = ['complete', 'reviewed', 'gate_pending', 'amend_pending', 'incomplete', 'interrupted', 'refused'];
@@ -247,6 +249,7 @@ export const ARTIFACT_SCHEMAS = {
base_red: { type: 'object', required: ['cases', 'files', 'base', 'lane', 'expires_at'], properties: { cases: strList, files: strList, base: str, lane: str, expires_at: str } },
review_depth: { type: 'object', required: ['depth', 'class', 'runtime_lines', 'rules', 'base', 'gate_scan'], properties: { depth: { enum: ['light', 'standard', 'full'] }, class: { enum: ['docs', 'tests', 'code', 'gate'] }, runtime_lines: { type: 'integer', minimum: 0 }, rules: str, base: str, registry: nullableStr, gate_scan: { type: 'object', required: ['tagged', 'head'], properties: { tagged: { type: ['integer', 'null'], minimum: 0 }, head: str } }, reasons: strList, risk: { type: 'array', items: { enum: ['money', 'auth', 'secret', 'security', 'data'] } } } },
review_targeted: { type: 'object', required: ['classes', 'rules', 'brief_sha', 'product_patch_id', 'status', 'findings', 'raised_by'], properties: { classes: { type: 'array', items: { enum: ['money', 'gate', 'auth', 'secret'] } }, rules: str, brief_sha: str, product_patch_id: str, status: { enum: ['done', 'failed', 'not-run'] }, findings: { type: 'integer', minimum: 0 }, raised_by: str } },
review_exclude: { type: 'object', required: ['source', 'patterns', 'applied', 'excluded', 'excluded_bytes', 'kept', 'gate_tagged', 'patterns_sha256', 'list_sha256', 'digest'], properties: { source: str, patterns: strList, applied: strList, excluded: { type: 'integer', minimum: 0 }, excluded_bytes: { type: 'integer', minimum: 0 }, kept: { type: 'integer', minimum: 0 }, gate_tagged: strList, patterns_sha256: str, list_sha256: str, digest: str } },
review_loop_exception: { type: 'object', required: ['accepted_by', 'date', 'via', 'reason', 'fix_check'], properties: { accepted_by: str, date: str, via: str, reason: str, fix_check: { type: 'object', required: ['head', 'findings'], properties: { head: str, findings: { type: 'integer', minimum: 0 }, files: strList } } } },
issues: { type: 'object', properties: { fixes: { type: 'array', items: { type: 'integer' } }, refs: { type: 'array', items: { type: 'integer' } }, flakes: { type: 'array', items: { type: 'object', required: ['issue', 'run'], properties: { issue: { type: 'integer' }, run: str } } }, related: { type: 'array', items: { type: 'object', required: ['pr', 'author', 'closes'], properties: { pr: { type: 'integer' }, repo: str, author: str, closes: { type: 'array', items: { type: 'integer' } }, supersedes: { type: 'boolean' }, credit: { type: 'boolean' } } } } } },
},
+10 -1
View File
@@ -165,6 +165,13 @@ export function parsePrTarget(raw: string | undefined, repo?: string): { repo?:
return { repo: resolved, pr };
}
/** The configured bot that owns a login: the longest name the lowercased login starts with (greptile-apps[bot] is greptile, notgreptile is nobody). Check names keep botOf's first-match rule. */
export function botOwning(login: string | null | undefined, bots: readonly string[]): string | null {
const lowered = (login ?? '').toLowerCase();
if (!lowered) return null;
return bots.filter(bot => bot !== '' && lowered.startsWith(bot)).sort((a, b) => b.length - a.length)[0] ?? null;
}
function botList(value: string | null, fallback: string[]): string[] {
if (value === null) return fallback;
return value.split(',').map(s => s.trim().toLowerCase()).filter(s => s && s !== 'none');
@@ -291,8 +298,10 @@ export class Watcher {
/** The configured bot a check name mentions, or (for a login) that the login starts with: greptile-apps[bot] is greptile, notgreptile is nobody. */
private botOf(text: string, login = false): string | null {
const bots = [...this.bots.gating, ...this.bots.advisory];
if (login) return botOwning(text, bots);
const lowered = text.toLowerCase();
return [...this.bots.gating, ...this.bots.advisory].find(bot => login ? lowered.startsWith(bot) : lowered.includes(bot)) ?? null;
return bots.find(bot => lowered.includes(bot)) ?? null;
}
private isAdvisory(name: string): boolean {
+4 -1
View File
@@ -17,6 +17,7 @@ import { isRegistryUnavailable, loadRegistryFromBase, loadRegistryFromHead, type
import { resolveStateRoot } from '../state-root';
import { RESULT_CODES, TROUBLESHOOTING_DOC, resultLine } from '../result-codes';
import { runScriptSync, wtreeFingerprint, type InvocationOptions, type WtreeResult } from '../script-invocation';
import { baseContainsHead } from '../default-base';
import { policyUnavailableLine, resolveBaseBranch } from '../ship-policy';
import { headPolicy } from './head-policy';
import { PREGATE_CONFIG_PATH, isConfigUnavailable, loadConfigFromBase, loadConfigFromUser, type ConfigError, type ConfigLoad, type PregateConfig } from './config';
@@ -48,7 +49,7 @@ export function readTestBackend(env: NodeJS.ProcessEnv, invocation?: InvocationO
return (r.status === 0 && (r.stdout ?? '').trim()) || 'local';
}
export interface ContextRefusal {
code: 'POLICY_SOURCE_UNAVAILABLE' | 'PREGATE_CONFIG_INVALID' | 'PREGATE_HEAD_POLICY_LOOSENS' | 'REGEN_REGISTRY_INVALID' | 'WTREE_UNAVAILABLE';
code: 'POLICY_SOURCE_UNAVAILABLE' | 'BASE_CONTAINS_HEAD' | 'PREGATE_CONFIG_INVALID' | 'PREGATE_HEAD_POLICY_LOOSENS' | 'REGEN_REGISTRY_INVALID' | 'WTREE_UNAVAILABLE';
message: string;
/** A cause-specific fix that replaces the code's own (a gstack-wtree exit names its repair; so does a `--base` that is no origin branch). */
fix?: string;
@@ -210,6 +211,8 @@ export function buildContext(o: BuildContextOptions): ContextLoad {
const baseRef = `origin/${base}`;
const baseSha = git(o.repoRoot, ['rev-parse', '--verify', '--quiet', `${baseRef}^{commit}`]);
if (!baseSha.ok || !baseSha.out) return { refused: notAnOriginBranch(o.repoRoot, base) ?? { code: 'POLICY_SOURCE_UNAVAILABLE', message: `${baseRef} is not a resolvable commit` } };
const self = baseContainsHead(o.repoRoot, baseRef);
if (self) return { refused: { code: 'BASE_CONTAINS_HEAD', message: self } };
const head = git(o.repoRoot, ['rev-parse', 'HEAD']).out;
const mergeBase = git(o.repoRoot, ['merge-base', baseSha.out, 'HEAD']).out || baseSha.out;
let configLoad: ConfigLoad = loadConfigFromBase(o.repoRoot, base, o.knownChecks);
+2 -2
View File
@@ -17,7 +17,7 @@
* finding (drives sterner AUQ wording in the skill) but never promotes a
* MEDIUM to HIGH. (TENSION-2-followup.)
* - Placeholder suppression is per-matched-span.
* - Tool-attributed fences (``` ```codex-review ``` / ``` ```greptile ```)
* - Tool-attributed fences (``` ```codex-review ``` / ``` ```review-bot ``` / ``` ```greptile ```)
* degrade credential findings to a non-blocking WARN — UNLESS the span is a
* live-format credential the doc-example heuristic can't excuse. No nonce,
* no trust exemption (the marker scheme was dropped as theater).
@@ -224,7 +224,7 @@ export function maskPreview(span: string): string {
// ── Tool-attributed fence detection ───────────────────────────────────────────
const TOOL_FENCE_INFO = /^```(codex-review|greptile|eval|codex|tool-output)\b/;
const TOOL_FENCE_INFO = /^```(codex-review|review-bot|greptile|eval|codex|tool-output)\b/;
/**
* Returns a sorted list of [start, end) offset ranges (in normalized text) that
+21
View File
@@ -254,6 +254,11 @@ export const RELEASE_RESULT_CODES = {
summary: 'no new base head passed the paused cases before gstack-base-red wait’s timeout',
fix: 'run gstack-base-red wait --pause <file> again (the pause stays valid until it expires), or check who owns the base red',
},
BASE_CONTAINS_HEAD: {
anchor: 'base-contains-head',
summary: 'the base is the head or already contains it while the branch has commits its real base lacks, so a base-relative step would compare the branch with itself and report no changes',
fix: 'compare against the branch’s real base (the PR’s base branch, usually main): pass it as the base, or run git remote set-head origin <base> when origin/HEAD names the checked-out branch',
},
QUEUE_RECEIPT_DEPTH_RAISED: {
anchor: 'queue-receipt-depth-raised',
summary: 'the PR body’s /ship receipt records a review depth below the one its head needs (the queue recomputed it from git), or a current receipt records no review_depth at all',
@@ -478,4 +483,20 @@ export const RELEASE_RESULT_CODES = {
summary: 'under --policy-from head, the branch’s .gstack/pregate.json removes, shrinks or loosens something the trusted policy declares; a head policy may only add test lanes',
fix: 'keep every key of the trusted policy as it is and only add dependencies globs with tests (or tests on an existing glob); land any other change, a [] declaration included, in its own PR on the base branch first',
},
// The reviewer diff ceiling and the base policy's review_exclude_paths (lib/review-exclude.ts, GST-508).
REVIEW_DIFF_TOO_LARGE: {
anchor: 'review-diff-too-large',
summary: 'one reviewer\'s diff is over the 2,000,000-byte review ceiling, so /ship stopped before writing a manifest or a brief instead of handing a reader more than it can read',
fix: 'follow the fix the line prints for where the base policy stands: land the printed review_exclude_paths JSON on the base as its own PR, then bring the base into the branch and rerun once; or narrow a pattern the kept-file bound skipped; or split the PR',
},
REVIEW_DIFF_UNREADABLE: {
anchor: 'review-diff-unreadable',
summary: 'git could not read a reviewer\'s diff (reason=timeout after timeout_ms, or reason=git-exit with git\'s exit code), so /ship stopped; the full stderr, with remote URLs redacted, is in the log= file',
fix: 'timeout: rerun once unchanged, then split the PR or exclude its largest data folders on the base; git exit 128: git fetch origin <base>, then rerun once; anything else: stop and ask a person',
},
REVIEW_EXCLUDE_REFERENCE: {
anchor: 'review-exclude-reference',
summary: 'informational: a kept code file names an excluded folder (a directory reference, not a full file path), so its files stay excluded and readers see that code without the files it reads',
fix: 'nothing to do for the review; if that code\'s correctness depends on the excluded files, narrow review_exclude_paths on the base so they stay in the diff',
},
} as const satisfies Record<string, ResultCode>;
+211
View File
@@ -0,0 +1,211 @@
/**
* review-bots: the PR review-bot comments /review Step 2.5 and /ship Step 10 triage (#3122).
*
* The bots are the two gstack-pr-wait lists (`pr_wait_gating_bots`, `pr_wait_advisory_bots`;
* `none` in both turns triage off), owned by login with gstack-pr-wait's rule (`botOwning`),
* and only Bot accounts count, so a human named `greptile-ci` is never triaged.
*
* fetch [--pr <n>] every root review comment, conversation comment and the newest review
* body of each configured bot, all pages; prints REVIEW_BOTS: skip|error
* (<reason>) or REVIEW_BOTS_DIR: <dir> with comments.jsonl and the counts
* record --dir D --id N --type fix|fp|already-fixed --category C
* append a triage outcome to review-bot-history.md (project and global); the bot
* and path come from comment N in D/comments.jsonl, never from a shell argument,
* so a contributor-named file cannot reach the command line
*
* fetch marks each record with the false-positive rows that could suppress it: same repo, same
* bot, matching file. Rows in the old greptile-history.md count as Greptile's.
* bin/gstack-review-bots is the CLI; tests drive `runReviewBots` with a fake runner.
*/
import * as fs from "fs";
import * as path from "path";
import { botLists, botOwning } from "./pr-wait";
import { remoteSlug } from "./remote-identity";
import { resolveStateRoot, type StateRootEnv } from "./state-root";
export interface ReviewBotsEnv {
run(cmd: string, args: string[]): { status: number; stdout: string; stderr: string };
vars: StateRootEnv;
/** Directory holding gstack-slug. */
binDir: string;
/** Parent directory for the private fetch directory. */
tmpDir: string;
now(): number;
out(text: string): void;
err(text: string): void;
}
export const HISTORY_FILE = "review-bot-history.md";
export const LEGACY_HISTORY_FILE = "greptile-history.md";
export const HISTORY_TYPES = ["fix", "fp", "already-fixed"] as const;
export interface BotComment {
id: number;
bot: string;
source: "line" | "top" | "review";
path?: string;
line?: number | null;
url: string;
body: string;
suppress_candidates: string[];
}
interface GhUser { login?: string; type?: string }
interface GhItem {
id: number; user?: GhUser | null; body?: string | null; html_url?: string; path?: string;
line?: number | null; position?: number | null; in_reply_to_id?: number | null; submitted_at?: string | null; subject_type?: string | null;
}
interface HistoryRow { repo: string; type: string; pattern: string; category: string; bot: string }
const firstLine = (text: string) => text.trim().split("\n")[0] || "no detail";
/** gh's answers when the checkout has no GitHub remote at all; any other repo lookup failure is an error, never a skip. */
const NO_GITHUB_REMOTE = /no git remotes found|none of the git remotes configured for this repository point to a known GitHub host|not a git repository/i;
/** The issue types a triage outcome may carry (review/greptile-triage.md, Suppressions Check). */
export const CATEGORIES = ["race-condition", "null-check", "error-handling", "style", "type-safety", "security", "performance", "correctness", "other"] as const;
/** The bot that owns an item's author, or null for humans, other bots and deleted accounts. */
function ownerOf(item: GhItem, bots: readonly string[]): string | null {
return item.user?.type === "Bot" ? botOwning(item.user.login, bots) : null;
}
function slug(env: ReviewBotsEnv): string {
const s = env.run(path.join(env.binDir, "gstack-slug"), ["--get", "SLUG"]);
if (s.status === 0 && s.stdout.trim()) return s.stdout.trim();
const origin = env.run("git", ["remote", "get-url", "origin"]);
return origin.status === 0 ? remoteSlug(origin.stdout.trim()) : "";
}
function historyFiles(env: ReviewBotsEnv, name: string): string[] {
const root = resolveStateRoot(env.vars);
const id = slug(env);
return [...(id ? [path.join(root, "projects", id, name)] : []), path.join(root, name)];
}
/** Rows of one history file; a five-column row is Greptile's (the old format). */
export function parseHistory(text: string): HistoryRow[] {
return text.split("\n").flatMap(line => {
const cells = line.split(" | ").map(c => c.trim());
if (cells.length !== 5 && cells.length !== 6) return [];
const [, repo, type, pattern, category, bot] = cells as [string, string, string, string, string, string?];
return repo && type && pattern && category ? [{ repo, type, pattern, category, bot: (bot || "greptile").toLowerCase() }] : [];
});
}
/** An exact path, or a glob where `*` stays inside one directory and `**` crosses them. */
export function patternMatches(pattern: string, file: string): boolean {
if (pattern === file) return true;
const re = pattern.split("**").map(part => part.split("*").map(s => s.replace(/[.+?^${}()|[\]\\]/g, "\\$&")).join("[^/]*")).join(".*");
return new RegExp(`^${re}$`).test(file);
}
function readOr(file: string): string {
try { return fs.readFileSync(file, "utf8"); } catch { return ""; }
}
/** All pages of one REST list, one item per line (no --slurp: gh before 2.48 lacks it), or the reason it could not be read whole. */
function pages(env: ReviewBotsEnv, endpoint: string): GhItem[] | string {
const r = env.run("gh", ["api", "--paginate", `${endpoint}?per_page=100`, "--jq", ".[] | tojson"]);
if (r.status !== 0) return `${endpoint}: ${firstLine(r.stderr)}`;
try {
const items: unknown[] = r.stdout.split("\n").filter(l => l.trim()).map(l => JSON.parse(l));
if (!items.every(i => i !== null && typeof i === "object" && !Array.isArray(i))) return `${endpoint}: not a list of objects`;
return items as GhItem[];
} catch { return `${endpoint}: invalid JSON`; }
}
function fetch(env: ReviewBotsEnv, args: string[]): number {
const skip = (reason: string) => { env.out(`REVIEW_BOTS: skip (${reason})\n`); return 0; };
const error = (reason: string) => { env.out(`REVIEW_BOTS: error (${reason})\n`); return 0; };
const lists = botLists(env.vars);
const bots = [...new Set([...lists.gating, ...lists.advisory])];
if (bots.length === 0) return skip("pr_wait_gating_bots and pr_wait_advisory_bots are both none");
if (env.run("gh", ["--version"]).status !== 0) return skip("gh not installed");
if (env.run("gh", ["auth", "status"]).status !== 0) return skip("gh not logged in");
const repoRun = env.run("gh", ["repo", "view", "--json", "nameWithOwner", "--jq", ".nameWithOwner"]);
const repo = repoRun.stdout.trim();
if (repoRun.status !== 0 || !repo) {
const why = firstLine(repoRun.stderr);
return NO_GITHUB_REMOTE.test(repoRun.stderr) ? skip(`not a GitHub remote: ${why}`) : error(`repository lookup failed: ${why}`);
}
const i = args.indexOf("--pr");
let pr = i >= 0 ? args[i + 1] ?? "" : "";
if (!pr) {
const view = env.run("gh", ["pr", "view", "--json", "number", "--jq", ".number"]);
if (view.status !== 0) return /no pull requests found/.test(view.stderr) ? skip("no PR yet") : error(`PR lookup failed: ${firstLine(view.stderr)}`);
pr = view.stdout.trim();
}
if (!/^[1-9][0-9]*$/.test(pr)) return error(`'${pr}' is not a PR number`);
const line = pages(env, `repos/${repo}/pulls/${pr}/comments`);
const top = pages(env, `repos/${repo}/issues/${pr}/comments`);
const reviews = pages(env, `repos/${repo}/pulls/${pr}/reviews`);
for (const r of [line, top, reviews]) if (typeof r === "string") return error(r);
const records: BotComment[] = [];
const add = (item: GhItem, bot: string, source: BotComment["source"], extra: Partial<BotComment> = {}) =>
records.push({ id: item.id, bot, source, url: item.html_url ?? "", body: item.body ?? "", suppress_candidates: [], ...extra });
for (const c of line as GhItem[]) {
const bot = ownerOf(c, bots);
if (bot && c.in_reply_to_id == null && (c.position != null || c.subject_type === "file")) add(c, bot, "line", { path: c.path, line: c.line ?? null });
}
for (const c of top as GhItem[]) {
const bot = ownerOf(c, bots);
if (bot) add(c, bot, "top");
}
const newest = new Map<string, GhItem>();
for (const r of reviews as GhItem[]) {
const bot = ownerOf(r, bots);
if (bot && (r.body ?? "").trim() && (r.submitted_at ?? "") >= (newest.get(bot)?.submitted_at ?? "")) newest.set(bot, r);
}
for (const [bot, r] of newest) add(r, bot, "review");
const id = slug(env);
const rows = id ? [HISTORY_FILE, LEGACY_HISTORY_FILE].flatMap(name => {
const rows = parseHistory(readOr(path.join(resolveStateRoot(env.vars), "projects", id, name)));
return name === LEGACY_HISTORY_FILE ? rows.map(r => ({ ...r, bot: "greptile" })) : rows;
}) : [];
for (const rec of records) {
if (!rec.path) continue;
rec.suppress_candidates = [...new Set(rows.filter(r => r.type === "fp" && r.repo === repo && r.bot === rec.bot && patternMatches(r.pattern, rec.path!)).map(r => r.category))];
}
const dir = fs.mkdtempSync(path.join(env.tmpDir, "gstack-review-bots-"));
fs.chmodSync(dir, 0o700);
fs.writeFileSync(path.join(dir, "comments.jsonl"), records.map(r => JSON.stringify(r) + "\n").join(""), { mode: 0o600 });
env.out(`REVIEW_BOTS_DIR: ${dir}\n`);
env.out(`REVIEW_BOTS_COUNTS: ${bots.map(b => `${b}=${records.filter(r => r.bot === b).length}`).join(" ")}\n`);
env.out(`REVIEW_BOTS_SUPPRESSION_CANDIDATES: ${records.filter(r => r.suppress_candidates.length > 0).length}\n`);
return 0;
}
function record(env: ReviewBotsEnv, args: string[]): number {
const flag = (name: string) => { const i = args.indexOf(name); return i >= 0 ? args[i + 1] : undefined; };
const [dir, id, type, category] = [flag("--dir"), flag("--id"), flag("--type"), flag("--category")];
const usage = "usage: gstack-review-bots record --dir <REVIEW_BOTS_DIR> --id <comment id> --type fix|fp|already-fixed --category <category>\n";
if (!dir || !id || !type || !category) { env.err(usage); return 64; }
if (!/^[1-9][0-9]*$/.test(id)) { env.err("gstack-review-bots: --id must be a comment id from comments.jsonl\n"); return 64; }
if (!(HISTORY_TYPES as readonly string[]).includes(type)) { env.err(`gstack-review-bots: --type must be one of ${HISTORY_TYPES.join(", ")}\n`); return 64; }
if (!(CATEGORIES as readonly string[]).includes(category)) { env.err(`gstack-review-bots: --category must be one of ${CATEGORIES.join(", ")}\n`); return 64; }
const jsonl = path.join(dir, "comments.jsonl");
if (!fs.existsSync(jsonl)) { env.err(`gstack-review-bots: ${jsonl} is gone; run fetch again and record by the new ids\n`); return 1; }
let matches: BotComment[];
try { matches = readOr(jsonl).split("\n").filter(Boolean).map(l => JSON.parse(l) as BotComment).filter(c => String(c.id) === id); }
catch { env.err(`gstack-review-bots: ${jsonl} is not valid JSONL; run fetch again\n`); return 1; }
if (matches.length !== 1) { env.err(`gstack-review-bots: ${matches.length ? "more than one" : "no"} comment ${id} in ${jsonl}\n`); return 1; }
const [bot, file] = [matches[0]!.bot, matches[0]!.path || "top-level"];
if (/[|*\r\n]/.test(file)) { env.err("gstack-review-bots: the comment's path contains |, * or a line break, so it cannot be a literal history row\n"); return 1; }
const repoRun = env.run("gh", ["repo", "view", "--json", "nameWithOwner", "--jq", ".nameWithOwner"]);
const repo = repoRun.status === 0 ? repoRun.stdout.trim() : "";
if (!repo) { env.err(`gstack-review-bots: cannot resolve the GitHub repository: ${firstLine(repoRun.stderr)}\n`); return 1; }
const row = `${new Date(env.now()).toISOString().slice(0, 10)} | ${repo} | ${type} | ${file} | ${category} | ${bot.toLowerCase()}\n`;
for (const target of historyFiles(env, HISTORY_FILE)) {
fs.mkdirSync(path.dirname(target), { recursive: true });
fs.appendFileSync(target, row);
}
env.out(`REVIEW_BOT_HISTORY: recorded ${type} for ${bot.toLowerCase()}\n`);
return 0;
}
export function runReviewBots(argv: string[], env: ReviewBotsEnv): number {
const [cmd, ...rest] = argv;
if (cmd === "fetch") return fetch(env, rest);
if (cmd === "record") return record(env, rest);
env.err("usage: gstack-review-bots {fetch [--pr <n>] | record --dir D --id N --type fix|fp|already-fixed --category C}\n");
return cmd === "help" || cmd === "--help" ? 0 : 64;
}
+17 -7
View File
@@ -13,6 +13,11 @@
* outside-voice diff_base, diff (the branch diff, which the bin reads from git)
* fix-check fix_list, hunks, tests, callers (scripts/resolvers/fix-check.ts)
* targeted classes, brief (review/specialists/targeted.md's text), checklist (its path), diff_base, stack, test_fw, learnings
*
* With the base policy's review_exclude_paths active (lib/review-exclude.ts) every role but fix-check also takes
* policy_exclude (the `:(top,exclude,glob)` items, which the fixture-handling command of the security specialist, the
* Red Team and the native adversarial pass gets), keep (the kept files' `:(top,literal)` items) and disclosure (the one
* line naming what was left out). Unset, every brief renders byte for byte as before.
*/
import { spawnSync } from 'node:child_process';
import { createHash, randomUUID } from 'node:crypto';
@@ -81,7 +86,7 @@ function fill(text: string, pairs: Array<[string, string]>, what: string): strin
}
const SPECIALIST_DIFF = 'run\n`DIFF_BASE=<diff-base> && git diff "$DIFF_BASE" <exclude>` to get the diff';
const PLACEHOLDERS = ['<diff-base>', '<exclude>', '<base>', '{checklist path}', '{STACK}', '{TEST_FW}', '{merged findings summary}', '{red-team checklist path}', '{learnings'];
const PLACEHOLDERS = ['<diff-base>', '<exclude>', '<list-exclude>', '<base>', '{checklist path}', '{STACK}', '{TEST_FW}', '{merged findings summary}', '{red-team checklist path}', '{learnings'];
/**
* Render one reader's brief. Throws BriefError on a missing slot, a section that no longer carries an anchor or placeholder, or a placeholder left unfilled.
@@ -94,7 +99,9 @@ export function renderBrief(role: BriefRole, slots: Slots, t: SectionTexts): str
const keys = ['findings', 'learnings'].filter(k => slots[k]?.trim());
const held: Slots = { ...slots };
for (const k of keys) held[k] = `\u0000${nonce}:${k}\u0000`;
const text = renderRole(role, held, t);
const rendered = renderRole(role, held, t);
const keep = slots.keep ? ` Read the files kept despite a match as well: \`git diff ${slots.diff_base} -- ${slots.keep}\`.` : '';
const text = slots.disclosure && role !== 'fix-check' && role !== 'outside-voice' ? `${rendered}\n\n${slots.disclosure}${keep}` : rendered;
const left = role === 'fix-check' || role === 'outside-voice' ? [] : PLACEHOLDERS.filter(p => text.includes(p));
if (left.length) throw new BriefError(`${role}: unfilled placeholder ${left.join(', ')}`);
return keys.reduce((out, k) => out.split(held[k]!).join(slots[k]!.trim()), text);
@@ -103,6 +110,8 @@ export function renderBrief(role: BriefRole, slots: Slots, t: SectionTexts): str
function renderRole(role: BriefRole, slots: Slots, t: SectionTexts): string {
const diffBase = role === 'fix-check' ? '' : need(slots, 'diff_base');
if (diffBase && !SHA.test(diffBase)) throw new BriefError(`slot diff_base must be a literal commit sha, got ${JSON.stringify(diffBase)}`);
const fixtures = (text: string) => text.split(' <exclude>`').join(slots.policy_exclude ? ` ${slots.policy_exclude}\`` : '`')
.split('<list-exclude>`').join(slots.policy_exclude ? ` -- . ${slots.policy_exclude}\`` : '`');
const common = (body: string, checklist: string) => fill(body, [
...(checklist ? [['{checklist path}', checklist] as [string, string]] : []), ['<diff-base>', diffBase],
['{TEST_FW}', slots.test_fw || 'unknown'], ['{STACK}', slots.stack || 'unknown'], ['{learnings or \'none\'}', slots.learnings?.trim() || 'none'],
@@ -112,7 +121,7 @@ function renderRole(role: BriefRole, slots: Slots, t: SectionTexts): string {
const name = need(slots, 'name');
const checklist = need(slots, 'checklist');
if (name !== 'security') return common(t.specialist, checklist);
const body = fill(t.specialist, [[SPECIALIST_DIFF, 'set `DIFF_BASE=<diff-base>` first. ' + t.fixtures + ' Then read the diff that way']], role);
const body = fill(t.specialist, [[SPECIALIST_DIFF, 'set `DIFF_BASE=<diff-base>` first. ' + fixtures(t.fixtures) + ' Then read the diff that way']], role);
return `${t.framing}\n\n${common(body, checklist)}`;
}
case 'specialist-batched': {
@@ -123,13 +132,14 @@ function renderRole(role: BriefRole, slots: Slots, t: SectionTexts): string {
case 'red-team': {
const body = fill(t.redTeam, [['{merged findings summary}', (slots.findings ?? '').trim() || '(none)'], ['by N specialists', `by ${need(slots, 'specialists')} specialists`],
['{red-team checklist path}', need(slots, 'checklist')], ['<diff-base>', diffBase]], role);
return `${t.framing}\n\n${t.fixtures}\n\n${body}`;
return fixtures(`${t.framing}\n\n${t.fixtures}\n\n${body}`);
}
case 'adversarial-native':
return fill(t.native, [['DIFF_BASE=$(git merge-base origin/<base> HEAD)', `DIFF_BASE=${diffBase}`]], role);
return fixtures(fill(t.native, [['DIFF_BASE=$(git merge-base origin/<base> HEAD)', `DIFF_BASE=${diffBase}`]], role));
case 'outside-voice': {
const body = fill(t.outside, [['DIFF_BASE=$(git merge-base origin/<base> HEAD)', `DIFF_BASE=${diffBase}`]], role);
return [body, '', `## Branch diff (git diff ${diffBase})`, FENCE + 'diff', (slots.diff ?? '').trim(), FENCE].join('\n');
const filled = fill(t.outside, [['DIFF_BASE=$(git merge-base origin/<base> HEAD)', `DIFF_BASE=${diffBase}`]], role);
const body = slots.policy_exclude ? filled.split('git diff "$DIFF_BASE".').join(`git diff "$DIFF_BASE" -- . ${slots.policy_exclude}${slots.keep ? ` and git diff "$DIFF_BASE" -- ${slots.keep}` : ''}.`) : filled;
return [body, '', ...(slots.disclosure ? [slots.disclosure, ''] : []), `## Branch diff (git diff ${diffBase})`, FENCE + 'diff', (slots.diff ?? '').trim(), FENCE].join('\n');
}
case 'targeted': {
const classes = parseClassList(need(slots, 'classes'));
+82 -13
View File
@@ -37,6 +37,14 @@ function git(root: string, args: string[]): { ok: boolean; status: number | null
const r = spawnSync('git', ['-c', 'core.quotePath=false', ...args], { cwd: root, encoding: 'utf8', timeout: 60_000, maxBuffer: 256 * 1024 * 1024 });
return { ok: r.status === 0, status: r.status, out: r.stdout ?? '', err: (r.stderr ?? '').trim() };
}
/** git with what is left of an overall deadline (epoch ms) as its timeout, for a read that may take minutes. */
function gitUntil(root: string, args: string[], deadline: number): { status: number | null; out: string; err: string; timedOut: boolean } {
const left = deadline - Date.now();
if (left <= 0) return { status: null, out: '', err: '', timedOut: true };
const r = spawnSync('git', ['-c', 'core.quotePath=false', ...args], { cwd: root, encoding: 'utf8', timeout: left, maxBuffer: 256 * 1024 * 1024 });
const status = r.status ?? null;
return { status, out: r.stdout ?? '', err: (r.stderr ?? '').trim() || ((r.error as NodeJS.ErrnoException | undefined)?.code ?? ''), timedOut: status === null && Date.now() >= deadline };
}
const scratch = (p: string) => p.startsWith('.gstack/tmp/');
const oneLine = (s: string) => s.replace(/[\0-\x1f]+/g, ' ').trim().slice(0, 200);
const unavailable = (input: string, detail: string): RegistryInfo => ({ sha: null, outputs: [], error: { code: 'registry_unavailable', detail: oneLine(detail), input, repair: 'fetch the base branch (git fetch origin <base>) so its commit and registry resolve' } });
@@ -101,36 +109,96 @@ function readDiff(root: string, range: string[], errors: DepthError[]): Pick<Col
export const CONSUMER_SCAN_CAP = 2000;
/** How many data files deep a chain of data naming data is followed (a manifest naming a data file) before the rest count as consumed. */
export const CONSUMER_SCAN_DEPTH = 4;
/**
* One consumer scan's whole budget, every git grep in it (GSTACK_CONSUMER_SCAN_TIMEOUT_MS overrides it); past it the scan fails
* closed. One scan stays a minute under the 180 s gstack-review-dispatch and gstack-ship-preflight give all of gstack-review-plan;
* a plan that also scans a fix range (--range-base) runs a second scan and can still reach that limit, which fails the plan.
*/
export const CONSUMER_SCAN_TIMEOUT_MS = 120_000;
/** Pathspec bytes per git grep call: a long list of files to search splits across calls instead of growing one command line. */
export const CONSUMER_SCAN_PATHSPEC_BYTES = 16_000;
/** Output bytes one consumer grep may return across all its calls, the buffer the single whole-tree call had; past it the scan fails closed. */
export const CONSUMER_SCAN_OUTPUT_BYTES = 256 * 1024 * 1024;
export function consumerScanTimeoutMs(): number {
const v = Number(process.env.GSTACK_CONSUMER_SCAN_TIMEOUT_MS);
return Number.isFinite(v) && v >= 1 ? Math.floor(v) : CONSUMER_SCAN_TIMEOUT_MS;
}
export type GrepFailure = { timedOut: boolean; status: number | null; err: string };
/**
* The lines `git grep -I -F -o --null` prints for `patterns` at `rev` ('' is the working tree with untracked files), for
* the files `keep` accepts, in some order. Every pattern holds its own extension, so a `-l` pass for the extensions alone
* lists every file that can match; the full pattern list then reads only those files, CONSUMER_SCAN_PATHSPEC_BYTES of
* paths per call under --literal-pathspecs, CONSUMER_SCAN_OUTPUT_BYTES of output in all (an inherited GIT_GLOB_PATHSPECS makes git refuse, which fails closed). A listed name
* that is not valid UTF-8 cannot be passed back as a path, so then one grep reads the whole tree, as the scan did before GST-544. Each call gets what is left of `deadline` (epoch ms).
*/
export function grepConsumers(root: string, patterns: readonly string[], rev: string, keep: (file: string) => boolean, deadline: number, chunkBytes = CONSUMER_SCAN_PATHSPEC_BYTES, outputBytes = CONSUMER_SCAN_OUTPUT_BYTES): string[] | GrepFailure {
const where = rev ? [rev, '--'] : ['--untracked', '--'];
const exts = [...new Set(patterns.map(b => b.slice(b.lastIndexOf('.'))))].flatMap(e => ['-e', e]);
const listed = gitUntil(root, ['grep', '-I', '-F', '-l', '-z', '--no-color', ...exts, ...where], deadline);
if (listed.status !== 0 && listed.status !== 1) return listed;
const names = listed.out.split('\0').filter(Boolean).map(f => (rev ? f.slice(rev.length + 1) : f));
const flags = ['--literal-pathspecs', 'grep', '-I', '-F', '-o', '--null', '--no-line-number', '--no-column', '--no-color', ...patterns.flatMap(b => ['-e', b]), ...where];
if (names.some(f => f.includes('\uFFFD'))) {
const whole = gitUntil(root, flags, deadline);
return whole.status !== 0 && whole.status !== 1 ? whole : whole.out.split('\n');
}
const files = names.filter(keep);
const lines: string[] = [];
let total = 0;
for (let i = 0; i < files.length;) {
const chunk: string[] = [];
for (let bytes = 0; i < files.length; i++) {
const spec = files[i]!;
if (chunk.length && bytes + spec.length > chunkBytes) break;
chunk.push(spec);
bytes += spec.length;
}
const r = gitUntil(root, [...flags, ...chunk], deadline);
if (r.status !== 0 && r.status !== 1) return r;
total += Buffer.byteLength(r.out);
if (total > outputBytes) return { status: null, err: 'ENOBUFS', timedOut: false };
for (const line of r.out.split('\n')) lines.push(line);
}
return lines;
}
/**
* The data candidates (lib/review-depth.ts dataCandidate) among the changed and deleted paths that a non-docs file names
* by basename, at each rev and, with `live`, in the working tree with untracked files. The merge base is one of the revs,
* so a consumer the change removes still counts. A data file named only by another data file is consumed when that one
* is, followed CONSUMER_SCAN_DEPTH files deep; anything still open after that counts as consumed. A git grep failure is
* a collector error, which fails the depth closed.
* is, followed CONSUMER_SCAN_DEPTH files deep; anything still open after that counts as consumed. A git grep failure, or
* a scan that outlives `timeoutMs`, is a collector error, which fails the depth closed.
*/
export function scanConsumers(root: string, paths: readonly string[], revs: readonly string[], live: boolean, classRoot: string | readonly string[], errors: DepthError[]): string[] | 'all' {
export function scanConsumers(root: string, paths: readonly string[], revs: readonly string[], live: boolean, classRoot: string | readonly string[], errors: DepthError[], timeoutMs = consumerScanTimeoutMs()): string[] | 'all' {
const candidates = [...new Set(paths)].filter(p => dataCandidate(p, classRoot) || svgCandidate(p, classRoot)).sort();
if (candidates.length > CONSUMER_SCAN_CAP) return 'all';
const deadline = Date.now() + timeoutMs;
const base = (p: string) => p.slice(p.lastIndexOf('/') + 1);
const read = (file: string) => !scratch(file) && classifyPath(file, classRoot) !== 'docs';
const named = new Map<string, Set<string>>();
const direct = new Set<string>();
const probed = new Set<string>();
let probe = candidates;
for (let depth = 0; probe.length && depth < CONSUMER_SCAN_DEPTH; depth++) {
const patterns = [...new Set(probe.map(base))].flatMap(b => ['-e', b]);
const runs = [...revs.map(rev => ({ at: rev, args: [rev, '--'] })), ...(live ? [{ at: '', args: ['--untracked', '--'] }] : [])];
const patterns = [...new Set(probe.map(base))];
const next = new Set<string>();
for (const run of runs) {
const r = git(root, ['grep', '-I', '-F', '-o', '--null', '--no-line-number', '--no-column', '--no-color', ...patterns, ...run.args]);
if (r.status !== 0 && r.status !== 1) { errors.push({ code: 'consumer_scan_failed', detail: oneLine(r.err || `git grep exited ${r.status}`), input: `git grep -F <data basenames> ${run.at || '--untracked'}`, repair: 'run the git grep by hand and fix what git reports' }); return 'all'; }
for (const line of r.out.split('\n')) {
for (const at of [...revs, ...(live ? [''] : [])]) {
const r = grepConsumers(root, patterns, at, read, deadline);
if (!Array.isArray(r)) {
const detail = r.timedOut ? `timed out: the scan's ${timeoutMs / 1000} s deadline passed (GSTACK_CONSUMER_SCAN_TIMEOUT_MS)` : `git grep exited ${r.status}${r.err ? `: ${r.err}` : ''} (deadline ${timeoutMs / 1000} s)`;
errors.push({ code: 'consumer_scan_failed', detail: oneLine(detail), input: `git grep -F <data basenames> ${at || '--untracked'}`, repair: r.timedOut ? 'raise GSTACK_CONSUMER_SCAN_TIMEOUT_MS (milliseconds) or run the git grep by hand to see why it is slow' : 'run the git grep by hand and fix what git reports' });
return 'all';
}
for (const line of r) {
const parts = line.split('\0');
const where = parts[0];
const match = parts.length > 1 ? parts[parts.length - 1] : undefined;
if (!where || !match) continue;
const file = run.at ? where.slice(run.at.length + 1) : where;
if (scratch(file) || classifyPath(file, classRoot) === 'docs') continue;
const file = at ? where.slice(at.length + 1) : where;
if (!read(file)) continue;
for (const p of probe) {
if (file === p || !match.includes(base(p))) continue;
if (!dataCandidate(file, classRoot)) { direct.add(p); continue; }
@@ -187,10 +255,11 @@ export function collectCommit(root: string, baseSha: string, headSha: string): C
return result(diff, { root: skillDirs, registry, errors, mergeBase, untracked: [], consumed, drawings });
}
/** The working tree against the merge base with origin/<base> (or <base>), untracked files counted per path; the registry is the base ref's commit. */
/** The working tree against the merge base with origin/<base> (or <base>; a remote-qualified base reads refs/remotes/<base> only), untracked files counted per path; the registry is the base ref's commit. */
export function collectLive(root: string, base: string): Collected {
const errors: DepthError[] = [];
const ref = [`refs/remotes/origin/${base}`, `refs/heads/${base}`].find(r => git(root, ['rev-parse', '--verify', '-q', `${r}^{commit}`]).ok) ?? base;
const qualified = base.startsWith('origin/') || git(root, ['remote']).out.split('\n').some(r => r && base.startsWith(`${r}/`));
const ref = (qualified ? [`refs/remotes/${base}`] : [`refs/remotes/origin/${base}`, `refs/heads/${base}`]).find(r => git(root, ['rev-parse', '--verify', '-q', `${r}^{commit}`]).ok) ?? base;
const registry = loadRegistryAt(root, ref);
const mb = git(root, ['merge-base', ref, 'HEAD']);
if (!mb.ok) {
+22 -6
View File
@@ -58,7 +58,12 @@ const GATE_PATH = [
/^lib\/review-(depth|collect|scope|targeted|risk)\.ts$/, /^review\/specialists\/targeted\.md$/, /^bin\/gstack-review-plan$/, /^lib\/ship-queue[^/]*\.ts$/, /^lib\/regen\.ts$/, /^\.gstack\/generated\.json$/, /^ship\/sections\/(review-army|adversarial)\.md\.tmpl$/, /^scripts\/resolvers\/review-army\.ts$/,
];
/** Data files: JSON, CSV, TSV and raster images. YAML and TOML are never data here: deploy configs and API specs stay code. An SVG is data only as a drawing (svgDrawing). */
/** The path rules lib/review-exclude.ts keeps a matched file for: a gate path, agent prose, or a data-shaped config file. */
export const isGatePath = (p: string) => GATE_PATH.some(r => r.test(p));
export const isAgentProse = (p: string) => AGENT_PROSE.some(r => r.test(p));
export const isDataConfig = (p: string) => DATA_CONFIG.some(r => r.test(p));
/** Data files: JSON, CSV, TSV and raster images. YAML and TOML are never data here: deploy configs and API specs stay code. An SVG is data only as a drawing (svgDrawing); executable SVG stays code. */
export const DATA_PATH = /\.(json|jsonl|ndjson|csv|tsv|png|jpe?g|gif|webp|bmp|ico|avif)$/i;
/** Data-shaped files tools read by convention, with no code naming them: always code. */
const DATA_CONFIG = [
@@ -222,13 +227,15 @@ export interface ClassInput {
/** Added and deleted lines per path (lib/review-collect.ts): the risk classes read them; without them no line signal applies. */
added?: Record<string, readonly string[]>;
removed?: Record<string, readonly string[]>;
/** Paths the base policy's review_exclude_paths leave out of every reader's diff (lib/review-exclude.ts): classified as docs. */
excluded?: ReadonlySet<string>;
}
export interface ClassResult { class: DiffClass; reasons: string[]; errors: DepthError[]; runtimeLines: number | null; risk: RiskClass[] }
const sanitize = (s: string) => s.replace(/[\0-\x1f]+/g, ' ').slice(0, 200);
const isOutput = (i: ClassInput, p: string) => !!i.registry?.outputs.length && matchesAny(i.registry.outputs, p);
const unconsumedData = (i: ClassInput, p: string) => Array.isArray(i.consumed) && !i.consumed.includes(p) && (dataCandidate(p, i.root) || (!!i.drawings?.includes(p) && svgCandidate(p, i.root)));
const pathClass = (i: ClassInput, p: string): PathClass => ((i.stampOnly ?? []).includes(p) || unconsumedData(i, p) ? 'docs' : classifyPath(p, i.root));
const pathClass = (i: ClassInput, p: string): PathClass => (i.excluded?.has(p) || (i.stampOnly ?? []).includes(p) || unconsumedData(i, p) ? 'docs' : classifyPath(p, i.root));
/** The diff's class, raised to gate by any gate signal; `errors` lists every input that fails the depth closed; `runtimeLines` counts code-class lines outside the base registry's outputs. */
export function classifyDiff(i: ClassInput): ClassResult {
@@ -250,7 +257,7 @@ export function classifyDiff(i: ClassInput): ClassResult {
raise('gate', 'gate scan unavailable');
errors.push({ code: 'gate_scan_unavailable', detail: 'gstack-gate-diff reported no GATE_SUMMARY', input: 'gstack-gate-diff <base>', repair: 'run gstack-gate-diff <base> and read its GATE_ERROR line, or pass the recorded scan as --gate-tagged <n>' });
} else if (i.gateTagged > 0) raise('gate', `gate scan tagged ${i.gateTagged} hunk(s)`);
const deletedGuarded = (i.deleted ?? []).filter(p => classifyPath(p, i.root) !== 'docs');
const deletedGuarded = (i.deleted ?? []).filter(p => !i.excluded?.has(p) && classifyPath(p, i.root) !== 'docs');
if (deletedGuarded.length) raise('gate', `deletes ${deletedGuarded.length} code or test path(s)`);
}
if (CLASS_RANK[cls] >= CLASS_RANK.code) {
@@ -412,9 +419,11 @@ export interface PlanInput extends ClassInput {
tiers?: boolean;
/** The targeted classes (lib/review-targeted.ts) and whether the targeted reader already read this product patch. */
targeted?: { classes: readonly string[]; ran?: boolean };
/** The base policy's exclusion: one `:(top,exclude,glob)` pathspec per applied pattern, added to every reader, and the kept files under them as `:(top,literal)` pathspecs. */
exclusion?: { specs: readonly string[]; keep: readonly string[] };
}
/** One reviewer. `exclude` holds git pathspec argv items the reviewer's diff leaves out; `checklists` names what the batched specialist reads. */
export interface Dispatch { role: Role; name: string; effort: Effort; exclude: string[]; checklists?: string[]; reason?: string }
export interface Dispatch { role: Role; name: string; effort: Effort; exclude: string[]; checklists?: string[]; reason?: string; keep?: string[] }
export interface ReviewPlan extends DepthResult {
/** The depth this step runs at: light raised to standard by a fixed blocker, a CRITICAL or an open blocker. */
stepDepth: Depth;
@@ -487,6 +496,11 @@ export function diffCommand(d: Dispatch): string {
return d.exclude.length ? `git diff <diff-base> -- . ${d.exclude.map(shellQuote).join(' ')}` : 'git diff <diff-base>';
}
/** The second command a reader runs when the base policy's exclusion keeps files its patterns match (a git exclude beats any positive pathspec); null otherwise. */
export function keepCommand(d: Dispatch): string | null {
return d.keep?.length ? `git diff <diff-base> -- ${d.keep.map(shellQuote).join(' ')}` : null;
}
/** The reviewers the next step dispatches at its depth, with each role's effort tier and excluded pathspecs. */
export function planReview(i: PlanInput): ReviewPlan {
const c = classifyDiff(i);
@@ -499,8 +513,10 @@ export function planReview(i: PlanInput): ReviewPlan {
const depthReasons = [...depth.reasons, ...(raise ? [`history:${h.fixedBlocking ? 'fixed-blocking' : h.blockingOpen ? 'blocking-open' : 'critical-found'}`] : [])];
const skipped: string[] = [];
const dispatches: Dispatch[] = [];
const add = (role: Role, name: string, effort: Effort, exclude: string[] = [], extra: Partial<Dispatch> = {}) => dispatches.push({ role, name, effort, exclude, ...extra });
const docsAndOutputs = i.paths.filter(p => (pathClass(i, p) === 'docs' && !(i.numbers.publishes && DOC_PATH.test(p))) || isOutput(i, p)).map(excludeSpec);
const x = i.exclusion;
const add = (role: Role, name: string, effort: Effort, exclude: string[] = [], extra: Partial<Dispatch> = {}) =>
dispatches.push({ role, name, effort, exclude: x ? [...exclude, ...x.specs] : exclude, ...extra, ...(x?.keep.length ? { keep: [...x.keep] } : {}) });
const docsAndOutputs = i.paths.filter(p => !i.excluded?.has(p) && ((pathClass(i, p) === 'docs' && !(i.numbers.publishes && DOC_PATH.test(p))) || isOutput(i, p))).map(excludeSpec);
const numbersChanged = i.numbers.publishes && i.reviewedNumbers !== undefined && i.reviewedNumbers !== i.numbers.fingerprint;
const firstVoices = h.fullPasses === 0 || !i.adversarialRan;
const specialist: Effort = i.tiers === false ? 'host' : ROLE_EFFORT.specialist;
+408
View File
@@ -0,0 +1,408 @@
/**
* review-exclude — the base policy's review_exclude_paths applied to one range, and the
* bounded read of each reviewer's diff (docs/ship-policy.md#review-exclude-paths, GST-508).
*
* The policy is read at the base commit the caller already resolved (loadPolicyAt), never
* from the head: a branch cannot exclude its own files. Git owns the glob dialect: every
* matched path comes from `git diff --raw -z` under one `:(top,glob)<pattern>` pathspec
* per pattern (untracked files never count), so the receipt, the plan line and what
* readers see are one set. A matched file is excluded only when it is
* content (data, docs, binary or a .log) that no rule keeps; every kept file carries one
* stable reason. Applied patterns become `:(top,exclude,glob)` pathspecs on every reader,
* and kept files under them come back through a second `:(top,literal)` command, because a
* git exclude beats any positive pathspec. That second command is bounded by its argument
* bytes: a pattern whose kept files would pass KEEP_ARG_MAX_BYTES is not applied.
*
* readSelection reads one reader's diff once, with a timeout, and counts its bytes;
* gstack-review-dispatch stops with REVIEW_DIFF_TOO_LARGE above REVIEW_DIFF_MAX_BYTES and
* with REVIEW_DIFF_UNREADABLE when git fails, before it writes a manifest or a brief.
*/
import { spawnSync } from 'node:child_process';
import { createHash } from 'node:crypto';
import * as fs from 'node:fs';
import * as os from 'node:os';
import * as path from 'node:path';
import { collectLive } from './review-collect';
import { DATA_PATH, classifyPath, isAgentProse, isDataConfig, isGatePath, type PathLines } from './review-depth';
import { secretLine } from './review-targeted';
import { cmpVersion, parseVersion } from './version-source';
import { REVIEW_EXCLUDE_KEY_VERSION, SHIP_POLICY_PATH, isUnavailable, policyUnavailableLine } from './ship-policy';
import { loadPolicyAt } from './ship-policy-at';
/** The most diff bytes one reader is handed (about 500K tokens); above it the pass stops instead of truncating. */
export const REVIEW_DIFF_MAX_BYTES = 2_000_000;
/** The most argument bytes the kept-file command may carry across all applied patterns (under Windows' 32,767-character command line). */
export const KEEP_ARG_MAX_BYTES = 24_000;
/** How long one reader's `git diff` may run. */
export const REVIEW_DIFF_TIMEOUT_MS = 120_000;
const MAX_BUFFER = 256 * 1024 * 1024;
export const KEPT_REASONS = ['policy-dir', 'gate-path', 'executable', 'agent-prose', 'config', 'gate-tagged', 'code', 'secret-line', 'named-by-code', 'consumer-scan-unavailable', 'gate-scan-unavailable', 'link'] as const;
export type KeptReason = (typeof KEPT_REASONS)[number];
/** What the receipt records and the queue and carry compare (by `digest`, never by `source`). */
export interface ExcludeRecord {
source: string; patterns: string[]; applied: string[]; skipped: Array<{ pattern: string; kept: number; bytes: number }>; unmatched: string[];
excluded: number; excluded_bytes: number; kept: number; gate_tagged: string[]; patterns_sha256: string; list_sha256: string; digest: string;
}
export interface ActiveExclusion {
state: 'active'; record: ExcludeRecord; excluded: Set<string>; specs: string[]; keep: string[];
kept: Array<{ path: string; reason: KeptReason }>; content: Record<ContentKind, number>; references: Array<{ prefix: string; files: string[] }>;
}
export type Exclusion =
| { state: 'none' }
| { state: 'head-only'; patterns: string[]; source: string }
| { state: 'refused'; code: 'POLICY_SOURCE_UNAVAILABLE' | 'POLICY_INVALID' | 'REVIEW_DIFF_UNREADABLE'; lines: string[] }
| ActiveExclusion;
export interface ExcludeInput {
root: string;
/** The base commit whose policy applies, and the ref it was resolved from (origin/<base> or <base>). */
baseSha: string; ref: string;
/** The merge base, and the head commit; no head reads the working tree with untracked files. */
mergeBase: string; head?: string;
/** The collector's per-path lines (binary flags) and added lines (lib/review-collect.ts). */
perPath: readonly PathLines[]; added: Record<string, readonly string[]>;
/** Skill directory names (commit mode) or the repository root (live): markdown in a skill directory is code. */
classRoot: string | readonly string[];
/** Paths whose hunks the gate scan tagged; null when unknown. */
gateTagged?: ReadonlySet<string> | null;
/** The command name the refusal lines start with. */
context: string;
}
const sha256 = (s: string) => createHash('sha256').update(s).digest('hex');
const shellQuote = (s: string) => `'${s.replace(/'/g, "'\\''")}'`;
const literal = (p: string) => `:(top,literal)${p}`;
const argBytes = (paths: Iterable<string>) => { let n = 0; for (const p of paths) n += Buffer.byteLength(shellQuote(literal(p))) + 1; return n; };
function git(root: string, args: string[], input?: string): { ok: boolean; status: number | null; out: string; err: string } {
const r = spawnSync('git', ['-c', 'core.quotePath=false', ...args], { cwd: root, input, encoding: 'utf8', timeout: 120_000, maxBuffer: MAX_BUFFER });
return { ok: r.status === 0, status: r.status, out: r.stdout ?? '', err: (r.stderr ?? '').trim() };
}
/** The patterns a policy text declares, or [] for anything unreadable: only a notice reads the head's copy. */
function declared(text: string | null): string[] {
try { const v = JSON.parse(text ?? 'null')?.review_exclude_paths; return Array.isArray(v) ? v.filter((p): p is string => typeof p === 'string') : []; } catch { return []; }
}
function headPatterns(i: ExcludeInput): string[] {
if (!i.head) { try { return declared(fs.readFileSync(path.join(i.root, SHIP_POLICY_PATH), 'utf8')); } catch { return []; } }
const r = git(i.root, ['show', `${i.head}:${SHIP_POLICY_PATH}`]);
return r.ok ? declared(r.out) : [];
}
/** A pattern's literal directory prefix (`receipts/` for `receipts/**`), or '' when it starts with a glob. */
export function literalPrefix(pattern: string): string {
const at = pattern.search(/[*?[]/);
const head = at < 0 ? pattern : pattern.slice(0, at);
return head.slice(0, head.lastIndexOf('/') + 1);
}
interface Matched { oldMode: string; newMode: string; oldSha: string; newSha: string }
/** Every changed path each pattern matches, with its modes, from git's raw diff; untracked files are never part of it, since no reader's git diff shows them and the receipt reads the committed head. */
function select(i: ExcludeInput, patterns: readonly string[]): { byPattern: Map<string, string[]>; files: Map<string, Matched> } | { error: string } {
const byPattern = new Map<string, string[]>();
const files = new Map<string, Matched>();
const range = i.head ? [i.mergeBase, i.head] : [i.mergeBase];
for (const p of patterns) {
const spec = `:(top,glob)${p}`;
const raw = git(i.root, ['diff', '--raw', '-z', '--no-abbrev', '--no-renames', ...range, '--', spec]);
if (!raw.ok) return { error: `git diff --raw ${range.join(' ')} -- ${spec}: ${raw.err.split('\n')[0] || `exit ${raw.status}`}` };
const parts = raw.out.split('\0');
const hit: string[] = [];
for (let k = 0; k + 1 < parts.length; k += 2) {
const [oldMode, newMode, oldSha, newSha] = parts[k]!.slice(1).split(' ');
const f = parts[k + 1]!;
if (!f) continue;
hit.push(f);
if (!files.has(f)) files.set(f, { oldMode: oldMode ?? '', newMode: newMode ?? '', oldSha: oldSha ?? '', newSha: newSha ?? '' });
}
byPattern.set(p, hit);
}
return { byPattern, files };
}
/**
* The matched paths git calls binary whose bytes hold a NUL in their first 8,000 (git's own test), read from the head blob,
* the file on a live tree, or the old blob for a deletion. A branch's .gitattributes can make git call a text file binary;
* only real bytes make it binary content here.
*/
function binaryContent(i: ExcludeInput, files: ReadonlyMap<string, Matched>, claimed: ReadonlySet<string>, unread: Set<string>): Set<string> {
const out = new Set<string>();
const real = (sha: string) => /^[0-9a-f]{40,64}$/.test(sha) && !/^0+$/.test(sha);
const bySha = new Map<string, string[]>();
for (const p of claimed) {
const m = files.get(p);
if (!m) continue;
if (!i.head && m.newMode !== '000000') {
try { const fd = fs.openSync(path.join(i.root, p), 'r'); const buf = Buffer.alloc(8000); const n = fs.readSync(fd, buf, 0, 8000, 0); fs.closeSync(fd); if (buf.subarray(0, n).includes(0)) out.add(p); } catch { unread.add(p); }
continue;
}
const sha = real(m.newSha) ? m.newSha : real(m.oldSha) ? m.oldSha : '';
if (sha) bySha.set(sha, [...(bySha.get(sha) ?? []), p]);
}
for (const [sha, paths] of bySha) {
const r = spawnSync('git', ['cat-file', 'blob', sha], { cwd: i.root, timeout: 30_000, maxBuffer: 8000 });
const read = r.status === 0 || (r.error as NodeJS.ErrnoException | undefined)?.code === 'ENOBUFS';
const binary = read && (r.stdout ?? Buffer.alloc(0)).subarray(0, 8000).includes(0);
for (const p of paths) if (!read) unread.add(p); else if (binary) out.add(p);
}
return out;
}
/** The kind of content a path holds when the policy may exclude it, or null for code (tests included). */
export type ContentKind = 'binary' | 'data' | 'docs' | 'log';
function contentKind(i: ExcludeInput, p: string, binary: ReadonlySet<string>): ContentKind | null {
if (binary.has(p)) return 'binary';
if (DATA_PATH.test(p) && !isDataConfig(p)) return 'data';
if (/\.log$/i.test(p)) return 'log';
return classifyPath(p, i.classRoot) === 'docs' ? 'docs' : null;
}
/** Source and script extensions: a NUL in one does not make it excludable binary content. */
const CODE_EXT = /\.(?:[cm]?[jt]sx?|py|rb|go|rs|java|kts?|scala|swift|exs?|php|cs|c|h|cc|cpp|hpp|mm?|dart|pl|pm|lua|sh|bash|zsh|fish|ps1|bat|cmd|vue|svelte|html?|wasm)$/i;
/** Why a matched file is kept by its path, mode and lines alone; null for content the policy may exclude. A binary file readers would only see as "Binary files differ" is content unless its directory, mode or gate tags keep it. */
function pathReason(i: ExcludeInput, p: string, m: Matched, binary: ReadonlySet<string>): KeptReason | null {
if (/^\.(gstack|github)\//.test(p)) return 'policy-dir';
if (isGatePath(p)) return 'gate-path';
if (m.oldMode === '100755' || m.newMode === '100755') return 'executable';
if ([m.oldMode, m.newMode].some(x => x === '120000' || x === '160000')) return 'link';
if (i.gateTagged?.has(p)) return 'gate-tagged';
if (binary.has(p)) return CODE_EXT.test(p) ? 'code' : null;
if (isAgentProse(p)) return 'agent-prose';
if (isDataConfig(p)) return 'config';
if (contentKind(i, p, binary) === null) return 'code';
if ((i.added[p] ?? []).some(secretLine)) return 'secret-line';
return null;
}
/** A file whose full-path reference keeps excluded content: a code or test file (not docs, data, a .log or binary) outside the would-be-excluded set. */
const consumer = (i: ExcludeInput, f: string, candidates: ReadonlySet<string>) =>
!candidates.has(f) && !f.startsWith('.gstack/tmp/') && contentKind(i, f, new Set()) === null;
/** An uncapped `git grep -F -o -f` of fixed strings at the head (or the working tree) and `revs`: string -> the consumer files holding it; null when git grep failed. */
function grepFixed(i: ExcludeInput, strings: readonly string[], candidates: ReadonlySet<string>, revs: readonly string[]): Map<string, Set<string>> | null {
const found = new Map<string, Set<string>>();
if (!strings.length) return found;
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-exclude-'));
try {
const list = path.join(dir, 'patterns');
fs.writeFileSync(list, strings.join('\n') + '\n');
const runs = [i.head ? { at: i.head, args: [i.head, '--'] } : { at: '', args: ['--'] }, ...revs.map(rev => ({ at: rev, args: [rev, '--'] }))];
for (const run of runs) {
const r = git(i.root, ['grep', '-I', '-F', '-o', '--null', '--no-line-number', '--no-column', '--no-color', '-f', list, ...run.args]);
if (r.status !== 0 && r.status !== 1) return null;
for (const line of r.out.split('\n')) {
const parts = line.split('\0');
const where = parts[0];
const match = parts.length > 1 ? parts[parts.length - 1] : undefined;
if (!where || !match) continue;
const file = run.at ? where.slice(run.at.length + 1) : where;
if (!consumer(i, file, candidates)) continue;
(found.get(match) ?? found.set(match, new Set()).get(match)!).add(file);
}
}
return found;
} finally { fs.rmSync(dir, { recursive: true, force: true }); }
}
/**
* The base policy's exclusion for one range. `none` when neither side declares a pattern (callers then print and change
* nothing); `head-only` when only the head does (a notice, never applied); `refused` when the base policy cannot be read or
* does not validate, or git cannot list the matches; `active` otherwise. Named-by-code keeps a file only when a kept code
* file names its full repository path (never its basename: receipt trees reuse result.json thousands of times); a git grep
* failure keeps every candidate (consumer-scan-unavailable), and so does an unknown gate scan (gate-scan-unavailable).
*/
export function reviewExclude(i: ExcludeInput): Exclusion {
const load = loadPolicyAt(i.root, i.baseSha, i.ref);
if (isUnavailable(load)) return { state: 'refused', code: 'POLICY_SOURCE_UNAVAILABLE', lines: [policyUnavailableLine(i.context, load.base, load.detail)] };
const source = `${i.ref}@${i.baseSha.slice(0, 12)}`;
const base = i.ref.replace(/^origin\//, '');
if (load.errors.length) {
return { state: 'refused', code: 'POLICY_INVALID', lines: load.errors.map(e => `${i.context}: the base policy ${source}:${SHIP_POLICY_PATH} does not validate, so its review_exclude_paths cannot be read (${e.path || SHIP_POLICY_PATH}: ${e.message}); fix: run ` + '`' + `gstack-ship-policy validate --policy-from base --base ${base}` + '`' + ` and fix the file on ${base} with its own PR; nothing was planned (${e.code})`) };
}
const patterns = load.policy.review_exclude_paths;
if (!patterns.length) { const head = headPatterns(i); return head.length ? { state: 'head-only', patterns: head, source } : { state: 'none' }; }
const sel = select(i, patterns);
if ('error' in sel) return { state: 'refused', code: 'REVIEW_DIFF_UNREADABLE', lines: [`${i.context}: git could not list the files review_exclude_paths matches (${sel.error}); fix: git fetch origin ${base}, then rerun once; nothing was planned (REVIEW_DIFF_UNREADABLE)`] };
const unread = new Set<string>();
const binary = binaryContent(i, sel.files, new Set(i.perPath.filter(e => e.binary && sel.files.has(e.path)).map(e => e.path)), unread);
const reasons = new Map<string, KeptReason | null>();
for (const [p, m] of sel.files) reasons.set(p, unread.has(p) ? 'consumer-scan-unavailable' : pathReason(i, p, m, binary));
const candidates = new Set([...reasons].filter(([, r]) => r === null).map(([p]) => p));
const named = grepFixed(i, [...candidates].filter(p => !p.includes('\n')), candidates, [i.mergeBase]);
for (const p of candidates) {
if (i.gateTagged === null) reasons.set(p, 'gate-scan-unavailable');
else if (named === null || p.includes('\n')) reasons.set(p, 'consumer-scan-unavailable');
else if (named.has(p)) reasons.set(p, 'named-by-code');
}
const applied: string[] = [];
const skipped: ExcludeRecord['skipped'] = [];
const keep = new Set<string>();
for (const p of patterns) {
const hit = sel.byPattern.get(p) ?? [];
if (!hit.some(f => reasons.get(f) === null)) continue;
const kept = hit.filter(f => reasons.get(f) !== null);
const bytes = argBytes(new Set([...keep, ...kept]));
if (bytes > KEEP_ARG_MAX_BYTES) { skipped.push({ pattern: p, kept: kept.length, bytes }); continue; }
applied.push(p);
for (const f of kept) keep.add(f);
}
const excluded = new Set(applied.flatMap(p => (sel.byPattern.get(p) ?? []).filter(f => reasons.get(f) === null)));
const keptList = [...reasons].filter(([, r]) => r !== null).map(([p, r]) => ({ path: p, reason: r! })).sort((a, b) => (a.path < b.path ? -1 : a.path > b.path ? 1 : 0));
const sortedExcluded = [...excluded].sort();
const content: Record<ContentKind, number> = { binary: 0, data: 0, docs: 0, log: 0 };
for (const p of sortedExcluded) content[contentKind(i, p, binary)!]++;
const prefixes = [...new Set(applied.map(literalPrefix).filter(Boolean))];
const refs = grepFixed(i, prefixes, excluded, []) ?? new Map<string, Set<string>>();
const patterns_sha256 = sha256(JSON.stringify(patterns));
const list_sha256 = sha256(JSON.stringify({ applied, excluded: sortedExcluded, kept: keptList.map(k => [k.path, k.reason]) }));
const record: ExcludeRecord = {
source, patterns: [...patterns], applied, skipped, unmatched: patterns.filter(p => !(sel.byPattern.get(p) ?? []).length),
excluded: excluded.size, excluded_bytes: excludedBytes(i, sortedExcluded, sel.files), kept: keptList.length,
gate_tagged: keptList.filter(k => k.reason === 'gate-tagged').map(k => k.path), patterns_sha256, list_sha256, digest: sha256(`${patterns_sha256}\n${list_sha256}`),
};
return {
state: 'active', record, excluded, specs: applied.map(p => `:(top,exclude,glob)${p}`), keep: [...keep].sort().map(literal), kept: keptList, content,
references: prefixes.filter(p => refs.has(p)).map(p => ({ prefix: p, files: [...refs.get(p)!].sort() })),
};
}
/** The head-side size of the excluded files: blob sizes at the head commit, file sizes on a live tree; a deletion counts 0. */
function excludedBytes(i: ExcludeInput, excluded: readonly string[], files: ReadonlyMap<string, Matched>): number {
if (!i.head) return excluded.reduce((n, p) => { try { return n + fs.statSync(path.join(i.root, p)).size; } catch { return n; } }, 0);
const shas = excluded.map(p => files.get(p)?.newSha ?? '').filter(s => /^[0-9a-f]{40,64}$/.test(s) && !/^0+$/.test(s));
if (!shas.length) return 0;
const r = git(i.root, ['cat-file', '--batch-check=%(objectsize)'], shas.join('\n') + '\n');
return r.out.split('\n').reduce((n, l) => n + (Number(l) || 0), 0);
}
/** The plan's lines for an exclusion: none prints nothing, so a repository without the key plans byte for byte as before. */
export function excludeLines(x: Exclusion, base: string): string[] {
if (x.state === 'none' || x.state === 'refused') return [];
if (x.state === 'head-only') return [`REVIEW_EXCLUDE: none (head sets review_exclude_paths; it applies after it lands on ${base}) patterns=${x.patterns.length} source=${x.source}`];
const r = x.record;
const skipped = r.skipped.length ? ` skipped=${r.skipped.map(s => `${s.pattern}:kept=${s.kept}:bytes=${s.bytes}`).join(',')} (kept files over ${KEEP_ARG_MAX_BYTES} argument bytes: pattern not applied)` : '';
const out = [`REVIEW_EXCLUDE: patterns=${r.applied.length}/${r.patterns.length} excluded=${r.excluded} bytes=${r.excluded_bytes} kept=${r.kept} content=${Object.entries(x.content).filter(([, n]) => n).map(([k, n]) => `${k}:${n}`).join(',') || 'none'} source=${r.source} digest=${r.digest}${skipped}${r.unmatched.length ? ` unmatched=${r.unmatched.join(',')}` : ''}`];
for (const k of x.kept.slice(0, 20)) out.push(`REVIEW_EXCLUDE_KEPT: ${k.reason} ${k.path}`);
if (x.kept.length > 20) out.push(`REVIEW_EXCLUDE_KEPT: +${x.kept.length - 20} more (gstack-review-plan --json lists every kept file)`);
for (const ref of x.references) out.push(`REVIEW_EXCLUDE_REFERENCE: ${ref.prefix} is named by ${ref.files.slice(0, 3).join(', ')}${ref.files.length > 3 ? ` (+${ref.files.length - 3} more)` : ''}; the files stay excluded, so readers see that code without the files it reads (REVIEW_EXCLUDE_REFERENCE)`);
return out;
}
/** The one line every brief gets under an active exclusion; it names the selection digest the receipt binds (verify's REVIEW_EXCLUDE_RECORD). */
export function disclosureLine(r: Pick<ExcludeRecord, 'source' | 'excluded' | 'excluded_bytes' | 'applied' | 'kept' | 'digest'>): string {
return `Excluded by base policy ${r.source} (selection ${r.digest}): ${r.excluded} files, ${r.excluded_bytes} bytes on this branch matching ${r.applied.join(' ') || 'no applied pattern'}. They are not in your diff; do not report on their absence. Kept despite a match: ${r.kept} files (listed in the plan).`;
}
/** The selection digest a disclosure line names, or undefined. */
export const disclosureDigest = (line: string | undefined) => line?.match(/^Excluded by base policy \S+ \(selection ([0-9a-f]{64})\)/)?.[1];
export function parseTail(tail: string | undefined): string[] {
const out: string[] = [];
const re = /'((?:[^']|'\\'')*)'|(\S+)/g;
for (let m = re.exec(tail ?? ''); m; m = re.exec(tail ?? '')) out.push(m[1] !== undefined ? m[1].replace(/'\\''/g, "'") : m[2]!);
return out;
}
export type ReaderDiff =
| { ok: true; text: string; bytes: number; byDir: Array<[string, number]> }
| { ok: false; reason: 'timeout' | 'git-exit'; exit: number | null; stderr: string; timeoutMs: number };
/** One reader's selection read once: `git diff <base> [-- pathspecs]`, then the kept-file command, with its bytes per top directory. */
export function readSelection(root: string, diffBase: string, pathspecs: readonly string[], keep: readonly string[] = [], timeoutMs = REVIEW_DIFF_TIMEOUT_MS): ReaderDiff {
let text = '';
for (const specs of [pathspecs, ...(keep.length ? [keep] : [])]) {
const tail = specs.length ? (specs[0] === '--' ? specs : ['--', ...specs]) : [];
const r = spawnSync('git', ['-c', 'core.quotePath=false', 'diff', '--no-color', '--no-ext-diff', diffBase, ...tail], { cwd: root, encoding: 'utf8', timeout: timeoutMs, maxBuffer: MAX_BUFFER });
const code = (r.error as NodeJS.ErrnoException | undefined)?.code;
if (code === 'ENOBUFS') { text += r.stdout ?? ''; break; }
if (code === 'ETIMEDOUT' || (r.signal && r.status === null)) return { ok: false, reason: 'timeout', exit: null, stderr: r.stderr ?? '', timeoutMs };
if (r.status !== 0) return { ok: false, reason: 'git-exit', exit: r.status, stderr: r.stderr ?? '', timeoutMs };
text += r.stdout;
}
const byDir = new Map<string, number>();
const starts = [...text.matchAll(/^diff --git a\/(\S+)/gm)];
starts.forEach((m, k) => {
const end = k + 1 < starts.length ? starts[k + 1]!.index! : text.length;
const dir = m[1]!.includes('/') ? m[1]!.split('/')[0]! : '(root)';
byDir.set(dir, (byDir.get(dir) ?? 0) + Buffer.byteLength(text.slice(m.index!, end)));
});
return { ok: true, text, bytes: Buffer.byteLength(text), byDir: [...byDir].sort((a, b) => b[1] - a[1]) };
}
/** Credential-bearing remote URLs and scp-style remotes out of git's stderr. */
export const redact = (s: string) => s.replace(/\b[a-z][a-z0-9+.-]*:\/\/[^\s'"]+/gi, '<url>').replace(/\b[\w.-]+@[\w.-]+:[^\s'"]+/g, '<url>');
/** Where the over-ceiling selection's policy stands: decides the fix the line prints. */
export interface CeilingContext { state: 'none' | 'head-only' | 'active'; base: string; source: string; floor?: string | null; headPatterns?: string[]; record?: Pick<ExcludeRecord, 'patterns' | 'skipped' | 'excluded'> }
/** The policy hunk to land, as JSON a person can paste: the patterns, and a floor at least this key's version. */
function policyJson(patterns: readonly string[], floor?: string | null): string {
const min = floor && parseVersion(floor) && cmpVersion(parseVersion(floor)!, parseVersion(REVIEW_EXCLUDE_KEY_VERSION)!) > 0 ? floor : REVIEW_EXCLUDE_KEY_VERSION;
return JSON.stringify({ review_exclude_paths: patterns, min_gstack_version: min });
}
/** Folders to suggest: the largest top directories, until what is left would fit. */
function suggest(byDir: ReadonlyArray<[string, number]>, bytes: number): string[] {
const out: string[] = [];
let left = bytes;
for (const [dir, n] of byDir) {
if (left <= REVIEW_DIFF_MAX_BYTES || out.length >= 3) break;
if (dir === '(root)') continue;
out.push(`${dir}/**`);
left -= n;
}
return out;
}
/** The REVIEW_DIFF_TOO_LARGE line and object: the reader's bytes, the largest folders and a fix chosen by where the policy stands. */
export function tooLargeLine(context: string, reader: string, d: { bytes: number; byDir: ReadonlyArray<[string, number]> }, c: CeilingContext): { line: string; json: Record<string, unknown> } {
const top = d.byDir.slice(0, 5).map(([dir, n]) => `${dir}:${n}`).join(',') || 'none';
const land = (patterns: readonly string[]) => `ask the repo owner to land ${policyJson(patterns, c.floor)} in ${SHIP_POLICY_PATH} on ${c.base} as its own PR (docs/ship-policy.md#review-exclude-paths), then bring ${c.base} into this branch and rerun once; if it is still over, split the PR`;
const dirs = suggest(d.byDir, d.bytes);
const s = c.record?.skipped[0];
const fix = c.state === 'head-only' ? `this branch declares review_exclude_paths, but only the copy on ${c.base} applies: ${land(c.headPatterns ?? [])}`
: s ? `pattern ${s.pattern} was not applied: its ${s.kept} kept files need ${s.bytes} argument bytes (more than ${KEEP_ARG_MAX_BYTES}); narrow the pattern on ${c.base} or move the kept files out of it, or split the PR`
: c.state === 'active' ? `still over after excluding ${c.record?.excluded ?? 0} files; ${dirs.length ? land([...(c.record?.patterns ?? []), ...dirs.filter(p => !c.record?.patterns.includes(p))]) : 'split the PR'}`
: dirs.length ? land(dirs) : 'split the PR';
const line = `${context}: the reader's diff is over the review ceiling (reader=${reader} bytes=${d.bytes} max=${REVIEW_DIFF_MAX_BYTES} top=${top} policy=${c.source}); fix: ${fix}; nothing was written (REVIEW_DIFF_TOO_LARGE)`;
return { line, json: { code: 'REVIEW_DIFF_TOO_LARGE', reader, bytes: d.bytes, max: REVIEW_DIFF_MAX_BYTES, top: d.byDir.slice(0, 5).map(([dir, bytes]) => ({ dir, bytes })), policy: c.source, state: c.state, fix, side_effects: 'none' } };
}
/** The REVIEW_DIFF_UNREADABLE line: timeout or git exit, the timeout, the base, a redacted first stderr line, and the whole (redacted) stderr written under `logDir`. */
export function unreadableLine(context: string, reader: string, d: Extract<ReaderDiff, { ok: false }>, diffBase: string, base: string, logDir: string): string {
fs.mkdirSync(logDir, { recursive: true, mode: 0o700 });
const log = path.join(logDir, `diff-${reader.replace(/[^a-zA-Z0-9._-]/g, '_')}.stderr`);
fs.writeFileSync(log, redact(d.stderr), { mode: 0o600 });
const first = redact(d.stderr.split('\n').find(l => l.trim()) ?? '').trim().slice(0, 200);
const fix = d.reason === 'timeout' ? `rerun once unchanged; if it times out again, split the PR or exclude its largest data folders on ${base}`
: d.exit === 128 ? `git fetch origin ${base}, then rerun once` : 'stop and ask a person: git failed in a way a rerun does not repair';
return `${context}: git could not read the reader's diff (reader=${reader} reason=${d.reason} exit=${d.exit ?? 'none'} timeout_ms=${d.timeoutMs} base=${diffBase.slice(0, 12)} stderr=${JSON.stringify(first)} log=${log}); fix: ${fix}; nothing else was written (REVIEW_DIFF_UNREADABLE)`;
}
/** The plan's review_exclude object (gstack-review-plan --json), or null for a repository without the key. */
export type PlanExclude = { state: string; patterns?: string[]; skipped?: ExcludeRecord['skipped']; excluded?: number } | null;
/**
* Where <base>'s policy stands for a REVIEW_DIFF_TOO_LARGE fix: the source commit, its min_gstack_version, and the exclusion
* state, from `known` (the plan's review_exclude, null when it had none) or, when the caller has no plan, a fresh live run.
*/
export function ceilingContext(root: string, base: string, known?: PlanExclude): CeilingContext {
const b = base.replace(/^origin\//, '');
const ref = git(root, ['rev-parse', '--verify', '-q', `origin/${b}^{commit}`]).ok ? `origin/${b}` : b;
const sha = git(root, ['rev-parse', '--verify', '-q', `${ref}^{commit}`]).out.trim();
const load = sha ? loadPolicyAt(root, sha, ref) : null;
const read = load && !isUnavailable(load) ? load : null;
const source = sha ? `${ref}@${sha.slice(0, 12)}${read && !read.present ? ` (no ${SHIP_POLICY_PATH})` : ''}` : ref;
let x = known;
if (x === undefined) {
const c = sha ? collectLive(root, b) : null;
const live = c?.mergeBase ? reviewExclude({ root, baseSha: sha, ref, mergeBase: c.mergeBase, perPath: c.perPath, added: c.added, classRoot: root, context: 'ceiling' }) : { state: 'none' as const };
x = live.state === 'active' ? { state: 'active', ...live.record } : live.state === 'head-only' ? live : null;
}
const floor = read?.policy.min_gstack_version ?? null;
if (!x) return { state: 'none', base: b, source, floor };
if (x.state === 'head-only') return { state: 'head-only', base: b, source, floor, headPatterns: x.patterns ?? [] };
return { state: 'active', base: b, source, floor, record: { patterns: x.patterns ?? [], skipped: x.skipped ?? [], excluded: x.excluded ?? 0 } };
}
+15 -6
View File
@@ -14,6 +14,7 @@
import * as fs from 'node:fs';
import * as path from 'node:path';
import { BriefError, briefFile, checkoutRoot, checkoutTree, promptHash, renderBrief, withWorkspace, type BriefRole, type SectionTexts, type Slots } from './review-brief';
import { disclosureDigest } from './review-exclude';
import { appendEvent, readerKey } from './review-ledger';
import { diffCommand, type Dispatch } from './review-depth';
import { TARGETED_RULES, parseClassList, targetedBriefSha, targetedRead } from './review-targeted';
@@ -31,6 +32,8 @@ export interface PassReader {
/** An audit reader: its prompt is a file the parent wrote from its section; its output is returned, never collected. */
prompt_file?: string; model?: PassModel;
file?: string; sha256?: string; bytes?: number; chunks?: number; findings_sha256?: string;
/** With the base policy's review_exclude_paths active: its `:(top,exclude,glob)` items, shell-quoted (the fixture command's slot), and the kept files' `:(top,literal)` items. */
policy_exclude?: string; keep?: string[];
/** The fingerprint the reader's reply must report: the checkout's HEAD^{tree} and toplevel when its brief was rendered (GST-482). */
tree?: string | null;
path?: string; output?: string; error?: string; state_at?: string;
@@ -42,6 +45,8 @@ export interface Manifest {
/** Step 11's own reading at its first dispatch, so its note never counts the pass's readers again. */
before_voices?: { tokens: number; credits: number };
log_starts?: Record<string, string>;
/** The plan's review_exclude object (lib/review-exclude.ts) without its file lists, and the line every brief carries; absent without the key. */
review_exclude?: Record<string, unknown>; disclosure?: string;
/** Decision 3.5's files for a fix-check reader: the fix list, touched hunks, test results and callers. */
fix_check?: FixCheckFiles;
}
@@ -75,6 +80,7 @@ export function detectTestFw(root: string): string {
return fw || 'unknown';
}
const shellQuote = (s: string) => `'${s.replace(/'/g, "'\\''")}'`;
const safe = (reader: string) => reader.replace(/[^a-zA-Z0-9._-]/g, '_');
const STAGE_OF: Record<string, Stage> = { specialist: 'specialists', 'fix-check': 'specialists', 'red-team': 'red-team', 'adversarial-native': 'voices', 'outside-voice': 'voices', targeted: 'voices' };
@@ -82,7 +88,7 @@ const STAGE_OF: Record<string, Stage> = { specialist: 'specialists', 'fix-check'
export function buildManifest(o: {
pass: string; out: string; ledger: string; base: string; diffBase: string; root: string; checklistRoot: string;
dispatches: Dispatch[]; next: string; planLines: string[]; learnings?: string | null; extras?: Extra[];
targetedClasses?: readonly string[]; mergeBase?: string; rangeBase?: string; fixCheck?: FixCheckFiles;
targetedClasses?: readonly string[]; mergeBase?: string; rangeBase?: string; fixCheck?: FixCheckFiles; reviewExclude?: Record<string, unknown>; disclosure?: string;
}): Manifest {
const readers: PassReader[] = [];
for (const d of o.dispatches) {
@@ -93,7 +99,9 @@ export function buildManifest(o: {
const name = role === 'specialist' ? d.name : undefined;
const reader = readerKey(role, name);
const exclude = d.exclude.length ? diffCommand(d).slice('git diff <diff-base> '.length) : undefined;
const policy = d.exclude.filter(s => s.startsWith(':(top,exclude,glob)'));
readers.push({ reader, stage, title: `ship ${o.pass} ${reader}`, state: 'planned', role, name, effort: d.effort, exclude, checklists: d.checklists,
...(policy.length ? { policy_exclude: policy.map(shellQuote).join(' ') } : {}), ...(d.keep?.length ? { keep: d.keep } : {}),
...(role === 'targeted' ? { classes: [...(o.targetedClasses ?? [])], diff_base: o.mergeBase ?? o.diffBase } : {}),
...(role === 'red-team' && o.rangeBase ? { diff_base: o.rangeBase } : {}) });
}
@@ -109,7 +117,7 @@ export function buildManifest(o: {
return {
version: 1, pass: o.pass, out, ledger: path.resolve(o.ledger), merged: path.join(out, 'merged.jsonl'), base: o.base, diff_base: o.diffBase,
stack: detectStack(o.root), test_fw: detectTestFw(o.root), learnings: o.learnings ?? null, checklist_root: o.checklistRoot,
next: o.next, plan_lines: o.planLines, readers,
next: o.next, plan_lines: o.planLines, readers, ...(o.reviewExclude ? { review_exclude: o.reviewExclude } : {}), ...(o.disclosure ? { disclosure: o.disclosure } : {}),
...(o.fixCheck ? { fix_check: Object.fromEntries(Object.entries(o.fixCheck).filter(([, v]) => v).map(([k, v]) => [k, path.resolve(v as string)])) as unknown as FixCheckFiles } : {}),
};
}
@@ -136,7 +144,7 @@ export function specialistFindings(mergedFile: string): string {
}
/** Render every planned reader of a stage. The Red Team renders only after every earlier reader is collected. */
export function renderStage(m: Manifest, stage: Stage, t: SectionTexts, diffFor: (base: string) => string): PassReader[] {
export function renderStage(m: Manifest, stage: Stage, t: SectionTexts, diffFor: (base: string, r: PassReader) => string): PassReader[] {
const open = m.readers.filter(r => (r.stage === 'first' || r.stage === 'specialists') && r.state !== 'collected');
if (stage === 'red-team' && open.length) throw new BriefError(`the Red Team renders after every earlier reader is collected; still open: ${open.map(r => r.reader).join(', ')}`);
const briefs = path.join(m.out, 'briefs');
@@ -157,7 +165,8 @@ export function renderStage(m: Manifest, stage: Stage, t: SectionTexts, diffFor:
}
const role = r.role!;
const slots: Slots = { name: r.name, diff_base: r.diff_base ?? m.diff_base, exclude: r.exclude, stack: m.stack, test_fw: m.test_fw, learnings: m.learnings ?? undefined,
checklists: r.checklists?.map(n => path.join(m.checklist_root, 'review/specialists', `${n}.md`)).join(',') };
checklists: r.checklists?.map(n => path.join(m.checklist_root, 'review/specialists', `${n}.md`)).join(','),
...(m.disclosure ? { disclosure: m.disclosure, policy_exclude: r.policy_exclude, keep: r.keep?.map(shellQuote).join(' ') } : {}) };
if (role === 'specialist') slots.checklist = path.join(m.checklist_root, 'review/specialists', `${r.name}.md`);
let findingsSha: string | undefined;
if (role === 'red-team') {
@@ -167,7 +176,7 @@ export function renderStage(m: Manifest, stage: Stage, t: SectionTexts, diffFor:
const specialists = m.readers.reduce((n, x) => n + (x.role === 'specialist' ? 1 : x.role === 'specialist-batched' ? (x.checklists ?? []).filter(c => c !== 'red-team').length : 0), 0);
Object.assign(slots, { checklist: path.join(m.checklist_root, 'review/specialists/red-team.md'), findings, specialists: String(specialists) });
}
if (role === 'outside-voice') slots.diff = diffFor(m.diff_base);
if (role === 'outside-voice') slots.diff = diffFor(m.diff_base, r);
if (role === 'fix-check') {
const read = (f: string | undefined, what: string) => { if (!f) return undefined; try { return fs.readFileSync(f, 'utf8'); } catch { throw new BriefError(`cannot read the fix-check ${what} ${f}`); } };
const fc = m.fix_check!;
@@ -186,7 +195,7 @@ export function renderStage(m: Manifest, stage: Stage, t: SectionTexts, diffFor:
fs.writeFileSync(file, text, { mode: 0o600 });
const sha = promptHash(text);
const bytes = Buffer.byteLength(text);
appendEvent(m.ledger, { event: 'rendered', reader: r.reader, role, name: r.name ?? null, file, sha256: sha, bytes, pass: m.pass, stage, workdir, tree, ...(findingsSha ? { findings_sha256: findingsSha } : {}), ...targeted });
appendEvent(m.ledger, { event: 'rendered', reader: r.reader, role, name: r.name ?? null, file, sha256: sha, bytes, pass: m.pass, stage, workdir, tree, ...(findingsSha ? { findings_sha256: findingsSha } : {}), ...targeted, ...(disclosureDigest(m.disclosure) ? { exclude: disclosureDigest(m.disclosure) } : {}) });
Object.assign(r, { file, sha256: sha, bytes, chunks: chunkCount(text), state: 'rendered', tree, ...(findingsSha ? { findings_sha256: findingsSha } : {}) });
done.push(r);
}
+3
View File
@@ -26,6 +26,9 @@ import { AUTH_PATH_PREFIXES, AUTH_PATH_TOKENS, KEY_NAME, MONEY_LINE, MONEY_LINE_
export const TARGETED_CLASSES = ['money', 'gate', 'auth', 'secret'] as const;
export type TargetedClass = (typeof TARGETED_CLASSES)[number];
/** Whether one added or deleted line reads a credential (SECRET_LINE): lib/review-exclude.ts keeps a matched file that has one. */
export const secretLine = (line: string) => SECRET_LINE.some(r => r.test(line));
/** A hash of the patterns that decide the classes, not of this file's bytes: a comment edit keeps it, a pattern edit changes it. */
export const TARGETED_RULES = createHash('sha256').update(JSON.stringify({
TARGETED_CLASSES, MONEY_PATH_TOKENS, MONEY_LINE_TOKENS, MONEY_PAIRS, AUTH_PATH_TOKENS, AUTH_PATH_PREFIXES, KEY_NAME,
+1
View File
@@ -32,6 +32,7 @@ export const PARTS: PartSpec[] = [
{ id: 'review-calibration', file: RA, from: '## Confidence Calibration', phase: 4 },
{ id: 'review-core', file: RA, from: '### Core checklist', phase: 4 },
{ id: 'checklist', file: 'review/checklist.md', from: null, phase: 4 },
{ id: 'checklist-pass-2', file: 'review/checklist.md', from: '### Pass 2 — INFORMATIONAL', phase: 4 },
{ id: 'review-design', file: RA, from: '## Design Review (conditional, diff-scoped)', phase: 4 },
{ id: 'review-9.2', file: RA, from: '### Step 9.2: Collect and merge findings', phase: 4 },
{ id: 'review-9.3', file: RA, from: '### Step 9.3: Cross-review finding dedup', phase: 4 },
+34
View File
@@ -0,0 +1,34 @@
/**
* ship-policy-at — the ship policy committed at a commit the caller already
* resolved (docs/ship-policy.md#review-exclude-paths). Unlike
* loadPolicyFromBase, any git failure is `unavailable`, never a silent
* default. Callers: lib/review-exclude.ts, lib/ship-receipt-depth.ts,
* bin/gstack-review-plan.
*/
import { spawnSync } from 'node:child_process';
import { SHIP_POLICY_PATH, finish, type PolicyLoad, type PolicySource } from './ship-policy';
function git(repoRoot: string, args: string[]): { ok: boolean; out: string; err: string } {
const r = spawnSync('git', args, { cwd: repoRoot, encoding: 'utf8', timeout: 30_000 });
return { ok: r.status === 0, out: (r.stdout ?? '').trim(), err: (r.stderr ?? '').trim() };
}
/**
* The policy committed at `sha` (a commit the caller already resolved, labeled `ref`): `cat-file -e` first, and only an
* `ls-tree` that succeeds with no entry is a missing file (defaults); any git failure, including a `show` failure after
* the file was found, is `unavailable`, never a silent default.
*/
export function loadPolicyAt(repoRoot: string, sha: string, ref: string): PolicyLoad {
const label = `${ref}@${sha.slice(0, 12)}`;
const source: PolicySource = { kind: 'base', ref, sha, label: `${label}:${SHIP_POLICY_PATH}` };
const spec = `${sha}:${SHIP_POLICY_PATH}`;
const base = ref.replace(/^origin\//, '');
if (!git(repoRoot, ['cat-file', '-e', spec]).ok) {
const tree = git(repoRoot, ['ls-tree', sha, '--', SHIP_POLICY_PATH]);
if (!tree.ok) return { unavailable: true, base, detail: `git ls-tree ${sha.slice(0, 12)} failed: ${tree.err.split('\n')[0] || 'no detail'}` };
if (!tree.out) return finish(null, { ...source, kind: 'none', label: `${label} (no ${SHIP_POLICY_PATH})` });
}
const show = git(repoRoot, ['show', spec]);
if (!show.ok) return { unavailable: true, base, detail: `git show ${label}:${SHIP_POLICY_PATH} failed: ${show.err.split('\n')[0] || 'no detail'}` };
return finish(show.out, source);
}
+28 -6
View File
@@ -15,6 +15,7 @@
import { spawnSync } from 'node:child_process';
import * as fs from 'node:fs';
import * as path from 'node:path';
import { defaultBaseBranch, noDefaultBaseLine } from './default-base';
import { RESULT_CODES, type ResultCodeName } from './result-codes';
import { cmpVersion, parseVersion } from './version-source';
@@ -53,6 +54,8 @@ export interface ShipPolicy {
stamp_sensitive_gates: string[];
/** The /ship review loop's finding scope (docs/ship-policy.md#review-scope). */
review_scope: ReviewScope;
/** Git glob pathspecs (root-anchored) whose data, docs, binary and log files /ship's reviewers leave out; read from the base only (docs/ship-policy.md#review-exclude-paths). */
review_exclude_paths: string[];
/** Consumed by the plan reviews (D1). */
pr_count: { default_pr_count: number; split_requires_owner: boolean };
/** Maintainer logins for contributor-PR mode (E3); empty means "use push permission". */
@@ -153,6 +156,7 @@ export const DEFAULT_POLICY: ShipPolicy = {
preregistration_shas: [],
stamp_sensitive_gates: [],
review_scope: 'frozen',
review_exclude_paths: [],
pr_count: { default_pr_count: 1, split_requires_owner: true },
maintainers: [],
stamp_files: [],
@@ -184,6 +188,7 @@ export const POLICY_EXPLANATIONS: Record<PolicyKey, string> = {
preregistration_shas: 'Commits that must be ancestors of HEAD before push (preregistered evals); checked by `gstack-evidence ancestor`.',
stamp_sensitive_gates: 'Evidence labels (gstack-evidence --label) that read VERSION or generated bytes; the tree receipt never marks them reusable.',
review_scope: 'frozen | diff-plus-blocking | all. frozen (default): /ship Step 9 fixes findings in the diff on pass 1, files blocking findings in code the PR did not change as issues, and later passes read only the range since the previous pass and fix only blocking findings; diff-plus-blocking: the same, but a blocking finding in code the PR did not change is fixed in this PR instead of filed; all: every finding is fixed now (the pre-1.91.249 loop).',
review_exclude_paths: 'Git glob pathspecs, root-anchored and case-sensitive (`receipts/**`, `**/*.jsonl`; at most 64, 256 characters each, no leading `:` or `!`): /ship\'s reviewers leave out the data, docs, binary and .log files they match. Read from the base branch only, so a branch cannot exclude its own files. Code, gate paths, agent prose, config, executables, .gstack/ and .github/ files, gate-tagged or secret-bearing files and files a kept code or test file names by full path stay reviewed (docs/ship-policy.md#review-exclude-paths).',
pr_count: '{ default_pr_count, split_requires_owner }: the PR-count row in plan reviews.',
maintainers: 'GitHub logins treated as maintainers by /review contributor mode; empty uses repository push permission.',
stamp_files: 'The repo’s stamp rule: globs of the only files that may differ by a stamp (version file, manifests, plugin trees, the PR’s own since: rows). Empty keeps gstack’s per-file rules; set, it applies before every rule.',
@@ -231,6 +236,7 @@ export const BUMP_NEVER_KEY_VERSION = '1.91.353.0';
export const BASE_RED_KEY_VERSION = '1.91.374.0';
/** `train.gates` and `train.land` are values of the older train key, so compatErrors checks them by value, as for bump: never. */
export const TRAIN_GATES_KEY_VERSION = '1.92.0.0';
export const REVIEW_EXCLUDE_KEY_VERSION = '1.93.33.0';
export const POLICY_KEY_INTRODUCED: Record<PolicyKey, string> = {
bump: '1.91.73.0', queue_mode: '1.91.73.0', release_tool: '1.91.73.0', release_outputs: '1.91.73.0', stamp_paths: '1.91.73.0',
changelog: '1.91.73.0', mirror: '1.91.73.0', history: '1.91.73.0', preregistration_shas: '1.91.73.0', stamp_sensitive_gates: '1.91.73.0',
@@ -238,7 +244,7 @@ export const POLICY_KEY_INTRODUCED: Record<PolicyKey, string> = {
migrations: '1.91.98.0', gate_command: SHIP_QUEUE_KEYS_VERSION, full_e2e: SHIP_QUEUE_KEYS_VERSION, min_gstack_version: SHIP_QUEUE_KEYS_VERSION,
train: TRAIN_KEYS_VERSION, review_scope: REVIEW_SCOPE_KEY_VERSION, docs: DOCS_CAPS_KEY_VERSION,
changelog_fragments: CHANGELOG_FRAGMENTS_KEY_VERSION,
base_red: BASE_RED_KEY_VERSION,
base_red: BASE_RED_KEY_VERSION, review_exclude_paths: REVIEW_EXCLUDE_KEY_VERSION,
};
const newer = (a: string, b: string): boolean => cmpVersion(parseVersion(a)!, parseVersion(b)!) > 0;
@@ -327,6 +333,16 @@ function globError(g: string): string | null {
return null;
}
/** `review_exclude_paths`: git glob pathspecs, each one repo-relative line with no pathspec magic of its own; every error is POLICY_INVALID at its index. */
function excludeErrors(value: unknown, out: PolicyError[]): void {
if (!Array.isArray(value)) { out.push({ code: 'POLICY_INVALID', path: 'review_exclude_paths', message: 'must be an array of git glob pathspecs' }); return; }
if (value.length > MAX_LIST) out.push({ code: 'POLICY_INVALID', path: 'review_exclude_paths', message: `more than ${MAX_LIST} patterns` });
value.forEach((p, i) => {
const why = typeof p !== 'string' ? 'must be a string' : containedRelative(p) ?? (/^[:!]/.test(p) ? 'must not start with : or ! (no pathspec magic or negation)' : /[\x00-\x1f\x7f]/.test(p) ? 'contains a control character' : p.replace(/\/+$/, '') === '' || p === '.' ? 'must not name the whole repository' : null);
if (why) out.push({ code: 'POLICY_INVALID', path: `review_exclude_paths[${i}]`, message: `${JSON.stringify(p)} ${why}` });
});
}
function templateErrors(value: unknown, out: PolicyError[]): void {
if (!Array.isArray(value)) { out.push({ code: 'POLICY_INVALID', path: 'stamp_patterns', message: 'must be an array of literal templates' }); return; }
if (value.length > MAX_TEMPLATES) out.push({ code: 'POLICY_CONTAINMENT', path: 'stamp_patterns', message: `more than ${MAX_TEMPLATES} templates` });
@@ -471,6 +487,7 @@ export function validatePolicy(raw: unknown, opts: { gstackVersion?: string } =
oneOf('queue_mode', ['claim-at-open', 'stamp-at-merge']);
oneOf('history', ['merge-only', 'squash-ok']);
oneOf('review_scope', ['frozen', 'diff-plus-blocking', 'all']);
if ('review_exclude_paths' in obj) excludeErrors(obj.review_exclude_paths, errors);
if ('release_tool' in obj && obj.release_tool !== null) {
const t = obj.release_tool;
if (typeof t !== 'string' || !t.trim()) errors.push({ code: 'POLICY_INVALID', path: 'release_tool', message: 'must be a non-empty string or null' });
@@ -635,15 +652,19 @@ function git(repoRoot: string, args: string[]): { ok: boolean; out: string; err:
return { ok: r.status === 0, out: (r.stdout ?? '').trim(), err: (r.stderr ?? '').trim() };
}
/** `--base` when given, else origin/HEAD's branch, else `main`. */
/**
* `--base` when given, else the default base (lib/default-base.ts): origin/HEAD's branch unless it is the checked-out
* one, the PR's base, origin/main, origin/master. When none resolves it stays `main`, which origin then lacks, so every
* reader of origin/<base> refuses; when origin/HEAD named the checked-out branch, stderr says why.
*/
export function resolveBaseBranch(repoRoot: string, explicit?: string): string {
if (explicit && explicit.trim()) return explicit.trim().replace(/^origin\//, '');
const r = git(repoRoot, ['symbolic-ref', '--quiet', '--short', 'refs/remotes/origin/HEAD']);
if (r.ok && r.out) return r.out.replace(/^origin\//, '');
return 'main';
const found = defaultBaseBranch(repoRoot);
if (!found.branch && found.skipped) process.stderr.write(`gstack: ${noDefaultBaseLine(found.skipped)}\n`);
return found.branch ?? 'main';
}
function finish(rawText: string | null, source: PolicySource): LoadedPolicy {
export function finish(rawText: string | null, source: PolicySource): LoadedPolicy {
if (rawText === null) return { policy: mergePolicy(null, source.label).policy, present: false, source, errors: [], values: mergePolicy(null, source.label).values };
let raw: unknown;
try { raw = JSON.parse(rawText); }
@@ -733,6 +754,7 @@ export function shipConsequences(policy: ShipPolicy, base: string): string[] {
: policy.review_scope === 'diff-plus-blocking'
? 'review_scope diff-plus-blocking: Step 9.4 runs `gstack-review-scope classify --policy diff-plus-blocking` each pass; pass 1 fixes the diff and every blocking finding, in unchanged code too, later passes read only the range since the previous pass and fix only blocking findings; notes go to the PR body'
: 'review_scope frozen: Step 9.4 runs `gstack-review-scope classify` each pass; pass 1 fixes the diff and defers blocking findings in unchanged code to issues, later passes read only the range since the previous pass and fix only blocking findings; notes go to the PR body');
if (policy.review_exclude_paths.length) out.push(`review_exclude_paths ${policy.review_exclude_paths.join(' ')}: Step 9's reviewers leave out the data, docs, binary and .log files these git globs match on this branch; gstack-review-plan prints REVIEW_EXCLUDE: with the counts and the kept files, and every brief names what was left out (docs/ship-policy.md#review-exclude-paths)`);
if (policy.preregistration_shas.length) out.push(`preregistration: \`gstack-evidence ancestor --from-policy --base ${base}\` must print only ANCESTOR: ok before push; the ship receipt records the result`);
if (policy.release_tool) out.push(`release_tool: \`${policy.release_tool}\` runs only inside gstack-restamp (staging worktree, outputs confined to release_outputs)`);
return out;
+16 -1
View File
@@ -37,7 +37,7 @@ function integrate(cwd: string, base: string, r: Report): boolean {
if (!branch || branch === base || branch === head) { r.stop(`on the base branch (${branch || 'detached HEAD'}): ship from a feature branch`); return false; }
const status = git(cwd, ['status', '--porcelain']).stdout.split('\n').filter(Boolean);
r.add(`STATUS: modified=${status.filter(l => !l.startsWith('??')).length} untracked=${status.filter(l => l.startsWith('??')).length} (uncommitted changes ship with this run)`);
const fetch = git(cwd, ['fetch', 'origin', base], 300_000);
const fetch = git(cwd, ['fetch', 'origin', `+refs/heads/${base}:refs/remotes/origin/${base}`], 300_000);
if (fetch.status !== 0) { r.stop(`git fetch origin ${base} failed: ${firstLine(fetch.stderr)}; restore access first`); return false; }
const stat = git(cwd, ['diff', forkPoint(cwd, base), '--shortstat']).stdout.trim();
const lines = [...stat.matchAll(/(\d+) (insertion|deletion)/g)].reduce((n, m) => n + Number(m[1]), 0);
@@ -48,6 +48,7 @@ function integrate(cwd: string, base: string, r: Report): boolean {
if (!pr) r.add('PR: not checked (origin is a local path, not a hosted remote gh can map; Step 0 uses git-native commands)');
else if (pr.status !== 0) r.add(`CARRY: not checked (gh pr list failed: ${firstLine(pr.stderr)})`);
else if (/^\d+$/.test(n)) {
if (!prSized(cwd, base, n, r)) return false;
const carry = bin(cwd, 'gstack-ship-receipt', ['carry', '--pr', n, '--base', base, '--dry-run'], 120_000);
const line = (carry.stdout.match(/^CARRY: .*$/m) ?? [`CARRY: check failed (${firstLine(carry.stderr)})`])[0];
r.add(`PR: #${n}`).add(line);
@@ -69,6 +70,20 @@ function integrate(cwd: string, base: string, r: Report): boolean {
return merge(cwd, base, r);
}
/** The diff against origin/<base> must be about the PR's size: far more files (a stale or wrong base, HARN-15's 17,000-file review of main's history) or fewer (a fork's origin ahead of the PR's base, which would leave commits out of the gate scan) stops before any reader runs. Checked only when HEAD is the PR's pushed head. */
function prSized(cwd: string, base: string, n: string, r: Report): boolean {
const view = gh(cwd, ['pr', 'view', n, '--json', 'changedFiles,headRefOid,baseRefName', '--jq', '"\\(.changedFiles) \\(.headRefOid) \\(.baseRefName)"']);
const [pr, prHead, prBase] = view.status === 0 ? view.stdout.trim().split(' ') : [];
if (!pr || !/^[1-9]\d*$/.test(pr) || prHead !== git(cwd, ['rev-parse', 'HEAD']).stdout.trim()) return true;
const local = git(cwd, ['diff', '--name-only', forkPoint(cwd, base), 'HEAD']).stdout.split('\n').filter(Boolean).length;
if (local >= Number(pr) && local <= 2 * Number(pr) + 50) return true;
const word = /^[A-Za-z0-9._\/-]+$/.test(prBase ?? '') ? prBase! : `'${(prBase ?? '').replace(/'/g, "'\\''")}'`;
r.stop(`the diff against origin/${base} lists ${local} files but PR #${n} changes ${pr} (its base is ${word}): ${prBase === base
? `origin is not the PR's base repository (a fork) or its ${base} has moved; fetch the PR's base repository's ${base} into origin/${base}, then rerun`
: `origin/${base} is not the PR's base; rerun with --base ${word} after git fetch origin +refs/heads/${word}:refs/remotes/origin/${word}`}`);
return false;
}
const forkPoint = (cwd: string, base: string) => git(cwd, ['merge-base', `origin/${base}`, 'HEAD']).stdout.trim() || `origin/${base}`;
function merge(cwd: string, base: string, r: Report): boolean {
+1 -1
View File
@@ -338,7 +338,7 @@ export function land(ctx: QueueContext, o: LandOptions): StepOutput {
const pin = j?.reviewed_pin ?? recovered ?? info.headRefOid;
const lines = [`LAND: #${o.pr} branch=${info.headRefName} pin=${pin}${j ? ` (resuming at ${j.phase})` : recovered ? ` (the head ${info.headRefOid.slice(0, 12)} is land's stamp commit; restamping from its pin)` : ''}`];
git(root, ['fetch', '-q', 'origin', ctx.base]);
const receipt = bindShippedReceipt(root, `origin/${ctx.base}`, info.body ?? '', o.pr, [pin]);
const receipt = bindShippedReceipt(root, `origin/${ctx.base}`, info.body ?? '', o.pr, [pin], mergedResume);
lines.push(receipt.line, ...(receipt.depthLine ? [receipt.depthLine] : []), ...(receipt.targetedLine ? [receipt.targetedLine] : []));
if (!receipt.ok) return fail(lines, { code: receipt.code, message: receipt.message, fix: 'run /ship at the pinned head first' }, run);
if (!policy.changelog_fragments) return fail(lines, { code: 'QUEUE_NOT_READY', message: 'the base policy has changelog_fragments off; land merges only under the key, where ready records the READY receipt it checks' }, run);
+79 -19
View File
@@ -19,8 +19,12 @@
*/
import { spawnSync } from 'node:child_process';
import * as path from 'node:path';
import { baseContainsHead } from './default-base';
import type { ShipReceipt } from './headless-artifacts';
import { collectCommit } from './review-collect';
import { reviewExclude, type Exclusion } from './review-exclude';
import { isUnavailable } from './ship-policy';
import { loadPolicyAt } from './ship-policy-at';
import { depthOfCommitInput, gateScanNeeded, type Depth } from './review-depth';
import { TARGETED_CLASSES, classList, targetedClasses, type TargetedClass } from './review-targeted';
import { runScriptSync } from './script-invocation';
@@ -42,16 +46,33 @@ function git(root: string, args: string[]): { ok: boolean; out: string } {
/** A base as a commit; origin/<branch> reads refs/remotes/origin/<branch> by its full name, so a tag named origin/main cannot stand in for it (GST-884). */
const baseCommit = (root: string, ref: string) => git(root, ['rev-parse', '--verify', '--quiet', `${/^origin\//.test(ref) ? `refs/remotes/${ref}` : ref}^{commit}`]);
/** gstack-diff-scope on the committed range `base`...`head` (`--head`): the same signals at write, carry and the queue. */
function scopeOfRange(root: string, binDir: string, base: string, head: string): Record<string, string> {
const sr = runScriptSync(path.join(binDir, 'gstack-diff-scope'), [base, '--head', head], { cwd: root, encoding: 'utf8', timeout: 60_000, maxBuffer: 16 * 1024 * 1024 });
/** gstack-diff-scope on the committed range `base`...`head` (`--head`): the same signals at write, carry and the queue; `merged` (`--merged`) for a head GitHub already merged into the base. */
function scopeOfRange(root: string, binDir: string, base: string, head: string, merged = false): Record<string, string> {
const sr = runScriptSync(path.join(binDir, 'gstack-diff-scope'), [base, '--head', head, ...(merged ? ['--merged'] : [])], { cwd: root, encoding: 'utf8', timeout: 60_000, maxBuffer: 16 * 1024 * 1024 });
const scope: Record<string, string> = {};
for (const l of (sr.stdout ?? '').split('\n')) { const m = /^(SCOPE_[A-Z_]+)=(.*)$/.exec(l); if (m) scope[m[1]!] = m[2]!; }
if (sr.status !== 0 && !scope.SCOPE_ERROR) scope.SCOPE_ERROR = `exit ${sr.status}`;
return scope;
}
export type ReceiptExclude = NonNullable<ShipReceipt['review_exclude']>;
/** The receipt's copy of an active exclusion: counts and hashes, plus the gate-tagged kept paths the queue cannot recompute. */
export function receiptExclude(x: Exclusion): ReceiptExclude | undefined {
if (x.state !== 'active') return undefined;
const { source, patterns, applied, excluded, excluded_bytes, kept, gate_tagged, patterns_sha256, list_sha256, digest } = x.record;
return { source, patterns, applied, excluded, excluded_bytes, kept, gate_tagged, patterns_sha256, list_sha256, digest };
}
/** The exclusion for the committed range `baseSha`...`head` (collectCommit's), with the gate-tagged paths the caller knows. */
function exclusionAt(root: string, ref: string, baseSha: string | null, head: string, c: ReturnType<typeof collectCommit>, gateTagged: ReadonlySet<string> | null, context: string): Exclusion {
if (!baseSha || !c.mergeBase) return { state: 'none' };
return reviewExclude({ root, baseSha, ref, mergeBase: c.mergeBase, head, perPath: c.perPath, added: c.added, classRoot: c.root ?? [], gateTagged, context });
}
/** Whether the base policy declares patterns, so the gate scan's per-path tags are worth reading. */
const declaresExclude = (root: string, ref: string, sha: string | null) => { if (!sha) return false; const l = loadPolicyAt(root, sha, ref); return !isUnavailable(l) && l.policy.review_exclude_paths.length > 0; };
/**
* The depth record for `head`: the range from git objects (collectCommit) and
* its diff-scope, as the queue recomputes them, plus, for a tests or code diff,
@@ -60,20 +81,36 @@ function scopeOfRange(root: string, binDir: string, base: string, head: string):
* unavailable and the depth fails closed to full.
*/
export function receiptDepthAtHead(repoRoot: string, base: string, binDir: string, head: string): ReceiptDepth {
return receiptReviewAtHead(repoRoot, base, binDir, head).depth;
}
/** receiptDepthAtHead plus the exclusion the head's review read under: the depth classifies excluded paths as docs (GST-508). */
export function receiptReviewAtHead(repoRoot: string, base: string, binDir: string, head: string): { depth: ReceiptDepth; exclude: Exclusion } {
const branch = base.replace(/^origin\//, '');
const baseSha = baseCommit(repoRoot, `origin/${branch}`);
const baseRev = baseSha.ok ? baseSha.out : branch;
const ref = baseSha.ok ? `origin/${branch}` : branch;
const sha = baseSha.ok ? baseSha.out : (git(repoRoot, ['rev-parse', '--verify', '--quiet', `${branch}^{commit}`]).out || null);
const c = collectCommit(repoRoot, baseRev, head);
const scope = scopeOfRange(repoRoot, binDir, baseRev, head);
const declares = declaresExclude(repoRoot, ref, sha);
let tagged: number | null = 0;
if (gateScanNeeded(c.paths, c.root)) {
let paths: Set<string> | null = declares ? new Set() : null;
if (declares || gateScanNeeded(c.paths, c.root)) {
const atHead = git(repoRoot, ['rev-parse', 'HEAD']).out === head && git(repoRoot, ['diff', '--quiet', 'HEAD', '--']).ok;
const g = atHead ? runScriptSync(path.join(binDir, 'gstack-gate-diff'), [baseRev], { cwd: repoRoot, encoding: 'utf8', timeout: 120_000, maxBuffer: 16 * 1024 * 1024 }) : null;
const g = atHead ? runScriptSync(path.join(binDir, 'gstack-gate-diff'), [baseRev, ...(declares ? ['--format', 'jsonl'] : [])], { cwd: repoRoot, encoding: 'utf8', timeout: 120_000, maxBuffer: 64 * 1024 * 1024 }) : null;
const m = /GATE_SUMMARY: .*?tagged=\{([^}]*)\}/.exec(g?.stdout ?? '');
tagged = g?.status === 0 && m ? m[1]!.split(',').reduce((n, kv) => n + (Number(kv.split(':')[1]) || 0), 0) : null;
paths = declares && tagged !== null ? taggedPaths(g?.stdout ?? '') : null;
}
const d = depthOfCommitInput({ ...c, scope, gateTagged: tagged });
return { depth: d.depth, class: d.class, runtime_lines: d.runtimeLines, rules: d.rules, base: baseRev, registry: c.registry.sha, gate_scan: { tagged, head }, reasons: d.reasons.slice(0, 12), risk: d.risk };
const exclude = exclusionAt(repoRoot, ref, sha, head, c, paths, 'gstack-ship-receipt');
const d = depthOfCommitInput({ ...c, scope, gateTagged: tagged, ...(exclude.state === 'active' ? { excluded: exclude.excluded } : {}) });
return { depth: { depth: d.depth, class: d.class, runtime_lines: d.runtimeLines, rules: d.rules, base: baseRev, registry: c.registry.sha, gate_scan: { tagged, head }, reasons: d.reasons.slice(0, 12), risk: d.risk }, exclude };
}
/** The paths of the hunks a `gstack-gate-diff --format jsonl` run tagged. */
export function taggedPaths(jsonl: string): Set<string> {
return new Set(jsonl.split('\n').slice(1).flatMap(l => { try { const h = JSON.parse(l) as { path?: string; tags?: unknown[] }; return h.path && h.tags?.length ? [h.path] : []; } catch { return []; } }));
}
export const depthBelow = (have: Depth, need: Depth) => RANK[have] < RANK[need];
@@ -175,11 +212,27 @@ export function receiptTargetedVerdict(required: readonly TargetedClass[], r: Sh
}
/** The head's committed range against the base, read once for the depth verdict and the targeted line. */
interface HeadInputs { baseSha: { ok: boolean; out: string }; c: ReturnType<typeof collectCommit>; scope: Record<string, string> }
function headInputs(root: string, baseRef: string, head: string): HeadInputs {
interface HeadInputs { baseSha: { ok: boolean; out: string }; c: ReturnType<typeof collectCommit>; scope: Record<string, string>; x: Exclusion }
function headInputs(root: string, baseRef: string, head: string, r?: ShipReceipt, merged = false): HeadInputs {
const baseSha = baseCommit(root, baseRef);
const baseRev = baseSha.ok ? baseSha.out : baseRef;
return { baseSha, c: collectCommit(root, baseRev, head), scope: scopeOfRange(root, BIN_DIR, baseRev, head) };
const c = collectCommit(root, baseRev, head);
const tagged = r && boundScan(r, head) !== null ? new Set(r.review_exclude?.gate_tagged ?? []) : null;
return { baseSha, c, scope: scopeOfRange(root, BIN_DIR, baseRev, head, merged), x: exclusionAt(root, baseRef, baseSha.ok ? baseSha.out : null, head, c, tagged, 'gstack-ship-queue') };
}
/**
* The queue's exclusion check: the digest the receipt recorded (none when absent) against the one `head` has under the current
* base policy. Only the digest counts (the policy's patterns and the excluded list), so a base merge or a moved base that leaves
* both unchanged binds; a changed selection needs the review rerun, which receipt write alone cannot stand in for.
*/
export function excludeMismatch(x: Exclusion, r: ShipReceipt, head: string, base: string): string | null {
const expected = x.state === 'active' ? x.record.digest : 'none';
const recorded = r.review_exclude?.digest ?? 'none';
if (expected === recorded) return null;
const source = x.state === 'active' ? x.record.source : r.review_exclude?.source ?? base;
const h = head.slice(0, 12);
return `field=review_exclude expected=${expected} recorded=${recorded} source=${source}: the reviewers read another selection than ${base}'s review_exclude_paths gives ${h}; the receipt is unchanged; fix: rerun /ship's review on ${h} (gstack-review-dispatch pass --base ${base.replace(/^(refs\/remotes\/)?origin\//, '')}), then gstack-ship-receipt write --pr <n> --review-exclude <d> --body <body> with the digest gstack-review-dispatch verify --ledger <ledger> prints on REVIEW_EXCLUDE_RECORD:; a receipt write alone does not clear this`;
}
/** The receipt's recorded gate scan when it names this head; otherwise null (unavailable). */
const boundScan = (r: ShipReceipt, head: string): number | null => {
@@ -197,10 +250,10 @@ const boundScan = (r: ShipReceipt, head: string): number | null => {
export function receiptDepthVerdict(root: string, baseRef: string, r: ShipReceipt, head: string, at?: HeadInputs): DepthVerdict {
const rd = r.review_depth;
if (!rd && !r.gstack_version) return { ok: true, line: 'REVIEW_DEPTH: legacy receipt (written before receipts recorded review_depth); accepted' };
const { baseSha, c, scope } = at ?? headInputs(root, baseRef, head);
const { baseSha, c, scope, x } = at ?? headInputs(root, baseRef, head, r);
const tagged = boundScan(r, head);
const scanBound = tagged !== null;
const need = depthOfCommitInput({ ...c, scope, gateTagged: tagged });
const need = depthOfCommitInput({ ...c, scope, gateTagged: tagged, ...(x.state === 'active' ? { excluded: x.excluded } : {}) });
const have = rd?.depth;
const inputs = `base@${short(baseSha.ok ? baseSha.out : null)},registry@${short(c.registry.sha)},gate_scan@${scanBound ? short(rd!.gate_scan.head) : 'missing'}`;
const line = `REVIEW_DEPTH: receipt=${have ?? 'none'} required=${need.depth} runtime_lines=${need.runtimeLines} class=${need.class} reasons=${need.reasons.join(',')} inputs=${inputs} rounds=${r.review_loop?.rounds ?? (r.review_loop ? r.review_loop.passes + (r.review_loop.fix_checks ?? 0) : 'none')}`;
@@ -210,19 +263,26 @@ export function receiptDepthVerdict(root: string, baseRef: string, r: ShipReceip
return { ok: false, line, message: `the receipt ${why}, but head ${head.slice(0, 12)} needs ${need.depth} (${need.reasons.join(', ')}${errs ? `; ${errs}` : ''}); the receipt is unchanged; run /ship on ${head.slice(0, 12)}` };
}
/** bindReceipt plus the owner exceptions, the review depth and the targeted pass: what READY, train build and land accept. */
export function bindShippedReceipt(root: string, baseRef: string, body: string, pr: string | undefined, accepted: readonly string[]):
/** bindReceipt plus the owner exceptions, the review depth and the targeted pass: what READY, train build and land accept. A base that already contains the head refuses (BASE_CONTAINS_HEAD, GST-774) before any depth is computed, except for a PR GitHub already merged (`merged`: land resuming after a merge-only true merge), whose depth then reads the range with diff-scope's `--merged` and whose exclusion digest is not recomputed over that emptied range. */
export function bindShippedReceipt(root: string, baseRef: string, body: string, pr: string | undefined, accepted: readonly string[], merged = false):
| { ok: true; receipt: ShipReceipt; head: string; line: string; depthLine: string; targetedLine: string }
| { ok: false; code: ReceiptBindingCode | 'QUEUE_RECEIPT_DEPTH_RAISED' | 'QUEUE_RECEIPT_EXCEPTION_INVALID' | 'QUEUE_RECEIPT_TARGETED_MISSING'; message: string; line: string; depthLine?: string; targetedLine?: string } {
| { ok: false; code: ReceiptBindingCode | 'BASE_CONTAINS_HEAD' | 'QUEUE_RECEIPT_DEPTH_RAISED' | 'QUEUE_RECEIPT_EXCEPTION_INVALID' | 'RECEIPT_INVALID' | 'QUEUE_RECEIPT_TARGETED_MISSING' | Extract<Exclusion, { state: 'refused' }>['code']; message: string; line: string; depthLine?: string; targetedLine?: string } {
const b = bindReceipt(body, pr, accepted);
if (!b.ok) return b;
const self = merged ? null : baseContainsHead(root, baseRef, b.head);
if (self) return { ok: false, code: 'BASE_CONTAINS_HEAD', message: self, line: b.line };
const ancestor = (commit: string) => b.receipt.carry?.ancestry === 'rewritten' || git(root, ['merge-base', '--is-ancestor', commit, b.head]).ok;
const problems = exceptionProblems(body, b.receipt, ancestor);
if (problems.length) return { ok: false, code: 'QUEUE_RECEIPT_EXCEPTION_INVALID', message: `the receipt's owner exception does not stand: ${problems.join('; ')}`, line: b.line };
const at = headInputs(root, baseRef, b.head);
const at = headInputs(root, baseRef, b.head, b.receipt, merged);
if (at.x.state === 'refused') return { ok: false, code: at.x.code, message: at.x.lines.join('; '), line: b.line };
const mismatch = merged ? null : excludeMismatch(at.x, b.receipt, b.head, baseRef);
if (mismatch) return { ok: false, code: 'RECEIPT_INVALID', message: mismatch, line: b.line };
const d = receiptDepthVerdict(root, baseRef, b.receipt, b.head, at);
const gateTagged = boundScan(b.receipt, b.head) ?? (gateScanNeeded(at.c.paths, at.c.root) ? null : 0);
const t = receiptTargetedVerdict(targetedClasses({ ...at.c, scope: at.scope, gateTagged }).classes, b.receipt);
const x = at.x.state === 'active' ? at.x.excluded : null;
const without = <T,>(rec: Record<string, T>) => (x ? Object.fromEntries(Object.entries(rec).filter(([p]) => !x.has(p))) : rec);
const t = receiptTargetedVerdict(targetedClasses({ ...at.c, ...(x ? { paths: at.c.paths.filter(p => !x.has(p)), added: without(at.c.added), removed: without(at.c.removed) } : {}), scope: at.scope, gateTagged }).classes, b.receipt);
if (!d.ok) return { ok: false, code: 'QUEUE_RECEIPT_DEPTH_RAISED', message: d.message!, line: b.line, depthLine: d.line, targetedLine: t.line };
if (!t.ok) return { ok: false, code: 'QUEUE_RECEIPT_TARGETED_MISSING', message: `${t.message}; the receipt is unchanged; run /ship on ${b.head.slice(0, 12)}`, line: b.line, depthLine: d.line, targetedLine: t.line };
return { ...b, depthLine: d.line, targetedLine: t.line };
@@ -235,7 +295,7 @@ export function bindShippedReceipt(root: string, baseRef: string, body: string,
*/
export function targetedClassesSince(root: string, from: string, head: string): readonly TargetedClass[] | null {
if (!git(root, ['merge-base', '--is-ancestor', from, head]).ok) return null;
const at = headInputs(root, from, head);
const at = headInputs(root, from, head, undefined, true);
let gateTagged: number | null = 0;
if (gateScanNeeded(at.c.paths, at.c.root)) {
const g = runScriptSync(path.join(BIN_DIR, 'gstack-gate-diff'), [from], { cwd: root, encoding: 'utf8', timeout: 120_000, maxBuffer: 16 * 1024 * 1024 });
+1 -1
View File
@@ -19,7 +19,7 @@ const KEY_ORDER: Array<keyof ShipReceipt | 'docs'> = [
'schema_version', 'pr', 'head', 'version', 'base', 'gated_tree', 'predecessor', 'tree', 'gate_reuse', 'gate', 'ci', 'spend_usd',
'session_kind', 'artifacts_consumed', 'run', 'thread', 'policy', 'queue_mode', 'preregistration', 'history',
'head_tree', 'product_patch_id', 'gstack_version', 'review_depth', 'pregate', 'requires_remote', 'issues', 'approval', 'docs', 'review_loop',
'review_loop_exception', 'review_targeted', 'carry', 'base_red',
'review_loop_exception', 'review_targeted', 'review_exclude', 'carry', 'base_red',
];
/** Keys in the fixed order so identifiers lead; undefined values are dropped. */