mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-15 04:47:23 +01:00
Merge commit '995e442521516c707125ecce388c698aad0ff432' into HEAD
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> # Conflicts: # bin/gstack-review-dispatch # lib/review-depth.ts # lib/review-targeted.ts # lib/ship-receipt-depth.ts # test/review-packets-removed.test.ts
This commit is contained in:
garrytan committed
commit
a7f5a044e8
31 files changed
+1378
-89
No files matched your search
@@ -119,6 +119,8 @@ export interface ShipReceipt {
|
||||
review_targeted?: { classes: string[]; rules: string; brief_sha: string; product_patch_id: string; status: 'done' | 'failed' | 'not-run'; findings: number; raised_by: string };
|
||||
/** Step 5 failures gstack-base-red proved are the base's own (lib/base-red.ts): a conditional receipt the queue refuses until /ship --resume rewrites it without this block. */
|
||||
base_red?: { cases: string[]; files: string[]; base: string; lane: string; expires_at: string };
|
||||
/** The base policy's review_exclude_paths at the reviewed head (lib/review-exclude.ts); absent when the base declares none. Carry and the queue compare `digest`. */
|
||||
review_exclude?: { source: string; patterns: string[]; applied: string[]; excluded: number; excluded_bytes: number; kept: number; gate_tagged: string[]; patterns_sha256: string; list_sha256: string; digest: string };
|
||||
}
|
||||
|
||||
export const RESULT_STATUSES: readonly ResultStatus[] = ['complete', 'reviewed', 'gate_pending', 'amend_pending', 'incomplete', 'interrupted', 'refused'];
|
||||
@@ -244,6 +246,7 @@ export const ARTIFACT_SCHEMAS = {
|
||||
base_red: { type: 'object', required: ['cases', 'files', 'base', 'lane', 'expires_at'], properties: { cases: strList, files: strList, base: str, lane: str, expires_at: str } },
|
||||
review_depth: { type: 'object', required: ['depth', 'class', 'runtime_lines', 'rules', 'base', 'gate_scan'], properties: { depth: { enum: ['light', 'standard', 'full'] }, class: { enum: ['docs', 'tests', 'code', 'gate'] }, runtime_lines: { type: 'integer', minimum: 0 }, rules: str, base: str, registry: nullableStr, gate_scan: { type: 'object', required: ['tagged', 'head'], properties: { tagged: { type: ['integer', 'null'], minimum: 0 }, head: str } }, reasons: strList, risk: { type: 'array', items: { enum: ['money', 'auth', 'secret', 'security', 'data'] } } } },
|
||||
review_targeted: { type: 'object', required: ['classes', 'rules', 'brief_sha', 'product_patch_id', 'status', 'findings', 'raised_by'], properties: { classes: { type: 'array', items: { enum: ['money', 'gate', 'auth', 'secret'] } }, rules: str, brief_sha: str, product_patch_id: str, status: { enum: ['done', 'failed', 'not-run'] }, findings: { type: 'integer', minimum: 0 }, raised_by: str } },
|
||||
review_exclude: { type: 'object', required: ['source', 'patterns', 'applied', 'excluded', 'excluded_bytes', 'kept', 'gate_tagged', 'patterns_sha256', 'list_sha256', 'digest'], properties: { source: str, patterns: strList, applied: strList, excluded: { type: 'integer', minimum: 0 }, excluded_bytes: { type: 'integer', minimum: 0 }, kept: { type: 'integer', minimum: 0 }, gate_tagged: strList, patterns_sha256: str, list_sha256: str, digest: str } },
|
||||
review_loop_exception: { type: 'object', required: ['accepted_by', 'date', 'via', 'reason', 'fix_check'], properties: { accepted_by: str, date: str, via: str, reason: str, fix_check: { type: 'object', required: ['head', 'findings'], properties: { head: str, findings: { type: 'integer', minimum: 0 }, files: strList } } } },
|
||||
issues: { type: 'object', properties: { fixes: { type: 'array', items: { type: 'integer' } }, refs: { type: 'array', items: { type: 'integer' } }, flakes: { type: 'array', items: { type: 'object', required: ['issue', 'run'], properties: { issue: { type: 'integer' }, run: str } } }, related: { type: 'array', items: { type: 'object', required: ['pr', 'author', 'closes'], properties: { pr: { type: 'integer' }, repo: str, author: str, closes: { type: 'array', items: { type: 'integer' } }, supersedes: { type: 'boolean' }, credit: { type: 'boolean' } } } } } },
|
||||
},
|
||||
|
||||
@@ -468,4 +468,20 @@ export const RELEASE_RESULT_CODES = {
|
||||
summary: 'under --policy-from head, the branch’s .gstack/pregate.json removes, shrinks or loosens something the trusted policy declares; a head policy may only add test lanes',
|
||||
fix: 'keep every key of the trusted policy as it is and only add dependencies globs with tests (or tests on an existing glob); land any other change, a [] declaration included, in its own PR on the base branch first',
|
||||
},
|
||||
// The reviewer diff ceiling and the base policy's review_exclude_paths (lib/review-exclude.ts, GST-508).
|
||||
REVIEW_DIFF_TOO_LARGE: {
|
||||
anchor: 'review-diff-too-large',
|
||||
summary: 'one reviewer\'s diff is over the 2,000,000-byte review ceiling, so /ship stopped before writing a manifest or a brief instead of handing a reader more than it can read',
|
||||
fix: 'follow the fix the line prints for where the base policy stands: land the printed review_exclude_paths JSON on the base as its own PR, then bring the base into the branch and rerun once; or narrow a pattern the kept-file bound skipped; or split the PR',
|
||||
},
|
||||
REVIEW_DIFF_UNREADABLE: {
|
||||
anchor: 'review-diff-unreadable',
|
||||
summary: 'git could not read a reviewer\'s diff (reason=timeout after timeout_ms, or reason=git-exit with git\'s exit code), so /ship stopped; the full stderr, with remote URLs redacted, is in the log= file',
|
||||
fix: 'timeout: rerun once unchanged, then split the PR or exclude its largest data folders on the base; git exit 128: git fetch origin <base>, then rerun once; anything else: stop and ask a person',
|
||||
},
|
||||
REVIEW_EXCLUDE_REFERENCE: {
|
||||
anchor: 'review-exclude-reference',
|
||||
summary: 'informational: a kept code file names an excluded folder (a directory reference, not a full file path), so its files stay excluded and readers see that code without the files it reads',
|
||||
fix: 'nothing to do for the review; if that code\'s correctness depends on the excluded files, narrow review_exclude_paths on the base so they stay in the diff',
|
||||
},
|
||||
} as const satisfies Record<string, ResultCode>;
|
||||
+17
-7
@@ -13,6 +13,11 @@
|
||||
* outside-voice diff_base, diff (the branch diff, which the bin reads from git)
|
||||
* fix-check fix_list, hunks, tests, callers (scripts/resolvers/fix-check.ts)
|
||||
* targeted classes, brief (review/specialists/targeted.md's text), checklist (its path), diff_base, stack, test_fw, learnings
|
||||
*
|
||||
* With the base policy's review_exclude_paths active (lib/review-exclude.ts) every role but fix-check also takes
|
||||
* policy_exclude (the `:(top,exclude,glob)` items, which the fixture-handling command of the security specialist, the
|
||||
* Red Team and the native adversarial pass gets), keep (the kept files' `:(top,literal)` items) and disclosure (the one
|
||||
* line naming what was left out). Unset, every brief renders byte for byte as before.
|
||||
*/
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import { createHash, randomUUID } from 'node:crypto';
|
||||
@@ -81,7 +86,7 @@ function fill(text: string, pairs: Array<[string, string]>, what: string): strin
|
||||
}
|
||||
const SPECIALIST_DIFF = 'run\n`DIFF_BASE=<diff-base> && git diff "$DIFF_BASE" <exclude>` to get the diff';
|
||||
|
||||
const PLACEHOLDERS = ['<diff-base>', '<exclude>', '<base>', '{checklist path}', '{STACK}', '{TEST_FW}', '{merged findings summary}', '{red-team checklist path}', '{learnings'];
|
||||
const PLACEHOLDERS = ['<diff-base>', '<exclude>', '<list-exclude>', '<base>', '{checklist path}', '{STACK}', '{TEST_FW}', '{merged findings summary}', '{red-team checklist path}', '{learnings'];
|
||||
|
||||
/**
|
||||
* Render one reader's brief. Throws BriefError on a missing slot, a section that no longer carries an anchor or placeholder, or a placeholder left unfilled.
|
||||
@@ -94,7 +99,9 @@ export function renderBrief(role: BriefRole, slots: Slots, t: SectionTexts): str
|
||||
const keys = ['findings', 'learnings'].filter(k => slots[k]?.trim());
|
||||
const held: Slots = { ...slots };
|
||||
for (const k of keys) held[k] = `\u0000${nonce}:${k}\u0000`;
|
||||
const text = renderRole(role, held, t);
|
||||
const rendered = renderRole(role, held, t);
|
||||
const keep = slots.keep ? ` Read the files kept despite a match as well: \`git diff ${slots.diff_base} -- ${slots.keep}\`.` : '';
|
||||
const text = slots.disclosure && role !== 'fix-check' && role !== 'outside-voice' ? `${rendered}\n\n${slots.disclosure}${keep}` : rendered;
|
||||
const left = role === 'fix-check' || role === 'outside-voice' ? [] : PLACEHOLDERS.filter(p => text.includes(p));
|
||||
if (left.length) throw new BriefError(`${role}: unfilled placeholder ${left.join(', ')}`);
|
||||
return keys.reduce((out, k) => out.split(held[k]!).join(slots[k]!.trim()), text);
|
||||
@@ -103,6 +110,8 @@ export function renderBrief(role: BriefRole, slots: Slots, t: SectionTexts): str
|
||||
function renderRole(role: BriefRole, slots: Slots, t: SectionTexts): string {
|
||||
const diffBase = role === 'fix-check' ? '' : need(slots, 'diff_base');
|
||||
if (diffBase && !SHA.test(diffBase)) throw new BriefError(`slot diff_base must be a literal commit sha, got ${JSON.stringify(diffBase)}`);
|
||||
const fixtures = (text: string) => text.split(' <exclude>`').join(slots.policy_exclude ? ` ${slots.policy_exclude}\`` : '`')
|
||||
.split('<list-exclude>`').join(slots.policy_exclude ? ` -- . ${slots.policy_exclude}\`` : '`');
|
||||
const common = (body: string, checklist: string) => fill(body, [
|
||||
...(checklist ? [['{checklist path}', checklist] as [string, string]] : []), ['<diff-base>', diffBase],
|
||||
['{TEST_FW}', slots.test_fw || 'unknown'], ['{STACK}', slots.stack || 'unknown'], ['{learnings or \'none\'}', slots.learnings?.trim() || 'none'],
|
||||
@@ -112,7 +121,7 @@ function renderRole(role: BriefRole, slots: Slots, t: SectionTexts): string {
|
||||
const name = need(slots, 'name');
|
||||
const checklist = need(slots, 'checklist');
|
||||
if (name !== 'security') return common(t.specialist, checklist);
|
||||
const body = fill(t.specialist, [[SPECIALIST_DIFF, 'set `DIFF_BASE=<diff-base>` first. ' + t.fixtures + ' Then read the diff that way']], role);
|
||||
const body = fill(t.specialist, [[SPECIALIST_DIFF, 'set `DIFF_BASE=<diff-base>` first. ' + fixtures(t.fixtures) + ' Then read the diff that way']], role);
|
||||
return `${t.framing}\n\n${common(body, checklist)}`;
|
||||
}
|
||||
case 'specialist-batched': {
|
||||
@@ -123,13 +132,14 @@ function renderRole(role: BriefRole, slots: Slots, t: SectionTexts): string {
|
||||
case 'red-team': {
|
||||
const body = fill(t.redTeam, [['{merged findings summary}', (slots.findings ?? '').trim() || '(none)'], ['by N specialists', `by ${need(slots, 'specialists')} specialists`],
|
||||
['{red-team checklist path}', need(slots, 'checklist')], ['<diff-base>', diffBase]], role);
|
||||
return `${t.framing}\n\n${t.fixtures}\n\n${body}`;
|
||||
return fixtures(`${t.framing}\n\n${t.fixtures}\n\n${body}`);
|
||||
}
|
||||
case 'adversarial-native':
|
||||
return fill(t.native, [['DIFF_BASE=$(git merge-base origin/<base> HEAD)', `DIFF_BASE=${diffBase}`]], role);
|
||||
return fixtures(fill(t.native, [['DIFF_BASE=$(git merge-base origin/<base> HEAD)', `DIFF_BASE=${diffBase}`]], role));
|
||||
case 'outside-voice': {
|
||||
const body = fill(t.outside, [['DIFF_BASE=$(git merge-base origin/<base> HEAD)', `DIFF_BASE=${diffBase}`]], role);
|
||||
return [body, '', `## Branch diff (git diff ${diffBase})`, FENCE + 'diff', (slots.diff ?? '').trim(), FENCE].join('\n');
|
||||
const filled = fill(t.outside, [['DIFF_BASE=$(git merge-base origin/<base> HEAD)', `DIFF_BASE=${diffBase}`]], role);
|
||||
const body = slots.policy_exclude ? filled.split('git diff "$DIFF_BASE".').join(`git diff "$DIFF_BASE" -- . ${slots.policy_exclude}${slots.keep ? ` and git diff "$DIFF_BASE" -- ${slots.keep}` : ''}.`) : filled;
|
||||
return [body, '', ...(slots.disclosure ? [slots.disclosure, ''] : []), `## Branch diff (git diff ${diffBase})`, FENCE + 'diff', (slots.diff ?? '').trim(), FENCE].join('\n');
|
||||
}
|
||||
case 'targeted': {
|
||||
const classes = parseClassList(need(slots, 'classes'));
|
||||
|
||||
+21
-5
@@ -58,6 +58,11 @@ const GATE_PATH = [
|
||||
/^lib\/review-(depth|collect|scope|targeted|risk)\.ts$/, /^review\/specialists\/targeted\.md$/, /^bin\/gstack-review-plan$/, /^lib\/ship-queue[^/]*\.ts$/, /^lib\/regen\.ts$/, /^\.gstack\/generated\.json$/, /^ship\/sections\/(review-army|adversarial)\.md\.tmpl$/, /^scripts\/resolvers\/review-army\.ts$/,
|
||||
];
|
||||
|
||||
/** The path rules lib/review-exclude.ts keeps a matched file for: a gate path, agent prose, or a data-shaped config file. */
|
||||
export const isGatePath = (p: string) => GATE_PATH.some(r => r.test(p));
|
||||
export const isAgentProse = (p: string) => AGENT_PROSE.some(r => r.test(p));
|
||||
export const isDataConfig = (p: string) => DATA_CONFIG.some(r => r.test(p));
|
||||
|
||||
/** Data files: JSON, CSV, TSV and raster images. YAML, TOML and SVG are never data here: deploy configs, API specs and executable SVG stay code. */
|
||||
export const DATA_PATH = /\.(json|jsonl|ndjson|csv|tsv|png|jpe?g|gif|webp|bmp|ico|avif)$/i;
|
||||
/** Data-shaped files tools read by convention, with no code naming them: always code. */
|
||||
@@ -136,13 +141,15 @@ export interface ClassInput {
|
||||
/** Added and deleted lines per path (lib/review-collect.ts): the risk classes read them; without them no line signal applies. */
|
||||
added?: Record<string, readonly string[]>;
|
||||
removed?: Record<string, readonly string[]>;
|
||||
/** Paths the base policy's review_exclude_paths leave out of every reader's diff (lib/review-exclude.ts): classified as docs. */
|
||||
excluded?: ReadonlySet<string>;
|
||||
}
|
||||
export interface ClassResult { class: DiffClass; reasons: string[]; errors: DepthError[]; runtimeLines: number | null; risk: RiskClass[] }
|
||||
|
||||
const sanitize = (s: string) => s.replace(/[\0-\x1f]+/g, ' ').slice(0, 200);
|
||||
const isOutput = (i: ClassInput, p: string) => !!i.registry?.outputs.length && matchesAny(i.registry.outputs, p);
|
||||
const unconsumedData = (i: ClassInput, p: string) => Array.isArray(i.consumed) && !i.consumed.includes(p) && dataCandidate(p, i.root);
|
||||
const pathClass = (i: ClassInput, p: string): PathClass => ((i.stampOnly ?? []).includes(p) || unconsumedData(i, p) ? 'docs' : classifyPath(p, i.root));
|
||||
const pathClass = (i: ClassInput, p: string): PathClass => (i.excluded?.has(p) || (i.stampOnly ?? []).includes(p) || unconsumedData(i, p) ? 'docs' : classifyPath(p, i.root));
|
||||
|
||||
/** The diff's class, raised to gate by any gate signal; `errors` lists every input that fails the depth closed; `runtimeLines` counts code-class lines outside the base registry's outputs. */
|
||||
export function classifyDiff(i: ClassInput): ClassResult {
|
||||
@@ -164,7 +171,7 @@ export function classifyDiff(i: ClassInput): ClassResult {
|
||||
raise('gate', 'gate scan unavailable');
|
||||
errors.push({ code: 'gate_scan_unavailable', detail: 'gstack-gate-diff reported no GATE_SUMMARY', input: 'gstack-gate-diff <base>', repair: 'run gstack-gate-diff <base> and read its GATE_ERROR line, or pass the recorded scan as --gate-tagged <n>' });
|
||||
} else if (i.gateTagged > 0) raise('gate', `gate scan tagged ${i.gateTagged} hunk(s)`);
|
||||
const deletedGuarded = (i.deleted ?? []).filter(p => classifyPath(p, i.root) !== 'docs');
|
||||
const deletedGuarded = (i.deleted ?? []).filter(p => !i.excluded?.has(p) && classifyPath(p, i.root) !== 'docs');
|
||||
if (deletedGuarded.length) raise('gate', `deletes ${deletedGuarded.length} code or test path(s)`);
|
||||
}
|
||||
if (CLASS_RANK[cls] >= CLASS_RANK.code) {
|
||||
@@ -309,9 +316,11 @@ export interface PlanInput extends ClassInput {
|
||||
tiers?: boolean;
|
||||
/** The targeted classes (lib/review-targeted.ts) and whether the targeted reader already read this product patch. */
|
||||
targeted?: { classes: readonly string[]; ran?: boolean };
|
||||
/** The base policy's exclusion: one `:(top,exclude,glob)` pathspec per applied pattern, added to every reader, and the kept files under them as `:(top,literal)` pathspecs. */
|
||||
exclusion?: { specs: readonly string[]; keep: readonly string[] };
|
||||
}
|
||||
/** One reviewer. `exclude` holds git pathspec argv items the reviewer's diff leaves out; `checklists` names what the batched specialist reads. */
|
||||
export interface Dispatch { role: Role; name: string; effort: Effort; exclude: string[]; checklists?: string[]; reason?: string }
|
||||
export interface Dispatch { role: Role; name: string; effort: Effort; exclude: string[]; checklists?: string[]; reason?: string; keep?: string[] }
|
||||
export interface ReviewPlan extends DepthResult {
|
||||
/** The depth this step runs at: light raised to standard by a fixed blocker, a CRITICAL or an open blocker. */
|
||||
stepDepth: Depth;
|
||||
@@ -384,6 +393,11 @@ export function diffCommand(d: Dispatch): string {
|
||||
return d.exclude.length ? `git diff <diff-base> -- . ${d.exclude.map(shellQuote).join(' ')}` : 'git diff <diff-base>';
|
||||
}
|
||||
|
||||
/** The second command a reader runs when the base policy's exclusion keeps files its patterns match (a git exclude beats any positive pathspec); null otherwise. */
|
||||
export function keepCommand(d: Dispatch): string | null {
|
||||
return d.keep?.length ? `git diff <diff-base> -- ${d.keep.map(shellQuote).join(' ')}` : null;
|
||||
}
|
||||
|
||||
/** The reviewers the next step dispatches at its depth, with each role's effort tier and excluded pathspecs. */
|
||||
export function planReview(i: PlanInput): ReviewPlan {
|
||||
const c = classifyDiff(i);
|
||||
@@ -396,8 +410,10 @@ export function planReview(i: PlanInput): ReviewPlan {
|
||||
const depthReasons = [...depth.reasons, ...(raise ? [`history:${h.fixedBlocking ? 'fixed-blocking' : h.blockingOpen ? 'blocking-open' : 'critical-found'}`] : [])];
|
||||
const skipped: string[] = [];
|
||||
const dispatches: Dispatch[] = [];
|
||||
const add = (role: Role, name: string, effort: Effort, exclude: string[] = [], extra: Partial<Dispatch> = {}) => dispatches.push({ role, name, effort, exclude, ...extra });
|
||||
const docsAndOutputs = i.paths.filter(p => (pathClass(i, p) === 'docs' && !(i.numbers.publishes && DOC_PATH.test(p))) || isOutput(i, p)).map(excludeSpec);
|
||||
const x = i.exclusion;
|
||||
const add = (role: Role, name: string, effort: Effort, exclude: string[] = [], extra: Partial<Dispatch> = {}) =>
|
||||
dispatches.push({ role, name, effort, exclude: x ? [...exclude, ...x.specs] : exclude, ...extra, ...(x?.keep.length ? { keep: [...x.keep] } : {}) });
|
||||
const docsAndOutputs = i.paths.filter(p => !i.excluded?.has(p) && ((pathClass(i, p) === 'docs' && !(i.numbers.publishes && DOC_PATH.test(p))) || isOutput(i, p))).map(excludeSpec);
|
||||
const numbersChanged = i.numbers.publishes && i.reviewedNumbers !== undefined && i.reviewedNumbers !== i.numbers.fingerprint;
|
||||
const firstVoices = h.fullPasses === 0 || !i.adversarialRan;
|
||||
const specialist: Effort = i.tiers === false ? 'host' : ROLE_EFFORT.specialist;
|
||||
|
||||
@@ -0,0 +1,415 @@
|
||||
/**
|
||||
* review-exclude — the base policy's review_exclude_paths applied to one range, and the
|
||||
* bounded read of each reviewer's diff (docs/ship-policy.md#review-exclude-paths, GST-508).
|
||||
*
|
||||
* The policy is read at the base commit the caller already resolved (loadPolicyAt), never
|
||||
* from the head: a branch cannot exclude its own files. Git owns the glob dialect: every
|
||||
* matched path comes from `git diff --raw -z` under one `:(top,glob)<pattern>` pathspec
|
||||
* per pattern (untracked files never count), so the receipt, the plan line and what
|
||||
* readers see are one set. A matched file is excluded only when it is
|
||||
* content (data, docs, binary or a .log) that no rule keeps; every kept file carries one
|
||||
* stable reason. Applied patterns become `:(top,exclude,glob)` pathspecs on every reader,
|
||||
* and kept files under them come back through a second `:(top,literal)` command, because a
|
||||
* git exclude beats any positive pathspec. That second command is bounded by its argument
|
||||
* bytes: a pattern whose kept files would pass KEEP_ARG_MAX_BYTES is not applied.
|
||||
*
|
||||
* readSelection reads one reader's diff once, with a timeout, and counts its bytes;
|
||||
* gstack-review-dispatch stops with REVIEW_DIFF_TOO_LARGE above REVIEW_DIFF_MAX_BYTES and
|
||||
* with REVIEW_DIFF_UNREADABLE when git fails, before it writes a manifest or a brief.
|
||||
*/
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import { createHash } from 'node:crypto';
|
||||
import * as fs from 'node:fs';
|
||||
import * as os from 'node:os';
|
||||
import * as path from 'node:path';
|
||||
import { collectLive } from './review-collect';
|
||||
import { DATA_PATH, classifyPath, isAgentProse, isDataConfig, isGatePath, type PathLines } from './review-depth';
|
||||
import { secretLine } from './review-targeted';
|
||||
import { cmpVersion, parseVersion } from './version-source';
|
||||
import { REVIEW_EXCLUDE_KEY_VERSION, SHIP_POLICY_PATH, isUnavailable, policyUnavailableLine } from './ship-policy';
|
||||
import { loadPolicyAt } from './ship-policy-at';
|
||||
|
||||
/** The most diff bytes one reader is handed (about 500K tokens); above it the pass stops instead of truncating. */
|
||||
export const REVIEW_DIFF_MAX_BYTES = 2_000_000;
|
||||
/** The most argument bytes the kept-file command may carry across all applied patterns (under Windows' 32,767-character command line). */
|
||||
export const KEEP_ARG_MAX_BYTES = 24_000;
|
||||
/** How long one reader's `git diff` may run. */
|
||||
export const REVIEW_DIFF_TIMEOUT_MS = 120_000;
|
||||
const MAX_BUFFER = 256 * 1024 * 1024;
|
||||
export const KEPT_REASONS = ['policy-dir', 'gate-path', 'executable', 'agent-prose', 'config', 'gate-tagged', 'code', 'secret-line', 'named-by-code', 'consumer-scan-unavailable', 'gate-scan-unavailable', 'link'] as const;
|
||||
export type KeptReason = (typeof KEPT_REASONS)[number];
|
||||
|
||||
/** What the receipt records and the queue and carry compare (by `digest`, never by `source`). */
|
||||
export interface ExcludeRecord {
|
||||
source: string; patterns: string[]; applied: string[]; skipped: Array<{ pattern: string; kept: number; bytes: number }>; unmatched: string[];
|
||||
excluded: number; excluded_bytes: number; kept: number; gate_tagged: string[]; patterns_sha256: string; list_sha256: string; digest: string;
|
||||
}
|
||||
export interface ActiveExclusion {
|
||||
state: 'active'; record: ExcludeRecord; excluded: Set<string>; specs: string[]; keep: string[];
|
||||
kept: Array<{ path: string; reason: KeptReason }>; content: Record<ContentKind, number>; references: Array<{ prefix: string; files: string[] }>;
|
||||
}
|
||||
export type Exclusion =
|
||||
| { state: 'none' }
|
||||
| { state: 'head-only'; patterns: string[]; source: string }
|
||||
| { state: 'refused'; code: 'POLICY_SOURCE_UNAVAILABLE' | 'POLICY_INVALID' | 'REVIEW_DIFF_UNREADABLE'; lines: string[] }
|
||||
| ActiveExclusion;
|
||||
|
||||
export interface ExcludeInput {
|
||||
root: string;
|
||||
/** The base commit whose policy applies, and the ref it was resolved from (origin/<base> or <base>). */
|
||||
baseSha: string; ref: string;
|
||||
/** The merge base, and the head commit; no head reads the working tree with untracked files. */
|
||||
mergeBase: string; head?: string;
|
||||
/** The collector's per-path lines (binary flags) and added lines (lib/review-collect.ts). */
|
||||
perPath: readonly PathLines[]; added: Record<string, readonly string[]>;
|
||||
/** Skill directory names (commit mode) or the repository root (live): markdown in a skill directory is code. */
|
||||
classRoot: string | readonly string[];
|
||||
/** Paths whose hunks the gate scan tagged; null when unknown. */
|
||||
gateTagged?: ReadonlySet<string> | null;
|
||||
/** The command name the refusal lines start with. */
|
||||
context: string;
|
||||
}
|
||||
|
||||
const sha256 = (s: string) => createHash('sha256').update(s).digest('hex');
|
||||
const shellQuote = (s: string) => `'${s.replace(/'/g, "'\\''")}'`;
|
||||
const literal = (p: string) => `:(top,literal)${p}`;
|
||||
const argBytes = (paths: Iterable<string>) => { let n = 0; for (const p of paths) n += Buffer.byteLength(shellQuote(literal(p))) + 1; return n; };
|
||||
|
||||
function git(root: string, args: string[], input?: string): { ok: boolean; status: number | null; out: string; err: string } {
|
||||
const r = spawnSync('git', ['-c', 'core.quotePath=false', ...args], { cwd: root, input, encoding: 'utf8', timeout: 120_000, maxBuffer: MAX_BUFFER });
|
||||
return { ok: r.status === 0, status: r.status, out: r.stdout ?? '', err: (r.stderr ?? '').trim() };
|
||||
}
|
||||
|
||||
/** The patterns a policy text declares, or [] for anything unreadable: only a notice reads the head's copy. */
|
||||
function declared(text: string | null): string[] {
|
||||
try { const v = JSON.parse(text ?? 'null')?.review_exclude_paths; return Array.isArray(v) ? v.filter((p): p is string => typeof p === 'string') : []; } catch { return []; }
|
||||
}
|
||||
function headPatterns(i: ExcludeInput): string[] {
|
||||
if (!i.head) { try { return declared(fs.readFileSync(path.join(i.root, SHIP_POLICY_PATH), 'utf8')); } catch { return []; } }
|
||||
const r = git(i.root, ['show', `${i.head}:${SHIP_POLICY_PATH}`]);
|
||||
return r.ok ? declared(r.out) : [];
|
||||
}
|
||||
|
||||
/** A pattern's literal directory prefix (`receipts/` for `receipts/**`), or '' when it starts with a glob. */
|
||||
export function literalPrefix(pattern: string): string {
|
||||
const at = pattern.search(/[*?[]/);
|
||||
const head = at < 0 ? pattern : pattern.slice(0, at);
|
||||
return head.slice(0, head.lastIndexOf('/') + 1);
|
||||
}
|
||||
|
||||
interface Matched { oldMode: string; newMode: string; oldSha: string; newSha: string }
|
||||
|
||||
/** Every changed path each pattern matches, with its modes, from git's raw diff; untracked files are never part of it, since no reader's git diff shows them and the receipt reads the committed head. */
|
||||
function select(i: ExcludeInput, patterns: readonly string[]): { byPattern: Map<string, string[]>; files: Map<string, Matched> } | { error: string } {
|
||||
const byPattern = new Map<string, string[]>();
|
||||
const files = new Map<string, Matched>();
|
||||
const range = i.head ? [i.mergeBase, i.head] : [i.mergeBase];
|
||||
for (const p of patterns) {
|
||||
const spec = `:(top,glob)${p}`;
|
||||
const raw = git(i.root, ['diff', '--raw', '-z', '--no-abbrev', '--no-renames', ...range, '--', spec]);
|
||||
if (!raw.ok) return { error: `git diff --raw ${range.join(' ')} -- ${spec}: ${raw.err.split('\n')[0] || `exit ${raw.status}`}` };
|
||||
const parts = raw.out.split('\0');
|
||||
const hit: string[] = [];
|
||||
for (let k = 0; k + 1 < parts.length; k += 2) {
|
||||
const [oldMode, newMode, oldSha, newSha] = parts[k]!.slice(1).split(' ');
|
||||
const f = parts[k + 1]!;
|
||||
if (!f) continue;
|
||||
hit.push(f);
|
||||
if (!files.has(f)) files.set(f, { oldMode: oldMode ?? '', newMode: newMode ?? '', oldSha: oldSha ?? '', newSha: newSha ?? '' });
|
||||
}
|
||||
byPattern.set(p, hit);
|
||||
}
|
||||
return { byPattern, files };
|
||||
}
|
||||
|
||||
/**
|
||||
* The matched paths git calls binary whose bytes hold a NUL in their first 8,000 (git's own test), read from the head blob,
|
||||
* the file on a live tree, or the old blob for a deletion. A branch's .gitattributes can make git call a text file binary;
|
||||
* only real bytes make it binary content here.
|
||||
*/
|
||||
function binaryContent(i: ExcludeInput, files: ReadonlyMap<string, Matched>, claimed: ReadonlySet<string>, unread: Set<string>): Set<string> {
|
||||
const out = new Set<string>();
|
||||
const real = (sha: string) => /^[0-9a-f]{40,64}$/.test(sha) && !/^0+$/.test(sha);
|
||||
const bySha = new Map<string, string[]>();
|
||||
for (const p of claimed) {
|
||||
const m = files.get(p);
|
||||
if (!m) continue;
|
||||
if (!i.head && m.newMode !== '000000') {
|
||||
try { const fd = fs.openSync(path.join(i.root, p), 'r'); const buf = Buffer.alloc(8000); const n = fs.readSync(fd, buf, 0, 8000, 0); fs.closeSync(fd); if (buf.subarray(0, n).includes(0)) out.add(p); } catch { unread.add(p); }
|
||||
continue;
|
||||
}
|
||||
const sha = real(m.newSha) ? m.newSha : real(m.oldSha) ? m.oldSha : '';
|
||||
if (sha) bySha.set(sha, [...(bySha.get(sha) ?? []), p]);
|
||||
}
|
||||
if (!bySha.size) return out;
|
||||
const r = spawnSync('git', ['cat-file', '--batch'], { cwd: i.root, input: [...bySha.keys()].join('\n') + '\n', timeout: 120_000, maxBuffer: MAX_BUFFER });
|
||||
const buf = r.stdout ?? Buffer.alloc(0);
|
||||
let at = 0;
|
||||
while (at < buf.length) {
|
||||
const nl = buf.indexOf(10, at);
|
||||
if (nl < 0) break;
|
||||
const [sha, , size] = buf.subarray(at, nl).toString('utf8').split(' ');
|
||||
const n = Number(size);
|
||||
if (!Number.isFinite(n)) { at = nl + 1; continue; }
|
||||
if (buf.subarray(nl + 1, nl + 1 + Math.min(n, 8000)).includes(0)) for (const p of bySha.get(sha!) ?? []) out.add(p);
|
||||
at = nl + 1 + n + 1;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** The kind of content a path holds when the policy may exclude it, or null for code (tests included). */
|
||||
export type ContentKind = 'binary' | 'data' | 'docs' | 'log';
|
||||
function contentKind(i: ExcludeInput, p: string, binary: ReadonlySet<string>): ContentKind | null {
|
||||
if (binary.has(p)) return 'binary';
|
||||
if (DATA_PATH.test(p) && !isDataConfig(p)) return 'data';
|
||||
if (/\.log$/i.test(p)) return 'log';
|
||||
return classifyPath(p, i.classRoot) === 'docs' ? 'docs' : null;
|
||||
}
|
||||
|
||||
/** Source and script extensions: a NUL in one does not make it excludable binary content. */
|
||||
const CODE_EXT = /\.(?:[cm]?[jt]sx?|py|rb|go|rs|java|kts?|scala|swift|exs?|php|cs|c|h|cc|cpp|hpp|mm?|dart|pl|pm|lua|sh|bash|zsh|fish|ps1|bat|cmd|vue|svelte|html?|wasm)$/i;
|
||||
|
||||
/** Why a matched file is kept by its path, mode and lines alone; null for content the policy may exclude. A binary file readers would only see as "Binary files differ" is content unless its directory, mode or gate tags keep it. */
|
||||
function pathReason(i: ExcludeInput, p: string, m: Matched, binary: ReadonlySet<string>): KeptReason | null {
|
||||
if (/^\.(gstack|github)\//.test(p)) return 'policy-dir';
|
||||
if (isGatePath(p)) return 'gate-path';
|
||||
if (m.oldMode === '100755' || m.newMode === '100755') return 'executable';
|
||||
if ([m.oldMode, m.newMode].some(x => x === '120000' || x === '160000')) return 'link';
|
||||
if (i.gateTagged?.has(p)) return 'gate-tagged';
|
||||
if (binary.has(p)) return CODE_EXT.test(p) ? 'code' : null;
|
||||
if (isAgentProse(p)) return 'agent-prose';
|
||||
if (isDataConfig(p)) return 'config';
|
||||
if (contentKind(i, p, binary) === null) return 'code';
|
||||
if ((i.added[p] ?? []).some(secretLine)) return 'secret-line';
|
||||
return null;
|
||||
}
|
||||
|
||||
/** A file whose full-path reference keeps excluded content: a code or test file (not docs, data, a .log or binary) outside the would-be-excluded set. */
|
||||
const consumer = (i: ExcludeInput, f: string, candidates: ReadonlySet<string>) =>
|
||||
!candidates.has(f) && !f.startsWith('.gstack/tmp/') && contentKind(i, f, new Set()) === null;
|
||||
|
||||
/** An uncapped `git grep -F -o -f` of fixed strings at the head (or the working tree) and `revs`: string -> the consumer files holding it; null when git grep failed. */
|
||||
function grepFixed(i: ExcludeInput, strings: readonly string[], candidates: ReadonlySet<string>, revs: readonly string[]): Map<string, Set<string>> | null {
|
||||
const found = new Map<string, Set<string>>();
|
||||
if (!strings.length) return found;
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-exclude-'));
|
||||
try {
|
||||
const list = path.join(dir, 'patterns');
|
||||
fs.writeFileSync(list, strings.join('\n') + '\n');
|
||||
const runs = [i.head ? { at: i.head, args: [i.head, '--'] } : { at: '', args: ['--'] }, ...revs.map(rev => ({ at: rev, args: [rev, '--'] }))];
|
||||
for (const run of runs) {
|
||||
const r = git(i.root, ['grep', '-I', '-F', '-o', '--null', '--no-line-number', '--no-column', '--no-color', '-f', list, ...run.args]);
|
||||
if (r.status !== 0 && r.status !== 1) return null;
|
||||
for (const line of r.out.split('\n')) {
|
||||
const parts = line.split('\0');
|
||||
const where = parts[0];
|
||||
const match = parts.length > 1 ? parts[parts.length - 1] : undefined;
|
||||
if (!where || !match) continue;
|
||||
const file = run.at ? where.slice(run.at.length + 1) : where;
|
||||
if (!consumer(i, file, candidates)) continue;
|
||||
(found.get(match) ?? found.set(match, new Set()).get(match)!).add(file);
|
||||
}
|
||||
}
|
||||
return found;
|
||||
} finally { fs.rmSync(dir, { recursive: true, force: true }); }
|
||||
}
|
||||
|
||||
/**
|
||||
* The base policy's exclusion for one range. `none` when neither side declares a pattern (callers then print and change
|
||||
* nothing); `head-only` when only the head does (a notice, never applied); `refused` when the base policy cannot be read or
|
||||
* does not validate, or git cannot list the matches; `active` otherwise. Named-by-code keeps a file only when a kept code
|
||||
* file names its full repository path (never its basename: receipt trees reuse result.json thousands of times); a git grep
|
||||
* failure keeps every candidate (consumer-scan-unavailable), and so does an unknown gate scan (gate-scan-unavailable).
|
||||
*/
|
||||
export function reviewExclude(i: ExcludeInput): Exclusion {
|
||||
const load = loadPolicyAt(i.root, i.baseSha, i.ref);
|
||||
if (isUnavailable(load)) return { state: 'refused', code: 'POLICY_SOURCE_UNAVAILABLE', lines: [policyUnavailableLine(i.context, load.base, load.detail)] };
|
||||
const source = `${i.ref}@${i.baseSha.slice(0, 12)}`;
|
||||
const base = i.ref.replace(/^origin\//, '');
|
||||
if (load.errors.length) {
|
||||
return { state: 'refused', code: 'POLICY_INVALID', lines: load.errors.map(e => `${i.context}: the base policy ${source}:${SHIP_POLICY_PATH} does not validate, so its review_exclude_paths cannot be read (${e.path || SHIP_POLICY_PATH}: ${e.message}); fix: run ` + '`' + `gstack-ship-policy validate --policy-from base --base ${base}` + '`' + ` and fix the file on ${base} with its own PR; nothing was planned (${e.code})`) };
|
||||
}
|
||||
const patterns = load.policy.review_exclude_paths;
|
||||
if (!patterns.length) { const head = headPatterns(i); return head.length ? { state: 'head-only', patterns: head, source } : { state: 'none' }; }
|
||||
const sel = select(i, patterns);
|
||||
if ('error' in sel) return { state: 'refused', code: 'REVIEW_DIFF_UNREADABLE', lines: [`${i.context}: git could not list the files review_exclude_paths matches (${sel.error}); fix: git fetch origin ${base}, then rerun once; nothing was planned (REVIEW_DIFF_UNREADABLE)`] };
|
||||
const unread = new Set<string>();
|
||||
const binary = binaryContent(i, sel.files, new Set(i.perPath.filter(e => e.binary && sel.files.has(e.path)).map(e => e.path)), unread);
|
||||
const reasons = new Map<string, KeptReason | null>();
|
||||
for (const [p, m] of sel.files) reasons.set(p, unread.has(p) ? 'consumer-scan-unavailable' : pathReason(i, p, m, binary));
|
||||
const candidates = new Set([...reasons].filter(([, r]) => r === null).map(([p]) => p));
|
||||
const named = grepFixed(i, [...candidates].filter(p => !p.includes('\n')), candidates, [i.mergeBase]);
|
||||
for (const p of candidates) {
|
||||
if (i.gateTagged === null) reasons.set(p, 'gate-scan-unavailable');
|
||||
else if (named === null || p.includes('\n')) reasons.set(p, 'consumer-scan-unavailable');
|
||||
else if (named.has(p)) reasons.set(p, 'named-by-code');
|
||||
}
|
||||
const applied: string[] = [];
|
||||
const skipped: ExcludeRecord['skipped'] = [];
|
||||
const keep = new Set<string>();
|
||||
for (const p of patterns) {
|
||||
const hit = sel.byPattern.get(p) ?? [];
|
||||
if (!hit.some(f => reasons.get(f) === null)) continue;
|
||||
const kept = hit.filter(f => reasons.get(f) !== null);
|
||||
const bytes = argBytes(new Set([...keep, ...kept]));
|
||||
if (bytes > KEEP_ARG_MAX_BYTES) { skipped.push({ pattern: p, kept: kept.length, bytes }); continue; }
|
||||
applied.push(p);
|
||||
for (const f of kept) keep.add(f);
|
||||
}
|
||||
const excluded = new Set(applied.flatMap(p => (sel.byPattern.get(p) ?? []).filter(f => reasons.get(f) === null)));
|
||||
const keptList = [...reasons].filter(([, r]) => r !== null).map(([p, r]) => ({ path: p, reason: r! })).sort((a, b) => (a.path < b.path ? -1 : a.path > b.path ? 1 : 0));
|
||||
const sortedExcluded = [...excluded].sort();
|
||||
const content: Record<ContentKind, number> = { binary: 0, data: 0, docs: 0, log: 0 };
|
||||
for (const p of sortedExcluded) content[contentKind(i, p, binary)!]++;
|
||||
const prefixes = [...new Set(applied.map(literalPrefix).filter(Boolean))];
|
||||
const refs = grepFixed(i, prefixes, excluded, []) ?? new Map<string, Set<string>>();
|
||||
const patterns_sha256 = sha256(JSON.stringify(patterns));
|
||||
const list_sha256 = sha256(JSON.stringify({ applied, excluded: sortedExcluded, kept: keptList.map(k => [k.path, k.reason]) }));
|
||||
const record: ExcludeRecord = {
|
||||
source, patterns: [...patterns], applied, skipped, unmatched: patterns.filter(p => !(sel.byPattern.get(p) ?? []).length),
|
||||
excluded: excluded.size, excluded_bytes: excludedBytes(i, sortedExcluded, sel.files), kept: keptList.length,
|
||||
gate_tagged: keptList.filter(k => k.reason === 'gate-tagged').map(k => k.path), patterns_sha256, list_sha256, digest: sha256(`${patterns_sha256}\n${list_sha256}`),
|
||||
};
|
||||
return {
|
||||
state: 'active', record, excluded, specs: applied.map(p => `:(top,exclude,glob)${p}`), keep: [...keep].sort().map(literal), kept: keptList, content,
|
||||
references: prefixes.filter(p => refs.has(p)).map(p => ({ prefix: p, files: [...refs.get(p)!].sort() })),
|
||||
};
|
||||
}
|
||||
|
||||
/** The head-side size of the excluded files: blob sizes at the head commit, file sizes on a live tree; a deletion counts 0. */
|
||||
function excludedBytes(i: ExcludeInput, excluded: readonly string[], files: ReadonlyMap<string, Matched>): number {
|
||||
if (!i.head) return excluded.reduce((n, p) => { try { return n + fs.statSync(path.join(i.root, p)).size; } catch { return n; } }, 0);
|
||||
const shas = excluded.map(p => files.get(p)?.newSha ?? '').filter(s => /^[0-9a-f]{40,64}$/.test(s) && !/^0+$/.test(s));
|
||||
if (!shas.length) return 0;
|
||||
const r = git(i.root, ['cat-file', '--batch-check=%(objectsize)'], shas.join('\n') + '\n');
|
||||
return r.out.split('\n').reduce((n, l) => n + (Number(l) || 0), 0);
|
||||
}
|
||||
|
||||
/** The plan's lines for an exclusion: none prints nothing, so a repository without the key plans byte for byte as before. */
|
||||
export function excludeLines(x: Exclusion, base: string): string[] {
|
||||
if (x.state === 'none' || x.state === 'refused') return [];
|
||||
if (x.state === 'head-only') return [`REVIEW_EXCLUDE: none (head sets review_exclude_paths; it applies after it lands on ${base}) patterns=${x.patterns.length} source=${x.source}`];
|
||||
const r = x.record;
|
||||
const skipped = r.skipped.length ? ` skipped=${r.skipped.map(s => `${s.pattern}:kept=${s.kept}:bytes=${s.bytes}`).join(',')} (kept files over ${KEEP_ARG_MAX_BYTES} argument bytes: pattern not applied)` : '';
|
||||
const out = [`REVIEW_EXCLUDE: patterns=${r.applied.length}/${r.patterns.length} excluded=${r.excluded} bytes=${r.excluded_bytes} kept=${r.kept} content=${Object.entries(x.content).filter(([, n]) => n).map(([k, n]) => `${k}:${n}`).join(',') || 'none'} source=${r.source} digest=${r.digest}${skipped}${r.unmatched.length ? ` unmatched=${r.unmatched.join(',')}` : ''}`];
|
||||
for (const k of x.kept.slice(0, 20)) out.push(`REVIEW_EXCLUDE_KEPT: ${k.reason} ${k.path}`);
|
||||
if (x.kept.length > 20) out.push(`REVIEW_EXCLUDE_KEPT: +${x.kept.length - 20} more (gstack-review-plan --json lists every kept file)`);
|
||||
for (const ref of x.references) out.push(`REVIEW_EXCLUDE_REFERENCE: ${ref.prefix} is named by ${ref.files.slice(0, 3).join(', ')}${ref.files.length > 3 ? ` (+${ref.files.length - 3} more)` : ''}; the files stay excluded, so readers see that code without the files it reads (REVIEW_EXCLUDE_REFERENCE)`);
|
||||
return out;
|
||||
}
|
||||
|
||||
/** The one line every brief gets under an active exclusion; it names the selection digest the receipt binds (verify's REVIEW_EXCLUDE_RECORD). */
|
||||
export function disclosureLine(r: Pick<ExcludeRecord, 'source' | 'excluded' | 'excluded_bytes' | 'applied' | 'kept' | 'digest'>): string {
|
||||
return `Excluded by base policy ${r.source} (selection ${r.digest}): ${r.excluded} files, ${r.excluded_bytes} bytes on this branch matching ${r.applied.join(' ') || 'no applied pattern'}. They are not in your diff; do not report on their absence. Kept despite a match: ${r.kept} files (listed in the plan).`;
|
||||
}
|
||||
/** The selection digest a disclosure line names, or undefined. */
|
||||
export const disclosureDigest = (line: string | undefined) => line?.match(/^Excluded by base policy \S+ \(selection ([0-9a-f]{64})\)/)?.[1];
|
||||
|
||||
export function parseTail(tail: string | undefined): string[] {
|
||||
const out: string[] = [];
|
||||
const re = /'((?:[^']|'\\'')*)'|(\S+)/g;
|
||||
for (let m = re.exec(tail ?? ''); m; m = re.exec(tail ?? '')) out.push(m[1] !== undefined ? m[1].replace(/'\\''/g, "'") : m[2]!);
|
||||
return out;
|
||||
}
|
||||
|
||||
export type ReaderDiff =
|
||||
| { ok: true; text: string; bytes: number; byDir: Array<[string, number]> }
|
||||
| { ok: false; reason: 'timeout' | 'git-exit'; exit: number | null; stderr: string; timeoutMs: number };
|
||||
|
||||
/** One reader's selection read once: `git diff <base> [-- pathspecs]`, then the kept-file command, with its bytes per top directory. */
|
||||
export function readSelection(root: string, diffBase: string, pathspecs: readonly string[], keep: readonly string[] = [], timeoutMs = REVIEW_DIFF_TIMEOUT_MS): ReaderDiff {
|
||||
let text = '';
|
||||
for (const specs of [pathspecs, ...(keep.length ? [keep] : [])]) {
|
||||
const tail = specs.length ? (specs[0] === '--' ? specs : ['--', ...specs]) : [];
|
||||
const r = spawnSync('git', ['-c', 'core.quotePath=false', 'diff', '--no-color', '--no-ext-diff', diffBase, ...tail], { cwd: root, encoding: 'utf8', timeout: timeoutMs, maxBuffer: MAX_BUFFER });
|
||||
const code = (r.error as NodeJS.ErrnoException | undefined)?.code;
|
||||
if (code === 'ENOBUFS') { text += r.stdout ?? ''; break; }
|
||||
if (code === 'ETIMEDOUT' || (r.signal && r.status === null)) return { ok: false, reason: 'timeout', exit: null, stderr: r.stderr ?? '', timeoutMs };
|
||||
if (r.status !== 0) return { ok: false, reason: 'git-exit', exit: r.status, stderr: r.stderr ?? '', timeoutMs };
|
||||
text += r.stdout;
|
||||
}
|
||||
const byDir = new Map<string, number>();
|
||||
const starts = [...text.matchAll(/^diff --git a\/(\S+)/gm)];
|
||||
starts.forEach((m, k) => {
|
||||
const end = k + 1 < starts.length ? starts[k + 1]!.index! : text.length;
|
||||
const dir = m[1]!.includes('/') ? m[1]!.split('/')[0]! : '(root)';
|
||||
byDir.set(dir, (byDir.get(dir) ?? 0) + Buffer.byteLength(text.slice(m.index!, end)));
|
||||
});
|
||||
return { ok: true, text, bytes: Buffer.byteLength(text), byDir: [...byDir].sort((a, b) => b[1] - a[1]) };
|
||||
}
|
||||
|
||||
/** Credential-bearing remote URLs and scp-style remotes out of git's stderr. */
|
||||
export const redact = (s: string) => s.replace(/\b[a-z][a-z0-9+.-]*:\/\/[^\s'"]+/gi, '<url>').replace(/\b[\w.-]+@[\w.-]+:[^\s'"]+/g, '<url>');
|
||||
|
||||
/** Where the over-ceiling selection's policy stands: decides the fix the line prints. */
|
||||
export interface CeilingContext { state: 'none' | 'head-only' | 'active'; base: string; source: string; floor?: string | null; headPatterns?: string[]; record?: Pick<ExcludeRecord, 'patterns' | 'skipped' | 'excluded'> }
|
||||
|
||||
/** The policy hunk to land, as JSON a person can paste: the patterns, and a floor at least this key's version. */
|
||||
function policyJson(patterns: readonly string[], floor?: string | null): string {
|
||||
const min = floor && parseVersion(floor) && cmpVersion(parseVersion(floor)!, parseVersion(REVIEW_EXCLUDE_KEY_VERSION)!) > 0 ? floor : REVIEW_EXCLUDE_KEY_VERSION;
|
||||
return JSON.stringify({ review_exclude_paths: patterns, min_gstack_version: min });
|
||||
}
|
||||
|
||||
/** Folders to suggest: the largest top directories, until what is left would fit. */
|
||||
function suggest(byDir: ReadonlyArray<[string, number]>, bytes: number): string[] {
|
||||
const out: string[] = [];
|
||||
let left = bytes;
|
||||
for (const [dir, n] of byDir) {
|
||||
if (left <= REVIEW_DIFF_MAX_BYTES || out.length >= 3) break;
|
||||
if (dir === '(root)') continue;
|
||||
out.push(`${dir}/**`);
|
||||
left -= n;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** The REVIEW_DIFF_TOO_LARGE line and object: the reader's bytes, the largest folders and a fix chosen by where the policy stands. */
|
||||
export function tooLargeLine(context: string, reader: string, d: { bytes: number; byDir: ReadonlyArray<[string, number]> }, c: CeilingContext): { line: string; json: Record<string, unknown> } {
|
||||
const top = d.byDir.slice(0, 5).map(([dir, n]) => `${dir}:${n}`).join(',') || 'none';
|
||||
const land = (patterns: readonly string[]) => `ask the repo owner to land ${policyJson(patterns, c.floor)} in ${SHIP_POLICY_PATH} on ${c.base} as its own PR (docs/ship-policy.md#review-exclude-paths), then bring ${c.base} into this branch and rerun once; if it is still over, split the PR`;
|
||||
const dirs = suggest(d.byDir, d.bytes);
|
||||
const s = c.record?.skipped[0];
|
||||
const fix = c.state === 'head-only' ? `this branch declares review_exclude_paths, but only the copy on ${c.base} applies: ${land(c.headPatterns ?? [])}`
|
||||
: s ? `pattern ${s.pattern} was not applied: its ${s.kept} kept files need ${s.bytes} argument bytes (more than ${KEEP_ARG_MAX_BYTES}); narrow the pattern on ${c.base} or move the kept files out of it, or split the PR`
|
||||
: c.state === 'active' ? `still over after excluding ${c.record?.excluded ?? 0} files; ${dirs.length ? land([...(c.record?.patterns ?? []), ...dirs.filter(p => !c.record?.patterns.includes(p))]) : 'split the PR'}`
|
||||
: dirs.length ? land(dirs) : 'split the PR';
|
||||
const line = `${context}: the reader's diff is over the review ceiling (reader=${reader} bytes=${d.bytes} max=${REVIEW_DIFF_MAX_BYTES} top=${top} policy=${c.source}); fix: ${fix}; nothing was written (REVIEW_DIFF_TOO_LARGE)`;
|
||||
return { line, json: { code: 'REVIEW_DIFF_TOO_LARGE', reader, bytes: d.bytes, max: REVIEW_DIFF_MAX_BYTES, top: d.byDir.slice(0, 5).map(([dir, bytes]) => ({ dir, bytes })), policy: c.source, state: c.state, fix, side_effects: 'none' } };
|
||||
}
|
||||
|
||||
/** The REVIEW_DIFF_UNREADABLE line: timeout or git exit, the timeout, the base, a redacted first stderr line, and the whole (redacted) stderr written under `logDir`. */
|
||||
export function unreadableLine(context: string, reader: string, d: Extract<ReaderDiff, { ok: false }>, diffBase: string, base: string, logDir: string): string {
|
||||
fs.mkdirSync(logDir, { recursive: true, mode: 0o700 });
|
||||
const log = path.join(logDir, `diff-${reader.replace(/[^a-zA-Z0-9._-]/g, '_')}.stderr`);
|
||||
fs.writeFileSync(log, redact(d.stderr), { mode: 0o600 });
|
||||
const first = redact(d.stderr.split('\n').find(l => l.trim()) ?? '').trim().slice(0, 200);
|
||||
const fix = d.reason === 'timeout' ? `rerun once unchanged; if it times out again, split the PR or exclude its largest data folders on ${base}`
|
||||
: d.exit === 128 ? `git fetch origin ${base}, then rerun once` : 'stop and ask a person: git failed in a way a rerun does not repair';
|
||||
return `${context}: git could not read the reader's diff (reader=${reader} reason=${d.reason} exit=${d.exit ?? 'none'} timeout_ms=${d.timeoutMs} base=${diffBase.slice(0, 12)} stderr=${JSON.stringify(first)} log=${log}); fix: ${fix}; nothing else was written (REVIEW_DIFF_UNREADABLE)`;
|
||||
}
|
||||
|
||||
/** The plan's review_exclude object (gstack-review-plan --json), or null for a repository without the key. */
|
||||
export type PlanExclude = { state: string; patterns?: string[]; skipped?: ExcludeRecord['skipped']; excluded?: number } | null;
|
||||
|
||||
/**
|
||||
* Where <base>'s policy stands for a REVIEW_DIFF_TOO_LARGE fix: the source commit, its min_gstack_version, and the exclusion
|
||||
* state, from `known` (the plan's review_exclude, null when it had none) or, when the caller has no plan, a fresh live run.
|
||||
*/
|
||||
export function ceilingContext(root: string, base: string, known?: PlanExclude): CeilingContext {
|
||||
const b = base.replace(/^origin\//, '');
|
||||
const ref = git(root, ['rev-parse', '--verify', '-q', `origin/${b}^{commit}`]).ok ? `origin/${b}` : b;
|
||||
const sha = git(root, ['rev-parse', '--verify', '-q', `${ref}^{commit}`]).out.trim();
|
||||
const load = sha ? loadPolicyAt(root, sha, ref) : null;
|
||||
const read = load && !isUnavailable(load) ? load : null;
|
||||
const source = sha ? `${ref}@${sha.slice(0, 12)}${read && !read.present ? ` (no ${SHIP_POLICY_PATH})` : ''}` : ref;
|
||||
let x = known;
|
||||
if (x === undefined) {
|
||||
const c = sha ? collectLive(root, b) : null;
|
||||
const live = c?.mergeBase ? reviewExclude({ root, baseSha: sha, ref, mergeBase: c.mergeBase, perPath: c.perPath, added: c.added, classRoot: root, context: 'ceiling' }) : { state: 'none' as const };
|
||||
x = live.state === 'active' ? { state: 'active', ...live.record } : live.state === 'head-only' ? live : null;
|
||||
}
|
||||
const floor = read?.policy.min_gstack_version ?? null;
|
||||
if (!x) return { state: 'none', base: b, source, floor };
|
||||
if (x.state === 'head-only') return { state: 'head-only', base: b, source, floor, headPatterns: x.patterns ?? [] };
|
||||
return { state: 'active', base: b, source, floor, record: { patterns: x.patterns ?? [], skipped: x.skipped ?? [], excluded: x.excluded ?? 0 } };
|
||||
}
|
||||
+15
-6
@@ -14,6 +14,7 @@
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import { BriefError, briefFile, checkoutRoot, checkoutTree, promptHash, renderBrief, withWorkspace, type BriefRole, type SectionTexts, type Slots } from './review-brief';
|
||||
import { disclosureDigest } from './review-exclude';
|
||||
import { appendEvent, readerKey } from './review-ledger';
|
||||
import { diffCommand, type Dispatch } from './review-depth';
|
||||
import { TARGETED_RULES, parseClassList, targetedBriefSha, targetedPatchId } from './review-targeted';
|
||||
@@ -31,6 +32,8 @@ export interface PassReader {
|
||||
/** An audit reader: its prompt is a file the parent wrote from its section; its output is returned, never collected. */
|
||||
prompt_file?: string; model?: PassModel;
|
||||
file?: string; sha256?: string; bytes?: number; chunks?: number; findings_sha256?: string;
|
||||
/** With the base policy's review_exclude_paths active: its `:(top,exclude,glob)` items, shell-quoted (the fixture command's slot), and the kept files' `:(top,literal)` items. */
|
||||
policy_exclude?: string; keep?: string[];
|
||||
/** The fingerprint the reader's reply must report: the checkout's HEAD^{tree} and toplevel when its brief was rendered (GST-482). */
|
||||
tree?: string | null;
|
||||
path?: string; output?: string; error?: string; state_at?: string;
|
||||
@@ -42,6 +45,8 @@ export interface Manifest {
|
||||
/** Step 11's own reading at its first dispatch, so its note never counts the pass's readers again. */
|
||||
before_voices?: { tokens: number; credits: number };
|
||||
log_starts?: Record<string, string>;
|
||||
/** The plan's review_exclude object (lib/review-exclude.ts) without its file lists, and the line every brief carries; absent without the key. */
|
||||
review_exclude?: Record<string, unknown>; disclosure?: string;
|
||||
/** Decision 3.5's files for a fix-check reader: the fix list, touched hunks, test results and callers. */
|
||||
fix_check?: FixCheckFiles;
|
||||
}
|
||||
@@ -75,6 +80,7 @@ export function detectTestFw(root: string): string {
|
||||
return fw || 'unknown';
|
||||
}
|
||||
|
||||
const shellQuote = (s: string) => `'${s.replace(/'/g, "'\\''")}'`;
|
||||
const safe = (reader: string) => reader.replace(/[^a-zA-Z0-9._-]/g, '_');
|
||||
const STAGE_OF: Record<string, Stage> = { specialist: 'specialists', 'fix-check': 'specialists', 'red-team': 'red-team', 'adversarial-native': 'voices', 'outside-voice': 'voices', targeted: 'voices' };
|
||||
|
||||
@@ -82,7 +88,7 @@ const STAGE_OF: Record<string, Stage> = { specialist: 'specialists', 'fix-check'
|
||||
export function buildManifest(o: {
|
||||
pass: string; out: string; ledger: string; base: string; diffBase: string; root: string; checklistRoot: string;
|
||||
dispatches: Dispatch[]; next: string; planLines: string[]; learnings?: string | null; extras?: Extra[];
|
||||
targetedClasses?: readonly string[]; mergeBase?: string; rangeBase?: string; fixCheck?: FixCheckFiles;
|
||||
targetedClasses?: readonly string[]; mergeBase?: string; rangeBase?: string; fixCheck?: FixCheckFiles; reviewExclude?: Record<string, unknown>; disclosure?: string;
|
||||
}): Manifest {
|
||||
const readers: PassReader[] = [];
|
||||
for (const d of o.dispatches) {
|
||||
@@ -93,7 +99,9 @@ export function buildManifest(o: {
|
||||
const name = role === 'specialist' ? d.name : undefined;
|
||||
const reader = readerKey(role, name);
|
||||
const exclude = d.exclude.length ? diffCommand(d).slice('git diff <diff-base> '.length) : undefined;
|
||||
const policy = d.exclude.filter(s => s.startsWith(':(top,exclude,glob)'));
|
||||
readers.push({ reader, stage, title: `ship ${o.pass} ${reader}`, state: 'planned', role, name, effort: d.effort, exclude, checklists: d.checklists,
|
||||
...(policy.length ? { policy_exclude: policy.map(shellQuote).join(' ') } : {}), ...(d.keep?.length ? { keep: d.keep } : {}),
|
||||
...(role === 'targeted' ? { classes: [...(o.targetedClasses ?? [])], diff_base: o.mergeBase ?? o.diffBase } : {}),
|
||||
...(role === 'red-team' && o.rangeBase ? { diff_base: o.rangeBase } : {}) });
|
||||
}
|
||||
@@ -109,7 +117,7 @@ export function buildManifest(o: {
|
||||
return {
|
||||
version: 1, pass: o.pass, out, ledger: path.resolve(o.ledger), merged: path.join(out, 'merged.jsonl'), base: o.base, diff_base: o.diffBase,
|
||||
stack: detectStack(o.root), test_fw: detectTestFw(o.root), learnings: o.learnings ?? null, checklist_root: o.checklistRoot,
|
||||
next: o.next, plan_lines: o.planLines, readers,
|
||||
next: o.next, plan_lines: o.planLines, readers, ...(o.reviewExclude ? { review_exclude: o.reviewExclude } : {}), ...(o.disclosure ? { disclosure: o.disclosure } : {}),
|
||||
...(o.fixCheck ? { fix_check: Object.fromEntries(Object.entries(o.fixCheck).filter(([, v]) => v).map(([k, v]) => [k, path.resolve(v as string)])) as unknown as FixCheckFiles } : {}),
|
||||
};
|
||||
}
|
||||
@@ -136,7 +144,7 @@ export function specialistFindings(mergedFile: string): string {
|
||||
}
|
||||
|
||||
/** Render every planned reader of a stage. The Red Team renders only after every earlier reader is collected. */
|
||||
export function renderStage(m: Manifest, stage: Stage, t: SectionTexts, diffFor: (base: string) => string): PassReader[] {
|
||||
export function renderStage(m: Manifest, stage: Stage, t: SectionTexts, diffFor: (base: string, r: PassReader) => string): PassReader[] {
|
||||
const open = m.readers.filter(r => (r.stage === 'first' || r.stage === 'specialists') && r.state !== 'collected');
|
||||
if (stage === 'red-team' && open.length) throw new BriefError(`the Red Team renders after every earlier reader is collected; still open: ${open.map(r => r.reader).join(', ')}`);
|
||||
const briefs = path.join(m.out, 'briefs');
|
||||
@@ -157,7 +165,8 @@ export function renderStage(m: Manifest, stage: Stage, t: SectionTexts, diffFor:
|
||||
}
|
||||
const role = r.role!;
|
||||
const slots: Slots = { name: r.name, diff_base: r.diff_base ?? m.diff_base, exclude: r.exclude, stack: m.stack, test_fw: m.test_fw, learnings: m.learnings ?? undefined,
|
||||
checklists: r.checklists?.map(n => path.join(m.checklist_root, 'review/specialists', `${n}.md`)).join(',') };
|
||||
checklists: r.checklists?.map(n => path.join(m.checklist_root, 'review/specialists', `${n}.md`)).join(','),
|
||||
...(m.disclosure ? { disclosure: m.disclosure, policy_exclude: r.policy_exclude, keep: r.keep?.map(shellQuote).join(' ') } : {}) };
|
||||
if (role === 'specialist') slots.checklist = path.join(m.checklist_root, 'review/specialists', `${r.name}.md`);
|
||||
let findingsSha: string | undefined;
|
||||
if (role === 'red-team') {
|
||||
@@ -167,7 +176,7 @@ export function renderStage(m: Manifest, stage: Stage, t: SectionTexts, diffFor:
|
||||
const specialists = m.readers.reduce((n, x) => n + (x.role === 'specialist' ? 1 : x.role === 'specialist-batched' ? (x.checklists ?? []).filter(c => c !== 'red-team').length : 0), 0);
|
||||
Object.assign(slots, { checklist: path.join(m.checklist_root, 'review/specialists/red-team.md'), findings, specialists: String(specialists) });
|
||||
}
|
||||
if (role === 'outside-voice') slots.diff = diffFor(m.diff_base);
|
||||
if (role === 'outside-voice') slots.diff = diffFor(m.diff_base, r);
|
||||
if (role === 'fix-check') {
|
||||
const read = (f: string | undefined, what: string) => { if (!f) return undefined; try { return fs.readFileSync(f, 'utf8'); } catch { throw new BriefError(`cannot read the fix-check ${what} ${f}`); } };
|
||||
const fc = m.fix_check!;
|
||||
@@ -186,7 +195,7 @@ export function renderStage(m: Manifest, stage: Stage, t: SectionTexts, diffFor:
|
||||
fs.writeFileSync(file, text, { mode: 0o600 });
|
||||
const sha = promptHash(text);
|
||||
const bytes = Buffer.byteLength(text);
|
||||
appendEvent(m.ledger, { event: 'rendered', reader: r.reader, role, name: r.name ?? null, file, sha256: sha, bytes, pass: m.pass, stage, workdir, tree, ...(findingsSha ? { findings_sha256: findingsSha } : {}), ...targeted });
|
||||
appendEvent(m.ledger, { event: 'rendered', reader: r.reader, role, name: r.name ?? null, file, sha256: sha, bytes, pass: m.pass, stage, workdir, tree, ...(findingsSha ? { findings_sha256: findingsSha } : {}), ...targeted, ...(disclosureDigest(m.disclosure) ? { exclude: disclosureDigest(m.disclosure) } : {}) });
|
||||
Object.assign(r, { file, sha256: sha, bytes, chunks: chunkCount(text), state: 'rendered', tree, ...(findingsSha ? { findings_sha256: findingsSha } : {}) });
|
||||
done.push(r);
|
||||
}
|
||||
|
||||
@@ -26,6 +26,9 @@ import { AUTH_PATH_PREFIXES, AUTH_PATH_TOKENS, KEY_NAME, MONEY_LINE, MONEY_LINE_
|
||||
export const TARGETED_CLASSES = ['money', 'gate', 'auth', 'secret'] as const;
|
||||
export type TargetedClass = (typeof TARGETED_CLASSES)[number];
|
||||
|
||||
/** Whether one added or deleted line reads a credential (SECRET_LINE): lib/review-exclude.ts keeps a matched file that has one. */
|
||||
export const secretLine = (line: string) => SECRET_LINE.some(r => r.test(line));
|
||||
|
||||
/** A hash of the patterns that decide the classes, not of this file's bytes: a comment edit keeps it, a pattern edit changes it. */
|
||||
export const TARGETED_RULES = createHash('sha256').update(JSON.stringify({
|
||||
TARGETED_CLASSES, MONEY_PATH_TOKENS, MONEY_LINE_TOKENS, MONEY_PAIRS, AUTH_PATH_TOKENS, AUTH_PATH_PREFIXES, KEY_NAME,
|
||||
|
||||
@@ -0,0 +1,34 @@
|
||||
/**
|
||||
* ship-policy-at — the ship policy committed at a commit the caller already
|
||||
* resolved (docs/ship-policy.md#review-exclude-paths). Unlike
|
||||
* loadPolicyFromBase, any git failure is `unavailable`, never a silent
|
||||
* default. Callers: lib/review-exclude.ts, lib/ship-receipt-depth.ts,
|
||||
* bin/gstack-review-plan.
|
||||
*/
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import { SHIP_POLICY_PATH, finish, type PolicyLoad, type PolicySource } from './ship-policy';
|
||||
|
||||
function git(repoRoot: string, args: string[]): { ok: boolean; out: string; err: string } {
|
||||
const r = spawnSync('git', args, { cwd: repoRoot, encoding: 'utf8', timeout: 30_000 });
|
||||
return { ok: r.status === 0, out: (r.stdout ?? '').trim(), err: (r.stderr ?? '').trim() };
|
||||
}
|
||||
|
||||
/**
|
||||
* The policy committed at `sha` (a commit the caller already resolved, labeled `ref`): `cat-file -e` first, and only an
|
||||
* `ls-tree` that succeeds with no entry is a missing file (defaults); any git failure, including a `show` failure after
|
||||
* the file was found, is `unavailable`, never a silent default.
|
||||
*/
|
||||
export function loadPolicyAt(repoRoot: string, sha: string, ref: string): PolicyLoad {
|
||||
const label = `${ref}@${sha.slice(0, 12)}`;
|
||||
const source: PolicySource = { kind: 'base', ref, sha, label: `${label}:${SHIP_POLICY_PATH}` };
|
||||
const spec = `${sha}:${SHIP_POLICY_PATH}`;
|
||||
const base = ref.replace(/^origin\//, '');
|
||||
if (!git(repoRoot, ['cat-file', '-e', spec]).ok) {
|
||||
const tree = git(repoRoot, ['ls-tree', sha, '--', SHIP_POLICY_PATH]);
|
||||
if (!tree.ok) return { unavailable: true, base, detail: `git ls-tree ${sha.slice(0, 12)} failed: ${tree.err.split('\n')[0] || 'no detail'}` };
|
||||
if (!tree.out) return finish(null, { ...source, kind: 'none', label: `${label} (no ${SHIP_POLICY_PATH})` });
|
||||
}
|
||||
const show = git(repoRoot, ['show', spec]);
|
||||
if (!show.ok) return { unavailable: true, base, detail: `git show ${label}:${SHIP_POLICY_PATH} failed: ${show.err.split('\n')[0] || 'no detail'}` };
|
||||
return finish(show.out, source);
|
||||
}
|
||||
+19
-2
@@ -53,6 +53,8 @@ export interface ShipPolicy {
|
||||
stamp_sensitive_gates: string[];
|
||||
/** The /ship review loop's finding scope (docs/ship-policy.md#review-scope). */
|
||||
review_scope: ReviewScope;
|
||||
/** Git glob pathspecs (root-anchored) whose data, docs, binary and log files /ship's reviewers leave out; read from the base only (docs/ship-policy.md#review-exclude-paths). */
|
||||
review_exclude_paths: string[];
|
||||
/** Consumed by the plan reviews (D1). */
|
||||
pr_count: { default_pr_count: number; split_requires_owner: boolean };
|
||||
/** Maintainer logins for contributor-PR mode (E3); empty means "use push permission". */
|
||||
@@ -153,6 +155,7 @@ export const DEFAULT_POLICY: ShipPolicy = {
|
||||
preregistration_shas: [],
|
||||
stamp_sensitive_gates: [],
|
||||
review_scope: 'frozen',
|
||||
review_exclude_paths: [],
|
||||
pr_count: { default_pr_count: 1, split_requires_owner: true },
|
||||
maintainers: [],
|
||||
stamp_files: [],
|
||||
@@ -184,6 +187,7 @@ export const POLICY_EXPLANATIONS: Record<PolicyKey, string> = {
|
||||
preregistration_shas: 'Commits that must be ancestors of HEAD before push (preregistered evals); checked by `gstack-evidence ancestor`.',
|
||||
stamp_sensitive_gates: 'Evidence labels (gstack-evidence --label) that read VERSION or generated bytes; the tree receipt never marks them reusable.',
|
||||
review_scope: 'frozen | diff-plus-blocking | all. frozen (default): /ship Step 9 fixes findings in the diff on pass 1, files blocking findings in code the PR did not change as issues, and later passes read only the range since the previous pass and fix only blocking findings; diff-plus-blocking: the same, but a blocking finding in code the PR did not change is fixed in this PR instead of filed; all: every finding is fixed now (the pre-1.91.249 loop).',
|
||||
review_exclude_paths: 'Git glob pathspecs, root-anchored and case-sensitive (`receipts/**`, `**/*.jsonl`; at most 64, 256 characters each, no leading `:` or `!`): /ship\'s reviewers leave out the data, docs, binary and .log files they match. Read from the base branch only, so a branch cannot exclude its own files. Code, gate paths, agent prose, config, executables, .gstack/ and .github/ files, gate-tagged or secret-bearing files and files a kept code or test file names by full path stay reviewed (docs/ship-policy.md#review-exclude-paths).',
|
||||
pr_count: '{ default_pr_count, split_requires_owner }: the PR-count row in plan reviews.',
|
||||
maintainers: 'GitHub logins treated as maintainers by /review contributor mode; empty uses repository push permission.',
|
||||
stamp_files: 'The repo’s stamp rule: globs of the only files that may differ by a stamp (version file, manifests, plugin trees, the PR’s own since: rows). Empty keeps gstack’s per-file rules; set, it applies before every rule.',
|
||||
@@ -231,6 +235,7 @@ export const BUMP_NEVER_KEY_VERSION = '1.91.353.0';
|
||||
export const BASE_RED_KEY_VERSION = '1.91.374.0';
|
||||
/** `train.gates` and `train.land` are values of the older train key, so compatErrors checks them by value, as for bump: never. */
|
||||
export const TRAIN_GATES_KEY_VERSION = '1.92.0.0';
|
||||
export const REVIEW_EXCLUDE_KEY_VERSION = '1.93.0.0';
|
||||
export const POLICY_KEY_INTRODUCED: Record<PolicyKey, string> = {
|
||||
bump: '1.91.73.0', queue_mode: '1.91.73.0', release_tool: '1.91.73.0', release_outputs: '1.91.73.0', stamp_paths: '1.91.73.0',
|
||||
changelog: '1.91.73.0', mirror: '1.91.73.0', history: '1.91.73.0', preregistration_shas: '1.91.73.0', stamp_sensitive_gates: '1.91.73.0',
|
||||
@@ -238,7 +243,7 @@ export const POLICY_KEY_INTRODUCED: Record<PolicyKey, string> = {
|
||||
migrations: '1.91.98.0', gate_command: SHIP_QUEUE_KEYS_VERSION, full_e2e: SHIP_QUEUE_KEYS_VERSION, min_gstack_version: SHIP_QUEUE_KEYS_VERSION,
|
||||
train: TRAIN_KEYS_VERSION, review_scope: REVIEW_SCOPE_KEY_VERSION, docs: DOCS_CAPS_KEY_VERSION,
|
||||
changelog_fragments: CHANGELOG_FRAGMENTS_KEY_VERSION,
|
||||
base_red: BASE_RED_KEY_VERSION,
|
||||
base_red: BASE_RED_KEY_VERSION, review_exclude_paths: REVIEW_EXCLUDE_KEY_VERSION,
|
||||
};
|
||||
|
||||
const newer = (a: string, b: string): boolean => cmpVersion(parseVersion(a)!, parseVersion(b)!) > 0;
|
||||
@@ -327,6 +332,16 @@ function globError(g: string): string | null {
|
||||
return null;
|
||||
}
|
||||
|
||||
/** `review_exclude_paths`: git glob pathspecs, each one repo-relative line with no pathspec magic of its own; every error is POLICY_INVALID at its index. */
|
||||
function excludeErrors(value: unknown, out: PolicyError[]): void {
|
||||
if (!Array.isArray(value)) { out.push({ code: 'POLICY_INVALID', path: 'review_exclude_paths', message: 'must be an array of git glob pathspecs' }); return; }
|
||||
if (value.length > MAX_LIST) out.push({ code: 'POLICY_INVALID', path: 'review_exclude_paths', message: `more than ${MAX_LIST} patterns` });
|
||||
value.forEach((p, i) => {
|
||||
const why = typeof p !== 'string' ? 'must be a string' : containedRelative(p) ?? (/^[:!]/.test(p) ? 'must not start with : or ! (no pathspec magic or negation)' : /[\x00-\x1f\x7f]/.test(p) ? 'contains a control character' : p.replace(/\/+$/, '') === '' || p === '.' ? 'must not name the whole repository' : null);
|
||||
if (why) out.push({ code: 'POLICY_INVALID', path: `review_exclude_paths[${i}]`, message: `${JSON.stringify(p)} ${why}` });
|
||||
});
|
||||
}
|
||||
|
||||
function templateErrors(value: unknown, out: PolicyError[]): void {
|
||||
if (!Array.isArray(value)) { out.push({ code: 'POLICY_INVALID', path: 'stamp_patterns', message: 'must be an array of literal templates' }); return; }
|
||||
if (value.length > MAX_TEMPLATES) out.push({ code: 'POLICY_CONTAINMENT', path: 'stamp_patterns', message: `more than ${MAX_TEMPLATES} templates` });
|
||||
@@ -471,6 +486,7 @@ export function validatePolicy(raw: unknown, opts: { gstackVersion?: string } =
|
||||
oneOf('queue_mode', ['claim-at-open', 'stamp-at-merge']);
|
||||
oneOf('history', ['merge-only', 'squash-ok']);
|
||||
oneOf('review_scope', ['frozen', 'diff-plus-blocking', 'all']);
|
||||
if ('review_exclude_paths' in obj) excludeErrors(obj.review_exclude_paths, errors);
|
||||
if ('release_tool' in obj && obj.release_tool !== null) {
|
||||
const t = obj.release_tool;
|
||||
if (typeof t !== 'string' || !t.trim()) errors.push({ code: 'POLICY_INVALID', path: 'release_tool', message: 'must be a non-empty string or null' });
|
||||
@@ -643,7 +659,7 @@ export function resolveBaseBranch(repoRoot: string, explicit?: string): string {
|
||||
return 'main';
|
||||
}
|
||||
|
||||
function finish(rawText: string | null, source: PolicySource): LoadedPolicy {
|
||||
export function finish(rawText: string | null, source: PolicySource): LoadedPolicy {
|
||||
if (rawText === null) return { policy: mergePolicy(null, source.label).policy, present: false, source, errors: [], values: mergePolicy(null, source.label).values };
|
||||
let raw: unknown;
|
||||
try { raw = JSON.parse(rawText); }
|
||||
@@ -733,6 +749,7 @@ export function shipConsequences(policy: ShipPolicy, base: string): string[] {
|
||||
: policy.review_scope === 'diff-plus-blocking'
|
||||
? 'review_scope diff-plus-blocking: Step 9.4 runs `gstack-review-scope classify --policy diff-plus-blocking` each pass; pass 1 fixes the diff and every blocking finding, in unchanged code too, later passes read only the range since the previous pass and fix only blocking findings; notes go to the PR body'
|
||||
: 'review_scope frozen: Step 9.4 runs `gstack-review-scope classify` each pass; pass 1 fixes the diff and defers blocking findings in unchanged code to issues, later passes read only the range since the previous pass and fix only blocking findings; notes go to the PR body');
|
||||
if (policy.review_exclude_paths.length) out.push(`review_exclude_paths ${policy.review_exclude_paths.join(' ')}: Step 9's reviewers leave out the data, docs, binary and .log files these git globs match on this branch; gstack-review-plan prints REVIEW_EXCLUDE: with the counts and the kept files, and every brief names what was left out (docs/ship-policy.md#review-exclude-paths)`);
|
||||
if (policy.preregistration_shas.length) out.push(`preregistration: \`gstack-evidence ancestor --from-policy --base ${base}\` must print only ANCESTOR: ok before push; the ship receipt records the result`);
|
||||
if (policy.release_tool) out.push(`release_tool: \`${policy.release_tool}\` runs only inside gstack-restamp (staging worktree, outputs confined to release_outputs)`);
|
||||
return out;
|
||||
|
||||
+69
-11
@@ -21,6 +21,9 @@ import { spawnSync } from 'node:child_process';
|
||||
import * as path from 'node:path';
|
||||
import type { ShipReceipt } from './headless-artifacts';
|
||||
import { collectCommit } from './review-collect';
|
||||
import { reviewExclude, type Exclusion } from './review-exclude';
|
||||
import { isUnavailable } from './ship-policy';
|
||||
import { loadPolicyAt } from './ship-policy-at';
|
||||
import { depthOfCommitInput, gateScanNeeded, type Depth } from './review-depth';
|
||||
import { TARGETED_CLASSES, classList, targetedClasses, type TargetedClass } from './review-targeted';
|
||||
import { runScriptSync } from './script-invocation';
|
||||
@@ -49,6 +52,24 @@ function scopeOfRange(root: string, binDir: string, base: string, head: string):
|
||||
return scope;
|
||||
}
|
||||
|
||||
export type ReceiptExclude = NonNullable<ShipReceipt['review_exclude']>;
|
||||
|
||||
/** The receipt's copy of an active exclusion: counts and hashes, plus the gate-tagged kept paths the queue cannot recompute. */
|
||||
export function receiptExclude(x: Exclusion): ReceiptExclude | undefined {
|
||||
if (x.state !== 'active') return undefined;
|
||||
const { source, patterns, applied, excluded, excluded_bytes, kept, gate_tagged, patterns_sha256, list_sha256, digest } = x.record;
|
||||
return { source, patterns, applied, excluded, excluded_bytes, kept, gate_tagged, patterns_sha256, list_sha256, digest };
|
||||
}
|
||||
|
||||
/** The exclusion for the committed range `baseSha`...`head` (collectCommit's), with the gate-tagged paths the caller knows. */
|
||||
function exclusionAt(root: string, ref: string, baseSha: string | null, head: string, c: ReturnType<typeof collectCommit>, gateTagged: ReadonlySet<string> | null, context: string): Exclusion {
|
||||
if (!baseSha || !c.mergeBase) return { state: 'none' };
|
||||
return reviewExclude({ root, baseSha, ref, mergeBase: c.mergeBase, head, perPath: c.perPath, added: c.added, classRoot: c.root ?? [], gateTagged, context });
|
||||
}
|
||||
|
||||
/** Whether the base policy declares patterns, so the gate scan's per-path tags are worth reading. */
|
||||
const declaresExclude = (root: string, ref: string, sha: string | null) => { if (!sha) return false; const l = loadPolicyAt(root, sha, ref); return !isUnavailable(l) && l.policy.review_exclude_paths.length > 0; };
|
||||
|
||||
/**
|
||||
* The depth record for `head`: the range from git objects (collectCommit) and
|
||||
* its diff-scope, as the queue recomputes them, plus, for a tests or code diff,
|
||||
@@ -57,20 +78,36 @@ function scopeOfRange(root: string, binDir: string, base: string, head: string):
|
||||
* unavailable and the depth fails closed to full.
|
||||
*/
|
||||
export function receiptDepthAtHead(repoRoot: string, base: string, binDir: string, head: string): ReceiptDepth {
|
||||
return receiptReviewAtHead(repoRoot, base, binDir, head).depth;
|
||||
}
|
||||
|
||||
/** receiptDepthAtHead plus the exclusion the head's review read under: the depth classifies excluded paths as docs (GST-508). */
|
||||
export function receiptReviewAtHead(repoRoot: string, base: string, binDir: string, head: string): { depth: ReceiptDepth; exclude: Exclusion } {
|
||||
const branch = base.replace(/^origin\//, '');
|
||||
const baseSha = git(repoRoot, ['rev-parse', '--verify', '--quiet', `origin/${branch}^{commit}`]);
|
||||
const baseRev = baseSha.ok ? baseSha.out : branch;
|
||||
const ref = baseSha.ok ? `origin/${branch}` : branch;
|
||||
const sha = baseSha.ok ? baseSha.out : (git(repoRoot, ['rev-parse', '--verify', '--quiet', `${branch}^{commit}`]).out || null);
|
||||
const c = collectCommit(repoRoot, baseRev, head);
|
||||
const scope = scopeOfRange(repoRoot, binDir, baseRev, head);
|
||||
const declares = declaresExclude(repoRoot, ref, sha);
|
||||
let tagged: number | null = 0;
|
||||
let paths: Set<string> | null = declares ? new Set() : null;
|
||||
if (gateScanNeeded(c.paths, c.root)) {
|
||||
const atHead = git(repoRoot, ['rev-parse', 'HEAD']).out === head && git(repoRoot, ['diff', '--quiet', 'HEAD', '--']).ok;
|
||||
const g = atHead ? runScriptSync(path.join(binDir, 'gstack-gate-diff'), [baseRev], { cwd: repoRoot, encoding: 'utf8', timeout: 120_000, maxBuffer: 16 * 1024 * 1024 }) : null;
|
||||
const g = atHead ? runScriptSync(path.join(binDir, 'gstack-gate-diff'), [baseRev, ...(declares ? ['--format', 'jsonl'] : [])], { cwd: repoRoot, encoding: 'utf8', timeout: 120_000, maxBuffer: 64 * 1024 * 1024 }) : null;
|
||||
const m = /GATE_SUMMARY: .*?tagged=\{([^}]*)\}/.exec(g?.stdout ?? '');
|
||||
tagged = g?.status === 0 && m ? m[1]!.split(',').reduce((n, kv) => n + (Number(kv.split(':')[1]) || 0), 0) : null;
|
||||
paths = declares && tagged !== null ? taggedPaths(g?.stdout ?? '') : null;
|
||||
}
|
||||
const d = depthOfCommitInput({ ...c, scope, gateTagged: tagged });
|
||||
return { depth: d.depth, class: d.class, runtime_lines: d.runtimeLines, rules: d.rules, base: baseRev, registry: c.registry.sha, gate_scan: { tagged, head }, reasons: d.reasons.slice(0, 12), risk: d.risk };
|
||||
const exclude = exclusionAt(repoRoot, ref, sha, head, c, paths, 'gstack-ship-receipt');
|
||||
const d = depthOfCommitInput({ ...c, scope, gateTagged: tagged, ...(exclude.state === 'active' ? { excluded: exclude.excluded } : {}) });
|
||||
return { depth: { depth: d.depth, class: d.class, runtime_lines: d.runtimeLines, rules: d.rules, base: baseRev, registry: c.registry.sha, gate_scan: { tagged, head }, reasons: d.reasons.slice(0, 12), risk: d.risk }, exclude };
|
||||
}
|
||||
|
||||
/** The paths of the hunks a `gstack-gate-diff --format jsonl` run tagged. */
|
||||
export function taggedPaths(jsonl: string): Set<string> {
|
||||
return new Set(jsonl.split('\n').slice(1).flatMap(l => { try { const h = JSON.parse(l) as { path?: string; tags?: unknown[] }; return h.path && h.tags?.length ? [h.path] : []; } catch { return []; } }));
|
||||
}
|
||||
|
||||
export const depthBelow = (have: Depth, need: Depth) => RANK[have] < RANK[need];
|
||||
@@ -154,11 +191,27 @@ export function receiptTargetedLine(required: readonly TargetedClass[], r: ShipR
|
||||
export interface DepthVerdict { ok: boolean; line: string; message?: string }
|
||||
|
||||
/** The head's committed range against the base, read once for the depth verdict and the targeted line. */
|
||||
interface HeadInputs { baseSha: { ok: boolean; out: string }; c: ReturnType<typeof collectCommit>; scope: Record<string, string> }
|
||||
function headInputs(root: string, baseRef: string, head: string): HeadInputs {
|
||||
interface HeadInputs { baseSha: { ok: boolean; out: string }; c: ReturnType<typeof collectCommit>; scope: Record<string, string>; x: Exclusion }
|
||||
function headInputs(root: string, baseRef: string, head: string, r?: ShipReceipt): HeadInputs {
|
||||
const baseSha = git(root, ['rev-parse', '--verify', '--quiet', `${baseRef}^{commit}`]);
|
||||
const baseRev = baseSha.ok ? baseSha.out : baseRef;
|
||||
return { baseSha, c: collectCommit(root, baseRev, head), scope: scopeOfRange(root, BIN_DIR, baseRev, head) };
|
||||
const c = collectCommit(root, baseRev, head);
|
||||
const tagged = r && boundScan(r, head) !== null ? new Set(r.review_exclude?.gate_tagged ?? []) : null;
|
||||
return { baseSha, c, scope: scopeOfRange(root, BIN_DIR, baseRev, head), x: exclusionAt(root, baseRef, baseSha.ok ? baseSha.out : null, head, c, tagged, 'gstack-ship-queue') };
|
||||
}
|
||||
|
||||
/**
|
||||
* The queue's exclusion check: the digest the receipt recorded (none when absent) against the one `head` has under the current
|
||||
* base policy. Only the digest counts (the policy's patterns and the excluded list), so a base merge or a moved base that leaves
|
||||
* both unchanged binds; a changed selection needs the review rerun, which receipt write alone cannot stand in for.
|
||||
*/
|
||||
export function excludeMismatch(x: Exclusion, r: ShipReceipt, head: string, base: string): string | null {
|
||||
const expected = x.state === 'active' ? x.record.digest : 'none';
|
||||
const recorded = r.review_exclude?.digest ?? 'none';
|
||||
if (expected === recorded) return null;
|
||||
const source = x.state === 'active' ? x.record.source : r.review_exclude?.source ?? base;
|
||||
const h = head.slice(0, 12);
|
||||
return `field=review_exclude expected=${expected} recorded=${recorded} source=${source}: the reviewers read another selection than ${base}'s review_exclude_paths gives ${h}; the receipt is unchanged; fix: rerun /ship's review on ${h} (gstack-review-dispatch pass --base ${base.replace(/^(refs\/remotes\/)?origin\//, '')}), then gstack-ship-receipt write --pr <n> --review-exclude <d> --body <body> with the digest gstack-review-dispatch verify --ledger <ledger> prints on REVIEW_EXCLUDE_RECORD:; a receipt write alone does not clear this`;
|
||||
}
|
||||
/** The receipt's recorded gate scan when it names this head; otherwise null (unavailable). */
|
||||
const boundScan = (r: ShipReceipt, head: string): number | null => {
|
||||
@@ -176,10 +229,10 @@ const boundScan = (r: ShipReceipt, head: string): number | null => {
|
||||
export function receiptDepthVerdict(root: string, baseRef: string, r: ShipReceipt, head: string, at?: HeadInputs): DepthVerdict {
|
||||
const rd = r.review_depth;
|
||||
if (!rd && !r.gstack_version) return { ok: true, line: 'REVIEW_DEPTH: legacy receipt (written before receipts recorded review_depth); accepted' };
|
||||
const { baseSha, c, scope } = at ?? headInputs(root, baseRef, head);
|
||||
const { baseSha, c, scope, x } = at ?? headInputs(root, baseRef, head, r);
|
||||
const tagged = boundScan(r, head);
|
||||
const scanBound = tagged !== null;
|
||||
const need = depthOfCommitInput({ ...c, scope, gateTagged: tagged });
|
||||
const need = depthOfCommitInput({ ...c, scope, gateTagged: tagged, ...(x.state === 'active' ? { excluded: x.excluded } : {}) });
|
||||
const have = rd?.depth;
|
||||
const inputs = `base@${short(baseSha.ok ? baseSha.out : null)},registry@${short(c.registry.sha)},gate_scan@${scanBound ? short(rd!.gate_scan.head) : 'missing'}`;
|
||||
const line = `REVIEW_DEPTH: receipt=${have ?? 'none'} required=${need.depth} runtime_lines=${need.runtimeLines} class=${need.class} reasons=${need.reasons.join(',')} inputs=${inputs} rounds=${r.review_loop?.rounds ?? (r.review_loop ? r.review_loop.passes + (r.review_loop.fix_checks ?? 0) : 'none')}`;
|
||||
@@ -192,16 +245,21 @@ export function receiptDepthVerdict(root: string, baseRef: string, r: ShipReceip
|
||||
/** bindReceipt plus the owner exceptions and the review depth: what READY and train build accept; the targeted line is reported, never refused on. */
|
||||
export function bindShippedReceipt(root: string, baseRef: string, body: string, pr: string | undefined, accepted: readonly string[]):
|
||||
| { ok: true; receipt: ShipReceipt; head: string; line: string; depthLine: string; targetedLine: string }
|
||||
| { ok: false; code: ReceiptBindingCode | 'QUEUE_RECEIPT_DEPTH_RAISED' | 'QUEUE_RECEIPT_EXCEPTION_INVALID'; message: string; line: string; depthLine?: string; targetedLine?: string } {
|
||||
| { ok: false; code: ReceiptBindingCode | 'QUEUE_RECEIPT_DEPTH_RAISED' | 'QUEUE_RECEIPT_EXCEPTION_INVALID' | 'RECEIPT_INVALID' | Extract<Exclusion, { state: 'refused' }>['code']; message: string; line: string; depthLine?: string; targetedLine?: string } {
|
||||
const b = bindReceipt(body, pr, accepted);
|
||||
if (!b.ok) return b;
|
||||
const ancestor = (commit: string) => b.receipt.carry?.ancestry === 'rewritten' || git(root, ['merge-base', '--is-ancestor', commit, b.head]).ok;
|
||||
const problems = exceptionProblems(body, b.receipt, ancestor);
|
||||
if (problems.length) return { ok: false, code: 'QUEUE_RECEIPT_EXCEPTION_INVALID', message: `the receipt's owner exception does not stand: ${problems.join('; ')}`, line: b.line };
|
||||
const at = headInputs(root, baseRef, b.head);
|
||||
const at = headInputs(root, baseRef, b.head, b.receipt);
|
||||
if (at.x.state === 'refused') return { ok: false, code: at.x.code, message: at.x.lines.join('; '), line: b.line };
|
||||
const mismatch = excludeMismatch(at.x, b.receipt, b.head, baseRef);
|
||||
if (mismatch) return { ok: false, code: 'RECEIPT_INVALID', message: mismatch, line: b.line };
|
||||
const d = receiptDepthVerdict(root, baseRef, b.receipt, b.head, at);
|
||||
const gateTagged = boundScan(b.receipt, b.head) ?? (gateScanNeeded(at.c.paths, at.c.root) ? null : 0);
|
||||
const targetedLine = receiptTargetedLine(targetedClasses({ ...at.c, scope: at.scope, gateTagged }).classes, b.receipt);
|
||||
const x = at.x.state === 'active' ? at.x.excluded : null;
|
||||
const without = <T,>(rec: Record<string, T>) => (x ? Object.fromEntries(Object.entries(rec).filter(([p]) => !x.has(p))) : rec);
|
||||
const targetedLine = receiptTargetedLine(targetedClasses({ ...at.c, ...(x ? { paths: at.c.paths.filter(p => !x.has(p)), added: without(at.c.added), removed: without(at.c.removed) } : {}), scope: at.scope, gateTagged }).classes, b.receipt);
|
||||
if (!d.ok) return { ok: false, code: 'QUEUE_RECEIPT_DEPTH_RAISED', message: d.message!, line: b.line, depthLine: d.line, targetedLine };
|
||||
return { ...b, depthLine: d.line, targetedLine };
|
||||
}
|
||||
|
||||
+1
-1
@@ -19,7 +19,7 @@ const KEY_ORDER: Array<keyof ShipReceipt | 'docs'> = [
|
||||
'schema_version', 'pr', 'head', 'version', 'base', 'gated_tree', 'predecessor', 'tree', 'gate_reuse', 'gate', 'ci', 'spend_usd',
|
||||
'session_kind', 'artifacts_consumed', 'run', 'thread', 'policy', 'queue_mode', 'preregistration', 'history',
|
||||
'head_tree', 'product_patch_id', 'gstack_version', 'review_depth', 'pregate', 'requires_remote', 'issues', 'approval', 'docs', 'review_loop',
|
||||
'review_loop_exception', 'review_targeted', 'carry', 'base_red',
|
||||
'review_loop_exception', 'review_targeted', 'review_exclude', 'carry', 'base_red',
|
||||
];
|
||||
|
||||
/** Keys in the fixed order so identifiers lead; undefined values are dropped. */
|
||||
|
||||
Reference in new issue
Block a user