mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-14 04:17:33 +01:00
Merge main bf2bcf14b into gsta258-c-review-depth (rule 19 grant, GSTA-36)
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
This commit is contained in:
commit
d67b8d39d5
33 files changed
+842
-104
No files matched your search
@@ -137,6 +137,27 @@ export function resolveBase(ref: string, cwd: string, head?: string): { ref: str
|
||||
return sha ? { ref, sha } : null;
|
||||
}
|
||||
|
||||
export type ResolvedBase = NonNullable<ReturnType<typeof resolveBase>>;
|
||||
|
||||
/** Why resolveBase read another ref than the name given (`local main is behind origin/main; base is origin/main`); null when the name stood as given. */
|
||||
export function baseNote(name: string, base: ResolvedBase): string | null {
|
||||
if (base.local === 'fallback') return `${base.unrelatedRemote ? `${base.unrelatedRemote} shares no history with HEAD` : `no remote-tracking branch for ${name}`}; base is the local ${name}, which may lag its remote`;
|
||||
if (!base.local) return null;
|
||||
return `${{ unrelated: `local ${name} shares no history with HEAD`, missing: `no local branch ${name}`, behind: `local ${name} is behind ${base.ref}`, ahead: `local ${name} is ahead of ${base.ref}`, diverged: `local ${name} has diverged from ${base.ref}` }[base.local]}; base is ${base.ref}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* A --base argument as the commit a review or ship bin reads (GST-1036): resolveBase against HEAD, so a bare `main`
|
||||
* reads origin/main over a stale local main, with the note on stderr as `<bin>: <note>` when it read another ref than
|
||||
* the name. An unresolvable name comes back as given, so the caller's own git error still names it.
|
||||
*/
|
||||
export function baseArg(bin: string, name: string, cwd: string): string {
|
||||
const base = resolveBase(name, cwd, 'HEAD');
|
||||
const note = base && baseNote(name, base);
|
||||
if (note) process.stderr.write(`${bin}: ${note}\n`);
|
||||
return base?.sha ?? name;
|
||||
}
|
||||
|
||||
/** A word for a printed shell line: as is when it is plain, single-quoted otherwise (ref names may hold `;`, `$`, `(`, `{1,2}`). */
|
||||
function shellWord(s: string): string {
|
||||
return /^[A-Za-z0-9._\/@^~+:-]+$/.test(s) ? s : `'${s.replace(/'/g, "'\\''")}'`;
|
||||
|
||||
+4
-1
@@ -17,6 +17,7 @@
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import { resolveBase } from './gate-diff/git';
|
||||
import { findSection } from './markdown-sections';
|
||||
import { scan } from './redact-engine';
|
||||
import type { ResultCodeName } from './result-codes';
|
||||
@@ -55,9 +56,11 @@ export function readCapture(file: string, roots: string[], what: string): string
|
||||
return cleanText(fs.readFileSync(real), `${what} file`);
|
||||
}
|
||||
|
||||
/** Resolve `ref` to a commit sha without letting it be read as an option. */
|
||||
/** Resolve `ref` to a commit sha without letting it be read as an option; a bare branch name reads origin's tracking branch first (resolveBase, GST-1036). */
|
||||
export function resolveCommit(cwd: string, ref: string): string | ShotError {
|
||||
if (!ref || ref.startsWith('-') || /\s/.test(ref)) return { code: 'PR_SHOTS_INPUT_REFUSED', message: `ref ${JSON.stringify(ref)} is not a plain ref` };
|
||||
const base = resolveBase(ref, cwd, 'HEAD');
|
||||
if (base) return base.sha;
|
||||
const r = git(cwd, ['rev-parse', '--verify', '--quiet', '--end-of-options', `${ref}^{commit}`]);
|
||||
return r.status === 0 ? r.stdout.trim() : { code: 'PR_SHOTS_INPUT_NOT_FOUND', message: `base ref ${ref}` };
|
||||
}
|
||||
|
||||
+12
-5
@@ -10,6 +10,7 @@
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import { resolveBase, type ResolvedBase } from './gate-diff/git';
|
||||
import { GENERATED_PATH, parseRegistry } from './regen';
|
||||
import { RESULT_CODES } from './result-codes';
|
||||
import { classifyPath, dataCandidate, svgCandidate, svgDrawing, type ClassInput, type DepthError, type PathLines, type RegistryInfo } from './review-depth';
|
||||
@@ -31,6 +32,8 @@ export interface Collected extends ClassInput {
|
||||
untracked: string[];
|
||||
consumed: string[] | 'all';
|
||||
drawings: string[];
|
||||
/** collectLive only: the ref resolveBase read for the base name (GST-1036), null when nothing resolved. */
|
||||
base?: ResolvedBase | null;
|
||||
}
|
||||
|
||||
function git(root: string, args: string[]): { ok: boolean; status: number | null; out: string; err: string } {
|
||||
@@ -255,16 +258,20 @@ export function collectCommit(root: string, baseSha: string, headSha: string): C
|
||||
return result(diff, { root: skillDirs, registry, errors, mergeBase, untracked: [], consumed, drawings });
|
||||
}
|
||||
|
||||
/** The working tree against the merge base with origin/<base> (or <base>; a remote-qualified base reads refs/remotes/<base> only), untracked files counted per path; the registry is the base ref's commit. */
|
||||
/**
|
||||
* The working tree against the merge base with the base resolveBase reads (lib/gate-diff/git.ts: a bare name is
|
||||
* origin/<base> when it shares history with HEAD, else the local branch; a remote-qualified base reads
|
||||
* refs/remotes/<base> only), untracked files counted per path; the registry is the base ref's commit.
|
||||
*/
|
||||
export function collectLive(root: string, base: string): Collected {
|
||||
const errors: DepthError[] = [];
|
||||
const qualified = base.startsWith('origin/') || git(root, ['remote']).out.split('\n').some(r => r && base.startsWith(`${r}/`));
|
||||
const ref = (qualified ? [`refs/remotes/${base}`] : [`refs/remotes/origin/${base}`, `refs/heads/${base}`]).find(r => git(root, ['rev-parse', '--verify', '-q', `${r}^{commit}`]).ok) ?? base;
|
||||
const resolved = resolveBase(base, root, 'HEAD');
|
||||
const ref = resolved?.sha ?? base;
|
||||
const registry = loadRegistryAt(root, ref);
|
||||
const mb = git(root, ['merge-base', ref, 'HEAD']);
|
||||
if (!mb.ok) {
|
||||
errors.push({ code: 'numstat_failed', detail: oneLine(mb.err || `no merge base with ${ref}`), input: `git merge-base ${ref} HEAD`, repair: `fetch the base first (git fetch origin ${base})` });
|
||||
return result({ perPath: [], deleted: [], stampOnly: [], created: [], added: {}, removed: {} }, { root, registry, errors, mergeBase: null, untracked: [], consumed: 'all' });
|
||||
return { ...result({ perPath: [], deleted: [], stampOnly: [], created: [], added: {}, removed: {} }, { root, registry, errors, mergeBase: null, untracked: [], consumed: 'all' }), base: resolved };
|
||||
}
|
||||
const mergeBase = mb.out.trim();
|
||||
const diff = readDiff(root, [mergeBase], errors);
|
||||
@@ -284,5 +291,5 @@ export function collectLive(root: string, base: string): Collected {
|
||||
const drawings = drawingsAmong(diff.perPath, root, p => {
|
||||
try { return fs.statSync(path.join(root, p)).size <= DRAWING_READ_CAP ? fs.readFileSync(path.join(root, p)) : null; } catch { return null; }
|
||||
});
|
||||
return result(diff, { root, registry, errors, mergeBase, untracked, consumed, drawings });
|
||||
return { ...result(diff, { root, registry, errors, mergeBase, untracked, consumed, drawings }), base: resolved };
|
||||
}
|
||||
+32
-21
@@ -95,7 +95,16 @@ const SVG_MARKUP = /^<(?:!--|!\[CDATA\[|\?xml\s|!DOCTYPE\s+svg(?:\s+PUBLIC\s+(["
|
||||
export const SVG_REFUSED = [/&(?!(?:amp|lt|gt|quot|apos);)/, /\\/, /@import/i, /url\((?!\s*#|['"]\s*#)/i, /expression\s*\(/i, /-moz-binding|\bbehavior\s*:/i];
|
||||
/** A drawing's CSS, a style element's text or a style attribute's value, holds declarations only: no @ rule and no function but url(#id). */
|
||||
const SVG_STYLE_OPEN = /<(?:[A-Za-z_][\w.-]*:)?style\b/gi;
|
||||
const SVG_STYLE_CLOSE = /^<\/(?:[A-Za-z_][\w.-]*:)?style\s*>/i;
|
||||
const SVG_STYLE_CLOSE = /^<\/(?:[A-Za-z_][\w.-]*:)?style[ \t\n\r]*>/i;
|
||||
/**
|
||||
* A style element's opening tag, read as quoted name="value" pairs separated by the ASCII whitespace XML and HTML share: any
|
||||
* other shape (a value holding < or >, an unquoted or valueless attribute, />, another space) refuses, so it cannot hide a closing tag.
|
||||
*/
|
||||
const SVG_STYLE_TAG = /^<(?:[A-Za-z_][\w.-]*:)?style(?:[ \t\n\r]+[\w.:-]+[ \t\n\r]*=[ \t\n\r]*(?:"[^"<>]*"|'[^'<>]*'))*[ \t\n\r]*>/i;
|
||||
/** An element name, ended by ASCII whitespace, / or >: a non-ASCII character in the name (a prefix such as gé:) refuses rather than reading as its ASCII start. */
|
||||
const SVG_TAG_NAME = /^<\/?(?:[A-Za-z_][\w.-]*:)?([A-Za-z_][\w.-]*)(?=[ \t\n\r/>]|$)/;
|
||||
/** Every name= in the text, non-ASCII letters included so a prefix such as xé:href is read whole and refused. */
|
||||
const SVG_ATTR = /(?<![\w.:\-\u0080-\uffff])([A-Za-z_\u0080-\uffff][\w.:\-\u0080-\uffff]*)\s*=\s*(?=(?:"([^"]*)"|'([^']*)'|([^\s"'<>=]+)))/g;
|
||||
const SVG_CSS_REF = /url\(\s*(['"]?)#[\w.:-]*\1\s*\)/gi;
|
||||
const SVG_CSS_REFUSED = /[@(]/;
|
||||
/** An href or src value a drawing may hold: a same-document reference, or a raster image inline. */
|
||||
@@ -104,10 +113,12 @@ const SVG_HREF = /^\s*(?:#|data:image\/(?:png|jpe?g|gif|webp);base64,[A-Za-z0-9+
|
||||
/**
|
||||
* Whether an SVG's whole text is a drawing (GST-911): every element in SVG_ELEMENTS, no event handler, every href or src a
|
||||
* same-document reference or an inline raster image, CSS of declarations only, and nothing SVG_REFUSED names, read after character references are
|
||||
* decoded. Every name= in the text is read, so a stray quote cannot hide a later attribute inside its value; an unquoted value
|
||||
* stops at the next =, which keeps the scan linear. A style element's CSS is walked with indexOf to its closing tag, dropping
|
||||
* comments and reading CDATA as CSS, so neither can hide a closing tag; any other markup inside it, or a style element that never
|
||||
* closes, refuses. Anything else, an SVG that can run code included, stays code.
|
||||
* decoded. Every name= in the text is read (SVG_ATTR), so a stray quote cannot hide a later attribute inside its value, and a name
|
||||
* holding a non-ASCII character refuses, like an element name (SVG_TAG_NAME); an unquoted value
|
||||
* stops at the next =, which keeps the scan linear. A style element's opening tag is read quote by quote (SVG_STYLE_TAG), and its CSS is walked with indexOf to its closing tag over the raw
|
||||
* text, so a character reference cannot fake a closing tag or a comment, and any & in it refuses (GST-940). Comments and CDATA are
|
||||
* read as CSS, so neither can hide a closing tag and an HTML-only comment end (<!--> or --!>) exposes nothing unread; any other
|
||||
* markup inside it, or a style element that never closes, refuses. Anything else, an SVG that can run code included, stays code.
|
||||
*/
|
||||
export function svgDrawing(text: string): boolean {
|
||||
const t = decodeCharRefs(text);
|
||||
@@ -115,36 +126,36 @@ export function svgDrawing(text: string): boolean {
|
||||
const cssDrawing = (css: string) => !SVG_CSS_REFUSED.test(css.replace(SVG_CSS_REF, ''));
|
||||
for (let at = 0; ;) {
|
||||
SVG_STYLE_OPEN.lastIndex = at;
|
||||
const open = SVG_STYLE_OPEN.exec(t);
|
||||
const open = SVG_STYLE_OPEN.exec(text);
|
||||
if (!open) break;
|
||||
let i = t.indexOf('>', open.index) + 1;
|
||||
if (!i) return false;
|
||||
const tag = SVG_STYLE_TAG.exec(text.slice(open.index, open.index + 4096));
|
||||
if (!tag) return false;
|
||||
let i = open.index + tag[0].length;
|
||||
let css = '';
|
||||
for (;;) {
|
||||
const lt = t.indexOf('<', i);
|
||||
const lt = text.indexOf('<', i);
|
||||
if (lt === -1) return false;
|
||||
css += t.slice(i, lt);
|
||||
if (t.startsWith('<!--', lt)) {
|
||||
i = t.indexOf('-->', lt + 4) + 3;
|
||||
if (i === 2) return false;
|
||||
} else if (t.startsWith('<![CDATA[', lt)) {
|
||||
const end = t.indexOf(']]>', lt + 9);
|
||||
css += text.slice(i, lt);
|
||||
const [start, close] = text.startsWith('<!--', lt) ? [lt + 4, '-->'] : text.startsWith('<![CDATA[', lt) ? [lt + 9, ']]>'] : [-1, ''];
|
||||
if (start !== -1) {
|
||||
const end = text.indexOf(close, start);
|
||||
if (end === -1) return false;
|
||||
css += t.slice(lt + 9, end);
|
||||
css += ' ' + text.slice(start, end) + ' ';
|
||||
i = end + 3;
|
||||
} else if (SVG_STYLE_CLOSE.test(t.slice(lt, lt + 256))) {
|
||||
} else if (SVG_STYLE_CLOSE.test(text.slice(lt, lt + 256))) {
|
||||
at = lt + 1;
|
||||
break;
|
||||
} else return false;
|
||||
}
|
||||
if (!cssDrawing(css)) return false;
|
||||
if (css.includes('&') || !cssDrawing(css)) return false;
|
||||
}
|
||||
for (let at = t.indexOf('<'); at !== -1; at = t.indexOf('<', at + 1)) {
|
||||
if (SVG_MARKUP.test(t.slice(at, at + 512))) continue;
|
||||
const tag = /^<\/?(?:[A-Za-z_][\w.-]*:)?([A-Za-z_][\w.-]*)/.exec(t.slice(at, at + 256));
|
||||
const tag = SVG_TAG_NAME.exec(t.slice(at, at + 256));
|
||||
if (!tag || !SVG_ELEMENTS.has(tag[1]!.toLowerCase())) return false;
|
||||
}
|
||||
for (const m of t.matchAll(/(?<![\w.:-])([A-Za-z_][\w.:-]*)\s*=\s*(?=(?:"([^"]*)"|'([^']*)'|([^\s"'<>=]+)))/g)) {
|
||||
for (const m of t.matchAll(SVG_ATTR)) {
|
||||
if (/[^\x00-\x7f]/.test(m[1]!)) return false;
|
||||
const local = m[1]!.slice(m[1]!.lastIndexOf(':') + 1).toLowerCase();
|
||||
const value = m[2] ?? m[3] ?? m[4] ?? '';
|
||||
if (local.startsWith('on') || ((local === 'href' || local === 'src') && !SVG_HREF.test(value)) || (local === 'style' && !cssDrawing(value))) return false;
|
||||
@@ -155,7 +166,7 @@ export function svgDrawing(text: string): boolean {
|
||||
/** A hash of the constants and patterns that decide a depth, not of this file's bytes: a comment edit keeps it, a threshold edit changes it. */
|
||||
export const DEPTH_RULES = createHash('sha256').update(JSON.stringify({
|
||||
RISK_RULES, SMALL_DIFF_LINES, RED_TEAM_LINES, LARGE_DIFF_LINES, FULL_PASS_CAP, LIGHT_FIXED, BATCH_SCOPED, SCOPED_RANGE_LINES, ROLE_EFFORT, SVG_ELEMENTS: [...SVG_ELEMENTS],
|
||||
patterns: Object.entries({ BINARY_DATA_PATH: [BINARY_DATA_PATH], GATE_PATH, AGENT_PROSE, TEST_PATH, RELEASE_METADATA, DOC_PATH: [DOC_PATH], DATA_PATH: [DATA_PATH], DATA_CONFIG, SVG_MARKUP: [SVG_MARKUP], SVG_REFUSED, SVG_HREF: [SVG_HREF], SVG_STYLE_OPEN: [SVG_STYLE_OPEN], SVG_STYLE_CLOSE: [SVG_STYLE_CLOSE], SVG_CSS_REF: [SVG_CSS_REF], SVG_CSS_REFUSED: [SVG_CSS_REFUSED] }).map(([k, v]) => [k, v.map(r => `/${r.source}/${r.flags}`)]),
|
||||
patterns: Object.entries({ BINARY_DATA_PATH: [BINARY_DATA_PATH], GATE_PATH, AGENT_PROSE, TEST_PATH, RELEASE_METADATA, DOC_PATH: [DOC_PATH], DATA_PATH: [DATA_PATH], DATA_CONFIG, SVG_MARKUP: [SVG_MARKUP], SVG_REFUSED, SVG_HREF: [SVG_HREF], SVG_STYLE_OPEN: [SVG_STYLE_OPEN], SVG_STYLE_CLOSE: [SVG_STYLE_CLOSE], SVG_STYLE_TAG: [SVG_STYLE_TAG], SVG_TAG_NAME: [SVG_TAG_NAME], SVG_ATTR: [SVG_ATTR], SVG_CSS_REF: [SVG_CSS_REF], SVG_CSS_REFUSED: [SVG_CSS_REFUSED] }).map(([k, v]) => [k, v.map(r => `/${r.source}/${r.flags}`)]),
|
||||
})).digest('hex').slice(0, 16);
|
||||
|
||||
/**
|
||||
|
||||
@@ -272,6 +272,9 @@ export function bindReview(rec: Record<string, any>, token: string, env = proces
|
||||
if (env.GSTACK_STAMP_TREE) rec.tree = env.GSTACK_STAMP_TREE;
|
||||
if (env.GSTACK_STAMP_DIRTY) rec.dirty = env.GSTACK_STAMP_DIRTY === 'true';
|
||||
stampGateFindings(rec.findings, env.GSTACK_STAMP_SESSION_KIND);
|
||||
if (rec.skill === 'review-fix-check' && (typeof rec.completed !== 'boolean' || typeof rec.converged !== 'boolean')) {
|
||||
throw new Error('review-log: a review-fix-check row must carry completed and converged as true or false (review-army item 6 fields; without them gstack-review-read marks the row UNVERIFIED and finish cannot count it)');
|
||||
}
|
||||
// A /ship pass whose findings were all fixed, or deferred or noted below the floor, left no open defect (review-army
|
||||
// item 6), so a claimed issues_found is recorded clean with zero counts and Step 11.5 can bind it (GST-430). Only the
|
||||
// logger-stamped action counts (never a caller's status), and the findings must account for every claimed issue.
|
||||
|
||||
+67
-14
@@ -122,20 +122,38 @@ function select(i: ExcludeInput, patterns: readonly string[]): { byPattern: Map<
|
||||
return { byPattern, files };
|
||||
}
|
||||
|
||||
/** Executable image magic: ELF, Mach-O (thin, universal and 64-bit universal; 0xcafebabe is also a Java class), PE, WebAssembly and a script's #!. */
|
||||
const NATIVE_MAGIC = [[0x7f, 0x45, 0x4c, 0x46], [0xfe, 0xed, 0xfa, 0xce], [0xfe, 0xed, 0xfa, 0xcf], [0xce, 0xfa, 0xed, 0xfe], [0xcf, 0xfa, 0xed, 0xfe], [0xca, 0xfe, 0xba, 0xbe], [0xca, 0xfe, 0xba, 0xbf], [0x4d, 0x5a], [0x00, 0x61, 0x73, 0x6d], [0x23, 0x21]];
|
||||
|
||||
/**
|
||||
* The matched paths git calls binary whose bytes hold a NUL in their first 8,000 (git's own test), read from the head blob,
|
||||
* the file on a live tree, or the old blob for a deletion. A branch's .gitattributes can make git call a text file binary;
|
||||
* only real bytes make it binary content here.
|
||||
* The matched paths git calls binary by their bytes alone: `git diff --numstat` with attributes read from the empty tree, so
|
||||
* a branch's .gitattributes (`diff`, `-binary`, `text`) cannot make an executable image read as text and skip the
|
||||
* magic check. null when git cannot run it (a git without --attr-source); the caller then reads every matched file's bytes.
|
||||
*/
|
||||
function binaryContent(i: ExcludeInput, files: ReadonlyMap<string, Matched>, claimed: ReadonlySet<string>, unread: Set<string>): Set<string> {
|
||||
function bytesBinary(i: ExcludeInput, patterns: readonly string[]): string[] | null {
|
||||
const empty = git(i.root, ['hash-object', '-t', 'tree', '--stdin'], '');
|
||||
if (!empty.ok) return null;
|
||||
const range = i.head ? [i.mergeBase, i.head] : [i.mergeBase];
|
||||
const r = git(i.root, [`--attr-source=${empty.out.trim()}`, 'diff', '--numstat', '-z', '--no-renames', ...range, '--', ...patterns.map(p => `:(top,glob)${p}`)]);
|
||||
return r.ok ? r.out.split('\0').filter(l => l.startsWith('-\t-\t')).map(l => l.slice(4)) : null;
|
||||
}
|
||||
const nativeImage = (b: Buffer) => NATIVE_MAGIC.some(m => m.every((x, k) => b[k] === x));
|
||||
|
||||
/**
|
||||
* The claimed paths (git calls them binary under the branch's attributes or under none: bytesBinary) whose bytes hold a NUL
|
||||
* in their first 8,000 (git's own test), read from the head blob, the file on a live tree, or the old blob for a deletion.
|
||||
* A branch's .gitattributes can make git call a text file binary; only real bytes make it binary content here. A binary file that starts with executable image magic also goes in `native`.
|
||||
*/
|
||||
function binaryContent(i: ExcludeInput, files: ReadonlyMap<string, Matched>, claimed: ReadonlySet<string>, unread: Set<string>, native: Set<string>): Set<string> {
|
||||
const out = new Set<string>();
|
||||
const sniff = (p: string, b: Buffer) => { if (!b.includes(0)) return; out.add(p); if (nativeImage(b)) native.add(p); };
|
||||
const real = (sha: string) => /^[0-9a-f]{40,64}$/.test(sha) && !/^0+$/.test(sha);
|
||||
const bySha = new Map<string, string[]>();
|
||||
for (const p of claimed) {
|
||||
const m = files.get(p);
|
||||
if (!m) continue;
|
||||
if (!i.head && m.newMode !== '000000') {
|
||||
try { const fd = fs.openSync(path.join(i.root, p), 'r'); const buf = Buffer.alloc(8000); const n = fs.readSync(fd, buf, 0, 8000, 0); fs.closeSync(fd); if (buf.subarray(0, n).includes(0)) out.add(p); } catch { unread.add(p); }
|
||||
try { const fd = fs.openSync(path.join(i.root, p), 'r'); const buf = Buffer.alloc(8000); const n = fs.readSync(fd, buf, 0, 8000, 0); fs.closeSync(fd); sniff(p, buf.subarray(0, n)); } catch { unread.add(p); }
|
||||
continue;
|
||||
}
|
||||
const sha = real(m.newSha) ? m.newSha : real(m.oldSha) ? m.oldSha : '';
|
||||
@@ -144,8 +162,36 @@ function binaryContent(i: ExcludeInput, files: ReadonlyMap<string, Matched>, cla
|
||||
for (const [sha, paths] of bySha) {
|
||||
const r = spawnSync('git', ['cat-file', 'blob', sha], { cwd: i.root, timeout: 30_000, maxBuffer: 8000 });
|
||||
const read = r.status === 0 || (r.error as NodeJS.ErrnoException | undefined)?.code === 'ENOBUFS';
|
||||
const binary = read && (r.stdout ?? Buffer.alloc(0)).subarray(0, 8000).includes(0);
|
||||
for (const p of paths) if (!read) unread.add(p); else if (binary) out.add(p);
|
||||
const head = (r.stdout ?? Buffer.alloc(0)).subarray(0, 8000);
|
||||
for (const p of paths) if (!read) unread.add(p); else sniff(p, head);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* The added lines of matched files git calls binary only because of their attributes (the bytes hold no NUL): the
|
||||
* collector's added lines are empty for them, so the secret-line rule reads `git diff --text` instead, which no
|
||||
* .gitattributes, textconv or external diff driver changes. One git diff per KEEP_ARG_MAX_BYTES of literal pathspecs;
|
||||
* git prints one section per path in byte order, so a chunk whose sections do not line up with its sorted paths, or a
|
||||
* git failure, puts every path of that chunk in `unread`.
|
||||
*/
|
||||
function attributeBinaryLines(i: ExcludeInput, paths: readonly string[], unread: Set<string>): Record<string, string[]> {
|
||||
const out: Record<string, string[]> = {};
|
||||
const range = i.head ? [i.mergeBase, i.head] : [i.mergeBase];
|
||||
const chunks: string[][] = [];
|
||||
let bytes = Infinity;
|
||||
for (const p of [...paths].sort((a, b) => Buffer.compare(Buffer.from(a), Buffer.from(b)))) {
|
||||
const n = argBytes([p]);
|
||||
if (bytes + n > KEEP_ARG_MAX_BYTES) { chunks.push([]); bytes = 0; }
|
||||
chunks[chunks.length - 1]!.push(p);
|
||||
bytes += n;
|
||||
}
|
||||
for (const chunk of chunks) {
|
||||
const r = git(i.root, ['diff', '--text', '--no-textconv', '--no-ext-diff', '--no-renames', '--src-prefix=a/', '--dst-prefix=b/', '-U0', ...range, '--', ...chunk.map(literal)]);
|
||||
const sections = r.ok ? r.out.split(/^(?=diff --git )/m).filter(s => s.startsWith('diff --git ')) : [];
|
||||
const aligned = sections.length === chunk.length && chunk.every((p, k) => /["\\\x00-\x1f\x7f]/.test(p) || sections[k]!.startsWith(`diff --git a/${p} b/${p}\n`));
|
||||
if (!aligned) { for (const p of chunk) unread.add(p); continue; }
|
||||
chunk.forEach((p, k) => { let hunk = false; out[p] = sections[k]!.split('\n').filter(l => (hunk ||= l.startsWith('@@')) && l.startsWith('+')).map(l => l.slice(1)); });
|
||||
}
|
||||
return out;
|
||||
}
|
||||
@@ -159,17 +205,21 @@ function contentKind(i: ExcludeInput, p: string, binary: ReadonlySet<string>): C
|
||||
return classifyPath(p, i.classRoot) === 'docs' ? 'docs' : null;
|
||||
}
|
||||
|
||||
/** Source and script extensions: a NUL in one does not make it excludable binary content. */
|
||||
const CODE_EXT = /\.(?:[cm]?[jt]sx?|py|rb|go|rs|java|kts?|scala|swift|exs?|php|cs|c|h|cc|cpp|hpp|mm?|dart|pl|pm|lua|sh|bash|zsh|fish|ps1|bat|cmd|vue|svelte|html?|wasm)$/i;
|
||||
/**
|
||||
* The only extensions a binary file may be excludable content under: images, fonts, media, documents, compressed archives
|
||||
* and data blobs. Any other binary (source with a NUL, a native library or executable, a key store, a pickle, no
|
||||
* extension) is code; so is one that starts with executable image magic whatever its extension.
|
||||
*/
|
||||
const BINARY_CONTENT_EXT = /\.(?:png|jpe?g|gif|webp|bmp|ico|avif|tiff?|heic|heif|woff2?|ttf|otf|eot|mp3|mp4|m4a|wav|ogg|oga|ogv|webm|mov|avi|mkv|flac|aac|opus|pdf|docx|xlsx|pptx|odt|ods|odp|gz|tgz|zip|tar|bz2|xz|zst|7z|bin|dat|parquet|arrow|feather|avro|orc|npy|npz|h5|hdf5|sqlite3?|db|safetensors)$/i;
|
||||
|
||||
/** Why a matched file is kept by its path, mode and lines alone; null for content the policy may exclude. A binary file readers would only see as "Binary files differ" is content unless its directory, mode or gate tags keep it. */
|
||||
function pathReason(i: ExcludeInput, p: string, m: Matched, binary: ReadonlySet<string>): KeptReason | null {
|
||||
/** Why a matched file is kept by its path, mode and lines alone; null for content the policy may exclude. A binary file readers would only see as "Binary files differ" is content unless its directory, mode, gate tags, extension or magic keep it. */
|
||||
function pathReason(i: ExcludeInput, p: string, m: Matched, binary: ReadonlySet<string>, native: ReadonlySet<string>): KeptReason | null {
|
||||
if (/^\.(gstack|github)\//.test(p)) return 'policy-dir';
|
||||
if (isGatePath(p)) return 'gate-path';
|
||||
if (m.oldMode === '100755' || m.newMode === '100755') return 'executable';
|
||||
if ([m.oldMode, m.newMode].some(x => x === '120000' || x === '160000')) return 'link';
|
||||
if (i.gateTagged?.has(p)) return 'gate-tagged';
|
||||
if (binary.has(p)) return CODE_EXT.test(p) ? 'code' : null;
|
||||
if (binary.has(p)) return BINARY_CONTENT_EXT.test(p) && !native.has(p) ? null : 'code';
|
||||
if (isAgentProse(p)) return 'agent-prose';
|
||||
if (isDataConfig(p)) return 'config';
|
||||
if (contentKind(i, p, binary) === null) return 'code';
|
||||
@@ -227,9 +277,12 @@ export function reviewExclude(i: ExcludeInput): Exclusion {
|
||||
const sel = select(i, patterns);
|
||||
if ('error' in sel) return { state: 'refused', code: 'REVIEW_DIFF_UNREADABLE', lines: [`${i.context}: git could not list the files review_exclude_paths matches (${sel.error}); fix: git fetch origin ${base}, then rerun once; nothing was planned (REVIEW_DIFF_UNREADABLE)`] };
|
||||
const unread = new Set<string>();
|
||||
const binary = binaryContent(i, sel.files, new Set(i.perPath.filter(e => e.binary && sel.files.has(e.path)).map(e => e.path)), unread);
|
||||
const native = new Set<string>();
|
||||
const claimed = i.perPath.filter(e => e.binary && sel.files.has(e.path)).map(e => e.path);
|
||||
const binary = binaryContent(i, sel.files, new Set([...claimed, ...(bytesBinary(i, patterns) ?? sel.files.keys())]), unread, native);
|
||||
const text = { ...i, added: { ...i.added, ...attributeBinaryLines(i, claimed.filter(p => !binary.has(p) && !unread.has(p)), unread) } };
|
||||
const reasons = new Map<string, KeptReason | null>();
|
||||
for (const [p, m] of sel.files) reasons.set(p, unread.has(p) ? 'consumer-scan-unavailable' : pathReason(i, p, m, binary));
|
||||
for (const [p, m] of sel.files) reasons.set(p, unread.has(p) ? 'consumer-scan-unavailable' : pathReason(text, p, m, binary, native));
|
||||
const candidates = new Set([...reasons].filter(([, r]) => r === null).map(([p]) => p));
|
||||
const named = grepFixed(i, [...candidates].filter(p => !p.includes('\n')), candidates, [i.mergeBase]);
|
||||
for (const p of candidates) {
|
||||
|
||||
@@ -254,7 +254,8 @@ function headInputs(root: string, baseRef: string, head: string, r?: ShipReceipt
|
||||
/**
|
||||
* The queue's exclusion check: the digest the receipt recorded (none when absent) against the one `head` has under the current
|
||||
* base policy. Only the digest counts (the policy's patterns and the excluded list), so a base merge or a moved base that leaves
|
||||
* both unchanged binds; a changed selection needs the review rerun, which receipt write alone cannot stand in for.
|
||||
* both unchanged binds; a changed selection needs the review rerun (write checks the same digest and does not read the
|
||||
* review ledger).
|
||||
*/
|
||||
export function excludeMismatch(x: Exclusion, r: ShipReceipt, head: string, base: string): string | null {
|
||||
const expected = x.state === 'active' ? x.record.digest : 'none';
|
||||
@@ -262,7 +263,7 @@ export function excludeMismatch(x: Exclusion, r: ShipReceipt, head: string, base
|
||||
if (expected === recorded) return null;
|
||||
const source = x.state === 'active' ? x.record.source : r.review_exclude?.source ?? base;
|
||||
const h = head.slice(0, 12);
|
||||
return `field=review_exclude expected=${expected} recorded=${recorded} source=${source}: the reviewers read another selection than ${base}'s review_exclude_paths gives ${h}; the receipt is unchanged; fix: rerun /ship's review on ${h} (gstack-review-dispatch pass --base ${base.replace(/^(refs\/remotes\/)?origin\//, '')}), then gstack-ship-receipt write --pr <n> --review-exclude <d> --body <body> with the digest gstack-review-dispatch verify --ledger <ledger> prints on REVIEW_EXCLUDE_RECORD:; a receipt write alone does not clear this`;
|
||||
return `field=review_exclude expected=${expected} recorded=${recorded} source=${source}: the receipt records another selection than ${base}'s review_exclude_paths gives ${h}; the receipt is unchanged; fix: rerun /ship's review on ${h} (gstack-review-dispatch pass --base ${base.replace(/^(refs\/remotes\/)?origin\//, '')}), then gstack-ship-receipt write --pr <n> --review-exclude <d> --body <body> with the digest gstack-review-dispatch verify --ledger <ledger> prints on REVIEW_EXCLUDE_RECORD: (write checks the digest against the base policy and does not read the review ledger, so only the rerun shows the reviewers read this selection)`;
|
||||
}
|
||||
/** The receipt's recorded gate scan when it names this head; otherwise null (unavailable). */
|
||||
const boundScan = (r: ShipReceipt, head: string): number | null => {
|
||||
|
||||
+28
-1
@@ -6,6 +6,7 @@
|
||||
* computes the P7 adoption census over merged PRs (deduplicated by run and
|
||||
* thread id against an owner-maintained thread cohort).
|
||||
*/
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import { ARTIFACT_SCHEMAS, checkSchema, type ShipReceipt, type ValidationError } from './headless-artifacts';
|
||||
import { approvalSummary } from './approval-citations';
|
||||
import { FRAGMENT_README } from './changelog-fragments';
|
||||
@@ -177,6 +178,8 @@ export interface CarryPlanInput {
|
||||
rangeDiff: string;
|
||||
/** The base the receipt was reviewed against when it differs from the carry's (a stack parent that landed, #3570): productPatchId.from is then the recorded diff against it. */
|
||||
retargetedFrom?: string;
|
||||
/** A commit inside the reviewed range whose product change the base already holds (a stack parent that squash-merged under a receipt written against the base, GST-1199): productPatchId.from is then the range from it. */
|
||||
stackParent?: string;
|
||||
/** A clean scoped review of product paths whose only difference is base context next to unchanged hunks (lib/ship-patch.ts, GST-384): the carry reason it adds. */
|
||||
scopedReview?: string;
|
||||
/** Stamp files the branch changed as product whose only change since the reviewed head is a stamp on the branch's own rows, the base untouched, and without which the two product patches match (GST-980). */
|
||||
@@ -253,7 +256,8 @@ export function planCarry(input: CarryPlanInput): CarryPlan {
|
||||
const named = input.changedProduct.slice(0, CARRY_NAMED_PATHS);
|
||||
return { ok: false, code: 'RECEIPT_CARRY_PRODUCT_CHANGED', message: `product diff changed between ${input.from.slice(0, 12)} and ${input.head.slice(0, 12)}: ${named.join(', ') || 'patch-id differs'}${input.changedProduct.length > named.length ? ` (+${input.changedProduct.length - named.length} more)` : ''}` };
|
||||
}
|
||||
const reasons: string[] = input.retargetedFrom ? [`stack parent ${input.retargetedFrom} landed`] : [];
|
||||
const parent = input.retargetedFrom ?? input.stackParent?.slice(0, 12);
|
||||
const reasons: string[] = parent ? [`stack parent ${parent} landed`] : [];
|
||||
if (input.mergeCommits > 0) reasons.push(input.mergeCommits === 1 ? 'base merge' : `${input.mergeCommits} base merges`);
|
||||
if (input.scopedReview && input.productPatchId.from !== input.productPatchId.head) reasons.push(input.scopedReview);
|
||||
if (input.stampedRows?.length && input.productPatchId.from !== input.productPatchId.head) reasons.push(`a stamp on the branch's own rows (${input.stampedRows.join(', ')})`);
|
||||
@@ -271,6 +275,29 @@ export function planCarry(input: CarryPlanInput): CarryPlan {
|
||||
};
|
||||
}
|
||||
|
||||
/** How many commits of the reviewed range landedStackParent tries, newest first. */
|
||||
export const STACK_PARENT_WALK = 100;
|
||||
|
||||
/**
|
||||
* The newest commit inside the reviewed range (rangeBase..reviewed, the reviewed head itself excluded) whose product
|
||||
* change the current base already holds: merging it into currentBase changes no product path. That is a stack parent
|
||||
* that squash-merged (or merged) into the base after the receipt was written against the base (GST-1199). A commit with no
|
||||
* product change of its own (a test-first commit) is no parent, since any base trivially holds it. Null when no
|
||||
* commit within STACK_PARENT_WALK qualifies or git cannot tell; a conflicted product path is never held.
|
||||
*/
|
||||
export function landedStackParent(repoRoot: string, s: { rangeBase: string; reviewed: string; currentBase: string }, product: readonly string[]): string | null {
|
||||
const run = (args: string[]) => spawnSync('git', args, { cwd: repoRoot, encoding: 'utf8', timeout: 120_000, maxBuffer: 64 * 1024 * 1024 });
|
||||
const commits = run(['rev-list', '--topo-order', `--max-count=${STACK_PARENT_WALK + 1}`, `${s.rangeBase}..${s.reviewed}`]);
|
||||
if (commits.status !== 0) return null;
|
||||
for (const commit of commits.stdout.split('\n').filter(c => c && c !== s.reviewed)) {
|
||||
if (run(['diff', '--quiet', s.rangeBase, commit, '--', ...product]).status !== 1) continue;
|
||||
const merged = run(['merge-tree', '--write-tree', '--no-messages', s.currentBase, commit]);
|
||||
const tree = merged.status === 0 || merged.status === 1 ? merged.stdout.split('\n')[0]?.trim() : undefined;
|
||||
if (tree && run(['diff', '--quiet', s.currentBase, tree, '--', ...product]).status === 0) return commit;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
export interface TreeCarryInput {
|
||||
from: string; head: string;
|
||||
/** HEAD^{tree}, and every tree the receipt already covers (its head_tree plus earlier carries' trees). */
|
||||
|
||||
+36
-4
@@ -15,6 +15,8 @@ import { mergeChangelogs, normalizeChangelog } from './changelog-check';
|
||||
import { versionAtRef } from './restamp';
|
||||
import { classifyPath } from './review-depth';
|
||||
import { comparePath, contextPacket, gateEditedPaths, isTestCode, PACKET_SECTION_CAP, readScopedReview, touchedPaths, type PatchSides } from './ship-patch';
|
||||
import { locate, readRecord } from './approval-record';
|
||||
import { reviewFreshness } from './review-evidence';
|
||||
import { parseReviewLoopException, type ReviewLoopException } from './ship-receipt-depth';
|
||||
import { parseReceiptDocs, type ReceiptDocs } from './ship-receipt-docs';
|
||||
import { resolveStateRoot } from './state-root';
|
||||
@@ -183,8 +185,14 @@ function harnessRepin(cwd: string, from: string, to: string, p: string): boolean
|
||||
|| (e?.since === b.version && typeof e?.nonBehavioral === 'string' && e.nonBehavioral.trim().length >= 10));
|
||||
}
|
||||
|
||||
/** Step 16 stage 2's route. With --reviewed-base, a path whose branch patch is the reviewed one counts as base (lib/ship-patch.ts). */
|
||||
type Routed = Record<'behavior' | 'docs' | 'metadata' | 'tests' | 'base' | 'context' | 'accepted', string[]>;
|
||||
/**
|
||||
* Step 16 stage 2's route. With --reviewed-base, a path whose branch patch is the reviewed one counts as base (lib/ship-patch.ts).
|
||||
* A code or test path the newest completed clean review-fix-check row read (inside its range_base..commit range, where
|
||||
* range_base holds the reviewed tree, and unchanged since) counts as fix_checked when an approved review_loop_grant names a stop
|
||||
* head inside range_base..commit: the owner's granted fix-check after Step 11 is a review (GST-1130). Authored docs keep the
|
||||
* stage 2.2 recheck.
|
||||
*/
|
||||
type Routed = Record<'behavior' | 'docs' | 'metadata' | 'tests' | 'base' | 'context' | 'accepted' | 'fix_checked', string[]>;
|
||||
function route(cwd: string, base: string, reviewed: string, reviewedBase: string | undefined, release: string[], opts: { docsRechecked: boolean; contextReviewed?: string; loop?: ReviewLoopException }, r: Report): Routed | null {
|
||||
const now = bin(cwd, 'gstack-wtree', [], 60_000).stdout.trim();
|
||||
if (!/^[0-9a-f]{40,64}$/.test(now)) { r.stop('gstack-wtree printed no tree; the stage 2 comparison is blocked'); return null; }
|
||||
@@ -204,7 +212,7 @@ function route(cwd: string, base: string, reviewed: string, reviewedBase: string
|
||||
sides = { reviewedBase: rb, reviewed, currentBase: cb, current: now };
|
||||
touched = new Set([...a, ...b]);
|
||||
}
|
||||
const by: Routed = { behavior: [], docs: [], metadata: [], tests: [], base: [], context: [], accepted: [] };
|
||||
const by: Routed = { behavior: [], docs: [], metadata: [], tests: [], base: [], context: [], accepted: [], fix_checked: [] };
|
||||
for (const p of [...changedSet].sort()) {
|
||||
const verdict = sides ? (touched.has(p) ? comparePath(cwd, sides, p) : 'same') : 'changed';
|
||||
by[verdict === 'same' ? 'base' : release.includes(p) || harnessRepin(cwd, reviewed, now, p) ? 'metadata' : sides && isTestCode(p) ? 'tests' : classifyPath(p, cwd) === 'docs' ? 'docs' : verdict === 'context' ? 'context' : 'behavior'].push(p);
|
||||
@@ -219,6 +227,30 @@ function route(cwd: string, base: string, reviewed: string, reviewedBase: string
|
||||
by.context = by.context.filter(p => !packetBody.over.includes(p));
|
||||
by.tests = by.tests.filter(p => !packetBody.over.includes(p));
|
||||
}
|
||||
const fixRow = bin(cwd, 'gstack-review-read', [], 60_000).stdout.split('\n---CONFIG---')[0].split('\n')
|
||||
.flatMap(l => { try { const v = JSON.parse(l); return v?.skill === 'review-fix-check' ? [v] : []; } catch { return []; } }).at(-1);
|
||||
const commitOf = (c: unknown) => typeof c === 'string' && /^[0-9a-f]{7,64}$/.test(c) ? git(cwd, ['rev-parse', '--verify', '--quiet', `${c}^{commit}`]).stdout.trim() : '';
|
||||
const onBranch = (c: string) => !!c && git(cwd, ['merge-base', '--is-ancestor', c, 'HEAD']).status === 0;
|
||||
const fixCommit = commitOf(fixRow?.commit_full), fixFrom = commitOf(fixRow?.range_base);
|
||||
const fresh = fixRow && reviewFreshness(fixRow, fixRow.wtree);
|
||||
const ancestor = (a: string, d: string) => git(cwd, ['merge-base', '--is-ancestor', a, d]).status === 0;
|
||||
const grants = fixRow ? (loc => 'file' in loc ? readRecord(loc).rows : [])(locate({ cwd, env: process.env })) : [];
|
||||
const grant = grants.find(g => g.outcome === 'approved' && `${g.ask} ${g.scope}`.includes('review_loop_grant')
|
||||
&& (`${g.ask} ${g.scope}`.match(/\b[0-9a-f]{7,40}\b/g) ?? []).some(s => { const c = commitOf(s); return !!c && !!fixFrom && !!fixCommit && ancestor(fixFrom, c) && ancestor(c, fixCommit); }));
|
||||
const notCounted = !fixRow ? '' : fresh?.status !== 'CURRENT' ? fresh?.reason ?? 'not a review row'
|
||||
: fixRow.tree !== fixRow.wtree || !onBranch(fixCommit) ? 'the row was logged on content no branch commit holds'
|
||||
: !onBranch(fixFrom) ? 'the row names no range_base commit on this branch'
|
||||
: git(cwd, ['rev-parse', `${fixFrom}^{tree}`]).stdout.trim() !== reviewed ? `range_base ${fixFrom.slice(0, 12)} does not hold the reviewed tree`
|
||||
: !grant ? 'no approved review_loop_grant names a stop head inside its range_base..commit' : '';
|
||||
if (notCounted) r.add(`FIX_CHECK: not counted (${notCounted})`);
|
||||
const counted = !!fixRow && !notCounted;
|
||||
if (counted) {
|
||||
const range = touchedPaths(cwd, fixFrom, fixCommit), after = touchedPaths(cwd, fixCommit, now);
|
||||
if (!range || !after) { r.stop(`git could not diff the fix-check range ${fixFrom.slice(0, 12)}..${fixCommit.slice(0, 12)} against the reviewed and current trees: stage 2 is blocked`); return null; }
|
||||
by.fix_checked = [...by.behavior, ...by.tests, ...by.context].filter(p => range.has(p) && !after.has(p)).sort();
|
||||
for (const k of ['behavior', 'tests', 'context'] as const) by[k] = by[k].filter(p => !by.fix_checked.includes(p));
|
||||
r.add(`FIX_CHECK: reviewed ${by.fix_checked.length ? short(by.fix_checked) : 'none'} (fix-check at ${fixCommit.slice(0, 12)}, range from ${fixFrom.slice(0, 12)}, grant ${grant!.id})`);
|
||||
}
|
||||
const loop = opts.loop;
|
||||
if (loop) {
|
||||
const head = loop.fix_check.head;
|
||||
@@ -233,7 +265,7 @@ function route(cwd: string, base: string, reviewed: string, reviewedBase: string
|
||||
by.behavior = by.behavior.filter(p => !by.accepted.includes(p));
|
||||
by.docs = by.docs.filter(p => !by.accepted.includes(p));
|
||||
}
|
||||
r.add(`ROUTE: reviewed=${reviewed.slice(0, 12)} current=${now.slice(0, 12)} behavior=${by.behavior.length} docs=${by.docs.length} metadata=${by.metadata.length} tests=${by.tests.length} base=${sides ? by.base.length : 'unknown'} context=${by.context.length}${loop ? ` accepted=${by.accepted.length}` : ''}`);
|
||||
r.add(`ROUTE: reviewed=${reviewed.slice(0, 12)} current=${now.slice(0, 12)} behavior=${by.behavior.length} docs=${by.docs.length} metadata=${by.metadata.length} tests=${by.tests.length} base=${sides ? by.base.length : 'unknown'} context=${by.context.length}${counted ? ` fix_checked=${by.fix_checked.length}` : ''}${loop ? ` accepted=${by.accepted.length}` : ''}`);
|
||||
if (loop) r.add(`REVIEW_LOOP_EXCEPTION: accepted ${by.accepted.length ? short(by.accepted) : 'none'} (changed since fix-check ${loop.fix_check.head.slice(0, 12)}; accepted by ${loop.accepted_by} ${loop.date} via ${loop.via})`);
|
||||
if (by.behavior.length) { r.stop(`behavior, tests or build inputs changed since review (${short(by.behavior.sort())}): insert 5–11.5 → 12–14 → 16 before Step 17 (stage 2.1)`); return null; }
|
||||
const scoped = [...by.context, ...by.tests].sort();
|
||||
|
||||
Reference in new issue
Block a user