A folder marked as published data can no longer hide source code or agent files from /ship's review (GST-3158) (#4516)

In a folder a repo marks as published data (`review_content_patterns`),
source code in any language, agent instruction files, config and data
that kept code names by full path now stay in /ship's review.

Why: #4511 let a `.java` file, a binary `AGENTS.md` or a file named by a
kept script leave review there, and reviewers must see every line of
code a PR changes.

Nothing to do: no base policy sets the key yet.

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
Review-Finding: p1/red-team#1 reader=red-team
Review-Finding: p1/specialist-batched#1 reader=specialist-batched
This commit is contained in:
Garry Tanandcapy-ai[bot] authored and GitHub committed 2026-11-19 11:15:51 +00:00
1 parent 944b41e24d
commit ed267248d3
4 files changed
+98 -22

No files matched your search

+21 -14
View File
@@ -13,7 +13,7 @@
* git exclude beats any positive pathspec. That second command is bounded by its argument
* bytes: a pattern whose kept files would pass KEEP_ARG_MAX_BYTES on it refuses the plan
* (REVIEW_EXCLUDE_KEEP_TOO_LARGE) instead of being dropped (GST-3158). A file the base policy's
* review_content_patterns also matches is content whatever its extension, unless it is a source file (SOURCE_EXT, any case): the policy says so.
* review_content_patterns also matches is content whatever its extension, unless it stays code anyway (declaredData): the policy says so.
*
* A file kept only for a secret line (a line that names a credential and carries a value, secretValueLine;
* no other reason, under an applied pattern, gate scan and consumer scan both read) is also listed in `excerpt`: pass readers
@@ -31,7 +31,6 @@ import * as os from 'node:os';
import * as path from 'node:path';
import { collectLive } from './review-collect';
import { DATA_PATH, classifyPath, isAgentProse, isDataConfig, isGatePath, type PathLines } from './review-depth';
import { SOURCE_EXT } from './pregate/import-graph';
import { scan } from './redact-engine';
import { shannonEntropy } from './redact-patterns';
import { KEY_NAME, SECRET_LINE, secretPath } from './review-risk';
@@ -265,15 +264,18 @@ export type ContentKind = 'binary' | 'data' | 'docs' | 'log' | 'declared';
* config name there is data; and the files a review_exclude_paths and a review_content_patterns pattern both match, which
* the base policy declares content.
*/
type Scope = ExcludeInput & { named?: ReadonlySet<string>; declared?: ReadonlySet<string> };
type Scope = ExcludeInput & { named?: ReadonlySet<string>; declared?: ReadonlySet<string>; keptDeclared?: ReadonlySet<string> };
const config = (i: Scope, p: string) => isDataConfig(p) && !i.named?.has(p);
const SOURCE = new RegExp(SOURCE_EXT.source, 'i');
/** Source code by its name, in any letter case: a program extension, or a build or task file name. A declared file with one stays code. */
const SOURCE = /(?:\.(?:[cm]?[jt]sx?|py[iw]?|pyx|rb|erb|rake|sh|bash|zsh|fish|ksh|csh|ps1|psm1|psd1|bat|cmd|vbs|go|rs|dart|java|kt|kts|scala|sc|groovy|gradle|clj[sc]?|swift|m|mm|c|h|cc|cpp|cxx|hpp|hh|hxx|ino|cs|fs|fsx|vb|php|phtml|pl|pm|lua|r|jl|ex|exs|erl|hrl|hs|ml|mli|elm|zig|nim|v|sol|sql|vue|svelte|astro|html?|xhtml|css|scss|sass|less|wasm|wat|tf|hcl|nix|mk|cmake|bzl|star|awk|sed|asm|s|applescript|jsonnet|tmpl|ejs|hbs|jinja2?|j2|mdx)|(?:^|\/)(?:Makefile|GNUmakefile|Dockerfile|Containerfile|Rakefile|Gemfile|Brewfile|Justfile|Procfile|Vagrantfile|Jenkinsfile|Earthfile|Tiltfile|BUILD|WORKSPACE|CMakeLists\.txt|meson\.build)(?:\.[^/]*)?)$/i;
/** A declared file that is data: none of source, agent prose, config, a gate path or a .gstack/ or .github/ file, which stay code whatever the policy says. */
const declaredData = (i: Scope, p: string) => !!i.declared?.has(p) && !SOURCE.test(p) && !isAgentProse(p) && !config(i, p) && !isGatePath(p) && !/^\.(gstack|github)\//.test(p);
function contentKind(i: Scope, p: string, binary: ReadonlySet<string>): ContentKind | null {
if (binary.has(p)) return 'binary';
if (DATA_PATH.test(p) && !config(i, p)) return 'data';
if (/\.log$/i.test(p)) return 'log';
if (classifyPath(p, i.classRoot) === 'docs') return 'docs';
return i.declared?.has(p) && !SOURCE.test(p) ? 'declared' : null;
if (classifyPath(p, i.classRoot) === 'docs' && !SOURCE.test(p)) return 'docs';
return declaredData(i, p) ? 'declared' : null;
}
/**
@@ -313,7 +315,7 @@ function pathReason(i: Scope, p: string, m: Matched, binary: ReadonlySet<string>
if (m.oldMode === '100755' || m.newMode === '100755') return 'executable';
if ([m.oldMode, m.newMode].some(x => x === '120000' || x === '160000')) return 'link';
if (i.gateTagged?.has(p)) return 'gate-tagged';
if (binary.has(p)) return (BINARY_CONTENT_EXT.test(p) || (i.declared?.has(p) && !SOURCE.test(p))) && !native.has(p) ? null : 'code';
if (binary.has(p)) return (BINARY_CONTENT_EXT.test(p) || declaredData(i, p)) && !native.has(p) ? null : 'code';
if (isAgentProse(p)) return 'agent-prose';
if (config(i, p)) return 'config';
if (contentKind(i, p, binary) === null) return 'code';
@@ -321,9 +323,13 @@ function pathReason(i: Scope, p: string, m: Matched, binary: ReadonlySet<string>
return null;
}
/** A file whose full-path reference keeps excluded content: a code or test file (not docs, data, a .log or binary) outside the would-be-excluded set. */
const consumer = (i: Scope, f: string, candidates: ReadonlySet<string>) =>
!candidates.has(f) && !f.startsWith('.gstack/tmp/') && contentKind(i, f, new Set()) === null;
/**
* A file whose full-path reference keeps excluded content: a code or test file (not docs, data, a .log or binary) outside the
* would-be-excluded set, or a declared file that stays in review (kept in this range, or an executable or link at `mode`).
*/
const consumer = (i: Scope, f: string, candidates: ReadonlySet<string>, mode?: string) =>
!candidates.has(f) && !f.startsWith('.gstack/tmp/') && (contentKind(i, f, new Set()) === null
|| (!!i.declared?.has(f) && (!!i.keptDeclared?.has(f) || mode === '100755' || mode === '120000')));
/**
* An uncapped `git grep -F -o -f` of fixed strings at the head (or the working tree) and `revs`: string -> the consumer files
@@ -350,9 +356,9 @@ function grepFixed(i: Scope, strings: readonly string[], candidates: ReadonlySet
for (const entry of ls.out.split('\0')) {
const tab = entry.indexOf('\t');
const f = entry.slice(tab + 1);
if (tab < 0 || !consumer(i, f, candidates)) continue;
if (f.includes('\uFFFD')) return null;
const meta = entry.slice(0, tab).split(' ');
if (tab < 0 || !consumer(i, f, candidates, at ? meta[0] : meta[1])) continue;
if (f.includes('\uFFFD')) return null;
if (at) { if (meta[1] === 'blob') cached.push(`${meta[0]} ${meta[2]}\t${f}`); continue; }
const [tag = '', mode, sha, stage] = meta;
(tag === 'S' || tag !== tag.toUpperCase() ? cached : worktree).push(`${mode} ${sha} ${stage}\t${f}`);
@@ -411,7 +417,8 @@ export function reviewExclude(i: ExcludeInput): Exclusion {
for (const [p, m] of sel.files) reasons.set(p, unread.has(p) ? 'consumer-scan-unavailable' : pathReason(text, p, m, binary, native) ?? (unseen(p) ? 'consumer-scan-unavailable' : null));
const candidates = new Set([...reasons].filter(([, r]) => r === null).map(([p]) => p));
const secretOnly = [...reasons].filter(([p, r]) => r === 'secret-line' && !p.includes('\n')).map(([p]) => p);
const named = grepFixed(x, [...candidates, ...secretOnly].filter(p => !p.includes('\n')), new Set([...candidates, ...secretOnly]), [i.mergeBase]);
const y: Scope = { ...x, keptDeclared: new Set([...reasons].filter(([p, r]) => r !== null && declared.has(p)).map(([p]) => p)) };
const named = grepFixed(y, [...candidates, ...secretOnly].filter(p => !p.includes('\n')), new Set([...candidates, ...secretOnly]), [i.mergeBase]);
for (const p of candidates) {
if (i.gateTagged === null) reasons.set(p, 'gate-scan-unavailable');
else if (named === null || p.includes('\n')) reasons.set(p, 'consumer-scan-unavailable');
@@ -441,7 +448,7 @@ export function reviewExclude(i: ExcludeInput): Exclusion {
const content: Record<ContentKind, number> = { binary: 0, data: 0, docs: 0, log: 0, declared: 0 };
for (const p of sortedExcluded) content[contentKind(x, p, binary)!]++;
const prefixes = [...new Set(applied.map(literalPrefix).filter(Boolean))];
const refs = grepFixed(x, prefixes, excluded, []) ?? new Map<string, Set<string>>();
const refs = grepFixed(y, prefixes, excluded, []) ?? new Map<string, Set<string>>();
const patterns_sha256 = sha256(JSON.stringify(patterns));
const list_sha256 = sha256(JSON.stringify({ applied, excluded: sortedExcluded, kept: keptList.map(k => [k.path, k.reason]) }));
const record: ExcludeRecord = {