Catch secrets split across lines in UTF-16 files and stop deleted binaries from blocking pushes (GST-2261) (#4261)

* Catch secrets split across lines in UTF-16 files and skip deleted files in the push scan (GST-2261)

Each reading of a file git calls binary is now scanned as its own text with the file's line numbers, so a keyword and its value on the next line stay neighbours. The pre-push hook diffs with --irreversible-delete, so a deleted binary's old bytes no longer fill its 64 MiB buffer.

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* Leave the landed binary-scan changelog fragment unchanged (GST-2261)

Review-Finding: changelog.d/scan-binary-files-for-secrets.md:1:release reader=core
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* List a token once when several readings of a binary file find it (GST-2261)

Review-Finding: p1/specialist-batched#1 reader=specialist-batched
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* Report a repeated token once only within one binary file, so an allowlisted token never hides another (GST-2261)

Review-Finding: p2/specialist-batched#1 reader=security
Review-Finding: p2/red-team#2 reader=red-team
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* Name the changelog fragment after the branch (GST-2261)

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* Keep the side-by-side reading of binary files so a key and its keyword in different readings still fail (GST-2261)

Review-Finding: audit/cross-family#1 reader=gpt-6.1-sol
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

---------

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
This commit is contained in:
Garry Tanandcapy-ai[bot] authored and GitHub committed 2026-11-10 02:12:29 +00:00
1 parent 46d51496b4
commit ccc3f0fc14
6 files changed
+148 -37

No files matched your search

+39 -26
View File
@@ -6,7 +6,7 @@
* never env.value); a scanner that does not finish or exits outside its
* contract is PREGATE_ENVIRONMENT (kind tool), never a pass, and so is a
* merge-base diff git did not finish. Files git calls binary are read too
* (scannableText). Runs regardless of lint tools.
* (scannableReadings). Runs regardless of lint tools.
*/
import { spawnSync } from 'node:child_process';
import * as fs from 'node:fs';
@@ -73,29 +73,35 @@ export function parsePostimages(diff: string): Map<string, string[]> {
}
/**
* Text the scanner can read from a file git calls binary, which the diff
* passes through with --text (GST-2195). Text with no NUL is returned as is.
* Each line is read three ways. The plain reading turns each run of NUL,
* Texts the scanner can read from a file git calls binary, which the diff
* passes through with --text (GST-2195). Text with no NUL is its own only
* reading. Otherwise the file is read three ways. The plain reading turns each run of NUL,
* other control bytes or bytes that are not UTF-8 (an image chunk, an
* executable's padding) into one space. The little- and big-endian UTF-16
* readings first drop the NULs inside every run of character-then-NUL or
* NUL-then-character pairs, which UTF-16 text puts beside each character of
* its ASCII range however much of the file is Cyrillic or CJK, and set the run
* apart with a space on each side, so the bytes of a CJK character beside it
* cannot join a token. Each distinct reading stays on the line, so a token is
* found in whichever one keeps it whole: a plain token that a NUL-flanked
* byte beside it would glue or cut in a UTF-16 reading, or a UTF-16 token
* whose neighbour pairs up with it in the other byte order. Line breaks stay, so line numbers still hold. Callers keep only credential
* findings from such a file: privacy patterns read its bytes as noise (a run
* of digits in a compressed icon reads as a phone number).
* cannot join a token. The first text is main's combined reading, each line
* holding its distinct readings side by side, so a keyword in one reading and
* its value in another still meet. Each distinct reading is then also a whole
* text of its own, scanned separately, so a token is found in whichever one keeps it whole (a
* plain token that a NUL-flanked byte beside it would glue or cut in a UTF-16
* reading, or a UTF-16 token whose neighbour pairs up with it in the other
* byte order), and a keyword and its value on the next line stay neighbours
* (GST-2261). Line breaks stay, so every reading keeps the file's line
* numbers. Callers keep only credential findings from such a file: privacy
* patterns read its bytes as noise (a run of digits in a compressed icon reads
* as a phone number).
*/
export function scannableText(text: string): string {
if (!text.includes('\0')) return text;
export function scannableReadings(text: string): string[] {
if (!text.includes('\0')) return [text];
const noise = /[\0-\x08\x0b\x0c\x0e-\x1f\x7f\uFFFD]+/g;
return text.split('\n').map(line => {
const utf16 = (pairs: RegExp) => line.replace(pairs, run => ` ${run.replaceAll('\0', '')} `).replace(noise, ' ');
return [...new Set([utf16(/(?:[^\0-\x08\x0b\x0c\x0e-\x1f\x7f\uFFFD]\0){2,}/g), utf16(/(?:\0[^\0-\x08\x0b\x0c\x0e-\x1f\x7f\uFFFD]){2,}/g), line.replace(noise, ' ')])].join(' ');
}).join('\n');
const lines = text.split('\n');
const utf16 = (pairs: RegExp) => lines.map(line => line.replace(pairs, run => ` ${run.replaceAll('\0', '')} `).replace(noise, ' '));
const readings = [utf16(/(?:[^\0-\x08\x0b\x0c\x0e-\x1f\x7f\uFFFD]\0){2,}/g), utf16(/(?:\0[^\0-\x08\x0b\x0c\x0e-\x1f\x7f\uFFFD]){2,}/g), lines.map(line => line.replace(noise, ' '))];
const combined = lines.map((_, i) => [...new Set(readings.map(reading => reading[i]))].join(' ')).join('\n');
return [...new Set([combined, ...readings.map(reading => reading.join('\n'))])];
}
/** Files whose source-syntax exemption needs the whole file: a line added inside an existing `'''`/`"""` block is a literal (#3630). */
@@ -116,10 +122,10 @@ const WHOLE_FILE_MAX_BYTES = 4 * 1024 * 1024;
* reads as clean. --text keeps a file git calls binary (a NUL byte, UTF-16, a
* `-diff` attribute the branch itself can add) from printing "Binary files
* differ" in place of its added lines; each such file is marked `binary`,
* read through scannableText and held to its credential findings. A diff git did not finish (an exit, a kill, a
* carries every reading scannableReadings gives and is held to its credential findings. A diff git did not finish (an exit, a kill, a
* full buffer) throws: its partial output is never scanned as complete.
*/
export function addedLines(ctx: PregateContext): Array<{ file: string; text: string; keep?: number[]; binary?: boolean }> {
export function addedLines(ctx: PregateContext): Array<{ file: string; texts: string[]; keep?: number[]; binary?: boolean }> {
const diff = spawnSync('git', ['diff', '--text', '--no-color', '--no-ext-diff', '--no-textconv', '--src-prefix=a/', '--dst-prefix=b/', '--unified=0', '--diff-filter=ACMR', ctx.mergeBase, '--'], { cwd: ctx.repoRoot, encoding: 'utf8', timeout: 60_000, maxBuffer: 256 * 1024 * 1024, env: ctx.env });
if (diff.error || diff.status !== 0) throw new Error(`git diff ${ctx.mergeBase.slice(0, 12)} did not finish (${diff.error?.message ?? (diff.status === null ? `killed by ${diff.signal}` : `exit ${diff.status}: ${(diff.stderr ?? '').trim().split('\n').pop() ?? ''}`)})`);
const files = parseAddedLines(diff.stdout ?? '');
@@ -149,8 +155,8 @@ export function addedLines(ctx: PregateContext): Array<{ file: string; text: str
}
return [...files]
.map(([file, lines]) => whole.get(file) ? { file, ...whole.get(file)! } : { file, text: lines.join('\n') })
.map(p => p.text.includes('\0') ? { ...p, text: scannableText(p.text), binary: true } : p)
.filter(p => p.text.trim());
.map(({ text, ...p }) => ({ ...p, texts: scannableReadings(text), ...(text.includes('\0') ? { binary: true } : {}) }))
.filter(p => p.texts.some(text => text.trim()));
}
type Finding = { id?: string; description?: string; severity?: string; category?: string; line?: number; col?: number; file: string; fileLine?: boolean };
type Scanned = Finding & AllowlistFinding;
@@ -171,21 +177,28 @@ function run(ctx: PregateContext): CheckResult {
let high = false;
let medium = false;
let parsedAll = true;
for (const [i, part] of parts.entries()) {
const file = path.join(tmp, String(i), (!part.keep && LEXICAL_CONTEXT_SOURCE.test(part.file) ? '' : path.basename(part.file)) || 'added.txt');
for (const [i, part] of parts.entries()) for (const [j, text] of part.texts.entries()) {
const file = path.join(tmp, `${i}.${j}`, (!part.keep && LEXICAL_CONTEXT_SOURCE.test(part.file) ? '' : path.basename(part.file)) || 'added.txt');
fs.mkdirSync(path.dirname(file), { mode: 0o700 });
fs.writeFileSync(file, part.text, { mode: 0o600 });
fs.writeFileSync(file, text, { mode: 0o600 });
const r = runScriptSync(path.join(BIN_DIR, 'gstack-redact'), ['--from-file', file, '--json', '--max-bytes', String(64 * 1024 * 1024)], { cwd: ctx.repoRoot, encoding: 'utf8', timeout: Math.min(ctx.timeoutMs, 300_000), maxBuffer: 64 * 1024 * 1024, env: ctx.env });
if (r.error || r.status === null) return environmentResult(base, launchCause('tool', `gstack-redact did not finish on ${part.file || 'added lines'} (${r.error?.message ?? 'killed'})`), `gstack-redact did not finish on ${part.file || 'added lines'}; an incomplete scan never passes`);
if (r.status !== 0 && r.status !== 2 && r.status !== 3) return environmentResult(base, launchCause('tool', `gstack-redact exited ${r.status} on ${part.file || 'added lines'}: ${(r.stderr ?? '').trim().split('\n').pop() ?? ''}`), `gstack-redact exited ${r.status}; an incomplete scan never passes`);
high ||= r.status === 3;
medium ||= r.status === 2;
const lines = part.text.split('\n');
const lines = text.split('\n');
try { for (const f of (JSON.parse(r.stdout).findings ?? []).filter((f: Omit<Finding, 'file'>) => (!part.binary || f.category === 'secret') && (!part.keep || f.id?.startsWith('engine.') || (f.line !== undefined && part.keep.includes(f.line)))).map((f: Omit<Finding, 'file'>) => ({ ...f, id: f.id ?? 'finding', category: f.category ?? '?', severity: f.severity ?? '?', file: part.file, fileLine: !!part.keep, lineText: f.line ? lines[f.line - 1] : undefined }))) findings.push(f); } catch { parsedAll = false; }
}
// The allowlist applies only to findings the scanner reported in full; an unparsed report keeps its exit-code verdict.
const applied = parsedAll && ctx.config.allowlist.length ? applyAllowlist(findings, ctx.config.allowlist) : { kept: findings, allowlisted: [], untested: [], missed: [] };
const kept = applied.kept;
// Readings of one binary file that keep a line unchanged report its token again: list each finding once.
const reported = new Set<string>();
const kept = applied.kept.filter(f => {
const key = JSON.stringify([f.file, f.id, f.line, f.col, f.lineText]);
if (reported.has(key)) return false;
reported.add(key);
return true;
});
if (parsedAll) { high = kept.some(f => f.severity === 'HIGH'); medium = kept.some(f => f.severity === 'MEDIUM'); }
const render = (f: Scanned) => `${f.severity} ${f.id} [${f.category}]: ${f.description ?? ''}${f.line ? ` (${f.file ? `${f.file}, ` : ''}${f.fileLine ? 'line' : 'added line'} ${f.line})` : ''}`;
const allowLines = [...applied.allowlisted.map(describeAllowlisted), ...applied.untested.map(describeUntested), ...applied.missed.map(describeMissed)];
@@ -197,7 +210,7 @@ function run(ctx: PregateContext): CheckResult {
const advisory = kept.filter(f => f.severity === 'MEDIUM' && f.category !== 'secret');
const binary = parts.filter(p => p.binary).length;
const scanned = `${parts.length} file${parts.length === 1 ? '' : 's'}${binary ? ` (${binary} binary)` : ''}`;
const lineCount = parts.reduce((n, p) => n + (p.keep?.length ?? p.text.split('\n').length), 0);
const lineCount = parts.reduce((n, p) => n + (p.keep?.length ?? p.texts[0].split('\n').length), 0);
const fail = (lines: string[], detail: string): CheckResult => ({ ...base, status: 'fail', cause: 'diff', lines: [...lines, ...allowLines], code: 'PREGATE_SECRETS', detail: `${detail}${allowNote}`, fix: 'rotate and remove the credential from the added lines (history included); secrets cannot be downgraded (a benign finding is allowlisted by id, path and pattern in the policy, never env.value)' });
if (!high && !medium) return { ...base, status: 'pass', ...(allowLines.length ? { lines: allowLines } : {}), detail: `${lineCount} added lines clean in ${scanned}${kept.length ? ` (${kept.length} WARN)` : ''}${allowNote}` };
if (blocking.length) return fail([...blocking, ...advisory].map(render), `${blocking.length} credential finding(s) in added lines (${blocking.map(f => f.id).join(', ')})`);