mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-17 05:41:50 +01:00
Catch secrets split across lines in UTF-16 files and stop deleted binaries from blocking pushes (GST-2261) (#4261)
* Catch secrets split across lines in UTF-16 files and skip deleted files in the push scan (GST-2261) Each reading of a file git calls binary is now scanned as its own text with the file's line numbers, so a keyword and its value on the next line stay neighbours. The pre-push hook diffs with --irreversible-delete, so a deleted binary's old bytes no longer fill its 64 MiB buffer. Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * Leave the landed binary-scan changelog fragment unchanged (GST-2261) Review-Finding: changelog.d/scan-binary-files-for-secrets.md:1:release reader=core Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * List a token once when several readings of a binary file find it (GST-2261) Review-Finding: p1/specialist-batched#1 reader=specialist-batched Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * Report a repeated token once only within one binary file, so an allowlisted token never hides another (GST-2261) Review-Finding: p2/specialist-batched#1 reader=security Review-Finding: p2/red-team#2 reader=red-team Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * Name the changelog fragment after the branch (GST-2261) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * Keep the side-by-side reading of binary files so a key and its keyword in different readings still fail (GST-2261) Review-Finding: audit/cross-family#1 reader=gpt-6.1-sol Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --------- Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
This commit is contained in:
1 parent
46d51496b4
commit
ccc3f0fc14
6 files changed
+148
-37
No files matched your search
@@ -6,7 +6,7 @@
|
||||
* never env.value); a scanner that does not finish or exits outside its
|
||||
* contract is PREGATE_ENVIRONMENT (kind tool), never a pass, and so is a
|
||||
* merge-base diff git did not finish. Files git calls binary are read too
|
||||
* (scannableText). Runs regardless of lint tools.
|
||||
* (scannableReadings). Runs regardless of lint tools.
|
||||
*/
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import * as fs from 'node:fs';
|
||||
@@ -73,29 +73,35 @@ export function parsePostimages(diff: string): Map<string, string[]> {
|
||||
}
|
||||
|
||||
/**
|
||||
* Text the scanner can read from a file git calls binary, which the diff
|
||||
* passes through with --text (GST-2195). Text with no NUL is returned as is.
|
||||
* Each line is read three ways. The plain reading turns each run of NUL,
|
||||
* Texts the scanner can read from a file git calls binary, which the diff
|
||||
* passes through with --text (GST-2195). Text with no NUL is its own only
|
||||
* reading. Otherwise the file is read three ways. The plain reading turns each run of NUL,
|
||||
* other control bytes or bytes that are not UTF-8 (an image chunk, an
|
||||
* executable's padding) into one space. The little- and big-endian UTF-16
|
||||
* readings first drop the NULs inside every run of character-then-NUL or
|
||||
* NUL-then-character pairs, which UTF-16 text puts beside each character of
|
||||
* its ASCII range however much of the file is Cyrillic or CJK, and set the run
|
||||
* apart with a space on each side, so the bytes of a CJK character beside it
|
||||
* cannot join a token. Each distinct reading stays on the line, so a token is
|
||||
* found in whichever one keeps it whole: a plain token that a NUL-flanked
|
||||
* byte beside it would glue or cut in a UTF-16 reading, or a UTF-16 token
|
||||
* whose neighbour pairs up with it in the other byte order. Line breaks stay, so line numbers still hold. Callers keep only credential
|
||||
* findings from such a file: privacy patterns read its bytes as noise (a run
|
||||
* of digits in a compressed icon reads as a phone number).
|
||||
* cannot join a token. The first text is main's combined reading, each line
|
||||
* holding its distinct readings side by side, so a keyword in one reading and
|
||||
* its value in another still meet. Each distinct reading is then also a whole
|
||||
* text of its own, scanned separately, so a token is found in whichever one keeps it whole (a
|
||||
* plain token that a NUL-flanked byte beside it would glue or cut in a UTF-16
|
||||
* reading, or a UTF-16 token whose neighbour pairs up with it in the other
|
||||
* byte order), and a keyword and its value on the next line stay neighbours
|
||||
* (GST-2261). Line breaks stay, so every reading keeps the file's line
|
||||
* numbers. Callers keep only credential findings from such a file: privacy
|
||||
* patterns read its bytes as noise (a run of digits in a compressed icon reads
|
||||
* as a phone number).
|
||||
*/
|
||||
export function scannableText(text: string): string {
|
||||
if (!text.includes('\0')) return text;
|
||||
export function scannableReadings(text: string): string[] {
|
||||
if (!text.includes('\0')) return [text];
|
||||
const noise = /[\0-\x08\x0b\x0c\x0e-\x1f\x7f\uFFFD]+/g;
|
||||
return text.split('\n').map(line => {
|
||||
const utf16 = (pairs: RegExp) => line.replace(pairs, run => ` ${run.replaceAll('\0', '')} `).replace(noise, ' ');
|
||||
return [...new Set([utf16(/(?:[^\0-\x08\x0b\x0c\x0e-\x1f\x7f\uFFFD]\0){2,}/g), utf16(/(?:\0[^\0-\x08\x0b\x0c\x0e-\x1f\x7f\uFFFD]){2,}/g), line.replace(noise, ' ')])].join(' ');
|
||||
}).join('\n');
|
||||
const lines = text.split('\n');
|
||||
const utf16 = (pairs: RegExp) => lines.map(line => line.replace(pairs, run => ` ${run.replaceAll('\0', '')} `).replace(noise, ' '));
|
||||
const readings = [utf16(/(?:[^\0-\x08\x0b\x0c\x0e-\x1f\x7f\uFFFD]\0){2,}/g), utf16(/(?:\0[^\0-\x08\x0b\x0c\x0e-\x1f\x7f\uFFFD]){2,}/g), lines.map(line => line.replace(noise, ' '))];
|
||||
const combined = lines.map((_, i) => [...new Set(readings.map(reading => reading[i]))].join(' ')).join('\n');
|
||||
return [...new Set([combined, ...readings.map(reading => reading.join('\n'))])];
|
||||
}
|
||||
|
||||
/** Files whose source-syntax exemption needs the whole file: a line added inside an existing `'''`/`"""` block is a literal (#3630). */
|
||||
@@ -116,10 +122,10 @@ const WHOLE_FILE_MAX_BYTES = 4 * 1024 * 1024;
|
||||
* reads as clean. --text keeps a file git calls binary (a NUL byte, UTF-16, a
|
||||
* `-diff` attribute the branch itself can add) from printing "Binary files
|
||||
* differ" in place of its added lines; each such file is marked `binary`,
|
||||
* read through scannableText and held to its credential findings. A diff git did not finish (an exit, a kill, a
|
||||
* carries every reading scannableReadings gives and is held to its credential findings. A diff git did not finish (an exit, a kill, a
|
||||
* full buffer) throws: its partial output is never scanned as complete.
|
||||
*/
|
||||
export function addedLines(ctx: PregateContext): Array<{ file: string; text: string; keep?: number[]; binary?: boolean }> {
|
||||
export function addedLines(ctx: PregateContext): Array<{ file: string; texts: string[]; keep?: number[]; binary?: boolean }> {
|
||||
const diff = spawnSync('git', ['diff', '--text', '--no-color', '--no-ext-diff', '--no-textconv', '--src-prefix=a/', '--dst-prefix=b/', '--unified=0', '--diff-filter=ACMR', ctx.mergeBase, '--'], { cwd: ctx.repoRoot, encoding: 'utf8', timeout: 60_000, maxBuffer: 256 * 1024 * 1024, env: ctx.env });
|
||||
if (diff.error || diff.status !== 0) throw new Error(`git diff ${ctx.mergeBase.slice(0, 12)} did not finish (${diff.error?.message ?? (diff.status === null ? `killed by ${diff.signal}` : `exit ${diff.status}: ${(diff.stderr ?? '').trim().split('\n').pop() ?? ''}`)})`);
|
||||
const files = parseAddedLines(diff.stdout ?? '');
|
||||
@@ -149,8 +155,8 @@ export function addedLines(ctx: PregateContext): Array<{ file: string; text: str
|
||||
}
|
||||
return [...files]
|
||||
.map(([file, lines]) => whole.get(file) ? { file, ...whole.get(file)! } : { file, text: lines.join('\n') })
|
||||
.map(p => p.text.includes('\0') ? { ...p, text: scannableText(p.text), binary: true } : p)
|
||||
.filter(p => p.text.trim());
|
||||
.map(({ text, ...p }) => ({ ...p, texts: scannableReadings(text), ...(text.includes('\0') ? { binary: true } : {}) }))
|
||||
.filter(p => p.texts.some(text => text.trim()));
|
||||
}
|
||||
type Finding = { id?: string; description?: string; severity?: string; category?: string; line?: number; col?: number; file: string; fileLine?: boolean };
|
||||
type Scanned = Finding & AllowlistFinding;
|
||||
@@ -171,21 +177,28 @@ function run(ctx: PregateContext): CheckResult {
|
||||
let high = false;
|
||||
let medium = false;
|
||||
let parsedAll = true;
|
||||
for (const [i, part] of parts.entries()) {
|
||||
const file = path.join(tmp, String(i), (!part.keep && LEXICAL_CONTEXT_SOURCE.test(part.file) ? '' : path.basename(part.file)) || 'added.txt');
|
||||
for (const [i, part] of parts.entries()) for (const [j, text] of part.texts.entries()) {
|
||||
const file = path.join(tmp, `${i}.${j}`, (!part.keep && LEXICAL_CONTEXT_SOURCE.test(part.file) ? '' : path.basename(part.file)) || 'added.txt');
|
||||
fs.mkdirSync(path.dirname(file), { mode: 0o700 });
|
||||
fs.writeFileSync(file, part.text, { mode: 0o600 });
|
||||
fs.writeFileSync(file, text, { mode: 0o600 });
|
||||
const r = runScriptSync(path.join(BIN_DIR, 'gstack-redact'), ['--from-file', file, '--json', '--max-bytes', String(64 * 1024 * 1024)], { cwd: ctx.repoRoot, encoding: 'utf8', timeout: Math.min(ctx.timeoutMs, 300_000), maxBuffer: 64 * 1024 * 1024, env: ctx.env });
|
||||
if (r.error || r.status === null) return environmentResult(base, launchCause('tool', `gstack-redact did not finish on ${part.file || 'added lines'} (${r.error?.message ?? 'killed'})`), `gstack-redact did not finish on ${part.file || 'added lines'}; an incomplete scan never passes`);
|
||||
if (r.status !== 0 && r.status !== 2 && r.status !== 3) return environmentResult(base, launchCause('tool', `gstack-redact exited ${r.status} on ${part.file || 'added lines'}: ${(r.stderr ?? '').trim().split('\n').pop() ?? ''}`), `gstack-redact exited ${r.status}; an incomplete scan never passes`);
|
||||
high ||= r.status === 3;
|
||||
medium ||= r.status === 2;
|
||||
const lines = part.text.split('\n');
|
||||
const lines = text.split('\n');
|
||||
try { for (const f of (JSON.parse(r.stdout).findings ?? []).filter((f: Omit<Finding, 'file'>) => (!part.binary || f.category === 'secret') && (!part.keep || f.id?.startsWith('engine.') || (f.line !== undefined && part.keep.includes(f.line)))).map((f: Omit<Finding, 'file'>) => ({ ...f, id: f.id ?? 'finding', category: f.category ?? '?', severity: f.severity ?? '?', file: part.file, fileLine: !!part.keep, lineText: f.line ? lines[f.line - 1] : undefined }))) findings.push(f); } catch { parsedAll = false; }
|
||||
}
|
||||
// The allowlist applies only to findings the scanner reported in full; an unparsed report keeps its exit-code verdict.
|
||||
const applied = parsedAll && ctx.config.allowlist.length ? applyAllowlist(findings, ctx.config.allowlist) : { kept: findings, allowlisted: [], untested: [], missed: [] };
|
||||
const kept = applied.kept;
|
||||
// Readings of one binary file that keep a line unchanged report its token again: list each finding once.
|
||||
const reported = new Set<string>();
|
||||
const kept = applied.kept.filter(f => {
|
||||
const key = JSON.stringify([f.file, f.id, f.line, f.col, f.lineText]);
|
||||
if (reported.has(key)) return false;
|
||||
reported.add(key);
|
||||
return true;
|
||||
});
|
||||
if (parsedAll) { high = kept.some(f => f.severity === 'HIGH'); medium = kept.some(f => f.severity === 'MEDIUM'); }
|
||||
const render = (f: Scanned) => `${f.severity} ${f.id} [${f.category}]: ${f.description ?? ''}${f.line ? ` (${f.file ? `${f.file}, ` : ''}${f.fileLine ? 'line' : 'added line'} ${f.line})` : ''}`;
|
||||
const allowLines = [...applied.allowlisted.map(describeAllowlisted), ...applied.untested.map(describeUntested), ...applied.missed.map(describeMissed)];
|
||||
@@ -197,7 +210,7 @@ function run(ctx: PregateContext): CheckResult {
|
||||
const advisory = kept.filter(f => f.severity === 'MEDIUM' && f.category !== 'secret');
|
||||
const binary = parts.filter(p => p.binary).length;
|
||||
const scanned = `${parts.length} file${parts.length === 1 ? '' : 's'}${binary ? ` (${binary} binary)` : ''}`;
|
||||
const lineCount = parts.reduce((n, p) => n + (p.keep?.length ?? p.text.split('\n').length), 0);
|
||||
const lineCount = parts.reduce((n, p) => n + (p.keep?.length ?? p.texts[0].split('\n').length), 0);
|
||||
const fail = (lines: string[], detail: string): CheckResult => ({ ...base, status: 'fail', cause: 'diff', lines: [...lines, ...allowLines], code: 'PREGATE_SECRETS', detail: `${detail}${allowNote}`, fix: 'rotate and remove the credential from the added lines (history included); secrets cannot be downgraded (a benign finding is allowlisted by id, path and pattern in the policy, never env.value)' });
|
||||
if (!high && !medium) return { ...base, status: 'pass', ...(allowLines.length ? { lines: allowLines } : {}), detail: `${lineCount} added lines clean in ${scanned}${kept.length ? ` (${kept.length} WARN)` : ''}${allowNote}` };
|
||||
if (blocking.length) return fail([...blocking, ...advisory].map(render), `${blocking.length} credential finding(s) in added lines (${blocking.map(f => f.id).join(', ')})`);
|
||||
|
||||
Reference in new issue
Block a user