From ccc3f0fc148600267f5040d612cdce024e8cc4d1 Mon Sep 17 00:00:00 2001 From: Garry Tan Date: Mon, 9 Nov 2026 18:12:29 -0800 Subject: [PATCH] Catch secrets split across lines in UTF-16 files and stop deleted binaries from blocking pushes (GST-2261) (#4261) * Catch secrets split across lines in UTF-16 files and skip deleted files in the push scan (GST-2261) Each reading of a file git calls binary is now scanned as its own text with the file's line numbers, so a keyword and its value on the next line stay neighbours. The pre-push hook diffs with --irreversible-delete, so a deleted binary's old bytes no longer fill its 64 MiB buffer. Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * Leave the landed binary-scan changelog fragment unchanged (GST-2261) Review-Finding: changelog.d/scan-binary-files-for-secrets.md:1:release reader=core Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * List a token once when several readings of a binary file find it (GST-2261) Review-Finding: p1/specialist-batched#1 reader=specialist-batched Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * Report a repeated token once only within one binary file, so an allowlisted token never hides another (GST-2261) Review-Finding: p2/specialist-batched#1 reader=security Review-Finding: p2/red-team#2 reader=red-team Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * Name the changelog fragment after the branch (GST-2261) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * Keep the side-by-side reading of binary files so a key and its keyword in different readings still fail (GST-2261) Review-Finding: audit/cross-family#1 reader=gpt-6.1-sol Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --------- Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- bin/gstack-redact-prepush | 24 +++++-- .../capy-catch-secrets-split-across.md | 4 ++ docs/pregate.md | 18 +++-- lib/pregate/checks/secrets.ts | 65 +++++++++++-------- test/pregate-secrets-source-context.test.ts | 26 ++++++++ test/redact-prepush-hook.test.ts | 48 ++++++++++++++ 6 files changed, 148 insertions(+), 37 deletions(-) create mode 100644 changelog.d/capy-catch-secrets-split-across.md diff --git a/bin/gstack-redact-prepush b/bin/gstack-redact-prepush index 4aee829bd..7658a3c74 100755 --- a/bin/gstack-redact-prepush +++ b/bin/gstack-redact-prepush @@ -8,7 +8,10 @@ * the most common real-world leak. It does NOT scan history, LFS-stored or * submodule content, or non-added lines. History scanning is /cso's job. Files * git calls binary (a NUL byte, UTF-16, a `-diff` attribute) are diffed with - * --text, read through scannableText and held to credential findings (GST-2195). + * --text, scanned once per reading scannableReadings gives and held to + * credential findings (GST-2195, GST-2261). Deleted files are left out of the + * diff (--irreversible-delete): a delete cannot add a secret, and a deleted + * binary's old bytes would fill the diff buffer. * * Git pre-push interface: refs are read from STDIN, one per line: * @@ -44,7 +47,7 @@ import * as fs from "fs"; import * as path from "path"; import { normalizeWithMap, scan, type Finding, type ScanOptions } from "../lib/redact-engine"; import { mkdirpSync } from "../lib/fs-utils"; -import { scannableText } from "../lib/pregate/checks/secrets"; +import { scannableReadings } from "../lib/pregate/checks/secrets"; import { resolveStateRoot } from "../lib/state-root"; import { evaluateSealed, publishedCommits, renderSealed, sealedErrorReport, type SealedReport } from "../lib/sealed-guard"; @@ -257,7 +260,7 @@ function addedLinesFromNewCommits(localSha: string, remoteSha: string): AddedFil // gitStrict: a failed diff must never read as "nothing added" (#1946). // Collected per commit so a finding names the commit whose file line it is. out.push(...collectAddedLines(gitStrict([ - "show", "--text", "--unified=0", "--no-color", "--no-ext-diff", "--no-textconv", + "show", "--text", "--irreversible-delete", "--unified=0", "--no-color", "--no-ext-diff", "--no-textconv", "--src-prefix=a/", "--dst-prefix=b/", "--cc", "--format=", sha, ]), sha)); } @@ -301,8 +304,11 @@ function addedLinesFor(localSha: string, remoteSha: string): AddedFile[] { // --text: without it a file git calls binary (a NUL byte, UTF-16, a // `-diff` attribute the push itself can add) prints "Binary files differ" // and no added lines, so its credentials pushed unscanned. (GST-2195) + // --irreversible-delete: a deleted file's old lines are not printed. They + // add nothing to scan, and with --text a deleted 70 MiB binary filled the + // 64 MiB buffer and blocked the push. (GST-2261) const diff = gitStrict([ - "diff", "--text", "--unified=0", "--no-color", "--no-ext-diff", "--no-textconv", + "diff", "--text", "--irreversible-delete", "--unified=0", "--no-color", "--no-ext-diff", "--no-textconv", "--src-prefix=a/", "--dst-prefix=b/", range, ]); return collectAddedLines(diff); @@ -331,7 +337,7 @@ function collectAddedLines(diff: string, commit?: string): AddedFile[] { const lines = releaseVersionExempt(file, fileLines); const text = lines.join("\n"); const binary = text.includes("\0"); - if (lines.length) added.push({ file, text: binary ? scannableText(text) : text, lines: lineNumbers, ...(commit ? { commit } : {}), ...(binary ? { binary } : {}) }); + if (lines.length) for (const reading of scannableReadings(text)) added.push({ file, text: reading, lines: lineNumbers, ...(commit ? { commit } : {}), ...(binary ? { binary } : {}) }); fileLines = []; lineNumbers = []; }; @@ -701,18 +707,26 @@ function main() { // so the report names exactly what they suppressed. const allowed: string[] = []; const findingKey = (f: Finding) => `${f.id}:${f.line}:${f.col}`; + // The readings of one binary file share its line list: a token every reading keeps is reported once. + const reported = new Map>(); for (const { part, allowlist } of parts) { const opts = { repoVisibility: "private" as const, ...emails.options, ...(part.file && !/\.(?:dart|pyi?)$/i.test(part.file) ? { sourcePath: part.file } : {}) }; const findings = scanAddedLines(part.text, opts); const kept = allowlist.length ? new Set(scanAddedLines(part.text, { ...opts, allowlist }).map(findingKey)) : null; + const seen = reported.get(part.lines) ?? new Set(); + reported.set(part.lines, seen); + const lineTexts = part.text.split("\n"); for (const f of findings) { if (part.binary && f.category !== "secret") continue; if (kept && !kept.has(findingKey(f))) { if (f.severity === "HIGH" || f.severity === "MEDIUM") allowed.push(` ${f.severity} ${f.id} ${findingWhere(f, part)}\n`); continue; } + const key = `${findingKey(f)}:${lineTexts[f.line - 1]}`; + if (seen.has(key)) continue; + seen.add(key); if (f.severity === "HIGH") allHigh.push(f); else if (f.severity === "MEDIUM") { mediums.push(mediumLine(f, part)); diff --git a/changelog.d/capy-catch-secrets-split-across.md b/changelog.d/capy-catch-secrets-split-across.md new file mode 100644 index 000000000..2ae07efcf --- /dev/null +++ b/changelog.d/capy-catch-secrets-split-across.md @@ -0,0 +1,4 @@ +--- +important: security +--- +- **A secret split across two lines of UTF-16 text is now caught, and deleting a large binary no longer blocks a push.** The secret check and the pre-push hook read a file git calls binary three ways (as bytes and as little- and big-endian UTF-16), but they kept all three readings on one line. So in a UTF-16 file, an AWS secret key on the line under its `aws_secret_access_key:` key passed both guards, because the keyword line ended with another reading. Each reading is now scanned as its own text with the file's line numbers, so that pair fails in UTF-16LE and UTF-16BE as it does in plain text. The pre-push hook also leaves deleted files out of the diff it scans (`--irreversible-delete`). A delete cannot add a secret, and the old bytes of a deleted 70 MiB binary used to fill the hook's 64 MiB buffer and block the push with `git diff failed (status=killed/overflow)`. Adding a binary that large still blocks, and a replaced binary's old bytes still count toward the limit. diff --git a/docs/pregate.md b/docs/pregate.md index ed2510d04..32383920c 100644 --- a/docs/pregate.md +++ b/docs/pregate.md @@ -220,13 +220,19 @@ Two stages with separate clocks, both printed: file is read whole from disk. A file over the size limit, or whose full diff does not parse, is scanned on its added lines alone, without that context. A file git calls binary (a NUL byte, UTF-16 text, or a `-diff` attribute) - is still scanned: the diff runs with `--text`, and each added line is read + is still scanned: the diff runs with `--text`, and its added lines are read three ways, as bytes with control runs turned into spaces and as little- and - big-endian UTF-16. Only credential findings count in such a file, so image - bytes that look like a phone number are not reported. A file whose scanned - text is over the scanner's 16 MiB limit fails the check, and the pre-push - hook blocks a push whose diff is over 64 MiB; keep large binaries in Git LFS, - whose small pointer files scan as text. A merge-base diff git does not finish + big-endian UTF-16. The three readings are scanned side by side on each line, + so a keyword in one reading still meets its value in another, and each + reading is also scanned as its own text with the file's line numbers, so a keyword and its value on the next line (an AWS secret key + under its YAML key) are caught in UTF-16 text as in plain text. Only + credential findings count in such a file, so image bytes that look like a + phone number are not reported. A file whose scanned text is over the + scanner's 16 MiB limit fails the check, and the pre-push hook blocks a push + whose diff is over 64 MiB. The hook leaves deleted files out of that diff, + so removing a large binary does not count toward the limit, but a replaced + binary's old bytes still do. Keep large binaries in Git LFS, whose small + pointer files scan as text. A merge-base diff git does not finish (an error, a kill or a full buffer) is `PREGATE_ENVIRONMENT`, never a pass. The pass line names the files read: `12 added lines clean in 3 files (1 binary)`. - `strays`: a new file at the repository root, a new top-level directory, or a diff --git a/lib/pregate/checks/secrets.ts b/lib/pregate/checks/secrets.ts index d08fee348..c8dda8fe2 100644 --- a/lib/pregate/checks/secrets.ts +++ b/lib/pregate/checks/secrets.ts @@ -6,7 +6,7 @@ * never env.value); a scanner that does not finish or exits outside its * contract is PREGATE_ENVIRONMENT (kind tool), never a pass, and so is a * merge-base diff git did not finish. Files git calls binary are read too - * (scannableText). Runs regardless of lint tools. + * (scannableReadings). Runs regardless of lint tools. */ import { spawnSync } from 'node:child_process'; import * as fs from 'node:fs'; @@ -73,29 +73,35 @@ export function parsePostimages(diff: string): Map { } /** - * Text the scanner can read from a file git calls binary, which the diff - * passes through with --text (GST-2195). Text with no NUL is returned as is. - * Each line is read three ways. The plain reading turns each run of NUL, + * Texts the scanner can read from a file git calls binary, which the diff + * passes through with --text (GST-2195). Text with no NUL is its own only + * reading. Otherwise the file is read three ways. The plain reading turns each run of NUL, * other control bytes or bytes that are not UTF-8 (an image chunk, an * executable's padding) into one space. The little- and big-endian UTF-16 * readings first drop the NULs inside every run of character-then-NUL or * NUL-then-character pairs, which UTF-16 text puts beside each character of * its ASCII range however much of the file is Cyrillic or CJK, and set the run * apart with a space on each side, so the bytes of a CJK character beside it - * cannot join a token. Each distinct reading stays on the line, so a token is - * found in whichever one keeps it whole: a plain token that a NUL-flanked - * byte beside it would glue or cut in a UTF-16 reading, or a UTF-16 token - * whose neighbour pairs up with it in the other byte order. Line breaks stay, so line numbers still hold. Callers keep only credential - * findings from such a file: privacy patterns read its bytes as noise (a run - * of digits in a compressed icon reads as a phone number). + * cannot join a token. The first text is main's combined reading, each line + * holding its distinct readings side by side, so a keyword in one reading and + * its value in another still meet. Each distinct reading is then also a whole + * text of its own, scanned separately, so a token is found in whichever one keeps it whole (a + * plain token that a NUL-flanked byte beside it would glue or cut in a UTF-16 + * reading, or a UTF-16 token whose neighbour pairs up with it in the other + * byte order), and a keyword and its value on the next line stay neighbours + * (GST-2261). Line breaks stay, so every reading keeps the file's line + * numbers. Callers keep only credential findings from such a file: privacy + * patterns read its bytes as noise (a run of digits in a compressed icon reads + * as a phone number). */ -export function scannableText(text: string): string { - if (!text.includes('\0')) return text; +export function scannableReadings(text: string): string[] { + if (!text.includes('\0')) return [text]; const noise = /[\0-\x08\x0b\x0c\x0e-\x1f\x7f\uFFFD]+/g; - return text.split('\n').map(line => { - const utf16 = (pairs: RegExp) => line.replace(pairs, run => ` ${run.replaceAll('\0', '')} `).replace(noise, ' '); - return [...new Set([utf16(/(?:[^\0-\x08\x0b\x0c\x0e-\x1f\x7f\uFFFD]\0){2,}/g), utf16(/(?:\0[^\0-\x08\x0b\x0c\x0e-\x1f\x7f\uFFFD]){2,}/g), line.replace(noise, ' ')])].join(' '); - }).join('\n'); + const lines = text.split('\n'); + const utf16 = (pairs: RegExp) => lines.map(line => line.replace(pairs, run => ` ${run.replaceAll('\0', '')} `).replace(noise, ' ')); + const readings = [utf16(/(?:[^\0-\x08\x0b\x0c\x0e-\x1f\x7f\uFFFD]\0){2,}/g), utf16(/(?:\0[^\0-\x08\x0b\x0c\x0e-\x1f\x7f\uFFFD]){2,}/g), lines.map(line => line.replace(noise, ' '))]; + const combined = lines.map((_, i) => [...new Set(readings.map(reading => reading[i]))].join(' ')).join('\n'); + return [...new Set([combined, ...readings.map(reading => reading.join('\n'))])]; } /** Files whose source-syntax exemption needs the whole file: a line added inside an existing `'''`/`"""` block is a literal (#3630). */ @@ -116,10 +122,10 @@ const WHOLE_FILE_MAX_BYTES = 4 * 1024 * 1024; * reads as clean. --text keeps a file git calls binary (a NUL byte, UTF-16, a * `-diff` attribute the branch itself can add) from printing "Binary files * differ" in place of its added lines; each such file is marked `binary`, - * read through scannableText and held to its credential findings. A diff git did not finish (an exit, a kill, a + * carries every reading scannableReadings gives and is held to its credential findings. A diff git did not finish (an exit, a kill, a * full buffer) throws: its partial output is never scanned as complete. */ -export function addedLines(ctx: PregateContext): Array<{ file: string; text: string; keep?: number[]; binary?: boolean }> { +export function addedLines(ctx: PregateContext): Array<{ file: string; texts: string[]; keep?: number[]; binary?: boolean }> { const diff = spawnSync('git', ['diff', '--text', '--no-color', '--no-ext-diff', '--no-textconv', '--src-prefix=a/', '--dst-prefix=b/', '--unified=0', '--diff-filter=ACMR', ctx.mergeBase, '--'], { cwd: ctx.repoRoot, encoding: 'utf8', timeout: 60_000, maxBuffer: 256 * 1024 * 1024, env: ctx.env }); if (diff.error || diff.status !== 0) throw new Error(`git diff ${ctx.mergeBase.slice(0, 12)} did not finish (${diff.error?.message ?? (diff.status === null ? `killed by ${diff.signal}` : `exit ${diff.status}: ${(diff.stderr ?? '').trim().split('\n').pop() ?? ''}`)})`); const files = parseAddedLines(diff.stdout ?? ''); @@ -149,8 +155,8 @@ export function addedLines(ctx: PregateContext): Array<{ file: string; text: str } return [...files] .map(([file, lines]) => whole.get(file) ? { file, ...whole.get(file)! } : { file, text: lines.join('\n') }) - .map(p => p.text.includes('\0') ? { ...p, text: scannableText(p.text), binary: true } : p) - .filter(p => p.text.trim()); + .map(({ text, ...p }) => ({ ...p, texts: scannableReadings(text), ...(text.includes('\0') ? { binary: true } : {}) })) + .filter(p => p.texts.some(text => text.trim())); } type Finding = { id?: string; description?: string; severity?: string; category?: string; line?: number; col?: number; file: string; fileLine?: boolean }; type Scanned = Finding & AllowlistFinding; @@ -171,21 +177,28 @@ function run(ctx: PregateContext): CheckResult { let high = false; let medium = false; let parsedAll = true; - for (const [i, part] of parts.entries()) { - const file = path.join(tmp, String(i), (!part.keep && LEXICAL_CONTEXT_SOURCE.test(part.file) ? '' : path.basename(part.file)) || 'added.txt'); + for (const [i, part] of parts.entries()) for (const [j, text] of part.texts.entries()) { + const file = path.join(tmp, `${i}.${j}`, (!part.keep && LEXICAL_CONTEXT_SOURCE.test(part.file) ? '' : path.basename(part.file)) || 'added.txt'); fs.mkdirSync(path.dirname(file), { mode: 0o700 }); - fs.writeFileSync(file, part.text, { mode: 0o600 }); + fs.writeFileSync(file, text, { mode: 0o600 }); const r = runScriptSync(path.join(BIN_DIR, 'gstack-redact'), ['--from-file', file, '--json', '--max-bytes', String(64 * 1024 * 1024)], { cwd: ctx.repoRoot, encoding: 'utf8', timeout: Math.min(ctx.timeoutMs, 300_000), maxBuffer: 64 * 1024 * 1024, env: ctx.env }); if (r.error || r.status === null) return environmentResult(base, launchCause('tool', `gstack-redact did not finish on ${part.file || 'added lines'} (${r.error?.message ?? 'killed'})`), `gstack-redact did not finish on ${part.file || 'added lines'}; an incomplete scan never passes`); if (r.status !== 0 && r.status !== 2 && r.status !== 3) return environmentResult(base, launchCause('tool', `gstack-redact exited ${r.status} on ${part.file || 'added lines'}: ${(r.stderr ?? '').trim().split('\n').pop() ?? ''}`), `gstack-redact exited ${r.status}; an incomplete scan never passes`); high ||= r.status === 3; medium ||= r.status === 2; - const lines = part.text.split('\n'); + const lines = text.split('\n'); try { for (const f of (JSON.parse(r.stdout).findings ?? []).filter((f: Omit) => (!part.binary || f.category === 'secret') && (!part.keep || f.id?.startsWith('engine.') || (f.line !== undefined && part.keep.includes(f.line)))).map((f: Omit) => ({ ...f, id: f.id ?? 'finding', category: f.category ?? '?', severity: f.severity ?? '?', file: part.file, fileLine: !!part.keep, lineText: f.line ? lines[f.line - 1] : undefined }))) findings.push(f); } catch { parsedAll = false; } } // The allowlist applies only to findings the scanner reported in full; an unparsed report keeps its exit-code verdict. const applied = parsedAll && ctx.config.allowlist.length ? applyAllowlist(findings, ctx.config.allowlist) : { kept: findings, allowlisted: [], untested: [], missed: [] }; - const kept = applied.kept; + // Readings of one binary file that keep a line unchanged report its token again: list each finding once. + const reported = new Set(); + const kept = applied.kept.filter(f => { + const key = JSON.stringify([f.file, f.id, f.line, f.col, f.lineText]); + if (reported.has(key)) return false; + reported.add(key); + return true; + }); if (parsedAll) { high = kept.some(f => f.severity === 'HIGH'); medium = kept.some(f => f.severity === 'MEDIUM'); } const render = (f: Scanned) => `${f.severity} ${f.id} [${f.category}]: ${f.description ?? ''}${f.line ? ` (${f.file ? `${f.file}, ` : ''}${f.fileLine ? 'line' : 'added line'} ${f.line})` : ''}`; const allowLines = [...applied.allowlisted.map(describeAllowlisted), ...applied.untested.map(describeUntested), ...applied.missed.map(describeMissed)]; @@ -197,7 +210,7 @@ function run(ctx: PregateContext): CheckResult { const advisory = kept.filter(f => f.severity === 'MEDIUM' && f.category !== 'secret'); const binary = parts.filter(p => p.binary).length; const scanned = `${parts.length} file${parts.length === 1 ? '' : 's'}${binary ? ` (${binary} binary)` : ''}`; - const lineCount = parts.reduce((n, p) => n + (p.keep?.length ?? p.text.split('\n').length), 0); + const lineCount = parts.reduce((n, p) => n + (p.keep?.length ?? p.texts[0].split('\n').length), 0); const fail = (lines: string[], detail: string): CheckResult => ({ ...base, status: 'fail', cause: 'diff', lines: [...lines, ...allowLines], code: 'PREGATE_SECRETS', detail: `${detail}${allowNote}`, fix: 'rotate and remove the credential from the added lines (history included); secrets cannot be downgraded (a benign finding is allowlisted by id, path and pattern in the policy, never env.value)' }); if (!high && !medium) return { ...base, status: 'pass', ...(allowLines.length ? { lines: allowLines } : {}), detail: `${lineCount} added lines clean in ${scanned}${kept.length ? ` (${kept.length} WARN)` : ''}${allowNote}` }; if (blocking.length) return fail([...blocking, ...advisory].map(render), `${blocking.length} credential finding(s) in added lines (${blocking.map(f => f.id).join(', ')})`); diff --git a/test/pregate-secrets-source-context.test.ts b/test/pregate-secrets-source-context.test.ts index f0d293ddd..0c26a363e 100644 --- a/test/pregate-secrets-source-context.test.ts +++ b/test/pregate-secrets-source-context.test.ts @@ -256,6 +256,32 @@ describe('pregate secrets: source context and diff integrity', () => { for (const file of ['ru.strings', 'zh.strings', 'zh-be.strings']) expect(r.lines).toContainEqual(expect.stringMatching(new RegExp(`^HIGH github\\.pat .*\\(${file.replace('.', '\\.')}, added line 9\\)$`))); }); + // Value: protects=a keyword and its value on the next line of UTF-16 text fail like the same pair in plain text; fails_when=the readings of a binary file share one line, so the keyword line ends with another reading and the next-line rule loses it; + // why_new=GSTA-474: an AWS secret key under its YAML key passed in UTF-16LE (#4243 advisory note 1); seam=fixture repo; no_claim=UTF-32 + test('an AWS secret key on the line under its keyword fails in UTF-16LE and UTF-16BE like plain text (GST-2261)', () => { + const pair = `aws_secret_access_key:\n ${['Zq3Xn7P2Lm4Rt8Wv', 'Bc5Df6Gh7Jk8Lm9Np0Qr1St2'].join('')}\n`; + const r = repo({ 'plain.yml': pair, 'le.yml': utf16(pair), 'be.yml': utf16(pair, true) }).secrets(); + expect(r.status).toBe('fail'); + for (const file of ['plain.yml', 'le.yml', 'be.yml']) expect(r.lines).toContainEqual(expect.stringMatching(new RegExp(`^HIGH aws\\.secret_key .*\\(${file.replace('.', '\\.')}, added line \\d\\)$`))); + }); + + // Value: protects=a plain token in a binary file that also holds UTF-16 text is listed once; fails_when=each reading of the file reports the same finding again; + // why_new=GST-2261 review (testing specialist): a mixed binary listed one token three times; seam=fixture repo; no_claim=a UTF-16 token found at different columns in both byte orders + test('a plain token in a binary file that also holds UTF-16 text is listed once (GST-2261)', () => { + const r = repo({ 'mixed.bin': Buffer.concat([Buffer.from(`token ${PAT}\n`), utf16('hello world\n')]) }).secrets(); + expect(r.status).toBe('fail'); + expect(r.lines.filter(l => /github\.pat .*mixed\.bin/.test(l))).toHaveLength(1); + }); + + // Value: protects=a credential whose keyword and value sit in different readings of one binary line still fails, as on main; fails_when=the readings are only scanned apart; + // why_new=GST-2261 cross-family audit: a plain-byte key before its UTF-16LE keyword passed once the combined reading was dropped; seam=fixture repo; no_claim=UTF-32 + test('a plain-byte AWS key before its UTF-16LE keyword on one line fails like main (GST-2261)', () => { + const mixed = Buffer.concat([Buffer.from(`\0x\0${['Zq3Xn7P2Lm4Rt8Wv', 'Bc5Df6Gh7Jk8Lm9Np0Qr1St2'].join('')}\0y\0 `, 'latin1'), Buffer.from('aws_secret_access_key:', 'utf16le'), Buffer.from('\n')]); + const r = repo({ 'mixed.bin': mixed }).secrets(); + expect(r.status).toBe('fail'); + expect(r.lines).toContainEqual(expect.stringMatching(/^HIGH aws\.secret_key .*\(mixed\.bin, added line 1\)$/)); + }); + // Value: protects=a UTF-16 token right beside a CJK character is found, and so is a plain token beside a run of character-and-NUL pairs; fails_when=the CJK character's bytes glue onto the token, or padding the run cuts the plain token; // why_new=GST-2195 review (outside voice): U+4E2D before a token read as "-N" glued to it; seam=fixture repo; no_claim=UTF-32 test('a token right beside a CJK character in UTF-16, or beside a NUL run in binary content, fails like plain text (GST-2195)', () => { diff --git a/test/redact-prepush-hook.test.ts b/test/redact-prepush-hook.test.ts index 4b30ed0e6..6c9d9ee43 100644 --- a/test/redact-prepush-hook.test.ts +++ b/test/redact-prepush-hook.test.ts @@ -818,6 +818,54 @@ describe("files git calls binary are scanned (GST-2195)", () => { expect({ code, pat: stderr.includes("github.pat") }).toEqual({ code: 1, pat: true }); }); + test("an AWS secret key on the line under its keyword blocks in UTF-16LE and UTF-16BE like plain text (GST-2261)", () => { + const base = git(["rev-parse", "HEAD"]); + const pair = `aws_secret_access_key:\n ${["Zq3Xn7P2Lm4Rt8Wv", "Bc5Df6Gh7Jk8Lm9Np0Qr1St2"].join("")}\n`; + const shapes: Array<[string, string | Buffer]> = [["plain", pair], ["utf-16le", utf16(pair)], ["utf-16be", utf16(pair, true)]]; + const results = shapes.map(([shape, content]) => { + git(["reset", "-q", "--hard", base]); + const head = commitFiles({ "creds.yml": content }, shape); + const { code, stderr } = runHook(`refs/heads/main ${head} refs/heads/main ${base}\n`); + return { shape, code, aws: stderr.includes("aws.secret_key") }; + }); + expect(results).toEqual(shapes.map(([shape]) => ({ shape, code: 1, aws: true }))); + }); + + test("a plain token in a binary file that also holds UTF-16 text is listed once (GST-2261)", () => { + const base = git(["rev-parse", "HEAD"]); + const head = commitFiles({ "mixed.bin": Buffer.concat([Buffer.from(`token ${FAKE_PAT}\n`), utf16("hello world\n")]) }, "mixed binary"); + const { code, stderr } = runHook(`refs/heads/main ${head} refs/heads/main ${base}\n`); + expect({ code, listed: stderr.split("\n").filter((l) => l.includes("github.pat")).length }).toEqual({ code: 1, listed: 1 }); + }); + + test("an allowlisted token in one file whose name git quotes does not hide another token at the same spot in a second such file (GST-2261)", () => { + const base = git(["rev-parse", "HEAD"]); + const other = ["ghp", "Xv7Nc4Rt9Wy1Kp6Hd3Fs5Gb0Jx8Ue2Zq8Lm2"].join("_"); + const head = commitFiles({ "café-1.txt": `token: ${FAKE_PAT}\n`, "café-2.txt": `token: ${other}\n`, ".gstack-redact-allowlist": `${FAKE_PAT}\n` }, "quoted names"); + const { code, stderr } = runHook(`refs/heads/main ${head} refs/heads/main ${base}\n`); + git(["reset", "-q", "--hard", base]); + expect({ code, blocked: stderr.includes("BLOCKED") }).toEqual({ code: 1, blocked: true }); + }); + + test("a plain-byte AWS key before its UTF-16LE keyword on one line blocks like main (GST-2261)", () => { + const base = git(["rev-parse", "HEAD"]); + const mixed = Buffer.concat([Buffer.from(`\0x\0${["Zq3Xn7P2Lm4Rt8Wv", "Bc5Df6Gh7Jk8Lm9Np0Qr1St2"].join("")}\0y\0 `, "latin1"), Buffer.from("aws_secret_access_key:", "utf16le"), Buffer.from("\n")]); + const head = commitFiles({ "mixed.bin": mixed }, "mixed"); + const { code, stderr } = runHook(`refs/heads/main ${head} refs/heads/main ${base}\n`); + expect({ code, aws: stderr.includes("aws.secret_key") }).toEqual({ code: 1, aws: true }); + }); + + test("deleting a 70 MiB binary pushes clean, and adding one still blocks on the 64 MiB diff limit (GST-2261)", () => { + const base = git(["rev-parse", "HEAD"]); + const big = commitFiles({ "big.bin": Buffer.alloc(70 * 1024 * 1024) }, "add big binary"); + const added = runHook(`refs/heads/main ${big} refs/heads/main ${base}\n`); + expect({ code: added.code, overflow: added.stderr.includes("killed/overflow") }).toEqual({ code: 1, overflow: true }); + git(["rm", "-q", "big.bin"]); + git(["commit", "-q", "-m", "drop big binary"]); + const dropped = git(["rev-parse", "HEAD"]); + expect(runHook(`refs/heads/main ${dropped} refs/heads/main ${big}\n`)).toEqual({ code: 0, stderr: "" }); + }); + test("the real binary assets this repo stores push clean", () => { const root = path.resolve(import.meta.dir, ".."); const assets = spawnSync("git", ["ls-files", "--eol"], { cwd: root, encoding: "utf8", timeout: 30_000 }).stdout.split("\n").filter((l) => l.startsWith("i/-text")).map((l) => l.split("\t").pop()!);