Catch secrets split across lines in UTF-16 files and stop deleted binaries from blocking pushes (GST-2261) (#4261)

* Catch secrets split across lines in UTF-16 files and skip deleted files in the push scan (GST-2261)

Each reading of a file git calls binary is now scanned as its own text with the file's line numbers, so a keyword and its value on the next line stay neighbours. The pre-push hook diffs with --irreversible-delete, so a deleted binary's old bytes no longer fill its 64 MiB buffer.

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* Leave the landed binary-scan changelog fragment unchanged (GST-2261)

Review-Finding: changelog.d/scan-binary-files-for-secrets.md:1:release reader=core
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* List a token once when several readings of a binary file find it (GST-2261)

Review-Finding: p1/specialist-batched#1 reader=specialist-batched
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* Report a repeated token once only within one binary file, so an allowlisted token never hides another (GST-2261)

Review-Finding: p2/specialist-batched#1 reader=security
Review-Finding: p2/red-team#2 reader=red-team
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* Name the changelog fragment after the branch (GST-2261)

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* Keep the side-by-side reading of binary files so a key and its keyword in different readings still fail (GST-2261)

Review-Finding: audit/cross-family#1 reader=gpt-6.1-sol
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

---------

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

test/pregate-secrets-source-context.test.ts

@@ -256,6 +256,32 @@ describe('pregate secrets: source context and diff integrity', () => {
     for (const file of ['ru.strings', 'zh.strings', 'zh-be.strings']) expect(r.lines).toContainEqual(expect.stringMatching(new RegExp(`^HIGH github\\.pat .*\\(${file.replace('.', '\\.')}, added line 9\\)$`)));
   });
 
+  // Value: protects=a keyword and its value on the next line of UTF-16 text fail like the same pair in plain text; fails_when=the readings of a binary file share one line, so the keyword line ends with another reading and the next-line rule loses it;
+  //   why_new=GSTA-474: an AWS secret key under its YAML key passed in UTF-16LE (#4243 advisory note 1); seam=fixture repo; no_claim=UTF-32
+  test('an AWS secret key on the line under its keyword fails in UTF-16LE and UTF-16BE like plain text (GST-2261)', () => {
+    const pair = `aws_secret_access_key:\n  ${['Zq3Xn7P2Lm4Rt8Wv', 'Bc5Df6Gh7Jk8Lm9Np0Qr1St2'].join('')}\n`;
+    const r = repo({ 'plain.yml': pair, 'le.yml': utf16(pair), 'be.yml': utf16(pair, true) }).secrets();
+    expect(r.status).toBe('fail');
+    for (const file of ['plain.yml', 'le.yml', 'be.yml']) expect(r.lines).toContainEqual(expect.stringMatching(new RegExp(`^HIGH aws\\.secret_key .*\\(${file.replace('.', '\\.')}, added line \\d\\)$`)));
+  });
+
+  // Value: protects=a plain token in a binary file that also holds UTF-16 text is listed once; fails_when=each reading of the file reports the same finding again;
+  //   why_new=GST-2261 review (testing specialist): a mixed binary listed one token three times; seam=fixture repo; no_claim=a UTF-16 token found at different columns in both byte orders
+  test('a plain token in a binary file that also holds UTF-16 text is listed once (GST-2261)', () => {
+    const r = repo({ 'mixed.bin': Buffer.concat([Buffer.from(`token ${PAT}\n`), utf16('hello world\n')]) }).secrets();
+    expect(r.status).toBe('fail');
+    expect(r.lines.filter(l => /github\.pat .*mixed\.bin/.test(l))).toHaveLength(1);
+  });
+
+  // Value: protects=a credential whose keyword and value sit in different readings of one binary line still fails, as on main; fails_when=the readings are only scanned apart;
+  //   why_new=GST-2261 cross-family audit: a plain-byte key before its UTF-16LE keyword passed once the combined reading was dropped; seam=fixture repo; no_claim=UTF-32
+  test('a plain-byte AWS key before its UTF-16LE keyword on one line fails like main (GST-2261)', () => {
+    const mixed = Buffer.concat([Buffer.from(`\0x\0${['Zq3Xn7P2Lm4Rt8Wv', 'Bc5Df6Gh7Jk8Lm9Np0Qr1St2'].join('')}\0y\0 `, 'latin1'), Buffer.from('aws_secret_access_key:', 'utf16le'), Buffer.from('\n')]);
+    const r = repo({ 'mixed.bin': mixed }).secrets();
+    expect(r.status).toBe('fail');
+    expect(r.lines).toContainEqual(expect.stringMatching(/^HIGH aws\.secret_key .*\(mixed\.bin, added line 1\)$/));
+  });
+
   // Value: protects=a UTF-16 token right beside a CJK character is found, and so is a plain token beside a run of character-and-NUL pairs; fails_when=the CJK character's bytes glue onto the token, or padding the run cuts the plain token;
   //   why_new=GST-2195 review (outside voice): U+4E2D before a token read as "-N" glued to it; seam=fixture repo; no_claim=UTF-32
   test('a token right beside a CJK character in UTF-16, or beside a NUL run in binary content, fails like plain text (GST-2195)', () => {

test/redact-prepush-hook.test.ts

@@ -818,6 +818,54 @@ describe("files git calls binary are scanned (GST-2195)", () => {
     expect({ code, pat: stderr.includes("github.pat") }).toEqual({ code: 1, pat: true });
   });
 
+  test("an AWS secret key on the line under its keyword blocks in UTF-16LE and UTF-16BE like plain text (GST-2261)", () => {
+    const base = git(["rev-parse", "HEAD"]);
+    const pair = `aws_secret_access_key:\n  ${["Zq3Xn7P2Lm4Rt8Wv", "Bc5Df6Gh7Jk8Lm9Np0Qr1St2"].join("")}\n`;
+    const shapes: Array<[string, string | Buffer]> = [["plain", pair], ["utf-16le", utf16(pair)], ["utf-16be", utf16(pair, true)]];
+    const results = shapes.map(([shape, content]) => {
+      git(["reset", "-q", "--hard", base]);
+      const head = commitFiles({ "creds.yml": content }, shape);
+      const { code, stderr } = runHook(`refs/heads/main ${head} refs/heads/main ${base}\n`);
+      return { shape, code, aws: stderr.includes("aws.secret_key") };
+    });
+    expect(results).toEqual(shapes.map(([shape]) => ({ shape, code: 1, aws: true })));
+  });
+
+  test("a plain token in a binary file that also holds UTF-16 text is listed once (GST-2261)", () => {
+    const base = git(["rev-parse", "HEAD"]);
+    const head = commitFiles({ "mixed.bin": Buffer.concat([Buffer.from(`token ${FAKE_PAT}\n`), utf16("hello world\n")]) }, "mixed binary");
+    const { code, stderr } = runHook(`refs/heads/main ${head} refs/heads/main ${base}\n`);
+    expect({ code, listed: stderr.split("\n").filter((l) => l.includes("github.pat")).length }).toEqual({ code: 1, listed: 1 });
+  });
+
+  test("an allowlisted token in one file whose name git quotes does not hide another token at the same spot in a second such file (GST-2261)", () => {
+    const base = git(["rev-parse", "HEAD"]);
+    const other = ["ghp", "Xv7Nc4Rt9Wy1Kp6Hd3Fs5Gb0Jx8Ue2Zq8Lm2"].join("_");
+    const head = commitFiles({ "café-1.txt": `token: ${FAKE_PAT}\n`, "café-2.txt": `token: ${other}\n`, ".gstack-redact-allowlist": `${FAKE_PAT}\n` }, "quoted names");
+    const { code, stderr } = runHook(`refs/heads/main ${head} refs/heads/main ${base}\n`);
+    git(["reset", "-q", "--hard", base]);
+    expect({ code, blocked: stderr.includes("BLOCKED") }).toEqual({ code: 1, blocked: true });
+  });
+
+  test("a plain-byte AWS key before its UTF-16LE keyword on one line blocks like main (GST-2261)", () => {
+    const base = git(["rev-parse", "HEAD"]);
+    const mixed = Buffer.concat([Buffer.from(`\0x\0${["Zq3Xn7P2Lm4Rt8Wv", "Bc5Df6Gh7Jk8Lm9Np0Qr1St2"].join("")}\0y\0 `, "latin1"), Buffer.from("aws_secret_access_key:", "utf16le"), Buffer.from("\n")]);
+    const head = commitFiles({ "mixed.bin": mixed }, "mixed");
+    const { code, stderr } = runHook(`refs/heads/main ${head} refs/heads/main ${base}\n`);
+    expect({ code, aws: stderr.includes("aws.secret_key") }).toEqual({ code: 1, aws: true });
+  });
+
+  test("deleting a 70 MiB binary pushes clean, and adding one still blocks on the 64 MiB diff limit (GST-2261)", () => {
+    const base = git(["rev-parse", "HEAD"]);
+    const big = commitFiles({ "big.bin": Buffer.alloc(70 * 1024 * 1024) }, "add big binary");
+    const added = runHook(`refs/heads/main ${big} refs/heads/main ${base}\n`);
+    expect({ code: added.code, overflow: added.stderr.includes("killed/overflow") }).toEqual({ code: 1, overflow: true });
+    git(["rm", "-q", "big.bin"]);
+    git(["commit", "-q", "-m", "drop big binary"]);
+    const dropped = git(["rev-parse", "HEAD"]);
+    expect(runHook(`refs/heads/main ${dropped} refs/heads/main ${big}\n`)).toEqual({ code: 0, stderr: "" });
+  });
+
   test("the real binary assets this repo stores push clean", () => {
     const root = path.resolve(import.meta.dir, "..");
     const assets = spawnSync("git", ["ls-files", "--eol"], { cwd: root, encoding: "utf8", timeout: 30_000 }).stdout.split("\n").filter((l) => l.startsWith("i/-text")).map((l) => l.split("\t").pop()!);