diff --git a/bin/gstack-redact-prepush b/bin/gstack-redact-prepush index 3da23c7ad..6d3f2a5c9 100755 --- a/bin/gstack-redact-prepush +++ b/bin/gstack-redact-prepush @@ -19,7 +19,8 @@ * * Behavior: * - HIGH finding in added lines → print + exit 1 (block), for public AND private. - * - MEDIUM → warn (non-blocking). LOW/WARN → silent. + * - MEDIUM → warn (non-blocking), one line per finding naming its rule, + * file line and fix, never the matched value. LOW/WARN → silent. * - GSTACK_REDACT_PREPUSH=skip → log + exit 0 (escape valve). * * Installed/uninstalled via `gstack-redact install-prepush-hook` (see the @@ -210,7 +211,7 @@ function unknownRemoteTipBase(localSha: string): string | null { * resolving a merge is still caught, while an ordinary merge contributes * nothing. Returns null when the notion does not apply, so callers fall back. */ -function addedLinesFromNewCommits(localSha: string, remoteSha: string): string | null { +function addedLinesFromNewCommits(localSha: string, remoteSha: string): AddedFile[] | null { // remoteSha is what git TELLS us the remote has, and it is authoritative in a // way `--remotes` is not: remote-tracking refs can be absent (a fresh clone // that never fetched, a push to a remote with no tracking ref) or stale. Drop @@ -236,26 +237,32 @@ function addedLinesFromNewCommits(localSha: string, remoteSha: string): string | const shas = narrowed.split("\n").filter(Boolean); // A rewrite of long history should fall back rather than shell out per commit. if (shas.length > 500) return null; - const out: string[] = []; + const out: AddedFile[] = []; for (const sha of shas) { // gitStrict: a failed diff must never read as "nothing added" (#1946). - out.push(gitStrict([ + // Collected per commit so a finding names the commit whose file line it is. + out.push(...collectAddedLines(gitStrict([ "show", "--unified=0", "--no-color", "--no-ext-diff", "--no-textconv", "--src-prefix=a/", "--dst-prefix=b/", "--cc", "--format=", sha, - ])); + ]), sha)); } - return out.join("\n"); + return out; } -/** Return the added-line text for a ref update being pushed. */ -/** Added lines grouped by the file they were added to (`+++ b/` path; empty when unknown). */ -interface AddedFile { file: string; text: string } +/** + * Added lines grouped by the file they were added to (`+++ b/` path; empty + * when unknown). `lines[i]` is the file line number of the i-th line of + * `text`, read from the hunk headers; `commit` is set when the lines came from + * one commit's diff rather than the net diff of the pushed range. + */ +interface AddedFile { file: string; text: string; lines: number[]; commit?: string } +/** Return the added lines for a ref update being pushed. */ function addedLinesFor(localSha: string, remoteSha: string): AddedFile[] { // Preferred ONLY when this push carries catch-up commits: scanning them again // is the bug. Every other shape falls through to the range logic below. const fromNew = addedLinesFromNewCommits(localSha, remoteSha); - if (fromNew !== null) return collectAddedLines(fromNew); + if (fromNew !== null) return fromNew; let range: string; if (ZERO.test(remoteSha) || !objectExists(remoteSha)) { @@ -287,7 +294,7 @@ function addedLinesFor(localSha: string, remoteSha: string): AddedFile[] { * Added-line text from a unified diff. Shared by both range strategies so the * hunk-aware header handling below cannot drift between them. */ -function collectAddedLines(diff: string): AddedFile[] { +function collectAddedLines(diff: string, commit?: string): AddedFile[] { const added: AddedFile[] = []; // Hunk-aware header skip (#2498): `+++ ` is only a FILE HEADER outside a // hunk. Inside a hunk, an added content line whose text begins with "++" @@ -296,10 +303,17 @@ function collectAddedLines(diff: string): AddedFile[] { let inHunk = false; let file = ""; let fileLines: string[] = []; + let lineNumbers: number[] = []; + // File line tracking from hunk headers. A combined (--cc) header has one + // more `@` than it has parents, and each body line one prefix column per + // parent; a line belongs to the resulting file unless a column says `-`. + let parents = 1; + let nextLine = 0; const flush = () => { const lines = releaseVersionExempt(file, fileLines); - if (lines.length) added.push({ file, text: lines.join("\n") }); + if (lines.length) added.push({ file, text: lines.join("\n"), lines: lineNumbers, ...(commit ? { commit } : {}) }); fileLines = []; + lineNumbers = []; }; for (const line of diff.split("\n")) { // `diff --` rather than `diff --git`: a merge scanned with --cc emits @@ -307,10 +321,20 @@ function collectAddedLines(diff: string): AddedFile[] { // boundaries and read the next file's `+++ b/...` header as content. Only // noise (it over-scans, never under-scans), but the boundary is real. if (line.startsWith("diff --")) { flush(); file = ""; inHunk = false; continue; } - if (line.startsWith("@@")) { inHunk = true; continue; } + if (line.startsWith("@@")) { + inHunk = true; + parents = (/^@+/.exec(line)?.[0].length ?? 2) - 1; + nextLine = Number(/ \+(\d+)(?:,\d+)? @+/.exec(line)?.[1] ?? 0); + continue; + } if (!inHunk && line.startsWith("+++ ")) { file = line.startsWith("+++ b/") ? line.slice(6) : ""; continue; } if (!inHunk && (line.startsWith("+++") || line.startsWith("---"))) continue; - if (line.startsWith("+")) fileLines.push(line.slice(1)); + const inResult = inHunk && !line.startsWith("\\") && !line.slice(0, parents).includes("-"); + const lineNumber = inResult ? nextLine++ : 0; + if (line.startsWith("+")) { + fileLines.push(line.slice(1)); + lineNumbers.push(lineNumber); + } } flush(); return added; @@ -366,9 +390,10 @@ const MAX_SCAN_BYTES = 1024 * 1024; * the engine intact, so a genuinely unscannable blob (minified bundle, * embedded base64) trips input_too_large and blocks exactly as before. * - * Findings' line/col are slice-relative, which is fine here — this hook only - * reads severity, id and preview. Do not lift this into the engine, where - * callers rely on absolute line numbers. + * The engine reports line/col relative to each slice; an admitted finding's + * `line` is rewritten to its 1-based line within `added`, which the caller + * maps to the file line. Do not lift this into the engine, where callers rely + * on absolute line numbers. */ function scanAddedLines(added: string, opts: Parameters[1]): Finding[] { const findings: Finding[] = []; @@ -426,6 +451,19 @@ function scanAddedLines(added: string, opts: Parameters[1]): Findin } }; + const addedLineStarts = [0]; + for (let i = 0; i < added.length; i++) if (added[i] === "\n") addedLineStarts.push(i + 1); + const lineAt = (offset: number): number => { + let lo = 0; + let hi = addedLineStarts.length - 1; + while (lo < hi) { + const mid = (lo + hi + 1) >> 1; + if (addedLineStarts[mid] <= offset) lo = mid; + else hi = mid - 1; + } + return lo + 1; + }; + for (const core of cores) { const left = contextStart(core.start); const text = added.slice(left, contextEnd(core.end)); @@ -438,7 +476,7 @@ function scanAddedLines(added: string, opts: Parameters[1]): Findin for (let i = 0; i < text.length; i++) if (text[i] === "\n") starts.push(i + 1); for (const finding of result.findings) { const offset = starts[finding.line - 1] + finding.col - 1 + left; - if (offset >= core.start && offset < core.end) findings.push(finding); + if (offset >= core.start && offset < core.end) findings.push({ ...finding, line: lineAt(offset) }); } } @@ -462,6 +500,14 @@ function logSkip(reason: string): void { } } +/** One MEDIUM report line: rule id, file line (and commit), and the fix. Never the matched value. */ +function mediumLine(f: Finding, part: AddedFile): string { + const line = part.lines[f.line - 1]; + const where = `${part.file || "(unknown file)"}${line ? `:${line}` : ""}` + + (part.commit ? ` in commit ${part.commit.slice(0, 12)}` : ""); + return ` MEDIUM ${f.id} ${where} fix: remove the value before it becomes public\n`; +} + function main() { if ((process.env.GSTACK_REDACT_PREPUSH || "").toLowerCase() === "skip") { logSkip(process.env.GSTACK_REDACT_PREPUSH_REASON || "env-skip"); @@ -476,8 +522,7 @@ function main() { .filter(Boolean) .map((l) => l.split(/\s+/)); - const allHigh: Finding[] = []; - let mediumCount = 0; + const parts: AddedFile[] = []; for (const fields of refs) { // Fail CLOSED on a ref line we cannot parse (#2498): git hands pre-push @@ -511,25 +556,30 @@ function main() { ); process.exit(1); } - if (!added.some((part) => part.text.trim())) continue; - // Visibility doesn't change HIGH behavior; pass private so nothing is treated - // as public-strict (HIGH blocks regardless either way). - // Sliced (see scanAddedLines) so a large-but-legitimate diff is actually - // scanned rather than blocked unscanned on the engine's size cap. - // Scanned per file so a pattern can use the file's type (#3048). - for (const part of added) { - for (const f of scanAddedLines(part.text, { repoVisibility: "private", ...(part.file ? { sourcePath: part.file } : {}) })) { - if (f.severity === "HIGH") allHigh.push(f); - else if (f.severity === "MEDIUM") mediumCount++; - } + parts.push(...added.filter((part) => part.text.trim())); + } + + const allHigh: Finding[] = []; + const mediums: string[] = []; + // Visibility doesn't change HIGH behavior; pass private so nothing is treated + // as public-strict (HIGH blocks regardless either way). + // Sliced (see scanAddedLines) so a large-but-legitimate diff is actually + // scanned rather than blocked unscanned on the engine's size cap. + // Scanned per file so a pattern can use the file's type (#3048). + for (const part of parts) { + const opts = { repoVisibility: "private" as const, ...(part.file ? { sourcePath: part.file } : {}) }; + for (const f of scanAddedLines(part.text, opts)) { + if (f.severity === "HIGH") allHigh.push(f); + else if (f.severity === "MEDIUM") mediums.push(mediumLine(f, part)); } } - if (mediumCount > 0) { + if (mediums.length > 0) { process.stderr.write( - `gstack-redact-prepush: ${mediumCount} MEDIUM finding(s) in pushed diff (PII/internal). ` + + `gstack-redact-prepush: ${mediums.length} MEDIUM finding(s) in pushed diff (PII/internal). ` + "Not blocking. Review before this becomes public.\n", ); + for (const line of mediums) process.stderr.write(line); } if (allHigh.length > 0) { diff --git a/test/redact-prepush-emails.test.ts b/test/redact-prepush-emails.test.ts new file mode 100644 index 000000000..519d6cb4d --- /dev/null +++ b/test/redact-prepush-emails.test.ts @@ -0,0 +1,186 @@ +/** + * gstack-redact-prepush: which email addresses the push guard reports, and + * where it says they are (#3060, CEO-7, CEO-23, DX-9, DX-17, ENG-8). + * + * An address that is the pusher's own, or already sits in the destination's + * commit metadata, is not a new leak, so `pii.email` must not report it. Every + * relaxation here has a paired control that still reports: a stranger's + * address, an address only another remote knows, a URL push that cannot vouch + * for tracking refs, and a HIGH secret next to an allowed address. + * + * The fixtures are real repositories with explicit per-commit identities and + * an isolated git config, so the machine's own identity can never leak in. + */ +import { afterEach, describe, expect, test } from "bun:test"; +import { spawnSync } from "node:child_process"; +import * as fs from "node:fs"; +import * as os from "node:os"; +import * as path from "node:path"; + +const PREPUSH = path.resolve(import.meta.dir, "../bin/gstack-redact-prepush"); +const REDACT = path.resolve(import.meta.dir, "../bin/gstack-redact"); +const ZERO = "0".repeat(40); +const AWS_KEY = ["AKIA", "1234567890ABCDEF"].join(""); +const roots: string[] = []; + +afterEach(() => { + for (const root of roots.splice(0)) fs.rmSync(root, { recursive: true, force: true }); +}); + +function baseEnv(root: string): Record { + const env: Record = {}; + for (const [k, v] of Object.entries(process.env)) { + if (v === undefined || /^GIT_(AUTHOR|COMMITTER)_/.test(k) || k.startsWith("GSTACK_REDACT_PREPUSH")) continue; + env[k] = v; + } + const globalConfig = path.join(root, "global.gitconfig"); + if (!fs.existsSync(globalConfig)) fs.writeFileSync(globalConfig, ""); + return { ...env, GIT_CONFIG_GLOBAL: globalConfig, GIT_CONFIG_NOSYSTEM: "1" }; +} + +interface Fixture { root: string; repo: string; origin: string; env: Record } + +function git(fx: Fixture, cwd: string, args: string[], extraEnv: Record = {}): string { + const r = spawnSync("git", args, { cwd, encoding: "utf8", env: { ...fx.env, ...extraEnv }, timeout: 60_000 }); + if (r.status !== 0) throw new Error(`git ${args.join(" ")} failed: ${r.stderr}`); + return (r.stdout ?? "").trim(); +} + +function as(email: string): Record { + return { GIT_AUTHOR_NAME: "Someone", GIT_AUTHOR_EMAIL: email, GIT_COMMITTER_NAME: "Someone", GIT_COMMITTER_EMAIL: email }; +} + +let clock = 1_800_000_000; + +/** + * Commit `file` as `email`. The commit object is written by hand rather than + * by `git commit`, so a git wrapper that exports its own GIT_AUTHOR_* (some CI + * and agent machines do) cannot replace the identity under test. + */ +function commitFile(fx: Fixture, cwd: string, file: string, body: string, email = "me@corp.io"): string { + fs.mkdirSync(path.dirname(path.join(cwd, file)), { recursive: true }); + fs.writeFileSync(path.join(cwd, file), body); + git(fx, cwd, ["add", file]); + const tree = git(fx, cwd, ["write-tree"]); + const parent = spawnSync("git", ["rev-parse", "--verify", "-q", "HEAD"], { cwd, encoding: "utf8", env: fx.env, timeout: 30_000 }).stdout?.trim(); + const stamp = `${clock++} +0000`; + const object = [`tree ${tree}`, ...(parent ? [`parent ${parent}`] : []), `author Someone <${email}> ${stamp}`, `committer Someone <${email}> ${stamp}`, "", `edit ${file}`, ""].join("\n"); + const made = spawnSync("git", ["hash-object", "-t", "commit", "-w", "--stdin"], { cwd, input: object, encoding: "utf8", env: fx.env, timeout: 30_000 }); + if (made.status !== 0) throw new Error(`hash-object failed: ${made.stderr}`); + const sha = made.stdout.trim(); + git(fx, cwd, ["update-ref", "HEAD", sha]); + return sha; +} + +/** A working repo with an origin that already holds one seed commit, and the managed hook installed. */ +function fixture(opts: { selfEmail?: string | null; seed?: boolean } = {}): Fixture { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "prepush-emails-")); + roots.push(root); + const fx: Fixture = { root, repo: path.join(root, "repo"), origin: path.join(root, "origin.git"), env: baseEnv(root) }; + fs.mkdirSync(fx.repo); + git(fx, root, ["init", "--bare", "-q", "-b", "main", fx.origin]); + git(fx, fx.repo, ["init", "-q", "-b", "main"]); + git(fx, fx.repo, ["config", "user.name", "Me"]); + if (opts.selfEmail !== null) git(fx, fx.repo, ["config", "user.email", opts.selfEmail ?? "me@corp.io"]); + git(fx, fx.repo, ["remote", "add", "origin", fx.origin]); + if (opts.seed !== false) { + commitFile(fx, fx.repo, "README.md", "seed\n", "seed@example.com"); + git(fx, fx.repo, ["push", "-q", "origin", "main"]); + } + const install = spawnSync("bun", [REDACT, "install-prepush-hook"], { cwd: fx.repo, encoding: "utf8", env: fx.env, timeout: 30_000 }); + expect(install.status).toBe(0); + return fx; +} + +function push(fx: Fixture, args: string[]): { code: number; stderr: string } { + const r = spawnSync("git", ["push", ...args], { cwd: fx.repo, encoding: "utf8", env: fx.env, timeout: 60_000 }); + return { code: r.status ?? -1, stderr: r.stderr ?? "" }; +} + +/** Run the hook binary directly with git's argv and stdin, for fixtures that stub `git`. */ +function runHook(fx: Fixture, stdin: string, argv: string[], extraEnv: Record = {}): { code: number; stderr: string } { + const r = spawnSync(process.execPath, [PREPUSH, ...argv], { + cwd: fx.repo, input: stdin, encoding: "utf8", env: { ...fx.env, ...extraEnv }, timeout: 60_000, + }); + return { code: r.status ?? -1, stderr: r.stderr ?? "" }; +} + +const mediumLines = (stderr: string): string[] => stderr.split("\n").filter((l) => /^\s+MEDIUM\s/.test(l)); +const limitedLine = /existing-email suppression was limited for this push/; + +/** A producer clone that pushes `branch` with one commit by `email` to `remote`. */ +function publishForeignBranch(fx: Fixture, remote: string, branch: string, email: string): void { + const producer = path.join(fx.root, `producer-${branch}`); + git(fx, fx.root, ["clone", "-q", remote, producer]); + commitFile(fx, producer, `${branch}.txt`, "their work\n", email); + git(fx, producer, ["push", "-q", "origin", `HEAD:refs/heads/${branch}`]); +} + +describe("MEDIUM findings are listed one per line", () => { + test("control: a stranger's address is reported with rule, file, line and fix, never the address", () => { + const fx = fixture(); + commitFile(fx, fx.repo, "notes.md", "line one\nline two\ncontact stranger@corp.io\n"); + const { code, stderr } = push(fx, ["origin", "main"]); + expect(code).toBe(0); + expect(stderr).toMatch(/1 MEDIUM finding/); + const lines = mediumLines(stderr); + expect(lines).toHaveLength(1); + expect(lines[0]).toContain("pii.email"); + expect(lines[0]).toContain("notes.md:3"); + expect(stderr).not.toContain("stranger@corp.io"); + }); + + test("control: a stranger's address in the first push of a repo with no history is reported", () => { + const fx = fixture({ seed: false }); + commitFile(fx, fx.repo, "notes.md", "contact stranger@corp.io\n"); + const { code, stderr } = push(fx, ["origin", "main"]); + expect(code).toBe(0); + const lines = mediumLines(stderr); + expect(lines).toHaveLength(1); + expect(lines[0]).toContain("notes.md:1"); + }); +}); + +describe("MEDIUM lines point at the real file line (ENG-8)", () => { + test("separated hunks in one file report each hunk's own line", () => { + const fx = fixture(); + const body = Array.from({ length: 30 }, (_, i) => `line ${i + 1}`); + commitFile(fx, fx.repo, "notes.md", body.join("\n") + "\n"); + expect(push(fx, ["origin", "main"]).code).toBe(0); + body[4] = "ask first@corp.io"; + body[24] = "ask second@corp.io"; + commitFile(fx, fx.repo, "notes.md", body.join("\n") + "\n"); + const found = mediumLines(push(fx, ["origin", "main"]).stderr); + expect(found).toHaveLength(2); + expect(found[0]).toContain("notes.md:5"); + expect(found[1]).toContain("notes.md:25"); + }); + + test("a finding past a scan chunk boundary reports its file line", () => { + const fx = fixture(); + const filler = Array.from({ length: 20_000 }, (_, i) => `filler line ${i} ${"x".repeat(40)}`); + filler.push("ask late@corp.io"); + commitFile(fx, fx.repo, "big.txt", filler.join("\n") + "\n"); + const found = mediumLines(push(fx, ["origin", "main"]).stderr); + expect(found).toHaveLength(1); + expect(found[0]).toContain("big.txt:20001"); + }); + + test("content removed before the pushed tip names the commit that added it", () => { + const fx = fixture(); + git(fx, fx.repo, ["checkout", "-q", "-b", "feature"]); + commitFile(fx, fx.repo, "notes.md", "start\n"); + expect(push(fx, ["origin", "feature"]).code).toBe(0); + git(fx, fx.repo, ["checkout", "-q", "main"]); + commitFile(fx, fx.repo, "upstream.txt", "landed upstream\n"); + expect(push(fx, ["origin", "main"]).code).toBe(0); + git(fx, fx.repo, ["checkout", "-q", "feature"]); + const added = commitFile(fx, fx.repo, "notes.md", "start\nask gone@corp.io\n"); + commitFile(fx, fx.repo, "notes.md", "start\n"); + git(fx, fx.repo, ["merge", "-q", "--no-edit", "main"], as("me@corp.io")); + const found = mediumLines(push(fx, ["origin", "feature"]).stderr); + expect(found).toHaveLength(1); + expect(found[0]).toContain("notes.md:2"); + expect(found[0]).toContain(added.slice(0, 12)); + }); +});