diff --git a/CHANGELOG.md b/CHANGELOG.md index 41057d8d1..48888b314 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,44 @@ # Changelog +## [1.87.7.0] - 2026-09-22 + +**Pre-push credential checks close four bypass paths.** +**Every target remote resolves against its own base.** + +This release closes four paths where the pre-push credential hook exited 0 on pushed diffs carrying live keys. Pushing to a remote other than origin no longer evaluates diff ranges against origin/main. Unfetched remote tips and boundary-crossing proximity patterns now fail closed or scan with overlap instead of slipping past the hook. + +### The four numbers that matter + +Source: scenarios in `test/redact-prepush-fail-open.sh`, comparing the baseline scanner from v1.87.5.0 with this release. Run `bash test/redact-prepush-fail-open.sh` or `bun test test/redact-prepush-fail-open.test.ts` to execute the full gate. These are deterministic pre-push protocol checks covering range resolution, boundary slicing, and credential detection. + +| Metric | Before | After | Δ | +|---|---:|---:|---:| +| Fail-open pre-push scenarios | 4 | 0 | -4 | +| Target remotes scoped to push target | No | Yes | Scoped | +| Proximity slice overlap | 0 KiB | 16 KiB | +16 KiB | +| Fail-open gate scenarios passing | 22/26 | 26/26 | +4 | + +Pushing to a secondary remote previously produced an empty diff range when local HEAD matched origin, letting secret-bearing commits ship with exit 0. That probe is now scoped directly to the push target. + +### What this means for developers + +Your pre-push hook blocks secrets when pushing to mirrors, staging targets, and non-origin remotes. Large minified bundles are sliced with overlap and scanned, reporting the matched rule instead of a generic size failure. Run `bun test test/redact-prepush-fail-open.test.ts` to verify your pre-push hook configuration. + +### Itemized changes + +#### Fixed + +- **Scoping default branch probes to push remote:** Pushing to a non-origin remote when HEAD matched origin/main previously resolved an empty diff range against origin, exiting 0 and allowing commits with credentials to push unscanned. The default branch probe now targets the destination remote and falls back to ref-listing when unresolvable. +- **Fail closed on absent remote tips:** Well-shaped 40-character hex tips that do not exist in the local object store previously triggered a merge-base guess that could resolve to an empty diff. The hook now treats absent remote tips as unscannable ranges, blocking the push and instructing the user to run git fetch. +- **Slice boundary overlap for proximity rules:** Added lines in large diffs are sliced in 768 KiB chunks. Patterns that require qualifying context within a character window previously missed secrets when the label and value straddled a boundary without overlap. Slices now overlap by 16 KiB. +- **Zero-width character stripping on ingest:** Normalization of zero-width characters now happens before slice budgeting so that raw byte counts match what the detection engine inspects. Invisible padding can no longer push proximity pairs across slice boundaries. +- **Dynamic empty-tree object resolution:** The fallback diff range used a hardcoded SHA-1 empty-tree object id, which does not exist in a SHA-256 repository and hard-blocked every legitimate first push of a new branch there. The id is now obtained from `git hash-object -t tree --stdin`, which is correct under either hash algorithm. +- **Over-budget single line slicing:** Minified files with single lines exceeding the chunk budget are now sliced into overlapping chunks and inspected, identifying the specific credential finding rather than exiting with an uninspected size error. + +#### Added + +- **Pre-push fail-open gate:** Added `test/redact-prepush-fail-open.sh` and `test/redact-prepush-fail-open.test.ts` verifying all 26 pre-push range resolution, fail-closed, and boundary detection invariants. + ## [1.87.5.0] - 2026-09-17 **Tests finish sooner without dropping checks.** diff --git a/VERSION b/VERSION index 133a1bb9d..0e386466d 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.87.5.0 +1.87.7.0 diff --git a/agents-digest/gstack-AGENTS.md b/agents-digest/gstack-AGENTS.md index 79fdcac47..64418a1e4 100644 --- a/agents-digest/gstack-AGENTS.md +++ b/agents-digest/gstack-AGENTS.md @@ -1,4 +1,4 @@ -# gstack digest v1.87.5.0 — regenerate/re-copy after upgrading gstack +# gstack digest v1.87.7.0 — regenerate/re-copy after upgrading gstack Behavioral rules from gstack (https://github.com/garrytan/gstack), compressed for agent hosts without a full skill install. The full skills add workflows, diff --git a/bin/gstack-redact-prepush b/bin/gstack-redact-prepush index 3e678f3a3..786d522a4 100755 --- a/bin/gstack-redact-prepush +++ b/bin/gstack-redact-prepush @@ -33,8 +33,9 @@ import { scan, type Finding } from "../lib/redact-engine"; import { mkdirpSync } from "../lib/fs-utils"; const ZERO = /^0+$/; -// The canonical empty-tree object; diffing against it yields all content as added. -const EMPTY_TREE = "4b825dc642cb6eb9a060e54bf8d69288fbee4904"; +// The canonical empty-tree object under SHA-1; diffing against it yields all +// content as added. Only a FALLBACK — see emptyTree(). +const EMPTY_TREE_SHA1 = "4b825dc642cb6eb9a060e54bf8d69288fbee4904"; /** * Permissive git for legitimately-fallible PROBES (symbolic-ref, rev-parse, @@ -65,12 +66,56 @@ function gitStrict(args: string[]): string { return r.stdout ?? ""; } +/** + * The empty-tree OID for THIS repository's hash algorithm. + * + * The SHA-1 constant does not exist in a SHA-256 repository, so the fallback + * range `..local` makes git error out, gitStrict throws, and + * every legitimate first push of a new branch is hard-blocked with a diff + * error. Asking git to hash the empty tree returns the right OID under either + * algorithm and needs no table of constants. + */ +let _emptyTree: string | undefined; +function emptyTree(): string { + if (_emptyTree === undefined) { + // --stdin with empty input rather than /dev/null: portable to Windows. + const r = spawnSync("git", ["hash-object", "-t", "tree", "--stdin"], { + encoding: "utf8", + input: "", + }); + const oid = r.status === 0 ? (r.stdout ?? "").trim() : ""; + _emptyTree = /^[0-9a-f]{40,64}$/i.test(oid) ? oid : EMPTY_TREE_SHA1; + } + return _emptyTree; +} + /** True when the object exists in the local odb (cat-file -e signals via exit code). */ function objectExists(sha: string): boolean { const r = spawnSync("git", ["cat-file", "-e", sha], { encoding: "utf8" }); return r.status === 0; } +/** + * The remote this push targets, as git hands it to pre-push in $1 (and its URL + * as $2); the installed hook wrapper forwards "$@". Null when the name is + * unavailable (stdin/CLI invocation) or does not name a CONFIGURED remote — a + * URL push has no remote-tracking namespace at all — and every caller then + * falls back to the historical origin-shaped behavior. + */ +let _pushRemote: string | null | undefined; +function pushRemote(): string | null { + if (_pushRemote === undefined) { + const name = process.argv[2]; + const configured = name + ? git(["remote"]).split("\n").map((s) => s.trim()).filter(Boolean).includes(name) + : false; + _pushRemote = configured ? name! : null; + } + // `?? null` only narrows away the not-yet-computed sentinel; by here the + // cache is always set, and null legitimately means "no tracking namespace". + return _pushRemote ?? null; +} + /** * The remote-tracking exclusion used when narrowing to "commits new to the * remote" (#2592 catch-up merges, #2573 rebased force-pushes). @@ -79,33 +124,43 @@ function objectExists(sha: string): boolean { * commits reachable from ANY remote-tracking ref, so a secret that had only * ever been fetched from (or pushed to) a private/local-path remote was never * scanned when later pushed to a PUBLIC remote — "already left this machine" - * is not "already reached THIS remote". Git hands pre-push the push remote's - * name as $1 (and its URL as $2); the installed hook wrapper forwards "$@". - * Fallbacks keep the historical all-remotes behavior when the name is - * unavailable (stdin/CLI invocation) or is not a configured remote (URL - * pushes have no remote-tracking namespace) — falling back scans LESS than - * the narrowed form would, but never less than the hook historically did. + * is not "already reached THIS remote". Falling back to the bare form scans + * LESS than the narrowed form would, but never less than the hook historically + * did. */ -let _remotesExclusion: string | undefined; function remotesExclusion(): string { - if (_remotesExclusion === undefined) { - const name = process.argv[2]; - const configured = name - ? git(["remote"]).split("\n").map((s) => s.trim()).filter(Boolean).includes(name) - : false; - _remotesExclusion = configured ? `--remotes=${name}/*` : "--remotes"; - } - return _remotesExclusion; + const name = pushRemote(); + return name ? `--remotes=${name}/*` : "--remotes"; } -function defaultRemoteBranch(): string { - // origin/HEAD → origin/main, fall back to main/master. - const sym = git(["symbolic-ref", "refs/remotes/origin/HEAD"]).trim(); +/** + * The push target's default branch, or null when it has none we can resolve. + * + * ⚠ SCOPED TO THE PUSH TARGET, NOT HARDCODED TO origin. + * + * remotesExclusion() above already reasons this way for the rev-list exclusion, + * but this probe kept asking origin regardless of where the push was going, and + * the two together are what pick the scan range. The consequence was a + * fail-OPEN, measured: with HEAD == origin/main, `git push publish HEAD:main` + * to a brand-new second remote resolved a merge-base of HEAD against + * origin/main, producing the range HEAD..HEAD — an empty diff — so the entire + * tree, credentials included, shipped to an unknown (possibly public) remote + * with exit 0 and nothing scanned. That is the exact moment a scan matters most. + * + * Returning null when the target remote has no resolvable default branch drops + * the caller through to the rev-list step, which is already scoped to the push + * target and correctly reports "all of this is new" for a never-fetched remote. + * Scanning more is the safe direction here and matches the S1 reasoning above. + */ +function defaultRemoteBranch(): string | null { + const remote = pushRemote() ?? "origin"; + // /HEAD → /main, fall back to main/master. + const sym = git(["symbolic-ref", `refs/remotes/${remote}/HEAD`]).trim(); if (sym) return sym.replace("refs/remotes/", ""); - for (const b of ["origin/main", "origin/master"]) { + for (const b of [`${remote}/main`, `${remote}/master`]) { if (git(["rev-parse", "--verify", b]).trim()) return b; } - return "origin/main"; + return null; } /** @@ -114,8 +169,11 @@ function defaultRemoteBranch(): string { * range, i.e. the whole history really is new content. */ function unknownRemoteTipBase(localSha: string): string | null { - // 1. The common case: a merge-base with the remote's default branch. - const base = git(["merge-base", localSha, defaultRemoteBranch()]).trim(); + // 1. The common case: a merge-base with the remote's default branch. Null + // means the PUSH TARGET has no default branch we can see (never fetched), + // in which case there is nothing legitimate to anchor on — fall through. + const def = defaultRemoteBranch(); + const base = def ? git(["merge-base", localSha, def]).trim() : ""; if (base) return base; // 2. No merge-base. defaultRemoteBranch() guessed a ref that does not exist @@ -235,13 +293,18 @@ function addedLinesFor(localSha: string, remoteSha: string): string { if (fromNew !== null) return collectAddedLines(fromNew); let range: string; + // Set when git NAMED a remote tip we do not have. Any base we derive locally + // is then a guess about content we cannot see — see the empty-result check + // at the end of this function. + let guessedFromUnresolvableTip = false; if (ZERO.test(remoteSha) || !objectExists(remoteSha)) { // Either a new branch (zero remote sha), or the remote tip object is absent // locally (shallow clone, force-push without a prior fetch, CI checkout) so // remote..local cannot resolve. Both need a base derived locally; scan MORE // rather than hard-blocking a legitimate push (adversarial review finding 8). + guessedFromUnresolvableTip = !ZERO.test(remoteSha); const base = unknownRemoteTipBase(localSha); - range = base ? `${base}..${localSha}` : `${EMPTY_TREE}..${localSha}`; + range = base ? `${base}..${localSha}` : `${emptyTree()}..${localSha}`; } else { // Existing branch (incl. force-push): net new content remote..local. range = `${remoteSha}..${localSha}`; @@ -261,7 +324,34 @@ function addedLinesFor(localSha: string, remoteSha: string): string { "diff", "--unified=0", "--no-color", "--no-ext-diff", "--no-textconv", range, ]); - return collectAddedLines(diff); + const added = collectAddedLines(diff); + + // ⚠ A GUESS THAT SCANNED NOTHING IS NOT EVIDENCE OF NOTHING. + // + // git told us the remote is at a sha we do not have, which is itself proof + // that our remote-tracking refs do not describe this ref. Every base above is + // then a guess, and when the guess yields an EMPTY diff we have not concluded + // "this push adds nothing" — we have concluded nothing at all, while git is + // about to send whatever the real difference is. + // + // Measured fail-open: a four-field ref line with a valid local sha and a + // well-shaped but absent remote sha, on a repo where HEAD == origin/main, + // resolved to HEAD..HEAD and allowed a push carrying a live credential with + // exit 0. The shape check in main() rejects junk like "not-a-sha"; it cannot + // reject 40 plausible hex digits. + // + // Narrow by construction: the legitimate cases this path exists for (shallow + // clone, CI checkout, force-push without a prior fetch) all produce a + // NON-empty range and are untouched. Only "we guessed and saw nothing" blocks, + // and it blocks with the fix in the message: fetch the remote. + if (guessedFromUnresolvableTip && !added.trim()) { + throw new Error( + `remote tip ${remoteSha.slice(0, 12)} is not in the local object database and no ` + + "local base narrowed this push, so nothing was scanned — run `git fetch` for this " + + "remote and push again", + ); + } + return added; } /** @@ -294,6 +384,99 @@ function collectAddedLines(diff: string): string { */ const SCAN_CHUNK_BYTES = 768 * 1024; +/** + * Bytes replayed from the end of one slice into the start of the next. + * + * ⚠ WHY SLICING WITHOUT OVERLAP IS NOT SAFE. + * + * The reasoning this replaced said overlap was unnecessary because "every + * pattern is single-line, so a line boundary cannot bisect a detectable + * secret". True of the MATCH, false of the PROXIMITY requirement: five + * patterns in redact-patterns.ts carry a `nearRegex` and only fire when their + * qualifying label appears within `nearWindow` characters — aws.secret_key + * (HIGH) needs `aws_secret_access_key` within 100, gcp.service_account (HIGH) + * needs `"private_key_id"` within 300. Put the label at the end of one slice + * and the secret at the start of the next and the pattern never fires at all. + * + * Measured, not hypothetical: a 40-char AWS secret with its label straddling + * the cut produced ZERO HIGH findings and exit 0 — the guard allowed a push it + * would have blocked had the same bytes sat 41 bytes earlier. + * + * 16 KiB is ~55x the largest nearWindow, so the qualifying context always + * survives the seam with room to spare. + */ +const SCAN_OVERLAP_BYTES = 16 * 1024; + +/** + * Per-line budget. Kept a full overlap below the slice budget so that + * `carried overlap + one unit` can never exceed SCAN_CHUNK_BYTES, which keeps + * every scan() call provably under the engine's 1 MiB cap. + */ +const LINE_BUDGET_BYTES = SCAN_CHUNK_BYTES - SCAN_OVERLAP_BYTES; + +/** + * Zero-width characters, matching redact-engine's ZERO_WIDTH set exactly. + * + * The engine strips these before matching, so they cost bytes here while + * contributing nothing to the text a pattern actually sees. Budgeting slices in + * RAW bytes therefore lets invisible padding decide where the seam falls: + * ~900 KB of U+200B between a label and its secret pushed them into different + * slices while their NORMALIZED distance stayed a couple of characters, and the + * push went out with exit 0. Stripping on ingest makes this file's budget and + * overlap measure the same thing the engine measures. + * + * Finding-preserving by construction — the engine would have removed exactly + * these characters anyway, so no match is created or destroyed. + */ +const ZERO_WIDTH = /[\u200B\u200C\u200D\u2060\uFEFF]/g; + +/** Never cut between the halves of a surrogate pair. */ +function safeCut(s: string, i: number): number { + if (i <= 0 || i >= s.length) return i; + const prev = s.charCodeAt(i - 1); + return prev >= 0xd800 && prev <= 0xdbff ? i - 1 : i; +} + +/** + * Cut one over-budget line into byte-bounded pieces that overlap. + * + * A minified bundle or a single-line JSON blob is one "line" of megabytes. Handed + * to the engine whole it trips input_too_large: the push is blocked, which is + * safe, but blocked WITHOUT the content ever being read — so the operator is + * told a size error where a credential may be sitting, and learns to bypass. + * Slicing with overlap reads it for real; the 2 MiB one-liner that used to + * report `engine.input_too_large` now names the aws.access_key inside it. + */ +function sliceLongLine(line: string, budget: number, overlap: number): string[] { + const pieces: string[] = []; + let start = 0; + while (start < line.length) { + // One char is never fewer than one byte, so `budget` chars is never short + // of the byte budget; shrink proportionally until it fits. + let end = Math.min(line.length, start + budget); + let bytes = Buffer.byteLength(line.slice(start, end), "utf8"); + while (end > start + 1 && bytes > budget) { + const scaled = start + Math.max(1, Math.floor((end - start) * (budget / bytes))); + end = scaled < end ? scaled : end - 1; + bytes = Buffer.byteLength(line.slice(start, end), "utf8"); + } + end = Math.max(start + 1, safeCut(line, end)); + pieces.push(line.slice(start, end)); + if (end >= line.length) break; + // Step back by the overlap so a secret sitting on the cut — and the label + // that qualifies it — land together in the next piece. + let back = 0; + let backBytes = 0; + while (end - back - 1 > start && backBytes < overlap) { + backBytes += Buffer.byteLength(line[end - back - 1]!, "utf8"); + back++; + } + // start strictly increases: the loop guarantees end - back > start. + start = Math.max(start + 1, safeCut(line, end - back)); + } + return pieces; +} + /** * Scan added lines in line-aligned slices, unioning the findings. * @@ -306,15 +489,9 @@ const SCAN_CHUNK_BYTES = 768 * 1024; * people to reach for --no-verify, defeating the guardrail far more thoroughly * than a large diff does. * - * Slicing loses NO detection coverage, because every pattern is single-line: - * none in redact-patterns.ts carries the `m` or `s` flag, the - * BEGIN-PRIVATE-KEY patterns capture only the header line rather than the key - * body, and the engine itself iterates line by line. A line boundary therefore - * cannot bisect a detectable secret, so no inter-slice overlap is needed. - * - * Fail-closed is preserved: a SINGLE line over the budget is still passed to - * the engine intact, so a genuinely unscannable blob (minified bundle, - * embedded base64) trips input_too_large and blocks exactly as before. + * Slices overlap by SCAN_OVERLAP_BYTES and are budgeted in zero-width-stripped + * bytes, so a seam cannot separate a proximity-qualified secret from its label; + * see those two constants for the fail-open each one closes. * * Findings' line/col are slice-relative, which is fine here — this hook only * reads severity, id and preview. Do not lift this into the engine, where @@ -322,26 +499,66 @@ const SCAN_CHUNK_BYTES = 768 * 1024; */ function scanAddedLines(added: string, opts: Parameters[1]): Finding[] { const findings: Finding[] = []; + // Overlap deliberately re-scans the seam, so one occurrence can surface + // twice. Dedup on id+preview: line/col are slice-relative and cannot identify + // an occurrence. Worst case two DIFFERENT occurrences share a preview and are + // counted once — that lowers a count, never a block. + const seen = new Set(); + let slice: string[] = []; let sliceBytes = 0; - const flush = () => { + const flush = (carryOverlap: boolean) => { if (slice.length === 0) return; - findings.push(...scan(slice.join("\n"), opts).findings); - slice = []; - sliceBytes = 0; + for (const f of scan(slice.join("\n"), opts).findings) { + const key = `${f.id}\u0000${f.preview}`; + if (seen.has(key)) continue; + seen.add(key); + findings.push(f); + } + if (!carryOverlap) { + slice = []; + sliceBytes = 0; + return; + } + // Re-seed the next slice with the trailing lines worth up to one overlap. + const tail: string[] = []; + let tailBytes = 0; + for (let i = slice.length - 1; i >= 0; i--) { + const b = Buffer.byteLength(slice[i]!, "utf8") + 1; + if (tailBytes + b > SCAN_OVERLAP_BYTES) break; + tail.unshift(slice[i]!); + tailBytes += b; + } + slice = tail; + sliceBytes = tailBytes; }; - for (const line of added.split("\n")) { - // +1 for the newline that rejoins it. + // unitBytes counts the newline that rejoins the unit. + const push = (unit: string, unitBytes: number) => { + // Close the current slice BEFORE overflowing it, carrying the overlap. + if (sliceBytes > 0 && sliceBytes + unitBytes > SCAN_CHUNK_BYTES) flush(true); + slice.push(unit); + sliceBytes += unitBytes; + }; + + for (const raw of added.split("\n")) { + const line = raw.replace(ZERO_WIDTH, ""); const lineBytes = Buffer.byteLength(line, "utf8") + 1; - // Close the current slice BEFORE overflowing it. A single oversized line - // lands in a slice of its own and is handed to the engine as-is. - if (sliceBytes > 0 && sliceBytes + lineBytes > SCAN_CHUNK_BYTES) flush(); - slice.push(line); - sliceBytes += lineBytes; + if (lineBytes > LINE_BUDGET_BYTES) { + // An over-budget line cannot share a slice with anything else. Close the + // current slice first (carrying its tail so context leading INTO the long + // line survives), then feed the pieces one slice at a time. + flush(true); + for (const piece of sliceLongLine(line, LINE_BUDGET_BYTES - 1, SCAN_OVERLAP_BYTES)) { + push(piece, Buffer.byteLength(piece, "utf8") + 1); + flush(true); + } + continue; + } + push(line, lineBytes); } - flush(); + flush(false); return findings; } diff --git a/test/redact-prepush-fail-open.sh b/test/redact-prepush-fail-open.sh new file mode 100755 index 000000000..f75fae3bf --- /dev/null +++ b/test/redact-prepush-fail-open.sh @@ -0,0 +1,383 @@ +#!/usr/bin/env bash +# +# test/redact-prepush-fail-open.sh — regression gate for pre-push credential +# redaction fail-open paths and adjacent range-resolution invariants. +# +# Usage: +# bash test/redact-prepush-fail-open.sh [scanner-path] +# +# Defaults to the in-tree scanner at bin/gstack-redact-prepush. +# +# Every fixture credential below is synthetic: assembled at runtime from +# halves that match nothing on their own. None is, or ever was, a live secret. +set -uo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +SCANNER="${1:-$REPO_ROOT/bin/gstack-redact-prepush}" + +command -v bun >/dev/null 2>&1 || { echo "bun not found"; exit 2; } +command -v git >/dev/null 2>&1 || { echo "git not found"; exit 2; } +command -v python3 >/dev/null 2>&1 || { echo "python3 not found"; exit 2; } +[ -f "$SCANNER" ] || { echo "no scanner at $SCANNER"; exit 2; } + +_K1='AKIA'; _K2='3QZ7YB2LKWVNPCJD'; KEY="${_K1}${_K2}" +_P1='-----BEGIN '; _P2='RSA PRIVATE KEY'; _P3='-----' +PEM_BEGIN="${_P1}${_P2}${_P3}"; PEM_END="${_P3}END ${_P2}${_P3}" +_S1='wJalrXUtnFEMI/K7MDENG'; _S2='/bPxRfiCYzQ9v2tKzXq'; SEC="${_S1}${_S2}"; export SEC +ZERO40=0000000000000000000000000000000000000000 + +WORK=$(mktemp -d -t prepush-gate-XXXXXX) +trap 'rm -rf "$WORK"' EXIT + +STAGE="$WORK/stage" +mkdir -p "$STAGE/bin" +ln -s "$REPO_ROOT/lib" "$STAGE/lib" +cp "$SCANNER" "$STAGE/bin/cand.ts" +CAND="$STAGE/bin/cand.ts" + +WINDOW=$(grep -oE 'const (WINDOW_BYTES|SCAN_CHUNK_BYTES) = [0-9]+ \* 1024' "$CAND" \ + | grep -oE '[0-9]+ \* 1024' | head -1 | awk '{print $1*1024}') +: "${WINDOW:=786432}" + +R="$WORK/repos" +mkdir -p "$R" + +mkrepo() { + local d="$R/$1" + rm -rf "$d" + mkdir -p "$d" + cd "$d" || exit 1 + git init -q . + git config user.email t@t.t + git config user.name t + git checkout -q -b probe +} + +run_probe() { + cd "$1" || exit 1 + local sha + sha=$(git rev-parse HEAD) + printf 'refs/heads/probe %s refs/heads/probe %s\n' "$sha" "${2:-$ZERO40}" \ + | bun "$CAND" 2>&1 + echo "___EXIT:$?" +} + +verdict() { + local out="$1" rc id + rc=$(printf '%s' "$out" | sed -n 's/^___EXIT:\(.*\)$/\1/p' | tail -1) + [ "$rc" = "0" ] && { echo "ALLOW"; return; } + id=$(printf '%s' "$out" | grep -oE 'HIGH [a-z0-9._]+' | head -1 | awk '{print $2}') + if [ -z "$id" ]; then + id=$(printf '%s' "$out" | grep -oE '^ engine\.[a-z_]+' | head -1 | tr -d ' ') + fi + if [ -n "$id" ]; then + echo "BLOCK($id)" + else + echo "BLOCK(unscannable)" + fi +} + +TOTAL=0 +FAILURES=0 + +row() { + local name="$1" out="$2" want="$3" + TOTAL=$((TOTAL + 1)) + local got + got=$(verdict "$out") + local mark="ok" + if [ "$want" = "BLOCK" ]; then + case "$got" in + BLOCK*) ;; + *) mark="FAIL (want $want)"; FAILURES=$((FAILURES + 1)) ;; + esac + elif [ "$got" != "$want" ]; then + mark="FAIL (want $want)" + FAILURES=$((FAILURES + 1)) + fi + printf '%-48s %-26s %-26s %s\n' "$name" "$want" "$got" "$mark" +} + +printf '\n%-48s %-26s %-26s %s\n' "SCENARIO" "EXPECTED" "ACTUAL" "GATE" +printf -- '-%.0s' {1..110}; printf '\n' + +# 1: 2 MiB innocuous +mkrepo s1; python3 -c " +with open('d.json','w') as f: + f.write('{\n') + for i in range(60000): f.write(' \"prop%07d\": {\"n\": %d},\n' % (i, i*7919%999983)) + f.write(' \"end\": 1\n}\n')" +git add -A >/dev/null; git commit -qm big +S1=$(run_probe "$R/s1") +row "1 clean 2 MiB, no credential" "$S1" "ALLOW" + +# 2: small + AWS key +mkrepo s2; printf 'cfg = 1\naws_key = "%s"\n' "$KEY" > app.py +git add -A >/dev/null; git commit -qm secret +S2=$(run_probe "$R/s2") +row "2 small + AWS key" "$S2" "BLOCK(aws.access_key)" + +# 3: key buried in 3 MiB +mkrepo s3; python3 -c " +k='$KEY' +with open('d.json','w') as f: + f.write('{\n') + for i in range(45000): f.write(' \"prop%07d\": {\"n\": %d},\n' % (i, i*7919%999983)) + f.write(' \"aws_key\": \"%s\",\n' % k) + for i in range(45000): f.write(' \"item%07d\": {\"n\": %d},\n' % (i, i*104729%999983)) + f.write(' \"end\": 1\n}\n')" +git add -A >/dev/null; git commit -qm buried +S3=$(run_probe "$R/s3") +row "3 AWS key buried in 3 MiB" "$S3" "BLOCK(aws.access_key)" + +# 4: PEM straddling the cut +mkrepo s4; PEM_BEGIN="$PEM_BEGIN" PEM_END="$PEM_END" python3 -c " +import base64, os +W=$WINDOW; line='f'*40; n=(W-500)//41 +body='\n'.join(base64.b64encode(bytes((i*37+j)%256 for i in range(48))).decode() for j in range(25)) +with open('d.txt','w') as f: + for i in range(n): f.write(line+'\n') + f.write(os.environ['PEM_BEGIN']+'\n'+body+'\n'+os.environ['PEM_END']+'\n') + for i in range(2000): f.write('t'*40+'\n')" +git add -A >/dev/null; git commit -qm pem +S4=$(run_probe "$R/s4") +row "4 PEM straddling window cut" "$S4" "BLOCK(pem.private_key)" + +# 5: key exactly at the cut +mkrepo s5; python3 -c " +k='$KEY'; W=$WINDOW; line='f'*40; n=(W-60)//41 +with open('d.txt','w') as f: + for i in range(n): f.write(line+'\n') + f.write('aws_key = \"%s\"\n' % k) + for i in range(2000): f.write('t'*40+'\n')" +git add -A >/dev/null; git commit -qm atcut +S5=$(run_probe "$R/s5") +row "5 AWS key exactly at window cut" "$S5" "BLOCK(aws.access_key)" + +# 6: bogus local sha -> range unreadable -> must fail closed +mkrepo s6; echo hi > a.txt; git add -A >/dev/null; git commit -qm init +run_badlocal() { cd "$1" || exit 1 + printf 'refs/heads/probe deadbeefdeadbeefdeadbeefdeadbeefdeadbeef refs/heads/probe 0000000000000000000000000000000000000000\n' \ + | bun "$CAND" 2>&1; echo "___EXIT:$?"; } +S6=$(run_badlocal "$R/s6") +row "6 unreadable range (fail-closed)" "$S6" "BLOCK(unscannable)" + +# 7: credential on a line whose own content starts with "++" +mkrepo s7; printf 'harmless\n++ aws_key = "%s"\n' "$KEY" > notes.patch +git add -A >/dev/null; git commit -qm plusplus +S7=$(run_probe "$R/s7") +row "7 credential on a '++...' line" "$S7" "BLOCK(aws.access_key)" + +# 8: malformed stdin +run_malformed() { cd "$1" || exit 1 + printf 'refs/heads/probe\n' | bun "$CAND" 2>&1; echo "___EXIT:$?"; } +mkrepo s8; echo hi > a.txt; git add -A >/dev/null; git commit -qm init +S8=$(run_malformed "$R/s8") +row "8 malformed stdin (fail-closed)" "$S8" "BLOCK(unscannable)" + +# 9: gcp key spanning the cut +mkrepo s9; PEM_BEGIN="$PEM_BEGIN" python3 -c " +import os +W=$WINDOW; line='f'*40; n=(W-20)//41 +with open('sa.json','w') as f: + for i in range(n): f.write(line+'\n') + f.write('\"private_key\"\n') + f.write(': \"'+os.environ['PEM_BEGIN']+'\n') + for i in range(2000): f.write('t'*40+'\n')" +git add -A >/dev/null; git commit -qm gcpsplit +S9=$(run_probe "$R/s9") +row "9 gcp key spanning the window cut" "$S9" "BLOCK(pem.private_key)" + +# 10: diff.external replaces the diff +mkrepo s10; printf 'cfg = 1\naws_key = "%s"\n' "$KEY" > app.py +git add -A >/dev/null; git commit -qm extdiff +git config diff.external /bin/echo +S10=$(run_probe "$R/s10") +row "10 diff.external set + real key" "$S10" "BLOCK(aws.access_key)" + +# 11: 2 MiB single minified line carrying a credential +mkrepo s11; KEY="$KEY" python3 -c " +import os +k=os.environ['KEY'] +with open('bundle.min.js','w') as f: + f.write('var d={'+','.join('\"k%05d\":%d'%(i,i) for i in range(90000))+',\"aws_key\":\"'+k+'\"};')" +git add -A >/dev/null; git commit -qm minified +S11=$(run_probe "$R/s11") +row "11 credential in a 2 MiB one-liner" "$S11" "BLOCK" + +# 12: push to a remote that is NOT origin +run_remote() { cd "$1" || exit 1; local sha; sha=$(git rev-parse HEAD) + printf 'refs/heads/main %s refs/heads/main 0000000000000000000000000000000000000000\n' "$sha" \ + | bun "$CAND" publish https://example.invalid/publish.git 2>&1; echo "___EXIT:$?"; } +mkrepo s12; git branch -m main 2>/dev/null || true +printf 'cfg = 1\naws_key = "%s"\n' "$KEY" > app.py +git add -A >/dev/null; git commit -qm seed +git update-ref refs/remotes/origin/main HEAD +git remote add origin https://example.invalid/origin.git +git remote add publish https://example.invalid/publish.git +S12=$(run_remote "$R/s12") +row "12 push to a non-origin remote" "$S12" "BLOCK(aws.access_key)" + +# 13: 4 fields, valid local sha, junk REMOTE sha +run_junkremote() { cd "$1" || exit 1; local sha; sha=$(git rev-parse HEAD) + printf 'refs/heads/main %s refs/heads/main not-a-sha\n' "$sha" \ + | bun "$CAND" origin https://example.invalid/origin.git 2>&1; echo "___EXIT:$?"; } +S13=$(run_junkremote "$R/s12") +row "13 junk REMOTE sha" "$S13" "BLOCK(unscannable)" + +# 14: zero-width padding between proximity label and secret +mkrepo s14; SEC="${_S1}${_S2}" python3 -c " +import os +zw='\u200b'*200000 +with open('conf.txt','w') as f: + f.write('aws_secret_access_key =\n') + f.write(zw+'\n') + f.write('\"'+os.environ['SEC']+'\"\n')" +git add -A >/dev/null; git commit -qm zerowidth +S14=$(run_probe "$R/s14") +row "14 zero-width padding at the seam" "$S14" "BLOCK(aws.secret_key)" + +# 15: localSha="0" is not a branch delete +run_shortzero() { cd "$1" || exit 1 + printf 'refs/heads/probe 0 refs/heads/probe 0000000000000000000000000000000000000000\n' \ + | bun "$CAND" origin url 2>&1; echo "___EXIT:$?"; } +S15=$(run_shortzero "$R/s2") +row "15 localSha=\"0\" is not a delete" "$S15" "BLOCK(unscannable)" + +# 16: missing-but-shaped remote sha +run_missingremote() { cd "$1" || exit 1; local sha; sha=$(git rev-parse HEAD) + printf 'refs/heads/main %s refs/heads/main cafebabecafebabecafebabecafebabecafebabe\n' "$sha" \ + | bun "$CAND" origin https://example.invalid/origin.git 2>&1; echo "___EXIT:$?"; } +S16=$(run_missingremote "$R/s12") +row "16 missing-but-shaped remote sha" "$S16" "BLOCK(unscannable)" + +# 17: zero-width across slice boundary +mkrepo s17; SEC="${_S1}${_S2}" python3 -c " +import os +zw='\u200b'*4500 +with open('conf.txt','w') as f: + f.write('aws_secret_access_key =\n') + for i in range(70): f.write(zw+'\n') + f.write('\"'+os.environ['SEC']+'\"\n')" +git add -A >/dev/null; git commit -qm zwslice +S17=$(run_probe "$R/s17") +row "17 zero-width across slice boundary" "$S17" "BLOCK(aws.secret_key)" + +# 18: proximity split by slice cut +mkrepo s18; SEC="${_S1}${_S2}" python3 -c " +import os +W=$WINDOW; line='-'*40 +lab='aws_secret_access_key =' +sec='\"'+os.environ['SEC']+'\"' +n=(W-len(lab)-1)//41 +assert 41*n+len(lab)+1+len(sec)+1 > W, 'fixture does not straddle the cut' +with open('conf.txt','w') as f: + for i in range(n): f.write(line+'\n') + f.write(lab+'\n') + f.write(sec+'\n') + for i in range(100): f.write('-'*40+'\n')" +git add -A >/dev/null; git commit -qm proxsplit +S18=$(run_probe "$R/s18") +row "18 proximity split by slice cut" "$S18" "BLOCK(aws.secret_key)" + +# E1: ordinary new-branch push must scan ONLY the new commit +mkrepo e1 +printf 'aws_key = "%s"\n' "$KEY" > old.py +git add -A >/dev/null; git commit -qm old >/dev/null +git update-ref refs/remotes/origin/main HEAD +git remote add origin https://example.invalid/o.git +git checkout -q -b feature +echo 'harmless = 1' > new.py; git add -A >/dev/null; git commit -qm new >/dev/null +E1=$(cd "$R/e1" && printf 'refs/heads/main %s refs/heads/main %s\n' "$(git rev-parse HEAD)" "$ZERO40" \ + | bun "$CAND" origin https://example.invalid/o.git 2>&1; echo "___EXIT:$?") +row "E1 new branch off origin/main stays narrow" "$E1" "ALLOW" + +# E2: default branch reachable via origin/HEAD -> trunk +mkrepo e2 +printf 'aws_key = "%s"\n' "$KEY" > old.py +git add -A >/dev/null; git commit -qm old >/dev/null +git update-ref refs/remotes/origin/trunk HEAD +git symbolic-ref refs/remotes/origin/HEAD refs/remotes/origin/trunk +git remote add origin https://example.invalid/o.git +git checkout -q -b feature +echo 'harmless = 1' > new.py; git add -A >/dev/null; git commit -qm new >/dev/null +E2=$(cd "$R/e2" && printf 'refs/heads/main %s refs/heads/main %s\n' "$(git rev-parse HEAD)" "$ZERO40" \ + | bun "$CAND" origin https://example.invalid/o.git 2>&1; echo "___EXIT:$?") +row "E2 origin/HEAD -> trunk resolves, stays narrow" "$E2" "ALLOW" + +# E3: fetched second remote anchors on its own default branch +mkrepo e3 +printf 'aws_key = "%s"\n' "$KEY" > old.py +git add -A >/dev/null; git commit -qm old >/dev/null +git update-ref refs/remotes/publish/main HEAD +git remote add origin https://example.invalid/o.git +git remote add publish https://example.invalid/p.git +git checkout -q -b feature +echo 'harmless = 1' > new.py; git add -A >/dev/null; git commit -qm new >/dev/null +E3=$(cd "$R/e3" && printf 'refs/heads/main %s refs/heads/main %s\n' "$(git rev-parse HEAD)" "$ZERO40" \ + | bun "$CAND" publish https://example.invalid/p.git 2>&1; echo "___EXIT:$?") +row "E3 fetched non-origin remote anchors on its own" "$E3" "ALLOW" + +# E4: real branch delete stays a skip +mkrepo e4; echo hi > a.txt; git add -A >/dev/null; git commit -qm init >/dev/null +E4=$(cd "$R/e4" && printf 'refs/heads/main %s refs/heads/main %s\n' "$ZERO40" "$(git rev-parse HEAD)" \ + | bun "$CAND" origin https://example.invalid/o.git 2>&1; echo "___EXIT:$?") +row "E4 branch delete skipped" "$E4" "ALLOW" + +# E5: URL push keeps historical origin-shaped fallback +mkrepo e5 +printf 'aws_key = "%s"\n' "$KEY" > old.py +git add -A >/dev/null; git commit -qm old >/dev/null +git update-ref refs/remotes/origin/main HEAD +git remote add origin https://example.invalid/o.git +git checkout -q -b feature +echo 'harmless = 1' > new.py; git add -A >/dev/null; git commit -qm new >/dev/null +E5=$(cd "$R/e5" && printf 'refs/heads/main %s refs/heads/main %s\n' "$(git rev-parse HEAD)" "$ZERO40" \ + | bun "$CAND" https://example.invalid/direct.git '' 2>&1; echo "___EXIT:$?") +row "E5 URL push falls back to origin (narrow)" "$E5" "ALLOW" + +# E6: long-line slicer survives multi-byte text +mkrepo e6; KEY="$KEY" python3 -c " +import os +k=os.environ['KEY'] +pad='žřáčě\U0001f600' +with open('b.min.js','w') as f: + f.write('var d=\"'+pad*160000+'\",aws_key=\"'+k+'\";')" +git add -A >/dev/null; git commit -qm mb >/dev/null +E6=$(cd "$R/e6" && printf 'refs/heads/main %s refs/heads/main %s\n' "$(git rev-parse HEAD)" "$ZERO40" \ + | bun "$CAND" origin https://example.invalid/o.git 2>&1; echo "___EXIT:$?") +row "E6 long-line slicer survives multi-byte text" "$E6" "BLOCK" + +# E7: clean repo, ordinary existing-branch push +mkrepo e7; echo 'a = 1' > a.py; git add -A >/dev/null; git commit -qm one >/dev/null +BASE=$(git rev-parse HEAD) +git update-ref refs/remotes/origin/main "$BASE" +git remote add origin https://example.invalid/o.git +echo 'b = 2' >> a.py; git add -A >/dev/null; git commit -qm two >/dev/null +E7=$(cd "$R/e7" && printf 'refs/heads/main %s refs/heads/main %s\n' "$(git rev-parse HEAD)" "$BASE" \ + | bun "$CAND" origin https://example.invalid/o.git 2>&1; echo "___EXIT:$?") +row "E7 ordinary push of clean content" "$E7" "ALLOW" + +# E8: sha256 repo, new branch +if git init -q --object-format=sha256 "$R/.probe256" 2>/dev/null; then + rm -rf "$R/.probe256" + d="$R/e8"; rm -rf "$d"; mkdir -p "$d"; cd "$d" || exit 1 + git init -q --object-format=sha256 . + git config user.email t@t.t; git config user.name t; git checkout -q -b main + printf 'cfg = 1\naws_key = "%s"\n' "$KEY" > app.py + git add -A >/dev/null; git commit -qm seed >/dev/null + git remote add origin https://example.invalid/o.git + ZERO64=$(printf '0%.0s' {1..64}) + E8=$(cd "$d" && printf 'refs/heads/main %s refs/heads/main %s\n' "$(git rev-parse HEAD)" "$ZERO64" \ + | bun "$CAND" origin https://example.invalid/o.git 2>&1; echo "___EXIT:$?") + row "E8 sha256 repo, new branch" "$E8" "BLOCK" +fi + +echo +if [ "$FAILURES" -eq 0 ]; then + echo "GATE: PASS ($TOTAL/$TOTAL)" + exit 0 +else + echo "GATE: FAIL" + exit 1 +fi diff --git a/test/redact-prepush-fail-open.test.ts b/test/redact-prepush-fail-open.test.ts new file mode 100644 index 000000000..e9a2b3f7a --- /dev/null +++ b/test/redact-prepush-fail-open.test.ts @@ -0,0 +1,27 @@ +import { describe, test, expect } from "bun:test"; +import { spawnSync } from "child_process"; +import * as path from "path"; + +const REPO_ROOT = path.resolve(import.meta.dir, ".."); +const GATE_SCRIPT = path.join(REPO_ROOT, "test", "redact-prepush-fail-open.sh"); + +describe("pre-push fail-open regression gate", () => { + test.skipIf(!Bun.which("git") || !Bun.which("bun") || !Bun.which("python3"))( + "executes full prepush fail-open gate cleanly (exit 0)", + () => { + const r = spawnSync("bash", [GATE_SCRIPT], { + cwd: REPO_ROOT, + encoding: "utf8", + timeout: 180_000, + env: { ...process.env }, + }); + if (r.status !== 0) { + console.error("Gate stdout:\n", r.stdout); + console.error("Gate stderr:\n", r.stderr); + } + expect(r.status).toBe(0); + expect(r.stdout).toContain("GATE: PASS"); + }, + 200_000, + ); +});