train T3: merge #3578 at bb5b6c665d

CHANGELOG.md

@@ -2,6 +2,17 @@
 
 ## [Unreleased]
 
+**The PII scanner stops flagging two kinds of ordinary text: a bare power-of-two or hash constant in code, and a package@version-style filename.** Eval artifacts and PR bodies that print a PRNG range or name a versioned results file no longer raise a MEDIUM phone or email finding, and real phone numbers and addresses next to them still report.
+
+### Fixed
+- **`lib/redact-patterns.ts`**: `pii.phone.e164` skips a bare number that is exactly a power of two from 2^30 through 2^49, one less than one, or one of eleven well-known 32-bit hash and PRNG constants. The same digits written with a `+`, spaces, dashes, dots or parentheses still report (#3577).
+- **`lib/redact-patterns.ts`**: `pii.email` skips a filename whose part after the `@` is a version or tag followed by file extensions no top-level domain uses, or a dotted version followed by an extension that doubles as one. A file named after a real address, an address at a numeric domain such as 163.com, and an address under a top-level domain that doubles as an extension (`.sh`, `.md`) still report (#3576).
+
+### Tests
+- `test/redact-constant-and-at-filename.test.ts`: pins both exemptions in both directions, including files named after a real address.
+
+## [Unreleased]
+
 **An install check run from inside /ship no longer upgrades gstack under it.** /ship's docs step runs `scripts/install.sh --for document-release`. That run fast-forwarded the checkout from 1.91.298 to 1.91.299 in the middle of a lane's /ship, so the receipt writer's rules changed partway through the run (#3571).
 
 ### Fixed

docs/troubleshooting.md

@@ -2396,6 +2396,14 @@ and an E.164 `+14155551234` still report, so the finding means the id stands
 without a run word (`the id is 40269992077`, `40269992077 failed`) or follows
 a word that is not one (`see 40323989742`).
 
+Separately, the scanner skips a bare number that is exactly a power of two
+from 2^30 through 2^49, one less than such a power, or one of eleven 32-bit
+hash and PRNG constants (Knuth's multiplicative hash, the golden-ratio
+constant, the glibc and Numerical Recipes LCGs, FNV-1a, MurmurHash2 and 3, the
+MT19937 seeder and the largest 32-bit prime; `isNumericConstant` in
+`lib/redact-patterns.ts`). Only the exact digits with no separator are exempt:
+the same digits with a `+`, spaces, dashes, dots or parentheses still report.
+
 **Fix.** Write the id next to a run word (`run 40269992077`) or as an
 `/actions/runs/` link. For anything else, rerun with `--confirm <token>`
 using the `TOKEN:` line the refusal printed; MEDIUM findings do not block
@@ -2436,6 +2444,15 @@ An address only another remote knows still reports. If the hook also printed
 `existing-email suppression was limited for this push`, it could not read the
 history in 5 seconds, so known authors may be listed too.
 
+A filename with an `@` in the package@version style is not an address and
+never reports: the part after the `@` is a version or tag (one dotless label
+or a dotted version, never a domain name) followed by file extensions no
+top-level domain uses, or a dotted all-digit version followed by
+an extension that doubles as a top-level domain (`looksLikeAtFilename` in
+`lib/redact-patterns.ts`). A file named after a real address, an address at an
+all-digit domain and an address whose top-level domain doubles as a file
+extension still report.
+
 **Fix.** Remove the address, or allow it for this repo:
 `git config --add gstack.redact.allowEmail <address>`. If it is your own
 address, check `git config user.email`. The allowlist only covers `pii.email`;

lib/redact-patterns.ts

@@ -201,6 +201,35 @@ function looksLikeCompactTimestamp(span: string): boolean {
   );
 }
 
+/**
+ * Bare integers that programs print as constants: 2^30 through 2^49 and each
+ * one minus one (`4294967296`, `2147483647`), and the 32-bit multipliers,
+ * offsets and primes of common hashes and PRNGs (Knuth's multiplicative hash,
+ * the golden-ratio constant, the glibc and Numerical Recipes LCGs, FNV-1a,
+ * MurmurHash2 and 3, the MT19937 seeder, the largest 32-bit prime). pii.phone.e164
+ * reads each as a bare 10-15 digit number. Only the exact separator-free value
+ * is exempt: the same digits with a `+`, spaces, dashes or dots still report,
+ * so an E.164 number, which carries its `+`, is never exempt.
+ */
+const NUMERIC_CONSTANTS: ReadonlySet<string> = new Set([
+  ...Array.from({ length: 20 }, (_, i) => 2n ** BigInt(30 + i)).flatMap((p) => [String(p), String(p - 1n)]),
+  "2654435761",
+  "2654435769",
+  "1103515245",
+  "1013904223",
+  "2166136261",
+  "1540483477",
+  "3432918353",
+  "2246822507",
+  "3266489909",
+  "1812433253",
+  "4294967291",
+]);
+
+export function isNumericConstant(span: string): boolean {
+  return NUMERIC_CONSTANTS.has(span);
+}
+
 /** Context window for pairing a normalized parcel ID with its punctuated form. */
 const PARCEL_CONTEXT_CHARS = 400;
 
@@ -446,6 +475,31 @@ export function isDotenvFilename(match: RegExpExecArray): boolean {
   return spanStart > 0 && input[spanStart - 1] === ".";
 }
 
+/** A trailing run of file extensions that are not top-level domains (checked
+ * against the IANA root zone list, 2026-10-28). Extensions that are also TLDs
+ * (`.md`, `.py`, `.sh`, `.rs`, `.so`, `.zip`, `.mov`, `.map`) are left out on
+ * purpose. */
+const NON_TLD_EXTENSIONS = /(?:\.(?:json|jsonc|jsonl|ndjson|ya?ml|toml|lock|txt|csv|tsv|log|ini|cfg|conf|xml|html?|[mc]?js|jsx|tsx?|tgz|gz|tar|bz2|xz|zst|whl|jar|gem|nupkg|deb|rpm|wasm|svg|png|jpe?g|pdf|patch|diff))+$/i;
+
+/**
+ * True when a pii.email span is a filename with an `@` in it rather than an
+ * address: `terminal-bench@2.0.json`, `lodash@4.17.21.tgz`,
+ * `eslint-config@next.yaml`. The package@version convention puts a version or
+ * tag after the `@`, so either sign decides: the "domain" ends in file
+ * extensions no TLD uses and what precedes them cannot be a domain (one dotless
+ * label or a dotted version), or it is a dotted version (two or more all-digit
+ * labels) followed by an extension that doubles as a TLD (`a@1.2.md`). A file
+ * named after an address (`jane@acme.com.json`), `user@163.com`,
+ * `alice@1.2.com` and an address at a TLD that doubles as an extension
+ * (`dev@acme.sh`, `ops@acme.md`) still report.
+ */
+export function looksLikeAtFilename(span: string): boolean {
+  const domain = span.slice(span.lastIndexOf("@") + 1);
+  const stem = domain.replace(NON_TLD_EXTENSIONS, "");
+  if (stem !== domain) return !/\.[A-Za-z]{2,}$/.test(stem);
+  return /^\d+(?:\.\d+)+\.(?:md|py|sh|rs|so|zip|mov|map)$/i.test(domain);
+}
+
 /** Extensions that mark a `<name>.local.<ext>` span as a file. Closed on
  * purpose: `.md` is also Moldova's TLD, but no internal suffix sits in front
  * of a public one, so after `.local` it can only be an extension. */
@@ -955,6 +1009,7 @@ export const PATTERNS: RedactPattern[] = [
     regex: /\b([A-Za-z0-9._%+\-]+@[A-Za-z0-9.\-]+\.[A-Za-z]{2,})\b/,
     autoRedactable: true,
     redactToken: "<REDACTED-EMAIL>",
+    validate: (span) => !looksLikeAtFilename(span),
     // Engine layers the email allowlist (example.com, noreply@, user's own,
     // repo-public authors) on top of this — see redact-engine.ts.
   },
@@ -969,12 +1024,14 @@ export const PATTERNS: RedactPattern[] = [
     // A digit-only UUID's hyphen groups read as national phone formatting, and
     // so do a county tax-map parcel ID (see looksLikeParcelId), vector
     // coordinates (looksLikeDecimalCoordinates) and seed/nonce/timestamp JSON
-    // values (isNumericMetadataValue), and GitHub Actions run and job ids in
-    // their id positions (isCiRunIdentifier).
+    // values (isNumericMetadataValue), GitHub Actions run and job ids in
+    // their id positions (isCiRunIdentifier), and bare powers of two and
+    // hash/PRNG constants (isNumericConstant).
     validate: (span, match) =>
       !insideUuid(match) &&
       span.replace(/\D/g, "").length >= 10 &&
       !looksLikeCompactTimestamp(span) &&
+      !isNumericConstant(span) &&
       !looksLikeParcelId(span, match) &&
       !looksLikeDecimalCoordinates(span) &&
       !isNumericMetadataValue(span, match) &&

test/redact-constant-and-at-filename.test.ts

@@ -0,0 +1,97 @@
+/**
+ * Two MEDIUM false positives from the gstack-evals runs (gstack 1.91.299-301):
+ * pii.phone.e164 on `4294967296` (2^32, a PRNG constant) and pii.email on
+ * `terminal-bench@2.0.json` (a package@version filename). Each guard is
+ * pinned in both directions: the false positive is clean, and the real shape
+ * next to it still reports.
+ */
+import { describe, test, expect } from "bun:test";
+import { scan } from "../lib/redact-engine";
+import { isNumericConstant, looksLikeAtFilename } from "../lib/redact-patterns";
+
+const flags = (id: string, s: string): boolean =>
+  scan(s, { repoVisibility: "private" }).findings.some((f) => f.id === id);
+
+describe("pii.phone.e164 and bare numeric constants", () => {
+  // Value: protects=a bare power of two (or one less) and the common 32-bit hash/PRNG constants raise no phone finding; fails_when=the guard is
+  // removed and `4294967296` reports pii.phone.e164 again; why_new=no constant guard existed; seam=none (scan); no_claim=other constants not listed
+  test("powers of two, their max values and hash/PRNG constants are not phone numbers", () => {
+    for (const s of [
+      "const RANGE = 4294967296;",
+      "mask = 4294967295",
+      "INT32_MAX 2147483647",
+      "x % 2147483648",
+      "1073741824 bytes",
+      "2^49 is 562949953421312",
+      "h *= 2654435761",
+      "seed ^= 2654435769",
+      "a = 1103515245",
+      "basis 2166136261",
+      "mt init 1812433253",
+      "p = 4294967291",
+    ]) {
+      expect([s, flags("pii.phone.e164", s)]).toEqual([s, false]);
+    }
+    expect(isNumericConstant("4294967296")).toBe(true);
+    expect(isNumericConstant("4294967297")).toBe(false);
+  });
+
+  // Value: protects=the same digits written as a phone number (a +, spaces, dashes, parens) and ordinary bare numbers still report; fails_when=the
+  // guard exempts by digits alone or by length, so +14294967296 or 429-496-7296 goes quiet; why_new=pins the narrowness of the new guard;
+  // seam=none (scan); no_claim=international formats beyond the ones listed
+  test("real phone numbers still report, including the constant's digits in phone form", () => {
+    for (const s of [
+      "tel: +14294967296",
+      "call 429-496-7296",
+      "reach +1 429 496 7296",
+      "phone (429) 496-7296",
+      "p 429.496.7296",
+      "call 4294967297 now",
+      "ring +44 20 7946 0958",
+      "office 4155550123",
+    ]) {
+      expect([s, flags("pii.phone.e164", s)]).toEqual([s, true]);
+    }
+  });
+});
+
+describe("pii.email and package@version filenames", () => {
+  // Value: protects=a filename with an @ (package@version, or a name ending in a file extension that is not a TLD) raises no email finding;
+  // fails_when=the guard is removed and terminal-bench@2.0.json reports pii.email again; why_new=no filename guard existed; seam=none (scan);
+  // no_claim=extensions outside the closed list
+  test("package@version and @-named files are not email addresses", () => {
+    for (const s of [
+      "results in terminal-bench@2.0.json",
+      "npm pack wrote lodash@4.17.21.tgz",
+      "config eslint-config@next.yaml",
+      "cache react-dom@18.2.0.tar.gz",
+      "pin left-pad@1.3.0.lock",
+      "notes a@1.2.md",
+    ]) {
+      expect([s, flags("pii.email", s)]).toEqual([s, false]);
+    }
+    expect(looksLikeAtFilename("terminal-bench@2.0.json")).toBe(true);
+  });
+
+  // Value: protects=real addresses still report, including numeric domain labels (163.com, 1.2.com), TLDs that double as file extensions (.sh, .md,
+  // .py, .zip) and a file named after an address; fails_when=the guard treats any digit label, any extension-like TLD or any trailing extension as a
+  // filename; why_new=pins the narrowness of the new guard; seam=none (scan); no_claim=every IANA TLD
+  test("real addresses still report, numeric domains and extension-shaped TLDs included", () => {
+    for (const s of [
+      "mail alex@acme-corp.com",
+      "reach li.wei@163.com",
+      "ops dev@acme.sh",
+      "docs ops@acme.md",
+      "py team@acme.py",
+      "files sales@acme.zip",
+      "contact jo@mail.acme.io",
+      "reach alice@1.2.com",
+      "export jane.doe@acme-corp.com.json",
+      "log at ops-team@acme.io.log",
+      "uploaded jdoe@acme-corp.com.PDF",
+      "archive li.wei@163.com.tar.gz",
+    ]) {
+      expect([s, flags("pii.email", s)]).toEqual([s, true]);
+    }
+  });
+});