Merge pull request #862 from tdurieux/fix/anonymizer-large-files

perf: index anonymization terms before scanning large files

scripts/benchmark-anonymizer.js

@@ -0,0 +1,31 @@
+// Run with node scripts/benchmark-anonymizer.js [path-to-anonymize-utils.ts].
+// Fixtures are synthetic; reported timings exclude fixture creation and I/O.
+const path = require("node:path");
+const { performance } = require("node:perf_hooks");
+require("ts-node/register/transpile-only");
+const { ContentAnonimizer } = require(process.argv[2]
+  ? path.resolve(process.argv[2]) : "../src/core/anonymize-utils");
+const terms = ["first.last", "user@example.org", "dept.example.edu", "lab.io", "project.org", "Author X. Name",
+  ...Array.from({ length: 41 }, (_, i) => `Researcher${i}`)];
+const row = "123,456,78.91234,-45.123456\n";
+const cases = [
+  ["2 MB CSV, 47 absent terms", () => row.repeat(70000), terms],
+  ["100 MB CSV, 47 absent terms", () => row.repeat(3600000), terms],
+  ["50 MB CSV, sparse matches", () => `first.last\n${row.repeat(1800000)}\nhttps://host/user@example.org`, terms],
+  ["22 MB text, 1.2 million replacements", () => "Alice 123 first.last,45.678,-23.456\n".repeat(600000), ["Alice", ...terms]],
+];
+for (const [name, fixture, rules] of cases) {
+  const input = fixture();
+  const startCompile = performance.now();
+  const anonymizer = new ContentAnonimizer({ terms: rules });
+  const compileMs = performance.now() - startCompile;
+  const started = performance.now();
+  try {
+    const output = anonymizer.anonymize(input);
+    console.log(JSON.stringify({ name, inputBytes: Buffer.byteLength(input), outputBytes: Buffer.byteLength(output),
+      compileMs: +compileMs.toFixed(1), matchMs: +(performance.now() - started).toFixed(1) }));
+  } catch (error) {
+    console.log(JSON.stringify({ name, inputBytes: Buffer.byteLength(input),
+      compileMs: +compileMs.toFixed(1), matchMs: +(performance.now() - started).toFixed(1), error: error.code || error.message }));
+  }
+}

src/core/anonymize-utils.ts

@@ -6,6 +6,7 @@ import { isBinaryFileSync } from "isbinaryfile";
 import { lookup as lookupMime } from "mime-types";
 
 import config from "../config";
+import { TermPresenceIndex } from "./term-presence";
 import {
   parseTermSpec,
   termVariants,
@@ -211,9 +212,13 @@ const markdownImageRegex =
 interface CompiledTermVariant {
   // RE2 for regular patterns; time-limited native fallback for JS extensions.
   pattern: RE2JS | RegExp;
-  // A boundary-free, fixed-width native search can cheaply rule out literal
+  // A boundary-free native search for fixed-width patterns can rule out
-  // terms before RE2 scans a large file. A hit still uses the usual matcher.
+  // absent terms. Candidate replacement retains RE2's boundary semantics.
   literalPrefilter?: RegExp;
+  asciiPattern?: RegExp;
+  asciiReplacement?: string;
+  asciiBefore?: boolean;
+  asciiAfter?: boolean;
   before: boolean;
   after: boolean;
   mask: string;
@@ -269,11 +274,14 @@ function compileTerms(terms: string[] | undefined): CompiledTermVariant[] {
     if (!useAsRegex || hasCatastrophicBacktracking(term)) {
       term = term.replace(/[-[\]{}()*+?.,\\^$|#]/g, "\\$&");
     }
-    // Only plain literals qualify: expanding their letters produces fixed
+    // Plain text and unquantified dots have fixed width in code points.
-    // character classes, never user-controlled backtracking. Do not apply
+    // Exclude all other regex syntax so the native search cannot backtrack
-    // this shortcut to regex syntax, including escaped literals.
+    // exponentially. This includes dotted names, emails and domains.
-    const isLiteral = !/[\\^$.*+?()[\]{}|]/.test(term);
+    const isFixedWidth = !/[\\^$*+?()[\]{}|]/.test(term);
     for (const variant of termVariants(term)) {
+      // Stripping a combining-mark-only term can produce an empty variant.
+      // Keep zero-width matches on RE2's progress-aware iterator.
+      const fixedWidth = isFixedWidth && variant.pattern.length > 0;
       const bounded = withWordBoundaries(variant.pattern, {
         sniffSource: variant.sniff,
         unicode: variant.unicode,
@@ -286,9 +294,15 @@ function compileTerms(terms: string[] | undefined): CompiledTermVariant[] {
           RE2JS.CASE_INSENSITIVE
         );
         compiled.push({ pattern, before, after, mask,
+          asciiPattern: fixedWidth ? new RegExp(bounded.replace(/\./g, "[^\\n]"), "giu") : undefined,
+          asciiReplacement: fixedWidth ? mask.replace(/\$/g, () => "$$") : undefined,
+          asciiBefore: !variant.unicode && bounded.startsWith("\\b"),
+          asciiAfter: !variant.unicode && bounded.endsWith("\\b"),
           // RE2 uses Unicode case folding even for the non-diacritic pass.
           // Omit boundaries so this search can only rule out absent terms.
-          literalPrefilter: isLiteral ? new RegExp(variant.pattern, "iu") : undefined });
+          literalPrefilter: fixedWidth
+            // RE2's dot excludes only LF; JS dot also excludes CR and U+2028/9.
+            ? new RegExp(variant.pattern.replace(/\./g, "[^\\n]"), "iu") : undefined });
       } catch {
         // Retain JavaScript-only syntax and large repetition counts under
         // the execution deadline; RE2 handles the common case without backtracking.
@@ -303,13 +317,37 @@ function compileTerms(terms: string[] | undefined): CompiledTermVariant[] {
   return compiled;
 }
 
+interface CompiledTermPlan {
+  terms: CompiledTermVariant[];
+  presence: TermPresenceIndex | null;
+}
+
+// Every file in a ZIP uses the same rules. Reuse immutable compiled patterns
+// across transformers, with bounded retention for unrelated repositories.
+const termPlans = new Map<string, CompiledTermPlan>();
+function compiledTermPlan(terms: string[] | undefined): CompiledTermPlan {
+  const key = JSON.stringify([config.ANONYMIZATION_MASK, terms || []]);
+  const cached = termPlans.get(key);
+  if (cached) return cached;
+  const compiled = compileTerms(terms);
+  const plan = {
+    terms: compiled,
+    presence: compiled.length && compiled.length <= 256 && compiled.every(term => term.literalPrefilter)
+      ? new TermPresenceIndex(compiled.map(term => term.literalPrefilter!)) : null,
+  };
+  if (key.length <= 8192 && compiled.length <= 256) {
+    if (termPlans.size >= 64) termPlans.delete(termPlans.keys().next().value!);
+    termPlans.set(key, plan);
+  }
+  return plan;
+}
+
 export class ContentAnonimizer {
   public wasAnonymized = false;
-  // Compiled once per instance and reused for every anonymize() call.
+  // Per-instance metadata references the bounded cache of compiled rules.
-  // Streamed files invoke anonymize() many times per file (one per chunk),
-  // so caching here avoids rebuilding regexes on every chunk.
   private compiledTerms: CompiledTermVariant[];
   private selfLinkRegexes: RegExp[] | null = null;
+  private termPresence: TermPresenceIndex | null = null;
 
   constructor(
     readonly opt: {
@@ -321,7 +359,11 @@ export class ContentAnonimizer {
       repoId?: string;
     }
   ) {
-    this.compiledTerms = compileTerms(opt.terms);
+    const plan = compiledTermPlan(opt.terms);
+    // Keep per-instance metadata independent. Regex matching is synchronous;
+    // native replacement resets lastIndex and RE2 creates a fresh matcher.
+    this.compiledTerms = plan.terms.map(term => ({ ...term }));
+    this.termPresence = plan.presence;
     if (opt.repoName && opt.branchName) {
       const r = escapeRegex(opt.repoName);
       const b = escapeRegex(opt.branchName);
@@ -370,19 +412,36 @@ export class ContentAnonimizer {
   }
 
   private replaceTerms(content: string): string {
-    for (const c of this.compiledTerms) {
+    if (!this.compiledTerms.length) return content;
-      // remove whole url if it contains the term
+    let candidates = this.termPresence?.candidates(content);
+    // On ASCII input both engines have identical fixed-width matching and
+    // word boundaries. Native replacement avoids allocating one JS match
+    // object per CSV value. Non-ASCII input keeps RE2 validation below.
+    let asciiContent = (!candidates || candidates.size > 0) && /^\p{ASCII}*$/u.test(content);
+    for (let index = 0; index < this.compiledTerms.length; index++) {
+      if (candidates && !candidates.has(index)) continue;
+      const c = this.compiledTerms[index];
+      // An absent term cannot occur inside a URL either.
+      if (c.literalPrefilter && !c.literalPrefilter.test(content)) continue;
+      const previous = content;
       content = content.replace(urlRegex, (match) => {
-        if (replaceTerm(match, c) !== match) {
+        if (replaceTerm(match, c, true) !== match) {
           this.wasAnonymized = true;
           return c.mask;
         }
         return match;
       });
-      // remove the term in the text
+      // URL replacement may already have inserted a non-ASCII custom mask.
-      const replaced = replaceTerm(content, c);
+      if (content !== previous) asciiContent = asciiContent && /^\p{ASCII}*$/u.test(c.mask);
+      const replaced = replaceTerm(content, c, asciiContent);
       if (replaced !== content) this.wasAnonymized = true;
       content = replaced;
+      // Masks and changed boundaries can introduce later matches. Re-index
+      // only after a real change, never once per absent term.
+      if (content !== previous) {
+        asciiContent = asciiContent && /^\p{ASCII}*$/u.test(c.mask);
+        candidates = this.termPresence?.candidates(content, index);
+      }
     }
     return content;
   }
@@ -437,38 +496,62 @@ function runWithAnonymizationDeadline(run: () => string): string {
   return anonymizationScript.runInNewContext({ run }, { timeout: 1000 });
 }
 
-function replaceTerm(content: string, term: CompiledTermVariant): string {
+function replaceTerm(content: string, term: CompiledTermVariant, asciiContent = false): string {
-  if (term.literalPrefilter && !term.literalPrefilter.test(content)) return content;
   if (term.pattern instanceof RegExp) {
     return content.replace(term.pattern, () => term.mask);
   }
-  // Fixed-width literal searches run natively. Validate each candidate with
+  if (term.literalPrefilter && asciiContent && term.asciiPattern) {
-  // RE2 on a small window to preserve its case folding and boundary semantics
+    return content.replace(term.asciiPattern, term.asciiReplacement!);
-  // without scanning megabytes of notebook outputs in the JS regex engine.
+  }
-  const candidates = term.literalPrefilter
+  if (term.literalPrefilter) {
-    ? content.matchAll(new RegExp(term.literalPrefilter.source, "giu"))
+    const validated = new Map<string, boolean>();
-    : null;
+    const candidates = new RegExp(term.literalPrefilter.source, "giu");
-  const matcher = candidates ? null : term.pattern.matcher(content);
+    const pieces: string[] = [];
+    let cursor = 0;
+    let match: RegExpExecArray | null;
+    while ((match = candidates.exec(content))) {
+      const candidate = match[0], start = match.index, end = start + candidate.length;
+      let valid = true;
+      if (term.before && /[\p{L}\p{N}_]$/u.test(content.slice(Math.max(0, start - 2), start))) valid = false;
+      if (term.after && /^[\p{L}\p{N}_]/u.test(content.slice(end, end + 2))) valid = false;
+      // On ASCII candidates the fixed-width native and RE2 searches agree.
+      // Check RE2's ASCII word boundaries explicitly: JS /iu\b/ also treats
+      // Kelvin sign and long s as word characters, while RE2 does not.
+      if (valid && /^\p{ASCII}*$/u.test(candidate)) {
+        if (term.asciiBefore && asciiWordAt(content, start - 1) === asciiWordAt(content, start)) valid = false;
+        if (term.asciiAfter && asciiWordAt(content, end - 1) === asciiWordAt(content, end)) valid = false;
+      } else if (valid) {
+        // Keep RE2 authoritative for Unicode case folding. Repeated values in
+        // datasets share small contexts; cap this cache independently of size.
+        const offset = Math.max(0, start - 2);
+        const window = content.slice(offset, end + 2);
+        const key = `${start - offset}:${window}`;
+        const cached = validated.get(key);
+        if (cached === undefined) {
+          const check = (term.pattern as RE2JS).matcher(window);
+          valid = check.find(start - offset) && check.start() === start - offset && check.end() === end - offset;
+          if (validated.size < 256) validated.set(key, valid);
+        } else valid = cached;
+      }
+      if (!valid) {
+        // A boundary-rejected candidate must not hide an overlapping match,
+        // e.g. the second a-a in "xa-a-a". Advance by one Unicode code point.
+        candidates.lastIndex = start + (content.codePointAt(start)! > 0xffff ? 2 : 1);
+        continue;
+      }
+      pieces.push(content.slice(cursor, start), term.mask);
+      cursor = end;
+    }
+    if (!pieces.length) return content;
+    pieces.push(content.slice(cursor));
+    return pieces.join("");
+  }
+
+  const matcher = term.pattern.matcher(content);
   const pieces: string[] = [];
   let cursor = 0;
-  const matches = function* () {
+  while (matcher.find()) {
-    if (candidates) {
+    const start = matcher.start(), end = matcher.end();
-      for (const candidate of candidates) {
-        const start = candidate.index!;
-        const end = start + candidate[0].length;
-        const offset = Math.max(0, start - 2);
-        const check = (term.pattern as RE2JS).matcher(content.slice(offset, end + 2));
-        if (check.find(start - offset) && check.start() === start - offset && check.end() === end - offset) {
-          yield { start, end };
-        }
-      }
-    } else {
-      while (matcher!.find()) yield { start: matcher!.start(), end: matcher!.end() };
-    }
-  };
-  for (const { start, end } of matches()) {
-    // RE2 has no lookahead. Check the generated Unicode word boundaries
-    // outside the engine, without executing any user-supplied native regex.
     if (term.before && /[\p{L}\p{N}_]$/u.test(content.slice(Math.max(0, start - 2), start))) continue;
     if (term.after && /^[\p{L}\p{N}_]/u.test(content.slice(end, end + 2))) continue;
     pieces.push(content.slice(cursor, start), term.mask);
@@ -478,3 +561,8 @@ function replaceTerm(content: string, term: CompiledTermVariant): string {
   pieces.push(content.slice(cursor));
   return pieces.join("");
 }
+
+function asciiWordAt(content: string, index: number): boolean {
+  const code = content.charCodeAt(index);
+  return code >= 48 && code <= 57 || code >= 65 && code <= 90 || code >= 97 && code <= 122 || code === 95;
+}

src/core/term-presence.ts

@@ -0,0 +1,67 @@
+/**
+ * Candidate index for nonempty, capture-free, fixed-width compiler patterns.
+ * False positives are safe; false negatives are not.
+ */
+export class TermPresenceIndex {
+  private groups: { indices: number[]; characters: string[] | null }[] = [];
+
+  constructor(private readonly patterns: RegExp[]) {
+    for (let index = 0; index < patterns.length; index++) {
+      const characters = firstCharacters(patterns[index].source);
+      const group = { indices: [index], characters };
+      // Merge groups whose first characters can overlap under Unicode folding.
+      // Then a combined search can identify one group without hiding another
+      // alternative at the same position (Alice versus AliceSmith, for example).
+      for (let i = this.groups.length - 1; i >= 0; i--) {
+        const other = this.groups[i];
+        if (overlap(group.characters, other.characters)) {
+          group.indices.push(...other.indices);
+          group.characters = group.characters && other.characters
+            ? [...new Set([...group.characters, ...other.characters])] : null;
+          this.groups.splice(i, 1);
+        }
+      }
+      this.groups.push(group);
+    }
+  }
+
+  candidates(content: string, after = -1): Set<number> {
+    const found = new Set<number>();
+    const remaining = this.groups.map(group => group.indices.filter(index => index > after))
+      .filter(indices => indices.length);
+    let offset = 0;
+    while (remaining.length) {
+      // Fixed-width inputs contain no captures or quantifiers. Removing a
+      // discovered group avoids revisiting millions of identical CSV values.
+      const search = new RegExp(remaining.map(indices =>
+        `(${indices.map(index => this.patterns[index].source).join("|")})`).join("|"), "giu");
+      search.lastIndex = offset;
+      const match = search.exec(content);
+      if (!match) break;
+      const selected = match.slice(1).findIndex(value => value !== undefined);
+      for (const index of remaining[selected]) found.add(index);
+      remaining.splice(selected, 1);
+      // Another group's match may start inside this one. Search from the next
+      // code point, not from the end of the match, to retain nested candidates.
+      offset = match.index + (content.codePointAt(match.index)! > 0xffff ? 2 : 1);
+    }
+    return found;
+  }
+}
+
+function firstCharacters(source: string): string[] | null {
+  if (source.startsWith("[")) {
+    const characters = source.slice(1, source.indexOf("]"));
+    // Only the compiler's expanded letter classes have an enumerable prefix.
+    // Dot's [^\n], escapes and any future syntax conservatively join all groups.
+    return /^[\p{L}\p{N}]+$/u.test(characters) ? Array.from(characters) : null;
+  }
+  if (source.startsWith("\\")) return null;
+  return [Array.from(source)[0]];
+}
+
+function overlap(left: string[] | null, right: string[] | null): boolean {
+  if (!left || !right) return true;
+  const pattern = new RegExp(`^(?:${left.map(char => char.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")).join("|")})$`, "iu");
+  return right.some(char => pattern.test(char));
+}

test/anonymizer-large-files.test.js

@@ -0,0 +1,28 @@
+const { expect } = require("chai");
+const { Readable } = require("node:stream");
+require("ts-node/register/transpile-only");
+const { AnonymizeTransformer, ContentAnonimizer } = require("../src/core/anonymize-utils");
+const terms = ["first.last", "user@example.org", "dept.example.edu", "lab.io", "project.org", "Author X. Name",
+  ...Array.from({ length: 41 }, (_, i) => `Researcher${i}`)];
+
+describe("large files with many anonymization terms", function () {
+  this.timeout(10000);
+  it("handles a 50 MB numeric dataset without exhausting the one-second deadline", function () {
+    const input = "123,456,78.91234,-45.123456\n".repeat(1800000);
+    expect(new ContentAnonimizer({ terms }).anonymize(input)).to.equal(input);
+  });
+  it("redacts dotted terms and their URLs around a large payload", async function () {
+    const payload = "123,456,78.91234,-45.123456\n".repeat(200000);
+    const input = Buffer.from(`first.last\n${payload}\nhttps://host/user@example.org/path\nfirstXlast`);
+    const stream = new AnonymizeTransformer({ filePath: "coords.csv", terms });
+    Readable.from([input.subarray(0, 6), input.subarray(6)]).pipe(stream);
+    const chunks = [];
+    for await (const chunk of stream) chunks.push(chunk);
+    expect(Buffer.concat(chunks).toString()).to.equal(`XXXX-1\n${payload}\nXXXX-2\nXXXX-1`);
+  });
+  it("handles many repeated matches while preserving the surrounding values", function () {
+    const input = "Alice,123,45.678,-23.456\n".repeat(100000);
+    expect(new ContentAnonimizer({ terms: ["Alice", ...terms] }).anonymize(input))
+      .to.equal("XXXX-1,123,45.678,-23.456\n".repeat(100000));
+  });
+});

test/term-presence.test.js

@@ -0,0 +1,75 @@
+const { expect } = require("chai");
+require("ts-node/register/transpile-only");
+const { TermPresenceIndex } = require("../src/core/term-presence");
+const { compileTerms, ContentAnonimizer } = require("../src/core/anonymize-utils");
+
+function reference(terms, text) {
+  const anonymizer = new ContentAnonimizer({ terms });
+  anonymizer.termPresence = null;
+  for (const term of anonymizer.compiledTerms) delete term.literalPrefilter;
+  return anonymizer.anonymize(text);
+}
+
+describe("term candidate index", function () {
+  it("retains shared prefixes, nested matches and Unicode-folded alternatives", function () {
+    const patterns = [/Alice/iu, /AliceSmith/iu, /Smith/iu, /[sśŝşšș]/iu, /ſ/iu, /K/iu, /k/iu, /研究/iu];
+    const index = new TermPresenceIndex(patterns);
+    for (const text of ["AliceSmith", "ALICESMITH K ſ 研究", "nothing", "ś", "k", "aAliceSmith"]) {
+      for (let after = -1; after < patterns.length; after++) {
+        const candidates = index.candidates(text, after);
+        patterns.forEach((pattern, i) => {
+          if (i > after && pattern.test(text)) expect(candidates.has(i), `${text}: ${i}`).to.equal(true);
+          if (i <= after) expect(candidates.has(i)).to.equal(false);
+        });
+      }
+    }
+  });
+
+  it("keeps dot's CR and Unicode line-separator matches", function () {
+    for (const text of ["first\rlast", "first\u2028last", "first\u2029last", "first\nlast", "first😀last"]) {
+      expect(new ContentAnonimizer({ terms: ["first.last"] }).anonymize(text)).to.equal(reference(["first.last"], text));
+    }
+  });
+
+  it("retains ordered cascades, URL removal, and matches created across replacement boundaries", function () {
+    const cases = [
+      [["Alice=>Bob", "Bob=>Hidden"], "Alice Bob https://host/Alice"],
+      [["Alice=>", "@Bob=>Hidden"], "@AliceBob"],
+      [["a-a", "Alice"], "xa-a-a Alice https://host/Alice/file"],
+      [["first.last", "firstXlast=>other"], "firstXlast first.last"],
+      [["Alice=>Bob", "Bob"], "https://host/Alice"],
+      [["k", "s"], "K ſ ék k_ xK Kx"],
+      [["Alice=>Bob", "B.b=>Done"], "Alice"],
+      [["k=>Kk"], "https://host/k"],
+      [["Alice=>$&", "Bob=>$$"], "Alice Bob"],
+      [["\u0301"], "abc"],
+      [["a.*b", "Alice"], "axxb Alice"],
+    ];
+    for (const [terms, text] of cases) {
+      expect(new ContentAnonimizer({ terms }).anonymize(text), JSON.stringify([terms, text])).to.equal(reference(terms, text));
+    }
+  });
+
+  it("matches the RE2 path for fixed-width terms and surrounding characters", function () {
+    const terms = ["a-a", "a.a", "k", "s", "k.s", "Alice", "@Alice", "Σ", "研究", "😀", "..", "a.", "a..b"];
+    const edges = ["", "x", "_", "é", "K", "ſ", "😀", "𐐀", "\r", "\n", "\u2028"];
+    for (const term of terms) {
+      const input = edges.flatMap(left => edges.map(right => `${left}${term}${right} ${left}${term.replaceAll(".", "\r")}${right}`)).join(" ");
+      expect(new ContentAnonimizer({ terms: [term] }).anonymize(input), term).to.equal(reference([term], input));
+    }
+  });
+
+  it("does not reuse stale rules after the caller edits its term list", function () {
+    const terms = ["Alice=>first"];
+    expect(new ContentAnonimizer({ terms }).anonymize("Alice")).to.equal("first");
+    terms[0] = "Alice=>second";
+    expect(new ContentAnonimizer({ terms }).anonymize("Alice")).to.equal("second");
+    expect(new ContentAnonimizer({ terms: ["Alice=>first"] }).anonymize("Alice")).to.equal("first");
+  });
+
+  it("does not index quantified, capturing, or escaped user patterns", function () {
+    for (const term of ["a.*b", "a+b", "(Alice)", "a\\.b", "a(?=b)"]) {
+      expect(compileTerms([term]).every(c => !c.literalPrefilter), term).to.equal(true);
+    }
+  });
+});