Merge pull request #917 from tdurieux/fix/anonymization-timeout-retries

perf: skip unmatched regex scans in large anonymized files

src/core/anonymize-utils.ts

@@ -317,27 +317,47 @@ function hasCatastrophicBacktracking(src: string): boolean {
   return false;
 }
 
-// A leading sequence of literal characters and simple letter classes must
-// occur in every match. Stop before unfamiliar syntax, drop a quantified last
-// atom, and reject alternation entirely. This is only an absence check; RE2
-// still decides every replacement and all boundaries.
+// A conservative native absence check for the mandatory leading sequence.
+// Assertions can be omitted (only broadening matches); escaped punctuation and
+// a bounded number of optional atoms are safe to retain. RE2 remains the
+// authority for replacements, boundaries, and syntax outside this subset.
 function requiredPrefix(pattern: string): RegExp | undefined {
   if (pattern.includes("|")) return undefined;
   const atoms: string[] = [];
   let offset = 0;
+  let optional = 0;
+  let mandatory = 0;
   while (offset < pattern.length) {
+    let atom: string;
     const char = String.fromCodePoint(pattern.codePointAt(offset)!);
-    if (char === "[") {
+    if (char === "\\") {
+      const next = pattern[offset + 1];
+      if (next === "b" || next === "B") { offset += 2; continue; }
+      if (!next || !"^$\\.*+?()[]{}|/".includes(next)) break;
+      atom = pattern.slice(offset, offset + 2); offset += 2;
+    } else if (char === "[") {
       const end = pattern.indexOf("]", offset + 1);
-      if (end < 0 || !/^[\p{L}\p{N}]+$/u.test(pattern.slice(offset + 1, end))) break;
-      atoms.push(pattern.slice(offset, end + 1)); offset = end + 1;
+      const body = pattern.slice(offset + 1, end);
+      // Enumerated characters only; a trailing/leading hyphen is literal.
+      if (end < 0 || !/^-?[\p{L}\p{N} _]+-?$/u.test(body)) break;
+      atom = pattern.slice(offset, end + 1); offset = end + 1;
     } else {
       if (/[\\^$.*+?()[\]{}]/.test(char)) break;
-      atoms.push(char); offset += char.length;
+      atom = char; offset += char.length;
+    }
+    const quantifier = pattern[offset];
+    if (quantifier === "*" || quantifier === "{") break;
+    if (quantifier === "?") {
+      // At most 16 optional paths, avoiding backtracking-heavy prefilters.
+      if (++optional > 4) break;
+      atoms.push(`${atom}?`); offset++;
+      if (pattern[offset] === "?") offset++;
+    } else {
+      atoms.push(atom); mandatory++;
+      if (quantifier === "+") break;
     }
   }
-  if (/[?*+{]/.test(pattern.charAt(offset)) && offset < pattern.length) atoms.pop();
-  return atoms.length ? new RegExp(atoms.join(""), "iu") : undefined;
+  return mandatory ? new RegExp(atoms.join(""), "iu") : undefined;
 }
 
 
@@ -647,11 +667,21 @@ function replaceTerm(content: string, term: CompiledTermVariant, asciiContent =
     return pieces.join("");
   }
 
+  // Boolean searches use RE2's DFA without capture bookkeeping. Avoid a
+  // full capture search when this rule has no match in a large document.
+  if (!pattern.test(content)) return content;
   const matcher = pattern.matcher(content);
+  const prefix = term.requiredPrefix && new RegExp(term.requiredPrefix.source, "giu");
   const pieces: string[] = [];
   let cursor = 0;
-  while (matcher.find()) {
+  while (true) {
+    // A required prefix is nonempty and starts at the match position. Skip
+    // unmatched tails using native search without slicing away anchor context.
+    const candidate = prefix?.exec(content);
+    if (prefix && !candidate) break;
+    if (!matcher.find(candidate ? candidate.index : undefined)) break;
     const start = matcher.start(), end = matcher.end();
+    if (prefix) prefix.lastIndex = end;
     if (term.before && /[\p{L}\p{N}_]$/u.test(content.slice(Math.max(0, start - 2), start))) continue;
     if (term.after && /^[\p{L}\p{N}_]/u.test(content.slice(end, end + 2))) continue;
     pieces.push(content.slice(cursor, start), term.mask);

test/anonymize-regex-prefix.test.js

@@ -0,0 +1,52 @@
+const { expect } = require("chai");
+require("ts-node/register/transpile-only");
+const { ContentAnonimizer } = require("../src/core/anonymize-utils");
+
+function unfiltered(terms, text) {
+  const a = new ContentAnonimizer({ terms });
+  a.termPresence = null;
+  for (const term of a.compiledTerms) delete term.requiredPrefix;
+  return a.anonymize(text);
+}
+
+describe("regex required-prefix searches", function () {
+  it("keeps RE2 results for assertions, escaped punctuation, optional atoms and quantifiers", function () {
+    const terms = [String.raw`\b[Aa]uthor\b`, String.raw`\Bauthor\B`, String.raw`author\.example\.org`,
+      "Research[ _-]?Institute", "a?b?c?d?e?required", "a??bc", "a+b", "a*b", "a{0,2}b", "a{2}b",
+      String.raw`\(Author\)`, String.raw`\ba?\b`, "[a-z]Name", "[^x]Name", "a|b", "(a)?b", "^Author", "Author$",
+      String.raw`Author\s+Name`, String.raw`Author\d+`, String.raw`Author\x41`, "Σ?Author", "😀?Author"];
+    const text = "Author author xauthorx author.example.org authorXexampleYorg Research-Institute Research Institute " +
+      "ResearchInstitute required aaaaab ab b aabc abc (Author) aName xName Author Name Author12 AuthorA ΣAuthor 😀Author";
+    for (const term of terms) {
+      const a = new ContentAnonimizer({ terms: [term] });
+      expect(a.anonymize(text), term).to.equal(unfiltered([term], text));
+    }
+  });
+
+  it("preserves cascades, anchors and external boundaries after skipping unmatched prefixes", function () {
+    const cases = [
+      [[String.raw`\bAuthor\b=>Research Institute`, "Research[ _-]?Institute=>Hidden"], "Author"],
+      [["a?bc=>Hidden"], "xbc abc bc abcx abc"],
+      [["a+b=>Hidden"], "xaaab aaab aabx ab"],
+      [[String.raw`\b[Aa]uthor\b=>Hidden`], "éAuthor Authoré Author xAuthor Author"],
+      [[String.raw`\bAuthor\b=>Hidden`], "😀Author\nAuthor😀"],
+      [["a?=>X"], "xyz"],
+      [["a{0}=>X"], "xyz"],
+      [[String.raw`\bAuthor\b=>First`, "^First=>Second", "Second$=>Last"], "Author"],
+    ];
+    for (const [terms, text] of cases) {
+      expect(new ContentAnonimizer({ terms }).anonymize(text), JSON.stringify(terms)).to.equal(unfiltered(terms, text));
+    }
+  });
+
+  it("processes sparse boundary and optional regex rules in a large file within the worker deadline", function () {
+    this.timeout(20000);
+    const body = "const value = [1,2,3,4,5]; // ordinary application data\n".repeat(290000);
+    const input = `Author Research Project\n${body}\nAuthor`;
+    const terms = [String.raw`\b[Aa]uthor\b=>Hidden`, "Research[ _-]?Institute=>Organization",
+      ...Array.from({ length: 24 }, (_, i) => String.raw`\bResearcher${i}\b=>Hidden`),
+      String.raw`research\.example\.org=>Website`];
+    const a = new ContentAnonimizer({ terms });
+    expect(a.anonymize(input, 10000)).to.equal(`Hidden Research Project\n${body}\nHidden`);
+  });
+});