Merge pull request #818 from tdurieux/fix/anonymizer-literal-performance

perf: avoid anonymization timeouts for absent literal terms

src/core/anonymize-utils.ts

@@ -211,6 +211,9 @@ const markdownImageRegex =
 interface CompiledTermVariant {
   // RE2 for regular patterns; time-limited native fallback for JS extensions.
   pattern: RE2JS | RegExp;
+  // A boundary-free, fixed-width native search can cheaply rule out literal
+  // terms before RE2 scans a large file. A hit still uses the usual matcher.
+  literalPrefilter?: RegExp;
   before: boolean;
   after: boolean;
   mask: string;
@@ -266,6 +269,10 @@ function compileTerms(terms: string[] | undefined): CompiledTermVariant[] {
     if (!useAsRegex || hasCatastrophicBacktracking(term)) {
       term = term.replace(/[-[\]{}()*+?.,\\^$|#]/g, "\\$&");
     }
+    // Only plain literals qualify: expanding their letters produces fixed
+    // character classes, never user-controlled backtracking. Do not apply
+    // this shortcut to regex syntax, including escaped literals.
+    const isLiteral = !/[\\^$.*+?()[\]{}|]/.test(term);
     for (const variant of termVariants(term)) {
       const bounded = withWordBoundaries(variant.pattern, {
         sniffSource: variant.sniff,
@@ -278,7 +285,10 @@ function compileTerms(terms: string[] | undefined): CompiledTermVariant[] {
           variant.unicode ? variant.pattern : bounded,
           RE2JS.CASE_INSENSITIVE
         );
-        compiled.push({ pattern, before, after, mask });
+        compiled.push({ pattern, before, after, mask,
+          // RE2 uses Unicode case folding even for the non-diacritic pass.
+          // Omit boundaries so this search can only rule out absent terms.
+          literalPrefilter: isLiteral ? new RegExp(variant.pattern, "iu") : undefined });
       } catch {
         // Retain JavaScript-only syntax and large repetition counts under
         // the execution deadline; RE2 handles the common case without backtracking.
@@ -428,6 +438,7 @@ function runWithAnonymizationDeadline(run: () => string): string {
 }
 
 function replaceTerm(content: string, term: CompiledTermVariant): string {
+  if (term.literalPrefilter && !term.literalPrefilter.test(content)) return content;
   if (term.pattern instanceof RegExp) {
     return content.replace(term.pattern, () => term.mask);
   }

test/anonymize-literal-performance.test.js

@@ -0,0 +1,41 @@
+const { expect } = require("chai");
+require("ts-node/register/transpile-only");
+const { Readable } = require("stream");
+const { ContentAnonimizer, AnonymizeTransformer } = require("../src/core/anonymize-utils");
+
+describe("literal term presence checks", function () {
+  it("serves a large JavaScript file with absent names within the anonymization deadline", async function () {
+    const input = Buffer.from('function render(){return "application content";}\n'.repeat(65000));
+    const transformer = new AnonymizeTransformer({
+      filePath: "main.js",
+      terms: Array.from({ length: 20 }, (_, i) => `Researcher${i}`),
+      image: true,
+      link: true,
+    });
+    const chunks = [];
+    Readable.from([input.subarray(0, 65536), input.subarray(65536)]).pipe(transformer);
+    for await (const chunk of transformer) chunks.push(chunk);
+    expect(Buffer.concat(chunks).equals(input)).to.equal(true);
+  });
+
+  it("preserves RE2 replacements, Unicode case folding, and boundaries", function () {
+    const cases = [
+      { terms: ["Alice"], text: "ALICE Alice Álîce Malice Alice2 éAlice Aliceé" },
+      { terms: ["Davo"], text: "Davó DAVO Davoé éDavo" },
+      { terms: ["Davó"], text: "Davo DAVÓ Davó" },
+      { terms: ["k", "s"], text: "k K K s S ſ éK ſé" },
+      { terms: ["Σ", "Ж", "研究"], text: "Σ σ ς Ж ж 研究" },
+      { terms: ["@Alice", "Alice-Bob"], text: "@Álice @Alice Alice-Bob" },
+      { terms: ["Alice=>Bob", "Bob=>$&"], text: "Alice and Bob" },
+      { terms: ["a.*b", "a(?=b)", "a\\.b"], text: "a.b ab axxb" },
+    ];
+    for (const { terms, text } of cases) {
+      const reference = new ContentAnonimizer({ terms });
+      // Compare against the existing matcher without its optional shortcut.
+      for (const term of reference.compiledTerms) delete term.literalPrefilter;
+      const optimized = new ContentAnonimizer({ terms });
+      expect(optimized.anonymize(text)).to.equal(reference.anonymize(text));
+      expect(optimized.wasAnonymized).to.equal(reference.wasAnonymized);
+    }
+  });
+});