diff --git a/src/core/anonymize-utils.ts b/src/core/anonymize-utils.ts index 57452b6..f5088e7 100644 --- a/src/core/anonymize-utils.ts +++ b/src/core/anonymize-utils.ts @@ -531,11 +531,12 @@ export class ContentAnonimizer { const replaced = replaceTerm(content, c, asciiContent); if (replaced !== content) this.wasAnonymized = true; content = replaced; - // Masks and changed boundaries can introduce later matches. Re-index - // only after a real change, never once per absent term. + // Masks and changed boundaries can introduce later matches. Invalidate + // the original-text index and check remaining rules directly; rebuilding + // the index after every change costs more than those presence checks. if (content !== previous) { asciiContent = asciiContent && /^\p{ASCII}*$/u.test(c.mask); - candidates = this.termPresence?.candidates(content, index); + candidates = undefined; } } return content; diff --git a/src/core/term-presence.ts b/src/core/term-presence.ts index 934344d..471e53b 100644 --- a/src/core/term-presence.ts +++ b/src/core/term-presence.ts @@ -4,6 +4,7 @@ */ export class TermPresenceIndex { private groups: { indices: number[]; characters: string[] | null }[] = []; + private searches: { indices: number[]; pattern: RegExp }[]; constructor(private readonly patterns: RegExp[]) { for (let index = 0; index < patterns.length; index++) { @@ -23,27 +24,20 @@ export class TermPresenceIndex { } this.groups.push(group); } + this.searches = this.groups.map(({ indices }) => ({ + indices, + pattern: new RegExp(indices.map(index => `(?:${patterns[index].source})`).join("|"), "iu"), + })); } candidates(content: string, after = -1): Set { const found = new Set(); - const remaining = this.groups.map(group => group.indices.filter(index => index > after)) - .filter(indices => indices.length); - let offset = 0; - while (remaining.length) { - // Fixed-width inputs contain no captures or quantifiers. Removing a - // discovered group avoids revisiting millions of identical CSV values. - const search = new RegExp(remaining.map(indices => - `(${indices.map(index => this.patterns[index].source).join("|")})`).join("|"), "giu"); - search.lastIndex = offset; - const match = search.exec(content); - if (!match) break; - const selected = match.slice(1).findIndex(value => value !== undefined); - for (const index of remaining[selected]) found.add(index); - remaining.splice(selected, 1); - // Another group's match may start inside this one. Search from the next - // code point, not from the end of the match, to retain nested candidates. - offset = match.index + (content.codePointAt(match.index)! > 0xffff ? 2 : 1); + // Each immutable group is tested independently, so overlapping matches + // cannot hide candidates. Non-global regexes retain no search position. + for (const { indices, pattern } of this.searches) { + if (indices.some(index => index > after) && pattern.test(content)) { + for (const index of indices) if (index > after) found.add(index); + } } return found; } diff --git a/test/term-presence.test.js b/test/term-presence.test.js index 63c7544..8e0445b 100644 --- a/test/term-presence.test.js +++ b/test/term-presence.test.js @@ -67,6 +67,24 @@ describe("term candidate index", function () { expect(new ContentAnonimizer({ terms: ["Alice=>first"] }).anonymize("Alice")).to.equal("first"); }); + it("handles many matching rules and late cascades within the normal deadline", function () { + this.timeout(10000); + const names = Array.from({ length: 120 }, (_, i) => `Researcher${i} Laboratory Department`); + const terms = names.map((name, i) => `${name}=>Person${i}`); + terms.push("Person119=>FinalPerson", "@Person0=>Contact"); + const input = names.map(name => `@${name} contributed to this research.\n`).join("").repeat(3); + const expected = names.map((_, i) => `${i === 0 ? "Contact" : i === 119 ? "@FinalPerson" : `@Person${i}`} contributed to this research.\n`).join("").repeat(3); + expect(new ContentAnonimizer({ terms }).anonymize(input)).to.equal(expected); + }); + + it("can reuse candidate groups after an absent search and with different cutoffs", function () { + const index = new TermPresenceIndex([/Alice/iu, /AliceSmith/iu, /Smith/iu, /研究/iu]); + expect([...index.candidates("nothing")]).to.deep.equal([]); + expect(index.candidates("AliceSmith 研究").has(2)).to.equal(true); + expect([...index.candidates("AliceSmith 研究", 2)]).to.deep.equal([3]); + expect(index.candidates("AliceSmith", -1).has(0)).to.equal(true); + }); + it("does not index quantified, capturing, or escaped user patterns", function () { for (const term of ["a.*b", "a+b", "(Alice)", "a\\.b", "a(?=b)"]) { expect(compileTerms([term]).every(c => !c.literalPrefilter), term).to.equal(true);