mirror of
https://github.com/tdurieux/anonymous_github.git
synced 2026-09-12 13:48:58 +02:00
fix: bound regex execution during anonymization
This commit is contained in:
Generated
+15
@@ -38,6 +38,7 @@
|
||||
"passport": "^0.6.0",
|
||||
"passport-github2": "^0.1.12",
|
||||
"rate-limit-redis": "^4.2.0",
|
||||
"re2js": "^2.8.6",
|
||||
"redis": "^4.6.13",
|
||||
"sanitize-html": "^2.17.2",
|
||||
"ts-custom-error": "^3.3.1",
|
||||
@@ -13436,6 +13437,15 @@
|
||||
"node": ">= 0.8"
|
||||
}
|
||||
},
|
||||
"node_modules/re2js": {
|
||||
"version": "2.8.6",
|
||||
"resolved": "https://registry.npmjs.org/re2js/-/re2js-2.8.6.tgz",
|
||||
"integrity": "sha512-xLgQil4kIUCrAzVk9fRSkxkFNwmygLFjVxXrLc65aE1F0+Zsb8rxumFBy4XKyvgMCTL6kilDq3EZ0piE2dP/Dg==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=18.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/readable-stream": {
|
||||
"version": "3.6.2",
|
||||
"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz",
|
||||
@@ -24892,6 +24902,11 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"re2js": {
|
||||
"version": "2.8.6",
|
||||
"resolved": "https://registry.npmjs.org/re2js/-/re2js-2.8.6.tgz",
|
||||
"integrity": "sha512-xLgQil4kIUCrAzVk9fRSkxkFNwmygLFjVxXrLc65aE1F0+Zsb8rxumFBy4XKyvgMCTL6kilDq3EZ0piE2dP/Dg=="
|
||||
},
|
||||
"readable-stream": {
|
||||
"version": "3.6.2",
|
||||
"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz",
|
||||
|
||||
@@ -63,6 +63,7 @@
|
||||
"passport": "^0.6.0",
|
||||
"passport-github2": "^0.1.12",
|
||||
"rate-limit-redis": "^4.2.0",
|
||||
"re2js": "^2.8.6",
|
||||
"redis": "^4.6.13",
|
||||
"sanitize-html": "^2.17.2",
|
||||
"ts-custom-error": "^3.3.1",
|
||||
|
||||
+68
-29
@@ -1,3 +1,5 @@
|
||||
import { RE2JS } from "re2js";
|
||||
import { Script } from "vm";
|
||||
import { basename } from "path";
|
||||
import { Transform, Readable } from "stream";
|
||||
import { isBinaryFileSync } from "isbinaryfile";
|
||||
@@ -207,11 +209,10 @@ const markdownImageRegex =
|
||||
/!\[[^\]]*\]\((?<filename>.*?)(?="|\))(?<optionalpart>".*")?\)/g;
|
||||
|
||||
interface CompiledTermVariant {
|
||||
// Global regex used to replace matches in content (and paths).
|
||||
replaceRegex: RegExp;
|
||||
// Non-global twin used inside the URL callback to test() without
|
||||
// mutating shared lastIndex state.
|
||||
testRegex: RegExp;
|
||||
// RE2 for regular patterns; time-limited native fallback for JS extensions.
|
||||
pattern: RE2JS | RegExp;
|
||||
before: boolean;
|
||||
after: boolean;
|
||||
mask: string;
|
||||
}
|
||||
|
||||
@@ -270,18 +271,23 @@ function compileTerms(terms: string[] | undefined): CompiledTermVariant[] {
|
||||
sniffSource: variant.sniff,
|
||||
unicode: variant.unicode,
|
||||
});
|
||||
const baseFlags = variant.unicode ? "iu" : "i";
|
||||
// A user-supplied regex can be valid without `u` but illegal with it
|
||||
// (e.g. `[\w-\.]` — a range between class shorthands is rejected only
|
||||
// in unicode mode). Skip variants that fail to compile so the other
|
||||
// variant still anonymizes.
|
||||
const before = variant.unicode && bounded.startsWith("(?<![\\p{L}\\p{N}_])");
|
||||
const after = variant.unicode && bounded.endsWith("(?![\\p{L}\\p{N}_])");
|
||||
try {
|
||||
const replaceRegex = new RegExp(bounded, "g" + baseFlags);
|
||||
const testRegex = new RegExp(bounded, baseFlags);
|
||||
compiled.push({ replaceRegex, testRegex, mask });
|
||||
const pattern = RE2JS.compile(
|
||||
variant.unicode ? variant.pattern : bounded,
|
||||
RE2JS.CASE_INSENSITIVE
|
||||
);
|
||||
compiled.push({ pattern, before, after, mask });
|
||||
} catch {
|
||||
continue;
|
||||
// Retain JavaScript-only syntax and large repetition counts under
|
||||
// the execution deadline; RE2 handles the common case without backtracking.
|
||||
try {
|
||||
compiled.push({ pattern: new RegExp(bounded, variant.unicode ? "giu" : "gi"),
|
||||
before: false, after: false, mask });
|
||||
} catch { /* The other variant may still compile. */ }
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
return compiled;
|
||||
@@ -357,27 +363,28 @@ export class ContentAnonimizer {
|
||||
for (const c of this.compiledTerms) {
|
||||
// remove whole url if it contains the term
|
||||
content = content.replace(urlRegex, (match) => {
|
||||
if (c.testRegex.test(match)) {
|
||||
if (replaceTerm(match, c) !== match) {
|
||||
this.wasAnonymized = true;
|
||||
return c.mask;
|
||||
}
|
||||
return match;
|
||||
});
|
||||
// remove the term in the text
|
||||
content = content.replace(c.replaceRegex, () => {
|
||||
this.wasAnonymized = true;
|
||||
return c.mask;
|
||||
});
|
||||
const replaced = replaceTerm(content, c);
|
||||
if (replaced !== content) this.wasAnonymized = true;
|
||||
content = replaced;
|
||||
}
|
||||
return content;
|
||||
}
|
||||
|
||||
anonymize(content: string) {
|
||||
content = this.removeImage(content);
|
||||
content = this.removeLink(content);
|
||||
content = this.replaceGitHubSelfLinks(content);
|
||||
content = this.replaceTerms(content);
|
||||
return content;
|
||||
anonymize(content: string): string {
|
||||
return runWithAnonymizationDeadline(() => {
|
||||
content = this.removeImage(content);
|
||||
content = this.removeLink(content);
|
||||
content = this.replaceGitHubSelfLinks(content);
|
||||
content = this.replaceTerms(content);
|
||||
return content;
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
@@ -397,10 +404,13 @@ export function anonymizePathCompiled(
|
||||
path: string,
|
||||
compiled: CompiledTermVariant[]
|
||||
) {
|
||||
for (const c of compiled) {
|
||||
path = path.replace(c.replaceRegex, c.mask);
|
||||
}
|
||||
return path;
|
||||
const replace = () => {
|
||||
for (const c of compiled) path = replaceTerm(path, c);
|
||||
return path;
|
||||
};
|
||||
return compiled.some((term) => term.pattern instanceof RegExp)
|
||||
? runWithAnonymizationDeadline(replace)
|
||||
: replace();
|
||||
}
|
||||
|
||||
export { compileTerms };
|
||||
@@ -409,3 +419,32 @@ export type { CompiledTermVariant };
|
||||
function escapeRegex(value: string): string {
|
||||
return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
||||
}
|
||||
|
||||
// V8 interrupts even a native RegExp that never returns to JavaScript. A
|
||||
// timeout aborts the operation instead of returning partially anonymized text.
|
||||
const anonymizationScript = new Script("run()");
|
||||
function runWithAnonymizationDeadline(run: () => string): string {
|
||||
return anonymizationScript.runInNewContext({ run }, { timeout: 1000 });
|
||||
}
|
||||
|
||||
function replaceTerm(content: string, term: CompiledTermVariant): string {
|
||||
if (term.pattern instanceof RegExp) {
|
||||
return content.replace(term.pattern, () => term.mask);
|
||||
}
|
||||
const matcher = term.pattern.matcher(content);
|
||||
const pieces: string[] = [];
|
||||
let cursor = 0;
|
||||
while (matcher.find()) {
|
||||
const start = matcher.start();
|
||||
const end = matcher.end();
|
||||
// RE2 has no lookahead. Check the generated Unicode word boundaries
|
||||
// outside the engine, without executing any user-supplied native regex.
|
||||
if (term.before && /[\p{L}\p{N}_]$/u.test(content.slice(Math.max(0, start - 2), start))) continue;
|
||||
if (term.after && /^[\p{L}\p{N}_]/u.test(content.slice(end, end + 2))) continue;
|
||||
pieces.push(content.slice(cursor, start), term.mask);
|
||||
cursor = end;
|
||||
}
|
||||
if (!pieces.length) return content;
|
||||
pieces.push(content.slice(cursor));
|
||||
return pieces.join("");
|
||||
}
|
||||
|
||||
@@ -36,6 +36,14 @@ describe("production regressions", function () {
|
||||
}
|
||||
});
|
||||
|
||||
it("executes adjacent repeated patterns without backtracking", function () {
|
||||
const start = Date.now();
|
||||
const text = "a".repeat(250);
|
||||
expect(new ContentAnonimizer({ terms: ["a+a+a+a+a+a+b"] }).anonymize(text)).to.equal(text);
|
||||
expect(anonymizePath(text, ["a+a+a+a+a+a+b"])).to.equal(text);
|
||||
expect(Date.now() - start).to.be.lessThan(1000);
|
||||
});
|
||||
|
||||
it("enforces the text buffering limit without emitting source bytes", async function () {
|
||||
stub(config, "MAX_FILE_SIZE", 8);
|
||||
const transformer = new AnonymizeTransformer({ filePath: "a.txt", terms: ["Alice"] });
|
||||
@@ -64,4 +72,10 @@ describe("production regressions", function () {
|
||||
expect(headers).not.to.have.property("Content-Length");
|
||||
});
|
||||
|
||||
it("fails closed when a JavaScript-only pattern exceeds its execution deadline", function () {
|
||||
this.timeout(3000);
|
||||
const anonymizer = new ContentAnonimizer({ terms: ["a+a+a+a+a+a+b(?=x)"] });
|
||||
expect(() => anonymizer.anonymize("a".repeat(250))).to.throw(/timed out/);
|
||||
});
|
||||
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user