mirror of
https://github.com/tdurieux/anonymous_github.git
synced 2026-09-17 08:05:25 +02:00
fix: bound regex execution during anonymization
This commit is contained in:
Generated
+15
@@ -38,6 +38,7 @@
|
|||||||
"passport": "^0.6.0",
|
"passport": "^0.6.0",
|
||||||
"passport-github2": "^0.1.12",
|
"passport-github2": "^0.1.12",
|
||||||
"rate-limit-redis": "^4.2.0",
|
"rate-limit-redis": "^4.2.0",
|
||||||
|
"re2js": "^2.8.6",
|
||||||
"redis": "^4.6.13",
|
"redis": "^4.6.13",
|
||||||
"sanitize-html": "^2.17.2",
|
"sanitize-html": "^2.17.2",
|
||||||
"ts-custom-error": "^3.3.1",
|
"ts-custom-error": "^3.3.1",
|
||||||
@@ -13436,6 +13437,15 @@
|
|||||||
"node": ">= 0.8"
|
"node": ">= 0.8"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"node_modules/re2js": {
|
||||||
|
"version": "2.8.6",
|
||||||
|
"resolved": "https://registry.npmjs.org/re2js/-/re2js-2.8.6.tgz",
|
||||||
|
"integrity": "sha512-xLgQil4kIUCrAzVk9fRSkxkFNwmygLFjVxXrLc65aE1F0+Zsb8rxumFBy4XKyvgMCTL6kilDq3EZ0piE2dP/Dg==",
|
||||||
|
"license": "MIT",
|
||||||
|
"engines": {
|
||||||
|
"node": ">=18.0.0"
|
||||||
|
}
|
||||||
|
},
|
||||||
"node_modules/readable-stream": {
|
"node_modules/readable-stream": {
|
||||||
"version": "3.6.2",
|
"version": "3.6.2",
|
||||||
"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz",
|
"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz",
|
||||||
@@ -24892,6 +24902,11 @@
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"re2js": {
|
||||||
|
"version": "2.8.6",
|
||||||
|
"resolved": "https://registry.npmjs.org/re2js/-/re2js-2.8.6.tgz",
|
||||||
|
"integrity": "sha512-xLgQil4kIUCrAzVk9fRSkxkFNwmygLFjVxXrLc65aE1F0+Zsb8rxumFBy4XKyvgMCTL6kilDq3EZ0piE2dP/Dg=="
|
||||||
|
},
|
||||||
"readable-stream": {
|
"readable-stream": {
|
||||||
"version": "3.6.2",
|
"version": "3.6.2",
|
||||||
"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz",
|
"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz",
|
||||||
|
|||||||
@@ -63,6 +63,7 @@
|
|||||||
"passport": "^0.6.0",
|
"passport": "^0.6.0",
|
||||||
"passport-github2": "^0.1.12",
|
"passport-github2": "^0.1.12",
|
||||||
"rate-limit-redis": "^4.2.0",
|
"rate-limit-redis": "^4.2.0",
|
||||||
|
"re2js": "^2.8.6",
|
||||||
"redis": "^4.6.13",
|
"redis": "^4.6.13",
|
||||||
"sanitize-html": "^2.17.2",
|
"sanitize-html": "^2.17.2",
|
||||||
"ts-custom-error": "^3.3.1",
|
"ts-custom-error": "^3.3.1",
|
||||||
|
|||||||
+68
-29
@@ -1,3 +1,5 @@
|
|||||||
|
import { RE2JS } from "re2js";
|
||||||
|
import { Script } from "vm";
|
||||||
import { basename } from "path";
|
import { basename } from "path";
|
||||||
import { Transform, Readable } from "stream";
|
import { Transform, Readable } from "stream";
|
||||||
import { isBinaryFileSync } from "isbinaryfile";
|
import { isBinaryFileSync } from "isbinaryfile";
|
||||||
@@ -207,11 +209,10 @@ const markdownImageRegex =
|
|||||||
/!\[[^\]]*\]\((?<filename>.*?)(?="|\))(?<optionalpart>".*")?\)/g;
|
/!\[[^\]]*\]\((?<filename>.*?)(?="|\))(?<optionalpart>".*")?\)/g;
|
||||||
|
|
||||||
interface CompiledTermVariant {
|
interface CompiledTermVariant {
|
||||||
// Global regex used to replace matches in content (and paths).
|
// RE2 for regular patterns; time-limited native fallback for JS extensions.
|
||||||
replaceRegex: RegExp;
|
pattern: RE2JS | RegExp;
|
||||||
// Non-global twin used inside the URL callback to test() without
|
before: boolean;
|
||||||
// mutating shared lastIndex state.
|
after: boolean;
|
||||||
testRegex: RegExp;
|
|
||||||
mask: string;
|
mask: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -270,18 +271,23 @@ function compileTerms(terms: string[] | undefined): CompiledTermVariant[] {
|
|||||||
sniffSource: variant.sniff,
|
sniffSource: variant.sniff,
|
||||||
unicode: variant.unicode,
|
unicode: variant.unicode,
|
||||||
});
|
});
|
||||||
const baseFlags = variant.unicode ? "iu" : "i";
|
const before = variant.unicode && bounded.startsWith("(?<![\\p{L}\\p{N}_])");
|
||||||
// A user-supplied regex can be valid without `u` but illegal with it
|
const after = variant.unicode && bounded.endsWith("(?![\\p{L}\\p{N}_])");
|
||||||
// (e.g. `[\w-\.]` — a range between class shorthands is rejected only
|
|
||||||
// in unicode mode). Skip variants that fail to compile so the other
|
|
||||||
// variant still anonymizes.
|
|
||||||
try {
|
try {
|
||||||
const replaceRegex = new RegExp(bounded, "g" + baseFlags);
|
const pattern = RE2JS.compile(
|
||||||
const testRegex = new RegExp(bounded, baseFlags);
|
variant.unicode ? variant.pattern : bounded,
|
||||||
compiled.push({ replaceRegex, testRegex, mask });
|
RE2JS.CASE_INSENSITIVE
|
||||||
|
);
|
||||||
|
compiled.push({ pattern, before, after, mask });
|
||||||
} catch {
|
} catch {
|
||||||
continue;
|
// Retain JavaScript-only syntax and large repetition counts under
|
||||||
|
// the execution deadline; RE2 handles the common case without backtracking.
|
||||||
|
try {
|
||||||
|
compiled.push({ pattern: new RegExp(bounded, variant.unicode ? "giu" : "gi"),
|
||||||
|
before: false, after: false, mask });
|
||||||
|
} catch { /* The other variant may still compile. */ }
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return compiled;
|
return compiled;
|
||||||
@@ -357,27 +363,28 @@ export class ContentAnonimizer {
|
|||||||
for (const c of this.compiledTerms) {
|
for (const c of this.compiledTerms) {
|
||||||
// remove whole url if it contains the term
|
// remove whole url if it contains the term
|
||||||
content = content.replace(urlRegex, (match) => {
|
content = content.replace(urlRegex, (match) => {
|
||||||
if (c.testRegex.test(match)) {
|
if (replaceTerm(match, c) !== match) {
|
||||||
this.wasAnonymized = true;
|
this.wasAnonymized = true;
|
||||||
return c.mask;
|
return c.mask;
|
||||||
}
|
}
|
||||||
return match;
|
return match;
|
||||||
});
|
});
|
||||||
// remove the term in the text
|
// remove the term in the text
|
||||||
content = content.replace(c.replaceRegex, () => {
|
const replaced = replaceTerm(content, c);
|
||||||
this.wasAnonymized = true;
|
if (replaced !== content) this.wasAnonymized = true;
|
||||||
return c.mask;
|
content = replaced;
|
||||||
});
|
|
||||||
}
|
}
|
||||||
return content;
|
return content;
|
||||||
}
|
}
|
||||||
|
|
||||||
anonymize(content: string) {
|
anonymize(content: string): string {
|
||||||
content = this.removeImage(content);
|
return runWithAnonymizationDeadline(() => {
|
||||||
content = this.removeLink(content);
|
content = this.removeImage(content);
|
||||||
content = this.replaceGitHubSelfLinks(content);
|
content = this.removeLink(content);
|
||||||
content = this.replaceTerms(content);
|
content = this.replaceGitHubSelfLinks(content);
|
||||||
return content;
|
content = this.replaceTerms(content);
|
||||||
|
return content;
|
||||||
|
});
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -397,10 +404,13 @@ export function anonymizePathCompiled(
|
|||||||
path: string,
|
path: string,
|
||||||
compiled: CompiledTermVariant[]
|
compiled: CompiledTermVariant[]
|
||||||
) {
|
) {
|
||||||
for (const c of compiled) {
|
const replace = () => {
|
||||||
path = path.replace(c.replaceRegex, c.mask);
|
for (const c of compiled) path = replaceTerm(path, c);
|
||||||
}
|
return path;
|
||||||
return path;
|
};
|
||||||
|
return compiled.some((term) => term.pattern instanceof RegExp)
|
||||||
|
? runWithAnonymizationDeadline(replace)
|
||||||
|
: replace();
|
||||||
}
|
}
|
||||||
|
|
||||||
export { compileTerms };
|
export { compileTerms };
|
||||||
@@ -409,3 +419,32 @@ export type { CompiledTermVariant };
|
|||||||
function escapeRegex(value: string): string {
|
function escapeRegex(value: string): string {
|
||||||
return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// V8 interrupts even a native RegExp that never returns to JavaScript. A
|
||||||
|
// timeout aborts the operation instead of returning partially anonymized text.
|
||||||
|
const anonymizationScript = new Script("run()");
|
||||||
|
function runWithAnonymizationDeadline(run: () => string): string {
|
||||||
|
return anonymizationScript.runInNewContext({ run }, { timeout: 1000 });
|
||||||
|
}
|
||||||
|
|
||||||
|
function replaceTerm(content: string, term: CompiledTermVariant): string {
|
||||||
|
if (term.pattern instanceof RegExp) {
|
||||||
|
return content.replace(term.pattern, () => term.mask);
|
||||||
|
}
|
||||||
|
const matcher = term.pattern.matcher(content);
|
||||||
|
const pieces: string[] = [];
|
||||||
|
let cursor = 0;
|
||||||
|
while (matcher.find()) {
|
||||||
|
const start = matcher.start();
|
||||||
|
const end = matcher.end();
|
||||||
|
// RE2 has no lookahead. Check the generated Unicode word boundaries
|
||||||
|
// outside the engine, without executing any user-supplied native regex.
|
||||||
|
if (term.before && /[\p{L}\p{N}_]$/u.test(content.slice(Math.max(0, start - 2), start))) continue;
|
||||||
|
if (term.after && /^[\p{L}\p{N}_]/u.test(content.slice(end, end + 2))) continue;
|
||||||
|
pieces.push(content.slice(cursor, start), term.mask);
|
||||||
|
cursor = end;
|
||||||
|
}
|
||||||
|
if (!pieces.length) return content;
|
||||||
|
pieces.push(content.slice(cursor));
|
||||||
|
return pieces.join("");
|
||||||
|
}
|
||||||
|
|||||||
@@ -36,6 +36,14 @@ describe("production regressions", function () {
|
|||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("executes adjacent repeated patterns without backtracking", function () {
|
||||||
|
const start = Date.now();
|
||||||
|
const text = "a".repeat(250);
|
||||||
|
expect(new ContentAnonimizer({ terms: ["a+a+a+a+a+a+b"] }).anonymize(text)).to.equal(text);
|
||||||
|
expect(anonymizePath(text, ["a+a+a+a+a+a+b"])).to.equal(text);
|
||||||
|
expect(Date.now() - start).to.be.lessThan(1000);
|
||||||
|
});
|
||||||
|
|
||||||
it("enforces the text buffering limit without emitting source bytes", async function () {
|
it("enforces the text buffering limit without emitting source bytes", async function () {
|
||||||
stub(config, "MAX_FILE_SIZE", 8);
|
stub(config, "MAX_FILE_SIZE", 8);
|
||||||
const transformer = new AnonymizeTransformer({ filePath: "a.txt", terms: ["Alice"] });
|
const transformer = new AnonymizeTransformer({ filePath: "a.txt", terms: ["Alice"] });
|
||||||
@@ -64,4 +72,10 @@ describe("production regressions", function () {
|
|||||||
expect(headers).not.to.have.property("Content-Length");
|
expect(headers).not.to.have.property("Content-Length");
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("fails closed when a JavaScript-only pattern exceeds its execution deadline", function () {
|
||||||
|
this.timeout(3000);
|
||||||
|
const anonymizer = new ContentAnonimizer({ terms: ["a+a+a+a+a+a+b(?=x)"] });
|
||||||
|
expect(() => anonymizer.anonymize("a".repeat(250))).to.throw(/timed out/);
|
||||||
|
});
|
||||||
|
|
||||||
});
|
});
|
||||||
|
|||||||
Reference in New Issue
Block a user