CalvinBackup/anonymous_github
1
0
Fork 0
mirror of https://github.com/tdurieux/anonymous_github.git synced 2026-06-29 18:50:00 +02:00
Code Issues Packages Projects Releases Wiki Activity

handle memory issues

Browse Source
This commit is contained in:
tdurieux
2026-05-07 21:01:07 +03:00
parent 369fd8edb2
commit 3eeed23609
6 changed files with 60 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/core/anonymize-utils.ts
+19 -2
View File
@@ -14,10 +14,27 @@ import {
const urlRegex =
/<?\b((https?|ftp|file):\/\/)[-A-Za-z0-9+&@#/%?=~_|!:,.;]+[-A-Za-z0-9+&@#/%=~_|]\b\/?>?/g;
export function streamToString(stream: Readable): Promise<string> {
export function streamToString(
stream: Readable,
maxBytes = 2 * 1024 * 1024
): Promise<string> {
const chunks: Buffer[] = [];
let totalBytes = 0;
return new Promise((resolve, reject) => {
stream.on("data", (chunk) => chunks.push(Buffer.from(chunk)));
stream.on("data", (chunk) => {
const buf = Buffer.from(chunk);
totalBytes += buf.length;
if (totalBytes > maxBytes) {
stream.destroy();
reject(
new Error(
`Stream exceeded ${maxBytes} bytes, refusing to buffer into memory`
)
);
return;
}
chunks.push(buf);
});
stream.on("error", (err) => reject(err));
stream.on("end", () => resolve(Buffer.concat(chunks).toString("utf8")));
});
src/server/index.ts
+13 -1
View File
@@ -93,7 +93,19 @@ export default async function start() {
const app = express();
app.use(express.json());
app.use(compression());
app.use(
compression({
filter: (req, res) => {
// Skip compression for streamed file content — these responses are
// piped from the streamer and can be very large. Compressing them
// forces the middleware to hold per-response zlib buffers that pile
// up under concurrent load and contribute to heap exhaustion.
// Binary files (images, archives) barely compress anyway.
if (req.path.match(/^\/api\/repo\/.+\/file\//)) return false;
return compression.filter(req, res);
},
})
);
app.set("etag", "strong");
// handle session and connection
src/server/routes/webview.ts
+8 -4
View File
@@ -149,10 +149,14 @@ async function webView(req: express.Request, res: express.Response) {
});
}
if (f.extension() == "md") {
const content = await streamToString(await f.anonymizedContent());
const body = sanitizeHtml(marked.marked(content, { headerIds: false, mangle: false }), sanitizeOptions);
const html = `<!DOCTYPE html><html><head><title>Content</title></head><link rel="stylesheet" href="/css/all.min.css" /><body><div class="container p-3 file-content markdown-body">${body}</div></body></html>`;
res.contentType("text/html").send(html);
try {
const content = await streamToString(await f.anonymizedContent());
const body = sanitizeHtml(marked.marked(content, { headerIds: false, mangle: false }), sanitizeOptions);
const html = `<!DOCTYPE html><html><head><title>Content</title></head><link rel="stylesheet" href="/css/all.min.css" /><body><div class="container p-3 file-content markdown-body">${body}</div></body></html>`;
res.contentType("text/html").send(html);
} catch {
f.send(res);
}
} else {
f.send(res);
}
src/streamer/index.ts
+11 -1
View File
@@ -15,7 +15,17 @@ const logger = createLogger("streamer");
const app = express();
app.use(express.json());
app.use(compression());
app.use(
compression({
filter: (req, res) => {
// The streamer serves file blobs that are often binary (images,
// archives) and can be very large. Compressing them holds zlib
// buffers per response that pile up under concurrent load.
if (req.path === "/api" && req.method === "POST") return false;
return compression.filter(req, res);
},
})
);
app.use("/api", router);