fix: bound large repository tree memory and cleanup

This commit is contained in:
Thomas Durieux committed 2026-11-02 21:57:33 +00:00
1 parent 19b1d5b546
commit af23ad9eb5
16 files changed
+474 -71

No files matched your search

+15 -5
View File
@@ -1,3 +1,4 @@
import { FILE_LISTING_LIMIT } from "../../core/file-tree";
import { repositoryFailureStatus } from "../../core/repository-failure-status";
import { anonymizePath } from "../../core/anonymize-utils";
import * as express from "express";
@@ -217,21 +218,30 @@ router.get(
{ $match: { repoId, treeGeneration: repo.model.treeGeneration || { $exists: false }, size: { $ne: null } } },
{ $project: { _id: 0, path: 1 } },
{ $group: { _id: "$path", count: { $sum: 1 } } },
{ $limit: FILE_LISTING_LIMIT + 1 },
]).exec();
if (results.length > FILE_LISTING_LIMIT) throw new AnonymousError("file_listing_too_large", { httpStatus: 413 });
const directCounts: Record<string, number> = {};
const directCounts: Record<string, number> = Object.create(null);
for (const r of results) {
directCounts[r._id ?? ""] = r.count;
}
const folderCounts: Record<string, number> = {};
const folderCounts: Record<string, number> = Object.create(null);
let folders = 0;
const addCount = (folder: string, count: number) => {
if (!Object.hasOwn(folderCounts, folder) && ++folders > FILE_LISTING_LIMIT) {
throw new AnonymousError("file_listing_too_large", { httpStatus: 413 });
}
folderCounts[folder] = (folderCounts[folder] || 0) + count;
};
for (const [folder, count] of Object.entries(directCounts)) {
let p = folder;
folderCounts[p] = (folderCounts[p] || 0) + count;
addCount(p, count);
while (p) {
const idx = p.lastIndexOf("/");
p = idx >= 0 ? p.substring(0, idx) : "";
folderCounts[p] = (folderCounts[p] || 0) + count;
addCount(p, count);
if (!p) break;
}
}
@@ -242,7 +252,7 @@ router.get(
"../../core/anonymize-utils"
);
const compiled = compileTerms(terms);
const anonymized: Record<string, number> = {};
const anonymized: Record<string, number> = Object.create(null);
for (const [folder, count] of Object.entries(folderCounts)) {
const anonFolder = anonymizePathCompiled(folder, compiled);
anonymized[anonFolder] = (anonymized[anonFolder] || 0) + count;
+1 -1
View File
@@ -156,7 +156,7 @@ export async function runRepositoryStatusCheck(now = new Date()) {
await expireContent(AnonymizedPullRequestModel, data => new PullRequest(data), now);
const retiredCursor = AnonymizedRepositoryModel.find({ $or: [
{ retiredTreeGenerations: { $exists: true } }, { retiredContentPrefixes: { $exists: true } },
{ retiredTreeGenerations: { $exists: true } }, { retiredPathIndexKeys: { $exists: true } }, { retiredContentPrefixes: { $exists: true } },
{ legacyContentCleanupPending: true, status: RepositoryStatus.READY },
{ "stagedFileTrees.until": { $lte: now } },
] }).cursor();