fix: bound large repository tree memory and cleanup

src/core/Repository.ts

@@ -1,3 +1,5 @@
+import { FILE_TREE_BATCH_SIZE, FILE_LISTING_LIMIT, deleteRowsInBatches } from "./file-tree";
+import { SourceBase } from "./source/Source";
 import { notifyOwnerAccessProblem, withOwnerAccessRecovery } from "./owner-notifications";
 import AnonymizedPathModel from "./model/anonymized-path";
 import storage from "./storage";
@@ -200,7 +202,12 @@ export default class Repository {
     if (pathQuery !== undefined) {
       query.path = pathQuery;
     }
-    return await FileModel.find(query).exec();
+    const files = await FileModel.find(query).select("name path repoId size sha upstream submodule")
+      .limit(FILE_LISTING_LIMIT + 1).lean().exec();
+    if (files.length > FILE_LISTING_LIMIT) {
+      throw new AnonymousError("file_listing_too_large", { httpStatus: 413 });
+    }
+    return files;
   }
 
   async submoduleMounts(): Promise<SubmoduleMount[]> {
@@ -238,18 +245,9 @@ export default class Repository {
       const previousGeneration = this.model.treeGeneration;
       const previousPrefix = contentGenerationPrefix(`${previousGeneration || "legacy"}:${this.model.anonymizeDate?.toISOString() || ""}`);
       const streamed = this.model.source.type !== "Zip";
-      const source = this.source;
-      const readFiles = () => source.getFiles(opt.progress);
-      const files = await (streamed ? withOwnerAccessRecovery(this.owner.id,
-        { kind: "repository", id: String(this.model._id) }, readFiles,
-        { "source.repositoryName": this.model.source.repositoryName, "source.commit": this.model.source.commit,
-          "githubAccess.revision": this.model.githubAccess?.revision ?? { $exists: false } }) : readFiles()).catch(error => {
-        void notifyOwnerAccessProblem(this.owner.id, error, { kind: "repository", id: String(this.model._id) });
-        throw error;
-      });
-      const sourceWithTruncation = source as unknown as { truncatedFolderList?: string[] };
-      const truncatedFolders = Array.isArray(sourceWithTruncation.truncatedFolderList)
-        ? [...sourceWithTruncation.truncatedFolderList] : [];
+      const source: SourceBase & { truncatedFolderList?: string[] } = this.source;
+      let fileCount = 0;
+      let truncatedFolders: string[] = [];
       const generation = randomUUID();
       const buildFilter = { _id: this.model._id,
         treeGeneration: previousGeneration || { $exists: false }, ...this.refreshFilter(),
@@ -271,9 +269,32 @@ export default class Repository {
         }, 30_000);
         heartbeat.unref();
       }
-      files.forEach(f => { f.repoId = this.repoId; f.treeGeneration = generation; if (isConnected) f.treeStaged = true; });
+      const readFiles = async () => {
+        let batch: IFile[] = [];
+        const insertBatch = async () => {
+          if (isConnected && !await AnonymizedRepositoryModel.exists({ ...buildFilter,
+            stagedFileTrees: { $elemMatch: { generation, until: { $gt: new Date() } } },
+          }).exec()) throw new AnonymousError("repository_changed", { httpStatus: 409 });
+          await FileModel.insertMany(batch); batch = [];
+        };
+        const files = source.iterateFiles ? source.iterateFiles(opt.progress) : await source.getFiles(opt.progress);
+        for await (const file of files) {
+          const row = file instanceof FileModel ? file.toObject() : file;
+          batch.push({ ...row, repoId: this.repoId, treeGeneration: generation, ...(isConnected ? { treeStaged: true } : {}) });
+          fileCount++;
+          if (batch.length === FILE_TREE_BATCH_SIZE) await insertBatch();
+        }
+        if (batch.length) await insertBatch();
+        truncatedFolders = Array.isArray(source.truncatedFolderList) ? [...source.truncatedFolderList] : [];
+      };
       try {
-        await FileModel.insertMany(files);
+        await (streamed ? withOwnerAccessRecovery(this.owner.id,
+          { kind: "repository", id: String(this.model._id) }, readFiles,
+          { "source.repositoryName": this.model.source.repositoryName, "source.commit": this.model.source.commit,
+            "githubAccess.revision": this.model.githubAccess?.revision ?? { $exists: false } }) : readFiles()).catch(error => {
+          void notifyOwnerAccessProblem(this.owner.id, error, { kind: "repository", id: String(this.model._id) });
+          throw error;
+        });
         if (isConnected) {
           const activated = await AnonymizedRepositoryModel.updateOne({ _id: this.model._id,
             treeGeneration: previousGeneration || { $exists: false }, ...this.refreshFilter(),
@@ -282,14 +303,15 @@ export default class Repository {
             stagedFileTrees: { $elemMatch: { generation, until: { $gt: new Date() } } },
           }, { $set: { treeGeneration: generation, truncatedFolders, size: { storage: 0, file: 0 },
             ...(streamed && !this.model.contentCacheVersion ? { contentCacheVersion: 1, legacyContentCleanupPending: true } : {}),
-            ...(!files.length ? { emptyTreeGeneration: generation } : {}),
+            ...(!fileCount ? { emptyTreeGeneration: generation } : {}),
           }, $unset: { pathIndexKey: "", pathIndexBuiltAt: "", sizeComputedAt: "",
-            ...(files.length ? { emptyTreeGeneration: "" } : {}),
+            ...(fileCount ? { emptyTreeGeneration: "" } : {}),
           }, $addToSet: { retiredTreeGenerations: previousGeneration || "",
             ...(streamed ? { retiredContentPrefixes: previousPrefix } : {}),
+            ...(this.model.pathIndexKey ? { retiredPathIndexKeys: this.model.pathIndexKey } : {}),
           }, $pull: { stagedFileTrees: { generation } } }).exec();
           if (!activated.matchedCount) {
-            await FileModel.deleteMany({ repoId: this.repoId, treeGeneration: generation }).exec();
+            await deleteRowsInBatches(FileModel.collection, { repoId: this.repoId, treeGeneration: generation });
             throw new AnonymousError("repository_changed", { httpStatus: 409 });
           }
         }
@@ -300,11 +322,11 @@ export default class Repository {
         await FileModel.updateMany({ repoId: this.repoId, treeGeneration: generation, treeStaged: true },
           { $unset: { treeStaged: "" } }).exec().catch(error => logger.warn("staged tree completion deferred", serializeError(error)));
       }
-      this.model.emptyTreeGeneration = files.length ? undefined : generation;
+      this.model.emptyTreeGeneration = fileCount ? undefined : generation;
       this.model.truncatedFolders = truncatedFolders;
       if (streamed) this.model.contentCacheVersion = 1;
       if (isConnected) {
-        await this.cleanupRetiredFileTrees().catch(error => logger.warn("retired tree cleanup deferred", serializeError(error)));
+        await this.cleanupRetiredFileTrees(new Date(), 10).catch(error => logger.warn("retired tree cleanup deferred", serializeError(error)));
       } else {
         await FileModel.deleteMany({ repoId: this.repoId, treeGeneration: previousGeneration || { $exists: false } }).exec();
       }
@@ -327,7 +349,7 @@ export default class Repository {
   }
 
   /** Activation records retirement atomically; maintenance retries interrupted deletion. */
-  async cleanupRetiredFileTrees(now = new Date()) {
+  async cleanupRetiredFileTrees(now = new Date(), maxBatches = Infinity) {
     const stages = await AnonymizedRepositoryModel.findById(this.model._id).select("stagedFileTrees").lean().exec();
     for (const stage of stages?.stagedFileTrees || []) {
       if (stage.until > now) continue;
@@ -337,7 +359,7 @@ export default class Repository {
         $addToSet: { retiredTreeGenerations: stage.generation } }).exec();
     }
     const current = await AnonymizedRepositoryModel.findById(this.model._id)
-      .select("treeGeneration retiredTreeGenerations retiredContentPrefixes legacyContentCleanupPending source.type status anonymizeDate").lean().exec();
+      .select("treeGeneration retiredTreeGenerations retiredContentPrefixes retiredPathIndexKeys pathIndexKey legacyContentCleanupPending source.type status anonymizeDate").lean().exec();
     if (!current) return;
     const activePrefix = contentGenerationPrefix(`${current.treeGeneration || "legacy"}:${current.anonymizeDate?.toISOString() || ""}`);
     const prefixes = (current.retiredContentPrefixes || []).filter(prefix => prefix !== activePrefix);
@@ -355,13 +377,21 @@ export default class Repository {
     }
     const retired = (current.retiredTreeGenerations || []).filter(generation => generation !== (current.treeGeneration || ""));
     if (retired.length) {
-      await FileModel.deleteMany({ repoId: this.repoId, $or: [
+      const complete = await deleteRowsInBatches(FileModel.collection, { repoId: this.repoId, $or: [
         { treeGeneration: { $in: retired.filter(Boolean) } },
         ...(retired.includes("") ? [{ treeGeneration: { $exists: false } }] : []),
-      ] }).exec();
-      await AnonymizedRepositoryModel.updateOne({ _id: this.model._id },
+      ] }, { maxBatches });
+      if (complete) await AnonymizedRepositoryModel.updateOne({ _id: this.model._id },
         { $pull: { retiredTreeGenerations: { $in: retired } } }).exec();
     }
+    const retiredKeys = (current.retiredPathIndexKeys || []).filter(key => key !== current.pathIndexKey);
+    if (retiredKeys.length && await deleteRowsInBatches(AnonymizedPathModel.collection,
+      { repoId: this.repoId, key: { $in: retiredKeys } }, { maxBatches })) {
+      await AnonymizedRepositoryModel.updateOne({ _id: this.model._id },
+        { $pull: { retiredPathIndexKeys: { $in: retiredKeys } } }).exec();
+    }
+    await AnonymizedRepositoryModel.updateOne({ _id: this.model._id, "retiredPathIndexKeys.0": { $exists: false } },
+      { $unset: { retiredPathIndexKeys: "" } }).exec();
     await AnonymizedRepositoryModel.updateOne({ _id: this.model._id, "retiredTreeGenerations.0": { $exists: false } },
       { $unset: { retiredTreeGenerations: "" } }).exec();
     await AnonymizedRepositoryModel.updateOne({ _id: this.model._id, "retiredContentPrefixes.0": { $exists: false } },
@@ -384,7 +414,7 @@ export default class Repository {
     if (current?.treeGeneration === generation && !terminal) {
       await FileModel.updateMany({ repoId, treeGeneration: generation, treeStaged: true }, { $unset: { treeStaged: "" } }).exec();
     } else if (terminal || !current?.stagedFileTrees?.some(stage => stage.generation === generation && stage.until > now)) {
-      await FileModel.deleteMany({ repoId, treeGeneration: generation, treeStaged: true }).exec();
+      await deleteRowsInBatches(FileModel.collection, { repoId, treeGeneration: generation, treeStaged: true });
     }
   }
 
@@ -423,7 +453,7 @@ export default class Repository {
         const current = await AnonymizedRepositoryModel.exists({ _id: this.model._id,
           fileMetadataRevision: metadataRevision || { $exists: false },
         }).exec();
-        if (!current) await AnonymizedPathModel.deleteMany({ repoId: this.repoId, key }).exec();
+        if (!current) await deleteRowsInBatches(AnonymizedPathModel.collection, { repoId: this.repoId, key });
         throw error;
       }
       // TTL retires superseded mappings. An older builder must never delete
@@ -802,11 +832,7 @@ export default class Repository {
     }
     this.model.increment();
     await this.updateStatus(RepositoryStatus.DOWNLOAD);
-    await this.files({
-      force: false,
-      progress,
-      recursive: false,
-    });
+    await this.ensureFileTree({ progress });
     // Previously inserted a dummy {path:"", name:"", size:0} FileModel
     // here for empty repos "to avoid errors" — but that record collides
     // with the special-case in AnonymizedFile.getFileInfo for the empty
@@ -972,11 +998,12 @@ export default class Repository {
       await removeTransformedRepositoryCache(this.repoId);
       await storage.rm(this.repoId);
       if (!await AnonymizedRepositoryModel.exists(lease).exec()) return;
-      await FileModel.deleteMany({ repoId: this.repoId }).exec();
-      await AnonymizedPathModel.deleteMany({ repoId: this.repoId }).exec();
+      const current = async () => !!await AnonymizedRepositoryModel.exists(lease).exec();
+      if (!await deleteRowsInBatches(FileModel.collection, { repoId: this.repoId }, { current })) return;
+      if (!await deleteRowsInBatches(AnonymizedPathModel.collection, { repoId: this.repoId }, { current })) return;
       const result = await AnonymizedRepositoryModel.updateOne(lease, { $set: { status: RepositoryStatus.EXPIRED,
         statusDate: new Date(), isReseted: true, size: { storage: 0, file: 0 } },
-        $unset: { cleanupToken: "", cleanupUntil: "", pathIndexKey: "", pathIndexBuiltAt: "", sizeComputedAt: "", emptyTreeGeneration: "", retiredTreeGenerations: "", retiredContentPrefixes: "", legacyContentCleanupPending: "", stagedFileTrees: "" } }).exec();
+        $unset: { cleanupToken: "", cleanupUntil: "", pathIndexKey: "", pathIndexBuiltAt: "", sizeComputedAt: "", emptyTreeGeneration: "", retiredTreeGenerations: "", retiredPathIndexKeys: "", retiredContentPrefixes: "", legacyContentCleanupPending: "", stagedFileTrees: "" } }).exec();
       if (result.matchedCount) { this.model.status = RepositoryStatus.EXPIRED; this.model.isReseted = true; this.model.size = { storage: 0, file: 0 }; this.model.sizeComputedAt = undefined; this.model.emptyTreeGeneration = undefined; }
     } finally {
       clearInterval(heartbeat);
@@ -1010,15 +1037,22 @@ export default class Repository {
   async resetSate(status?: RepositoryStatus, statusMessage?: string) {
     this.assertNotArchived();
     const revision = randomUUID();
+    const defer = isConnected && (status || this.status) === RepositoryStatus.PREPARING;
+    const previousGeneration = this.model.treeGeneration;
+    const generation = defer ? randomUUID() : previousGeneration;
     this.model.emptyTreeGeneration = undefined;
     if (isConnected) {
       const result = await AnonymizedRepositoryModel.updateOne({ _id: this.model._id,
       treeGeneration: this.model.treeGeneration || { $exists: false },
-      }, { $set: { fileMetadataRevision: revision },
+      }, { $set: { fileMetadataRevision: revision, ...(defer ? { treeGeneration: generation } : {}) },
+        ...(defer ? { $addToSet: { retiredTreeGenerations: previousGeneration || "",
+          ...(this.model.pathIndexKey ? { retiredPathIndexKeys: this.model.pathIndexKey } : {}),
+        } } : {}),
         $unset: { emptyTreeGeneration: "", pathIndexKey: "", pathIndexBuiltAt: "" },
       }).exec();
       if (!result.matchedCount) throw new AnonymousError("repository_changed", { httpStatus: 409 });
     }
+    this.model.treeGeneration = generation;
     this.model.fileMetadataRevision = revision;
     this.model.pathIndexKey = undefined;
     this.model.pathIndexBuiltAt = undefined;
@@ -1027,10 +1061,17 @@ export default class Repository {
     if (status) {
       await this.updateStatus(status, statusMessage);
     }
-    // remove cache
+    const current = async () => !isConnected || !!await AnonymizedRepositoryModel.exists({ _id: this.model._id,
+      treeGeneration: generation || { $exists: false }, fileMetadataRevision: revision, status: this.model.status,
+    }).exec();
+    const clearRows = async (collection: Parameters<typeof deleteRowsInBatches>[0]) => {
+      if (!await deleteRowsInBatches(collection, { repoId: this.repoId }, { current })) {
+        throw new AnonymousError("repository_changed", { httpStatus: 409 });
+      }
+    };
+    // A preparation reset retires metadata durably; maintenance performs its deletion.
     await Promise.all([
-      FileModel.deleteMany({ repoId: this.repoId }).exec(),
-      AnonymizedPathModel.deleteMany({ repoId: this.repoId }).exec(),
+      ...(!defer ? [clearRows(FileModel.collection), clearRows(AnonymizedPathModel.collection)] : []),
       this.removeCache(),
     ]);
     logger.info("reset", { repoId: this._model.repoId });

src/core/file-tree.ts

@@ -0,0 +1,32 @@
+import { mongo } from "mongoose";
+import { setImmediate } from "timers/promises";
+import { createLogger } from "./logger";
+
+export const FILE_TREE_BATCH_SIZE = 500;
+// A folder response must fit in memory alongside other API requests.
+export const FILE_LISTING_LIMIT = 10000;
+const logger = createLogger("file-tree");
+
+/** Keep each Mongo write bounded, and let other work run between batches. */
+export async function deleteRowsInBatches(
+  collection: Pick<mongo.Collection, "find" | "deleteMany">,
+  filter: mongo.Filter<mongo.Document> & { repoId: string },
+  options: { maxBatches?: number; current?: () => Promise<boolean> } = {}
+): Promise<boolean> {
+  const started = Date.now();
+  let deleted = 0;
+  for (let batch = 0; batch < (options.maxBatches ?? Infinity); batch++) {
+    if (options.current && !await options.current()) return false;
+    const rows = await collection.find(filter, { projection: { _id: 1 }, limit: FILE_TREE_BATCH_SIZE }).toArray();
+    if (!rows.length) {
+      if (deleted) logger.info("metadata cleanup complete", { repoId: filter.repoId, deleted, ms: Date.now() - started });
+      return true;
+    }
+    if (options.current && !await options.current()) return false;
+    const result = await collection.deleteMany({ ...filter, _id: { $in: rows.map(row => row._id) } }, { hint: "_id_" });
+    deleted += result.deletedCount;
+    await setImmediate();
+  }
+  logger.info("metadata cleanup deferred", { repoId: filter.repoId, deleted, ms: Date.now() - started });
+  return false;
+}

src/core/model/anonymizedRepositories/anonymizedRepositories.schema.ts

@@ -59,6 +59,7 @@ const AnonymizedRepositorySchema = new Schema({
   treeGeneration: String,
   stagedFileTrees: { type: [{ generation: String, until: Date }], default: undefined },
   retiredTreeGenerations: { type: [String], default: undefined },
+  retiredPathIndexKeys: { type: [String], default: undefined },
   retiredContentPrefixes: { type: [String], default: undefined },
   legacyContentCleanupPending: Boolean,
   contentCacheVersion: Number,
@@ -110,6 +111,7 @@ const AnonymizedRepositorySchema = new Schema({
 AnonymizedRepositorySchema.index({ "source.repositoryName": 1 });
 AnonymizedRepositorySchema.index({ "coauthors.githubId": 1 });
 AnonymizedRepositorySchema.index({ retiredTreeGenerations: 1 }, { sparse: true });
+AnonymizedRepositorySchema.index({ retiredPathIndexKeys: 1 }, { sparse: true });
 AnonymizedRepositorySchema.index({ "stagedFileTrees.until": 1 }, { sparse: true });
 AnonymizedRepositorySchema.index({ retiredContentPrefixes: 1 }, { sparse: true });
 AnonymizedRepositorySchema.index({ legacyContentCleanupPending: 1, status: 1 },

src/core/model/anonymizedRepositories/anonymizedRepositories.types.ts

@@ -34,6 +34,7 @@ export interface IAnonymizedRepository {
   treeGeneration?: string;
   stagedFileTrees?: { generation: string; until: Date }[];
   retiredTreeGenerations?: string[];
+  retiredPathIndexKeys?: string[];
   retiredContentPrefixes?: string[];
   legacyContentCleanupPending?: boolean;
   contentCacheVersion?: number;

src/core/source/GitHubStream.ts

@@ -17,8 +17,7 @@ import { basename, dirname } from "path";
 import * as stream from "stream";
 import AnonymousError from "../AnonymousError";
 import { octokit, waitForTokenGate, RateLimitDelayError } from "../GitHubUtils";
-import FileModel from "../model/files/files.model";
-import { expandSubmodules, validUpstream } from "./github-submodules";
+import { expandSubmodules, validUpstream, MAX_SUBMODULE_ENTRIES } from "./github-submodules";
 import { IFile, UpstreamFile } from "../model/files/files.types";
 import { createLogger, serializeError } from "../logger";
 import { measureStage, recordCacheHit } from "../request-monitoring";
@@ -32,6 +31,7 @@ import { cachedContentMiss, classifyContentMiss } from "./github-content-miss";
 const logger = createLogger("gh-stream");
 
 const GH_API_CONCURRENCY = 6;
+const MAX_PENDING_TREE_FOLDERS = 100000;
 const cacheFills = new Map<string, Promise<void>>();
 const missingContent = new ExpiringMap<string>(2048);
 
@@ -472,8 +472,17 @@ export default class GitHubStream extends GitHubBase {
   }
 
   async getFiles(progress?: (status: string) => void) {
+    const files: IFile[] = [];
+    for await (const file of this.iterateFiles(progress)) files.push(file);
+    return files;
+  }
+
+  async *iterateFiles(progress?: (status: string) => void): AsyncGenerator<IFile> {
     this._truncatedFolders = [];
-    return expandSubmodules(await this.getTruncatedTree(this.data.commit, progress), this.data);
+    const budget = { modules: 0, remaining: MAX_SUBMODULE_ENTRIES };
+    for await (const batch of this.iterateTreeBatches(this.data.commit, progress)) {
+      yield* await expandSubmodules(batch, this.data, budget);
+    }
   }
 
   /**
@@ -577,7 +586,7 @@ export default class GitHubStream extends GitHubBase {
     }
   }
 
-  private async getTruncatedTree(
+  private async *iterateTreeBatches(
     sha: string,
     progress?: (status: string) => void,
     parentPath: string = ""
@@ -588,21 +597,23 @@ export default class GitHubStream extends GitHubBase {
       request: 0,
       file: 0,
     };
-    const output: IFile[] = [];
     let data = await this.getGHTree(oct, token, sha, count, {
       recursive: true,
       callback: () => { progress?.("List file: " + count.file); },
     });
-    if (!data.truncated) return this.tree2Tree(data.tree, parentPath);
+    if (!data.truncated) { yield this.tree2Tree(data.tree, parentPath); return; }
     data = await this.getGHTree(oct, token, sha, count, {
       recursive: false,
       callback: () => { progress?.("List file: " + count.file); },
     });
     if (data.truncated) this._truncatedFolders.push(parentPath);
-    output.push(...this.tree2Tree(data.tree, parentPath));
+    yield this.tree2Tree(data.tree, parentPath);
     const subtrees: { sha: string; parentPath: string }[] = [];
     for (const file of data.tree) {
       if (file.type == "tree" && file.path && file.sha) {
+        if (subtrees.length >= MAX_PENDING_TREE_FOLDERS) {
+          throw new AnonymousError("repository_tree_too_large", { httpStatus: 413 });
+        }
         subtrees.push({
           sha: file.sha,
           parentPath: path.join(parentPath, file.path),
@@ -655,11 +666,15 @@ export default class GitHubStream extends GitHubBase {
         GH_API_CONCURRENCY
       );
       for (const result of batchResults) {
-        output.push(...result.files);
-        queue.push(...result.children);
+        yield result.files;
+        for (const child of result.children) {
+          if (queue.length >= MAX_PENDING_TREE_FOLDERS) {
+            throw new AnonymousError("repository_tree_too_large", { httpStatus: 413 });
+          }
+          queue.push(child);
+        }
       }
     }
-    return output;
   }
 
   private tree2Tree(
@@ -679,14 +694,14 @@ export default class GitHubStream extends GitHubBase {
       if (pathFile === ".") {
         pathFile = "";
       }
-      return new FileModel({
+      return {
         name: basename(fullPath),
         path: pathFile,
         repoId: this.data.repoId,
         size: elem.size,
-        ...(elem.type === "commit" ? { submodule: "unsupported" } : {}),
+        ...(elem.type === "commit" ? { submodule: "unsupported" as const } : {}),
         sha: elem.sha,
-      });
+      };
     });
   }
 }

src/core/source/Source.ts

@@ -21,4 +21,7 @@ export interface SourceBase {
    * Get all the files from a specific source
    */
   getFiles(progress?: (status: string) => void): Promise<IFile[]>;
+
+  /** Read large trees without retaining every entry. */
+  iterateFiles?(progress?: (status: string) => void): AsyncIterable<IFile>;
 }

src/core/source/github-submodules.ts

@@ -36,12 +36,12 @@ export function submoduleRepository(url: unknown, parent: GitHubBaseData): { org
 }
 
 /** Resolve only direct public GitHub dependencies. Never send the parent token to a child. */
-export async function expandSubmodules(files: IFile[], data: GitHubBaseData): Promise<IFile[]> {
+export async function expandSubmodules(files: IFile[], data: GitHubBaseData,
+  budget = { modules: 0, remaining: MAX_SUBMODULE_ENTRIES }): Promise<IFile[]> {
   const output = files.filter(file => file.name !== ".gitmodules");
   const modules = output.filter(file => file.submodule);
-  let remaining = MAX_SUBMODULE_ENTRIES;
-  for (const [index, mount] of modules.entries()) {
-    if (index >= MAX_SUBMODULES) { mount.submodule = "limit"; continue; }
+  for (const mount of modules) {
+    if (budget.modules++ >= MAX_SUBMODULES) { mount.submodule = "limit"; continue; }
     const mountPath = [mount.path, mount.name].filter(Boolean).join("/");
     if (!safeRepositoryPath(mountPath) || !mount.sha || !/^[a-f0-9]{40}$/.test(mount.sha)) continue;
     const token = await data.getToken();
@@ -59,10 +59,10 @@ export async function expandSubmodules(files: IFile[], data: GitHubBaseData): Pr
       const child = await octokit("").git.getTree({
         owner: repository.organization, repo: repository.repoName, tree_sha: mount.sha, recursive: "1",
       });
-      if (child.data.truncated || child.data.tree.length > remaining) { mount.submodule = "limit"; continue; }
+      if (child.data.truncated || child.data.tree.length > budget.remaining) { mount.submodule = "limit"; continue; }
       const entries = child.data.tree;
       if (entries.some(entry => !safeRepositoryPath(entry.path))) { mount.submodule = "unsupported"; continue; }
-      remaining -= entries.length;
+      budget.remaining -= entries.length;
       mount.submodule = "included";
       mount.upstream = { ...repository, commit: mount.sha, path: "" };
       for (const entry of entries) {

src/scripts/create-performance-indexes.ts

@@ -17,6 +17,7 @@ async function main() {
   await File.collection.createIndex({ metadataPending: 1, repoId: 1, treeGeneration: 1 },
     { partialFilterExpression: { metadataPending: true } });
   await Repository.collection.createIndex({ retiredTreeGenerations: 1 }, { sparse: true });
+  await Repository.collection.createIndex({ retiredPathIndexKeys: 1 }, { sparse: true });
   await Repository.collection.createIndex({ "stagedFileTrees.until": 1 }, { sparse: true });
   await File.collection.createIndex({ treeStaged: 1, repoId: 1, treeGeneration: 1 },
     { partialFilterExpression: { treeStaged: true } });

src/server/routes/repository-public.ts

@@ -1,3 +1,4 @@
+import { FILE_LISTING_LIMIT } from "../../core/file-tree";
 import { repositoryFailureStatus } from "../../core/repository-failure-status";
 import { anonymizePath } from "../../core/anonymize-utils";
 import * as express from "express";
@@ -217,21 +218,30 @@ router.get(
         { $match: { repoId, treeGeneration: repo.model.treeGeneration || { $exists: false }, size: { $ne: null } } },
         { $project: { _id: 0, path: 1 } },
         { $group: { _id: "$path", count: { $sum: 1 } } },
+        { $limit: FILE_LISTING_LIMIT + 1 },
       ]).exec();
+      if (results.length > FILE_LISTING_LIMIT) throw new AnonymousError("file_listing_too_large", { httpStatus: 413 });
 
-      const directCounts: Record<string, number> = {};
+      const directCounts: Record<string, number> = Object.create(null);
       for (const r of results) {
         directCounts[r._id ?? ""] = r.count;
       }
 
-      const folderCounts: Record<string, number> = {};
+      const folderCounts: Record<string, number> = Object.create(null);
+      let folders = 0;
+      const addCount = (folder: string, count: number) => {
+        if (!Object.hasOwn(folderCounts, folder) && ++folders > FILE_LISTING_LIMIT) {
+          throw new AnonymousError("file_listing_too_large", { httpStatus: 413 });
+        }
+        folderCounts[folder] = (folderCounts[folder] || 0) + count;
+      };
       for (const [folder, count] of Object.entries(directCounts)) {
         let p = folder;
-        folderCounts[p] = (folderCounts[p] || 0) + count;
+        addCount(p, count);
         while (p) {
           const idx = p.lastIndexOf("/");
           p = idx >= 0 ? p.substring(0, idx) : "";
-          folderCounts[p] = (folderCounts[p] || 0) + count;
+          addCount(p, count);
           if (!p) break;
         }
       }
@@ -242,7 +252,7 @@ router.get(
           "../../core/anonymize-utils"
         );
         const compiled = compileTerms(terms);
-        const anonymized: Record<string, number> = {};
+        const anonymized: Record<string, number> = Object.create(null);
         for (const [folder, count] of Object.entries(folderCounts)) {
           const anonFolder = anonymizePathCompiled(folder, compiled);
           anonymized[anonFolder] = (anonymized[anonFolder] || 0) + count;

src/server/schedule.ts

@@ -156,7 +156,7 @@ export async function runRepositoryStatusCheck(now = new Date()) {
   await expireContent(AnonymizedPullRequestModel, data => new PullRequest(data), now);
 
   const retiredCursor = AnonymizedRepositoryModel.find({ $or: [
-    { retiredTreeGenerations: { $exists: true } }, { retiredContentPrefixes: { $exists: true } },
+    { retiredTreeGenerations: { $exists: true } }, { retiredPathIndexKeys: { $exists: true } }, { retiredContentPrefixes: { $exists: true } },
     { legacyContentCleanupPending: true, status: RepositoryStatus.READY },
     { "stagedFileTrees.until": { $lte: now } },
   ] }).cursor();