merge: preserve billing admission with bounded tree preparation

src/core/Repository.ts

@@ -1,4 +1,6 @@
 import { allowance, reserveRepository } from "./billing/quotas";
+import { FILE_TREE_BATCH_SIZE, FILE_LISTING_LIMIT, deleteRowsInBatches } from "./file-tree";
+import { SourceBase, sourceStorageSize } from "./source/Source";
 import { notifyOwnerAccessProblem, withOwnerAccessRecovery } from "./owner-notifications";
 import AnonymizedPathModel from "./model/anonymized-path";
 import storage from "./storage";
@@ -37,6 +39,24 @@ import AnonymizedRepositoryModel from "./model/anonymizedRepositories/anonymized
 import { createLogger, serializeError } from "./logger";
 
 const logger = createLogger("repository");
+const snapshotValue = <T>(value: T | undefined) => value === undefined ? { $exists: false } : value;
+/** Capture and retire the stored key while clearing it, including concurrent index publication. */
+async function updateRetiringPathIndex(
+  filter: () => FilterQuery<IAnonymizedRepositoryDocument>,
+  update: UpdateQuery<IAnonymizedRepositoryDocument>
+): Promise<{ matchedCount: number }> {
+  for (let attempt = 0; attempt < 5; attempt++) {
+    const current = await AnonymizedRepositoryModel.findOne(filter()).select("pathIndexKey").lean().exec();
+    if (!current) return { matchedCount: 0 };
+    const result = await AnonymizedRepositoryModel.updateOne({ ...filter(),
+      pathIndexKey: current.pathIndexKey === undefined ? { $exists: false } : current.pathIndexKey,
+    }, { ...update, ...(current.pathIndexKey ? { $addToSet: {
+      ...update.$addToSet, retiredPathIndexKeys: current.pathIndexKey,
+    } } : {}) }).exec();
+    if (result.matchedCount) return result;
+  }
+  return { matchedCount: 0 };
+}
 const pathIndexes = new Map<string, Promise<void>>();
 async function buildPathIndex(repoId: string, key: string, terms: string[], generation?: string) {
   const cacheKey = `${repoId}:${key}`;
@@ -68,7 +88,7 @@ async function buildPathIndex(repoId: string, key: string, terms: string[], gene
 }
 import { IFile, IFileDocument } from "./model/files/files.types";
 import AnonymizedFile from "./AnonymizedFile";
-import { FilterQuery } from "mongoose";
+import { FilterQuery, UpdateQuery } from "mongoose";
 function anonymizeTreeRecursive(
   tree: IFile[],
   terms: string[],
@@ -201,7 +221,12 @@ export default class Repository {
     if (pathQuery !== undefined) {
       query.path = pathQuery;
     }
-    return await FileModel.find(query).exec();
+    const files = await FileModel.find(query).select("name path repoId size sha upstream submodule")
+      .limit(FILE_LISTING_LIMIT + 1).lean().exec();
+    if (files.length > FILE_LISTING_LIMIT) {
+      throw new AnonymousError("file_listing_too_large", { httpStatus: 413 });
+    }
+    return files;
   }
 
   async submoduleMounts(): Promise<SubmoduleMount[]> {
@@ -239,19 +264,9 @@ export default class Repository {
       const previousGeneration = this.model.treeGeneration;
       const previousPrefix = contentGenerationPrefix(`${previousGeneration || "legacy"}:${this.model.anonymizeDate?.toISOString() || ""}`);
       const streamed = this.model.source.type !== "Zip";
-      const source = this.source;
-      const readFiles = () => source.getFiles(opt.progress);
-      const files = await (streamed ? withOwnerAccessRecovery(this.owner.id,
-        { kind: "repository", id: String(this.model._id) }, readFiles,
-        { "source.repositoryName": this.model.source.repositoryName, "source.commit": this.model.source.commit,
-          "githubAccess.revision": this.model.githubAccess?.revision ?? { $exists: false } }) : readFiles()).catch(error => {
-        void notifyOwnerAccessProblem(this.owner.id, error, { kind: "repository", id: String(this.model._id) });
-        throw error;
-      });
-      if (this.model.billingManaged) await reserveRepository(String(this.model.owner), this.repoId, files.reduce((sum, file) => sum + (file.size || 0), 0), this.model.conference);
-      const sourceWithTruncation = source as unknown as { truncatedFolderList?: string[] };
-      const truncatedFolders = Array.isArray(sourceWithTruncation.truncatedFolderList)
-        ? [...sourceWithTruncation.truncatedFolderList] : [];
+      const source: SourceBase & { truncatedFolderList?: string[] } = this.source;
+      let fileCount = 0, storageBytes = 0;
+      let truncatedFolders: string[] = [];
       const generation = randomUUID();
       const buildFilter = { _id: this.model._id,
         treeGeneration: previousGeneration || { $exists: false }, ...this.refreshFilter(),
@@ -273,25 +288,50 @@ export default class Repository {
         }, 30_000);
         heartbeat.unref();
       }
-      files.forEach(f => { f.repoId = this.repoId; f.treeGeneration = generation; if (isConnected) f.treeStaged = true; });
+      const readFiles = async () => {
+        let batch: IFile[] = [];
+        const insertBatch = async () => {
+          if (isConnected && !await AnonymizedRepositoryModel.exists({ ...buildFilter,
+            stagedFileTrees: { $elemMatch: { generation, until: { $gt: new Date() } } },
+          }).exec()) throw new AnonymousError("repository_changed", { httpStatus: 409 });
+          await FileModel.insertMany(batch); batch = [];
+        };
+        const files = source.iterateFiles ? source.iterateFiles(opt.progress) : await source.getFiles(opt.progress);
+        for await (const file of files) {
+          const row = file instanceof FileModel ? file.toObject() : file;
+          batch.push({ ...row, repoId: this.repoId, treeGeneration: generation, ...(isConnected ? { treeStaged: true } : {}) });
+          fileCount++;
+          storageBytes += file.size || 0;
+          if (batch.length === FILE_TREE_BATCH_SIZE) await insertBatch();
+        }
+        if (batch.length) await insertBatch();
+        truncatedFolders = Array.isArray(source.truncatedFolderList) ? [...source.truncatedFolderList] : [];
+      };
       try {
-        await FileModel.insertMany(files);
+        await (streamed ? withOwnerAccessRecovery(this.owner.id,
+          { kind: "repository", id: String(this.model._id) }, readFiles,
+          { "source.repositoryName": this.model.source.repositoryName, "source.commit": this.model.source.commit,
+            "githubAccess.revision": this.model.githubAccess?.revision ?? { $exists: false } }) : readFiles()).catch(error => {
+          void notifyOwnerAccessProblem(this.owner.id, error, { kind: "repository", id: String(this.model._id) });
+          throw error;
+        });
+        if (this.model.billingManaged) await reserveRepository(String(this.model.owner), this.repoId, storageBytes, this.model.conference);
         if (isConnected) {
-          const activated = await AnonymizedRepositoryModel.updateOne({ _id: this.model._id,
+          const activated = await updateRetiringPathIndex(() => ({ _id: this.model._id,
             treeGeneration: previousGeneration || { $exists: false }, ...this.refreshFilter(),
             status: this.model.status, statusDate: this.model.statusDate,
             "githubAccess.revision": this.model.githubAccess?.revision || { $exists: false },
             stagedFileTrees: { $elemMatch: { generation, until: { $gt: new Date() } } },
-          }, { $set: { treeGeneration: generation, truncatedFolders, size: { storage: 0, file: 0 },
+          }), { $set: { treeGeneration: generation, truncatedFolders, size: { storage: 0, file: 0 },
             ...(streamed && !this.model.contentCacheVersion ? { contentCacheVersion: 1, legacyContentCleanupPending: true } : {}),
-            ...(!files.length ? { emptyTreeGeneration: generation } : {}),
+            ...(!fileCount ? { emptyTreeGeneration: generation } : {}),
           }, $unset: { pathIndexKey: "", pathIndexBuiltAt: "", sizeComputedAt: "",
-            ...(files.length ? { emptyTreeGeneration: "" } : {}),
+            ...(fileCount ? { emptyTreeGeneration: "" } : {}),
           }, $addToSet: { retiredTreeGenerations: previousGeneration || "",
             ...(streamed ? { retiredContentPrefixes: previousPrefix } : {}),
-          }, $pull: { stagedFileTrees: { generation } } }).exec();
+          }, $pull: { stagedFileTrees: { generation } } });
           if (!activated.matchedCount) {
-            await FileModel.deleteMany({ repoId: this.repoId, treeGeneration: generation }).exec();
+            await deleteRowsInBatches(FileModel.collection, { repoId: this.repoId, treeGeneration: generation });
             throw new AnonymousError("repository_changed", { httpStatus: 409 });
           }
         }
@@ -302,11 +342,11 @@ export default class Repository {
         await FileModel.updateMany({ repoId: this.repoId, treeGeneration: generation, treeStaged: true },
           { $unset: { treeStaged: "" } }).exec().catch(error => logger.warn("staged tree completion deferred", serializeError(error)));
       }
-      this.model.emptyTreeGeneration = files.length ? undefined : generation;
+      this.model.emptyTreeGeneration = fileCount ? undefined : generation;
       this.model.truncatedFolders = truncatedFolders;
       if (streamed) this.model.contentCacheVersion = 1;
       if (isConnected) {
-        await this.cleanupRetiredFileTrees().catch(error => logger.warn("retired tree cleanup deferred", serializeError(error)));
+        await this.cleanupRetiredFileTrees(new Date(), 10).catch(error => logger.warn("retired tree cleanup deferred", serializeError(error)));
       } else {
         await FileModel.deleteMany({ repoId: this.repoId, treeGeneration: previousGeneration || { $exists: false } }).exec();
       }
@@ -329,7 +369,7 @@ export default class Repository {
   }
 
   /** Activation records retirement atomically; maintenance retries interrupted deletion. */
-  async cleanupRetiredFileTrees(now = new Date()) {
+  async cleanupRetiredFileTrees(now = new Date(), maxBatches = Infinity) {
     const stages = await AnonymizedRepositoryModel.findById(this.model._id).select("stagedFileTrees").lean().exec();
     for (const stage of stages?.stagedFileTrees || []) {
       if (stage.until > now) continue;
@@ -339,7 +379,7 @@ export default class Repository {
         $addToSet: { retiredTreeGenerations: stage.generation } }).exec();
     }
     const current = await AnonymizedRepositoryModel.findById(this.model._id)
-      .select("treeGeneration retiredTreeGenerations retiredContentPrefixes legacyContentCleanupPending source.type status anonymizeDate").lean().exec();
+      .select("treeGeneration retiredTreeGenerations retiredContentPrefixes retiredPathIndexKeys pathIndexKey legacyContentCleanupPending source.type status anonymizeDate").lean().exec();
     if (!current) return;
     const activePrefix = contentGenerationPrefix(`${current.treeGeneration || "legacy"}:${current.anonymizeDate?.toISOString() || ""}`);
     const prefixes = (current.retiredContentPrefixes || []).filter(prefix => prefix !== activePrefix);
@@ -357,13 +397,21 @@ export default class Repository {
     }
     const retired = (current.retiredTreeGenerations || []).filter(generation => generation !== (current.treeGeneration || ""));
     if (retired.length) {
-      await FileModel.deleteMany({ repoId: this.repoId, $or: [
+      const complete = await deleteRowsInBatches(FileModel.collection, { repoId: this.repoId, $or: [
         { treeGeneration: { $in: retired.filter(Boolean) } },
         ...(retired.includes("") ? [{ treeGeneration: { $exists: false } }] : []),
-      ] }).exec();
-      await AnonymizedRepositoryModel.updateOne({ _id: this.model._id },
+      ] }, { maxBatches });
+      if (complete) await AnonymizedRepositoryModel.updateOne({ _id: this.model._id },
         { $pull: { retiredTreeGenerations: { $in: retired } } }).exec();
     }
+    const retiredKeys = (current.retiredPathIndexKeys || []).filter(key => key !== current.pathIndexKey);
+    if (retiredKeys.length && await deleteRowsInBatches(AnonymizedPathModel.collection,
+      { repoId: this.repoId, key: { $in: retiredKeys } }, { maxBatches })) {
+      await AnonymizedRepositoryModel.updateOne({ _id: this.model._id },
+        { $pull: { retiredPathIndexKeys: { $in: retiredKeys } } }).exec();
+    }
+    await AnonymizedRepositoryModel.updateOne({ _id: this.model._id, "retiredPathIndexKeys.0": { $exists: false } },
+      { $unset: { retiredPathIndexKeys: "" } }).exec();
     await AnonymizedRepositoryModel.updateOne({ _id: this.model._id, "retiredTreeGenerations.0": { $exists: false } },
       { $unset: { retiredTreeGenerations: "" } }).exec();
     await AnonymizedRepositoryModel.updateOne({ _id: this.model._id, "retiredContentPrefixes.0": { $exists: false } },
@@ -386,7 +434,7 @@ export default class Repository {
     if (current?.treeGeneration === generation && !terminal) {
       await FileModel.updateMany({ repoId, treeGeneration: generation, treeStaged: true }, { $unset: { treeStaged: "" } }).exec();
     } else if (terminal || !current?.stagedFileTrees?.some(stage => stage.generation === generation && stage.until > now)) {
-      await FileModel.deleteMany({ repoId, treeGeneration: generation, treeStaged: true }).exec();
+      await deleteRowsInBatches(FileModel.collection, { repoId, treeGeneration: generation, treeStaged: true });
     }
   }
 
@@ -425,7 +473,7 @@ export default class Repository {
         const current = await AnonymizedRepositoryModel.exists({ _id: this.model._id,
           fileMetadataRevision: metadataRevision || { $exists: false },
         }).exec();
-        if (!current) await AnonymizedPathModel.deleteMany({ repoId: this.repoId, key }).exec();
+        if (!current) await deleteRowsInBatches(AnonymizedPathModel.collection, { repoId: this.repoId, key });
         throw error;
       }
       // TTL retires superseded mappings. An older builder must never delete
@@ -747,10 +795,10 @@ export default class Repository {
           const candidateModel = new AnonymizedRepositoryModel(this.model.toObject());
           candidateModel.source.commit = newCommit;
           const candidate = new Repository(candidateModel);
-          const files = await candidate.source.getFiles();
+          const storageBytes = await sourceStorageSize(candidate.source);
           try {
             await reserveRepository(String(this.model.owner), this.repoId,
-              files.reduce((sum, file) => sum + (file.size || 0), 0), this.model.conference);
+              storageBytes, this.model.conference);
           } catch (error) {
             if (!opt?.force && error instanceof Error && error.message === "quota_exceeded") return;
             throw error;
@@ -822,11 +870,7 @@ export default class Repository {
     }
     this.model.increment();
     await this.updateStatus(RepositoryStatus.DOWNLOAD);
-    await this.files({
-      force: false,
-      progress,
-      recursive: false,
-    });
+    await this.ensureFileTree({ progress });
     // Previously inserted a dummy {path:"", name:"", size:0} FileModel
     // here for empty repos "to avoid errors" — but that record collides
     // with the special-case in AnonymizedFile.getFileInfo for the empty
@@ -992,11 +1036,12 @@ export default class Repository {
       await removeTransformedRepositoryCache(this.repoId);
       await storage.rm(this.repoId);
       if (!await AnonymizedRepositoryModel.exists(lease).exec()) return;
-      await FileModel.deleteMany({ repoId: this.repoId }).exec();
-      await AnonymizedPathModel.deleteMany({ repoId: this.repoId }).exec();
+      const current = async () => !!await AnonymizedRepositoryModel.exists(lease).exec();
+      if (!await deleteRowsInBatches(FileModel.collection, { repoId: this.repoId }, { current })) return;
+      if (!await deleteRowsInBatches(AnonymizedPathModel.collection, { repoId: this.repoId }, { current })) return;
       const result = await AnonymizedRepositoryModel.updateOne(lease, { $set: { status: RepositoryStatus.EXPIRED,
         statusDate: new Date(), isReseted: true, size: { storage: 0, file: 0 } },
-        $unset: { cleanupToken: "", cleanupUntil: "", pathIndexKey: "", pathIndexBuiltAt: "", sizeComputedAt: "", emptyTreeGeneration: "", retiredTreeGenerations: "", retiredContentPrefixes: "", legacyContentCleanupPending: "", stagedFileTrees: "" } }).exec();
+        $unset: { cleanupToken: "", cleanupUntil: "", pathIndexKey: "", pathIndexBuiltAt: "", sizeComputedAt: "", emptyTreeGeneration: "", retiredTreeGenerations: "", retiredPathIndexKeys: "", retiredContentPrefixes: "", legacyContentCleanupPending: "", stagedFileTrees: "" } }).exec();
       if (result.matchedCount) { this.model.status = RepositoryStatus.EXPIRED; this.model.isReseted = true; this.model.size = { storage: 0, file: 0 }; this.model.sizeComputedAt = undefined; this.model.emptyTreeGeneration = undefined; }
     } finally {
       clearInterval(heartbeat);
@@ -1004,6 +1049,19 @@ export default class Repository {
     }
   }
 
+  /** Fields that identify the cached source and its current lifecycle owner. */
+  private fileSnapshotFilter() {
+    return {
+      treeGeneration: snapshotValue(this.model.treeGeneration),
+      status: snapshotValue(this.model.status), statusDate: snapshotValue(this.model.statusDate),
+      anonymizeDate: snapshotValue(this.model.anonymizeDate),
+      fileMetadataRevision: snapshotValue(this.model.fileMetadataRevision),
+      "source.type": snapshotValue(this.model.source.type), "source.repositoryName": snapshotValue(this.model.source.repositoryName),
+      "source.commit": snapshotValue(this.model.source.commit), "source.branch": snapshotValue(this.model.source.branch),
+      "githubAccess.revision": snapshotValue(this.model.githubAccess?.revision),
+    };
+  }
+
   /**
    * Remove the repository
    */
@@ -1011,12 +1069,15 @@ export default class Repository {
     if (expected) {
       // Claim the lifecycle before deleting files; migration rejects REMOVING.
       this.assertNotArchived();
+      const statusDate = new Date();
       const result = await AnonymizedRepositoryModel.updateOne({ _id: this.model._id,
-        status: this.model.status,
-        "githubAccess.revision": expected.accessRevision || { $exists: false },
-      }, { $set: { status: RepositoryStatus.REMOVING, statusDate: new Date() } });
+        ...this.fileSnapshotFilter(), ...this.refreshFilter(),
+        settingsSavedAt: snapshotValue(this.model.settingsSavedAt),
+        "githubAccess.revision": snapshotValue(expected.accessRevision),
+      }, { $set: { status: RepositoryStatus.REMOVING, statusDate } });
       if (!result.matchedCount) throw new AnonymousError("connection_changed", { httpStatus: 409 });
       this.model.status = RepositoryStatus.REMOVING;
+      this.model.statusDate = statusDate;
     } else {
       await this.updateStatus(RepositoryStatus.REMOVING);
     }
@@ -1031,15 +1092,28 @@ export default class Repository {
   async resetSate(status?: RepositoryStatus, statusMessage?: string) {
     this.assertNotArchived();
     const revision = randomUUID();
-    this.model.emptyTreeGeneration = undefined;
+    const defer = isConnected && (status || this.status) === RepositoryStatus.PREPARING;
+    const previousGeneration = this.model.treeGeneration;
+    const generation = defer ? randomUUID() : previousGeneration;
+    const refreshToken = this.refreshToken;
+    // Options-only saves apply at read time and leave this source snapshot valid.
+    // They must not cancel a queued worker without scheduling replacement work.
+    const snapshot = this.fileSnapshotFilter();
     if (isConnected) {
-      const result = await AnonymizedRepositoryModel.updateOne({ _id: this.model._id,
-      treeGeneration: this.model.treeGeneration || { $exists: false },
-      }, { $set: { fileMetadataRevision: revision },
+      const result = await updateRetiringPathIndex(() => ({ _id: this.model._id,
+        ...snapshot,
+        ...(refreshToken ? { refreshToken, refreshUntil: { $gt: new Date() } } : {}),
+      }), { $set: { fileMetadataRevision: revision, ...(defer ? { treeGeneration: generation } : {}) },
+        ...(defer ? { $addToSet: { retiredTreeGenerations: previousGeneration || "",
+        } } : {}),
         $unset: { emptyTreeGeneration: "", pathIndexKey: "", pathIndexBuiltAt: "" },
-      }).exec();
-      if (!result.matchedCount) throw new AnonymousError("repository_changed", { httpStatus: 409 });
+      });
+      this.checkRefreshWrite(result);
+      if (!result.matchedCount) throw new AnonymousError(this.protectLifecycle ? "repository_job_cancelled" : "repository_changed",
+        { httpStatus: this.protectLifecycle ? 410 : 409 });
     }
+    this.model.emptyTreeGeneration = undefined;
+    this.model.treeGeneration = generation;
     this.model.fileMetadataRevision = revision;
     this.model.pathIndexKey = undefined;
     this.model.pathIndexBuiltAt = undefined;
@@ -1048,10 +1122,17 @@ export default class Repository {
     if (status) {
       await this.updateStatus(status, statusMessage);
     }
-    // remove cache
+    const current = async () => !isConnected || !!await AnonymizedRepositoryModel.exists({ _id: this.model._id,
+      treeGeneration: generation || { $exists: false }, fileMetadataRevision: revision, status: this.model.status,
+    }).exec();
+    const clearRows = async (collection: Parameters<typeof deleteRowsInBatches>[0]) => {
+      if (!await deleteRowsInBatches(collection, { repoId: this.repoId }, { current })) {
+        throw new AnonymousError("repository_changed", { httpStatus: 409 });
+      }
+    };
+    // A preparation reset retires metadata durably; maintenance performs its deletion.
     await Promise.all([
-      FileModel.deleteMany({ repoId: this.repoId }).exec(),
-      AnonymizedPathModel.deleteMany({ repoId: this.repoId }).exec(),
+      ...(!defer ? [clearRows(FileModel.collection), clearRows(AnonymizedPathModel.collection)] : []),
       this.removeCache(),
     ]);
     logger.info("reset", { repoId: this._model.repoId });

src/core/file-tree.ts

@@ -0,0 +1,32 @@
+import { mongo } from "mongoose";
+import { setImmediate } from "timers/promises";
+import { createLogger } from "./logger";
+
+export const FILE_TREE_BATCH_SIZE = 500;
+// A folder response must fit in memory alongside other API requests.
+export const FILE_LISTING_LIMIT = 10000;
+const logger = createLogger("file-tree");
+
+/** Keep each Mongo write bounded, and let other work run between batches. */
+export async function deleteRowsInBatches(
+  collection: Pick<mongo.Collection, "find" | "deleteMany">,
+  filter: mongo.Filter<mongo.Document> & { repoId: string },
+  options: { maxBatches?: number; current?: () => Promise<boolean> } = {}
+): Promise<boolean> {
+  const started = Date.now();
+  let deleted = 0;
+  for (let batch = 0; batch < (options.maxBatches ?? Infinity); batch++) {
+    if (options.current && !await options.current()) return false;
+    const rows = await collection.find(filter, { projection: { _id: 1 }, limit: FILE_TREE_BATCH_SIZE }).toArray();
+    if (!rows.length) {
+      if (deleted) logger.info("metadata cleanup complete", { repoId: filter.repoId, deleted, ms: Date.now() - started });
+      return true;
+    }
+    if (options.current && !await options.current()) return false;
+    const result = await collection.deleteMany({ ...filter, _id: { $in: rows.map(row => row._id) } }, { hint: "_id_" });
+    deleted += result.deletedCount;
+    await setImmediate();
+  }
+  logger.info("metadata cleanup deferred", { repoId: filter.repoId, deleted, ms: Date.now() - started });
+  return false;
+}

src/core/model/anonymizedRepositories/anonymizedRepositories.schema.ts

@@ -61,6 +61,7 @@ const AnonymizedRepositorySchema = new Schema({
   treeGeneration: String,
   stagedFileTrees: { type: [{ generation: String, until: Date }], default: undefined },
   retiredTreeGenerations: { type: [String], default: undefined },
+  retiredPathIndexKeys: { type: [String], default: undefined },
   retiredContentPrefixes: { type: [String], default: undefined },
   legacyContentCleanupPending: Boolean,
   contentCacheVersion: Number,
@@ -112,6 +113,7 @@ const AnonymizedRepositorySchema = new Schema({
 AnonymizedRepositorySchema.index({ "source.repositoryName": 1 });
 AnonymizedRepositorySchema.index({ "coauthors.githubId": 1 });
 AnonymizedRepositorySchema.index({ retiredTreeGenerations: 1 }, { sparse: true });
+AnonymizedRepositorySchema.index({ retiredPathIndexKeys: 1 }, { sparse: true });
 AnonymizedRepositorySchema.index({ "stagedFileTrees.until": 1 }, { sparse: true });
 AnonymizedRepositorySchema.index({ retiredContentPrefixes: 1 }, { sparse: true });
 AnonymizedRepositorySchema.index({ legacyContentCleanupPending: 1, status: 1 },

src/core/model/anonymizedRepositories/anonymizedRepositories.types.ts

@@ -36,6 +36,7 @@ export interface IAnonymizedRepository {
   treeGeneration?: string;
   stagedFileTrees?: { generation: string; until: Date }[];
   retiredTreeGenerations?: string[];
+  retiredPathIndexKeys?: string[];
   retiredContentPrefixes?: string[];
   legacyContentCleanupPending?: boolean;
   contentCacheVersion?: number;

src/core/source/GitHubStream.ts

@@ -20,8 +20,7 @@ import { basename, dirname } from "path";
 import * as stream from "stream";
 import AnonymousError from "../AnonymousError";
 import { octokit, waitForTokenGate, RateLimitDelayError } from "../GitHubUtils";
-import FileModel from "../model/files/files.model";
-import { expandSubmodules, validUpstream } from "./github-submodules";
+import { expandSubmodules, validUpstream, MAX_SUBMODULE_ENTRIES } from "./github-submodules";
 import { IFile, UpstreamFile } from "../model/files/files.types";
 import { createLogger, serializeError } from "../logger";
 import { measureStage, recordCacheHit } from "../request-monitoring";
@@ -35,6 +34,7 @@ import { cachedContentMiss, classifyContentMiss } from "./github-content-miss";
 const logger = createLogger("gh-stream");
 
 const GH_API_CONCURRENCY = 6;
+const MAX_PENDING_TREE_FOLDERS = 100000;
 const cacheFills = new Map<string, Promise<void>>();
 const missingContent = new ExpiringMap<string>(2048);
 
@@ -583,8 +583,17 @@ export default class GitHubStream extends GitHubBase {
   }
 
   async getFiles(progress?: (status: string) => void) {
+    const files: IFile[] = [];
+    for await (const file of this.iterateFiles(progress)) files.push(file);
+    return files;
+  }
+
+  async *iterateFiles(progress?: (status: string) => void): AsyncGenerator<IFile> {
     this._truncatedFolders = [];
-    return expandSubmodules(await this.getTruncatedTree(this.data.commit, progress), this.data);
+    const budget = { modules: 0, remaining: MAX_SUBMODULE_ENTRIES };
+    for await (const batch of this.iterateTreeBatches(this.data.commit, progress)) {
+      yield* await expandSubmodules(batch, this.data, budget);
+    }
   }
 
   /**
@@ -688,7 +697,7 @@ export default class GitHubStream extends GitHubBase {
     }
   }
 
-  private async getTruncatedTree(
+  private async *iterateTreeBatches(
     sha: string,
     progress?: (status: string) => void,
     parentPath: string = ""
@@ -699,21 +708,23 @@ export default class GitHubStream extends GitHubBase {
       request: 0,
       file: 0,
     };
-    const output: IFile[] = [];
     let data = await this.getGHTree(oct, token, sha, count, {
       recursive: true,
       callback: () => { progress?.("List file: " + count.file); },
     });
-    if (!data.truncated) return this.tree2Tree(data.tree, parentPath);
+    if (!data.truncated) { yield this.tree2Tree(data.tree, parentPath); return; }
     data = await this.getGHTree(oct, token, sha, count, {
       recursive: false,
       callback: () => { progress?.("List file: " + count.file); },
     });
     if (data.truncated) this._truncatedFolders.push(parentPath);
-    output.push(...this.tree2Tree(data.tree, parentPath));
+    yield this.tree2Tree(data.tree, parentPath);
     const subtrees: { sha: string; parentPath: string }[] = [];
     for (const file of data.tree) {
       if (file.type == "tree" && file.path && file.sha) {
+        if (subtrees.length >= MAX_PENDING_TREE_FOLDERS) {
+          throw new AnonymousError("repository_tree_too_large", { httpStatus: 413 });
+        }
         subtrees.push({
           sha: file.sha,
           parentPath: path.join(parentPath, file.path),
@@ -766,11 +777,15 @@ export default class GitHubStream extends GitHubBase {
         GH_API_CONCURRENCY
       );
       for (const result of batchResults) {
-        output.push(...result.files);
-        queue.push(...result.children);
+        yield result.files;
+        for (const child of result.children) {
+          if (queue.length >= MAX_PENDING_TREE_FOLDERS) {
+            throw new AnonymousError("repository_tree_too_large", { httpStatus: 413 });
+          }
+          queue.push(child);
+        }
       }
     }
-    return output;
   }
 
   private tree2Tree(
@@ -790,14 +805,14 @@ export default class GitHubStream extends GitHubBase {
       if (pathFile === ".") {
         pathFile = "";
       }
-      return new FileModel({
+      return {
         name: basename(fullPath),
         path: pathFile,
         repoId: this.data.repoId,
         size: elem.size,
-        ...(elem.type === "commit" ? { submodule: "unsupported" } : {}),
+        ...(elem.type === "commit" ? { submodule: "unsupported" as const } : {}),
         sha: elem.sha,
-      });
+      };
     });
   }
 }

src/core/source/Source.ts

@@ -21,4 +21,15 @@ export interface SourceBase {
    * Get all the files from a specific source
    */
   getFiles(progress?: (status: string) => void): Promise<IFile[]>;
+
+  /** Read large trees without retaining every entry. */
+  iterateFiles?(progress?: (status: string) => void): AsyncIterable<IFile>;
+}
+
+/** Sum admission size without materializing a large source tree. */
+export async function sourceStorageSize(source: SourceBase): Promise<number> {
+  let bytes = 0;
+  const files = source.iterateFiles ? source.iterateFiles() : await source.getFiles();
+  for await (const file of files) bytes += file.size || 0;
+  return bytes;
 }

src/core/source/github-submodules.ts

@@ -36,12 +36,12 @@ export function submoduleRepository(url: unknown, parent: GitHubBaseData): { org
 }
 
 /** Resolve only direct public GitHub dependencies. Never send the parent token to a child. */
-export async function expandSubmodules(files: IFile[], data: GitHubBaseData): Promise<IFile[]> {
+export async function expandSubmodules(files: IFile[], data: GitHubBaseData,
+  budget = { modules: 0, remaining: MAX_SUBMODULE_ENTRIES }): Promise<IFile[]> {
   const output = files.filter(file => file.name !== ".gitmodules");
   const modules = output.filter(file => file.submodule);
-  let remaining = MAX_SUBMODULE_ENTRIES;
-  for (const [index, mount] of modules.entries()) {
-    if (index >= MAX_SUBMODULES) { mount.submodule = "limit"; continue; }
+  for (const mount of modules) {
+    if (budget.modules++ >= MAX_SUBMODULES) { mount.submodule = "limit"; continue; }
     const mountPath = [mount.path, mount.name].filter(Boolean).join("/");
     if (!safeRepositoryPath(mountPath) || !mount.sha || !/^[a-f0-9]{40}$/.test(mount.sha)) continue;
     const token = await data.getToken();
@@ -59,10 +59,10 @@ export async function expandSubmodules(files: IFile[], data: GitHubBaseData): Pr
       const child = await octokit("").git.getTree({
         owner: repository.organization, repo: repository.repoName, tree_sha: mount.sha, recursive: "1",
       });
-      if (child.data.truncated || child.data.tree.length > remaining) { mount.submodule = "limit"; continue; }
+      if (child.data.truncated || child.data.tree.length > budget.remaining) { mount.submodule = "limit"; continue; }
       const entries = child.data.tree;
       if (entries.some(entry => !safeRepositoryPath(entry.path))) { mount.submodule = "unsupported"; continue; }
-      remaining -= entries.length;
+      budget.remaining -= entries.length;
       mount.submodule = "included";
       mount.upstream = { ...repository, commit: mount.sha, path: "" };
       for (const entry of entries) {

src/scripts/create-performance-indexes.ts

@@ -17,6 +17,7 @@ async function main() {
   await File.collection.createIndex({ metadataPending: 1, repoId: 1, treeGeneration: 1 },
     { partialFilterExpression: { metadataPending: true } });
   await Repository.collection.createIndex({ retiredTreeGenerations: 1 }, { sparse: true });
+  await Repository.collection.createIndex({ retiredPathIndexKeys: 1 }, { sparse: true });
   await Repository.collection.createIndex({ "stagedFileTrees.until": 1 }, { sparse: true });
   await File.collection.createIndex({ treeStaged: 1, repoId: 1, treeGeneration: 1 },
     { partialFilterExpression: { treeStaged: true } });

src/server/routes/repository-private.ts

@@ -1,3 +1,5 @@
+import Repository from "../../core/Repository";
+import { sourceStorageSize } from "../../core/source/Source";
 import { neverExpirationDeadline, validateBillingExpiration } from "../../core/billing/expiration";
 import config from "../../config";
 import { assertConferenceAdmission, reserveRepository } from "../../core/billing/quotas";
@@ -535,19 +537,22 @@ router.post(
       const sourceChanged = reconnecting || hasRepositorySourceChanged(repo.model, repoUpdate);
       const previousAccessRevision = repo.model.githubAccess?.revision;
 
+      // Keep proposed settings separate until cleanup passes the persisted fences.
+      const proposed = new AnonymizedRepositoryModel(repo.model.toObject());
       if (repo.model.billingManaged) validateBillingExpiration(repoUpdate.options);
       if (config.BILLING_ENABLED && repoUpdate.options.expirationMode === "never" &&
           (repo.model.options.expirationMode !== "never" || repo.model.billingNeverExpiresUntil)) {
-        repo.model.billingNeverExpiresUntil = await neverExpirationDeadline(String(repo.model.owner),
+        proposed.billingNeverExpiresUntil = await neverExpirationDeadline(String(repo.model.owner),
           repo.model.options.expirationMode === "never" ? repo.model.billingNeverExpiresUntil : undefined);
       } else if (repoUpdate.options.expirationMode !== "never") {
-        repo.model.billingNeverExpiresUntil = undefined;
+        proposed.billingNeverExpiresUntil = undefined;
       }
-      updateRepoModel(repo.model, repoUpdate);
-      const reactivating = shouldReactivateInactiveRepository(repo.model);
+      updateRepoModel(proposed, repoUpdate);
+      const reactivating = shouldReactivateInactiveRepository(proposed);
+
 
       if (reactivating) {
-        repo.model.anonymizeDate = new Date();
+        proposed.anonymizeDate = new Date();
       }
 
       const needsAdmission = sourceChanged || reactivating || repoUpdate.conference !== repo.model.conference;
@@ -606,20 +611,22 @@ router.post(
         const commit = await repository.getCommitInfo(repoUpdate.source.commit, {
           accessToken: sourceAccess.token,
         });
-        if (reconnecting) repo.model.source.commitDate = commit.commit.committer?.date
+        if (reconnecting) proposed.source.commitDate = commit.commit.committer?.date
           ? new Date(commit.commit.committer.date) : undefined;
-        repo.model.githubAccess = { ...sourceAccess.binding, revision: randomUUID() };
-        repo.model.source.repositoryId = repository.model.id;
-        repo.model.source.repositoryName =
+        proposed.githubAccess = { ...sourceAccess.binding, revision: randomUUID() };
+        proposed.source.repositoryId = repository.model.id;
+        proposed.source.repositoryName =
           repository.fullName || repoUpdate.fullName;
-        repo.model.anonymizeDate = new Date();
+        proposed.anonymizeDate = new Date();
         if (repo.model.billingManaged) {
-          const files = await repo.source.getFiles();
-          await reserveRepository(String(repo.model.owner), repo.repoId,
-            files.reduce((sum, file) => sum + (file.size || 0), 0), repoUpdate.conference);
+          const storageBytes = await sourceStorageSize(new Repository(proposed).source);
+          await reserveRepository(String(repo.model.owner), repo.repoId, storageBytes, repoUpdate.conference);
         }
         await repo.remove({ accessRevision: previousAccessRevision, replacement: true });
       }
+      repo.model.set({ options: proposed.options, source: proposed.source,
+        githubAccess: proposed.githubAccess, anonymizeDate: proposed.anonymizeDate,
+        billingNeverExpiresUntil: proposed.billingNeverExpiresUntil });
 
       const removeRepoFromConference = async (conferenceID: string) => {
         const conf = await ConferenceModel.findOne({

src/server/routes/repository-public.ts

@@ -1,3 +1,4 @@
+import { FILE_LISTING_LIMIT } from "../../core/file-tree";
 import { repositoryFailureStatus } from "../../core/repository-failure-status";
 import { anonymizePath } from "../../core/anonymize-utils";
 import * as express from "express";
@@ -218,21 +219,30 @@ router.get(
         { $match: { repoId, treeGeneration: repo.model.treeGeneration || { $exists: false }, size: { $ne: null } } },
         { $project: { _id: 0, path: 1 } },
         { $group: { _id: "$path", count: { $sum: 1 } } },
+        { $limit: FILE_LISTING_LIMIT + 1 },
       ]).exec();
+      if (results.length > FILE_LISTING_LIMIT) throw new AnonymousError("file_listing_too_large", { httpStatus: 413 });
 
-      const directCounts: Record<string, number> = {};
+      const directCounts: Record<string, number> = Object.create(null);
       for (const r of results) {
         directCounts[r._id ?? ""] = r.count;
       }
 
-      const folderCounts: Record<string, number> = {};
+      const folderCounts: Record<string, number> = Object.create(null);
+      let folders = 0;
+      const addCount = (folder: string, count: number) => {
+        if (!Object.hasOwn(folderCounts, folder) && ++folders > FILE_LISTING_LIMIT) {
+          throw new AnonymousError("file_listing_too_large", { httpStatus: 413 });
+        }
+        folderCounts[folder] = (folderCounts[folder] || 0) + count;
+      };
       for (const [folder, count] of Object.entries(directCounts)) {
         let p = folder;
-        folderCounts[p] = (folderCounts[p] || 0) + count;
+        addCount(p, count);
         while (p) {
           const idx = p.lastIndexOf("/");
           p = idx >= 0 ? p.substring(0, idx) : "";
-          folderCounts[p] = (folderCounts[p] || 0) + count;
+          addCount(p, count);
           if (!p) break;
         }
       }
@@ -243,7 +253,7 @@ router.get(
           "../../core/anonymize-utils"
         );
         const compiled = compileTerms(terms);
-        const anonymized: Record<string, number> = {};
+        const anonymized: Record<string, number> = Object.create(null);
         for (const [folder, count] of Object.entries(folderCounts)) {
           const anonFolder = anonymizePathCompiled(folder, compiled);
           anonymized[anonFolder] = (anonymized[anonFolder] || 0) + count;

src/server/schedule.ts

@@ -158,7 +158,7 @@ export async function runRepositoryStatusCheck(now = new Date()) {
   await expireContent(AnonymizedPullRequestModel, data => new PullRequest(data), now);
 
   const retiredCursor = AnonymizedRepositoryModel.find({ $or: [
-    { retiredTreeGenerations: { $exists: true } }, { retiredContentPrefixes: { $exists: true } },
+    { retiredTreeGenerations: { $exists: true } }, { retiredPathIndexKeys: { $exists: true } }, { retiredContentPrefixes: { $exists: true } },
     { legacyContentCleanupPending: true, status: RepositoryStatus.READY },
     { "stagedFileTrees.until": { $lte: now } },
   ] }).cursor();