perf: bound request work and resource usage (#866)

* perf: bound request work and resource usage

* fix: address performance review regressions

* fix: guard content expiration and dashboard state

* fix: tolerate cache outages and retire derived paths

* fix: revalidate expiration claims and status pagination

* fix: retry interrupted repository metadata cleanup

* fix: retire obsolete content cache storage

* fix: activate tree metadata and recheck expiration claims

* fix: fence cache publication across expiration

* fix: recover staged trees and enforce cache lifecycle cleanup

* fix: prune expired transforms during database outages

* fix: finish conference removals and reload expiration candidates

src/core/AnonymizedFile.ts

@@ -1,3 +1,4 @@
+import { streamResponse } from "./response-stream";
 import { join, basename, dirname } from "path";
 import { Response } from "express";
 import { Readable } from "stream";
@@ -9,6 +10,8 @@ import { RepositoryStatus } from "./types";
 import config from "../config";
 import {
   anonymizePath,
+  anonymizePathCompiled,
+  compileTerms,
   hasCustomTermReplacement,
   isTextFile,
 } from "./anonymize-utils";
@@ -168,10 +171,12 @@ export default class AnonymizedFile {
     const exactQuery: FilterQuery<IFile> = {
       repoId: this.repository.repoId,
       path: fileDir,
+      treeGeneration: this.repository.model?.treeGeneration || { $exists: false },
     };
     if (filename != "") exactQuery.name = filename;
     const exact = await FileModel.findOne(exactQuery);
     if (exact) {
+      if (exact.metadataPending) await this.repository.completeRecoveredFileMetadata(exact);
       this._file = exact;
       return exact;
     }
@@ -203,6 +208,18 @@ export default class AnonymizedFile {
     // narrow Mongo query. Fetch the repository's paths and verify them by
     // re-applying anonymization. Default XXXX-N masks retain the optimized,
     // anchored query.
+    const compiled = compileTerms(terms);
+    if (typeof this.repository.findAnonymizedPath === "function") {
+      const indexed = await this.repository.findAnonymizedPath(this.anonymizedPath);
+      if (indexed) { this._file = indexed; return indexed; }
+      if (indexed === null) {
+        if (anonymizePathCompiled(this.anonymizedPath, compiled) === this.anonymizedPath) {
+          const recovered = await this.recoverTruncatedFile(fileDir);
+          if (recovered) { this._file = recovered; return recovered; }
+        }
+        throw new AnonymousError("file_not_found", { httpStatus: 404, object: this });
+      }
+    }
     const candidates = usesCustomReplacement
       ? await FileModel.find({ repoId: this.repository.repoId }).exec()
       : await FileModel.find({
@@ -214,8 +231,9 @@ export default class AnonymizedFile {
     for (const candidate of candidates) {
       const candidatePath = join(candidate.path, candidate.name);
       if (
-        anonymizePath(candidatePath, terms) == this.anonymizedPath
+        anonymizePathCompiled(candidatePath, compiled) == this.anonymizedPath
       ) {
+        if (candidate.metadataPending) await this.repository.completeRecoveredFileMetadata(candidate);
         this._file = candidate;
         return candidate;
       }
@@ -253,19 +271,24 @@ export default class AnonymizedFile {
     const recovered = await source.fetchFileInfoFromPath(this.anonymizedPath);
     if (!recovered) return null;
     recovered.repoId = this.repository.repoId;
+    recovered.treeGeneration = this.repository.model.treeGeneration;
     logger.info("recovered file from truncated tree", {
       repoId: this.repository.repoId,
       path: this.anonymizedPath,
     });
+    let cached;
     try {
       // Cache it so the next request is served from the database.
-      await FileModel.create(recovered);
+      cached = await FileModel.create({ ...recovered, metadataPending: true });
     } catch (error) {
       logger.warn(
         "failed to cache recovered file",
         serializeError(error as Error)
       );
     }
+    // Once persisted, invalidation is required. Keep the pending marker on
+    // failure so both a subsequent lookup and maintenance retry the work.
+    if (cached) await this.repository.completeRecoveredFileMetadata(cached);
     return recovered;
   }
 
@@ -360,6 +383,8 @@ export default class AnonymizedFile {
         token: await githubTokenForStreamer(await this.repository.getToken(), this.repository.model.source.repositoryName),
         repoFullName: this.repository.model.source.repositoryName,
         commit: this.repository.model.source.commit,
+        cacheGeneration: `${this.repository.model.treeGeneration || "legacy"}:${this.repository.model.anonymizeDate?.toISOString() || ""}`,
+        cacheRevision: this.repository.model.contentCacheRevision,
         branch: this.repository.model.source.branch,
         repoId: this.repository.repoId,
         filePath: this.filePath,
@@ -407,24 +432,13 @@ export default class AnonymizedFile {
                 token: await githubTokenForStreamer(token, this.repository.model.source.repositoryName),
                 repoFullName: this.repository.model.source.repositoryName,
                 commit: this.repository.model.source.commit,
+        cacheGeneration: `${this.repository.model.treeGeneration || "legacy"}:${this.repository.model.anonymizeDate?.toISOString() || ""}`,
+                cacheRevision: this.repository.model.contentCacheRevision,
                 branch: this.repository.model.source.branch,
                 repoId: this.repository.repoId,
                 filePath: this.filePath,
                 anonymizerOptions: anonymizer.opt,
               },
-            })
-            .on("error", (err: Error) => {
-              const { error } = streamerErrorToAnonymous(
-                err as Error & {
-                  response?: { statusCode?: number; body?: unknown };
-                },
-                {
-                  repoId: this.repository.repoId,
-                  filePath: this.anonymizedPath,
-                }
-              );
-              error.value = this;
-              handleError(error, res);
             });
           // Forward Content-Type from the streamer's upstream response.
           // got.stream(...).pipe(res) forwards body bytes only — without
@@ -441,20 +455,21 @@ export default class AnonymizedFile {
               else if (isTextFile(this.anonymizedPath)) res.contentType("text/plain");
             }
           });
-          resStream.pipe(res);
-          // Resolve as soon as the response is fully written rather than
-          // waiting for the socket to close — keep-alive sockets stay open
-          // long after the body is delivered, and we don't want to delay
-          // post-send work like countView() that long.
-          res.on("finish", () => {
-            resolve();
-          });
-          res.on("close", () => {
-            resolve();
-          });
-          res.on("error", (err) => {
-            reject(err);
+          await streamResponse(resStream, res, (err: Error) => {
+              const { error } = streamerErrorToAnonymous(
+                err as Error & {
+                  response?: { statusCode?: number; body?: unknown };
+                },
+                {
+                  repoId: this.repository.repoId,
+                  filePath: this.anonymizedPath,
+                }
+              );
+              error.value = this;
+              handleError(error, res);
+              if (res.headersSent && !res.writableEnded) res.destroy();
           });
+          resolve();
           return;
         }
 
@@ -480,6 +495,7 @@ export default class AnonymizedFile {
           }
         });
         const content = await this.content();
+        if (res.destroyed || res.writableEnded) { content.destroy(); anonymizer.destroy(); resolve(); return; }
         function handleStreamError(error: Error) {
           if (!content.closed && !content.destroyed) {
             content.destroy();
@@ -505,6 +521,7 @@ export default class AnonymizedFile {
             if (!content.closed && !content.destroyed) {
               content.destroy();
             }
+            anonymizer.destroy();
             resolve();
           });
       } catch (error) {

src/core/Conference.ts

@@ -33,9 +33,7 @@ export default class Conference {
    * Expire the conference
    */
   async expire() {
-    await Promise.all(
-      (await this.repositories()).map(async (conf) => await conf.expire())
-    );
+    for (const repo of await this.repositories()) await repo.markExpired(true);
     await this.updateStatus("expired");
   }
 
@@ -44,9 +42,12 @@ export default class Conference {
    */
   async remove() {
     await this.updateStatus("removed");
-    await Promise.all(
-      (await this.repositories()).map(async (conf) => await conf.remove())
-    );
+    const failures: unknown[] = [];
+    for (const repo of await this.repositories()) {
+      try { await repo.remove(); }
+      catch (error) { failures.push(error); }
+    }
+    if (failures.length) throw new AggregateError(failures, "Conference repository removal failed");
   }
 
   /**

src/core/Gist.ts

@@ -1,3 +1,5 @@
+import { isConnected } from "../server/database";
+import { expireEmbeddedContent } from "./content-expiration";
 import { APP_PROVIDER, appError } from "./github-app";
 import CredentialModel from "./model/credentials/credentials.model";
 import { getCredentialToken } from "./credentials";
@@ -131,7 +133,7 @@ export default class Gist {
       this._model.options.expirationDate
     ) {
       if (this._model.options.expirationDate <= new Date()) {
-        await this.expire();
+        await this.markExpired();
       }
     }
     if (
@@ -217,10 +219,29 @@ export default class Gist {
     ).exec();
   }
 
+  async markExpired() {
+    const now = new Date();
+    if (isConnected) {
+      const result = await AnonymizedGistModel.updateOne({ _id: this.model._id,
+        status: RepositoryStatus.READY, "options.expirationMode": { $ne: "never" },
+        "options.expirationDate": { $lte: now },
+      }, { $set: { status: RepositoryStatus.EXPIRING, statusDate: now } }).exec();
+      if (!result.matchedCount) {
+        const current = await AnonymizedGistModel.findById(this.model._id).exec();
+        if (!current) throw new AnonymousError("gist_expired", { object: this, httpStatus: 410 });
+        this._model = current;
+        this._gistPayload = undefined;
+        return;
+      }
+    }
+    this.model.status = RepositoryStatus.EXPIRING;
+    this.model.statusDate = now;
+  }
+
   async expire() {
-    await this.updateStatus(RepositoryStatus.EXPIRING);
-    await this.resetSate();
-    await this.updateStatus(RepositoryStatus.EXPIRED);
+    await expireEmbeddedContent(AnonymizedGistModel, this.model, {
+      "gist.comments": [], "gist.description": "", "gist.files": [], "gist.ownerLogin": "",
+    });
   }
 
   async remove() {

src/core/PullRequest.ts

@@ -1,3 +1,5 @@
+import { isConnected } from "../server/database";
+import { expireEmbeddedContent } from "./content-expiration";
 import { boundAppToken } from "./github-app";
 import { getCredentialToken } from "./credentials";
 import { RepositoryStatus } from "./types";
@@ -121,7 +123,7 @@ export default class PullRequest {
       this._model.options.expirationDate
     ) {
       if (this._model.options.expirationDate <= new Date()) {
-        await this.expire();
+        await this.markExpired();
       }
     }
     if (
@@ -228,10 +230,30 @@ export default class PullRequest {
   /**
    * Expire the pullRequest
    */
+  async markExpired() {
+    const now = new Date();
+    if (isConnected) {
+      const result = await AnonymizedPullRequestModel.updateOne({ _id: this.model._id,
+        status: RepositoryStatus.READY, "options.expirationMode": { $ne: "never" },
+        "options.expirationDate": { $lte: now },
+      }, { $set: { status: RepositoryStatus.EXPIRING, statusDate: now } }).exec();
+      if (!result.matchedCount) {
+        const current = await AnonymizedPullRequestModel.findById(this.model._id).exec();
+        if (!current) throw new AnonymousError("pull_request_expired", { object: this, httpStatus: 410 });
+        this._model = current;
+        return;
+      }
+    }
+    this.model.status = RepositoryStatus.EXPIRING;
+    this.model.statusDate = now;
+  }
+
   async expire() {
-    await this.updateStatus(RepositoryStatus.EXPIRING);
-    await this.resetSate();
-    await this.updateStatus(RepositoryStatus.EXPIRED);
+    await expireEmbeddedContent(AnonymizedPullRequestModel, this.model, {
+      "pullRequest.comments": [], "pullRequest.body": "", "pullRequest.title": "", "pullRequest.diff": "",
+      "pullRequest.baseRepositoryFullName": "", "pullRequest.headRepositoryFullName": "",
+      "pullRequest.merged": false, "pullRequest.state": "closed", "pullRequest.draft": false,
+    }, { "pullRequest.mergedDate": "" });
   }
 
   /**

src/core/Repository.ts

@@ -1,5 +1,8 @@
+import AnonymizedPathModel from "./model/anonymized-path";
 import storage from "./storage";
-import { randomUUID } from "crypto";
+import { contentGenerationPrefix, contentRetirementMarker } from "./content-generation";
+import { removeTransformedRepositoryCache } from "./transformed-cache";
+import { createHash, randomUUID } from "crypto";
 import { RepositoryStatus } from "./types";
 import { Readable } from "stream";
 import * as sha1 from "crypto-js/sha1";
@@ -31,7 +34,36 @@ import AnonymizedRepositoryModel from "./model/anonymizedRepositories/anonymized
 import { createLogger, serializeError } from "./logger";
 
 const logger = createLogger("repository");
-import { IFile } from "./model/files/files.types";
+const pathIndexes = new Map<string, Promise<void>>();
+async function buildPathIndex(repoId: string, key: string, terms: string[], generation?: string) {
+  const cacheKey = `${repoId}:${key}`;
+  const existing = pathIndexes.get(cacheKey);
+  if (existing) return existing;
+  const task = (async () => {
+    const compiled = compileTerms(terms);
+    const cursor = FileModel.find({ repoId, treeGeneration: generation || { $exists: false } })
+      .select("name path sha size").sort({ path: 1, name: 1 }).lean().cursor();
+    let batch: Parameters<typeof AnonymizedPathModel.bulkWrite>[0] = [];
+    try {
+      for await (const file of cursor) {
+        const name = anonymizePathCompiled(file.name, compiled);
+        const directory = anonymizePathCompiled(file.path, compiled);
+        batch.push({ updateOne: { filter: { repoId, key, originalId: file._id }, update: { $set: {
+          repoId, key, originalId: file._id, name: file.name, path: file.path, sha: file.sha, size: file.size,
+          anonymousName: name, anonymousDirectory: directory, createdAt: new Date(),
+          searchName: name.toLowerCase(), searchDirectory: directory.toLowerCase(),
+          anonymousPath: anonymizePathCompiled(file.path ? `${file.path}/${file.name}` : file.name, compiled),
+        } }, upsert: true } });
+        if (batch.length === 250) { await AnonymizedPathModel.bulkWrite(batch); batch = []; }
+      }
+      if (batch.length) await AnonymizedPathModel.bulkWrite(batch);
+    } finally { await cursor.close(); }
+  })();
+  // Retain only in-flight work. A subsequent recovery can add file rows.
+  pathIndexes.set(cacheKey, task);
+  try { await task; } finally { pathIndexes.delete(cacheKey); }
+}
+import { IFile, IFileDocument } from "./model/files/files.types";
 import AnonymizedFile from "./AnonymizedFile";
 import { FilterQuery } from "mongoose";
 function anonymizeTreeRecursive(
@@ -87,6 +119,8 @@ export default class Repository {
     return new GitHubStream({
       repoId: this.repoId,
       commit: this.model.source.commit || "HEAD",
+      cacheGeneration: `${this.model.treeGeneration || "legacy"}:${this.model.anonymizeDate?.toISOString() || ""}`,
+      cacheRevision: this.model.contentCacheRevision,
       organization: ghRepo.owner,
       repoName: ghRepo.repo,
       getToken: () => this.getToken(),
@@ -133,51 +167,8 @@ export default class Repository {
       force: false,
     }
   ): Promise<IFile[]> {
-    this.assertNotArchived();
+    await this.ensureFileTree(opt);
     const terms = this._model.options.terms || [];
-    let hasFile = await FileModel.exists({ repoId: this.repoId }).exec();
-    // Files created by GitHubDownload don't carry a valid 40-char GitHub
-    // blob SHA.  When the source type later switches to GitHubStream the
-    // stale entries cause blob-API 404s.  Detect this by sampling a file
-    // with a sha and checking its length; force a re-fetch if it doesn't
-    // look like a GitHub SHA.
-    if (hasFile && this.source instanceof GitHubStream) {
-      const sample = await FileModel.findOne(
-        { repoId: this.repoId, sha: { $exists: true, $ne: null } },
-        { sha: 1 }
-      ).exec();
-      if (sample?.sha && sample.sha.length !== 40) {
-        hasFile = null;
-      }
-    }
-    if (!hasFile || opt.force) {
-      await FileModel.deleteMany({ repoId: this.repoId }).exec();
-      const source = this.source;
-      const files = await source.getFiles(opt.progress);
-      files.forEach((f) => (f.repoId = this.repoId));
-      await FileModel.insertMany(files);
-
-      const sourceWithTruncation = source as unknown as {
-        truncatedFolderList?: string[];
-      };
-      if (Array.isArray(sourceWithTruncation.truncatedFolderList)) {
-        this._model.truncatedFolders = sourceWithTruncation.truncatedFolderList;
-      }
-
-      this._model.size = { storage: 0, file: 0 };
-      await this.computeSize();
-      if (isConnected) {
-        await AnonymizedRepositoryModel.updateOne(
-          { _id: this._model._id },
-          {
-            $set: {
-              truncatedFolders: this._model.truncatedFolders,
-              size: this._model.size,
-            },
-          }
-        ).exec();
-      }
-    }
     if (
       opt.path &&
       (opt.path.includes(config.ANONYMIZATION_MASK) ||
@@ -202,6 +193,7 @@ export default class Repository {
 
     const query: FilterQuery<IFile> = {
       repoId: this.repoId,
+      treeGeneration: this.model.treeGeneration || { $exists: false },
     };
     if (pathQuery !== undefined) {
       query.path = pathQuery;
@@ -209,6 +201,268 @@ export default class Repository {
     return await FileModel.find(query).exec();
   }
 
+  private async ensureFileTree(opt: { force?: boolean; progress?: (status: string) => void } = {}) {
+    this.assertNotArchived();
+    let hasFile = this.hasEmptyTree() || await FileModel.exists({ repoId: this.repoId, treeGeneration: this.model.treeGeneration || { $exists: false } }).exec();
+    // Files created by GitHubDownload don't carry a valid 40-char GitHub
+    // blob SHA.  When the source type later switches to GitHubStream the
+    // stale entries cause blob-API 404s.  Detect this by sampling a file
+    // with a sha and checking its length; force a re-fetch if it doesn't
+    // look like a GitHub SHA.
+    if (hasFile && this.source instanceof GitHubStream) {
+      const sample = await FileModel.findOne(
+        { repoId: this.repoId, treeGeneration: this.model.treeGeneration || { $exists: false }, sha: { $exists: true, $ne: null } },
+        { sha: 1 }
+      ).exec();
+      if (sample?.sha && sample.sha.length !== 40) {
+        hasFile = null;
+      }
+    }
+    if (!hasFile || opt.force) {
+      const previousGeneration = this.model.treeGeneration;
+      const previousPrefix = contentGenerationPrefix(`${previousGeneration || "legacy"}:${this.model.anonymizeDate?.toISOString() || ""}`);
+      const streamed = this.model.source.type !== "Zip";
+      const source = this.source;
+      const files = await source.getFiles(opt.progress);
+      const sourceWithTruncation = source as unknown as { truncatedFolderList?: string[] };
+      const truncatedFolders = Array.isArray(sourceWithTruncation.truncatedFolderList)
+        ? [...sourceWithTruncation.truncatedFolderList] : [];
+      const generation = randomUUID();
+      const buildFilter = { _id: this.model._id,
+        treeGeneration: previousGeneration || { $exists: false }, ...this.refreshFilter(),
+        status: this.model.status, statusDate: this.model.statusDate,
+        "githubAccess.revision": this.model.githubAccess?.revision || { $exists: false },
+      };
+      let heartbeat: NodeJS.Timeout | undefined;
+      if (isConnected) {
+        // Register before insertion so a crash cannot leave an unknown UUID.
+        const registered = await AnonymizedRepositoryModel.updateOne(buildFilter, { $push: {
+          stagedFileTrees: { generation, until: new Date(Date.now() + 15 * 60_000) },
+        } }).exec();
+        if (!registered.matchedCount) throw new AnonymousError("repository_changed", { httpStatus: 409 });
+        heartbeat = setInterval(() => {
+          const now = new Date();
+          void AnonymizedRepositoryModel.updateOne({ ...buildFilter,
+            stagedFileTrees: { $elemMatch: { generation, until: { $gt: now } } },
+          }, { $set: { "stagedFileTrees.$.until": new Date(now.getTime() + 15 * 60_000) } }).exec().catch(() => {});
+        }, 30_000);
+        heartbeat.unref();
+      }
+      files.forEach(f => { f.repoId = this.repoId; f.treeGeneration = generation; if (isConnected) f.treeStaged = true; });
+      try {
+        await FileModel.insertMany(files);
+        if (isConnected) {
+          const activated = await AnonymizedRepositoryModel.updateOne({ _id: this.model._id,
+            treeGeneration: previousGeneration || { $exists: false }, ...this.refreshFilter(),
+            status: this.model.status, statusDate: this.model.statusDate,
+            "githubAccess.revision": this.model.githubAccess?.revision || { $exists: false },
+            stagedFileTrees: { $elemMatch: { generation, until: { $gt: new Date() } } },
+          }, { $set: { treeGeneration: generation, truncatedFolders, size: { storage: 0, file: 0 },
+            ...(streamed && !this.model.contentCacheVersion ? { contentCacheVersion: 1, legacyContentCleanupPending: true } : {}),
+            ...(!files.length ? { emptyTreeGeneration: generation } : {}),
+          }, $unset: { pathIndexKey: "", pathIndexBuiltAt: "", sizeComputedAt: "",
+            ...(files.length ? { emptyTreeGeneration: "" } : {}),
+          }, $addToSet: { retiredTreeGenerations: previousGeneration || "",
+            ...(streamed ? { retiredContentPrefixes: previousPrefix } : {}),
+          }, $pull: { stagedFileTrees: { generation } } }).exec();
+          if (!activated.matchedCount) {
+            await FileModel.deleteMany({ repoId: this.repoId, treeGeneration: generation }).exec();
+            throw new AnonymousError("repository_changed", { httpStatus: 409 });
+          }
+        }
+      } finally { clearInterval(heartbeat); }
+      this.model.treeGeneration = generation;
+      if (isConnected) {
+        // A crash here is safe: maintenance recognizes the active generation.
+        await FileModel.updateMany({ repoId: this.repoId, treeGeneration: generation, treeStaged: true },
+          { $unset: { treeStaged: "" } }).exec().catch(error => logger.warn("staged tree completion deferred", serializeError(error)));
+      }
+      this.model.emptyTreeGeneration = files.length ? undefined : generation;
+      this.model.truncatedFolders = truncatedFolders;
+      if (streamed) this.model.contentCacheVersion = 1;
+      if (isConnected) {
+        await this.cleanupRetiredFileTrees().catch(error => logger.warn("retired tree cleanup deferred", serializeError(error)));
+      } else {
+        await FileModel.deleteMany({ repoId: this.repoId, treeGeneration: previousGeneration || { $exists: false } }).exec();
+      }
+
+      this._model.size = { storage: 0, file: 0 };
+      this._model.sizeComputedAt = undefined;
+      await this.computeSize();
+      if (isConnected) {
+        await AnonymizedRepositoryModel.updateOne(
+          { _id: this._model._id, treeGeneration: this.model.treeGeneration },
+          {
+            $set: {
+              treeGeneration: this._model.treeGeneration,
+              size: this._model.size,
+            },
+          }
+        ).exec();
+      }
+    }
+  }
+
+  /** Activation records retirement atomically; maintenance retries interrupted deletion. */
+  async cleanupRetiredFileTrees(now = new Date()) {
+    const stages = await AnonymizedRepositoryModel.findById(this.model._id).select("stagedFileTrees").lean().exec();
+    for (const stage of stages?.stagedFileTrees || []) {
+      if (stage.until > now) continue;
+      await AnonymizedRepositoryModel.updateOne({ _id: this.model._id, treeGeneration: { $ne: stage.generation },
+        stagedFileTrees: { $elemMatch: { generation: stage.generation, until: { $lte: now } } },
+      }, { $pull: { stagedFileTrees: { generation: stage.generation } },
+        $addToSet: { retiredTreeGenerations: stage.generation } }).exec();
+    }
+    const current = await AnonymizedRepositoryModel.findById(this.model._id)
+      .select("treeGeneration retiredTreeGenerations retiredContentPrefixes legacyContentCleanupPending source.type status anonymizeDate").lean().exec();
+    if (!current) return;
+    const activePrefix = contentGenerationPrefix(`${current.treeGeneration || "legacy"}:${current.anonymizeDate?.toISOString() || ""}`);
+    const prefixes = (current.retiredContentPrefixes || []).filter(prefix => prefix !== activePrefix);
+    for (const prefix of prefixes) {
+      // Fence producers before removal. A late publisher rechecks this marker
+      // and removes its own output instead of recreating a retired cache.
+      await storage.write(this.repoId, contentRetirementMarker(prefix), "retired");
+      await storage.rm(this.repoId, prefix);
+    }
+    if (prefixes.length) await AnonymizedRepositoryModel.updateOne({ _id: current._id },
+      { $pull: { retiredContentPrefixes: { $in: prefixes } } }).exec();
+    if (current.legacyContentCleanupPending && current.status === RepositoryStatus.READY && current.source?.type !== "Zip") {
+      await storage.removeLegacyContent(this.repoId);
+      await AnonymizedRepositoryModel.updateOne({ _id: current._id }, { $unset: { legacyContentCleanupPending: "" } }).exec();
+    }
+    const retired = (current.retiredTreeGenerations || []).filter(generation => generation !== (current.treeGeneration || ""));
+    if (retired.length) {
+      await FileModel.deleteMany({ repoId: this.repoId, $or: [
+        { treeGeneration: { $in: retired.filter(Boolean) } },
+        ...(retired.includes("") ? [{ treeGeneration: { $exists: false } }] : []),
+      ] }).exec();
+      await AnonymizedRepositoryModel.updateOne({ _id: this.model._id },
+        { $pull: { retiredTreeGenerations: { $in: retired } } }).exec();
+    }
+    await AnonymizedRepositoryModel.updateOne({ _id: this.model._id, "retiredTreeGenerations.0": { $exists: false } },
+      { $unset: { retiredTreeGenerations: "" } }).exec();
+    await AnonymizedRepositoryModel.updateOne({ _id: this.model._id, "retiredContentPrefixes.0": { $exists: false } },
+      { $unset: { retiredContentPrefixes: "" } }).exec();
+    await AnonymizedRepositoryModel.updateOne({ _id: this.model._id, "stagedFileTrees.0": { $exists: false } },
+      { $unset: { stagedFileTrees: "" } }).exec();
+  }
+
+  /** Indexed row markers also discover inserts that finish after a build lease is retired. */
+  static async cleanupStagedFileTree(repoId: string, generation: string, now = new Date()) {
+    let current = await AnonymizedRepositoryModel.findOne({ repoId }).exec();
+    if (current && current.treeGeneration !== generation &&
+      current.stagedFileTrees?.some(stage => stage.generation === generation && stage.until > now)) return;
+    if (current) {
+      await new Repository(current).cleanupRetiredFileTrees(now);
+      current = await AnonymizedRepositoryModel.findById(current._id).exec();
+    }
+    const terminal = current?.status && [RepositoryStatus.EXPIRING, RepositoryStatus.EXPIRED,
+      RepositoryStatus.REMOVING, RepositoryStatus.REMOVED].includes(current.status);
+    if (current?.treeGeneration === generation && !terminal) {
+      await FileModel.updateMany({ repoId, treeGeneration: generation, treeStaged: true }, { $unset: { treeStaged: "" } }).exec();
+    } else if (terminal || !current?.stagedFileTrees?.some(stage => stage.generation === generation && stage.until > now)) {
+      await FileModel.deleteMany({ repoId, treeGeneration: generation, treeStaged: true }).exec();
+    }
+  }
+
+  private pathKey() {
+    return createHash("sha256").update(JSON.stringify([
+      "paths-v2", config.ANONYMIZATION_MASK, this.options.terms,
+      this.model.treeGeneration, this.model.fileMetadataRevision, this.model.source.commit, this.model.anonymizeDate,
+    ])).digest("hex");
+  }
+
+  private async indexPaths() {
+    if (this.status && [RepositoryStatus.EXPIRING, RepositoryStatus.EXPIRED,
+      RepositoryStatus.REMOVING, RepositoryStatus.REMOVED].includes(this.status)) {
+      throw new AnonymousError("repository_changed", { httpStatus: 409 });
+    }
+    await this.ensureFileTree();
+    const metadataRevision = this.model.fileMetadataRevision;
+    const key = this.pathKey();
+    if (this.model.pathIndexKey !== key || !this.model.pathIndexBuiltAt || this.model.pathIndexBuiltAt.getTime() < Date.now() - 24 * 3600_000) {
+      try {
+        await buildPathIndex(this.repoId, key, this.options.terms || [], this.model.treeGeneration);
+        const builtAt = new Date();
+        const result = await AnonymizedRepositoryModel.updateOne({ _id: this.model._id,
+          status: this.model.status, statusDate: this.model.statusDate,
+          treeGeneration: this.model.treeGeneration || { $exists: false },
+          fileMetadataRevision: metadataRevision || { $exists: false },
+          "source.commit": this.model.source.commit || { $exists: false },
+          "options.terms": this.options.terms || [], anonymizeDate: this.model.anonymizeDate,
+        }, { $set: { pathIndexKey: key, pathIndexBuiltAt: builtAt } }).exec();
+        if (!result.matchedCount) throw new AnonymousError("repository_changed", { httpStatus: 409 });
+        this.model.pathIndexKey = key;
+        this.model.pathIndexBuiltAt = builtAt;
+      } catch (error) {
+        // A reset invalidates this revision. Never delete a valid shared key
+        // merely because one of its concurrent builders failed.
+        const current = await AnonymizedRepositoryModel.exists({ _id: this.model._id,
+          fileMetadataRevision: metadataRevision || { $exists: false },
+        }).exec();
+        if (!current) await AnonymizedPathModel.deleteMany({ repoId: this.repoId, key }).exec();
+        throw error;
+      }
+      // TTL retires superseded mappings. An older builder must never delete
+      // mappings activated by a newer settings or metadata generation.
+    }
+    return key;
+  }
+
+  async findAnonymizedPath(path: string): Promise<IFile | null | undefined> {
+    if (!isConnected) return undefined;
+    const key = await this.indexPaths();
+    // Original paths retain priority; collisions between transformed paths
+    // resolve consistently in original path/name order.
+    return AnonymizedPathModel.findOne({ repoId: this.repoId, key, anonymousPath: path })
+      .sort({ path: 1, name: 1 }).lean().exec();
+  }
+
+  async invalidateFileMetadata() {
+    const revision = randomUUID();
+    this.model.pathIndexKey = undefined;
+    this.model.pathIndexBuiltAt = undefined;
+    this.model.sizeComputedAt = undefined;
+    this.model.size = { storage: 0, file: 0 };
+    if (isConnected) {
+      const result = await AnonymizedRepositoryModel.updateOne({ _id: this.model._id,
+        treeGeneration: this.model.treeGeneration || { $exists: false },
+        status: this.model.status, statusDate: this.model.statusDate,
+        anonymizeDate: this.model.anonymizeDate,
+      }, { $set: { size: this.model.size, fileMetadataRevision: revision }, $unset: { pathIndexKey: "", pathIndexBuiltAt: "", sizeComputedAt: "" } }).exec();
+      if (!result.matchedCount) throw new AnonymousError("repository_changed", { httpStatus: 409 });
+    }
+    this.model.fileMetadataRevision = revision;
+  }
+
+  async completeRecoveredFileMetadata(file: IFileDocument) {
+    await this.invalidateFileMetadata();
+    await FileModel.updateOne({ _id: file._id, metadataPending: true },
+      { $unset: { metadataPending: "" } }).exec();
+    file.metadataPending = undefined;
+  }
+
+  async searchFiles(query: string): Promise<Partial<IFile>[]> {
+    const q = query.toLowerCase();
+    if (q.includes("/")) return [];
+    if (isConnected) {
+      const key = await this.indexPaths();
+      const pattern = new RegExp(q.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"));
+      const rows = await AnonymizedPathModel.find({ repoId: this.repoId, key,
+        $or: [{ searchName: pattern }, { searchDirectory: pattern }] })
+        .select("anonymousName anonymousDirectory size").limit(500).lean().exec();
+      return rows.map(f => ({ name: f.anonymousName, path: f.anonymousDirectory, size: f.size }));
+    }
+    const matches: Partial<IFile>[] = [];
+    for (const f of await this.anonymizedFiles({ includeSha: false, recursive: true })) {
+      if (f.name?.toLowerCase().includes(q) || (f.path || "").split("/").some(segment => segment.toLowerCase().includes(q))) {
+        matches.push({ name: f.name, path: f.path, size: f.size });
+        if (matches.length === 500) break;
+      }
+    }
+    return matches;
+  }
+
   /**
    * Check the status of the repository
    */
@@ -226,7 +480,7 @@ export default class Repository {
       this._model.options.expirationDate
     ) {
       if (this._model.options.expirationDate <= new Date()) {
-        await this.expire();
+        await this.markExpired();
       }
     }
     if (
@@ -280,7 +534,9 @@ export default class Repository {
   generateAnonymizeTransformer(filePath: string) {
     return new AnonymizeTransformer({
       filePath: filePath,
+      cacheGeneration: `${this.model.treeGeneration || "legacy"}:${this.model.anonymizeDate?.toISOString() || ""}`,
       terms: this.options.terms,
+      cacheRevision: this.model.contentCacheRevision,
       image: this.options.image,
       link: this.options.link,
       repoId: this.repoId,
@@ -291,7 +547,7 @@ export default class Repository {
 
   async isReady() {
     if (this.status !== RepositoryStatus.READY) return false;
-    if (!(await FileModel.exists({ repoId: this.repoId }).exec())) {
+    if (!this.hasEmptyTree() && !(await FileModel.exists({ repoId: this.repoId, treeGeneration: this.model.treeGeneration || { $exists: false } }).exec())) {
       this.model.status = RepositoryStatus.PREPARING;
       await this.updateIfNeeded({ force: true });
       return false;
@@ -299,6 +555,10 @@ export default class Repository {
     return true;
   }
 
+  private hasEmptyTree() {
+    return !!this.model.treeGeneration && this.model.emptyTreeGeneration === this.model.treeGeneration;
+  }
+
   private refreshToken?: string;
 
   private refreshFilter() {
@@ -578,10 +838,85 @@ export default class Repository {
   /**
    * Expire the repository
    */
+  async markExpired(force = false) {
+    const inactive = [RepositoryStatus.ARCHIVED, RepositoryStatus.REMOVED,
+      RepositoryStatus.REMOVING, RepositoryStatus.EXPIRED, RepositoryStatus.EXPIRING];
+    if (force && this.status && inactive.includes(this.status)) return;
+    const now = new Date();
+    if (isConnected) {
+      const result = await AnonymizedRepositoryModel.updateOne({ _id: this.model._id,
+        status: force ? { $nin: inactive } : RepositoryStatus.READY, ...(force ? {} : { "options.expirationMode": { $ne: "never" },
+        "options.expirationDate": { $lte: now } }),
+      }, { $set: { status: RepositoryStatus.EXPIRING, statusDate: now } }).exec();
+      if (!result.matchedCount) {
+        if (force) return;
+        const current = await AnonymizedRepositoryModel.findById(this.model._id).exec();
+        if (!current) throw new AnonymousError("repository_expired", { object: this, httpStatus: 410 });
+        this._model = current;
+        this.assertNotArchived();
+        return;
+      }
+    }
+    this.model.status = RepositoryStatus.EXPIRING;
+    this.model.statusDate = now;
+  }
+
   async expire() {
-    await this.updateStatus(RepositoryStatus.EXPIRING);
-    await this.resetSate();
-    await this.updateStatus(RepositoryStatus.EXPIRED);
+    if (!isConnected) {
+      await this.updateStatus(RepositoryStatus.EXPIRING);
+      await this.resetSate();
+      await this.updateStatus(RepositoryStatus.EXPIRED);
+      return;
+    }
+    const now = new Date(), token = randomUUID(), revision = randomUUID();
+    const pending = this.status === RepositoryStatus.EXPIRING || this.status === RepositoryStatus.EXPIRED;
+    if (!pending && (this.options.expirationMode === "never" || !this.options.expirationDate
+      || this.options.expirationDate > now)) return;
+    if (this.status && [RepositoryStatus.ARCHIVED, RepositoryStatus.REMOVING, RepositoryStatus.REMOVED].includes(this.status)) return;
+    const claimed = await AnonymizedRepositoryModel.updateOne({ _id: this.model._id,
+      status: this.model.status, statusDate: this.model.statusDate,
+      anonymizeDate: this.model.anonymizeDate,
+      treeGeneration: this.model.treeGeneration || { $exists: false },
+      fileMetadataRevision: this.model.fileMetadataRevision || { $exists: false },
+      settingsSavedAt: this.model.settingsSavedAt || { $exists: false },
+      "options.expirationMode": this.options.expirationMode || { $exists: false },
+      "options.expirationDate": this.options.expirationDate || { $exists: false },
+      "githubAccess.revision": this.model.githubAccess?.revision || { $exists: false },
+      $or: [{ cleanupUntil: { $exists: false } }, { cleanupUntil: { $lte: now } }],
+    }, { $set: { status: RepositoryStatus.EXPIRING, statusDate: now,
+      fileMetadataRevision: revision, contentCacheRevision: revision, cleanupToken: token, cleanupUntil: new Date(now.getTime() + 15 * 60_000) },
+      $unset: { pathIndexKey: "", pathIndexBuiltAt: "" },
+    }).exec();
+    if (!claimed.matchedCount) return;
+    this.model.status = RepositoryStatus.EXPIRING;
+    this.model.statusDate = now;
+    this.model.fileMetadataRevision = revision;
+    this.model.contentCacheRevision = revision;
+    this.model.pathIndexKey = undefined;
+    this.model.pathIndexBuiltAt = undefined;
+    const lease = { _id: this.model._id, status: RepositoryStatus.EXPIRING, cleanupToken: token,
+      treeGeneration: this.model.treeGeneration || { $exists: false }, fileMetadataRevision: revision };
+    const heartbeat = setInterval(() => {
+      void AnonymizedRepositoryModel.updateOne(lease, { $set: { cleanupUntil: new Date(Date.now() + 15 * 60_000) } }).exec().catch(() => {});
+    }, 30_000);
+    heartbeat.unref();
+    try {
+      // The claimed lifecycle blocks restoration until the entire storage
+      // root, including empty directories and older cache generations, is gone.
+      if (!await AnonymizedRepositoryModel.exists(lease).exec()) return;
+      await removeTransformedRepositoryCache(this.repoId);
+      await storage.rm(this.repoId);
+      if (!await AnonymizedRepositoryModel.exists(lease).exec()) return;
+      await FileModel.deleteMany({ repoId: this.repoId }).exec();
+      await AnonymizedPathModel.deleteMany({ repoId: this.repoId }).exec();
+      const result = await AnonymizedRepositoryModel.updateOne(lease, { $set: { status: RepositoryStatus.EXPIRED,
+        statusDate: new Date(), isReseted: true, size: { storage: 0, file: 0 } },
+        $unset: { cleanupToken: "", cleanupUntil: "", pathIndexKey: "", pathIndexBuiltAt: "", sizeComputedAt: "", emptyTreeGeneration: "", retiredTreeGenerations: "", retiredContentPrefixes: "", legacyContentCleanupPending: "", stagedFileTrees: "" } }).exec();
+      if (result.matchedCount) { this.model.status = RepositoryStatus.EXPIRED; this.model.isReseted = true; this.model.size = { storage: 0, file: 0 }; this.model.sizeComputedAt = undefined; this.model.emptyTreeGeneration = undefined; }
+    } finally {
+      clearInterval(heartbeat);
+      await AnonymizedRepositoryModel.updateOne({ _id: this.model._id, cleanupToken: token }, { $unset: { cleanupToken: "", cleanupUntil: "" } }).exec();
+    }
   }
 
   /**
@@ -609,6 +944,19 @@ export default class Repository {
    */
   async resetSate(status?: RepositoryStatus, statusMessage?: string) {
     this.assertNotArchived();
+    const revision = randomUUID();
+    this.model.emptyTreeGeneration = undefined;
+    if (isConnected) {
+      const result = await AnonymizedRepositoryModel.updateOne({ _id: this.model._id,
+      treeGeneration: this.model.treeGeneration || { $exists: false },
+      }, { $set: { fileMetadataRevision: revision },
+        $unset: { emptyTreeGeneration: "", pathIndexKey: "", pathIndexBuiltAt: "" },
+      }).exec();
+      if (!result.matchedCount) throw new AnonymousError("repository_changed", { httpStatus: 409 });
+    }
+    this.model.fileMetadataRevision = revision;
+    this.model.pathIndexKey = undefined;
+    this.model.pathIndexBuiltAt = undefined;
     // remove attribute
     this._model.size = { storage: 0, file: 0 };
     if (status) {
@@ -617,6 +965,7 @@ export default class Repository {
     // remove cache
     await Promise.all([
       FileModel.deleteMany({ repoId: this.repoId }).exec(),
+      AnonymizedPathModel.deleteMany({ repoId: this.repoId }).exec(),
       this.removeCache(),
     ]);
     logger.info("reset", { repoId: this._model.repoId });
@@ -629,14 +978,24 @@ export default class Repository {
   async removeCache() {
     // Archive cleanup is handled by the resumable recovery script. Preserve DB metadata.
     if (this.status === RepositoryStatus.ARCHIVED) return;
+    const cacheRevision = randomUUID();
+    if (isConnected) {
+      const result = await AnonymizedRepositoryModel.updateOne({ _id: this.model._id,
+        treeGeneration: this.model.treeGeneration || { $exists: false },
+      }, { $set: { contentCacheRevision: cacheRevision } }).exec();
+      if (!result.matchedCount) throw new AnonymousError("repository_changed", { httpStatus: 409 });
+    }
+    this.model.contentCacheRevision = cacheRevision;
+    await removeTransformedRepositoryCache(this.repoId);
     await storage.rm(this.repoId);
     this.model.isReseted = true;
+    this.model.sizeComputedAt = undefined;
     this.model.size = { storage: 0, file: 0 };
     if (isConnected) {
       try {
         await AnonymizedRepositoryModel.updateOne(
           { _id: this._model._id },
-          { $set: { isReseted: true, size: this._model.size } }
+          { $set: { isReseted: true, size: this._model.size }, $unset: { sizeComputedAt: "" } }
         ).exec();
       } catch (error) {
         logger.error("removeCache save failed", serializeError(error));
@@ -675,11 +1034,12 @@ export default class Repository {
   }> {
     if (this.status !== RepositoryStatus.READY)
       return { storage: 0, file: 0 };
-    if (this._model.size.file) return this._model.size;
+    if (this._model.size.file || this._model.sizeComputedAt) return this._model.size;
     const res = await FileModel.aggregate([
       {
         $match: {
           repoId: this.repoId,
+          treeGeneration: this.model.treeGeneration || { $exists: false },
         },
       },
       {
@@ -694,10 +1054,12 @@ export default class Repository {
       storage: res[0]?.storage || 0,
       file: res[0]?.file || 0,
     };
+    this._model.sizeComputedAt = new Date();
     if (isConnected) {
       await AnonymizedRepositoryModel.updateOne(
-        { _id: this._model._id },
-        { $set: { size: this._model.size } }
+        { _id: this._model._id, treeGeneration: this.model.treeGeneration || { $exists: false },
+          fileMetadataRevision: this.model.fileMetadataRevision || { $exists: false } },
+        { $set: { size: this._model.size, sizeComputedAt: this._model.sizeComputedAt } }
       ).exec();
     }
     return this._model.size;

src/core/User.ts

@@ -14,6 +14,33 @@ import Gist from "./Gist";
 import AnonymizedGistModel from "./model/anonymizedGists/anonymizedGists.model";
 import { octokit } from "./GitHubUtils";
 
+import { isConnected } from "../server/database";
+async function markListExpired<T extends { _id?: unknown; status?: string; statusDate?: Date;
+  options: { expirationMode?: string; expirationDate?: Date } }>(model: {
+  updateMany(filter: Record<string, unknown>, update: Record<string, unknown>): { exec(): Promise<unknown> };
+  find(filter: Record<string, unknown>): { exec(): Promise<T[]> };
+}, rows: T[]): Promise<T[]> {
+  const now = new Date();
+  const expired = rows.filter(row => row.status === "ready" && row.options.expirationMode !== "never"
+    && row.options.expirationDate && row.options.expirationDate <= now);
+  if (!expired.length) return rows;
+  if (isConnected) {
+    const ids = expired.map(row => row._id);
+    await model.updateMany({ _id: { $in: ids }, status: "ready",
+      "options.expirationMode": { $ne: "never" }, "options.expirationDate": { $lte: now },
+    }, { $set: { status: "expiring", statusDate: now } }).exec();
+    const current = new Map((await model.find({ _id: { $in: ids } }).exec()).map(row => [String(row._id), row]));
+    const affected = new Set(ids.map(String));
+    return rows.flatMap(row => {
+      if (!affected.has(String(row._id))) return [row];
+      const fresh = current.get(String(row._id));
+      return fresh ? [fresh] : [];
+    });
+  }
+  for (const row of expired) { row.status = "expiring"; row.statusDate = now; }
+  return rows;
+}
+
 /**
  * Model for a user
  */
@@ -151,6 +178,30 @@ export default class User {
    * @returns the list of anonymized repositories
    */
   async getRepositories() {
+    const query = this.repositoryMembership();
+    const repositories = await AnonymizedRepositoryModel.find(query).exec();
+    return (await markListExpired<typeof repositories[number]>(AnonymizedRepositoryModel, repositories)).map(d => new Repository(d));
+  }
+  /**
+   * Get the lost of anonymized repositories
+   * @returns the list of anonymized repositories
+   */
+  async getPullRequests() {
+    const pullRequests = await AnonymizedPullRequestModel.find({
+      owner: this.id,
+    }).exec();
+    return (await markListExpired<typeof pullRequests[number]>(AnonymizedPullRequestModel, pullRequests)).map(d => new PullRequest(d));
+  }
+
+  /**
+   * Get the list of anonymized gists
+   */
+  async getGists() {
+    const gists = await AnonymizedGistModel.find({ owner: this.id }).exec();
+    return (await markListExpired<typeof gists[number]>(AnonymizedGistModel, gists)).map(d => new Gist(d));
+  }
+
+  repositoryMembership() {
     const memberships: Record<string, unknown>[] = [{ owner: this.id }];
     const githubId = this.model.externalIDs?.github;
     if (githubId) memberships.push({ "coauthors.githubId": githubId });
@@ -160,71 +211,14 @@ export default class User {
         $or: [{ githubId: { $exists: false } }, { githubId: null }, { githubId: "" }],
       } } });
     }
-    const query = { $or: memberships };
-    const repositories = (
-      await AnonymizedRepositoryModel.find(query).exec()
-    ).map((d) => new Repository(d));
-    const promises = [];
-    for (const repo of repositories) {
-      if (
-        repo.status == "ready" &&
-        repo.options.expirationMode != "never" &&
-        repo.options.expirationDate != null &&
-        repo.options.expirationDate < new Date()
-      ) {
-        // expire the repository
-        promises.push(repo.expire());
-      }
-    }
-    await Promise.all(promises);
-    return repositories;
-  }
-  /**
-   * Get the lost of anonymized repositories
-   * @returns the list of anonymized repositories
-   */
-  async getPullRequests() {
-    const pullRequests = (
-      await AnonymizedPullRequestModel.find({
-        owner: this.id,
-      }).exec()
-    ).map((d) => new PullRequest(d));
-    const promises = [];
-    for (const repo of pullRequests) {
-      if (
-        repo.status == "ready" &&
-        repo.options.expirationMode != "never" &&
-        repo.options.expirationDate != null &&
-        repo.options.expirationDate < new Date()
-      ) {
-        // expire the repository
-        promises.push(repo.expire());
-      }
-    }
-    await Promise.all(promises);
-    return pullRequests;
+    return { $or: memberships };
   }
 
-  /**
-   * Get the list of anonymized gists
-   */
-  async getGists() {
-    const gists = (
-      await AnonymizedGistModel.find({ owner: this.id }).exec()
-    ).map((d) => new Gist(d));
-    const promises = [];
-    for (const g of gists) {
-      if (
-        g.status == "ready" &&
-        g.options.expirationMode != "never" &&
-        g.options.expirationDate != null &&
-        g.options.expirationDate < new Date()
-      ) {
-        promises.push(g.expire());
-      }
-    }
-    await Promise.all(promises);
-    return gists;
+  async quotaRepositories() {
+    return AnonymizedRepositoryModel.find({ owner: this.id, status: "ready",
+      $or: [{ "options.expirationMode": "never" }, { "options.expirationDate": { $exists: false } },
+        { "options.expirationDate": null }, { "options.expirationDate": { $gt: new Date() } }] })
+      .select("repoId size sizeComputedAt treeGeneration fileMetadataRevision").lean().exec();
   }
 
   get model() {

src/core/anonymization-pool.ts

@@ -0,0 +1,91 @@
+import { Worker } from "worker_threads";
+import { existsSync } from "fs";
+import { join } from "path";
+
+const MEMORY_BUDGET = Math.max(128, Number(process.env.ANONYMIZATION_MEMORY_MB) || 512) * 1024 * 1024;
+const MAX_WORKERS = Math.min(4, Math.max(1, Number(process.env.ANONYMIZATION_WORKERS) || 1));
+interface Job {
+  message: { input: string; output: string; options: unknown; maxOutput: number; context: { mask: string; hostname: string } };
+  weight: number;
+  signal: AbortSignal;
+  resolve: (changed: boolean) => void;
+  reject: (error: Error) => void;
+  timer?: NodeJS.Timeout;
+  abort?: () => void;
+}
+const waiting: Job[] = [];
+const idle: Worker[] = [];
+let running = 0;
+let bytes = 0;
+
+export function anonymizeOnWorker(message: Job["message"], size: number, signal: AbortSignal): Promise<boolean> {
+  const weight = 64 * 1024 * 1024 + size * 4;
+  if (weight > MEMORY_BUDGET) return Promise.reject(new Error("anonymization_memory_budget_exceeded"));
+  if (waiting.length >= 16) return Promise.reject(new Error("anonymization_queue_full"));
+  return new Promise((resolve, reject) => {
+    const job: Job = { message, weight, signal, resolve, reject };
+    job.abort = () => {
+      const index = waiting.indexOf(job);
+      if (index < 0) return;
+      waiting.splice(index, 1);
+      clearTimeout(job.timer);
+      signal.removeEventListener("abort", job.abort!);
+      reject(new Error("anonymization_cancelled"));
+    };
+    if (signal.aborted) return reject(new Error("anonymization_cancelled"));
+    signal.addEventListener("abort", job.abort, { once: true });
+    job.timer = setTimeout(job.abort, 30_000);
+    waiting.push(job);
+    pump();
+  });
+}
+function pump() {
+  while (waiting.length && running < MAX_WORKERS && bytes + waiting[0].weight <= MEMORY_BUDGET) {
+    const job = waiting.shift()!;
+    clearTimeout(job.timer);
+    job.signal.removeEventListener("abort", job.abort!);
+    running++;
+    bytes += job.weight;
+    const compiled = join(__dirname, "anonymization.worker.js");
+    let worker: Worker;
+    try { worker = idle.pop() || new Worker(existsSync(compiled) ? compiled : join(__dirname, "anonymization.worker.ts"), {
+      execArgv: existsSync(compiled) ? [] : ["-r", require.resolve("ts-node/register/transpile-only")],
+      resourceLimits: { maxOldGenerationSizeMb: Math.ceil(MEMORY_BUDGET / 1024 / 1024) },
+    }); } catch (error) { running--; bytes -= job.weight; job.reject(error as Error); continue; }
+    worker.ref();
+    let finished = false;
+    const finish = (error?: Error, changed = false) => {
+      if (finished) return;
+      finished = true;
+      clearTimeout(deadline);
+      job.signal.removeEventListener("abort", cancel);
+      worker.removeListener("message", message);
+      worker.removeListener("error", failed);
+      worker.removeListener("exit", exited);
+      const release = () => { running--; bytes -= job.weight; pump(); };
+      // Large jobs can leave substantial unreachable buffers in V8 until GC.
+      // Release their workers before admitting the next job.
+      if (error || job.weight >= 96 * 1024 * 1024) {
+        void worker.terminate().then(release, release);
+      } else { idle.push(worker); worker.unref(); release(); }
+      if (error) job.reject(error); else job.resolve(changed);
+    };
+    const message = (result: { error?: string; changed?: boolean }) => finish(result.error ? new Error(result.error) : undefined, result.changed);
+    const failed = (error: Error) => finish(error);
+    const exited = () => finish(new Error("anonymization_worker_exited"));
+    const cancel = () => finish(new Error("anonymization_cancelled"));
+    const deadline = setTimeout(() => finish(new Error("anonymization_deadline_exceeded")), 15_000);
+    worker.once("message", message);
+    worker.once("error", failed);
+    worker.once("exit", exited);
+    job.signal.addEventListener("abort", cancel, { once: true });
+    if (job.signal.aborted) cancel(); else worker.postMessage(job.message);
+  }
+}
+
+let spoolBytes = 0;
+export function reserveSpool(bytes: number) {
+  if (spoolBytes + bytes > MEMORY_BUDGET * 2) throw new Error("anonymization_spool_budget_exceeded");
+  spoolBytes += bytes;
+}
+export function releaseSpool(bytes: number) { spoolBytes -= bytes; }

src/core/anonymization.worker.ts

@@ -0,0 +1,22 @@
+import config from "../config";
+import { parentPort } from "worker_threads";
+import { readFileSync, writeFileSync } from "fs";
+import { ContentAnonimizer } from "./anonymize-utils";
+
+parentPort!.on("message", ({ input, output, options, maxOutput, context }) => {
+  try {
+    config.ANONYMIZATION_MASK = context.mask;
+    config.APP_HOSTNAME = context.hostname;
+    const original = readFileSync(input);
+    const text = original.toString("utf8");
+    const anonymizer = new ContentAnonimizer(options);
+    const transformed = anonymizer.anonymize(text);
+    if (Buffer.byteLength(transformed, "utf8") > maxOutput) throw new Error("anonymized_output_too_large");
+    const bytes = transformed === text ? original : Buffer.from(transformed, "utf8");
+    if (bytes.length > maxOutput) throw new Error("anonymized_output_too_large");
+    writeFileSync(output, bytes, { flag: "wx" });
+    parentPort!.postMessage({ changed: anonymizer.wasAnonymized });
+  } catch (error) {
+    parentPort!.postMessage({ error: (error as Error).message });
+  }
+});

src/core/anonymize-utils.ts

@@ -1,3 +1,10 @@
+import { createHash } from "crypto";
+import { transformedFile } from "./transformed-cache";
+import { promises as fs, createReadStream, ReadStream } from "fs";
+import { tmpdir } from "os";
+import { join } from "path";
+import { anonymizeOnWorker, reserveSpool, releaseSpool } from "./anonymization-pool";
+import { maintainTextSpools } from "./temporary-storage";
 import { RE2JS } from "re2js";
 import { Script } from "vm";
 import { basename } from "path";
@@ -136,11 +143,20 @@ export class AnonymizeTransformer extends Transform {
   anonimizer: ContentAnonimizer;
   private wholeChunks: Buffer[] = [];
   private wholeBytes = 0;
+  private digest = createHash("sha256");
+  private spool?: string;
+  private reservedSpool = 0;
+  private output?: ReadStream;
+  private cancelled = new AbortController();
+  private spoolOperation?: Promise<void>;
+  private readonly memoryThreshold = 256 * 1024;
   private readonly bufferWholeStream: boolean;
 
   constructor(
     readonly opt: {
       filePath: string;
+      cacheGeneration?: string;
+      cacheRevision?: string;
     } & ConstructorParameters<typeof ContentAnonimizer>[0]
   ) {
     super();
@@ -171,12 +187,18 @@ export class AnonymizeTransformer extends Transform {
       this.nameVerdict = this.isText;
     }
     if (this.isText && this.bufferWholeStream) {
+      this.digest.update(chunk);
       this.wholeBytes += chunk.length;
       if (this.wholeBytes > config.MAX_FILE_SIZE) {
         this.wholeChunks = [];
         return callback(new Error(`Text file exceeded ${config.MAX_FILE_SIZE} bytes`));
       }
-      this.wholeChunks.push(chunk);
+      if (this.wholeBytes <= this.memoryThreshold) this.wholeChunks.push(Buffer.from(chunk));
+      else {
+        this.spoolOperation = this.spoolChunk(chunk);
+        void this.spoolOperation.then(() => callback(), error => callback(error));
+        return;
+      }
     } else {
       this.emit("transform", { isText: this.isText, wasAnonimized: false, chunk });
       this.push(chunk);
@@ -184,7 +206,54 @@ export class AnonymizeTransformer extends Transform {
     callback();
   }
 
+  private async spoolChunk(chunk: Buffer) {
+    const additional = chunk.length + (this.spool ? 0 : this.wholeBytes - chunk.length);
+    reserveSpool(additional);
+    this.reservedSpool += additional;
+    if (!this.spool) {
+      await maintainTextSpools();
+      this.spool = await fs.mkdtemp(join(tmpdir(), "anonymous-text-"));
+      await fs.writeFile(join(this.spool, "input"), Buffer.concat(this.wholeChunks));
+      this.wholeChunks = [];
+    }
+    if (this.cancelled.signal.aborted) throw new Error("anonymization_cancelled");
+    await fs.appendFile(join(this.spool, "input"), chunk);
+  }
+
+  _read(size: number) {
+    super._read(size);
+    this.output?.resume();
+  }
+
+  _destroy(error: Error | null, callback: (error?: Error | null) => void) {
+    this.cancelled.abort();
+    this.output?.destroy();
+    this.wholeChunks = [];
+    void (async () => {
+      await this.spoolOperation?.catch(() => {});
+      try { if (this.spool) await fs.rm(this.spool, { recursive: true, force: true }); }
+      finally { releaseSpool(this.reservedSpool); this.reservedSpool = 0; }
+    })().then(() => callback(error), () => callback(error));
+  }
+
   _flush(callback: (error?: Error) => void) {
+    if (this.spool) {
+      const output = join(this.spool, "output");
+      void transformedFile(this.digest.digest("hex"), this.opt, output, () => anonymizeOnWorker({ input: join(this.spool!, "input"), output,
+        options: this.opt, maxOutput: config.MAX_FILE_SIZE * 4, context: { mask: config.ANONYMIZATION_MASK, hostname: config.APP_HOSTNAME } }, this.wholeBytes, this.cancelled.signal))
+        .then(changed => {
+          if (this.destroyed) return;
+          this.anonimizer.wasAnonymized = changed;
+          this.output = createReadStream(output);
+          this.output.on("data", chunk => {
+            this.emit("transform", { isText: true, wasAnonimized: changed, chunk });
+            if (!this.push(chunk)) this.output!.pause();
+          });
+          this.output.once("error", callback);
+          this.output.once("end", () => callback());
+        }, error => callback(error));
+      return;
+    }
     try {
       if (this.nameVerdict === null) this.isText = true;
       if (this.wholeBytes) {

src/core/archive-entry.ts

@@ -0,0 +1,17 @@
+import { Archiver, EntryData } from "archiver";
+import { Readable } from "stream";
+/** Allow only one active entry; compression and the response provide backpressure. */
+export function appendArchiveEntry(archive: Archiver, source: Readable, options: EntryData): Promise<void> {
+  return new Promise((resolve, reject) => {
+    const cleanup = () => {
+      archive.removeListener("entry", done); archive.removeListener("error", fail);
+      archive.removeListener("close", closed); source.removeListener("error", fail);
+    };
+    const done = () => { cleanup(); resolve(); };
+    const fail = (error: Error) => { cleanup(); source.destroy(); reject(error); };
+    const closed = () => fail(new Error("archive_cancelled"));
+    archive.once("entry", done); archive.once("error", fail); archive.once("close", closed);
+    source.once("error", fail);
+    archive.append(source, options);
+  });
+}

src/core/async-cache.ts

@@ -0,0 +1,22 @@
+/** Bounded TTL cache with one calculation per key and retry after failure. */
+export class AsyncCache<T> {
+  private entries = new Map<string, { expires: number; value: Promise<T> }>();
+  constructor(private ttl: number, private capacity: number, private now = () => Date.now()) {}
+  get size() { this.prune(); return this.entries.size; }
+  clear() { this.entries.clear(); }
+  private prune() {
+    for (const [key, entry] of this.entries) if (entry.expires <= this.now()) this.entries.delete(key);
+  }
+  get(key: string, calculate: () => Promise<T>): Promise<T> {
+    this.prune();
+    const cached = this.entries.get(key);
+    if (cached) return cached.value;
+    if (this.entries.size >= this.capacity) this.entries.delete(this.entries.keys().next().value!);
+    const entry = { expires: Infinity, value: Promise.resolve().then(calculate) };
+    this.entries.set(key, entry);
+    entry.value.then(() => { entry.expires = this.now() + this.ttl; }, () => {
+      if (this.entries.get(key) === entry) this.entries.delete(key);
+    });
+    return entry.value;
+  }
+}

src/core/cache-coordination.ts

@@ -0,0 +1,83 @@
+import { createClient } from "redis";
+import { randomUUID } from "crypto";
+import config from "../config";
+
+let connection: ReturnType<typeof createClient> | undefined;
+let connecting: Promise<ReturnType<typeof createClient> | undefined> | undefined;
+let retryAt = 0;
+/** Bound commands even when a connected server stops replying. */
+export async function cacheCommand<T>(command: Promise<T>, client: ReturnType<typeof createClient>, timeoutMs = 1000): Promise<T> {
+  let timer: NodeJS.Timeout | undefined;
+  try {
+    return await Promise.race([command, new Promise<never>((_, reject) => {
+      timer = setTimeout(() => {
+        retryAt = Date.now() + 30_000;
+        if (connection === client) connection = undefined;
+        if (client.isOpen) void client.disconnect().catch(() => {});
+        reject(new Error("cache_command_timeout"));
+      }, timeoutMs);
+    })]);
+  } finally { clearTimeout(timer); }
+}
+export async function cacheRedis() {
+  if (connection?.isReady) return connection;
+  if (connecting) return connecting;
+  if (Date.now() < retryAt) return undefined;
+  connecting = (async () => {
+    const client = createClient({ disableOfflineQueue: true, socket: {
+      host: config.REDIS_HOSTNAME, port: config.REDIS_PORT,
+      connectTimeout: 500, reconnectStrategy: false,
+    } });
+    client.on("error", () => {});
+    try { await cacheCommand(client.connect(), client, 500); connection = client; return client; }
+    catch { retryAt = Date.now() + 30_000; if (client.isOpen) await client.disconnect().catch(() => {}); return undefined; }
+    finally { connecting = undefined; }
+  })();
+  return connecting;
+}
+
+/** A renewable lease. Followers recheck durable output; they never share a Readable. */
+export async function coordinatedFill<T>(
+  key: string, read: () => Promise<T | undefined>, produce: () => Promise<T>,
+  waitMs = 120_000
+): Promise<T> {
+  const redis = await cacheRedis();
+  if (!redis) return produce();
+  const lockKey = `perf:lease:${key}`;
+  const token = randomUUID();
+  const deadline = Date.now() + waitMs;
+  for (;;) {
+    const value = await read();
+    if (value !== undefined) return value;
+    let owned: string | null;
+    try { owned = await cacheCommand(redis.set(lockKey, token, { NX: true, PX: 30_000 }), redis); }
+    catch { return produce(); }
+    if (owned) {
+      const heartbeat = setInterval(() => {
+        void cacheCommand(redis.eval("if redis.call('get',KEYS[1]) == ARGV[1] then return redis.call('pexpire',KEYS[1],ARGV[2]) else return 0 end", {
+          keys: [lockKey], arguments: [token, "30000"],
+        }), redis).catch(() => {});
+      }, 10_000);
+      heartbeat.unref();
+      try {
+        // Coordination is optional. A lost heartbeat cannot invalidate output
+        // that the producer has already completed and verified.
+        return await produce();
+      } finally {
+        clearInterval(heartbeat);
+        await cacheCommand(redis.eval("if redis.call('get',KEYS[1]) == ARGV[1] then return redis.call('del',KEYS[1]) else return 0 end", {
+          keys: [lockKey], arguments: [token],
+        }), redis).catch(() => {});
+      }
+    }
+    if (Date.now() >= deadline) throw new Error("cache_fill_timeout");
+    await new Promise(resolve => setTimeout(resolve, 100));
+  }
+}
+
+export async function closeCacheRedis() {
+  const current = connection;
+  connection = undefined;
+  retryAt = 0;
+  if (current?.isOpen) await current.disconnect();
+}

src/core/content-expiration.ts

@@ -0,0 +1,35 @@
+import { Document, Model } from "mongoose";
+import { RepositoryStatus } from "./types";
+
+type ContentSnapshot = Document & {
+  status?: RepositoryStatus;
+  statusDate?: Date;
+  anonymizeDate?: Date;
+  options: { expirationMode?: string; expirationDate?: Date };
+  githubAccess?: { revision?: string };
+};
+
+// Embedded content and lifecycle state live in one document. Claim and clear
+// them in one conditional write so an extension or rebuild cannot interleave.
+export async function expireEmbeddedContent<T extends ContentSnapshot>(
+  model: Model<T>, data: T, fields: Record<string, unknown>, unset: Record<string, ""> = {}
+) {
+  const now = new Date();
+  const result = await model.updateOne({
+    _id: data._id,
+    status: data.status ?? { $exists: false },
+    statusDate: data.statusDate ?? { $exists: false },
+    anonymizeDate: data.anonymizeDate ?? { $exists: false },
+    "options.expirationMode": data.options.expirationMode ?? { $exists: false },
+    "options.expirationDate": data.options.expirationDate ?? { $exists: false },
+    "githubAccess.revision": data.githubAccess?.revision ?? { $exists: false },
+    $and: [
+      { status: { $in: [RepositoryStatus.READY, RepositoryStatus.EXPIRING] } },
+      { "options.expirationMode": { $ne: "never" } },
+      { "options.expirationDate": { $lte: now } },
+    ],
+  }, { $set: { ...fields, status: RepositoryStatus.EXPIRED, statusDate: now }, $unset: unset }).exec();
+  if (!result.matchedCount) return;
+  data.set({ ...fields, status: RepositoryStatus.EXPIRED, statusDate: now });
+  for (const key of Object.keys(unset)) data.set(key, undefined);
+}

src/core/content-generation.ts

@@ -0,0 +1,9 @@
+import { createHash } from "crypto";
+export function contentGenerationPrefix(generation: string) {
+  return `__content/${createHash("sha256").update(generation).digest("hex")}`;
+}
+
+export function contentRetirementMarker(prefix: string) {
+  if (!/^__content\/[a-f0-9]{64}$/.test(prefix)) throw new Error("Invalid content generation prefix");
+  return `__content/__retired/${prefix.slice("__content/".length)}.txt`;
+}

src/core/model/anonymized-path.ts

@@ -0,0 +1,11 @@
+import { IFile } from "./files/files.types";
+import { model, Schema } from "mongoose";
+const schema = new Schema({ repoId: { type: String, required: true }, key: { type: String, required: true }, originalId: Schema.Types.ObjectId,
+  name: { type: String, required: true }, path: { type: String, default: "", required: true }, sha: String, size: Number,
+  anonymousName: String, anonymousDirectory: String, anonymousPath: String,
+  searchName: String, searchDirectory: String,
+  createdAt: { type: Date, default: Date.now } });
+schema.index({ repoId: 1, key: 1, anonymousPath: 1, path: 1, name: 1 });
+schema.index({ repoId: 1, key: 1, originalId: 1 }, { unique: true });
+schema.index({ createdAt: 1 }, { expireAfterSeconds: 30 * 24 * 3600 });
+export default model<IFile & { key: string; originalId: Schema.Types.ObjectId; anonymousName: string; anonymousDirectory: string; anonymousPath: string; searchName: string; searchDirectory: string; createdAt: Date }>("AnonymizedPath", schema);

src/core/model/anonymizedGists/anonymizedGists.schema.ts

@@ -68,4 +68,6 @@ const AnonymizedGistSchema = new Schema({
   },
 });
 
+AnonymizedGistSchema.index({ status: 1, "options.expirationDate": 1 });
+
 export default AnonymizedGistSchema;

src/core/model/anonymizedPullRequests/anonymizedPullRequests.schema.ts

@@ -65,4 +65,6 @@ const AnonymizedPullRequestSchema = new Schema({
   },
 });
 
+AnonymizedPullRequestSchema.index({ status: 1, "options.expirationDate": 1 });
+
 export default AnonymizedPullRequestSchema;

src/core/model/anonymizedRepositories/anonymizedRepositories.schema.ts

@@ -47,6 +47,19 @@ const AnonymizedRepositorySchema = new Schema({
     repositoryName: String,
     accessToken: { type: String, select: false },
   },
+  cleanupToken: { type: String, select: false },
+  cleanupUntil: { type: Date, select: false },
+  treeGeneration: String,
+  stagedFileTrees: { type: [{ generation: String, until: Date }], default: undefined },
+  retiredTreeGenerations: { type: [String], default: undefined },
+  retiredContentPrefixes: { type: [String], default: undefined },
+  legacyContentCleanupPending: Boolean,
+  contentCacheVersion: Number,
+  contentCacheRevision: String,
+  emptyTreeGeneration: String,
+  fileMetadataRevision: String,
+  pathIndexKey: String,
+  pathIndexBuiltAt: Date,
   truncatedFolders: {
     type: [String],
     default: [],
@@ -70,6 +83,7 @@ const AnonymizedRepositorySchema = new Schema({
     type: Date,
     default: Date.now,
   },
+  sizeComputedAt: Date,
   size: {
     storage: {
       type: Number,
@@ -87,10 +101,17 @@ const AnonymizedRepositorySchema = new Schema({
 });
 
 AnonymizedRepositorySchema.index({ "source.repositoryName": 1 });
+AnonymizedRepositorySchema.index({ "coauthors.githubId": 1 });
+AnonymizedRepositorySchema.index({ retiredTreeGenerations: 1 }, { sparse: true });
+AnonymizedRepositorySchema.index({ "stagedFileTrees.until": 1 }, { sparse: true });
+AnonymizedRepositorySchema.index({ retiredContentPrefixes: 1 }, { sparse: true });
+AnonymizedRepositorySchema.index({ legacyContentCleanupPending: 1, status: 1 },
+  { partialFilterExpression: { legacyContentCleanupPending: true } });
 AnonymizedRepositorySchema.index({ status: 1, statusDate: 1 });
 AnonymizedRepositorySchema.index({ lastView: 1 });
 AnonymizedRepositorySchema.index({ anonymizeDate: 1 });
 AnonymizedRepositorySchema.index({ status: 1, isReseted: 1, lastView: 1 });
+AnonymizedRepositorySchema.index({ status: 1, "options.expirationDate": 1 });
 AnonymizedRepositorySchema.index({
   status: 1,
   isReseted: 1,

src/core/model/anonymizedRepositories/anonymizedRepositories.types.ts

@@ -30,6 +30,17 @@ export interface IAnonymizedRepository {
     photo?: string;
     addedAt?: Date;
   }[];
+  treeGeneration?: string;
+  stagedFileTrees?: { generation: string; until: Date }[];
+  retiredTreeGenerations?: string[];
+  retiredContentPrefixes?: string[];
+  legacyContentCleanupPending?: boolean;
+  contentCacheVersion?: number;
+  contentCacheRevision?: string;
+  emptyTreeGeneration?: string;
+  fileMetadataRevision?: string;
+  pathIndexKey?: string;
+  pathIndexBuiltAt?: Date;
   truncatedFolders: string[];
   conference: string;
   options: {
@@ -49,6 +60,7 @@ export interface IAnonymizedRepository {
   };
   pageView: number;
   lastView: Date;
+  sizeComputedAt?: Date;
   size: {
     storage: number;
     file: number;

src/core/model/dashboard-name.ts

@@ -0,0 +1,5 @@
+import { model, Schema } from "mongoose";
+const schema = new Schema({ owner: { type: Schema.Types.ObjectId, required: true },
+  type: { type: String, required: true }, artifactId: { type: String, required: true }, name: String });
+schema.index({ owner: 1, type: 1, artifactId: 1 }, { unique: true });
+export default model("DashboardName", schema);

src/core/model/files/files.schema.ts

@@ -4,6 +4,9 @@ const FileSchema = new Schema({
   name: { type: String, index: true },
   path: { type: String, index: true },
   repoId: { type: String, index: true },
+  treeGeneration: String,
+  treeStaged: Boolean,
+  metadataPending: Boolean,
   sha: {
     type: String,
   },
@@ -13,6 +16,11 @@ const FileSchema = new Schema({
 });
 
 FileSchema.index({ path: 1, repoId: 1 });
+FileSchema.index({ metadataPending: 1, repoId: 1, treeGeneration: 1 },
+  { partialFilterExpression: { metadataPending: true } });
+FileSchema.index({ treeStaged: 1, repoId: 1, treeGeneration: 1 },
+  { partialFilterExpression: { treeStaged: true } });
+FileSchema.index({ repoId: 1, treeGeneration: 1, path: 1, name: 1 });
 FileSchema.index(
   { repoId: 1, size: 1, path: 1 },
   { name: "repoId_1_size_1_path_1" }

src/core/model/files/files.types.ts

@@ -4,6 +4,9 @@ export interface IFile {
   name: string;
   path: string;
   repoId: string;
+  treeGeneration?: string;
+  treeStaged?: boolean;
+  metadataPending?: boolean;
   sha?: string;
   size?: number;
 }

src/core/response-stream.ts

@@ -0,0 +1,24 @@
+import { Readable, Writable } from "stream";
+
+/** Cancel upstream on every response terminal event, including a pre-header close. */
+export function streamResponse(source: Readable, response: Writable,
+  onError: (error: Error) => void): Promise<void> {
+  return new Promise((resolve, reject) => {
+    const cleanup = () => {
+      response.removeListener("close", done);
+      response.removeListener("finish", done);
+      response.removeListener("error", failed);
+      source.removeListener("error", upstreamFailed);
+      source.unpipe(response);
+      source.destroy();
+    };
+    const done = () => { cleanup(); resolve(); };
+    const failed = (error: Error) => { cleanup(); reject(error); };
+    const upstreamFailed = (error: Error) => { onError(error); done(); };
+    response.once("close", done);
+    response.once("finish", done);
+    response.once("error", failed);
+    source.once("error", upstreamFailed);
+    if (response.destroyed || response.writableEnded) done(); else source.pipe(response);
+  });
+}

src/core/source/GitHubBase.ts

@@ -14,6 +14,8 @@ export interface GitHubBaseData {
   organization: string;
   repoName: string;
   commit: string;
+  cacheGeneration?: string;
+  cacheRevision?: string;
 }
 
 export default abstract class GitHubBase implements SourceBase {

src/core/source/GitHubDownload.ts

@@ -1,6 +1,5 @@
 import got from "got";
 import { Readable } from "stream";
-import { OctokitResponse } from "@octokit/types";
 
 import storage from "../storage";
 import GitHubBase, {
@@ -18,7 +17,7 @@ export default class GitHubDownload extends GitHubBase {
     super(data);
   }
 
-  public async getZipUrl(): Promise<OctokitResponse<unknown, 302>> {
+  public async getZipUrl() {
     const oct = octokit(await this.data.getToken());
     return oct.rest.repos.downloadZipballArchive({
       owner: this.data.organization,
@@ -29,7 +28,7 @@ export default class GitHubDownload extends GitHubBase {
   }
 
   async download(progress?: (status: string) => void) {
-    let response: OctokitResponse<unknown, number>;
+    let response: Awaited<ReturnType<GitHubDownload["getZipUrl"]>>;
     try {
       response = await this.getZipUrl();
     } catch (error) {

src/core/source/GitHubStream.ts

@@ -1,3 +1,6 @@
+import { contentGenerationPrefix, contentRetirementMarker } from "../content-generation";
+import { createHash } from "crypto";
+import { coordinatedFill } from "../cache-coordination";
 import { githubTokenContext } from "../github-token-context";
 import AnonymizedFile from "../AnonymizedFile";
 import GitHubBase, {
@@ -11,17 +14,20 @@ import { basename, dirname } from "path";
 
 import * as stream from "stream";
 import AnonymousError from "../AnonymousError";
-import { FILE_TYPE } from "../storage/Storage";
 import { octokit, waitForTokenGate } from "../GitHubUtils";
 import FileModel from "../model/files/files.model";
 import { IFile } from "../model/files/files.types";
 import { createLogger, serializeError } from "../logger";
 import config from "../../config";
+import { isConnected } from "../../server/database";
+import AnonymizedRepositoryModel from "../model/anonymizedRepositories/anonymizedRepositories.model";
+import { RepositoryStatus } from "../types";
 
 
 const logger = createLogger("gh-stream");
 
 const GH_API_CONCURRENCY = 6;
+const cacheFills = new Map<string, Promise<void>>();
 
 async function pMap<T, R>(
   items: T[],
@@ -238,108 +244,89 @@ export default class GitHubStream extends GitHubBase {
     const meta = await fileMeta();
     const expected: { sha: string; size?: number } =
       typeof meta === "string" ? { sha: meta } : meta;
-    const fileInfo = await storage.exists(repoId, filePath);
-    if (fileInfo == FILE_TYPE.FILE) {
-      // If we know the upstream size, validate the cached entry. A cached
-      // file smaller than the upstream size means a previous fetch was
-      // truncated — likely a network error during the GitHub fetch left a
-      // 0-byte or partial blob behind. Treat it as a miss and re-fetch.
-      // Cached size >= expected is accepted: equal for normal files, and
-      // larger for Git LFS files where FileModel.size is the pointer's
-      // size but the cached bytes are the resolved LFS content.
-      if (expected.size != null && expected.size > 0) {
-        try {
-          const stat = await storage.fileInfo(repoId, filePath);
-          if (stat.size != null && stat.size < expected.size) {
-            await storage.rm(repoId, filePath);
-          } else {
-            return storage.read(repoId, filePath);
-          }
-        } catch {
-          // fall through and re-fetch
+    const generation = createHash("sha256").update(JSON.stringify([
+      this.data.commit, expected.sha, this.data.cacheGeneration, this.data.cacheRevision,
+    ])).digest("hex");
+    const cachePath = this.data.cacheGeneration ? `${contentGenerationPrefix(this.data.cacheGeneration)}/${generation}/${filePath}` : filePath;
+    const assertActive = async () => {
+      // Retirement markers live inside the storage root and disappear when
+      // expiration removes it. The persisted lifecycle also fences late fills.
+      if (isConnected) {
+        const current = await AnonymizedRepositoryModel.findOne({ repoId })
+          .select("status treeGeneration anonymizeDate contentCacheRevision").lean().exec();
+        if (!current || !current.status || [RepositoryStatus.EXPIRING, RepositoryStatus.EXPIRED,
+          RepositoryStatus.REMOVING, RepositoryStatus.REMOVED, RepositoryStatus.ARCHIVED].includes(current.status)
+          || (this.data.cacheGeneration && `${current.treeGeneration || "legacy"}:${current.anonymizeDate?.toISOString() || ""}` !== this.data.cacheGeneration)
+          || this.data.cacheRevision !== current.contentCacheRevision) {
+          throw new AnonymousError("repository_changed", { httpStatus: 409 });
         }
-      } else {
-        return storage.read(repoId, filePath);
       }
-    } else if (fileInfo == FILE_TYPE.FOLDER) {
-      throw new AnonymousError("folder_not_supported", {
-        httpStatus: 400,
-        object: filePath,
-      });
-    }
-
-    // GitHub's blob API rejects blobs larger than 100 MB with HTTP 422.
-    // Skip the download entirely when the tree already tells us the file is
-    // over the cap, so we surface a clean `file_too_big` instead of paying
-    // the round-trip just to translate a 422.
+      if (!this.data.cacheGeneration) return;
+      try {
+        await storage.fileInfo(repoId, contentRetirementMarker(contentGenerationPrefix(this.data.cacheGeneration)));
+      } catch (error) {
+        const missing = error as { code?: string; name?: string; $metadata?: { httpStatusCode?: number }; httpStatus?: number };
+        if (missing.code === "ENOENT" || missing.name === "NotFound" || missing.name === "NoSuchKey" ||
+          missing.$metadata?.httpStatusCode === 404 || missing.httpStatus === 404) return;
+        throw error;
+      }
+      throw new AnonymousError("repository_changed", { httpStatus: 409 });
+    };
+    const key = `${repoId}:${generation}:${filePath}`;
+    const cached = async () => {
+      try {
+        const info = await storage.fileInfo(repoId, cachePath);
+        if (info.contentType === "application/x-directory") {
+          throw new AnonymousError("folder_not_supported", { httpStatus: 400 });
+        }
+        if (expected.size != null && expected.size > 0 && (info.size == null || info.size < expected.size)) return undefined;
+        return true;
+      } catch (error) {
+        if (error instanceof AnonymousError && error.message === "folder_not_supported") throw error;
+        return undefined;
+      }
+    };
+    if (await cached()) return storage.read(repoId, cachePath);
     if (expected.size != null && expected.size > config.MAX_FILE_SIZE) {
-      throw new AnonymousError("file_too_big", {
-        httpStatus: 413,
-        object: filePath,
-      });
+      throw new AnonymousError("file_too_big", { httpStatus: 413, object: filePath });
     }
-    const token = await this.data.getToken();
-
-    // Try the blob API first, but fall back to the raw URL on recoverable
-    // blob misses/caps while still preserving LFS pointer handling.
-    const content = await this.downloadWithFallback(
-      token,
-      expected.sha,
-      filePath
-    );
-
-    // duplicate the stream to write it to the storage
-    const stream1 = content.pipe(new stream.PassThrough());
-    const stream2 = content.pipe(new stream.PassThrough());
-
-    // Safety net: guarantee an `error` listener exists on both branches
-    // before any error can be emitted. storage.write attaches its listener
-    // only after an `await mk(...)`, and the route handler attaches its
-    // listener after awaiting this function — both leave a window where
-    // an upstream error would have no listener and escalate to
-    // uncaughtException, crashing the streamer.
-    const noop = () => {};
-    stream1.on("error", noop);
-    stream2.on("error", noop);
-
-    content.on("error", (error) => {
-      const httpStatus =
-        (error as { response?: { statusCode?: number } })?.response
-          ?.statusCode ??
-        (error as { status?: number })?.status ??
-        (error as { httpStatus?: number })?.httpStatus;
-      const errCode = (error as { code?: string })?.code;
-      const isTransient =
-        !httpStatus &&
-        (errCode === "ECONNRESET" ||
-          errCode === "ETIMEDOUT" ||
-          errCode === "ERR_BODY_PARSE_FAILURE" ||
-          error.name === "ReadError");
-      const code =
-        httpStatus === 422
-          ? "file_too_big"
-          : httpStatus === 403
-          ? "file_not_accessible"
-          : isTransient
-          ? "upstream_error"
-          : "file_not_found";
-      const wrapped = new AnonymousError(code, {
-        httpStatus: isTransient ? 502 : httpStatus,
-        cause: error as Error,
-        object: filePath,
-      });
-      stream1.destroy(wrapped);
-      stream2.destroy(wrapped);
-    });
-
-    // Fire-and-forget: storage.write logs its own failures inside FileSystem
-    // (`[fs] write failed`). Swallow the rejection here so an upstream error
-    // (e.g. GitHub 422 on a too-big blob) doesn't surface as an unhandled
-    // promise rejection and crash the streamer process.
-    storage
-      .write(repoId, filePath, stream1, this.type, expected.size)
-      .catch(() => {});
-    return stream2;
+    let filling = cacheFills.get(key);
+    if (!filling) {
+      if (cacheFills.size >= 64) throw new AnonymousError("cache_busy", { httpStatus: 503 });
+      filling = coordinatedFill(key, cached, async () => {
+        await assertActive();
+        const content = await this.downloadWithFallback(await this.data.getToken(), expected.sha, filePath);
+        let count = 0;
+        const limited = new stream.Transform({ transform(chunk, encoding, callback) {
+          count += chunk.length;
+          callback(count > config.MAX_FILE_SIZE ? new AnonymousError("file_too_big", { httpStatus: 413 }) : null, chunk);
+        } });
+        content.on("error", error => {
+          if (error instanceof AnonymousError) { limited.destroy(error); return; }
+          const status = (error as { response?: { statusCode?: number }; status?: number; httpStatus?: number });
+          const httpStatus = status.response?.statusCode ?? status.status ?? status.httpStatus;
+          const code = httpStatus === 403 ? "file_not_accessible"
+            : httpStatus === 404 ? "file_not_found"
+            : httpStatus === 422 ? "file_too_big" : "upstream_error";
+          limited.destroy(new AnonymousError(code, { httpStatus: httpStatus || 502, cause: error, object: filePath }));
+        });
+        limited.once("close", () => content.destroy());
+        limited.on("error", () => {});
+        content.pipe(limited);
+        const deadline = setTimeout(() => limited.destroy(new AnonymousError("upstream_error", { httpStatus: 502 })), 120_000);
+        try {
+          await storage.write(repoId, cachePath, limited, this.type, expected.size);
+          try { await assertActive(); }
+          catch (error) { await storage.rm(repoId, cachePath); throw error; }
+        }
+        finally { clearTimeout(deadline); limited.destroy(); content.destroy(); }
+        return true;
+      }).then(() => {});
+      cacheFills.set(key, filling);
+      void filling.finally(() => { if (cacheFills.get(key) === filling) cacheFills.delete(key); }).catch(() => {});
+    }
+    await filling;
+    return storage.read(repoId, cachePath);
   }
 
   async getFileContent(file: AnonymizedFile): Promise<stream.Readable> {
@@ -444,6 +431,15 @@ export default class GitHubStream extends GitHubBase {
     const output: IFile[] = [];
     let data;
     try {
+      data = await this.getGHTree(oct, token, sha, count, {
+        recursive: true,
+        callback: () => {
+          if (progress) {
+            progress("List file: " + count.file);
+          }
+        },
+      });
+      if (!data.truncated) return this.tree2Tree(data.tree, parentPath);
       data = await this.getGHTree(oct, token, sha, count, {
         recursive: false,
         callback: () => {

src/core/storage/FileSystem.ts

@@ -1,7 +1,8 @@
+import { appendArchiveEntry } from "../archive-entry";
 import config from "../../config";
 import * as fs from "fs";
 import { Extract } from "unzip-stream";
-import { join, basename, dirname } from "path";
+import { join, dirname } from "path";
 import { Response } from "express";
 import { Readable, pipeline, Transform } from "stream";
 import * as archiver from "archiver";
@@ -144,6 +145,24 @@ export default class FileSystem extends StorageBase {
     });
   }
 
+  async removeLegacyContent(repoId: string): Promise<void> {
+    let directory: fs.Dir;
+    try { directory = await fs.promises.opendir(join(config.FOLDER, this.repoPath(repoId))); }
+    catch (error) {
+      if ((error as NodeJS.ErrnoException).code === "ENOENT") return;
+      throw error;
+    }
+    for await (const entry of directory) {
+      if (entry.name !== "__content" || !entry.isDirectory()) { await this.rm(repoId, entry.name); continue; }
+      const content = await fs.promises.opendir(join(config.FOLDER, this.repoPath(repoId), "__content"));
+      for await (const generation of content) {
+        if (generation.name !== "__retired" && !/^[a-f0-9]{64}$/.test(generation.name)) {
+          await this.rm(repoId, join("__content", generation.name));
+        }
+      }
+    }
+  }
+
   /** @override */
   async mk(repoId: string, dir: string = ""): Promise<void> {
     this.assertSafePath(dir);
@@ -243,26 +262,28 @@ export default class FileSystem extends StorageBase {
   ) {
     this.assertSafePath(dir);
     const archive = archiver(opt?.format || "zip", {});
-    const fullPath = join(config.FOLDER, this.repoPath(repoId), dir);
-
-    await this.listFiles(repoId, dir, {
-      onEntry: async (file) => {
-        let rs: Readable = await this.read(repoId, file.path);
+    let active: Readable | undefined;
+    let stopped = false;
+    archive.once("close", () => { stopped = true; active?.destroy(); });
+    void (async () => {
+      for (const file of await this.listFiles(repoId, dir)) {
+        if (stopped || file.size == null) continue;
+        const filename = join(file.path, file.name);
+        const source = await this.read(repoId, filename);
+        if (stopped) { source.destroy(); return; }
+        active = source;
+        let output = source;
         if (opt?.fileTransformer) {
-          const src = rs;
-          const transformer = opt.fileTransformer(file.path);
-          src.on("error", (err) => transformer.destroy(err));
-          rs = src.pipe(transformer);
+          const transformer = opt.fileTransformer(filename);
+          source.on("error", error => transformer.destroy(error));
+          transformer.once("close", () => source.destroy());
+          output = source.pipe(transformer);
+          active = output;
         }
-        const f = file.path.replace(fullPath, "");
-        archive.append(rs, {
-          name: basename(f),
-          prefix: dirname(f),
-        });
-      },
-    }).then(() => {
-      archive.finalize();
-    });
+        await appendArchiveEntry(archive, output, { name: filename.slice(dir ? dir.length + 1 : 0) });
+      }
+      if (!stopped) await archive.finalize();
+    })().catch(error => archive.destroy(error));
     return archive;
   }
 }

src/core/storage/S3.ts

@@ -1,3 +1,4 @@
+import { appendArchiveEntry } from "../archive-entry";
 import {
   GetObjectCommand,
   ListObjectsV2CommandOutput,
@@ -7,7 +8,7 @@ import {
 import { Upload } from "@aws-sdk/lib-storage";
 import { NodeHttpHandler } from "@smithy/node-http-handler";
 import config from "../../config";
-import { pipeline, Readable, Transform, PassThrough } from "stream";
+import { pipeline, Readable, Transform } from "stream";
 import ArchiveStreamToS3 from "decompress-stream-to-s3";
 import { Response } from "express";
 import { lookup } from "mime-types";
@@ -32,23 +33,30 @@ export default class S3Storage extends StorageBase {
       });
   }
 
+  private clients = new Map<number, S3>();
+
   private client(timeout = 10000) {
+    const existing = this.clients.get(timeout);
+    if (existing) return existing;
     if (!config.S3_CLIENT_ID) throw new Error("S3_CLIENT_ID not set");
     if (!config.S3_CLIENT_SECRET) throw new Error("S3_CLIENT_SECRET not set");
     if (!config.S3_REGION) throw new Error("S3_REGION not set");
     if (!config.S3_ENDPOINT) throw new Error("S3_ENDPOINT not set");
-    return new S3({
+    const client = new S3({
       credentials: {
         accessKeyId: config.S3_CLIENT_ID,
         secretAccessKey: config.S3_CLIENT_SECRET,
       },
       region: config.S3_REGION,
       endpoint: config.S3_ENDPOINT,
+      forcePathStyle: true,
       requestHandler: new NodeHttpHandler({
         requestTimeout: timeout,
         connectionTimeout: timeout,
       }),
     });
+    this.clients.set(timeout, client);
+    return client;
   }
 
   /** @override */
@@ -82,6 +90,18 @@ export default class S3Storage extends StorageBase {
     if (!config.S3_BUCKET) throw new Error("S3_BUCKET not set");
     this.assertSafePath(dir);
     const prefix = join(this.repoPath(repoId), dir).replace(/\/$/, "");
+    await this.removeMatching(prefix, key => key === prefix || key.startsWith(prefix + "/"));
+  }
+
+  async removeLegacyContent(repoId: string): Promise<void> {
+    if (!config.S3_BUCKET) throw new Error("S3_BUCKET not set");
+    const prefix = this.repoPath(repoId);
+    await this.removeMatching(prefix, key => key.startsWith(prefix) &&
+      !/^__content\/(?:[a-f0-9]{64}|__retired)(?:\/|$)/.test(key.slice(prefix.length)));
+  }
+
+  private async removeMatching(prefix: string, shouldRemove: (key: string) => boolean) {
+    if (!config.S3_BUCKET) throw new Error("S3_BUCKET not set");
     const client = this.client(200000);
     let continuationToken: string | undefined;
     do {
@@ -92,7 +112,7 @@ export default class S3Storage extends StorageBase {
         ContinuationToken: continuationToken,
       });
       const objects = (data.Contents || [])
-        .filter(({ Key }) => Key === prefix || Key?.startsWith(prefix + "/"))
+        .filter(({ Key }) => Key && shouldRemove(Key))
         .map(({ Key }) => ({ Key: Key! }));
       if (objects.length) {
         const result = await client.deleteObjects({
@@ -212,11 +232,14 @@ export default class S3Storage extends StorageBase {
     let bytesWritten = -1;
     if (typeof expectedSize === "number" && expectedSize > 0 && typeof data !== "string") {
       bytesWritten = 0;
-      const counter = new PassThrough();
-      counter.on("data", (chunk: Buffer) => {
-        bytesWritten += chunk.length;
+      const counter = new Transform({
+        transform(chunk, _encoding, callback) { bytesWritten += chunk.length; callback(null, chunk); },
+        flush(callback) {
+          callback(bytesWritten < expectedSize! ? new AnonymousError("storage_write_size_mismatch", { httpStatus: 502 }) : undefined);
+        },
       });
       data.on("error", (err: Error) => counter.destroy(err));
+      counter.once("close", () => (data as Readable).destroy());
       data.pipe(counter);
       body = counter;
     }
@@ -351,39 +374,36 @@ export default class S3Storage extends StorageBase {
     const archive = archiver(opt?.format || "zip", {});
     if (dir && dir[dir.length - 1] != "/") dir = dir + "/";
 
-    let req: ListObjectsV2CommandOutput;
-    let nextContinuationToken: string | undefined;
-    do {
-      req = await this.client(30000).listObjectsV2({
-        Bucket: config.S3_BUCKET,
-        Prefix: join(this.repoPath(repoId), dir),
-        MaxKeys: 250,
-        ContinuationToken: nextContinuationToken,
-      });
-
-      nextContinuationToken = req.NextContinuationToken;
-      for (const f of req.Contents || []) {
-        if (!f.Key) continue;
-        const filename = basename(f.Key);
-        const prefix = dirname(
-          f.Key.replace(join(this.repoPath(repoId), dir), "")
-        );
-
-        let rs: Readable = await this.read(repoId, f.Key);
-        if (opt?.fileTransformer) {
-          const src = rs;
-          const transformer = opt.fileTransformer(f.Key);
-          src.on("error", (err) => transformer.destroy(err));
-          rs = src.pipe(transformer);
+    let active: Readable | undefined;
+    let stopped = false;
+    archive.once("close", () => { stopped = true; active?.destroy(); });
+    void (async () => {
+      let next: string | undefined;
+      do {
+        const listing = await this.client(30000).listObjectsV2({ Bucket: config.S3_BUCKET!,
+          Prefix: join(this.repoPath(repoId), dir), MaxKeys: 250, ContinuationToken: next });
+        next = listing.IsTruncated ? listing.NextContinuationToken : undefined;
+        for (const file of listing.Contents || []) {
+          if (stopped) return;
+          if (!file.Key || file.Key.endsWith("/")) continue;
+          const filename = file.Key.slice(this.repoPath(repoId).length);
+          const source = await this.read(repoId, filename);
+          if (stopped) { source.destroy(); return; }
+          active = source;
+          let output = source;
+          if (opt?.fileTransformer) {
+            const transformer = opt.fileTransformer(filename);
+            source.on("error", error => transformer.destroy(error));
+            transformer.once("close", () => source.destroy());
+            output = source.pipe(transformer);
+            active = output;
+          }
+          await appendArchiveEntry(archive, output, { name: filename.slice(dir.length) });
         }
-
-        archive.append(rs, {
-          name: filename,
-          prefix,
-        });
-      }
-    } while (req && req.Contents?.length && req.IsTruncated);
-    archive.finalize();
+        if (listing.IsTruncated && !next) throw new Error("Invalid S3 continuation token");
+      } while (next && !stopped);
+      if (!stopped) await archive.finalize();
+    })().catch(error => archive.destroy(error));
     return archive;
   }
 }

src/core/storage/Storage.ts

@@ -120,6 +120,9 @@ export default abstract class StorageBase {
    */
   abstract mk(repoId: string, dir: string): Promise<void>;
 
+  /** Remove unversioned stream caches, preserving the managed content namespace. */
+  abstract removeLegacyContent(repoId: string): Promise<void>;
+
   repoPath(repoId: string) {
     return (
       join(repoId, "original") + (process.platform === "win32" ? "\\" : "/")

src/core/temporary-storage.ts

@@ -0,0 +1,47 @@
+import { promises as fs } from "fs";
+import { join } from "path";
+import { tmpdir } from "os";
+import { pruneTransformedCache } from "./transformed-cache";
+import { createLogger } from "./logger";
+
+const logger = createLogger("temporary-storage");
+
+let nextCleanup = 0;
+let cleanup: Promise<void> | undefined;
+/** Jobs time out in minutes; an hour-old spool belongs to an interrupted process. */
+export async function removeStaleTextSpools(directory = tmpdir(), now = Date.now()) {
+  const entries = await fs.opendir(directory);
+  for await (const entry of entries) {
+    if (!entry.isDirectory() || !entry.name.startsWith("anonymous-text-")) continue;
+    const path = join(directory, entry.name);
+    try {
+      if (now - (await fs.stat(path)).mtimeMs > 3600_000) await fs.rm(path, { recursive: true, force: true });
+    } catch { /* another process can finish cleanup first */ }
+  }
+}
+export function maintainTextSpools(force = false) {
+  if (cleanup) return cleanup;
+  if (!force && Date.now() < nextCleanup) return Promise.resolve();
+  nextCleanup = Date.now() + 60_000;
+  cleanup = removeStaleTextSpools().catch(() => {}).finally(() => { cleanup = undefined; });
+  return cleanup;
+}
+
+/** Recover on startup and continue while idle, without overlapping scans. */
+export async function startTemporaryStorageMaintenance(intervalMs = 60_000) {
+  let running = false;
+  const run = async () => {
+    if (running) return;
+    running = true;
+    try {
+      const results = await Promise.allSettled([maintainTextSpools(true), pruneTransformedCache()]);
+      for (const result of results) if (result.status === "rejected") {
+        logger.warn("temporary storage cleanup deferred", { message: (result.reason as Error).message });
+      }
+    } finally { running = false; }
+  };
+  await run();
+  const timer = setInterval(() => { void run(); }, intervalMs);
+  timer.unref();
+  return () => clearInterval(timer);
+}

src/core/transformed-cache.ts

@@ -0,0 +1,154 @@
+import { promises as fs } from "fs";
+import { join } from "path";
+import { tmpdir } from "os";
+import { createHash, randomUUID } from "crypto";
+import config from "../config";
+import { coordinatedFill } from "./cache-coordination";
+import { isConnected } from "../server/database";
+import RepositoryModel from "./model/anonymizedRepositories/anonymizedRepositories.model";
+import { RepositoryStatus } from "./types";
+import AnonymousError from "./AnonymousError";
+const root = join(tmpdir(), "anonymous-transformed-v1");
+const TTL = 3600_000;
+const DISK_BUDGET = 512 * 1024 * 1024;
+const fills = new Map<string, Promise<boolean>>();
+interface CacheScope { repoId: string; generation?: string; revision?: string }
+
+async function scopeActive(scope?: CacheScope) {
+  if (!scope || !isConnected) return true;
+  const current = await RepositoryModel.findOne({ repoId: scope.repoId })
+    .select("status treeGeneration anonymizeDate contentCacheRevision").lean().exec();
+  return !!current?.status && ![RepositoryStatus.EXPIRING, RepositoryStatus.EXPIRED,
+    RepositoryStatus.REMOVING, RepositoryStatus.REMOVED, RepositoryStatus.ARCHIVED].includes(current.status)
+    && (!scope.generation || scope.generation === `${current.treeGeneration || "legacy"}:${current.anonymizeDate?.toISOString() || ""}`)
+    && scope.revision === current.contentCacheRevision;
+}
+
+async function removeEntry(metadata: string) {
+  await Promise.all([metadata, metadata.replace(/\.json$/, ".data")].map(path => fs.rm(path, { force: true })));
+}
+
+/** The key includes actual content, all rules, generation, and matcher version. */
+export async function transformedFile(digest: string, options: unknown, output: string,
+  produce: () => Promise<boolean>): Promise<boolean> {
+  const settings = options as { repoId?: string; cacheGeneration?: string; cacheRevision?: string } | undefined;
+  const scope: CacheScope | undefined = typeof settings?.repoId === "string"
+    ? { repoId: settings.repoId, generation: settings.cacheGeneration, revision: settings.cacheRevision } : undefined;
+  const assertActive = async () => {
+    if (!await scopeActive(scope)) throw new AnonymousError("repository_changed", { httpStatus: 409 });
+  };
+  const produceActive = async () => {
+    await assertActive();
+    const changed = await produce();
+    try { await assertActive(); }
+    catch (error) { await fs.rm(output, { force: true }); throw error; }
+    return changed;
+  };
+  await assertActive();
+  const key = createHash("sha256").update(JSON.stringify(["matcher-v1", digest, options,
+    config.ANONYMIZATION_MASK, config.APP_HOSTNAME])).digest("hex");
+  const data = join(root, `${key}.data`), metadata = join(root, `${key}.json`);
+  const read = async () => {
+    try {
+      const record = JSON.parse(await fs.readFile(metadata, "utf8"));
+      if (record.version !== 2 || Date.now() - record.created > TTL || (await fs.stat(data)).size !== record.size) return undefined;
+      return Boolean(record.changed);
+    } catch { return undefined; }
+  };
+  const copyCached = async (changed: boolean) => {
+    await fs.copyFile(data, output, fs.constants.COPYFILE_EXCL);
+    try { await assertActive(); } catch (error) { await fs.rm(output, { force: true }); throw error; }
+    return changed;
+  };
+  const hit = await read();
+  if (hit !== undefined) {
+    try { return await copyCached(hit); } catch { /* eviction: retry the transformation */ }
+  }
+  const pending = fills.get(key);
+  if (pending) {
+    await pending.catch(() => {});
+    const cached = await read();
+    if (cached !== undefined) { try { return await copyCached(cached); } catch { /* retry */ } }
+  }
+  if (fills.size >= 64) return produceActive();
+  let producedHere = false;
+  const task = coordinatedFill(`transformed:${key}`, read, async () => {
+    const changed = await produceActive();
+    producedHere = true;
+    const size = (await fs.stat(output)).size;
+    if (size > DISK_BUDGET / 4) return changed;
+    const token = randomUUID();
+    const tempData = `${data}.${token}.tmp`, tempMetadata = `${metadata}.${token}.tmp`;
+    try {
+      await fs.mkdir(root, { recursive: true, mode: 0o700 });
+      await fs.copyFile(output, tempData);
+      await fs.writeFile(tempMetadata, JSON.stringify({ version: 2, changed, size, created: Date.now(), scope }));
+      await fs.rename(tempData, data);
+      await fs.rename(tempMetadata, metadata);
+      await pruneTransformedCache(false);
+    } catch { /* A valid private output remains usable when cache storage fails. */
+    } finally {
+      await Promise.all([tempData, tempMetadata].map(path => fs.rm(path, { force: true }).catch(() => {})));
+    }
+    try { await assertActive(); }
+    catch (error) { await removeEntry(metadata); await fs.rm(output, { force: true }); throw error; }
+    return changed;
+  });
+  fills.set(key, task);
+  try {
+    const changed = await task;
+    if (producedHere) return changed;
+    try { return await copyCached(changed); } catch { return produceActive(); }
+  } finally { if (fills.get(key) === task) fills.delete(key); }
+}
+export async function pruneTransformedCache(validateLifecycle = true) {
+  const records = [];
+  let lifecycleError: Error | undefined;
+  let entries;
+  try { entries = await fs.opendir(root); }
+  catch (error) { if ((error as NodeJS.ErrnoException).code === "ENOENT") return; throw error; }
+  for await (const entry of entries) {
+    const name = entry.name;
+    if (name.endsWith(".tmp") || name.endsWith(".data")) {
+      const path = join(root, name);
+      try { if (Date.now() - (await fs.stat(path)).mtimeMs > TTL) await fs.rm(path, { force: true }); } catch { /* another publisher */ }
+    }
+    if (!name.endsWith(".json")) continue;
+    const path = join(root, name);
+    let record;
+    try { record = JSON.parse(await fs.readFile(path, "utf8")); }
+    catch (error) { if ((error as NodeJS.ErrnoException).code !== "ENOENT") await removeEntry(path); continue; }
+    if (record.version !== 2 || !Number.isFinite(record.created) || !Number.isFinite(record.size)
+      || Date.now() - record.created > TTL) { await removeEntry(path); continue; }
+    if (validateLifecycle && !lifecycleError) {
+      try { if (!await scopeActive(record.scope)) { await removeEntry(path); continue; } }
+      catch (error) { lifecycleError = error as Error; }
+    }
+    records.push({ path, ...record });
+  }
+  records.sort((a, b) => b.created - a.created);
+  let bytes = 0;
+  for (let i = 0; i < records.length; i++) {
+    const record = records[i]; bytes += record.size;
+    if (i >= 128 || bytes > DISK_BUDGET || Date.now() - record.created > TTL) {
+      await removeEntry(record.path);
+    }
+  }
+  // Storage age and budget cleanup must finish even when MongoDB is unavailable.
+  if (lifecycleError) throw lifecycleError;
+}
+
+/** Lifecycle changes remove local entries; connected replicas also prune independently. */
+export async function removeTransformedRepositoryCache(repoId: string) {
+  let entries;
+  try { entries = await fs.opendir(root); }
+  catch (error) { if ((error as NodeJS.ErrnoException).code === "ENOENT") return; throw error; }
+  for await (const entry of entries) {
+    if (!entry.name.endsWith(".json")) continue;
+    const metadata = join(root, entry.name);
+    let record;
+    try { record = JSON.parse(await fs.readFile(metadata, "utf8")); }
+    catch (error) { if ((error as NodeJS.ErrnoException).code !== "ENOENT") await removeEntry(metadata); continue; }
+    if (record.version !== 2 || record.scope?.repoId === repoId) await removeEntry(metadata);
+  }
+}

src/core/zipStream.ts

@@ -1,5 +1,5 @@
 import got from "got";
-import { Readable, Transform } from "stream";
+import { Readable, Transform, Writable } from "stream";
 import { Parse } from "unzip-stream";
 import archiver = require("archiver");
 
@@ -67,8 +67,8 @@ function isEntryAllowed(
  * Stream the GitHub source zip for a repository, anonymize each entry on the
  * fly, and pipe the resulting archive into the provided writable response.
  *
- * No data is written to local storage — the zip flows GitHub → unzip → per
- * file anonymizer → archiver → response.
+ * Large text entries spool privately before worker anonymization. The parser
+ * admits the next entry only after the archive output drains to the response.
  */
 export async function streamAnonymizedZip(
   opt: StreamAnonymizedZipOptions,
@@ -118,6 +118,25 @@ export async function streamAnonymizedZip(
   const activeStreams = new Set<Readable>();
   const compiledTerms = compileTerms(opt.anonymizerOptions.terms || []);
   let stopped = false;
+  let entryWaiting = false;
+  const resumeParser = () => {
+    if (!stopped && entryWaiting && archive.readableLength === 0 && output.writableLength === 0) {
+      entryWaiting = false;
+      parser.resume();
+    }
+  };
+  // Wait for response writes to complete before admitting the next entry.
+  const output = new Writable({
+    write(chunk, encoding, callback) {
+      res.write(chunk, encoding, (error?: Error | null) => {
+        callback(error);
+        setImmediate(resumeParser);
+      });
+    },
+    final(callback) { res.end(); callback(); },
+  });
+  output.on("error", error => fail(error));
+  archive.on("data", () => { if (entryWaiting) setImmediate(resumeParser); });
   const stop = () => {
     if (stopped) return;
     stopped = true;
@@ -125,6 +144,7 @@ export async function streamAnonymizedZip(
     parser.destroy();
     for (const stream of activeStreams) stream.destroy();
     activeStreams.clear();
+    output.destroy();
     archive.abort();
   };
   const fail = (error: Error) => {
@@ -145,7 +165,7 @@ export async function streamAnonymizedZip(
   res.on("error", (error) => fail(error as Error));
   res.on("close", stop);
   archive.on("error", fail);
-  archive.pipe(res);
+  archive.pipe(output);
 
   downloadStream
     .on("error", fail)
@@ -172,6 +192,9 @@ export async function streamAnonymizedZip(
           // there to decide whether the entry is text (and therefore should be
           // anonymized) vs binary (passthrough). Assigning afterwards leaves
           // isText=false for every file, so the zip ships unanonymized.
+          parser.pause();
+          const onArchived = () => { entryWaiting = true; setImmediate(resumeParser); };
+          archive.once("entry", onArchived);
           const anonymizer = new AnonymizeTransformer({
             ...opt.anonymizerOptions,
             filePath: entry.path,
@@ -189,7 +212,7 @@ export async function streamAnonymizedZip(
       }
     })
     .on("error", fail)
-    .on("finish", () => {
+    .on("end", () => {
       if (stopped) return;
       void archive.finalize().catch(fail);
     });