Merge commit 'refs/security-review/pr846' into HEAD

# Conflicts:
#	public/script/main.js

src/core/AnonymizedFile.ts

@@ -1,3 +1,4 @@
+import { streamResponse } from "./response-stream";
 import { join, basename, dirname } from "path";
 import { Response } from "express";
 import { Readable } from "stream";
@@ -9,6 +10,8 @@ import { RepositoryStatus } from "./types";
 import config from "../config";
 import {
   anonymizePath,
+  anonymizePathCompiled,
+  compileTerms,
   hasCustomTermReplacement,
   isTextFile,
 } from "./anonymize-utils";
@@ -19,6 +22,7 @@ import { IFile } from "./model/files/files.types";
 import { FilterQuery } from "mongoose";
 import { createLogger, serializeError } from "./logger";
 import { githubTokenForStreamer } from "./github-token-context";
+import { requestHeaders, startStage } from "./request-monitoring";
 
 const logger = createLogger("anonymized-file");
 
@@ -168,10 +172,12 @@ export default class AnonymizedFile {
     const exactQuery: FilterQuery<IFile> = {
       repoId: this.repository.repoId,
       path: fileDir,
+      treeGeneration: this.repository.model?.treeGeneration || { $exists: false },
     };
     if (filename != "") exactQuery.name = filename;
     const exact = await FileModel.findOne(exactQuery);
     if (exact) {
+      if (exact.metadataPending) await this.repository.completeRecoveredFileMetadata(exact);
       this._file = exact;
       return exact;
     }
@@ -203,6 +209,18 @@ export default class AnonymizedFile {
     // narrow Mongo query. Fetch the repository's paths and verify them by
     // re-applying anonymization. Default XXXX-N masks retain the optimized,
     // anchored query.
+    const compiled = compileTerms(terms);
+    if (typeof this.repository.findAnonymizedPath === "function") {
+      const indexed = await this.repository.findAnonymizedPath(this.anonymizedPath);
+      if (indexed) { this._file = indexed; return indexed; }
+      if (indexed === null) {
+        if (anonymizePathCompiled(this.anonymizedPath, compiled) === this.anonymizedPath) {
+          const recovered = await this.recoverTruncatedFile(fileDir);
+          if (recovered) { this._file = recovered; return recovered; }
+        }
+        throw new AnonymousError("file_not_found", { httpStatus: 404, object: this });
+      }
+    }
     const candidates = usesCustomReplacement
       ? await FileModel.find({ repoId: this.repository.repoId }).exec()
       : await FileModel.find({
@@ -214,8 +232,9 @@ export default class AnonymizedFile {
     for (const candidate of candidates) {
       const candidatePath = join(candidate.path, candidate.name);
       if (
-        anonymizePath(candidatePath, terms) == this.anonymizedPath
+        anonymizePathCompiled(candidatePath, compiled) == this.anonymizedPath
       ) {
+        if (candidate.metadataPending) await this.repository.completeRecoveredFileMetadata(candidate);
         this._file = candidate;
         return candidate;
       }
@@ -253,19 +272,24 @@ export default class AnonymizedFile {
     const recovered = await source.fetchFileInfoFromPath(this.anonymizedPath);
     if (!recovered) return null;
     recovered.repoId = this.repository.repoId;
+    recovered.treeGeneration = this.repository.model.treeGeneration;
     logger.info("recovered file from truncated tree", {
       repoId: this.repository.repoId,
       path: this.anonymizedPath,
     });
+    let cached;
     try {
       // Cache it so the next request is served from the database.
-      await FileModel.create(recovered);
+      cached = await FileModel.create({ ...recovered, metadataPending: true });
     } catch (error) {
       logger.warn(
         "failed to cache recovered file",
         serializeError(error as Error)
       );
     }
+    // Once persisted, invalidation is required. Keep the pending marker on
+    // failure so both a subsequent lookup and maintenance retry the work.
+    if (cached) await this.repository.completeRecoveredFileMetadata(cached);
     return recovered;
   }
 
@@ -356,10 +380,13 @@ export default class AnonymizedFile {
     // use the streamer service
     return got.stream(join(config.STREAMER_ENTRYPOINT, "api"), {
       method: "POST",
+      headers: requestHeaders(),
       json: {
         token: await githubTokenForStreamer(await this.repository.getToken(), this.repository.model.source.repositoryName),
         repoFullName: this.repository.model.source.repositoryName,
         commit: this.repository.model.source.commit,
+        cacheGeneration: `${this.repository.model.treeGeneration || "legacy"}:${this.repository.model.anonymizeDate?.toISOString() || ""}`,
+        cacheRevision: this.repository.model.contentCacheRevision,
         branch: this.repository.model.source.branch,
         repoId: this.repository.repoId,
         filePath: this.filePath,
@@ -398,22 +425,44 @@ export default class AnonymizedFile {
             this.size(),
             this.repository.getToken(),
           ]);
+          const headersDone = startStage("streamer_headers");
           const resStream = got
             .stream(join(config.STREAMER_ENTRYPOINT, "api"), {
               method: "POST",
+              headers: requestHeaders(),
               json: {
                 sha,
                 size,
                 token: await githubTokenForStreamer(token, this.repository.model.source.repositoryName),
                 repoFullName: this.repository.model.source.repositoryName,
                 commit: this.repository.model.source.commit,
+        cacheGeneration: `${this.repository.model.treeGeneration || "legacy"}:${this.repository.model.anonymizeDate?.toISOString() || ""}`,
+                cacheRevision: this.repository.model.contentCacheRevision,
                 branch: this.repository.model.source.branch,
                 repoId: this.repository.repoId,
                 filePath: this.filePath,
                 anonymizerOptions: anonymizer.opt,
               },
-            })
-            .on("error", (err: Error) => {
+            });
+          // Forward Content-Type from the streamer's upstream response.
+          // got.stream(...).pipe(res) forwards body bytes only — without
+          // this, the parent response has no Content-Type and the browser
+          // guesses (text renders as download, images as octet-stream).
+          resStream.on("response", (upstream: { headers: Record<string, string | string[] | undefined> }) => {
+            headersDone();
+            if (res.headersSent) return;
+            const ct = upstream.headers["content-type"];
+            if (typeof ct === "string") {
+              res.contentType(ct);
+            } else {
+              const fallback = lookup(this.anonymizedPath);
+              if (fallback) res.contentType(fallback);
+              else if (isTextFile(this.anonymizedPath)) res.contentType("text/plain");
+            }
+          });
+          resStream.once("error", () => headersDone(true));
+          resStream.once("close", () => headersDone());
+          await streamResponse(resStream, res, (err: Error) => {
               const { error } = streamerErrorToAnonymous(
                 err as Error & {
                   response?: { statusCode?: number; body?: unknown };
@@ -425,36 +474,9 @@ export default class AnonymizedFile {
               );
               error.value = this;
               handleError(error, res);
-            });
-          // Forward Content-Type from the streamer's upstream response.
-          // got.stream(...).pipe(res) forwards body bytes only — without
-          // this, the parent response has no Content-Type and the browser
-          // guesses (text renders as download, images as octet-stream).
-          resStream.on("response", (upstream: { headers: Record<string, string | string[] | undefined> }) => {
-            if (res.headersSent) return;
-            const ct = upstream.headers["content-type"];
-            if (typeof ct === "string") {
-              res.contentType(ct);
-            } else {
-              const fallback = lookup(this.anonymizedPath);
-              if (fallback) res.contentType(fallback);
-              else if (isTextFile(this.anonymizedPath)) res.contentType("text/plain");
-            }
-          });
-          resStream.pipe(res);
-          // Resolve as soon as the response is fully written rather than
-          // waiting for the socket to close — keep-alive sockets stay open
-          // long after the body is delivered, and we don't want to delay
-          // post-send work like countView() that long.
-          res.on("finish", () => {
-            resolve();
-          });
-          res.on("close", () => {
-            resolve();
-          });
-          res.on("error", (err) => {
-            reject(err);
+              if (res.headersSent && !res.writableEnded) res.destroy();
           });
+          resolve();
           return;
         }
 
@@ -480,6 +502,7 @@ export default class AnonymizedFile {
           }
         });
         const content = await this.content();
+        if (res.destroyed || res.writableEnded) { content.destroy(); anonymizer.destroy(); resolve(); return; }
         function handleStreamError(error: Error) {
           if (!content.closed && !content.destroyed) {
             content.destroy();
@@ -505,6 +528,7 @@ export default class AnonymizedFile {
             if (!content.closed && !content.destroyed) {
               content.destroy();
             }
+            anonymizer.destroy();
             resolve();
           });
       } catch (error) {

src/core/Conference.ts

@@ -33,9 +33,7 @@ export default class Conference {
    * Expire the conference
    */
   async expire() {
-    await Promise.all(
-      (await this.repositories()).map(async (conf) => await conf.expire())
-    );
+    for (const repo of await this.repositories()) await repo.markExpired(true);
     await this.updateStatus("expired");
   }
 
@@ -44,9 +42,12 @@ export default class Conference {
    */
   async remove() {
     await this.updateStatus("removed");
-    await Promise.all(
-      (await this.repositories()).map(async (conf) => await conf.remove())
-    );
+    const failures: unknown[] = [];
+    for (const repo of await this.repositories()) {
+      try { await repo.remove(); }
+      catch (error) { failures.push(error); }
+    }
+    if (failures.length) throw new AggregateError(failures, "Conference repository removal failed");
   }
 
   /**

src/core/Gist.ts

@@ -1,4 +1,7 @@
-import { APP_PROVIDER, appError } from "./github-app";
+import { registerGitHubToken } from "./github-token-context";
+import { isConnected } from "../server/database";
+import { expireEmbeddedContent } from "./content-expiration";
+import { APP_PROVIDER, appUserToken } from "./github-app";
 import CredentialModel from "./model/credentials/credentials.model";
 import { getCredentialToken } from "./credentials";
 import { RepositoryStatus } from "./types";
@@ -48,16 +51,39 @@ export default class Gist {
     this.owner.model.isNew = false;
   }
 
-  async getToken() {
-    if (config.GITHUB_APP_ENABLED && !(await getCredentialToken(this.owner.id)) && await CredentialModel.exists({ ownerId: this.owner.id, provider: APP_PROVIDER })) {
-      throw appError("github_oauth_required");
+  private async getAccess() {
+    if (config.GITHUB_APP_ENABLED && await CredentialModel.exists({ ownerId: this.owner.id, provider: APP_PROVIDER })) {
+      const ownerId = this.owner.id;
+      const token = await appUserToken(ownerId);
+      registerGitHubToken(token, { quotaKey: `app-user:${ownerId}`, renew: () => appUserToken(ownerId) });
+      return { token, connection: "github-app" };
     }
-    return (await getCredentialToken(this.owner.id, "github", { collection: "anonymizedgists", id: this._model._id })) || config.GITHUB_TOKEN;
+    return { token: (await getCredentialToken(this.owner.id, "github", { collection: "anonymizedgists", id: this._model._id })) || config.GITHUB_TOKEN,
+      connection: "oauth" };
+  }
+
+  async getToken() {
+    return (await this.getAccess()).token;
   }
 
   async download() {
+    const access = await this.getAccess();
+    try {
+      await this.downloadContent(access.token);
+    } catch (error) {
+      // A stored credential can exist but have been revoked on GitHub.
+      // Ask the user to reconnect the provider that rejected the credential.
+      if ((error as { status?: number })?.status === 401) {
+        throw new AnonymousError(access.connection === "github-app" ? "github_app_reconnect_required" : "github_oauth_required",
+          { httpStatus: 403, cause: error as Error });
+      }
+      throw error;
+    }
+  }
+
+  private async downloadContent(token: string) {
     logger.info("downloading gist", { gistId: this._model.source.gistId });
-    const oct = octokit(await this.getToken());
+    const oct = octokit(token);
 
     const gist_id = this._model.source.gistId;
 
@@ -131,7 +157,7 @@ export default class Gist {
       this._model.options.expirationDate
     ) {
       if (this._model.options.expirationDate <= new Date()) {
-        await this.expire();
+        await this.markExpired();
       }
     }
     if (
@@ -217,10 +243,29 @@ export default class Gist {
     ).exec();
   }
 
+  async markExpired() {
+    const now = new Date();
+    if (isConnected) {
+      const result = await AnonymizedGistModel.updateOne({ _id: this.model._id,
+        status: RepositoryStatus.READY, "options.expirationMode": { $ne: "never" },
+        "options.expirationDate": { $lte: now },
+      }, { $set: { status: RepositoryStatus.EXPIRING, statusDate: now } }).exec();
+      if (!result.matchedCount) {
+        const current = await AnonymizedGistModel.findById(this.model._id).exec();
+        if (!current) throw new AnonymousError("gist_expired", { object: this, httpStatus: 410 });
+        this._model = current;
+        this._gistPayload = undefined;
+        return;
+      }
+    }
+    this.model.status = RepositoryStatus.EXPIRING;
+    this.model.statusDate = now;
+  }
+
   async expire() {
-    await this.updateStatus(RepositoryStatus.EXPIRING);
-    await this.resetSate();
-    await this.updateStatus(RepositoryStatus.EXPIRED);
+    await expireEmbeddedContent(AnonymizedGistModel, this.model, {
+      "gist.comments": [], "gist.description": "", "gist.files": [], "gist.ownerLogin": "",
+    });
   }
 
   async remove() {

src/core/GitHubUtils.ts

@@ -11,6 +11,7 @@ import Repository from "./Repository";
 import { getCredential, replaceCredential, getCredentialToken } from "./credentials";
 import config from "../config";
 import { createLogger } from "./logger";
+import { measureStage } from "./request-monitoring";
 
 const logger = createLogger("github");
 
@@ -154,12 +155,12 @@ function ensureRedisGateClient(): Promise<RedisClientType | null> {
         socket: {
           host: config.REDIS_HOSTNAME,
           port: config.REDIS_PORT,
-          reconnectStrategy: () => false as any,
+          reconnectStrategy: false,
         },
       }) as RedisClientType;
       c.on("error", () => {
         redisGateDisabled = true;
-        c.disconnect().catch(() => {});
+        if (c.isOpen) c.destroy();
         redisGateReady = null;
       });
       await c.connect();
@@ -281,6 +282,7 @@ export function octokit(token: string) {
     }
     throw err;
   });
+  oct.hook.wrap("request", (request, options) => measureStage("upstream", async () => request(options)));
   return oct;
 }
 
@@ -308,6 +310,10 @@ export async function checkToken(token: string) {
 const checkedRepositoryTokens = new WeakMap<Repository, string>();
 
 export async function getToken(repository: Repository) {
+  return measureStage("authorization", () => resolveRepositoryToken(repository));
+}
+
+async function resolveRepositoryToken(repository: Repository) {
   repository.assertNotArchived();
   logger.debug("getToken", { repoId: repository.repoId });
   if (isConnected && !repository.model.isNew) {

src/core/PullRequest.ts

@@ -1,3 +1,5 @@
+import { isConnected } from "../server/database";
+import { expireEmbeddedContent } from "./content-expiration";
 import { boundAppToken } from "./github-app";
 import { getCredentialToken } from "./credentials";
 import { RepositoryStatus } from "./types";
@@ -121,7 +123,7 @@ export default class PullRequest {
       this._model.options.expirationDate
     ) {
       if (this._model.options.expirationDate <= new Date()) {
-        await this.expire();
+        await this.markExpired();
       }
     }
     if (
@@ -228,10 +230,30 @@ export default class PullRequest {
   /**
    * Expire the pullRequest
    */
+  async markExpired() {
+    const now = new Date();
+    if (isConnected) {
+      const result = await AnonymizedPullRequestModel.updateOne({ _id: this.model._id,
+        status: RepositoryStatus.READY, "options.expirationMode": { $ne: "never" },
+        "options.expirationDate": { $lte: now },
+      }, { $set: { status: RepositoryStatus.EXPIRING, statusDate: now } }).exec();
+      if (!result.matchedCount) {
+        const current = await AnonymizedPullRequestModel.findById(this.model._id).exec();
+        if (!current) throw new AnonymousError("pull_request_expired", { object: this, httpStatus: 410 });
+        this._model = current;
+        return;
+      }
+    }
+    this.model.status = RepositoryStatus.EXPIRING;
+    this.model.statusDate = now;
+  }
+
   async expire() {
-    await this.updateStatus(RepositoryStatus.EXPIRING);
-    await this.resetSate();
-    await this.updateStatus(RepositoryStatus.EXPIRED);
+    await expireEmbeddedContent(AnonymizedPullRequestModel, this.model, {
+      "pullRequest.comments": [], "pullRequest.body": "", "pullRequest.title": "", "pullRequest.diff": "",
+      "pullRequest.baseRepositoryFullName": "", "pullRequest.headRepositoryFullName": "",
+      "pullRequest.merged": false, "pullRequest.state": "closed", "pullRequest.draft": false,
+    }, { "pullRequest.mergedDate": "" });
   }
 
   /**

src/core/Repository.ts

@@ -1,5 +1,8 @@
+import AnonymizedPathModel from "./model/anonymized-path";
 import storage from "./storage";
-import { randomUUID } from "crypto";
+import { contentGenerationPrefix, contentRetirementMarker } from "./content-generation";
+import { removeTransformedRepositoryCache } from "./transformed-cache";
+import { createHash, randomUUID } from "crypto";
 import { RepositoryStatus } from "./types";
 import { Readable } from "stream";
 import * as sha1 from "crypto-js/sha1";
@@ -31,7 +34,36 @@ import AnonymizedRepositoryModel from "./model/anonymizedRepositories/anonymized
 import { createLogger, serializeError } from "./logger";
 
 const logger = createLogger("repository");
-import { IFile } from "./model/files/files.types";
+const pathIndexes = new Map<string, Promise<void>>();
+async function buildPathIndex(repoId: string, key: string, terms: string[], generation?: string) {
+  const cacheKey = `${repoId}:${key}`;
+  const existing = pathIndexes.get(cacheKey);
+  if (existing) return existing;
+  const task = (async () => {
+    const compiled = compileTerms(terms);
+    const cursor = FileModel.find({ repoId, treeGeneration: generation || { $exists: false } })
+      .select("name path sha size").sort({ path: 1, name: 1 }).lean().cursor();
+    let batch: Parameters<typeof AnonymizedPathModel.bulkWrite>[0] = [];
+    try {
+      for await (const file of cursor) {
+        const name = anonymizePathCompiled(file.name, compiled);
+        const directory = anonymizePathCompiled(file.path, compiled);
+        batch.push({ updateOne: { filter: { repoId, key, originalId: file._id }, update: { $set: {
+          repoId, key, originalId: file._id, name: file.name, path: file.path, sha: file.sha, size: file.size,
+          anonymousName: name, anonymousDirectory: directory, createdAt: new Date(),
+          searchName: name.toLowerCase(), searchDirectory: directory.toLowerCase(),
+          anonymousPath: anonymizePathCompiled(file.path ? `${file.path}/${file.name}` : file.name, compiled),
+        } }, upsert: true } });
+        if (batch.length === 250) { await AnonymizedPathModel.bulkWrite(batch); batch = []; }
+      }
+      if (batch.length) await AnonymizedPathModel.bulkWrite(batch);
+    } finally { await cursor.close(); }
+  })();
+  // Retain only in-flight work. A subsequent recovery can add file rows.
+  pathIndexes.set(cacheKey, task);
+  try { await task; } finally { pathIndexes.delete(cacheKey); }
+}
+import { IFile, IFileDocument } from "./model/files/files.types";
 import AnonymizedFile from "./AnonymizedFile";
 import { FilterQuery } from "mongoose";
 function anonymizeTreeRecursive(
@@ -87,6 +119,8 @@ export default class Repository {
     return new GitHubStream({
       repoId: this.repoId,
       commit: this.model.source.commit || "HEAD",
+      cacheGeneration: `${this.model.treeGeneration || "legacy"}:${this.model.anonymizeDate?.toISOString() || ""}`,
+      cacheRevision: this.model.contentCacheRevision,
       organization: ghRepo.owner,
       repoName: ghRepo.repo,
       getToken: () => this.getToken(),
@@ -133,51 +167,8 @@ export default class Repository {
       force: false,
     }
   ): Promise<IFile[]> {
-    this.assertNotArchived();
+    await this.ensureFileTree(opt);
     const terms = this._model.options.terms || [];
-    let hasFile = await FileModel.exists({ repoId: this.repoId }).exec();
-    // Files created by GitHubDownload don't carry a valid 40-char GitHub
-    // blob SHA.  When the source type later switches to GitHubStream the
-    // stale entries cause blob-API 404s.  Detect this by sampling a file
-    // with a sha and checking its length; force a re-fetch if it doesn't
-    // look like a GitHub SHA.
-    if (hasFile && this.source instanceof GitHubStream) {
-      const sample = await FileModel.findOne(
-        { repoId: this.repoId, sha: { $exists: true, $ne: null } },
-        { sha: 1 }
-      ).exec();
-      if (sample?.sha && sample.sha.length !== 40) {
-        hasFile = null;
-      }
-    }
-    if (!hasFile || opt.force) {
-      await FileModel.deleteMany({ repoId: this.repoId }).exec();
-      const source = this.source;
-      const files = await source.getFiles(opt.progress);
-      files.forEach((f) => (f.repoId = this.repoId));
-      await FileModel.insertMany(files);
-
-      const sourceWithTruncation = source as unknown as {
-        truncatedFolderList?: string[];
-      };
-      if (Array.isArray(sourceWithTruncation.truncatedFolderList)) {
-        this._model.truncatedFolders = sourceWithTruncation.truncatedFolderList;
-      }
-
-      this._model.size = { storage: 0, file: 0 };
-      await this.computeSize();
-      if (isConnected) {
-        await AnonymizedRepositoryModel.updateOne(
-          { _id: this._model._id },
-          {
-            $set: {
-              truncatedFolders: this._model.truncatedFolders,
-              size: this._model.size,
-            },
-          }
-        ).exec();
-      }
-    }
     if (
       opt.path &&
       (opt.path.includes(config.ANONYMIZATION_MASK) ||
@@ -202,6 +193,7 @@ export default class Repository {
 
     const query: FilterQuery<IFile> = {
       repoId: this.repoId,
+      treeGeneration: this.model.treeGeneration || { $exists: false },
     };
     if (pathQuery !== undefined) {
       query.path = pathQuery;
@@ -209,6 +201,268 @@ export default class Repository {
     return await FileModel.find(query).exec();
   }
 
+  private async ensureFileTree(opt: { force?: boolean; progress?: (status: string) => void } = {}) {
+    this.assertNotArchived();
+    let hasFile = this.hasEmptyTree() || await FileModel.exists({ repoId: this.repoId, treeGeneration: this.model.treeGeneration || { $exists: false } }).exec();
+    // Files created by GitHubDownload don't carry a valid 40-char GitHub
+    // blob SHA.  When the source type later switches to GitHubStream the
+    // stale entries cause blob-API 404s.  Detect this by sampling a file
+    // with a sha and checking its length; force a re-fetch if it doesn't
+    // look like a GitHub SHA.
+    if (hasFile && this.source instanceof GitHubStream) {
+      const sample = await FileModel.findOne(
+        { repoId: this.repoId, treeGeneration: this.model.treeGeneration || { $exists: false }, sha: { $exists: true, $ne: null } },
+        { sha: 1 }
+      ).exec();
+      if (sample?.sha && sample.sha.length !== 40) {
+        hasFile = null;
+      }
+    }
+    if (!hasFile || opt.force) {
+      const previousGeneration = this.model.treeGeneration;
+      const previousPrefix = contentGenerationPrefix(`${previousGeneration || "legacy"}:${this.model.anonymizeDate?.toISOString() || ""}`);
+      const streamed = this.model.source.type !== "Zip";
+      const source = this.source;
+      const files = await source.getFiles(opt.progress);
+      const sourceWithTruncation = source as unknown as { truncatedFolderList?: string[] };
+      const truncatedFolders = Array.isArray(sourceWithTruncation.truncatedFolderList)
+        ? [...sourceWithTruncation.truncatedFolderList] : [];
+      const generation = randomUUID();
+      const buildFilter = { _id: this.model._id,
+        treeGeneration: previousGeneration || { $exists: false }, ...this.refreshFilter(),
+        status: this.model.status, statusDate: this.model.statusDate,
+        "githubAccess.revision": this.model.githubAccess?.revision || { $exists: false },
+      };
+      let heartbeat: NodeJS.Timeout | undefined;
+      if (isConnected) {
+        // Register before insertion so a crash cannot leave an unknown UUID.
+        const registered = await AnonymizedRepositoryModel.updateOne(buildFilter, { $push: {
+          stagedFileTrees: { generation, until: new Date(Date.now() + 15 * 60_000) },
+        } }).exec();
+        if (!registered.matchedCount) throw new AnonymousError("repository_changed", { httpStatus: 409 });
+        heartbeat = setInterval(() => {
+          const now = new Date();
+          void AnonymizedRepositoryModel.updateOne({ ...buildFilter,
+            stagedFileTrees: { $elemMatch: { generation, until: { $gt: now } } },
+          }, { $set: { "stagedFileTrees.$.until": new Date(now.getTime() + 15 * 60_000) } }).exec().catch(() => {});
+        }, 30_000);
+        heartbeat.unref();
+      }
+      files.forEach(f => { f.repoId = this.repoId; f.treeGeneration = generation; if (isConnected) f.treeStaged = true; });
+      try {
+        await FileModel.insertMany(files);
+        if (isConnected) {
+          const activated = await AnonymizedRepositoryModel.updateOne({ _id: this.model._id,
+            treeGeneration: previousGeneration || { $exists: false }, ...this.refreshFilter(),
+            status: this.model.status, statusDate: this.model.statusDate,
+            "githubAccess.revision": this.model.githubAccess?.revision || { $exists: false },
+            stagedFileTrees: { $elemMatch: { generation, until: { $gt: new Date() } } },
+          }, { $set: { treeGeneration: generation, truncatedFolders, size: { storage: 0, file: 0 },
+            ...(streamed && !this.model.contentCacheVersion ? { contentCacheVersion: 1, legacyContentCleanupPending: true } : {}),
+            ...(!files.length ? { emptyTreeGeneration: generation } : {}),
+          }, $unset: { pathIndexKey: "", pathIndexBuiltAt: "", sizeComputedAt: "",
+            ...(files.length ? { emptyTreeGeneration: "" } : {}),
+          }, $addToSet: { retiredTreeGenerations: previousGeneration || "",
+            ...(streamed ? { retiredContentPrefixes: previousPrefix } : {}),
+          }, $pull: { stagedFileTrees: { generation } } }).exec();
+          if (!activated.matchedCount) {
+            await FileModel.deleteMany({ repoId: this.repoId, treeGeneration: generation }).exec();
+            throw new AnonymousError("repository_changed", { httpStatus: 409 });
+          }
+        }
+      } finally { clearInterval(heartbeat); }
+      this.model.treeGeneration = generation;
+      if (isConnected) {
+        // A crash here is safe: maintenance recognizes the active generation.
+        await FileModel.updateMany({ repoId: this.repoId, treeGeneration: generation, treeStaged: true },
+          { $unset: { treeStaged: "" } }).exec().catch(error => logger.warn("staged tree completion deferred", serializeError(error)));
+      }
+      this.model.emptyTreeGeneration = files.length ? undefined : generation;
+      this.model.truncatedFolders = truncatedFolders;
+      if (streamed) this.model.contentCacheVersion = 1;
+      if (isConnected) {
+        await this.cleanupRetiredFileTrees().catch(error => logger.warn("retired tree cleanup deferred", serializeError(error)));
+      } else {
+        await FileModel.deleteMany({ repoId: this.repoId, treeGeneration: previousGeneration || { $exists: false } }).exec();
+      }
+
+      this._model.size = { storage: 0, file: 0 };
+      this._model.sizeComputedAt = undefined;
+      await this.computeSize();
+      if (isConnected) {
+        await AnonymizedRepositoryModel.updateOne(
+          { _id: this._model._id, treeGeneration: this.model.treeGeneration },
+          {
+            $set: {
+              treeGeneration: this._model.treeGeneration,
+              size: this._model.size,
+            },
+          }
+        ).exec();
+      }
+    }
+  }
+
+  /** Activation records retirement atomically; maintenance retries interrupted deletion. */
+  async cleanupRetiredFileTrees(now = new Date()) {
+    const stages = await AnonymizedRepositoryModel.findById(this.model._id).select("stagedFileTrees").lean().exec();
+    for (const stage of stages?.stagedFileTrees || []) {
+      if (stage.until > now) continue;
+      await AnonymizedRepositoryModel.updateOne({ _id: this.model._id, treeGeneration: { $ne: stage.generation },
+        stagedFileTrees: { $elemMatch: { generation: stage.generation, until: { $lte: now } } },
+      }, { $pull: { stagedFileTrees: { generation: stage.generation } },
+        $addToSet: { retiredTreeGenerations: stage.generation } }).exec();
+    }
+    const current = await AnonymizedRepositoryModel.findById(this.model._id)
+      .select("treeGeneration retiredTreeGenerations retiredContentPrefixes legacyContentCleanupPending source.type status anonymizeDate").lean().exec();
+    if (!current) return;
+    const activePrefix = contentGenerationPrefix(`${current.treeGeneration || "legacy"}:${current.anonymizeDate?.toISOString() || ""}`);
+    const prefixes = (current.retiredContentPrefixes || []).filter(prefix => prefix !== activePrefix);
+    for (const prefix of prefixes) {
+      // Fence producers before removal. A late publisher rechecks this marker
+      // and removes its own output instead of recreating a retired cache.
+      await storage.write(this.repoId, contentRetirementMarker(prefix), "retired");
+      await storage.rm(this.repoId, prefix);
+    }
+    if (prefixes.length) await AnonymizedRepositoryModel.updateOne({ _id: current._id },
+      { $pull: { retiredContentPrefixes: { $in: prefixes } } }).exec();
+    if (current.legacyContentCleanupPending && current.status === RepositoryStatus.READY && current.source?.type !== "Zip") {
+      await storage.removeLegacyContent(this.repoId);
+      await AnonymizedRepositoryModel.updateOne({ _id: current._id }, { $unset: { legacyContentCleanupPending: "" } }).exec();
+    }
+    const retired = (current.retiredTreeGenerations || []).filter(generation => generation !== (current.treeGeneration || ""));
+    if (retired.length) {
+      await FileModel.deleteMany({ repoId: this.repoId, $or: [
+        { treeGeneration: { $in: retired.filter(Boolean) } },
+        ...(retired.includes("") ? [{ treeGeneration: { $exists: false } }] : []),
+      ] }).exec();
+      await AnonymizedRepositoryModel.updateOne({ _id: this.model._id },
+        { $pull: { retiredTreeGenerations: { $in: retired } } }).exec();
+    }
+    await AnonymizedRepositoryModel.updateOne({ _id: this.model._id, "retiredTreeGenerations.0": { $exists: false } },
+      { $unset: { retiredTreeGenerations: "" } }).exec();
+    await AnonymizedRepositoryModel.updateOne({ _id: this.model._id, "retiredContentPrefixes.0": { $exists: false } },
+      { $unset: { retiredContentPrefixes: "" } }).exec();
+    await AnonymizedRepositoryModel.updateOne({ _id: this.model._id, "stagedFileTrees.0": { $exists: false } },
+      { $unset: { stagedFileTrees: "" } }).exec();
+  }
+
+  /** Indexed row markers also discover inserts that finish after a build lease is retired. */
+  static async cleanupStagedFileTree(repoId: string, generation: string, now = new Date()) {
+    let current = await AnonymizedRepositoryModel.findOne({ repoId }).exec();
+    if (current && current.treeGeneration !== generation &&
+      current.stagedFileTrees?.some(stage => stage.generation === generation && stage.until > now)) return;
+    if (current) {
+      await new Repository(current).cleanupRetiredFileTrees(now);
+      current = await AnonymizedRepositoryModel.findById(current._id).exec();
+    }
+    const terminal = current?.status && [RepositoryStatus.EXPIRING, RepositoryStatus.EXPIRED,
+      RepositoryStatus.REMOVING, RepositoryStatus.REMOVED].includes(current.status);
+    if (current?.treeGeneration === generation && !terminal) {
+      await FileModel.updateMany({ repoId, treeGeneration: generation, treeStaged: true }, { $unset: { treeStaged: "" } }).exec();
+    } else if (terminal || !current?.stagedFileTrees?.some(stage => stage.generation === generation && stage.until > now)) {
+      await FileModel.deleteMany({ repoId, treeGeneration: generation, treeStaged: true }).exec();
+    }
+  }
+
+  private pathKey() {
+    return createHash("sha256").update(JSON.stringify([
+      "paths-v2", config.ANONYMIZATION_MASK, this.options.terms,
+      this.model.treeGeneration, this.model.fileMetadataRevision, this.model.source.commit, this.model.anonymizeDate,
+    ])).digest("hex");
+  }
+
+  private async indexPaths() {
+    if (this.status && [RepositoryStatus.EXPIRING, RepositoryStatus.EXPIRED,
+      RepositoryStatus.REMOVING, RepositoryStatus.REMOVED].includes(this.status)) {
+      throw new AnonymousError("repository_changed", { httpStatus: 409 });
+    }
+    await this.ensureFileTree();
+    const metadataRevision = this.model.fileMetadataRevision;
+    const key = this.pathKey();
+    if (this.model.pathIndexKey !== key || !this.model.pathIndexBuiltAt || this.model.pathIndexBuiltAt.getTime() < Date.now() - 24 * 3600_000) {
+      try {
+        await buildPathIndex(this.repoId, key, this.options.terms || [], this.model.treeGeneration);
+        const builtAt = new Date();
+        const result = await AnonymizedRepositoryModel.updateOne({ _id: this.model._id,
+          status: this.model.status, statusDate: this.model.statusDate,
+          treeGeneration: this.model.treeGeneration || { $exists: false },
+          fileMetadataRevision: metadataRevision || { $exists: false },
+          "source.commit": this.model.source.commit || { $exists: false },
+          "options.terms": this.options.terms || [], anonymizeDate: this.model.anonymizeDate,
+        }, { $set: { pathIndexKey: key, pathIndexBuiltAt: builtAt } }).exec();
+        if (!result.matchedCount) throw new AnonymousError("repository_changed", { httpStatus: 409 });
+        this.model.pathIndexKey = key;
+        this.model.pathIndexBuiltAt = builtAt;
+      } catch (error) {
+        // A reset invalidates this revision. Never delete a valid shared key
+        // merely because one of its concurrent builders failed.
+        const current = await AnonymizedRepositoryModel.exists({ _id: this.model._id,
+          fileMetadataRevision: metadataRevision || { $exists: false },
+        }).exec();
+        if (!current) await AnonymizedPathModel.deleteMany({ repoId: this.repoId, key }).exec();
+        throw error;
+      }
+      // TTL retires superseded mappings. An older builder must never delete
+      // mappings activated by a newer settings or metadata generation.
+    }
+    return key;
+  }
+
+  async findAnonymizedPath(path: string): Promise<IFile | null | undefined> {
+    if (!isConnected) return undefined;
+    const key = await this.indexPaths();
+    // Original paths retain priority; collisions between transformed paths
+    // resolve consistently in original path/name order.
+    return AnonymizedPathModel.findOne({ repoId: this.repoId, key, anonymousPath: path })
+      .sort({ path: 1, name: 1 }).lean().exec();
+  }
+
+  async invalidateFileMetadata() {
+    const revision = randomUUID();
+    this.model.pathIndexKey = undefined;
+    this.model.pathIndexBuiltAt = undefined;
+    this.model.sizeComputedAt = undefined;
+    this.model.size = { storage: 0, file: 0 };
+    if (isConnected) {
+      const result = await AnonymizedRepositoryModel.updateOne({ _id: this.model._id,
+        treeGeneration: this.model.treeGeneration || { $exists: false },
+        status: this.model.status, statusDate: this.model.statusDate,
+        anonymizeDate: this.model.anonymizeDate,
+      }, { $set: { size: this.model.size, fileMetadataRevision: revision }, $unset: { pathIndexKey: "", pathIndexBuiltAt: "", sizeComputedAt: "" } }).exec();
+      if (!result.matchedCount) throw new AnonymousError("repository_changed", { httpStatus: 409 });
+    }
+    this.model.fileMetadataRevision = revision;
+  }
+
+  async completeRecoveredFileMetadata(file: IFileDocument) {
+    await this.invalidateFileMetadata();
+    await FileModel.updateOne({ _id: file._id, metadataPending: true },
+      { $unset: { metadataPending: "" } }).exec();
+    file.metadataPending = undefined;
+  }
+
+  async searchFiles(query: string): Promise<Partial<IFile>[]> {
+    const q = query.toLowerCase();
+    if (q.includes("/")) return [];
+    if (isConnected) {
+      const key = await this.indexPaths();
+      const pattern = new RegExp(q.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"));
+      const rows = await AnonymizedPathModel.find({ repoId: this.repoId, key,
+        $or: [{ searchName: pattern }, { searchDirectory: pattern }] })
+        .select("anonymousName anonymousDirectory size").limit(500).lean().exec();
+      return rows.map(f => ({ name: f.anonymousName, path: f.anonymousDirectory, size: f.size }));
+    }
+    const matches: Partial<IFile>[] = [];
+    for (const f of await this.anonymizedFiles({ includeSha: false, recursive: true })) {
+      if (f.name?.toLowerCase().includes(q) || (f.path || "").split("/").some(segment => segment.toLowerCase().includes(q))) {
+        matches.push({ name: f.name, path: f.path, size: f.size });
+        if (matches.length === 500) break;
+      }
+    }
+    return matches;
+  }
+
   /**
    * Check the status of the repository
    */
@@ -226,7 +480,7 @@ export default class Repository {
       this._model.options.expirationDate
     ) {
       if (this._model.options.expirationDate <= new Date()) {
-        await this.expire();
+        await this.markExpired();
       }
     }
     if (
@@ -280,7 +534,9 @@ export default class Repository {
   generateAnonymizeTransformer(filePath: string) {
     return new AnonymizeTransformer({
       filePath: filePath,
+      cacheGeneration: `${this.model.treeGeneration || "legacy"}:${this.model.anonymizeDate?.toISOString() || ""}`,
       terms: this.options.terms,
+      cacheRevision: this.model.contentCacheRevision,
       image: this.options.image,
       link: this.options.link,
       repoId: this.repoId,
@@ -291,7 +547,7 @@ export default class Repository {
 
   async isReady() {
     if (this.status !== RepositoryStatus.READY) return false;
-    if (!(await FileModel.exists({ repoId: this.repoId }).exec())) {
+    if (!this.hasEmptyTree() && !(await FileModel.exists({ repoId: this.repoId, treeGeneration: this.model.treeGeneration || { $exists: false } }).exec())) {
       this.model.status = RepositoryStatus.PREPARING;
       await this.updateIfNeeded({ force: true });
       return false;
@@ -299,6 +555,10 @@ export default class Repository {
     return true;
   }
 
+  private hasEmptyTree() {
+    return !!this.model.treeGeneration && this.model.emptyTreeGeneration === this.model.treeGeneration;
+  }
+
   private refreshToken?: string;
 
   private refreshFilter() {
@@ -578,10 +838,85 @@ export default class Repository {
   /**
    * Expire the repository
    */
+  async markExpired(force = false) {
+    const inactive = [RepositoryStatus.ARCHIVED, RepositoryStatus.REMOVED,
+      RepositoryStatus.REMOVING, RepositoryStatus.EXPIRED, RepositoryStatus.EXPIRING];
+    if (force && this.status && inactive.includes(this.status)) return;
+    const now = new Date();
+    if (isConnected) {
+      const result = await AnonymizedRepositoryModel.updateOne({ _id: this.model._id,
+        status: force ? { $nin: inactive } : RepositoryStatus.READY, ...(force ? {} : { "options.expirationMode": { $ne: "never" },
+        "options.expirationDate": { $lte: now } }),
+      }, { $set: { status: RepositoryStatus.EXPIRING, statusDate: now } }).exec();
+      if (!result.matchedCount) {
+        if (force) return;
+        const current = await AnonymizedRepositoryModel.findById(this.model._id).exec();
+        if (!current) throw new AnonymousError("repository_expired", { object: this, httpStatus: 410 });
+        this._model = current;
+        this.assertNotArchived();
+        return;
+      }
+    }
+    this.model.status = RepositoryStatus.EXPIRING;
+    this.model.statusDate = now;
+  }
+
   async expire() {
-    await this.updateStatus(RepositoryStatus.EXPIRING);
-    await this.resetSate();
-    await this.updateStatus(RepositoryStatus.EXPIRED);
+    if (!isConnected) {
+      await this.updateStatus(RepositoryStatus.EXPIRING);
+      await this.resetSate();
+      await this.updateStatus(RepositoryStatus.EXPIRED);
+      return;
+    }
+    const now = new Date(), token = randomUUID(), revision = randomUUID();
+    const pending = this.status === RepositoryStatus.EXPIRING || this.status === RepositoryStatus.EXPIRED;
+    if (!pending && (this.options.expirationMode === "never" || !this.options.expirationDate
+      || this.options.expirationDate > now)) return;
+    if (this.status && [RepositoryStatus.ARCHIVED, RepositoryStatus.REMOVING, RepositoryStatus.REMOVED].includes(this.status)) return;
+    const claimed = await AnonymizedRepositoryModel.updateOne({ _id: this.model._id,
+      status: this.model.status, statusDate: this.model.statusDate,
+      anonymizeDate: this.model.anonymizeDate,
+      treeGeneration: this.model.treeGeneration || { $exists: false },
+      fileMetadataRevision: this.model.fileMetadataRevision || { $exists: false },
+      settingsSavedAt: this.model.settingsSavedAt || { $exists: false },
+      "options.expirationMode": this.options.expirationMode || { $exists: false },
+      "options.expirationDate": this.options.expirationDate || { $exists: false },
+      "githubAccess.revision": this.model.githubAccess?.revision || { $exists: false },
+      $or: [{ cleanupUntil: { $exists: false } }, { cleanupUntil: { $lte: now } }],
+    }, { $set: { status: RepositoryStatus.EXPIRING, statusDate: now,
+      fileMetadataRevision: revision, contentCacheRevision: revision, cleanupToken: token, cleanupUntil: new Date(now.getTime() + 15 * 60_000) },
+      $unset: { pathIndexKey: "", pathIndexBuiltAt: "" },
+    }).exec();
+    if (!claimed.matchedCount) return;
+    this.model.status = RepositoryStatus.EXPIRING;
+    this.model.statusDate = now;
+    this.model.fileMetadataRevision = revision;
+    this.model.contentCacheRevision = revision;
+    this.model.pathIndexKey = undefined;
+    this.model.pathIndexBuiltAt = undefined;
+    const lease = { _id: this.model._id, status: RepositoryStatus.EXPIRING, cleanupToken: token,
+      treeGeneration: this.model.treeGeneration || { $exists: false }, fileMetadataRevision: revision };
+    const heartbeat = setInterval(() => {
+      void AnonymizedRepositoryModel.updateOne(lease, { $set: { cleanupUntil: new Date(Date.now() + 15 * 60_000) } }).exec().catch(() => {});
+    }, 30_000);
+    heartbeat.unref();
+    try {
+      // The claimed lifecycle blocks restoration until the entire storage
+      // root, including empty directories and older cache generations, is gone.
+      if (!await AnonymizedRepositoryModel.exists(lease).exec()) return;
+      await removeTransformedRepositoryCache(this.repoId);
+      await storage.rm(this.repoId);
+      if (!await AnonymizedRepositoryModel.exists(lease).exec()) return;
+      await FileModel.deleteMany({ repoId: this.repoId }).exec();
+      await AnonymizedPathModel.deleteMany({ repoId: this.repoId }).exec();
+      const result = await AnonymizedRepositoryModel.updateOne(lease, { $set: { status: RepositoryStatus.EXPIRED,
+        statusDate: new Date(), isReseted: true, size: { storage: 0, file: 0 } },
+        $unset: { cleanupToken: "", cleanupUntil: "", pathIndexKey: "", pathIndexBuiltAt: "", sizeComputedAt: "", emptyTreeGeneration: "", retiredTreeGenerations: "", retiredContentPrefixes: "", legacyContentCleanupPending: "", stagedFileTrees: "" } }).exec();
+      if (result.matchedCount) { this.model.status = RepositoryStatus.EXPIRED; this.model.isReseted = true; this.model.size = { storage: 0, file: 0 }; this.model.sizeComputedAt = undefined; this.model.emptyTreeGeneration = undefined; }
+    } finally {
+      clearInterval(heartbeat);
+      await AnonymizedRepositoryModel.updateOne({ _id: this.model._id, cleanupToken: token }, { $unset: { cleanupToken: "", cleanupUntil: "" } }).exec();
+    }
   }
 
   /**
@@ -609,6 +944,19 @@ export default class Repository {
    */
   async resetSate(status?: RepositoryStatus, statusMessage?: string) {
     this.assertNotArchived();
+    const revision = randomUUID();
+    this.model.emptyTreeGeneration = undefined;
+    if (isConnected) {
+      const result = await AnonymizedRepositoryModel.updateOne({ _id: this.model._id,
+      treeGeneration: this.model.treeGeneration || { $exists: false },
+      }, { $set: { fileMetadataRevision: revision },
+        $unset: { emptyTreeGeneration: "", pathIndexKey: "", pathIndexBuiltAt: "" },
+      }).exec();
+      if (!result.matchedCount) throw new AnonymousError("repository_changed", { httpStatus: 409 });
+    }
+    this.model.fileMetadataRevision = revision;
+    this.model.pathIndexKey = undefined;
+    this.model.pathIndexBuiltAt = undefined;
     // remove attribute
     this._model.size = { storage: 0, file: 0 };
     if (status) {
@@ -617,6 +965,7 @@ export default class Repository {
     // remove cache
     await Promise.all([
       FileModel.deleteMany({ repoId: this.repoId }).exec(),
+      AnonymizedPathModel.deleteMany({ repoId: this.repoId }).exec(),
       this.removeCache(),
     ]);
     logger.info("reset", { repoId: this._model.repoId });
@@ -629,14 +978,24 @@ export default class Repository {
   async removeCache() {
     // Archive cleanup is handled by the resumable recovery script. Preserve DB metadata.
     if (this.status === RepositoryStatus.ARCHIVED) return;
+    const cacheRevision = randomUUID();
+    if (isConnected) {
+      const result = await AnonymizedRepositoryModel.updateOne({ _id: this.model._id,
+        treeGeneration: this.model.treeGeneration || { $exists: false },
+      }, { $set: { contentCacheRevision: cacheRevision } }).exec();
+      if (!result.matchedCount) throw new AnonymousError("repository_changed", { httpStatus: 409 });
+    }
+    this.model.contentCacheRevision = cacheRevision;
+    await removeTransformedRepositoryCache(this.repoId);
     await storage.rm(this.repoId);
     this.model.isReseted = true;
+    this.model.sizeComputedAt = undefined;
     this.model.size = { storage: 0, file: 0 };
     if (isConnected) {
       try {
         await AnonymizedRepositoryModel.updateOne(
           { _id: this._model._id },
-          { $set: { isReseted: true, size: this._model.size } }
+          { $set: { isReseted: true, size: this._model.size }, $unset: { sizeComputedAt: "" } }
         ).exec();
       } catch (error) {
         logger.error("removeCache save failed", serializeError(error));
@@ -675,11 +1034,12 @@ export default class Repository {
   }> {
     if (this.status !== RepositoryStatus.READY)
       return { storage: 0, file: 0 };
-    if (this._model.size.file) return this._model.size;
+    if (this._model.size.file || this._model.sizeComputedAt) return this._model.size;
     const res = await FileModel.aggregate([
       {
         $match: {
           repoId: this.repoId,
+          treeGeneration: this.model.treeGeneration || { $exists: false },
         },
       },
       {
@@ -694,10 +1054,12 @@ export default class Repository {
       storage: res[0]?.storage || 0,
       file: res[0]?.file || 0,
     };
+    this._model.sizeComputedAt = new Date();
     if (isConnected) {
       await AnonymizedRepositoryModel.updateOne(
-        { _id: this._model._id },
-        { $set: { size: this._model.size } }
+        { _id: this._model._id, treeGeneration: this.model.treeGeneration || { $exists: false },
+          fileMetadataRevision: this.model.fileMetadataRevision || { $exists: false } },
+        { $set: { size: this._model.size, sizeComputedAt: this._model.sizeComputedAt } }
       ).exec();
     }
     return this._model.size;

src/core/User.ts

@@ -14,6 +14,33 @@ import Gist from "./Gist";
 import AnonymizedGistModel from "./model/anonymizedGists/anonymizedGists.model";
 import { octokit } from "./GitHubUtils";
 
+import { isConnected } from "../server/database";
+async function markListExpired<T extends { _id?: unknown; status?: string; statusDate?: Date;
+  options: { expirationMode?: string; expirationDate?: Date } }>(model: {
+  updateMany(filter: Record<string, unknown>, update: Record<string, unknown>): { exec(): Promise<unknown> };
+  find(filter: Record<string, unknown>): { exec(): Promise<T[]> };
+}, rows: T[]): Promise<T[]> {
+  const now = new Date();
+  const expired = rows.filter(row => row.status === "ready" && row.options.expirationMode !== "never"
+    && row.options.expirationDate && row.options.expirationDate <= now);
+  if (!expired.length) return rows;
+  if (isConnected) {
+    const ids = expired.map(row => row._id);
+    await model.updateMany({ _id: { $in: ids }, status: "ready",
+      "options.expirationMode": { $ne: "never" }, "options.expirationDate": { $lte: now },
+    }, { $set: { status: "expiring", statusDate: now } }).exec();
+    const current = new Map((await model.find({ _id: { $in: ids } }).exec()).map(row => [String(row._id), row]));
+    const affected = new Set(ids.map(String));
+    return rows.flatMap(row => {
+      if (!affected.has(String(row._id))) return [row];
+      const fresh = current.get(String(row._id));
+      return fresh ? [fresh] : [];
+    });
+  }
+  for (const row of expired) { row.status = "expiring"; row.statusDate = now; }
+  return rows;
+}
+
 /**
  * Model for a user
  */
@@ -151,6 +178,30 @@ export default class User {
    * @returns the list of anonymized repositories
    */
   async getRepositories() {
+    const query = this.repositoryMembership();
+    const repositories = await AnonymizedRepositoryModel.find(query).exec();
+    return (await markListExpired<typeof repositories[number]>(AnonymizedRepositoryModel, repositories)).map(d => new Repository(d));
+  }
+  /**
+   * Get the lost of anonymized repositories
+   * @returns the list of anonymized repositories
+   */
+  async getPullRequests() {
+    const pullRequests = await AnonymizedPullRequestModel.find({
+      owner: this.id,
+    }).exec();
+    return (await markListExpired<typeof pullRequests[number]>(AnonymizedPullRequestModel, pullRequests)).map(d => new PullRequest(d));
+  }
+
+  /**
+   * Get the list of anonymized gists
+   */
+  async getGists() {
+    const gists = await AnonymizedGistModel.find({ owner: this.id }).exec();
+    return (await markListExpired<typeof gists[number]>(AnonymizedGistModel, gists)).map(d => new Gist(d));
+  }
+
+  repositoryMembership() {
     const memberships: Record<string, unknown>[] = [{ owner: this.id }];
     const githubId = this.model.externalIDs?.github;
     if (githubId) memberships.push({ "coauthors.githubId": githubId });
@@ -160,71 +211,14 @@ export default class User {
         $or: [{ githubId: { $exists: false } }, { githubId: null }, { githubId: "" }],
       } } });
     }
-    const query = { $or: memberships };
-    const repositories = (
-      await AnonymizedRepositoryModel.find(query).exec()
-    ).map((d) => new Repository(d));
-    const promises = [];
-    for (const repo of repositories) {
-      if (
-        repo.status == "ready" &&
-        repo.options.expirationMode != "never" &&
-        repo.options.expirationDate != null &&
-        repo.options.expirationDate < new Date()
-      ) {
-        // expire the repository
-        promises.push(repo.expire());
-      }
-    }
-    await Promise.all(promises);
-    return repositories;
-  }
-  /**
-   * Get the lost of anonymized repositories
-   * @returns the list of anonymized repositories
-   */
-  async getPullRequests() {
-    const pullRequests = (
-      await AnonymizedPullRequestModel.find({
-        owner: this.id,
-      }).exec()
-    ).map((d) => new PullRequest(d));
-    const promises = [];
-    for (const repo of pullRequests) {
-      if (
-        repo.status == "ready" &&
-        repo.options.expirationMode != "never" &&
-        repo.options.expirationDate != null &&
-        repo.options.expirationDate < new Date()
-      ) {
-        // expire the repository
-        promises.push(repo.expire());
-      }
-    }
-    await Promise.all(promises);
-    return pullRequests;
+    return { $or: memberships };
   }
 
-  /**
-   * Get the list of anonymized gists
-   */
-  async getGists() {
-    const gists = (
-      await AnonymizedGistModel.find({ owner: this.id }).exec()
-    ).map((d) => new Gist(d));
-    const promises = [];
-    for (const g of gists) {
-      if (
-        g.status == "ready" &&
-        g.options.expirationMode != "never" &&
-        g.options.expirationDate != null &&
-        g.options.expirationDate < new Date()
-      ) {
-        promises.push(g.expire());
-      }
-    }
-    await Promise.all(promises);
-    return gists;
+  async quotaRepositories() {
+    return AnonymizedRepositoryModel.find({ owner: this.id, status: "ready",
+      $or: [{ "options.expirationMode": "never" }, { "options.expirationDate": { $exists: false } },
+        { "options.expirationDate": null }, { "options.expirationDate": { $gt: new Date() } }] })
+      .select("repoId size sizeComputedAt treeGeneration fileMetadataRevision").lean().exec();
   }
 
   get model() {

src/core/anonymization-pool.ts

@@ -0,0 +1,106 @@
+import { Worker } from "worker_threads";
+import { existsSync } from "fs";
+import { join } from "path";
+import { startStage } from "./request-monitoring";
+import { AsyncResource } from "async_hooks";
+
+const MEMORY_BUDGET = Math.max(128, Number(process.env.ANONYMIZATION_MEMORY_MB) || 512) * 1024 * 1024;
+const MAX_WORKERS = Math.min(4, Math.max(1, Number(process.env.ANONYMIZATION_WORKERS) || 1));
+interface Job {
+  message: { input: string; output: string; options: unknown; maxOutput: number; context: { mask: string; hostname: string } };
+  weight: number;
+  signal: AbortSignal;
+  resolve: (changed: boolean) => void;
+  reject: (error: Error) => void;
+  timer?: NodeJS.Timeout;
+  abort?: () => void;
+  waitingDone: ReturnType<typeof startStage>;
+  transformDone: () => ReturnType<typeof startStage>;
+}
+const waiting: Job[] = [];
+const idle: Worker[] = [];
+let running = 0;
+let bytes = 0;
+
+export function anonymizeOnWorker(message: Job["message"], size: number, signal: AbortSignal): Promise<boolean> {
+  const weight = 64 * 1024 * 1024 + size * 4;
+  if (weight > MEMORY_BUDGET) return Promise.reject(new Error("anonymization_memory_budget_exceeded"));
+  if (waiting.length >= 16) return Promise.reject(new Error("anonymization_queue_full"));
+  return new Promise((resolve, reject) => {
+    const waitingDone = startStage("worker_wait");
+    // Bind the stage factory to the submitting request, not the preceding job.
+    const transformDone = AsyncResource.bind(() => startStage("anonymize"));
+    const job: Job = { message, weight, signal, resolve, reject, waitingDone, transformDone };
+    job.abort = () => {
+      const index = waiting.indexOf(job);
+      if (index < 0) return;
+      waiting.splice(index, 1);
+      clearTimeout(job.timer);
+      signal.removeEventListener("abort", job.abort!);
+      waitingDone(true);
+      reject(new Error("anonymization_cancelled"));
+    };
+    if (signal.aborted) { waitingDone(true); return reject(new Error("anonymization_cancelled")); }
+    signal.addEventListener("abort", job.abort, { once: true });
+    job.timer = setTimeout(job.abort, 30_000);
+    waiting.push(job);
+    pump();
+  });
+}
+function pump() {
+  while (waiting.length && running < MAX_WORKERS && bytes + waiting[0].weight <= MEMORY_BUDGET) {
+    const job = waiting.shift()!;
+    job.waitingDone();
+    const transformDone = job.transformDone();
+    clearTimeout(job.timer);
+    job.signal.removeEventListener("abort", job.abort!);
+    running++;
+    bytes += job.weight;
+    const compiled = join(__dirname, "anonymization.worker.js");
+    let worker: Worker;
+    try { worker = idle.pop() || new Worker(existsSync(compiled) ? compiled : join(__dirname, "anonymization.worker.ts"), {
+      execArgv: existsSync(compiled) ? [] : ["-r", require.resolve("ts-node/register/transpile-only")],
+      resourceLimits: { maxOldGenerationSizeMb: Math.ceil(MEMORY_BUDGET / 1024 / 1024) },
+    }); } catch (error) { running--; bytes -= job.weight; transformDone(true); job.reject(error as Error); continue; }
+    worker.ref();
+    let finished = false;
+    const finish = (error?: Error, changed = false) => {
+      if (finished) return;
+      finished = true;
+      transformDone(!!error);
+      clearTimeout(deadline);
+      job.signal.removeEventListener("abort", cancel);
+      worker.removeListener("message", message);
+      worker.removeListener("error", failed);
+      worker.removeListener("exit", exited);
+      const release = () => { running--; bytes -= job.weight; pump(); };
+      // Large jobs can leave substantial unreachable buffers in V8 until GC.
+      // Release their workers before admitting the next job.
+      if (error || job.weight >= 96 * 1024 * 1024) {
+        void worker.terminate().then(release, release);
+      } else { idle.push(worker); worker.unref(); release(); }
+      if (error) job.reject(error); else job.resolve(changed);
+    };
+    const message = (result: { error?: string; changed?: boolean }) => finish(result.error ? new Error(result.error) : undefined, result.changed);
+    const failed = (error: Error) => finish(error);
+    const exited = () => finish(new Error("anonymization_worker_exited"));
+    const cancel = () => finish(new Error("anonymization_cancelled"));
+    const deadline = setTimeout(() => finish(new Error("anonymization_deadline_exceeded")), 15_000);
+    worker.once("message", message);
+    worker.once("error", failed);
+    worker.once("exit", exited);
+    job.signal.addEventListener("abort", cancel, { once: true });
+    if (job.signal.aborted) cancel(); else worker.postMessage(job.message);
+  }
+}
+
+let spoolBytes = 0;
+export function reserveSpool(bytes: number) {
+  if (spoolBytes + bytes > MEMORY_BUDGET * 2) throw new Error("anonymization_spool_budget_exceeded");
+  spoolBytes += bytes;
+}
+export function releaseSpool(bytes: number) { spoolBytes -= bytes; }
+export function getAnonymizationPoolStats() {
+  return { running, waiting: waiting.length, idle: idle.length, reservedBytes: bytes, spoolBytes,
+    maxWorkers: MAX_WORKERS, maxWaiting: 16, memoryBudgetBytes: MEMORY_BUDGET };
+}

src/core/anonymization.worker.ts

@@ -0,0 +1,26 @@
+import config from "../config";
+import { parentPort } from "worker_threads";
+import { readFileSync, writeFileSync } from "fs";
+import { ContentAnonimizer } from "./anonymize-utils";
+
+parentPort!.on("message", ({ input, output, options, maxOutput, context }) => {
+  try {
+    config.ANONYMIZATION_MASK = context.mask;
+    config.APP_HOSTNAME = context.hostname;
+    const original = readFileSync(input);
+    const text = original.toString("utf8");
+    const anonymizer = new ContentAnonimizer(options);
+    // Large RE2 jobs legitimately take more than the synchronous one-second
+    // limit. The pool also terminates this worker after 15 seconds, including
+    // input/output and compilation. Native backtracking patterns retain their
+    // own one-second deadline inside ContentAnonimizer.
+    const transformed = anonymizer.anonymize(text, 10_000);
+    if (Buffer.byteLength(transformed, "utf8") > maxOutput) throw new Error("anonymized_output_too_large");
+    const bytes = transformed === text ? original : Buffer.from(transformed, "utf8");
+    if (bytes.length > maxOutput) throw new Error("anonymized_output_too_large");
+    writeFileSync(output, bytes, { flag: "wx" });
+    parentPort!.postMessage({ changed: anonymizer.wasAnonymized });
+  } catch (error) {
+    parentPort!.postMessage({ error: (error as Error).message });
+  }
+});

src/core/anonymize-utils.ts

@@ -1,3 +1,10 @@
+import { createHash } from "crypto";
+import { transformedFile } from "./transformed-cache";
+import { promises as fs, createReadStream, ReadStream } from "fs";
+import { tmpdir } from "os";
+import { join } from "path";
+import { anonymizeOnWorker, reserveSpool, releaseSpool } from "./anonymization-pool";
+import { maintainTextSpools } from "./temporary-storage";
 import { RE2JS } from "re2js";
 import { Script } from "vm";
 import { basename } from "path";
@@ -6,6 +13,7 @@ import { isBinaryFileSync } from "isbinaryfile";
 import { lookup as lookupMime } from "mime-types";
 
 import config from "../config";
+import { TermPresenceIndex } from "./term-presence";
 import {
   parseTermSpec,
   termVariants,
@@ -135,11 +143,20 @@ export class AnonymizeTransformer extends Transform {
   anonimizer: ContentAnonimizer;
   private wholeChunks: Buffer[] = [];
   private wholeBytes = 0;
+  private digest = createHash("sha256");
+  private spool?: string;
+  private reservedSpool = 0;
+  private output?: ReadStream;
+  private cancelled = new AbortController();
+  private spoolOperation?: Promise<void>;
+  private readonly memoryThreshold = 256 * 1024;
   private readonly bufferWholeStream: boolean;
 
   constructor(
     readonly opt: {
       filePath: string;
+      cacheGeneration?: string;
+      cacheRevision?: string;
     } & ConstructorParameters<typeof ContentAnonimizer>[0]
   ) {
     super();
@@ -170,12 +187,18 @@ export class AnonymizeTransformer extends Transform {
       this.nameVerdict = this.isText;
     }
     if (this.isText && this.bufferWholeStream) {
+      this.digest.update(chunk);
       this.wholeBytes += chunk.length;
       if (this.wholeBytes > config.MAX_FILE_SIZE) {
         this.wholeChunks = [];
         return callback(new Error(`Text file exceeded ${config.MAX_FILE_SIZE} bytes`));
       }
-      this.wholeChunks.push(chunk);
+      if (this.wholeBytes <= this.memoryThreshold) this.wholeChunks.push(Buffer.from(chunk));
+      else {
+        this.spoolOperation = this.spoolChunk(chunk);
+        void this.spoolOperation.then(() => callback(), error => callback(error));
+        return;
+      }
     } else {
       this.emit("transform", { isText: this.isText, wasAnonimized: false, chunk });
       this.push(chunk);
@@ -183,7 +206,54 @@ export class AnonymizeTransformer extends Transform {
     callback();
   }
 
+  private async spoolChunk(chunk: Buffer) {
+    const additional = chunk.length + (this.spool ? 0 : this.wholeBytes - chunk.length);
+    reserveSpool(additional);
+    this.reservedSpool += additional;
+    if (!this.spool) {
+      await maintainTextSpools();
+      this.spool = await fs.mkdtemp(join(tmpdir(), "anonymous-text-"));
+      await fs.writeFile(join(this.spool, "input"), Buffer.concat(this.wholeChunks));
+      this.wholeChunks = [];
+    }
+    if (this.cancelled.signal.aborted) throw new Error("anonymization_cancelled");
+    await fs.appendFile(join(this.spool, "input"), chunk);
+  }
+
+  _read(size: number) {
+    super._read(size);
+    this.output?.resume();
+  }
+
+  _destroy(error: Error | null, callback: (error?: Error | null) => void) {
+    this.cancelled.abort();
+    this.output?.destroy();
+    this.wholeChunks = [];
+    void (async () => {
+      await this.spoolOperation?.catch(() => {});
+      try { if (this.spool) await fs.rm(this.spool, { recursive: true, force: true }); }
+      finally { releaseSpool(this.reservedSpool); this.reservedSpool = 0; }
+    })().then(() => callback(error), () => callback(error));
+  }
+
   _flush(callback: (error?: Error) => void) {
+    if (this.spool) {
+      const output = join(this.spool, "output");
+      void transformedFile(this.digest.digest("hex"), this.opt, output, () => anonymizeOnWorker({ input: join(this.spool!, "input"), output,
+        options: this.opt, maxOutput: config.MAX_FILE_SIZE * 4, context: { mask: config.ANONYMIZATION_MASK, hostname: config.APP_HOSTNAME } }, this.wholeBytes, this.cancelled.signal))
+        .then(changed => {
+          if (this.destroyed) return;
+          this.anonimizer.wasAnonymized = changed;
+          this.output = createReadStream(output);
+          this.output.on("data", chunk => {
+            this.emit("transform", { isText: true, wasAnonimized: changed, chunk });
+            if (!this.push(chunk)) this.output!.pause();
+          });
+          this.output.once("error", callback);
+          this.output.once("end", () => callback());
+        }, error => callback(error));
+      return;
+    }
     try {
       if (this.nameVerdict === null) this.isText = true;
       if (this.wholeBytes) {
@@ -211,9 +281,14 @@ const markdownImageRegex =
 interface CompiledTermVariant {
   // RE2 for regular patterns; time-limited native fallback for JS extensions.
   pattern: RE2JS | RegExp;
-  // A boundary-free, fixed-width native search can cheaply rule out literal
-  // terms before RE2 scans a large file. A hit still uses the usual matcher.
+  // A boundary-free native search for fixed-width patterns can rule out
+  // absent terms. Candidate replacement retains RE2's boundary semantics.
   literalPrefilter?: RegExp;
+  requiredPrefix?: RegExp;
+  asciiPattern?: RegExp;
+  asciiReplacement?: string;
+  asciiBefore?: boolean;
+  asciiAfter?: boolean;
   before: boolean;
   after: boolean;
   mask: string;
@@ -242,6 +317,29 @@ function hasCatastrophicBacktracking(src: string): boolean {
   return false;
 }
 
+// A leading sequence of literal characters and simple letter classes must
+// occur in every match. Stop before unfamiliar syntax, drop a quantified last
+// atom, and reject alternation entirely. This is only an absence check; RE2
+// still decides every replacement and all boundaries.
+function requiredPrefix(pattern: string): RegExp | undefined {
+  if (pattern.includes("|")) return undefined;
+  const atoms: string[] = [];
+  let offset = 0;
+  while (offset < pattern.length) {
+    const char = String.fromCodePoint(pattern.codePointAt(offset)!);
+    if (char === "[") {
+      const end = pattern.indexOf("]", offset + 1);
+      if (end < 0 || !/^[\p{L}\p{N}]+$/u.test(pattern.slice(offset + 1, end))) break;
+      atoms.push(pattern.slice(offset, end + 1)); offset = end + 1;
+    } else {
+      if (/[\\^$.*+?()[\]{}]/.test(char)) break;
+      atoms.push(char); offset += char.length;
+    }
+  }
+  if (/[?*+{]/.test(pattern.charAt(offset)) && offset < pattern.length) atoms.pop();
+  return atoms.length ? new RegExp(atoms.join(""), "iu") : undefined;
+}
+
 
 function compileTerms(terms: string[] | undefined): CompiledTermVariant[] {
   if (!terms || terms.length === 0) return [];
@@ -269,11 +367,14 @@ function compileTerms(terms: string[] | undefined): CompiledTermVariant[] {
     if (!useAsRegex || hasCatastrophicBacktracking(term)) {
       term = term.replace(/[-[\]{}()*+?.,\\^$|#]/g, "\\$&");
     }
-    // Only plain literals qualify: expanding their letters produces fixed
-    // character classes, never user-controlled backtracking. Do not apply
-    // this shortcut to regex syntax, including escaped literals.
-    const isLiteral = !/[\\^$.*+?()[\]{}|]/.test(term);
+    // Plain text and unquantified dots have fixed width in code points.
+    // Exclude all other regex syntax so the native search cannot backtrack
+    // exponentially. This includes dotted names, emails and domains.
+    const isFixedWidth = !/[\\^$*+?()[\]{}|]/.test(term);
     for (const variant of termVariants(term)) {
+      // Stripping a combining-mark-only term can produce an empty variant.
+      // Keep zero-width matches on RE2's progress-aware iterator.
+      const fixedWidth = isFixedWidth && variant.pattern.length > 0;
       const bounded = withWordBoundaries(variant.pattern, {
         sniffSource: variant.sniff,
         unicode: variant.unicode,
@@ -286,9 +387,16 @@ function compileTerms(terms: string[] | undefined): CompiledTermVariant[] {
           RE2JS.CASE_INSENSITIVE
         );
         compiled.push({ pattern, before, after, mask,
+          requiredPrefix: fixedWidth ? undefined : requiredPrefix(variant.pattern),
+          asciiPattern: fixedWidth ? new RegExp(bounded.replace(/\./g, "[^\\n]"), "giu") : undefined,
+          asciiReplacement: fixedWidth ? mask.replace(/\$/g, () => "$$") : undefined,
+          asciiBefore: !variant.unicode && bounded.startsWith("\\b"),
+          asciiAfter: !variant.unicode && bounded.endsWith("\\b"),
           // RE2 uses Unicode case folding even for the non-diacritic pass.
           // Omit boundaries so this search can only rule out absent terms.
-          literalPrefilter: isLiteral ? new RegExp(variant.pattern, "iu") : undefined });
+          literalPrefilter: fixedWidth
+            // RE2's dot excludes only LF; JS dot also excludes CR and U+2028/9.
+            ? new RegExp(variant.pattern.replace(/\./g, "[^\\n]"), "iu") : undefined });
       } catch {
         // Retain JavaScript-only syntax and large repetition counts under
         // the execution deadline; RE2 handles the common case without backtracking.
@@ -303,13 +411,37 @@ function compileTerms(terms: string[] | undefined): CompiledTermVariant[] {
   return compiled;
 }
 
+interface CompiledTermPlan {
+  terms: CompiledTermVariant[];
+  presence: TermPresenceIndex | null;
+}
+
+// Every file in a ZIP uses the same rules. Reuse immutable compiled patterns
+// across transformers, with bounded retention for unrelated repositories.
+const termPlans = new Map<string, CompiledTermPlan>();
+function compiledTermPlan(terms: string[] | undefined): CompiledTermPlan {
+  const key = JSON.stringify([config.ANONYMIZATION_MASK, terms || []]);
+  const cached = termPlans.get(key);
+  if (cached) return cached;
+  const compiled = compileTerms(terms);
+  const plan = {
+    terms: compiled,
+    presence: compiled.length && compiled.length <= 256 && compiled.every(term => term.literalPrefilter)
+      ? new TermPresenceIndex(compiled.map(term => term.literalPrefilter!)) : null,
+  };
+  if (key.length <= 8192 && compiled.length <= 256) {
+    if (termPlans.size >= 64) termPlans.delete(termPlans.keys().next().value!);
+    termPlans.set(key, plan);
+  }
+  return plan;
+}
+
 export class ContentAnonimizer {
   public wasAnonymized = false;
-  // Compiled once per instance and reused for every anonymize() call.
-  // Streamed files invoke anonymize() many times per file (one per chunk),
-  // so caching here avoids rebuilding regexes on every chunk.
+  // Per-instance metadata references the bounded cache of compiled rules.
   private compiledTerms: CompiledTermVariant[];
   private selfLinkRegexes: RegExp[] | null = null;
+  private termPresence: TermPresenceIndex | null = null;
 
   constructor(
     readonly opt: {
@@ -321,7 +453,11 @@ export class ContentAnonimizer {
       repoId?: string;
     }
   ) {
-    this.compiledTerms = compileTerms(opt.terms);
+    const plan = compiledTermPlan(opt.terms);
+    // Keep per-instance metadata independent. Regex matching is synchronous;
+    // native replacement resets lastIndex and RE2 creates a fresh matcher.
+    this.compiledTerms = plan.terms.map(term => ({ ...term }));
+    this.termPresence = plan.presence;
     if (opt.repoName && opt.branchName) {
       const r = escapeRegex(opt.repoName);
       const b = escapeRegex(opt.branchName);
@@ -370,31 +506,49 @@ export class ContentAnonimizer {
   }
 
   private replaceTerms(content: string): string {
-    for (const c of this.compiledTerms) {
-      // remove whole url if it contains the term
+    if (!this.compiledTerms.length) return content;
+    let candidates = this.termPresence?.candidates(content);
+    // On ASCII input both engines have identical fixed-width matching and
+    // word boundaries. Native replacement avoids allocating one JS match
+    // object per CSV value. Non-ASCII input keeps RE2 validation below.
+    let asciiContent = (!candidates || candidates.size > 0) && /^\p{ASCII}*$/u.test(content);
+    for (let index = 0; index < this.compiledTerms.length; index++) {
+      if (candidates && !candidates.has(index)) continue;
+      const c = this.compiledTerms[index];
+      // An absent term cannot occur inside a URL either.
+      if (c.literalPrefilter && !c.literalPrefilter.test(content)) continue;
+      if (c.requiredPrefix && !c.requiredPrefix.test(content)) continue;
+      const previous = content;
       content = content.replace(urlRegex, (match) => {
-        if (replaceTerm(match, c) !== match) {
+        if (replaceTerm(match, c, true) !== match) {
           this.wasAnonymized = true;
           return c.mask;
         }
         return match;
       });
-      // remove the term in the text
-      const replaced = replaceTerm(content, c);
+      // URL replacement may already have inserted a non-ASCII custom mask.
+      if (content !== previous) asciiContent = asciiContent && /^\p{ASCII}*$/u.test(c.mask);
+      const replaced = replaceTerm(content, c, asciiContent);
       if (replaced !== content) this.wasAnonymized = true;
       content = replaced;
+      // Masks and changed boundaries can introduce later matches. Re-index
+      // only after a real change, never once per absent term.
+      if (content !== previous) {
+        asciiContent = asciiContent && /^\p{ASCII}*$/u.test(c.mask);
+        candidates = this.termPresence?.candidates(content, index);
+      }
     }
     return content;
   }
 
-  anonymize(content: string): string {
+  anonymize(content: string, timeoutMs = 1000): string {
     return runWithAnonymizationDeadline(() => {
       content = this.removeImage(content);
       content = this.removeLink(content);
       content = this.replaceGitHubSelfLinks(content);
       content = this.replaceTerms(content);
       return content;
-    });
+    }, timeoutMs);
   }
 }
 
@@ -433,42 +587,70 @@ function escapeRegex(value: string): string {
 // V8 interrupts even a native RegExp that never returns to JavaScript. A
 // timeout aborts the operation instead of returning partially anonymized text.
 const anonymizationScript = new Script("run()");
-function runWithAnonymizationDeadline(run: () => string): string {
-  return anonymizationScript.runInNewContext({ run }, { timeout: 1000 });
+function runWithAnonymizationDeadline(run: () => string, timeoutMs = 1000): string {
+  return anonymizationScript.runInNewContext({ run }, { timeout: timeoutMs });
 }
 
-function replaceTerm(content: string, term: CompiledTermVariant): string {
-  if (term.literalPrefilter && !term.literalPrefilter.test(content)) return content;
-  if (term.pattern instanceof RegExp) {
-    return content.replace(term.pattern, () => term.mask);
+function replaceTerm(content: string, term: CompiledTermVariant, asciiContent = false): string {
+  const pattern = term.pattern;
+  if (pattern instanceof RegExp) {
+    // JavaScript-only patterns can backtrack even inside an isolated worker.
+    // Keep their short limit independent of the whole-file processing budget.
+    return runWithAnonymizationDeadline(() => content.replace(pattern, () => term.mask));
   }
-  // Fixed-width literal searches run natively. Validate each candidate with
-  // RE2 on a small window to preserve its case folding and boundary semantics
-  // without scanning megabytes of notebook outputs in the JS regex engine.
-  const candidates = term.literalPrefilter
-    ? content.matchAll(new RegExp(term.literalPrefilter.source, "giu"))
-    : null;
-  const matcher = candidates ? null : term.pattern.matcher(content);
+  if (term.requiredPrefix && !term.requiredPrefix.test(content)) return content;
+  if (term.literalPrefilter && asciiContent && term.asciiPattern) {
+    return content.replace(term.asciiPattern, term.asciiReplacement!);
+  }
+  if (term.literalPrefilter) {
+    const validated = new Map<string, boolean>();
+    const candidates = new RegExp(term.literalPrefilter.source, "giu");
+    const pieces: string[] = [];
+    let cursor = 0;
+    let match: RegExpExecArray | null;
+    while ((match = candidates.exec(content))) {
+      const candidate = match[0], start = match.index, end = start + candidate.length;
+      let valid = true;
+      if (term.before && /[\p{L}\p{N}_]$/u.test(content.slice(Math.max(0, start - 2), start))) valid = false;
+      if (term.after && /^[\p{L}\p{N}_]/u.test(content.slice(end, end + 2))) valid = false;
+      // On ASCII candidates the fixed-width native and RE2 searches agree.
+      // Check RE2's ASCII word boundaries explicitly: JS /iu\b/ also treats
+      // Kelvin sign and long s as word characters, while RE2 does not.
+      if (valid && /^\p{ASCII}*$/u.test(candidate)) {
+        if (term.asciiBefore && asciiWordAt(content, start - 1) === asciiWordAt(content, start)) valid = false;
+        if (term.asciiAfter && asciiWordAt(content, end - 1) === asciiWordAt(content, end)) valid = false;
+      } else if (valid) {
+        // Keep RE2 authoritative for Unicode case folding. Repeated values in
+        // datasets share small contexts; cap this cache independently of size.
+        const offset = Math.max(0, start - 2);
+        const window = content.slice(offset, end + 2);
+        const key = `${start - offset}:${window}`;
+        const cached = validated.get(key);
+        if (cached === undefined) {
+          const check = (term.pattern as RE2JS).matcher(window);
+          valid = check.find(start - offset) && check.start() === start - offset && check.end() === end - offset;
+          if (validated.size < 256) validated.set(key, valid);
+        } else valid = cached;
+      }
+      if (!valid) {
+        // A boundary-rejected candidate must not hide an overlapping match,
+        // e.g. the second a-a in "xa-a-a". Advance by one Unicode code point.
+        candidates.lastIndex = start + (content.codePointAt(start)! > 0xffff ? 2 : 1);
+        continue;
+      }
+      pieces.push(content.slice(cursor, start), term.mask);
+      cursor = end;
+    }
+    if (!pieces.length) return content;
+    pieces.push(content.slice(cursor));
+    return pieces.join("");
+  }
+
+  const matcher = pattern.matcher(content);
   const pieces: string[] = [];
   let cursor = 0;
-  const matches = function* () {
-    if (candidates) {
-      for (const candidate of candidates) {
-        const start = candidate.index!;
-        const end = start + candidate[0].length;
-        const offset = Math.max(0, start - 2);
-        const check = (term.pattern as RE2JS).matcher(content.slice(offset, end + 2));
-        if (check.find(start - offset) && check.start() === start - offset && check.end() === end - offset) {
-          yield { start, end };
-        }
-      }
-    } else {
-      while (matcher!.find()) yield { start: matcher!.start(), end: matcher!.end() };
-    }
-  };
-  for (const { start, end } of matches()) {
-    // RE2 has no lookahead. Check the generated Unicode word boundaries
-    // outside the engine, without executing any user-supplied native regex.
+  while (matcher.find()) {
+    const start = matcher.start(), end = matcher.end();
     if (term.before && /[\p{L}\p{N}_]$/u.test(content.slice(Math.max(0, start - 2), start))) continue;
     if (term.after && /^[\p{L}\p{N}_]/u.test(content.slice(end, end + 2))) continue;
     pieces.push(content.slice(cursor, start), term.mask);
@@ -478,3 +660,8 @@ function replaceTerm(content: string, term: CompiledTermVariant): string {
   pieces.push(content.slice(cursor));
   return pieces.join("");
 }
+
+function asciiWordAt(content: string, index: number): boolean {
+  const code = content.charCodeAt(index);
+  return code >= 48 && code <= 57 || code >= 65 && code <= 90 || code >= 97 && code <= 122 || code === 95;
+}

src/core/archive-entry.ts

@@ -0,0 +1,17 @@
+import { Archiver, EntryData } from "archiver";
+import { Readable } from "stream";
+/** Allow only one active entry; compression and the response provide backpressure. */
+export function appendArchiveEntry(archive: Archiver, source: Readable, options: EntryData): Promise<void> {
+  return new Promise((resolve, reject) => {
+    const cleanup = () => {
+      archive.removeListener("entry", done); archive.removeListener("error", fail);
+      archive.removeListener("close", closed); source.removeListener("error", fail);
+    };
+    const done = () => { cleanup(); resolve(); };
+    const fail = (error: Error) => { cleanup(); source.destroy(); reject(error); };
+    const closed = () => fail(new Error("archive_cancelled"));
+    archive.once("entry", done); archive.once("error", fail); archive.once("close", closed);
+    source.once("error", fail);
+    archive.append(source, options);
+  });
+}

src/core/async-cache.ts

@@ -0,0 +1,22 @@
+/** Bounded TTL cache with one calculation per key and retry after failure. */
+export class AsyncCache<T> {
+  private entries = new Map<string, { expires: number; value: Promise<T> }>();
+  constructor(private ttl: number, private capacity: number, private now = () => Date.now()) {}
+  get size() { this.prune(); return this.entries.size; }
+  clear() { this.entries.clear(); }
+  private prune() {
+    for (const [key, entry] of this.entries) if (entry.expires <= this.now()) this.entries.delete(key);
+  }
+  get(key: string, calculate: () => Promise<T>): Promise<T> {
+    this.prune();
+    const cached = this.entries.get(key);
+    if (cached) return cached.value;
+    if (this.entries.size >= this.capacity) this.entries.delete(this.entries.keys().next().value!);
+    const entry = { expires: Infinity, value: Promise.resolve().then(calculate) };
+    this.entries.set(key, entry);
+    entry.value.then(() => { entry.expires = this.now() + this.ttl; }, () => {
+      if (this.entries.get(key) === entry) this.entries.delete(key);
+    });
+    return entry.value;
+  }
+}

src/core/cache-coordination.ts

@@ -0,0 +1,83 @@
+import { createClient } from "redis";
+import { randomUUID } from "crypto";
+import config from "../config";
+
+let connection: ReturnType<typeof createClient> | undefined;
+let connecting: Promise<ReturnType<typeof createClient> | undefined> | undefined;
+let retryAt = 0;
+/** Bound commands even when a connected server stops replying. */
+export async function cacheCommand<T>(command: Promise<T>, client: ReturnType<typeof createClient>, timeoutMs = 1000): Promise<T> {
+  let timer: NodeJS.Timeout | undefined;
+  try {
+    return await Promise.race([command, new Promise<never>((_, reject) => {
+      timer = setTimeout(() => {
+        retryAt = Date.now() + 30_000;
+        if (connection === client) connection = undefined;
+        if (client.isOpen) client.destroy();
+        reject(new Error("cache_command_timeout"));
+      }, timeoutMs);
+    })]);
+  } finally { clearTimeout(timer); }
+}
+export async function cacheRedis() {
+  if (connection?.isReady) return connection;
+  if (connecting) return connecting;
+  if (Date.now() < retryAt) return undefined;
+  connecting = (async () => {
+    const client = createClient({ disableOfflineQueue: true, socket: {
+      host: config.REDIS_HOSTNAME, port: config.REDIS_PORT,
+      connectTimeout: 500, reconnectStrategy: false,
+    } });
+    client.on("error", () => {});
+    try { await cacheCommand(client.connect(), client, 500); connection = client; return client; }
+    catch { retryAt = Date.now() + 30_000; if (client.isOpen) client.destroy(); return undefined; }
+    finally { connecting = undefined; }
+  })();
+  return connecting;
+}
+
+/** A renewable lease. Followers recheck durable output; they never share a Readable. */
+export async function coordinatedFill<T>(
+  key: string, read: () => Promise<T | undefined>, produce: () => Promise<T>,
+  waitMs = 120_000
+): Promise<T> {
+  const redis = await cacheRedis();
+  if (!redis) return produce();
+  const lockKey = `perf:lease:${key}`;
+  const token = randomUUID();
+  const deadline = Date.now() + waitMs;
+  for (;;) {
+    const value = await read();
+    if (value !== undefined) return value;
+    let owned: string | null;
+    try { owned = await cacheCommand(redis.set(lockKey, token, { NX: true, PX: 30_000 }), redis); }
+    catch { return produce(); }
+    if (owned) {
+      const heartbeat = setInterval(() => {
+        void cacheCommand(redis.eval("if redis.call('get',KEYS[1]) == ARGV[1] then return redis.call('pexpire',KEYS[1],ARGV[2]) else return 0 end", {
+          keys: [lockKey], arguments: [token, "30000"],
+        }), redis).catch(() => {});
+      }, 10_000);
+      heartbeat.unref();
+      try {
+        // Coordination is optional. A lost heartbeat cannot invalidate output
+        // that the producer has already completed and verified.
+        return await produce();
+      } finally {
+        clearInterval(heartbeat);
+        await cacheCommand(redis.eval("if redis.call('get',KEYS[1]) == ARGV[1] then return redis.call('del',KEYS[1]) else return 0 end", {
+          keys: [lockKey], arguments: [token],
+        }), redis).catch(() => {});
+      }
+    }
+    if (Date.now() >= deadline) throw new Error("cache_fill_timeout");
+    await new Promise(resolve => setTimeout(resolve, 100));
+  }
+}
+
+export async function closeCacheRedis() {
+  const current = connection;
+  connection = undefined;
+  retryAt = 0;
+  if (current?.isOpen) current.destroy();
+}

src/core/content-expiration.ts

@@ -0,0 +1,35 @@
+import { Document, Model } from "mongoose";
+import { RepositoryStatus } from "./types";
+
+type ContentSnapshot = Document & {
+  status?: RepositoryStatus;
+  statusDate?: Date;
+  anonymizeDate?: Date;
+  options: { expirationMode?: string; expirationDate?: Date };
+  githubAccess?: { revision?: string };
+};
+
+// Embedded content and lifecycle state live in one document. Claim and clear
+// them in one conditional write so an extension or rebuild cannot interleave.
+export async function expireEmbeddedContent<T extends ContentSnapshot>(
+  model: Model<T>, data: T, fields: Record<string, unknown>, unset: Record<string, ""> = {}
+) {
+  const now = new Date();
+  const result = await model.updateOne({
+    _id: data._id,
+    status: data.status ?? { $exists: false },
+    statusDate: data.statusDate ?? { $exists: false },
+    anonymizeDate: data.anonymizeDate ?? { $exists: false },
+    "options.expirationMode": data.options.expirationMode ?? { $exists: false },
+    "options.expirationDate": data.options.expirationDate ?? { $exists: false },
+    "githubAccess.revision": data.githubAccess?.revision ?? { $exists: false },
+    $and: [
+      { status: { $in: [RepositoryStatus.READY, RepositoryStatus.EXPIRING] } },
+      { "options.expirationMode": { $ne: "never" } },
+      { "options.expirationDate": { $lte: now } },
+    ],
+  }, { $set: { ...fields, status: RepositoryStatus.EXPIRED, statusDate: now }, $unset: unset }).exec();
+  if (!result.matchedCount) return;
+  data.set({ ...fields, status: RepositoryStatus.EXPIRED, statusDate: now });
+  for (const key of Object.keys(unset)) data.set(key, undefined);
+}

src/core/content-generation.ts

@@ -0,0 +1,9 @@
+import { createHash } from "crypto";
+export function contentGenerationPrefix(generation: string) {
+  return `__content/${createHash("sha256").update(generation).digest("hex")}`;
+}
+
+export function contentRetirementMarker(prefix: string) {
+  if (!/^__content\/[a-f0-9]{64}$/.test(prefix)) throw new Error("Invalid content generation prefix");
+  return `__content/__retired/${prefix.slice("__content/".length)}.txt`;
+}

src/core/github-app.ts

@@ -253,23 +253,23 @@ export async function boundAppToken(ownerId: string, binding: RepositoryAccess,
   return token;
 }
 
-export async function selectRepositoryAccess(ownerId: string, fullName: string, choice?: unknown): Promise<{ token: string; binding: RepositoryAccess }> {
+export async function selectRepositoryAccess(ownerId: string, fullName: string, choice?: unknown, repositoryId?: number): Promise<{ token: string; binding: RepositoryAccess; fullName?: string }> {
   if (!/^[^/\s]+\/[^/\s]+$/.test(fullName)) throw appError("repo_not_found", 400);
   if (choice !== undefined && choice !== "oauth" && choice !== "github-app") throw appError("invalid_connection", 400);
   const hasApp = config.GITHUB_APP_ENABLED && await CredentialModel.exists({ ownerId, provider: APP_PROVIDER });
   if (choice === "github-app" || (choice === undefined && hasApp)) {
-    const repo = (await appRepositories(ownerId)).find(r => r.full_name.toLowerCase() === fullName.toLowerCase());
+    const repo = (await appRepositories(ownerId)).find(r => repositoryId === undefined ? r.full_name.toLowerCase() === fullName.toLowerCase() : r.id === repositoryId);
     if (!repo) {
       const userToken = await appUserToken(ownerId);
       const publicRepo = await githubRequest<GitHubRepositoryInfo>(
-        `/repos/${fullName.split("/").map(encodeURIComponent).join("/")}`, userToken);
+        repositoryId === undefined ? `/repos/${fullName.split("/").map(encodeURIComponent).join("/")}` : `/repositories/${repositoryId}`, userToken);
       if (publicRepo.private !== false || publicRepo.visibility !== "public" || !Number.isSafeInteger(publicRepo.id)) throw appError("github_app_access_required");
       const binding: RepositoryAccess = { kind: "github-app", publicRead: true,
         repositoryId: publicRepo.id, revision: randomUUID() };
-      return { binding, token: await boundAppToken(ownerId, binding, fullName) };
+      return { binding, fullName: publicRepo.full_name, token: await boundAppToken(ownerId, binding, publicRepo.full_name) };
     }
     const binding: RepositoryAccess = { kind: "github-app", repositoryId: repo.id, installationId: repo.installationId, revision: randomUUID() };
-    return { binding, token: await boundAppToken(ownerId, binding, fullName) };
+    return { binding, fullName: repo.full_name, token: await boundAppToken(ownerId, binding, repo.full_name) };
   }
   const token = await getCredentialToken(ownerId);
   if (!token) throw appError("github_oauth_required");

src/core/logger.ts

@@ -74,7 +74,7 @@ function getRedis(): RedisClientType | null {
       const c = redisClient;
       redisClient = null;
       if (c?.isOpen) {
-        c.disconnect().catch(() => {});
+        c.destroy();
       }
     });
     redisClient.connect().catch(() => {

src/core/model/anonymized-path.ts

@@ -0,0 +1,11 @@
+import { IFile } from "./files/files.types";
+import { model, Schema } from "mongoose";
+const schema = new Schema({ repoId: { type: String, required: true }, key: { type: String, required: true }, originalId: Schema.Types.ObjectId,
+  name: { type: String, required: true }, path: { type: String, default: "", required: true }, sha: String, size: Number,
+  anonymousName: String, anonymousDirectory: String, anonymousPath: String,
+  searchName: String, searchDirectory: String,
+  createdAt: { type: Date, default: Date.now } });
+schema.index({ repoId: 1, key: 1, anonymousPath: 1, path: 1, name: 1 });
+schema.index({ repoId: 1, key: 1, originalId: 1 }, { unique: true });
+schema.index({ createdAt: 1 }, { expireAfterSeconds: 30 * 24 * 3600 });
+export default model<IFile & { key: string; originalId: Schema.Types.ObjectId; anonymousName: string; anonymousDirectory: string; anonymousPath: string; searchName: string; searchDirectory: string; createdAt: Date }>("AnonymizedPath", schema);

src/core/model/anonymizedGists/anonymizedGists.schema.ts

@@ -68,4 +68,6 @@ const AnonymizedGistSchema = new Schema({
   },
 });
 
+AnonymizedGistSchema.index({ status: 1, "options.expirationDate": 1 });
+
 export default AnonymizedGistSchema;

src/core/model/anonymizedPullRequests/anonymizedPullRequests.schema.ts

@@ -65,4 +65,6 @@ const AnonymizedPullRequestSchema = new Schema({
   },
 });
 
+AnonymizedPullRequestSchema.index({ status: 1, "options.expirationDate": 1 });
+
 export default AnonymizedPullRequestSchema;

src/core/model/anonymizedRepositories/anonymizedRepositories.schema.ts

@@ -48,6 +48,19 @@ const AnonymizedRepositorySchema = new Schema({
     repositoryName: String,
     accessToken: { type: String, select: false },
   },
+  cleanupToken: { type: String, select: false },
+  cleanupUntil: { type: Date, select: false },
+  treeGeneration: String,
+  stagedFileTrees: { type: [{ generation: String, until: Date }], default: undefined },
+  retiredTreeGenerations: { type: [String], default: undefined },
+  retiredContentPrefixes: { type: [String], default: undefined },
+  legacyContentCleanupPending: Boolean,
+  contentCacheVersion: Number,
+  contentCacheRevision: String,
+  emptyTreeGeneration: String,
+  fileMetadataRevision: String,
+  pathIndexKey: String,
+  pathIndexBuiltAt: Date,
   truncatedFolders: {
     type: [String],
     default: [],
@@ -71,6 +84,7 @@ const AnonymizedRepositorySchema = new Schema({
     type: Date,
     default: Date.now,
   },
+  sizeComputedAt: Date,
   size: {
     storage: {
       type: Number,
@@ -88,10 +102,17 @@ const AnonymizedRepositorySchema = new Schema({
 });
 
 AnonymizedRepositorySchema.index({ "source.repositoryName": 1 });
+AnonymizedRepositorySchema.index({ "coauthors.githubId": 1 });
+AnonymizedRepositorySchema.index({ retiredTreeGenerations: 1 }, { sparse: true });
+AnonymizedRepositorySchema.index({ "stagedFileTrees.until": 1 }, { sparse: true });
+AnonymizedRepositorySchema.index({ retiredContentPrefixes: 1 }, { sparse: true });
+AnonymizedRepositorySchema.index({ legacyContentCleanupPending: 1, status: 1 },
+  { partialFilterExpression: { legacyContentCleanupPending: true } });
 AnonymizedRepositorySchema.index({ status: 1, statusDate: 1 });
 AnonymizedRepositorySchema.index({ lastView: 1 });
 AnonymizedRepositorySchema.index({ anonymizeDate: 1 });
 AnonymizedRepositorySchema.index({ status: 1, isReseted: 1, lastView: 1 });
+AnonymizedRepositorySchema.index({ status: 1, "options.expirationDate": 1 });
 AnonymizedRepositorySchema.index({
   status: 1,
   isReseted: 1,

src/core/model/anonymizedRepositories/anonymizedRepositories.types.ts

@@ -30,6 +30,17 @@ export interface IAnonymizedRepository {
     photo?: string;
     addedAt?: Date;
   }[];
+  treeGeneration?: string;
+  stagedFileTrees?: { generation: string; until: Date }[];
+  retiredTreeGenerations?: string[];
+  retiredContentPrefixes?: string[];
+  legacyContentCleanupPending?: boolean;
+  contentCacheVersion?: number;
+  contentCacheRevision?: string;
+  emptyTreeGeneration?: string;
+  fileMetadataRevision?: string;
+  pathIndexKey?: string;
+  pathIndexBuiltAt?: Date;
   truncatedFolders: string[];
   conference: string;
   options: {
@@ -49,6 +60,7 @@ export interface IAnonymizedRepository {
   };
   pageView: number;
   lastView: Date;
+  sizeComputedAt?: Date;
   size: {
     storage: number;
     file: number;

src/core/model/dashboard-name.ts

@@ -0,0 +1,5 @@
+import { model, Schema } from "mongoose";
+const schema = new Schema({ owner: { type: Schema.Types.ObjectId, required: true },
+  type: { type: String, required: true }, artifactId: { type: String, required: true }, name: String });
+schema.index({ owner: 1, type: 1, artifactId: 1 }, { unique: true });
+export default model("DashboardName", schema);

src/core/model/files/files.schema.ts

@@ -4,6 +4,9 @@ const FileSchema = new Schema({
   name: { type: String, index: true },
   path: { type: String, index: true },
   repoId: { type: String, index: true },
+  treeGeneration: String,
+  treeStaged: Boolean,
+  metadataPending: Boolean,
   sha: {
     type: String,
   },
@@ -13,6 +16,11 @@ const FileSchema = new Schema({
 });
 
 FileSchema.index({ path: 1, repoId: 1 });
+FileSchema.index({ metadataPending: 1, repoId: 1, treeGeneration: 1 },
+  { partialFilterExpression: { metadataPending: true } });
+FileSchema.index({ treeStaged: 1, repoId: 1, treeGeneration: 1 },
+  { partialFilterExpression: { treeStaged: true } });
+FileSchema.index({ repoId: 1, treeGeneration: 1, path: 1, name: 1 });
 FileSchema.index(
   { repoId: 1, size: 1, path: 1 },
   { name: "repoId_1_size_1_path_1" }

src/core/model/files/files.types.ts

@@ -4,6 +4,9 @@ export interface IFile {
   name: string;
   path: string;
   repoId: string;
+  treeGeneration?: string;
+  treeStaged?: boolean;
+  metadataPending?: boolean;
   sha?: string;
   size?: number;
 }

src/core/model/users/users.schema.ts

@@ -31,6 +31,7 @@ const UserSchema = new Schema({
     },
   ],
   photo: String,
+  projectNames: { type: Map, of: String },
   repositories: [
     {
       type: String,

src/core/model/users/users.types.ts

@@ -25,6 +25,7 @@ export interface IUser {
     default: boolean;
   }[];
   photo?: string;
+  projectNames?: Map<string, string>;
 
   repositories?: number[];
   default?: {

src/core/performance-monitoring.ts

@@ -0,0 +1,216 @@
+import { createClient, RedisClientType } from "redis";
+import { hostname } from "os";
+import { readdir, readFile, readlink } from "fs/promises";
+import { monitorEventLoopDelay, performance } from "perf_hooks";
+import config from "../config";
+import { AsyncCache } from "./async-cache";
+import { activeRequestCount, LATENCY_BUCKETS, Metric, requestMetrics, Service } from "./request-monitoring";
+
+const PREFIX = "performance:v1:";
+const RETENTION = 2 * 3600;
+let client: RedisClientType | undefined;
+let stop: (() => Promise<void>) | undefined;
+let flushNow: (() => Promise<void>) | undefined;
+const reports = new AsyncCache<unknown>(10000, 2);
+const emptyMetric = (metric: string, route: string, method: string): Metric => ({ metric, route, method,
+  count: 0, sumMs: 0, bytes: 0, aborted: 0, errors: 0, slow: 0, buckets: Array(LATENCY_BUCKETS.length + 1).fill(0) });
+
+async function containerMemoryLimit() {
+  for (const path of ["/sys/fs/cgroup/memory.max", "/sys/fs/cgroup/memory/memory.limit_in_bytes"]) {
+    const raw = await readFile(path, "utf8").catch(() => "");
+    const value = Number(raw.trim());
+    if (value > 0 && value < Number.MAX_SAFE_INTEGER) return value;
+  }
+  return null;
+}
+
+async function redisDeadline<T>(work: Promise<T>, redis: RedisClientType): Promise<T> {
+  let timer: NodeJS.Timeout | undefined;
+  try {
+    return await Promise.race([work, new Promise<never>((_resolve, reject) => {
+      timer = setTimeout(() => {
+        reject(new Error("performance_monitoring_timeout"));
+        if (redis.isOpen) {
+          redis.destroy();
+          if (client === redis) void redis.connect().catch(() => {});
+        }
+      }, 2000); timer.unref();
+    })]);
+  } finally { clearTimeout(timer); }
+}
+
+export async function flushPerformanceMetrics() { await flushNow?.(); reports.clear(); }
+
+export function percentileUpperBound(buckets: number[], percentile: number): number | null {
+  const total = buckets.reduce((sum, count) => sum + count, 0);
+  if (!total) return null;
+  let count = 0;
+  for (let i = 0; i < buckets.length; i++) {
+    count += buckets[i];
+    if (count >= Math.ceil(total * percentile)) return LATENCY_BUCKETS[i] ?? null;
+  }
+  return null;
+}
+
+/** PID-owned sockets only: /proc/net also includes healthcheck subprocesses. */
+export async function socketSnapshot() {
+  try {
+    const descriptors = (await readdir("/proc/self/fd")).slice(0, 4096);
+    const sockets = new Set<string>();
+    for (let i = 0; i < descriptors.length; i += 32) {
+      const links = await Promise.all(descriptors.slice(i, i + 32).map(fd => readlink(`/proc/self/fd/${fd}`).catch(() => "")));
+      for (const link of links) { const match = /^socket:\[(\d+)\]$/.exec(link); if (match) sockets.add(match[1]); }
+    }
+    const states: Record<string, number> = {};
+    for (const file of ["/proc/self/net/tcp", "/proc/self/net/tcp6"]) {
+      const table = await readFile(file, "utf8");
+      for (const line of table.trim().split("\n").slice(1)) {
+        const columns = line.trim().split(/\s+/);
+        if (sockets.has(columns[9])) states[columns[3]] = (states[columns[3]] || 0) + 1;
+      }
+    }
+    return { descriptors: sockets.size, established: states["01"] || 0, closeWait: states["08"] || 0,
+      listening: states["0A"] || 0, truncated: descriptors.length === 4096 };
+  } catch { return null; }
+}
+
+/** One bounded batch every 15s. Redis outages never queue commands or hold HTTP requests. */
+export function startPerformanceMonitoring(service: Service, workers: () => unknown = () => null) {
+  if (stop) return stop;
+  const redis = createClient({ disableOfflineQueue: true, socket: { host: config.REDIS_HOSTNAME,
+    port: config.REDIS_PORT, connectTimeout: 2000, reconnectStrategy: (retries: number) => Math.min(5000, 250 * (retries + 1)) } }) as RedisClientType;
+  client = redis;
+  redis.on("error", () => {});
+  void redis.connect().catch(() => {});
+  const loop = monitorEventLoopDelay({ resolution: 20 }); loop.enable();
+  let previousUtilization = performance.eventLoopUtilization();
+  let previousCpu = process.cpuUsage();
+  let previousCpuAt = performance.now();
+  let busy = false;
+  let droppedBatches = 0;
+  let lastFlushAt: number | null = null;
+  let stopped = false;
+  const instance = `${service}:${hostname()}`;
+  const flush = async () => {
+    if (busy || stopped) return;
+    busy = true;
+    const entries = requestMetrics.drain();
+    try {
+      if (!redis.isReady) { droppedBatches += entries.length ? 1 : 0; return; }
+      const utilization = performance.eventLoopUtilization(previousUtilization);
+      previousUtilization = performance.eventLoopUtilization();
+      const cpu = process.cpuUsage(previousCpu);
+      const cpuAt = performance.now();
+      const cpuPercent = (cpu.user + cpu.system) / ((cpuAt - previousCpuAt) * 1000) * 100;
+      previousCpu = process.cpuUsage(); previousCpuAt = cpuAt;
+      const runtime = { instance, service, sampledAt: Date.now(), uptime: process.uptime(),
+        cpuPercent, memoryLimitBytes: await containerMemoryLimit(),
+        memory: process.memoryUsage(), activeRequests: activeRequestCount(), sockets: await socketSnapshot(), workers: workers(),
+        eventLoop: { utilization: utilization.utilization, p95Ms: loop.percentile(95) / 1e6,
+          maxMs: Number.isFinite(loop.max) ? loop.max / 1e6 : 0 },
+        droppedMetrics: requestMetrics.dropped, droppedBatches, lastFlushAt };
+      loop.reset();
+      const tx = redis.multi();
+      const touched = new Set<string>();
+      for (const { minute, data } of entries) {
+        const key = `${PREFIX}minute:${service}:${minute}`;
+        touched.add(key);
+        const label = JSON.stringify([data.metric, data.route, data.method]);
+        const fields: Record<string, number> = { count: data.count, sumMs: data.sumMs, bytes: data.bytes,
+          aborted: data.aborted, errors: data.errors, slow: data.slow };
+        data.buckets.forEach((count, i) => { fields[`b${i}`] = count; });
+        for (const [field, value] of Object.entries(fields)) if (value) tx.hIncrBy(key, `${label}|${field}`, value);
+      }
+      for (const key of touched) tx.expire(key, RETENTION);
+      tx.set(`${PREFIX}instance:${instance}`, JSON.stringify(runtime), { EX: 90 });
+      tx.zAdd(`${PREFIX}instances`, { score: runtime.sampledAt, value: instance });
+      tx.zRemRangeByScore(`${PREFIX}instances`, 0, runtime.sampledAt - 90000);
+      tx.expire(`${PREFIX}instances`, 120);
+      tx.lPush(`${PREFIX}runtime:${service}`, JSON.stringify(runtime));
+      tx.lTrim(`${PREFIX}runtime:${service}`, 0, service === "api" ? 239 : 959);
+      tx.expire(`${PREFIX}runtime:${service}`, RETENTION);
+      await redisDeadline(tx.exec(), redis); lastFlushAt = Date.now();
+    } catch { droppedBatches++; }
+    finally { busy = false; }
+  };
+  const timer = setInterval(() => { void flush(); }, 15000); timer.unref();
+  flushNow = flush;
+  void flush();
+  stop = async () => {
+    stopped = true; clearInterval(timer); loop.disable();
+    if (redis.isOpen) redis.destroy();
+    client = undefined; stop = undefined; flushNow = undefined; reports.clear();
+  };
+  return stop;
+}
+
+export async function performanceReport(minutes: number) {
+  const windowMinutes = minutes === 60 ? 60 : 15;
+  if (!client?.isReady) return { available: false, windowMinutes, instances: [], routes: [], stages: [] };
+  const redis = client;
+  return reports.get(String(windowMinutes), async () => {
+    const minute = Math.floor(Date.now() / 60000) * 60000;
+    const groups = new Map<string, Metric & { service: Service }>();
+    const tx = redis.multi();
+    const requests: { service: Service; minute: number }[] = [];
+    for (const service of ["api", "streamer"] as const) {
+      for (let i = 0; i < windowMinutes; i++) {
+        requests.push({ service, minute: minute - i * 60000 });
+        tx.hGetAll(`${PREFIX}minute:${service}:${minute - i * 60000}`);
+      }
+    }
+    const values = await redisDeadline(tx.exec(), redis) as unknown as Record<string, string>[];
+    const series = new Map<number, { minute: number; requests: number; aborted: number; errors: number; sumMs: number }>();
+    values.forEach((hash, index) => {
+      for (const [key, raw] of Object.entries(hash || {})) {
+        const split = key.lastIndexOf("|");
+        const label = key.slice(0, split); const field = key.slice(split + 1);
+        const [metric, route, method] = JSON.parse(label) as string[];
+        const service = requests[index].service;
+        const id = `${service}:${label}`;
+        let row = groups.get(id);
+        if (!row) { row = { ...emptyMetric(metric, route, method), service }; groups.set(id, row); }
+        const value = Number(raw) || 0;
+        if (/^b\d+$/.test(field)) row.buckets[Number(field.slice(1))] += value;
+        else if (["count", "sumMs", "bytes", "aborted", "errors", "slow"].includes(field)) {
+          const numericField = field as "count" | "sumMs" | "bytes" | "aborted" | "errors" | "slow";
+          row[numericField] += value;
+        }
+        if (metric === "request" && service === "api") {
+          const timestamp = requests[index].minute;
+          const point = series.get(timestamp) || { minute: timestamp, requests: 0, aborted: 0, errors: 0, sumMs: 0 };
+          if (field === "count") point.requests += value;
+          if (field === "sumMs") point.sumMs += value;
+          if (field === "aborted") point.aborted += value;
+          if (field === "errors") point.errors += value;
+          series.set(timestamp, point);
+        }
+      }
+    });
+    const ids = await redisDeadline(redis.zRangeByScore(`${PREFIX}instances`, Date.now() - 90000, "+inf", { LIMIT: { offset: 0, count: 32 } }), redis);
+    const samples = ids.length ? await redisDeadline(redis.mGet(ids.map(id => `${PREFIX}instance:${id}`)), redis) : [];
+    const instances = samples.filter((sample): sample is string => !!sample).map(sample => JSON.parse(sample));
+    const runtimeSeries: { instance: string; service: string; sampledAt: number; rss: number; heapUsed: number; external: number;
+      cpuPercent: number; loopP95Ms: number; sockets: number | null; closeWait: number | null; waiting: number; reservedBytes: number }[] = [];
+    for (const service of ["api", "streamer"]) {
+      const samples = await redisDeadline(redis.lRange(`${PREFIX}runtime:${service}`, 0, service === "api" ? 239 : 959), redis);
+      for (const raw of samples) {
+        const sample = JSON.parse(raw);
+        if (sample.sampledAt >= Date.now() - windowMinutes * 60000) runtimeSeries.push({ service, instance: sample.instance,
+          sampledAt: sample.sampledAt, rss: sample.memory.rss, heapUsed: sample.memory.heapUsed, external: sample.memory.external,
+          cpuPercent: sample.cpuPercent, loopP95Ms: sample.eventLoop.p95Ms, sockets: sample.sockets?.descriptors ?? null,
+          closeWait: sample.sockets?.closeWait ?? null, waiting: sample.workers?.waiting || 0, reservedBytes: sample.workers?.reservedBytes || 0 });
+      }
+    }
+    const rows = [...groups.values()].map(row => ({ ...row, avgMs: row.count ? Math.round(row.sumMs / row.count) : 0,
+      p95UpperMs: percentileUpperBound(row.buckets, .95), p99UpperMs: percentileUpperBound(row.buckets, .99) }));
+    return { available: true, generatedAt: Date.now(), windowMinutes, instances,
+      routes: rows.filter(row => row.metric === "request").map(row => {
+        const firstByte = rows.find(sample => sample.metric === "first_byte" && sample.service === row.service && sample.route === row.route && sample.method === row.method);
+        return { ...row, firstByteCount: firstByte?.count || 0, firstByteP95UpperMs: firstByte?.p95UpperMs ?? null };
+      }).sort((a, b) => b.sumMs - a.sumMs),
+      stages: rows.filter(row => row.metric !== "request"), series: [...series.values()].sort((a, b) => a.minute - b.minute),
+      runtimeSeries, latencyBucketsMs: LATENCY_BUCKETS,
+      percentileNote: "Percentiles are histogram bucket upper bounds; null means no samples or above 60s." };
+  });
+}

src/core/request-monitoring.ts

@@ -0,0 +1,159 @@
+import { AsyncLocalStorage } from "async_hooks";
+import { randomUUID } from "crypto";
+import { performance } from "perf_hooks";
+import { RequestHandler } from "express";
+import { createLogger } from "./logger";
+
+export const LATENCY_BUCKETS = [10, 50, 100, 250, 500, 1000, 2500, 5000, 10000, 30000, 60000];
+export type Service = "api" | "streamer";
+export type Stage = "repository" | "authorization" | "upstream" | "cache_lookup" | "cache_fill" | "worker_wait" | "anonymize" | "streamer_headers";
+export interface Metric {
+  metric: string; route: string; method: string; count: number; sumMs: number;
+  bytes: number; aborted: number; errors: number; slow: number; buckets: number[];
+}
+const logger = createLogger("requests");
+const context = new AsyncLocalStorage<{ id: string; service: Service; stages: Partial<Record<Stage, number>> }>();
+const methods = new Set(["GET", "HEAD", "POST", "PUT", "PATCH", "DELETE", "OPTIONS"]);
+const staticRoutes = new Set([
+  "/", "/dashboard", "/anonymize", "/healthcheck", "/api/healthcheck", "/api/stat", "/api/stat/history",
+  "/api/options", "/api/message", "/api/user", "/api/user/quota", "/api/user/dashboard", "/api/repo",
+  "/api/anonymize-preview", "/api/admin/overview", "/api/admin/performance", "/api/admin/errors",
+  "/api/admin/errors/stats", "/api/admin/queues", "/api/admin/queues/metrics", "/github/app/webhook",
+]);
+
+/** Fixed labels: never use repository IDs, file names or query strings as metric keys. */
+export function requestRoute(url: string, service: Service): string {
+  const path = url.split("?", 1)[0].replace(/\/+$/, "") || "/";
+  if (service === "streamer") return ["/api/download", "/api", "/healthcheck"].includes(path) ? path : "other";
+  if (staticRoutes.has(path)) return path;
+  if (/^\/api\/repo\//.test(path)) {
+    const match = path.match(/^\/api\/repo\/[^/]+\/(file|files|zip|options|refresh|coauthors|extend)(?:\/|$)/);
+    if (match) {
+      const subroute = match[1] === "files" && /\/(counts|search)$/.exec(path);
+      return `/api/repo/:repoId/${match[1]}${match[1] === "file" ? "/:path" : subroute ? `/${subroute[1]}` : ""}`;
+    }
+    const github = path.match(/^\/api\/repo\/[^/]+\/[^/]+\/(readme|branches)$/);
+    if (github) return `/api/repo/:owner/:repo/${github[1]}`;
+    return "/api/repo/:repoId";
+  }
+  if (path.startsWith("/w/")) return "/w/:repoId/:path";
+  if (/^\/(r|repository)\//.test(path)) return "/r/:repoId/:path";
+  if (/^\/api\/(pr|gist)\//.test(path)) return `/api/${path.split("/")[2]}/:id/:path`;
+  if (/^\/(script|css|favicon|fonts|partials)\//.test(path)) return "/assets/:path";
+  if (path.startsWith("/github/")) return "/github/:action";
+  if (path.startsWith("/api/admin/")) return "/api/admin/:action";
+  if (path.startsWith("/api/user/")) return "/api/user/:action";
+  return "other";
+}
+
+export class RequestMetrics {
+  private pending = new Map<string, { minute: number; data: Metric }>();
+  dropped = 0;
+  constructor(private capacity = 512, private now = () => Date.now()) {}
+  get size() { return this.pending.size; }
+  observe(metric: string, route: string, method: string, ms: number,
+    detail: { bytes?: number; aborted?: boolean; error?: boolean; slow?: boolean } = {}) {
+    const minute = Math.floor(this.now() / 60000) * 60000;
+    const key = JSON.stringify([minute, metric, route, method]);
+    let entry = this.pending.get(key);
+    if (!entry) {
+      if (this.pending.size >= this.capacity) { this.dropped++; return; }
+      entry = { minute, data: { metric, route, method, count: 0, sumMs: 0, bytes: 0,
+        aborted: 0, errors: 0, slow: 0, buckets: Array(LATENCY_BUCKETS.length + 1).fill(0) } };
+      this.pending.set(key, entry);
+    }
+    const duration = Math.max(0, Number.isFinite(ms) ? ms : 0);
+    const data = entry.data;
+    data.count++; data.sumMs += Math.round(duration); data.bytes += detail.bytes || 0;
+    data.aborted += Number(!!detail.aborted); data.errors += Number(!!detail.error); data.slow += Number(!!detail.slow);
+    let bucket = LATENCY_BUCKETS.findIndex(bound => duration <= bound);
+    if (bucket < 0) bucket = LATENCY_BUCKETS.length;
+    data.buckets[bucket]++;
+  }
+  drain() { const entries = [...this.pending.values()]; this.pending.clear(); return entries; }
+}
+export const requestMetrics = new RequestMetrics();
+let activeRequests = 0;
+export function activeRequestCount() { return activeRequests; }
+export function requestHeaders(): Record<string, string> {
+  const id = context.getStore()?.id;
+  return id ? { "x-request-id": id } : {};
+}
+export function recordCacheHit(cache: "source" | "transformed", hit: boolean) {
+  if (context.getStore()) requestMetrics.observe(`${cache}_${hit ? "hit" : "miss"}`, "all", "all", 0);
+}
+
+/** Capture the context now: worker admission can resume from another request's callback. */
+export function startStage(stage: Stage) {
+  const started = performance.now();
+  const trace = context.getStore();
+  let ended = false;
+  return (error = false) => {
+    if (ended) return; ended = true;
+    if (!trace) return;
+    const ms = performance.now() - started;
+    trace.stages[stage] = (trace.stages[stage] || 0) + ms;
+    requestMetrics.observe(stage, "all", "all", ms, { error });
+  };
+}
+export async function measureStage<T>(stage: Stage, work: () => Promise<T>): Promise<T> {
+  const done = startStage(stage);
+  try { const value = await work(); done(); return value; }
+  catch (error) { done(true); throw error; }
+}
+
+export function monitorRequests(service: Service, metrics = requestMetrics,
+  log: Pick<typeof logger, "info" | "warn"> = logger): RequestHandler {
+  return (req, res, next) => {
+    const forwarded = service === "streamer" ? req.get("x-request-id") : undefined;
+    const id = forwarded && /^[a-f0-9-]{36}$/.test(forwarded) ? forwarded : randomUUID();
+    const trace = { id, service, stages: {} };
+    const route = requestRoute(req.originalUrl, service);
+    const method = methods.has(req.method) ? req.method : "OTHER";
+    const started = performance.now();
+    let firstByteMs: number | undefined;
+    let bytes = 0;
+    let ended = false;
+    activeRequests++;
+    res.setHeader("X-Request-ID", id);
+    const writeHead = res.writeHead;
+    res.writeHead = function (this: typeof res, ...args: unknown[]): typeof res {
+      firstByteMs ??= performance.now() - started;
+      return Reflect.apply(writeHead, this, args);
+    };
+    const count = (chunk: unknown, encoding: unknown) => {
+      if (req.method === "HEAD") return;
+      if (typeof chunk === "string") bytes += Buffer.byteLength(chunk, typeof encoding === "string" ? encoding as BufferEncoding : undefined);
+      else if (chunk instanceof Uint8Array) bytes += chunk.byteLength;
+    };
+    // Preserve callbacks, return values and receiver. Installed before compression,
+    // these count encoded body bytes without changing stream backpressure.
+    const write = res.write;
+    res.write = function (this: typeof res, ...args: unknown[]): boolean {
+      count(args[0], args[1]);
+      return Reflect.apply(write, this, args);
+    };
+    const end = res.end;
+    res.end = function (this: typeof res, ...args: unknown[]): typeof res {
+      count(args[0], args[1]);
+      return Reflect.apply(end, this, args);
+    };
+    const done = () => {
+      if (ended) return; ended = true;
+      activeRequests--;
+      res.removeListener("finish", done); res.removeListener("close", done);
+      const ms = performance.now() - started;
+      const aborted = !res.writableFinished;
+      const transfer = /\/(file|zip)(?:\/|$)/.test(route) || route.startsWith("/w/") || service === "streamer";
+      const slow = ms > (transfer ? 10000 : 1000);
+      metrics.observe("request", route, method, ms, { bytes, aborted, error: res.statusCode >= 500, slow });
+      if (firstByteMs != null) metrics.observe("first_byte", route, method, firstByteMs);
+      const detail = { requestId: id, service, method, route, status: res.statusCode,
+        outcome: aborted ? "interrupted" : "completed", ms: Math.round(ms), firstByteMs: firstByteMs == null ? null : Math.round(firstByteMs), bytes, stages: trace.stages };
+      if (slow || aborted) log.warn(aborted ? "request interrupted" : "slow request", detail);
+      else if (!route.includes("healthcheck")) log.info("request", detail);
+    };
+    res.once("finish", done); res.once("close", done);
+    context.run(trace, next);
+  };
+}

src/core/response-stream.ts

@@ -0,0 +1,24 @@
+import { Readable, Writable } from "stream";
+
+/** Cancel upstream on every response terminal event, including a pre-header close. */
+export function streamResponse(source: Readable, response: Writable,
+  onError: (error: Error) => void): Promise<void> {
+  return new Promise((resolve, reject) => {
+    const cleanup = () => {
+      response.removeListener("close", done);
+      response.removeListener("finish", done);
+      response.removeListener("error", failed);
+      source.removeListener("error", upstreamFailed);
+      source.unpipe(response);
+      source.destroy();
+    };
+    const done = () => { cleanup(); resolve(); };
+    const failed = (error: Error) => { cleanup(); reject(error); };
+    const upstreamFailed = (error: Error) => { onError(error); done(); };
+    response.once("close", done);
+    response.once("finish", done);
+    response.once("error", failed);
+    source.once("error", upstreamFailed);
+    if (response.destroyed || response.writableEnded) done(); else source.pipe(response);
+  });
+}

src/core/source/GitHubBase.ts

@@ -14,6 +14,8 @@ export interface GitHubBaseData {
   organization: string;
   repoName: string;
   commit: string;
+  cacheGeneration?: string;
+  cacheRevision?: string;
 }
 
 export default abstract class GitHubBase implements SourceBase {

src/core/source/GitHubDownload.ts

@@ -1,6 +1,5 @@
 import got from "got";
 import { Readable } from "stream";
-import { OctokitResponse } from "@octokit/types";
 
 import storage from "../storage";
 import GitHubBase, {
@@ -18,7 +17,7 @@ export default class GitHubDownload extends GitHubBase {
     super(data);
   }
 
-  public async getZipUrl(): Promise<OctokitResponse<unknown, 302>> {
+  public async getZipUrl() {
     const oct = octokit(await this.data.getToken());
     return oct.rest.repos.downloadZipballArchive({
       owner: this.data.organization,
@@ -29,7 +28,7 @@ export default class GitHubDownload extends GitHubBase {
   }
 
   async download(progress?: (status: string) => void) {
-    let response: OctokitResponse<unknown, number>;
+    let response: Awaited<ReturnType<GitHubDownload["getZipUrl"]>>;
     try {
       response = await this.getZipUrl();
     } catch (error) {

src/core/source/GitHubStream.ts

@@ -1,3 +1,6 @@
+import { contentGenerationPrefix, contentRetirementMarker } from "../content-generation";
+import { createHash } from "crypto";
+import { coordinatedFill } from "../cache-coordination";
 import { githubTokenContext } from "../github-token-context";
 import AnonymizedFile from "../AnonymizedFile";
 import GitHubBase, {
@@ -11,17 +14,21 @@ import { basename, dirname } from "path";
 
 import * as stream from "stream";
 import AnonymousError from "../AnonymousError";
-import { FILE_TYPE } from "../storage/Storage";
 import { octokit, waitForTokenGate } from "../GitHubUtils";
 import FileModel from "../model/files/files.model";
 import { IFile } from "../model/files/files.types";
 import { createLogger, serializeError } from "../logger";
+import { measureStage, recordCacheHit } from "../request-monitoring";
 import config from "../../config";
+import { isConnected } from "../../server/database";
+import AnonymizedRepositoryModel from "../model/anonymizedRepositories/anonymizedRepositories.model";
+import { RepositoryStatus } from "../types";
 
 
 const logger = createLogger("gh-stream");
 
 const GH_API_CONCURRENCY = 6;
+const cacheFills = new Map<string, Promise<void>>();
 
 async function pMap<T, R>(
   items: T[],
@@ -238,108 +245,94 @@ export default class GitHubStream extends GitHubBase {
     const meta = await fileMeta();
     const expected: { sha: string; size?: number } =
       typeof meta === "string" ? { sha: meta } : meta;
-    const fileInfo = await storage.exists(repoId, filePath);
-    if (fileInfo == FILE_TYPE.FILE) {
-      // If we know the upstream size, validate the cached entry. A cached
-      // file smaller than the upstream size means a previous fetch was
-      // truncated — likely a network error during the GitHub fetch left a
-      // 0-byte or partial blob behind. Treat it as a miss and re-fetch.
-      // Cached size >= expected is accepted: equal for normal files, and
-      // larger for Git LFS files where FileModel.size is the pointer's
-      // size but the cached bytes are the resolved LFS content.
-      if (expected.size != null && expected.size > 0) {
-        try {
-          const stat = await storage.fileInfo(repoId, filePath);
-          if (stat.size != null && stat.size < expected.size) {
-            await storage.rm(repoId, filePath);
-          } else {
-            return storage.read(repoId, filePath);
-          }
-        } catch {
-          // fall through and re-fetch
+    const generation = createHash("sha256").update(JSON.stringify([
+      this.data.commit, expected.sha, this.data.cacheGeneration, this.data.cacheRevision,
+    ])).digest("hex");
+    const cachePath = this.data.cacheGeneration ? `${contentGenerationPrefix(this.data.cacheGeneration)}/${generation}/${filePath}` : filePath;
+    const assertActive = async () => {
+      // Retirement markers live inside the storage root and disappear when
+      // expiration removes it. The persisted lifecycle also fences late fills.
+      if (isConnected) {
+        const current = await AnonymizedRepositoryModel.findOne({ repoId })
+          .select("status treeGeneration anonymizeDate contentCacheRevision").lean().exec();
+        if (!current || !current.status || [RepositoryStatus.EXPIRING, RepositoryStatus.EXPIRED,
+          RepositoryStatus.REMOVING, RepositoryStatus.REMOVED, RepositoryStatus.ARCHIVED].includes(current.status)
+          || (this.data.cacheGeneration && `${current.treeGeneration || "legacy"}:${current.anonymizeDate?.toISOString() || ""}` !== this.data.cacheGeneration)
+          || this.data.cacheRevision !== current.contentCacheRevision) {
+          throw new AnonymousError("repository_changed", { httpStatus: 409 });
         }
-      } else {
-        return storage.read(repoId, filePath);
       }
-    } else if (fileInfo == FILE_TYPE.FOLDER) {
-      throw new AnonymousError("folder_not_supported", {
-        httpStatus: 400,
-        object: filePath,
-      });
-    }
-
-    // GitHub's blob API rejects blobs larger than 100 MB with HTTP 422.
-    // Skip the download entirely when the tree already tells us the file is
-    // over the cap, so we surface a clean `file_too_big` instead of paying
-    // the round-trip just to translate a 422.
+      if (!this.data.cacheGeneration) return;
+      try {
+        await storage.fileInfo(repoId, contentRetirementMarker(contentGenerationPrefix(this.data.cacheGeneration)));
+      } catch (error) {
+        const missing = error as { code?: string; name?: string; $metadata?: { httpStatusCode?: number }; httpStatus?: number };
+        if (missing.code === "ENOENT" || missing.name === "NotFound" || missing.name === "NoSuchKey" ||
+          missing.$metadata?.httpStatusCode === 404 || missing.httpStatus === 404) return;
+        throw error;
+      }
+      throw new AnonymousError("repository_changed", { httpStatus: 409 });
+    };
+    const key = `${repoId}:${generation}:${filePath}`;
+    const cached = async () => {
+      try {
+        const info = await storage.fileInfo(repoId, cachePath);
+        if (info.contentType === "application/x-directory") {
+          throw new AnonymousError("folder_not_supported", { httpStatus: 400 });
+        }
+        if (expected.size != null && expected.size > 0 && (info.size == null || info.size < expected.size)) return undefined;
+        return true;
+      } catch (error) {
+        if (error instanceof AnonymousError && error.message === "folder_not_supported") throw error;
+        return undefined;
+      }
+    };
+    const hit = await measureStage("cache_lookup", cached);
+    recordCacheHit("source", !!hit);
+    if (hit) return storage.read(repoId, cachePath);
     if (expected.size != null && expected.size > config.MAX_FILE_SIZE) {
-      throw new AnonymousError("file_too_big", {
-        httpStatus: 413,
-        object: filePath,
-      });
+      throw new AnonymousError("file_too_big", { httpStatus: 413, object: filePath });
     }
-    const token = await this.data.getToken();
-
-    // Try the blob API first, but fall back to the raw URL on recoverable
-    // blob misses/caps while still preserving LFS pointer handling.
-    const content = await this.downloadWithFallback(
-      token,
-      expected.sha,
-      filePath
-    );
-
-    // duplicate the stream to write it to the storage
-    const stream1 = content.pipe(new stream.PassThrough());
-    const stream2 = content.pipe(new stream.PassThrough());
-
-    // Safety net: guarantee an `error` listener exists on both branches
-    // before any error can be emitted. storage.write attaches its listener
-    // only after an `await mk(...)`, and the route handler attaches its
-    // listener after awaiting this function — both leave a window where
-    // an upstream error would have no listener and escalate to
-    // uncaughtException, crashing the streamer.
-    const noop = () => {};
-    stream1.on("error", noop);
-    stream2.on("error", noop);
-
-    content.on("error", (error) => {
-      const httpStatus =
-        (error as { response?: { statusCode?: number } })?.response
-          ?.statusCode ??
-        (error as { status?: number })?.status ??
-        (error as { httpStatus?: number })?.httpStatus;
-      const errCode = (error as { code?: string })?.code;
-      const isTransient =
-        !httpStatus &&
-        (errCode === "ECONNRESET" ||
-          errCode === "ETIMEDOUT" ||
-          errCode === "ERR_BODY_PARSE_FAILURE" ||
-          error.name === "ReadError");
-      const code =
-        httpStatus === 422
-          ? "file_too_big"
-          : httpStatus === 403
-          ? "file_not_accessible"
-          : isTransient
-          ? "upstream_error"
-          : "file_not_found";
-      const wrapped = new AnonymousError(code, {
-        httpStatus: isTransient ? 502 : httpStatus,
-        cause: error as Error,
-        object: filePath,
-      });
-      stream1.destroy(wrapped);
-      stream2.destroy(wrapped);
-    });
-
-    // Fire-and-forget: storage.write logs its own failures inside FileSystem
-    // (`[fs] write failed`). Swallow the rejection here so an upstream error
-    // (e.g. GitHub 422 on a too-big blob) doesn't surface as an unhandled
-    // promise rejection and crash the streamer process.
-    storage
-      .write(repoId, filePath, stream1, this.type, expected.size)
-      .catch(() => {});
-    return stream2;
+    let filling = cacheFills.get(key);
+    if (!filling) {
+      if (cacheFills.size >= 64) throw new AnonymousError("cache_busy", { httpStatus: 503 });
+      filling = coordinatedFill(key, cached, async () => {
+        await assertActive();
+        const token = await this.data.getToken();
+        return measureStage("upstream", async () => {
+          const content = await this.downloadWithFallback(token, expected.sha, filePath);
+          let count = 0;
+          const limited = new stream.Transform({ transform(chunk, encoding, callback) {
+            count += chunk.length;
+            callback(count > config.MAX_FILE_SIZE ? new AnonymousError("file_too_big", { httpStatus: 413 }) : null, chunk);
+          } });
+          content.on("error", error => {
+            if (error instanceof AnonymousError) { limited.destroy(error); return; }
+            const status = (error as { response?: { statusCode?: number }; status?: number; httpStatus?: number });
+            const httpStatus = status.response?.statusCode ?? status.status ?? status.httpStatus;
+            const code = httpStatus === 403 ? "file_not_accessible"
+              : httpStatus === 404 ? "file_not_found"
+              : httpStatus === 422 ? "file_too_big" : "upstream_error";
+            limited.destroy(new AnonymousError(code, { httpStatus: httpStatus || 502, cause: error, object: filePath }));
+          });
+          limited.once("close", () => content.destroy());
+          limited.on("error", () => {});
+          content.pipe(limited);
+          const deadline = setTimeout(() => limited.destroy(new AnonymousError("upstream_error", { httpStatus: 502 })), 120_000);
+          try {
+            await storage.write(repoId, cachePath, limited, this.type, expected.size);
+            try { await assertActive(); }
+            catch (error) { await storage.rm(repoId, cachePath); throw error; }
+          }
+          finally { clearTimeout(deadline); limited.destroy(); content.destroy(); }
+          return true;
+        });
+      }).then(() => {});
+      cacheFills.set(key, filling);
+      void filling.finally(() => { if (cacheFills.get(key) === filling) cacheFills.delete(key); }).catch(() => {});
+    }
+    await measureStage("cache_fill", () => filling!);
+    return storage.read(repoId, cachePath);
   }
 
   async getFileContent(file: AnonymizedFile): Promise<stream.Readable> {
@@ -444,6 +437,15 @@ export default class GitHubStream extends GitHubBase {
     const output: IFile[] = [];
     let data;
     try {
+      data = await this.getGHTree(oct, token, sha, count, {
+        recursive: true,
+        callback: () => {
+          if (progress) {
+            progress("List file: " + count.file);
+          }
+        },
+      });
+      if (!data.truncated) return this.tree2Tree(data.tree, parentPath);
       data = await this.getGHTree(oct, token, sha, count, {
         recursive: false,
         callback: () => {

src/core/storage/FileSystem.ts

@@ -1,7 +1,8 @@
+import { appendArchiveEntry } from "../archive-entry";
 import config from "../../config";
 import * as fs from "fs";
 import { Extract } from "unzip-stream";
-import { join, basename, dirname } from "path";
+import { join, dirname } from "path";
 import { Response } from "express";
 import { Readable, pipeline, Transform } from "stream";
 import * as archiver from "archiver";
@@ -83,6 +84,7 @@ export default class FileSystem extends StorageBase {
     const tmpPath = `${fullPath}.tmp.${process.pid}.${Date.now()}.${Math.random()
       .toString(36)
       .slice(2, 8)}`;
+    let sourceFailed = false;
     try {
       if (typeof data === "string") {
         await fs.promises.writeFile(tmpPath, data);
@@ -100,7 +102,10 @@ export default class FileSystem extends StorageBase {
               resolve();
             }
           };
-          data.on("error", finish);
+          data.on("error", (error) => {
+            if (!settled) sourceFailed = true;
+            finish(error);
+          });
           ws.on("error", finish);
           ws.on("finish", () => finish());
           data.pipe(ws);
@@ -123,7 +128,8 @@ export default class FileSystem extends StorageBase {
       }
       await fs.promises.rename(tmpPath, fullPath);
     } catch (err) {
-      logger.error("write failed", serializeError(err));
+      if (sourceFailed) logger.warn("source stream failed; cache write discarded", serializeError(err));
+      else logger.error("write failed", serializeError(err));
       await fs.promises.rm(tmpPath, { force: true }).catch(() => undefined);
       throw err;
     }
@@ -139,6 +145,24 @@ export default class FileSystem extends StorageBase {
     });
   }
 
+  async removeLegacyContent(repoId: string): Promise<void> {
+    let directory: fs.Dir;
+    try { directory = await fs.promises.opendir(join(config.FOLDER, this.repoPath(repoId))); }
+    catch (error) {
+      if ((error as NodeJS.ErrnoException).code === "ENOENT") return;
+      throw error;
+    }
+    for await (const entry of directory) {
+      if (entry.name !== "__content" || !entry.isDirectory()) { await this.rm(repoId, entry.name); continue; }
+      const content = await fs.promises.opendir(join(config.FOLDER, this.repoPath(repoId), "__content"));
+      for await (const generation of content) {
+        if (generation.name !== "__retired" && !/^[a-f0-9]{64}$/.test(generation.name)) {
+          await this.rm(repoId, join("__content", generation.name));
+        }
+      }
+    }
+  }
+
   /** @override */
   async mk(repoId: string, dir: string = ""): Promise<void> {
     this.assertSafePath(dir);
@@ -238,26 +262,28 @@ export default class FileSystem extends StorageBase {
   ) {
     this.assertSafePath(dir);
     const archive = archiver(opt?.format || "zip", {});
-    const fullPath = join(config.FOLDER, this.repoPath(repoId), dir);
-
-    await this.listFiles(repoId, dir, {
-      onEntry: async (file) => {
-        let rs: Readable = await this.read(repoId, file.path);
+    let active: Readable | undefined;
+    let stopped = false;
+    archive.once("close", () => { stopped = true; active?.destroy(); });
+    void (async () => {
+      for (const file of await this.listFiles(repoId, dir)) {
+        if (stopped || file.size == null) continue;
+        const filename = join(file.path, file.name);
+        const source = await this.read(repoId, filename);
+        if (stopped) { source.destroy(); return; }
+        active = source;
+        let output = source;
         if (opt?.fileTransformer) {
-          const src = rs;
-          const transformer = opt.fileTransformer(file.path);
-          src.on("error", (err) => transformer.destroy(err));
-          rs = src.pipe(transformer);
+          const transformer = opt.fileTransformer(filename);
+          source.on("error", error => transformer.destroy(error));
+          transformer.once("close", () => source.destroy());
+          output = source.pipe(transformer);
+          active = output;
         }
-        const f = file.path.replace(fullPath, "");
-        archive.append(rs, {
-          name: basename(f),
-          prefix: dirname(f),
-        });
-      },
-    }).then(() => {
-      archive.finalize();
-    });
+        await appendArchiveEntry(archive, output, { name: filename.slice(dir ? dir.length + 1 : 0) });
+      }
+      if (!stopped) await archive.finalize();
+    })().catch(error => archive.destroy(error));
     return archive;
   }
 }

src/core/storage/S3.ts

@@ -1,3 +1,4 @@
+import { appendArchiveEntry } from "../archive-entry";
 import {
   GetObjectCommand,
   ListObjectsV2CommandOutput,
@@ -7,7 +8,7 @@ import {
 import { Upload } from "@aws-sdk/lib-storage";
 import { NodeHttpHandler } from "@smithy/node-http-handler";
 import config from "../../config";
-import { pipeline, Readable, Transform, PassThrough } from "stream";
+import { pipeline, Readable, Transform } from "stream";
 import ArchiveStreamToS3 from "decompress-stream-to-s3";
 import { Response } from "express";
 import { lookup } from "mime-types";
@@ -32,23 +33,30 @@ export default class S3Storage extends StorageBase {
       });
   }
 
+  private clients = new Map<number, S3>();
+
   private client(timeout = 10000) {
+    const existing = this.clients.get(timeout);
+    if (existing) return existing;
     if (!config.S3_CLIENT_ID) throw new Error("S3_CLIENT_ID not set");
     if (!config.S3_CLIENT_SECRET) throw new Error("S3_CLIENT_SECRET not set");
     if (!config.S3_REGION) throw new Error("S3_REGION not set");
     if (!config.S3_ENDPOINT) throw new Error("S3_ENDPOINT not set");
-    return new S3({
+    const client = new S3({
       credentials: {
         accessKeyId: config.S3_CLIENT_ID,
         secretAccessKey: config.S3_CLIENT_SECRET,
       },
       region: config.S3_REGION,
       endpoint: config.S3_ENDPOINT,
+      forcePathStyle: true,
       requestHandler: new NodeHttpHandler({
         requestTimeout: timeout,
         connectionTimeout: timeout,
       }),
     });
+    this.clients.set(timeout, client);
+    return client;
   }
 
   /** @override */
@@ -82,6 +90,18 @@ export default class S3Storage extends StorageBase {
     if (!config.S3_BUCKET) throw new Error("S3_BUCKET not set");
     this.assertSafePath(dir);
     const prefix = join(this.repoPath(repoId), dir).replace(/\/$/, "");
+    await this.removeMatching(prefix, key => key === prefix || key.startsWith(prefix + "/"));
+  }
+
+  async removeLegacyContent(repoId: string): Promise<void> {
+    if (!config.S3_BUCKET) throw new Error("S3_BUCKET not set");
+    const prefix = this.repoPath(repoId);
+    await this.removeMatching(prefix, key => key.startsWith(prefix) &&
+      !/^__content\/(?:[a-f0-9]{64}|__retired)(?:\/|$)/.test(key.slice(prefix.length)));
+  }
+
+  private async removeMatching(prefix: string, shouldRemove: (key: string) => boolean) {
+    if (!config.S3_BUCKET) throw new Error("S3_BUCKET not set");
     const client = this.client(200000);
     let continuationToken: string | undefined;
     do {
@@ -92,7 +112,7 @@ export default class S3Storage extends StorageBase {
         ContinuationToken: continuationToken,
       });
       const objects = (data.Contents || [])
-        .filter(({ Key }) => Key === prefix || Key?.startsWith(prefix + "/"))
+        .filter(({ Key }) => Key && shouldRemove(Key))
         .map(({ Key }) => ({ Key: Key! }));
       if (objects.length) {
         const result = await client.deleteObjects({
@@ -212,11 +232,14 @@ export default class S3Storage extends StorageBase {
     let bytesWritten = -1;
     if (typeof expectedSize === "number" && expectedSize > 0 && typeof data !== "string") {
       bytesWritten = 0;
-      const counter = new PassThrough();
-      counter.on("data", (chunk: Buffer) => {
-        bytesWritten += chunk.length;
+      const counter = new Transform({
+        transform(chunk, _encoding, callback) { bytesWritten += chunk.length; callback(null, chunk); },
+        flush(callback) {
+          callback(bytesWritten < expectedSize! ? new AnonymousError("storage_write_size_mismatch", { httpStatus: 502 }) : undefined);
+        },
       });
       data.on("error", (err: Error) => counter.destroy(err));
+      counter.once("close", () => (data as Readable).destroy());
       data.pipe(counter);
       body = counter;
     }
@@ -351,39 +374,36 @@ export default class S3Storage extends StorageBase {
     const archive = archiver(opt?.format || "zip", {});
     if (dir && dir[dir.length - 1] != "/") dir = dir + "/";
 
-    let req: ListObjectsV2CommandOutput;
-    let nextContinuationToken: string | undefined;
-    do {
-      req = await this.client(30000).listObjectsV2({
-        Bucket: config.S3_BUCKET,
-        Prefix: join(this.repoPath(repoId), dir),
-        MaxKeys: 250,
-        ContinuationToken: nextContinuationToken,
-      });
-
-      nextContinuationToken = req.NextContinuationToken;
-      for (const f of req.Contents || []) {
-        if (!f.Key) continue;
-        const filename = basename(f.Key);
-        const prefix = dirname(
-          f.Key.replace(join(this.repoPath(repoId), dir), "")
-        );
-
-        let rs: Readable = await this.read(repoId, f.Key);
-        if (opt?.fileTransformer) {
-          const src = rs;
-          const transformer = opt.fileTransformer(f.Key);
-          src.on("error", (err) => transformer.destroy(err));
-          rs = src.pipe(transformer);
+    let active: Readable | undefined;
+    let stopped = false;
+    archive.once("close", () => { stopped = true; active?.destroy(); });
+    void (async () => {
+      let next: string | undefined;
+      do {
+        const listing = await this.client(30000).listObjectsV2({ Bucket: config.S3_BUCKET!,
+          Prefix: join(this.repoPath(repoId), dir), MaxKeys: 250, ContinuationToken: next });
+        next = listing.IsTruncated ? listing.NextContinuationToken : undefined;
+        for (const file of listing.Contents || []) {
+          if (stopped) return;
+          if (!file.Key || file.Key.endsWith("/")) continue;
+          const filename = file.Key.slice(this.repoPath(repoId).length);
+          const source = await this.read(repoId, filename);
+          if (stopped) { source.destroy(); return; }
+          active = source;
+          let output = source;
+          if (opt?.fileTransformer) {
+            const transformer = opt.fileTransformer(filename);
+            source.on("error", error => transformer.destroy(error));
+            transformer.once("close", () => source.destroy());
+            output = source.pipe(transformer);
+            active = output;
+          }
+          await appendArchiveEntry(archive, output, { name: filename.slice(dir.length) });
         }
-
-        archive.append(rs, {
-          name: filename,
-          prefix,
-        });
-      }
-    } while (req && req.Contents?.length && req.IsTruncated);
-    archive.finalize();
+        if (listing.IsTruncated && !next) throw new Error("Invalid S3 continuation token");
+      } while (next && !stopped);
+      if (!stopped) await archive.finalize();
+    })().catch(error => archive.destroy(error));
     return archive;
   }
 }

src/core/storage/Storage.ts

@@ -120,6 +120,9 @@ export default abstract class StorageBase {
    */
   abstract mk(repoId: string, dir: string): Promise<void>;
 
+  /** Remove unversioned stream caches, preserving the managed content namespace. */
+  abstract removeLegacyContent(repoId: string): Promise<void>;
+
   repoPath(repoId: string) {
     return (
       join(repoId, "original") + (process.platform === "win32" ? "\\" : "/")

src/core/temporary-storage.ts

@@ -0,0 +1,47 @@
+import { promises as fs } from "fs";
+import { join } from "path";
+import { tmpdir } from "os";
+import { pruneTransformedCache } from "./transformed-cache";
+import { createLogger } from "./logger";
+
+const logger = createLogger("temporary-storage");
+
+let nextCleanup = 0;
+let cleanup: Promise<void> | undefined;
+/** Jobs time out in minutes; an hour-old spool belongs to an interrupted process. */
+export async function removeStaleTextSpools(directory = tmpdir(), now = Date.now()) {
+  const entries = await fs.opendir(directory);
+  for await (const entry of entries) {
+    if (!entry.isDirectory() || !entry.name.startsWith("anonymous-text-")) continue;
+    const path = join(directory, entry.name);
+    try {
+      if (now - (await fs.stat(path)).mtimeMs > 3600_000) await fs.rm(path, { recursive: true, force: true });
+    } catch { /* another process can finish cleanup first */ }
+  }
+}
+export function maintainTextSpools(force = false) {
+  if (cleanup) return cleanup;
+  if (!force && Date.now() < nextCleanup) return Promise.resolve();
+  nextCleanup = Date.now() + 60_000;
+  cleanup = removeStaleTextSpools().catch(() => {}).finally(() => { cleanup = undefined; });
+  return cleanup;
+}
+
+/** Recover on startup and continue while idle, without overlapping scans. */
+export async function startTemporaryStorageMaintenance(intervalMs = 60_000) {
+  let running = false;
+  const run = async () => {
+    if (running) return;
+    running = true;
+    try {
+      const results = await Promise.allSettled([maintainTextSpools(true), pruneTransformedCache()]);
+      for (const result of results) if (result.status === "rejected") {
+        logger.warn("temporary storage cleanup deferred", { message: (result.reason as Error).message });
+      }
+    } finally { running = false; }
+  };
+  await run();
+  const timer = setInterval(() => { void run(); }, intervalMs);
+  timer.unref();
+  return () => clearInterval(timer);
+}

src/core/term-presence.ts

@@ -0,0 +1,67 @@
+/**
+ * Candidate index for nonempty, capture-free, fixed-width compiler patterns.
+ * False positives are safe; false negatives are not.
+ */
+export class TermPresenceIndex {
+  private groups: { indices: number[]; characters: string[] | null }[] = [];
+
+  constructor(private readonly patterns: RegExp[]) {
+    for (let index = 0; index < patterns.length; index++) {
+      const characters = firstCharacters(patterns[index].source);
+      const group = { indices: [index], characters };
+      // Merge groups whose first characters can overlap under Unicode folding.
+      // Then a combined search can identify one group without hiding another
+      // alternative at the same position (Alice versus AliceSmith, for example).
+      for (let i = this.groups.length - 1; i >= 0; i--) {
+        const other = this.groups[i];
+        if (overlap(group.characters, other.characters)) {
+          group.indices.push(...other.indices);
+          group.characters = group.characters && other.characters
+            ? [...new Set([...group.characters, ...other.characters])] : null;
+          this.groups.splice(i, 1);
+        }
+      }
+      this.groups.push(group);
+    }
+  }
+
+  candidates(content: string, after = -1): Set<number> {
+    const found = new Set<number>();
+    const remaining = this.groups.map(group => group.indices.filter(index => index > after))
+      .filter(indices => indices.length);
+    let offset = 0;
+    while (remaining.length) {
+      // Fixed-width inputs contain no captures or quantifiers. Removing a
+      // discovered group avoids revisiting millions of identical CSV values.
+      const search = new RegExp(remaining.map(indices =>
+        `(${indices.map(index => this.patterns[index].source).join("|")})`).join("|"), "giu");
+      search.lastIndex = offset;
+      const match = search.exec(content);
+      if (!match) break;
+      const selected = match.slice(1).findIndex(value => value !== undefined);
+      for (const index of remaining[selected]) found.add(index);
+      remaining.splice(selected, 1);
+      // Another group's match may start inside this one. Search from the next
+      // code point, not from the end of the match, to retain nested candidates.
+      offset = match.index + (content.codePointAt(match.index)! > 0xffff ? 2 : 1);
+    }
+    return found;
+  }
+}
+
+function firstCharacters(source: string): string[] | null {
+  if (source.startsWith("[")) {
+    const characters = source.slice(1, source.indexOf("]"));
+    // Only the compiler's expanded letter classes have an enumerable prefix.
+    // Dot's [^\n], escapes and any future syntax conservatively join all groups.
+    return /^[\p{L}\p{N}]+$/u.test(characters) ? Array.from(characters) : null;
+  }
+  if (source.startsWith("\\")) return null;
+  return [Array.from(source)[0]];
+}
+
+function overlap(left: string[] | null, right: string[] | null): boolean {
+  if (!left || !right) return true;
+  const pattern = new RegExp(`^(?:${left.map(char => char.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")).join("|")})$`, "iu");
+  return right.some(char => pattern.test(char));
+}

src/core/transformed-cache.ts

@@ -0,0 +1,156 @@
+import { promises as fs } from "fs";
+import { join } from "path";
+import { tmpdir } from "os";
+import { createHash, randomUUID } from "crypto";
+import config from "../config";
+import { coordinatedFill } from "./cache-coordination";
+import { isConnected } from "../server/database";
+import RepositoryModel from "./model/anonymizedRepositories/anonymizedRepositories.model";
+import { RepositoryStatus } from "./types";
+import AnonymousError from "./AnonymousError";
+import { recordCacheHit } from "./request-monitoring";
+const root = join(tmpdir(), "anonymous-transformed-v1");
+const TTL = 3600_000;
+const DISK_BUDGET = 512 * 1024 * 1024;
+const fills = new Map<string, Promise<boolean>>();
+interface CacheScope { repoId: string; generation?: string; revision?: string }
+
+async function scopeActive(scope?: CacheScope) {
+  if (!scope || !isConnected) return true;
+  const current = await RepositoryModel.findOne({ repoId: scope.repoId })
+    .select("status treeGeneration anonymizeDate contentCacheRevision").lean().exec();
+  return !!current?.status && ![RepositoryStatus.EXPIRING, RepositoryStatus.EXPIRED,
+    RepositoryStatus.REMOVING, RepositoryStatus.REMOVED, RepositoryStatus.ARCHIVED].includes(current.status)
+    && (!scope.generation || scope.generation === `${current.treeGeneration || "legacy"}:${current.anonymizeDate?.toISOString() || ""}`)
+    && scope.revision === current.contentCacheRevision;
+}
+
+async function removeEntry(metadata: string) {
+  await Promise.all([metadata, metadata.replace(/\.json$/, ".data")].map(path => fs.rm(path, { force: true })));
+}
+
+/** The key includes actual content, all rules, generation, and matcher version. */
+export async function transformedFile(digest: string, options: unknown, output: string,
+  produce: () => Promise<boolean>): Promise<boolean> {
+  const settings = options as { repoId?: string; cacheGeneration?: string; cacheRevision?: string } | undefined;
+  const scope: CacheScope | undefined = typeof settings?.repoId === "string"
+    ? { repoId: settings.repoId, generation: settings.cacheGeneration, revision: settings.cacheRevision } : undefined;
+  const assertActive = async () => {
+    if (!await scopeActive(scope)) throw new AnonymousError("repository_changed", { httpStatus: 409 });
+  };
+  const produceActive = async () => {
+    await assertActive();
+    const changed = await produce();
+    try { await assertActive(); }
+    catch (error) { await fs.rm(output, { force: true }); throw error; }
+    return changed;
+  };
+  await assertActive();
+  const key = createHash("sha256").update(JSON.stringify(["matcher-v1", digest, options,
+    config.ANONYMIZATION_MASK, config.APP_HOSTNAME])).digest("hex");
+  const data = join(root, `${key}.data`), metadata = join(root, `${key}.json`);
+  const read = async () => {
+    try {
+      const record = JSON.parse(await fs.readFile(metadata, "utf8"));
+      if (record.version !== 2 || Date.now() - record.created > TTL || (await fs.stat(data)).size !== record.size) return undefined;
+      return Boolean(record.changed);
+    } catch { return undefined; }
+  };
+  const copyCached = async (changed: boolean) => {
+    await fs.copyFile(data, output, fs.constants.COPYFILE_EXCL);
+    try { await assertActive(); } catch (error) { await fs.rm(output, { force: true }); throw error; }
+    return changed;
+  };
+  const hit = await read();
+  if (hit !== undefined) {
+    try { const copied = await copyCached(hit); recordCacheHit("transformed", true); return copied; } catch { /* eviction: retry the transformation */ }
+  }
+  recordCacheHit("transformed", false);
+  const pending = fills.get(key);
+  if (pending) {
+    await pending.catch(() => {});
+    const cached = await read();
+    if (cached !== undefined) { try { return await copyCached(cached); } catch { /* retry */ } }
+  }
+  if (fills.size >= 64) return produceActive();
+  let producedHere = false;
+  const task = coordinatedFill(`transformed:${key}`, read, async () => {
+    const changed = await produceActive();
+    producedHere = true;
+    const size = (await fs.stat(output)).size;
+    if (size > DISK_BUDGET / 4) return changed;
+    const token = randomUUID();
+    const tempData = `${data}.${token}.tmp`, tempMetadata = `${metadata}.${token}.tmp`;
+    try {
+      await fs.mkdir(root, { recursive: true, mode: 0o700 });
+      await fs.copyFile(output, tempData);
+      await fs.writeFile(tempMetadata, JSON.stringify({ version: 2, changed, size, created: Date.now(), scope }));
+      await fs.rename(tempData, data);
+      await fs.rename(tempMetadata, metadata);
+      await pruneTransformedCache(false);
+    } catch { /* A valid private output remains usable when cache storage fails. */
+    } finally {
+      await Promise.all([tempData, tempMetadata].map(path => fs.rm(path, { force: true }).catch(() => {})));
+    }
+    try { await assertActive(); }
+    catch (error) { await removeEntry(metadata); await fs.rm(output, { force: true }); throw error; }
+    return changed;
+  });
+  fills.set(key, task);
+  try {
+    const changed = await task;
+    if (producedHere) return changed;
+    try { return await copyCached(changed); } catch { return produceActive(); }
+  } finally { if (fills.get(key) === task) fills.delete(key); }
+}
+export async function pruneTransformedCache(validateLifecycle = true) {
+  const records = [];
+  let lifecycleError: Error | undefined;
+  let entries;
+  try { entries = await fs.opendir(root); }
+  catch (error) { if ((error as NodeJS.ErrnoException).code === "ENOENT") return; throw error; }
+  for await (const entry of entries) {
+    const name = entry.name;
+    if (name.endsWith(".tmp") || name.endsWith(".data")) {
+      const path = join(root, name);
+      try { if (Date.now() - (await fs.stat(path)).mtimeMs > TTL) await fs.rm(path, { force: true }); } catch { /* another publisher */ }
+    }
+    if (!name.endsWith(".json")) continue;
+    const path = join(root, name);
+    let record;
+    try { record = JSON.parse(await fs.readFile(path, "utf8")); }
+    catch (error) { if ((error as NodeJS.ErrnoException).code !== "ENOENT") await removeEntry(path); continue; }
+    if (record.version !== 2 || !Number.isFinite(record.created) || !Number.isFinite(record.size)
+      || Date.now() - record.created > TTL) { await removeEntry(path); continue; }
+    if (validateLifecycle && !lifecycleError) {
+      try { if (!await scopeActive(record.scope)) { await removeEntry(path); continue; } }
+      catch (error) { lifecycleError = error as Error; }
+    }
+    records.push({ path, ...record });
+  }
+  records.sort((a, b) => b.created - a.created);
+  let bytes = 0;
+  for (let i = 0; i < records.length; i++) {
+    const record = records[i]; bytes += record.size;
+    if (i >= 128 || bytes > DISK_BUDGET || Date.now() - record.created > TTL) {
+      await removeEntry(record.path);
+    }
+  }
+  // Storage age and budget cleanup must finish even when MongoDB is unavailable.
+  if (lifecycleError) throw lifecycleError;
+}
+
+/** Lifecycle changes remove local entries; connected replicas also prune independently. */
+export async function removeTransformedRepositoryCache(repoId: string) {
+  let entries;
+  try { entries = await fs.opendir(root); }
+  catch (error) { if ((error as NodeJS.ErrnoException).code === "ENOENT") return; throw error; }
+  for await (const entry of entries) {
+    if (!entry.name.endsWith(".json")) continue;
+    const metadata = join(root, entry.name);
+    let record;
+    try { record = JSON.parse(await fs.readFile(metadata, "utf8")); }
+    catch (error) { if ((error as NodeJS.ErrnoException).code !== "ENOENT") await removeEntry(metadata); continue; }
+    if (record.version !== 2 || record.scope?.repoId === repoId) await removeEntry(metadata);
+  }
+}

src/core/zipStream.ts

@@ -1,5 +1,5 @@
 import got from "got";
-import { Readable, Transform } from "stream";
+import { Readable, Transform, Writable } from "stream";
 import { Parse } from "unzip-stream";
 import archiver = require("archiver");
 
@@ -12,6 +12,7 @@ import {
   compileTerms,
 } from "./anonymize-utils";
 import { createLogger, serializeError } from "./logger";
+import { startStage } from "./request-monitoring";
 
 const logger = createLogger("zip-stream");
 
@@ -67,8 +68,8 @@ function isEntryAllowed(
  * Stream the GitHub source zip for a repository, anonymize each entry on the
  * fly, and pipe the resulting archive into the provided writable response.
  *
- * No data is written to local storage — the zip flows GitHub → unzip → per
- * file anonymizer → archiver → response.
+ * Large text entries spool privately before worker anonymization. The parser
+ * admits the next entry only after the archive output drains to the response.
  */
 export async function streamAnonymizedZip(
   opt: StreamAnonymizedZipOptions,
@@ -111,20 +112,58 @@ export async function streamAnonymizedZip(
       },
     });
   }
+  const upstreamDone = startStage("upstream");
   const downloadStream = got.stream(response.url);
+  downloadStream.once("end", () => upstreamDone());
+  downloadStream.once("error", () => upstreamDone(true));
+  downloadStream.once("close", () => upstreamDone(!downloadStream.readableEnded));
 
   const archive = archiver("zip", {});
   const parser = Parse() as Transform;
   const activeStreams = new Set<Readable>();
   const compiledTerms = compileTerms(opt.anonymizerOptions.terms || []);
   let stopped = false;
+  let entryWaiting = false;
+  let cleanupDrain: (() => void) | undefined;
+  const resumeParser = () => {
+    if (!stopped && entryWaiting && archive.readableLength === 0 && output.writableLength === 0) {
+      entryWaiting = false;
+      parser.resume();
+    }
+  };
+  // Middleware can omit write callbacks. Honor the writable backpressure
+  // contract instead so queued output stays bounded.
+  const output = new Writable({
+    write(chunk, encoding, callback) {
+      const written = () => {
+        cleanupDrain?.();
+        cleanupDrain = undefined;
+        callback();
+        setImmediate(resumeParser);
+      };
+      if (res.write(chunk, encoding)) written();
+      else {
+        // Compression forwards drain subscriptions to its underlying stream.
+        // Remove the listener from the emitter returned by on(), rather than
+        // accumulating wrappers on that stream through response.once().
+        const emitter = res.on("drain", written) as NodeJS.EventEmitter;
+        cleanupDrain = () => emitter.removeListener("drain", written);
+      }
+    },
+    final(callback) { res.end(); callback(); },
+  });
+  output.on("error", error => fail(error));
+  archive.on("data", () => { if (entryWaiting) setImmediate(resumeParser); });
   const stop = () => {
     if (stopped) return;
     stopped = true;
+    cleanupDrain?.();
+    cleanupDrain = undefined;
     downloadStream.destroy();
     parser.destroy();
     for (const stream of activeStreams) stream.destroy();
     activeStreams.clear();
+    output.destroy();
     archive.abort();
   };
   const fail = (error: Error) => {
@@ -145,7 +184,7 @@ export async function streamAnonymizedZip(
   res.on("error", (error) => fail(error as Error));
   res.on("close", stop);
   archive.on("error", fail);
-  archive.pipe(res);
+  archive.pipe(output);
 
   downloadStream
     .on("error", fail)
@@ -172,6 +211,9 @@ export async function streamAnonymizedZip(
           // there to decide whether the entry is text (and therefore should be
           // anonymized) vs binary (passthrough). Assigning afterwards leaves
           // isText=false for every file, so the zip ships unanonymized.
+          parser.pause();
+          const onArchived = () => { entryWaiting = true; setImmediate(resumeParser); };
+          archive.once("entry", onArchived);
           const anonymizer = new AnonymizeTransformer({
             ...opt.anonymizerOptions,
             filePath: entry.path,
@@ -189,7 +231,7 @@ export async function streamAnonymizedZip(
       }
     })
     .on("error", fail)
-    .on("finish", () => {
+    .on("end", () => {
       if (stopped) return;
       void archive.finalize().catch(fail);
     });

src/queue/queueMetrics.ts

@@ -15,12 +15,12 @@ function getClient(): RedisClientType | null {
       socket: {
         host: config.REDIS_HOSTNAME,
         port: config.REDIS_PORT,
-        reconnectStrategy: () => false as any,
+        reconnectStrategy: false,
       },
     }) as RedisClientType;
     client.on("error", () => {
       disabled = true;
-      client?.disconnect().catch(() => {});
+      if (client?.isOpen) client.destroy();
       client = null;
     });
     client.connect().catch(() => {

src/scripts/create-performance-indexes.ts

@@ -0,0 +1,39 @@
+import "dotenv/config";
+import mongoose from "mongoose";
+import config from "../config";
+import Repository from "../core/model/anonymizedRepositories/anonymizedRepositories.model";
+import Gist from "../core/model/anonymizedGists/anonymizedGists.model";
+import PullRequest from "../core/model/anonymizedPullRequests/anonymizedPullRequests.model";
+import File from "../core/model/files/files.model";
+import Path from "../core/model/anonymized-path";
+import Name from "../core/model/dashboard-name";
+
+async function main() {
+  const uri = config.MONGODB_URI || `mongodb://${config.DB_USERNAME}:${config.DB_PASSWORD}@${config.DB_HOSTNAME}:27017/production`;
+  await mongoose.connect(uri, { autoIndex: false, ...(config.MONGODB_URI ? {} : { authSource: "admin" }) });
+  // Additive only. Never use syncIndexes here, which can drop deployed indexes.
+  await Repository.collection.createIndex({ "coauthors.githubId": 1 });
+  await File.collection.createIndex({ repoId: 1, treeGeneration: 1, path: 1, name: 1 });
+  await File.collection.createIndex({ metadataPending: 1, repoId: 1, treeGeneration: 1 },
+    { partialFilterExpression: { metadataPending: true } });
+  await Repository.collection.createIndex({ retiredTreeGenerations: 1 }, { sparse: true });
+  await Repository.collection.createIndex({ "stagedFileTrees.until": 1 }, { sparse: true });
+  await File.collection.createIndex({ treeStaged: 1, repoId: 1, treeGeneration: 1 },
+    { partialFilterExpression: { treeStaged: true } });
+  await Repository.collection.createIndex({ retiredContentPrefixes: 1 }, { sparse: true });
+  await Repository.collection.createIndex({ legacyContentCleanupPending: 1, status: 1 },
+    { partialFilterExpression: { legacyContentCleanupPending: true } });
+  // Register legacy cleanup once; ZIP roots remain the source of their files.
+  await Repository.updateMany({ "source.type": { $ne: "Zip" }, contentCacheVersion: { $exists: false } },
+    { $set: { contentCacheVersion: 1, legacyContentCleanupPending: true } }).exec();
+  for (const model of [Repository, Gist, PullRequest]) {
+    await model.collection.createIndex({ status: 1, "options.expirationDate": 1 });
+  }
+  await Repository.collection.createIndex({ status: 1, isReseted: 1, lastView: 1 });
+  for (const model of [Path, Name]) await model.createIndexes();
+  process.stdout.write("Performance indexes created. Existing indexes retained.\n");
+}
+main().catch(() => {
+  process.stderr.write("Performance index creation failed; check database configuration and connectivity.\n");
+  process.exitCode = 1;
+}).finally(() => mongoose.disconnect());

src/scripts/purge-legacy-sessions.ts

@@ -11,26 +11,28 @@ async function main() {
     await client.connect();
     let found = 0;
     let removed = 0;
-    for await (const key of client.scanIterator({ MATCH: "anoGH_session:*", COUNT: 100 })) {
-      const raw = await client.get(key);
-      if (!raw) continue;
-      let legacy = false;
-      try {
-        const value = JSON.parse(raw);
-        legacy = !!value.passport?.user && typeof value.passport.user !== "string";
-      } catch { legacy = true; }
-      if (!legacy) continue;
-      found++;
-      if (args.includes("--apply")) {
-        // Do not remove a session replaced since the scan read it.
-        removed += Number(await client.eval(
-          'if redis.call("GET", KEYS[1]) == ARGV[1] then return redis.call("DEL", KEYS[1]) else return 0 end',
-          { keys: [key], arguments: [raw] }
-        ));
+    for await (const keys of client.scanIterator({ MATCH: "anoGH_session:*", COUNT: 100 })) {
+      for (const key of keys) {
+        const raw = await client.get(key);
+        if (!raw) continue;
+        let legacy = false;
+        try {
+          const value = JSON.parse(raw);
+          legacy = !!value.passport?.user && typeof value.passport.user !== "string";
+        } catch { legacy = true; }
+        if (!legacy) continue;
+        found++;
+        if (args.includes("--apply")) {
+          // Do not remove a session replaced since the scan read it.
+          removed += Number(await client.eval(
+            'if redis.call("GET", KEYS[1]) == ARGV[1] then return redis.call("DEL", KEYS[1]) else return 0 end',
+            { keys: [key], arguments: [raw] }
+          ));
+        }
       }
     }
     process.stdout.write(JSON.stringify({ found, removed }) + "\n");
-  } finally { if (client.isOpen) await client.quit(); }
+  } finally { if (client.isOpen) await client.close(); }
 }
 main().catch(() => {
   process.stderr.write("Legacy session cleanup failed; check Redis configuration and connectivity.\n");

src/server/dailyStatsSnapshot.ts

@@ -109,3 +109,42 @@ export async function ensureTodaySnapshot(): Promise<void> {
     logger.error("ensureTodaySnapshot failed", serializeError(error));
   }
 }
+
+// Shared by /stat and /stat/history; zero counts are valid cache entries.
+import { AsyncCache } from "../core/async-cache";
+import { cacheRedis, cacheCommand, coordinatedFill } from "../core/cache-coordination";
+const currentStatsCache = new AsyncCache<HomeStats>(60 * 60_000, 1);
+const historyCache = new AsyncCache<HomeStatsHistoryRow[]>(60 * 60_000, 16);
+export function clearStatsCache() { currentStatsCache.clear(); historyCache.clear(); }
+export function getCurrentStats(): Promise<HomeStats> {
+  return currentStatsCache.get("current", async () => {
+    const redis = await cacheRedis();
+    const key = "perf:home-stats:v1";
+    const read = async () => {
+      if (!redis?.isReady) return undefined;
+      const value = await cacheCommand(redis.get(key), redis).catch(() => null);
+      if (!value) return undefined;
+      try { return JSON.parse(value) as HomeStats; } catch { return undefined; }
+    };
+    return coordinatedFill(key, read, async () => {
+      const stats = await computeStats();
+      if (redis?.isReady) await cacheCommand(redis.set(key, JSON.stringify(stats), { PX: 60 * 60_000 }), redis).catch(() => {});
+      return stats;
+    });
+  });
+}
+export function getStatsHistory(days: number, now = new Date()) {
+  const count = Math.min(365, Math.max(1, Math.floor(days) || 30));
+  const today = utcMidnight(now);
+  return historyCache.get(`${today.toISOString()}:${count}`, async () => {
+    const since = new Date(today);
+    since.setUTCDate(since.getUTCDate() - count + 1);
+    const [docs, current] = await Promise.all([
+      DailyStatsModel.find({ date: { $gte: since } }).sort({ date: 1 }).lean(),
+      getCurrentStats(),
+    ]);
+    return mergeCurrentStatsIntoHistory(docs.map(d => ({ date: d.date,
+      nbRepositories: d.nbRepositories, nbUsers: d.nbUsers,
+      nbPageViews: d.nbPageViews, nbPullRequests: d.nbPullRequests })), current, now);
+  });
+}

src/server/database.ts

@@ -22,12 +22,13 @@ export const database = mongoose.connection;
 
 export let isConnected = false;
 
-export async function connect() {
+export async function connect(overrides: Pick<ConnectOptions, "appName" | "maxPoolSize" | "minPoolSize"> = {}) {
   credentialCipher(); // Refuse to serve persisted credentials without a valid keyring.
   mongoose.set("strictQuery", false);
   const options: ConnectOptions = {
     appName: "Anonymous GitHub Server",
     compressors: "zstd",
+    ...overrides,
   };
   if (!config.MONGODB_URI) options.authSource = "admin";
   await mongoose.connect(getMongoUrl(), options);

src/server/index.ts

@@ -12,6 +12,7 @@ import * as express from "express";
 import * as compression from "compression";
 import * as passport from "passport";
 import { connect } from "./database";
+import { startTemporaryStorageMaintenance } from "../core/temporary-storage";
 import { initSession, router as connectionRouter } from "./routes/connection";
 import { bearerTokenAuth } from "./routes/token-auth";
 import router from "./routes";
@@ -27,16 +28,18 @@ import {
   recoverStuckRemoving,
 } from "../queue";
 import {
-  computeStats,
+  getCurrentStats,
+  getStatsHistory,
   ensureTodaySnapshot,
-  HomeStatsHistoryRow,
-  mergeCurrentStatsIntoHistory,
 } from "./dailyStatsSnapshot";
-import DailyStatsModel from "../core/model/dailyStats/dailyStats.model";
 import { getUser } from "./routes/route-utils";
 import config from "../config";
-import { resolveTrustProxy, isCloudflareIP } from "./trustProxy";
+import { resolveTrustProxy } from "./trustProxy";
+import { requestRateLimitKey } from "./rate-limit-key";
 import { createLogger, serializeError } from "../core/logger";
+import { monitorRequests } from "../core/request-monitoring";
+import { startPerformanceMonitoring } from "../core/performance-monitoring";
+import { getAnonymizationPoolStats } from "../core/anonymization-pool";
 
 import { createReviewCapabilities } from "./service/review-capabilities";
 
@@ -106,6 +109,8 @@ function indexResponse(req: express.Request, res: express.Response) {
 
 export default async function start() {
   const app = express();
+  app.use(monitorRequests("api"));
+  startPerformanceMonitoring("api", getAnonymizationPoolStats);
   app.set("query parser", "extended");
   app.use("/service", createReviewCapabilities(process.env.REVIEW_SERVICE_KEYS));
   app.use("/github/app/webhook", githubAppWebhook);
@@ -159,37 +164,6 @@ export default async function start() {
 
   await redisClient.connect();
 
-  function keyGenerator(
-    request: express.Request,
-    _response: express.Response
-  ): string {
-    // Use request.ip, which Express resolves from X-Forwarded-For honouring
-    // the configured "trust proxy" setting. Do NOT key off the
-    // cf-connecting-ip header unconditionally: when the server isn't actually
-    // behind Cloudflare a client can set that header to an arbitrary value
-    // per request and trivially bypass the rate limiter (CWE-290).
-    let ip = request.ip;
-    if (!ip && request.socket.remoteAddress) {
-      logger.warn("request.ip is missing");
-      ip = request.socket.remoteAddress;
-    }
-    // If resolution stopped at a Cloudflare edge address (X-Forwarded-For no
-    // longer contains the visitor, e.g. after a Cloudflare-side change), fall
-    // back to cf-connecting-ip. This is safe: request.ip can only be a
-    // Cloudflare address when every hop up to it is trusted, i.e. the request
-    // genuinely traversed Cloudflare. A direct client forging the header is
-    // keyed by its own untrusted address instead, so the CWE-290 bypass above
-    // does not apply.
-    if (ip && isCloudflareIP(ip)) {
-      const cfConnectingIP = request.headers["cf-connecting-ip"];
-      if (typeof cfConnectingIP === "string" && cfConnectingIP) {
-        ip = cfConnectingIP.trim();
-      }
-    }
-    // remove port number from IPv4 addresses
-    return (ip || "").replace(/:\d+[^:]*$/, "");
-  }
-
   const rate = rateLimit({
     store: new RedisStore({
       sendCommand: (...args: string[]) => redisClient.sendCommand(args),
@@ -204,7 +178,7 @@ export default async function start() {
       }
       return false;
     },
-    max: async (request: express.Request, _response: express.Response) => {
+    limit: async (request: express.Request, _response: express.Response) => {
       try {
         const user = await getUser(request);
         if (user) return config.RATE_LIMIT;
@@ -214,8 +188,8 @@ export default async function start() {
       // if not logged in, limit to half the rate
       return config.RATE_LIMIT / 2;
     },
-    keyGenerator,
-    standardHeaders: true,
+    keyGenerator: requestRateLimitKey,
+    standardHeaders: "draft-6",
     legacyHeaders: false,
     message: (_request: express.Request, _response: express.Response) => {
       return `You can only make ${config.RATE_LIMIT} requests every 15min. Please try again later.`;
@@ -226,28 +200,14 @@ export default async function start() {
     delayAfter: 50,
     delayMs: () => 150,
     maxDelayMs: 5000,
-    keyGenerator,
+    keyGenerator: requestRateLimitKey,
   });
   const webViewSpeedLimiter = slowDown({
     windowMs: 15 * 60 * 1000, // 15 minutes
     delayAfter: 200,
     delayMs: () => 150,
     maxDelayMs: 5000,
-    keyGenerator,
-  });
-
-  app.use(function (req, res, next) {
-    const start = Date.now();
-    res.on("finish", function () {
-      const time = Date.now() - start;
-      logger.info("request", {
-        method: req.method,
-        status: res.statusCode,
-        url: join(req.baseUrl || "", req.url || ""),
-        ms: time,
-      });
-    });
-    next();
+    keyGenerator: requestRateLimitKey,
   });
 
   app.use("/github", rate, speedLimiter, githubAppRouter);
@@ -285,53 +245,14 @@ export default async function start() {
     res.sendStatus(404);
   });
 
-  let stat: Record<string, unknown> = {};
-  let history: HomeStatsHistoryRow[] | null = null;
-  let historyKey: number | null = null;
-
-  setInterval(() => {
-    stat = {};
-    history = null;
-    historyKey = null;
-  }, 1000 * 60 * 60);
-
   apiRouter.get("/healthcheck", async (_, res) => {
     res.json({ status: "ok" });
   });
   apiRouter.get("/stat", async (_, res) => {
-    if (stat.nbRepositories) {
-      res.json(stat);
-      return;
-    }
-    stat = { ...(await computeStats()) };
-    res.json(stat);
+    res.json(await getCurrentStats());
   });
-
   apiRouter.get("/stat/history", async (req, res) => {
-    const days = Math.min(
-      Math.max(parseInt(req.query.days as string) || 30, 1),
-      365
-    );
-    if (history && historyKey === days) {
-      res.json(history);
-      return;
-    }
-    const since = new Date();
-    since.setUTCDate(since.getUTCDate() - days + 1);
-    since.setUTCHours(0, 0, 0, 0);
-    const docs = await DailyStatsModel.find({ date: { $gte: since } })
-      .sort({ date: 1 })
-      .lean();
-    const rows = docs.map((d) => ({
-      date: d.date,
-      nbRepositories: d.nbRepositories,
-      nbUsers: d.nbUsers,
-      nbPageViews: d.nbPageViews,
-      nbPullRequests: d.nbPullRequests,
-    }));
-    history = mergeCurrentStatsIntoHistory(rows, await computeStats());
-    historyKey = days;
-    res.json(history);
+    res.json(await getStatsHistory(parseInt(String(req.query.days)) || 30));
   });
 
   // web view
@@ -386,6 +307,7 @@ export default async function start() {
   dailyStatsSnapshot();
 
   await connect();
+  await startTemporaryStorageMaintenance();
   app.listen(config.PORT);
   logger.info("server started", { port: config.PORT });
   ensureTodaySnapshot().catch((err) =>

src/server/rate-limit-key.ts

@@ -0,0 +1,16 @@
+import { Request } from "express";
+import { ipKeyGenerator } from "express-rate-limit";
+import { isCloudflareIP } from "./trustProxy";
+
+export function requestRateLimitKey(request: Request): string {
+  // Express resolves request.ip using the configured trusted proxies.
+  let ip = request.ip || request.socket.remoteAddress || "";
+  // Only accept Cloudflare's visitor header when resolution reached its edge.
+  if (isCloudflareIP(ip)) {
+    const visitor = request.headers["cf-connecting-ip"];
+    if (typeof visitor === "string" && visitor.trim()) ip = visitor.trim();
+  }
+  // Some proxies append a port to IPv4 addresses. Do not truncate IPv6.
+  ip = ip.replace(/^(\d{1,3}(?:\.\d{1,3}){3}):\d+$/, "$1");
+  return ipKeyGenerator(ip);
+}

src/server/routes/admin.ts

@@ -1,5 +1,7 @@
+import { AsyncCache } from "../../core/async-cache";
 import * as os from "os";
-import { execSync } from "child_process";
+import { execFile } from "child_process";
+import { promisify } from "util";
 import { Queue, JobType } from "bullmq";
 import * as express from "express";
 import AnonymousError from "../../core/AnonymousError";
@@ -29,8 +31,11 @@ import {
 } from "../../core/logger";
 import { createClient, RedisClientType } from "redis";
 import config from "../../config";
+import { performanceReport } from "../../core/performance-monitoring";
 
 const logger = createLogger("admin");
+const execFileAsync = promisify(execFile);
+const diskUsageCache = new AsyncCache<string>(30000, 1);
 
 let errorLogClient: RedisClientType | null = null;
 
@@ -77,6 +82,12 @@ router.use(
 
 router.use("/tokens", adminTokensRouter);
 
+router.get("/performance", async (req, res) => {
+  res.set("Cache-Control", "no-store");
+  try { res.json(await performanceReport(Number(req.query.minutes))); }
+  catch { res.json({ available: false, instances: [], routes: [], stages: [] }); }
+});
+
 function dashboardCache(
   _req: express.Request,
   res: express.Response,
@@ -342,18 +353,11 @@ router.get("/queues/metrics", dashboardCache, async (req, res) => {
   }
 });
 
-const queuesCache = new Map<string, { data: unknown; ts: number }>();
-const QUEUES_CACHE_TTL = 10_000;
+const queuesCache = new AsyncCache<{ queues: unknown; selectedQueue: string; jobs: Record<string, unknown>[] }>(10_000, 3);
 
 router.get("/queues", dashboardCache, async (req, res) => {
   const search = req.query.search ? String(req.query.search).toLowerCase() : "";
   const queueName = req.query.queue ? String(req.query.queue) : "";
-  const cacheKey = `${queueName}|${search}`;
-  const cached = queuesCache.get(cacheKey);
-  if (cached && Date.now() - cached.ts < QUEUES_CACHE_TTL) {
-    return res.json(cached.data);
-  }
-
   const allQueues: { key: string; label: string; queue: Queue }[] = [
     { key: "download", label: "Download", queue: downloadQueue },
     { key: "remove", label: "Remove", queue: removeQueue },
@@ -365,6 +369,7 @@ router.get("/queues", dashboardCache, async (req, res) => {
     : allQueues[0];
   const targetQueue = target ? target.queue : downloadQueue;
 
+  const data = await queuesCache.get(target?.key || "download", async () => {
   const [statsResults, ...jobsByState] = await Promise.all([
     Promise.all(
       allQueues.map(async (q) => ({
@@ -376,7 +381,16 @@ router.get("/queues", dashboardCache, async (req, res) => {
     ...QUEUE_STATES.map(async (state) => {
       const jobs = await targetQueue.getJobs([state], 0, 199);
       return jobs.map((j) => {
-        const json: Record<string, unknown> = { ...j.asJSON(), _state: state };
+        // asJSON() serializes these fields to strings; the dashboard needs
+        // the objects (payload repoId, stack lines, failure reason).
+        const json: Record<string, unknown> = {
+          ...j.asJSON(),
+          data: j.data,
+          failedReason: j.failedReason,
+          stacktrace: j.stacktrace,
+          returnvalue: j.returnvalue,
+          _state: state,
+        };
         if (state === "delayed" && j.delay > 0) {
           json.delayUntil = j.timestamp + j.delay;
         }
@@ -385,6 +399,19 @@ router.get("/queues", dashboardCache, async (req, res) => {
     }),
   ]);
 
+  const allJobs = (jobsByState as Record<string, unknown>[][]).flat();
+
+  const stateOrder: Record<string, number> = {
+    active: 0, waiting: 1, delayed: 2, failed: 3, completed: 4,
+  };
+  allJobs.sort((a, b) => (stateOrder[a._state as string] ?? 9) - (stateOrder[b._state as string] ?? 9));
+
+  return {
+    queues: statsResults,
+    selectedQueue: target?.key || "download",
+    jobs: allJobs,
+  };
+  });
   const matches = (job: { id?: string | undefined; name?: string }) => {
     if (!search) return true;
     return (
@@ -392,20 +419,7 @@ router.get("/queues", dashboardCache, async (req, res) => {
       (job.name || "").toLowerCase().includes(search)
     );
   };
-  const allJobs = (jobsByState as Record<string, unknown>[][]).flat().filter(matches);
-
-  const stateOrder: Record<string, number> = {
-    active: 0, waiting: 1, delayed: 2, failed: 3, completed: 4,
-  };
-  allJobs.sort((a, b) => (stateOrder[a._state as string] ?? 9) - (stateOrder[b._state as string] ?? 9));
-
-  const data = {
-    queues: statsResults,
-    selectedQueue: target?.key || "download",
-    jobs: allJobs,
-  };
-  queuesCache.set(cacheKey, { data, ts: Date.now() });
-  res.json(data);
+  res.json({ ...data, jobs: data.jobs.filter(matches) });
 });
 
 // Errors captured by the logger sink. Server-paginated to avoid pulling
@@ -595,15 +609,15 @@ router.delete("/errors", async (req, res) => {
     // SCAN the hourly counter keys and del them along with the list and
     // dropped counter so the admin page comes back to a clean slate.
     const hourlyKeys: string[] = [];
-    let cursor = 0;
+    let cursor = "0";
     do {
       const reply = await client.scan(cursor, {
         MATCH: `${ERROR_LOG_HOURLY_PREFIX}*`,
         COUNT: 100,
       });
-      cursor = Number(reply.cursor);
+      cursor = reply.cursor;
       for (const k of reply.keys) hourlyKeys.push(k);
-    } while (cursor !== 0);
+    } while (cursor !== "0");
     const pipe = client.multi();
     pipe.del(ERROR_LOG_KEY);
     pipe.del(ERROR_LOG_DROPPED_KEY);
@@ -632,7 +646,10 @@ router.get("/overview", async (req, res) => {
     // Disk usage via df (root partition)
     let diskTotal = 0, diskUsed = 0, diskFree = 0, diskPercent = 0, diskMount = "/";
     try {
-      const dfOut = execSync("df -k / 2>/dev/null", { timeout: 3000 }).toString();
+      const dfOut = await diskUsageCache.get("root", async () => {
+        const result = await execFileAsync("df", ["-k", "/"], { timeout: 3000, maxBuffer: 16384 });
+        return result.stdout;
+      });
       const lines = dfOut.trim().split("\n");
       if (lines.length >= 2) {
         const cols = lines[1].split(/\s+/);

src/server/routes/connection.ts

@@ -1,7 +1,7 @@
 import { createClient } from "redis";
 import * as passport from "passport";
 import * as session from "express-session";
-import RedisStore from "connect-redis";
+import { RedisStore } from "connect-redis";
 import * as OAuth2Strategy from "passport-oauth2";
 import { Profile, Strategy } from "passport-github2";
 import * as express from "express";
@@ -166,7 +166,6 @@ passport.deserializeUser(async (id: string, done) => {
 
 export function initSession() {
   const redisClient = createClient({
-    legacyMode: false,
     socket: {
       port: config.REDIS_PORT,
       host: config.REDIS_HOSTNAME,

src/server/routes/dashboard-names.ts

@@ -0,0 +1,48 @@
+import { isConnected } from "../database";
+import { AsyncCache } from "../../core/async-cache";
+import DashboardName from "../../core/model/dashboard-name";
+import Repo from "../../core/model/anonymizedRepositories/anonymizedRepositories.model";
+import PR from "../../core/model/anonymizedPullRequests/anonymizedPullRequests.model";
+import Gist from "../../core/model/anonymizedGists/anonymizedGists.model";
+import User from "../../core/User";
+import { projectNameKey } from "./project-names";
+const backfills = new AsyncCache<void>(5 * 60_000, 64);
+export function invalidateDashboardNames() { backfills.clear(); }
+/** Backfill legacy private names using ID-only cursors, never artifact bodies. */
+export function ensureDashboardNames(user: User) {
+  if (!isConnected) return Promise.resolve();
+  return backfills.get(user.id, async () => {
+    // The user map is authoritative, including after an interrupted dual write.
+    const existing = DashboardName.find({ owner: user.model._id }).lean().cursor();
+    try {
+      let batch: Parameters<typeof DashboardName.bulkWrite>[0] = [];
+      for await (const row of existing) {
+        const name = user.model.projectNames?.get(projectNameKey(row.type, row.artifactId));
+        batch.push(name ? { updateOne: { filter: { _id: row._id }, update: { $set: { name } } } }
+          : { deleteOne: { filter: { _id: row._id } } });
+        if (batch.length === 250) { await DashboardName.bulkWrite(batch); batch = []; }
+      }
+      if (batch.length) await DashboardName.bulkWrite(batch);
+    } finally { await existing.close(); }
+    if (!user.model.projectNames?.size) return;
+    const sources = [
+      { type: "repo", cursor: Repo.find(user.repositoryMembership()).select("repoId").lean().cursor(), field: "repoId" },
+      { type: "pr", cursor: PR.find({ owner: user.id }).select("pullRequestId").lean().cursor(), field: "pullRequestId" },
+      { type: "gist", cursor: Gist.find({ owner: user.id }).select("gistId").lean().cursor(), field: "gistId" },
+    ];
+    for (const { type, cursor, field } of sources) {
+      try {
+        let batch: Parameters<typeof DashboardName.bulkWrite>[0] = [];
+        for await (const row of cursor) {
+          const id = String((row as unknown as Record<string, unknown>)[field] || "");
+          const name = user.model.projectNames!.get(projectNameKey(type, id));
+          if (!name) continue;
+          batch.push({ updateOne: { filter: { owner: user.model._id, type, artifactId: id },
+            update: { $set: { name } }, upsert: true } });
+          if (batch.length === 250) { await DashboardName.bulkWrite(batch); batch = []; }
+        }
+        if (batch.length) await DashboardName.bulkWrite(batch);
+      } finally { await cursor.close(); }
+    }
+  });
+}

src/server/routes/dashboard-summary.ts

@@ -0,0 +1,102 @@
+import DashboardName from "../../core/model/dashboard-name";
+import { ensureDashboardNames } from "./dashboard-names";
+import { PipelineStage, Types } from "mongoose";
+import User from "../../core/User";
+import Repo from "../../core/model/anonymizedRepositories/anonymizedRepositories.model";
+import PullRequest from "../../core/model/anonymizedPullRequests/anonymizedPullRequests.model";
+import Gist from "../../core/model/anonymizedGists/anonymizedGists.model";
+import AnonymousError from "../../core/AnonymousError";
+
+const fields: Record<string, string> = { _name: "_anonymousId", _label: "_label", status: "status",
+  anonymizeDate: "anonymizeDate", lastView: "lastView", pageView: "pageView", "options.expirationDate": "options.expirationDate" };
+export function dashboardPipeline(user: User, query: Record<string, unknown>, now = new Date()): PipelineStage[] {
+  const sort = String(query.sort || "-anonymizeDate");
+  const field = sort.replace(/^-/, "");
+  if (!fields[field]) throw new AnonymousError("invalid_sort", { httpStatus: 400 });
+  const direction = sort.startsWith("-") ? -1 : 1;
+  const limit = Math.min(100, Math.max(1, Math.floor(Number(query.limit)) || 50));
+  const owner = new Types.ObjectId(user.id);
+
+  function project(type: string, id: string, source: unknown, label: unknown, extra: Record<string, unknown>) {
+    return [
+      { $project: { _id: 1, status: 1, statusMessage: 1, anonymizeDate: 1, lastView: 1, pageView: { $ifNull: ["$pageView", 0] },
+        options: 1, conference: 1, ...extra, _type: { $literal: type }, _anonymousId: { $ifNull: [`$${id}`, ""] },
+        _source: source, _fallbackLabel: label,
+
+      } },
+    ];
+  }
+  const repoProject = project("repo", "repoId", "$source.repositoryName", "$source.repositoryName", {
+    repoId: 1, source: { fullName: "$source.repositoryName", commit: "$source.commit" }, coauthors: 1, size: 1,
+    role: { $cond: [{ $eq: ["$owner", owner] }, "owner", "coauthor"] },
+  });
+  const prProject = project("pr", "pullRequestId", { $concat: [{ $ifNull: ["$source.repositoryFullName", ""] }, "#", { $toString: "$source.pullRequestId" }] },
+    { $concat: [{ $ifNull: ["$source.repositoryFullName", ""] }, "#", { $toString: "$source.pullRequestId" }] },
+    { pullRequestId: 1, source: { repositoryFullName: "$source.repositoryFullName", pullRequestId: "$source.pullRequestId" } });
+  const gistProject = project("gist", "gistId", "$source.gistId", { $concat: ["Gist ", { $ifNull: ["$gistId", "$source.gistId"] }] },
+    { gistId: 1, source: { gistId: "$source.gistId" }, isPublic: "$gist.isPublic" });
+  const pipeline: PipelineStage[] = [
+    { $match: { $or: user.repositoryMembership().$or.map(clause => clause.owner ? { owner } : clause) } }, ...repoProject as PipelineStage[],
+    { $unionWith: { coll: PullRequest.collection.name, pipeline: [{ $match: { owner } }, ...prProject] } },
+    { $unionWith: { coll: Gist.collection.name, pipeline: [{ $match: { owner } }, ...gistProject] } },
+    { $lookup: { from: DashboardName.collection.name, let: { type: "$_type", id: "$_anonymousId" },
+      pipeline: [{ $match: { owner, $expr: { $and: [{ $eq: ["$type", "$$type"] }, { $eq: ["$artifactId", "$$id"] }] } } },
+        { $project: { name: 1 } }], as: "_privateName" } },
+    { $set: { projectName: { $ifNull: [{ $arrayElemAt: ["$_privateName.name", 0] }, ""] } } },
+    { $unset: "_privateName" },
+    { $set: { status: { $cond: [{ $and: [{ $eq: ["$status", "ready"] }, { $ne: ["$options.expirationMode", "never"] },
+      { $ne: [{ $ifNull: ["$options.expirationDate", null] }, null] }, { $lte: ["$options.expirationDate", now] }] }, "expired", "$status"] },
+      _label: { $cond: [{ $ne: ["$projectName", ""] }, "$projectName", { $ifNull: ["$_fallbackLabel", "$_anonymousId"] }] } } },
+    { $set: { _statusKey: { $switch: { branches: [
+      { case: { $in: ["$status", ["ready", "error"]] }, then: "$status" },
+      { case: { $in: ["$status", ["expired", "expiring"]] }, then: "expired" },
+      { case: { $in: ["$status", ["removed", "removing"]] }, then: "removed" },
+    ], default: "progress" } } } },
+    { $set: { _attention: { $or: [{ $eq: ["$status", "error"] }, { $eq: ["$_anonymousId", ""] },
+      { $and: [{ $eq: ["$_statusKey", "progress"] }, { $ne: [{ $ifNull: ["$anonymizeDate", "$lastView"] }, null] },
+        { $lt: [{ $ifNull: ["$anonymizeDate", "$lastView"] }, new Date(now.getTime() - 2 * 3600_000)] }] }] },
+      _sort: { $ifNull: [`$${fields[field]}`, ["pageView", "anonymizeDate", "lastView", "options.expirationDate"].includes(field)
+        ? (["pageView"].includes(field) ? 0 : new Date(0)) : ""] } } },
+  ];
+  if (["_name", "_label", "status"].includes(field)) pipeline.push({ $set: { _sort: { $toLower: "$_sort" } } });
+  const filter: Record<string, unknown> = {};
+  if (["repo", "pr", "gist"].includes(String(query.type))) filter._type = query.type;
+  if (query.attention === "true") filter._attention = true;
+  const statuses = String(query.statuses || "ready,progress,error,expired").split(",").filter(s => ["ready", "progress", "error", "expired", "removed"].includes(s));
+  filter._statusKey = { $in: statuses };
+  const search = String(query.q || "").trim().slice(0, 256);
+  if (search) {
+    const pattern = search.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+    filter.$or = ["_anonymousId", "_label", "_source", "conference"].map(key => ({ [key]: { $regex: pattern, $options: "i" } }));
+  }
+  const page: PipelineStage.FacetPipelineStage[] = [{ $match: filter }];
+  if (query.cursor) {
+    try {
+      if (String(query.cursor).length > 2048) throw new Error();
+      const cursor = JSON.parse(Buffer.from(String(query.cursor), "base64url").toString());
+      if (cursor.sort !== sort || typeof cursor.type !== "string" || !Types.ObjectId.isValid(cursor.id)) throw new Error();
+      const value = ["anonymizeDate", "lastView", "options.expirationDate"].includes(field) ? new Date(cursor.value) : cursor.value;
+      if ((value instanceof Date && isNaN(value.getTime())) || !(typeof value === "string" || typeof value === "number" || value instanceof Date)) throw new Error();
+      page.push({ $match: { $or: [ { _sort: { [direction === 1 ? "$gt" : "$lt"]: value } },
+        { _sort: value, _type: { $gt: cursor.type } }, { _sort: value, _type: cursor.type, _id: { $gt: new Types.ObjectId(cursor.id) } },
+      ] } });
+    } catch { throw new AnonymousError("invalid_cursor", { httpStatus: 400 }); }
+  }
+  page.push({ $sort: { _sort: direction, _type: 1, _id: 1 } }, { $limit: limit + 1 });
+  pipeline.push({ $facet: { totals: [{ $group: { _id: null, total: { $sum: 1 }, attention: { $sum: { $cond: ["$_attention", 1, 0] } } } }],
+    filtered: [{ $match: filter }, { $count: "count" }], items: page } });
+  return pipeline;
+}
+export async function dashboardSummary(user: User, query: Record<string, unknown>) {
+  await ensureDashboardNames(user);
+  const [result] = await Repo.aggregate(dashboardPipeline(user, query));
+  const limit = Math.min(100, Math.max(1, Math.floor(Number(query.limit)) || 50));
+  const hasMore = result.items.length > limit;
+  const items = result.items.slice(0, limit);
+  const last = items[items.length - 1];
+  const cursor = hasMore ? Buffer.from(JSON.stringify({ sort: String(query.sort || "-anonymizeDate"),
+    value: last._sort, type: last._type, id: last._id })).toString("base64url") : null;
+  for (const item of items) { delete item._sort; delete item._id; }
+  return { items, cursor, total: result.totals[0]?.total || 0, attention: result.totals[0]?.attention || 0,
+    filtered: result.filtered[0]?.count || 0 };
+}

src/server/routes/file.ts

@@ -3,6 +3,9 @@ import AnonymizedFile from "../../core/AnonymizedFile";
 import AnonymousError from "../../core/AnonymousError";
 import { getRepo, handleError } from "./route-utils";
 import { fileETag } from "./file-etag";
+import config from "../../config";
+import { lookup } from "mime-types";
+import { isTextFile } from "../../core/anonymize-utils";
 
 export const router = express.Router();
 
@@ -142,9 +145,17 @@ router.get(
       // anonymization term list left old anonymizations cached under the
       // same URL.
       res.header("Cache-Control", "private, no-cache, must-revalidate");
-      if (req.headers["if-none-match"] === etag) {
+      if (req.fresh) {
         return res.status(304).end();
       }
+      if (req.method === "HEAD") {
+        const size = await f.size();
+        if (size != null && size > config.MAX_FILE_SIZE) throw new AnonymousError("file_too_big", { httpStatus: 413 });
+        const mime = lookup(originalPath);
+        if (mime && !originalPath.endsWith(".ts")) res.type(mime);
+        else if (isTextFile(originalPath)) res.type("text/plain");
+        res.end(); return;
+      }
       await f.send(res);
       await repo.countView();
     } catch (error) {

src/server/routes/github-app.ts

@@ -5,6 +5,7 @@ import UserModel from "../../core/model/users/users.model";
 import CredentialModel from "../../core/model/credentials/credentials.model";
 import InstallationModel from "../../core/model/github-installation";
 import RepositoryModel from "../../core/model/anonymizedRepositories/anonymizedRepositories.model";
+import CachedRepositoryModel from "../../core/model/repositories/repositories.model";
 import PullRequestModel from "../../core/model/anonymizedPullRequests/anonymizedPullRequests.model";
 import GistModel from "../../core/model/anonymizedGists/anonymizedGists.model";
 import { getCredentialToken } from "../../core/credentials";
@@ -163,14 +164,15 @@ router.get("/connections", async (req, res) => {
       }
       catch (error) { appErrorCode = error instanceof Error ? error.message : "github_app_reconnect_required"; }
     }
-    const repos = await RepositoryModel.find({ owner: user.id, status: { $ne: "removed" } }).select("repoId source.repositoryName githubAccess status").lean();
-    const prs = await PullRequestModel.find({ owner: user.id, status: { $ne: "removed" } }).select("pullRequestId source.repositoryFullName githubAccess status").lean();
+    const repos = await RepositoryModel.find({ owner: user.id, status: { $ne: "removed" } }).select("repoId source.repositoryName githubAccess status statusDate").lean();
+    const prs = await PullRequestModel.find({ owner: user.id, status: { $ne: "removed" } }).select("pullRequestId source.repositoryFullName githubAccess status statusDate").lean();
     const gistCount = await GistModel.countDocuments({ owner: user.id, status: { $ne: "removed" } });
     res.json({ csrf: req.session.githubConnectionCSRF, appEnabled: config.GITHUB_APP_ENABLED && config.GITHUB_APP_NEW_CONNECTIONS,
+      gistConnection: config.GITHUB_APP_ENABLED && credentials.some(c => c.provider === APP_PROVIDER) ? "github-app" : "oauth",
       oauthEnabled: config.GITHUB_OAUTH_ENABLED, oauthConnected: !!(await getCredentialToken(user.id)), appConnected, appError: appErrorCode,
       installations, gistCount, resources: [
-        ...repos.map(r => ({ type: "repository", id: r.repoId, name: r.source.repositoryName, connection: r.githubAccess?.kind || "oauth", status: r.status })),
-        ...prs.map(r => ({ type: "pull-request", id: r.pullRequestId, name: r.source.repositoryFullName, connection: r.githubAccess?.kind || "oauth", status: r.status })),
+        ...repos.map(r => ({ type: "repository", id: r.repoId, name: r.source.repositoryName, connection: r.githubAccess?.kind || "oauth", status: r.status, statusDate: r.statusDate })),
+        ...prs.map(r => ({ type: "pull-request", id: r.pullRequestId, name: r.source.repositoryFullName, connection: r.githubAccess?.kind || "oauth", status: r.status, statusDate: r.statusDate })),
       ] });
   } catch (error) { handleError(error, res, req); }
 });
@@ -191,16 +193,36 @@ router.post("/connections/migrate", async (req, res) => {
     const isRepo = type === "repository";
     const model = isRepo ? await RepositoryModel.findOne({ repoId: id, owner: user.id }) : await PullRequestModel.findOne({ pullRequestId: id, owner: user.id });
     if (!model || ["removed", "archived"].includes(model.status || "")) throw appError("repo_not_found", 404);
-    if (["preparing", "removing", "expiring"].includes(model.status || "")) throw appError("repository_busy", 409);
-    const source = model.source as { repositoryName?: string; repositoryFullName?: string; commit?: string; pullRequestId?: number };
+    const stalePullRequest = !isRepo && model.status === "download" &&
+      (!model.statusDate || model.statusDate.getTime() <= Date.now() - 5 * 60 * 1000);
+    if (!stalePullRequest && ["preparing", "queue", "download", "removing", "expiring"].includes(model.status || "")) throw appError("repository_busy", 409);
+    const source = model.source as { repositoryName?: string; repositoryFullName?: string; repositoryId?: string; commit?: string; pullRequestId?: number };
     const name = source.repositoryName || source.repositoryFullName || "";
-    const selected = await selectRepositoryAccess(user.id, name, connection);
-    const parts = name.split("/").map(encodeURIComponent).join("/");
-    await githubRequest(`/repos/${parts}/${isRepo ? `commits/${encodeURIComponent(source.commit || "")}` : `pulls/${source.pullRequestId}`}`, selected.token);
+    const repositoryId = model.githubAccess?.kind === "github-app" && connection === "github-app"
+      ? model.githubAccess.repositoryId : undefined;
+    const selected = await selectRepositoryAccess(user.id, name, connection, repositoryId);
+    // Repair an installation binding without silently adopting a repository
+    // recreated at the same name. Replacing a source requires its own preview.
+    if (model.githubAccess?.kind === "github-app" && selected.binding.kind === "github-app" &&
+      model.githubAccess.repositoryId !== selected.binding.repositoryId) throw appError("connection_changed", 409);
+    const parts = (selected.fullName || name).split("/").map(encodeURIComponent).join("/");
+    const repositoryPath = `/repos/${parts}`;
+    const token = selected.binding.publicRead ? await appUserToken(user.id) : selected.token;
+    await githubRequest(`${repositoryPath}/${isRepo ? `commits/${encodeURIComponent(source.commit || "")}` : `pulls/${source.pullRequestId}`}`, token);
     if (preview === true) return res.json({ eligible: true, connection });
-    const filter = { _id: model._id, owner: user.id, status: model.status, source: model.source,
+    // This shared cache stores GitHub identity metadata, independently of the
+    // resource binding. Match the numeric ID so a replacement is never renamed.
+    if (isRepo && source.repositoryId && selected.fullName && selected.binding.kind === "github-app") {
+      await CachedRepositoryModel.updateOne({ _id: source.repositoryId, externalId: `gh_${selected.binding.repositoryId}` },
+        { $set: { name: selected.fullName, url: `https://github.com/${selected.fullName}` } });
+    }
+    const filter = { _id: model._id, owner: user.id, status: model.status,
+      ...(stalePullRequest ? { statusDate: model.statusDate ?? null } : {}), source: model.source,
       githubAccess: model.githubAccess ? model.githubAccess : { $exists: false } };
-    const change = { $set: { githubAccess: selected.binding } };
+    const change = { $set: { githubAccess: selected.binding,
+      ...(selected.fullName && selected.fullName !== name
+        ? { [isRepo ? "source.repositoryName" : "source.repositoryFullName"]: selected.fullName } : {}),
+    } };
     const result = isRepo ? await RepositoryModel.updateOne(filter, change) : await PullRequestModel.updateOne(filter, change);
     if (!result.modifiedCount) throw appError("connection_changed", 409);
     res.json({ connection });
@@ -225,9 +247,10 @@ router.post("/connections/disconnect-oauth", async (req, res) => {
   try {
     const user = await getUser(req);
     if (!(await CredentialModel.exists({ ownerId: user.id, provider: APP_PROVIDER, revoked: { $ne: true } }))) throw appError("another_login_required", 409);
-    await appUserToken(user.id);
+    // Confirm the remaining login still works on GitHub before revoking OAuth.
+    await userInstallations(user.id);
     const active = { owner: user.id, status: { $ne: "removed" }, "githubAccess.kind": { $ne: "github-app" } };
-    if (await RepositoryModel.exists(active) || await PullRequestModel.exists(active) || await GistModel.exists({ owner: user.id, status: { $ne: "removed" } })) {
+    if (await RepositoryModel.exists(active) || await PullRequestModel.exists(active)) {
       throw appError("oauth_resources_remaining", 409);
     }
     await revokeGrant(user.id, "github");

src/server/routes/project-names.ts

@@ -0,0 +1,44 @@
+import { isConnected } from "../database";
+import DashboardName from "../../core/model/dashboard-name";
+import { invalidateDashboardNames } from "./dashboard-names";
+import { createHash } from "crypto";
+import AnonymousError from "../../core/AnonymousError";
+import User from "../../core/User";
+import UserModel from "../../core/model/users/users.model";
+
+export function projectNameKey(type: string, id: string): string {
+  return createHash("sha256").update(`${type}:${id}`).digest("hex");
+}
+
+// Names belong to the user's dashboard, rather than the shared artifact.
+export async function saveProjectName(user: User, input: unknown): Promise<string> {
+  const body = input as { type?: unknown; id?: unknown; name?: unknown } | null;
+  if (!body || typeof body.type !== "string" || !["repo", "pr", "gist"].includes(body.type)
+      || typeof body.id !== "string" || !body.id || body.id.length > 250
+      || typeof body.name !== "string" || body.name.trim().length > 100
+      || Array.from(body.name).some(character => {
+        const code = character.charCodeAt(0);
+        return code < 32 || code === 127;
+      })) {
+    throw new AnonymousError("invalid_project_name", { httpStatus: 400 });
+  }
+  const type = String(body.type);
+  const records = type === "repo" ? await user.getRepositories()
+    : type === "pr" ? await user.getPullRequests() : await user.getGists();
+  const field = type === "repo" ? "repoId" : type === "pr" ? "pullRequestId" : "gistId";
+  if (!records.some(record => (record as unknown as Record<string, unknown>)[field] === body.id)) {
+    throw new AnonymousError("not_authorized", { httpStatus: 403 });
+  }
+  const key = projectNameKey(type, body.id);
+  const name = body.name.trim();
+  const path = `projectNames.${key}`;
+  await UserModel.updateOne({ _id: user.model._id }, name
+    ? { $set: { [path]: name } } : { $unset: { [path]: "" } });
+  invalidateDashboardNames();
+  if (name) user.model.set(path, name);
+  else user.model.projectNames?.delete(key);
+  if (isConnected && name) await DashboardName.updateOne({ owner: user.model._id, type, artifactId: body.id },
+    { $set: { name } }, { upsert: true });
+  else if (isConnected) await DashboardName.deleteOne({ owner: user.model._id, type, artifactId: body.id });
+  return name;
+}

src/server/routes/repository-private.ts

@@ -38,6 +38,12 @@ async function previewToken(req: express.Request) {
     const resource = await db.getRepository(req.query.anonymizedRepoId);
     isOwnerCoauthorOrAdmin(resource, user);
     if (resource.model.source.repositoryName?.toLowerCase() !== `${req.params.owner}/${req.params.repo}`.toLowerCase()) throw appError("repo_not_found", 404);
+    if (req.query.reconnect === "1") {
+      if (user.id !== resource.owner.id) throw appError("not_owner", 403);
+      resource.assertNotArchived();
+      return (await selectRepositoryAccess(user.id, resource.model.source.repositoryName,
+        resource.model.githubAccess?.kind || "oauth")).token;
+    }
     return getToken(resource);
   }
   return (await selectRepositoryAccess(user.id, `${req.params.owner}/${req.params.repo}`, req.query.connection)).token;
@@ -186,10 +192,11 @@ router.post(
         repo.model.anonymizeDate = new Date();
         updates.anonymizeDate = repo.model.anonymizeDate;
       }
-      await AnonymizedRepositoryModel.updateOne(
-        { _id: repo.model._id },
+      const saved = await AnonymizedRepositoryModel.updateOne(
+        { _id: repo.model._id, status: repo.status, statusDate: repo.model.statusDate },
         { $set: updates }
       ).exec();
+      if (saved && saved.matchedCount === 0) throw new AnonymousError("invalid_status", { httpStatus: 409 });
 
       if (reactivating) {
         // Expiration removes the cached files. Rebuild the saved commit
@@ -247,8 +254,8 @@ router.get(
         owner: req.params.owner,
         repo: req.params.repo,
         accessToken: token,
-        repositoryID: req.query.repositoryID as string,
-        force: req.query.force == "1",
+        repositoryID: req.query.reconnect === "1" ? undefined : req.query.repositoryID as string,
+        force: req.query.reconnect === "1" || req.query.force == "1",
       });
       res.json(repo.toJSON());
     } catch (error) {
@@ -266,13 +273,13 @@ router.get(
         accessToken: token,
         owner: req.params.owner,
         repo: req.params.repo,
-        repositoryID: req.query.repositoryID as string,
-        force: req.query.force == "1",
+        repositoryID: req.query.reconnect === "1" ? undefined : req.query.repositoryID as string,
+        force: req.query.reconnect === "1" || req.query.force == "1",
       });
       return res.json(
         await repository.branches({
           accessToken: token,
-          force: req.query.force == "1",
+          force: req.query.reconnect === "1" || req.query.force == "1",
         })
       );
     } catch (error) {
@@ -291,8 +298,8 @@ router.get(
         owner: req.params.owner,
         repo: req.params.repo,
         accessToken: token,
-        repositoryID: req.query.repositoryID as string,
-        force: req.query.force == "1",
+        repositoryID: req.query.reconnect === "1" ? undefined : req.query.repositoryID as string,
+        force: req.query.reconnect === "1" || req.query.force == "1",
       });
       if (!repo) {
         throw new AnonymousError("repo_not_found", {
@@ -303,7 +310,7 @@ router.get(
       return res.send(
         await repo.readme({
           accessToken: token,
-          force: req.query.force == "1",
+          force: req.query.reconnect === "1" || req.query.force == "1",
           branch: req.query.branch as string,
         })
       );
@@ -475,6 +482,10 @@ router.post(
 
       const repoUpdate = req.body;
 
+      if (repo.status === RepositoryStatus.EXPIRING || repo.status === RepositoryStatus.REMOVING) {
+        throw appError("invalid_status", 409);
+      }
+
       validateNewRepo(repoUpdate);
 
       // Only the source repository/commit/branch backs the cached FileModel —
@@ -482,7 +493,17 @@ router.post(
       // the fly per request. Re-running the download queue is therefore only
       // needed when the underlying snapshot moves. Other edits (e.g. turning
       // off auto-update — see #360) just persist and return.
-      const sourceChanged = hasRepositorySourceChanged(repo.model, repoUpdate);
+      const reconnecting = repoUpdate.reconnectRepositoryId !== undefined;
+      if (reconnecting) {
+        if (user.id !== repo.owner.id) throw appError("not_owner", 403);
+        if (typeof repoUpdate.reconnectRepositoryId !== "string" ||
+          !/^gh_[1-9][0-9]*$/.test(repoUpdate.reconnectRepositoryId) ||
+          repoUpdate.fullName !== repo.model.source.repositoryName) throw appError("repo_not_found", 400);
+        if (repo.status && [RepositoryStatus.PREPARING, RepositoryStatus.DOWNLOAD, RepositoryStatus.REMOVING,
+          RepositoryStatus.EXPIRING, RepositoryStatus.QUEUE].includes(repo.status)) throw appError("invalid_status", 409);
+        repo.assertNotArchived();
+      }
+      const sourceChanged = reconnecting || hasRepositorySourceChanged(repo.model, repoUpdate);
       const previousAccessRevision = repo.model.githubAccess?.revision;
 
       updateRepoModel(repo.model, repoUpdate);
@@ -501,13 +522,14 @@ router.post(
           });
         }
         if (repoUpdate.fullName !== repo.model.source.repositoryName && user.id !== repo.owner.id) throw appError("not_owner", 403);
-        const sourceAccess = repo.model.githubAccess?.kind === "github-app" && repoUpdate.fullName === repo.model.source.repositoryName
+        const sourceAccess = !reconnecting && repo.model.githubAccess?.kind === "github-app" && repoUpdate.fullName === repo.model.source.repositoryName
           ? { token: await boundAppToken(repo.owner.id, repo.model.githubAccess, repo.model.source.repositoryName), binding: repo.model.githubAccess }
           : await selectRepositoryAccess(repo.owner.id, `${parsedRepository.owner}/${parsedRepository.name}`, repo.model.githubAccess?.kind || "oauth");
         const repository = await getRepositoryFromGitHub({
           accessToken: sourceAccess.token,
           owner: parsedRepository.owner,
           repo: parsedRepository.name,
+          force: reconnecting,
         });
         if (!repository) {
           throw new AnonymousError("repo_not_found", {
@@ -515,9 +537,18 @@ router.post(
             httpStatus: 404,
           });
         }
-        await repository.getCommitInfo(repoUpdate.source.commit, {
+        // Pin the owner's preview to a GitHub identity, even if the name is
+        // deleted and recreated again before Save.
+        if (reconnecting && (repository.id !== repoUpdate.reconnectRepositoryId ||
+          (sourceAccess.binding.kind === "github-app" &&
+            repository.id !== `gh_${sourceAccess.binding.repositoryId}`))) throw appError("connection_changed", 409);
+        if (reconnecting && !(await repository.branches({ accessToken: sourceAccess.token, force: true }))
+          .some(branch => branch.name === repoUpdate.source.branch)) throw appError("branch_not_specified", 400);
+        const commit = await repository.getCommitInfo(repoUpdate.source.commit, {
           accessToken: sourceAccess.token,
         });
+        if (reconnecting) repo.model.source.commitDate = commit.commit.committer?.date
+          ? new Date(commit.commit.committer.date) : undefined;
         repo.model.githubAccess = { ...sourceAccess.binding, revision: randomUUID() };
         repo.model.source.repositoryId = repository.model.id;
         repo.model.source.repositoryName =
@@ -576,7 +607,8 @@ router.post(
       }
       repo.model.conference = repoUpdate.conference;
       const saved = await AnonymizedRepositoryModel.updateOne(
-        { _id: repo.model._id, "githubAccess.revision": previousAccessRevision || { $exists: false } },
+        { _id: repo.model._id, status: repo.status, statusDate: repo.model.statusDate,
+          "githubAccess.revision": previousAccessRevision || { $exists: false } },
         {
           $set: {
             settingsSavedAt: new Date(),

src/server/routes/repository-public.ts

@@ -14,6 +14,7 @@ import FileModel from "../../core/model/files/files.model";
 import { createLogger, serializeError } from "../../core/logger";
 import { githubTokenForStreamer } from "../../core/github-token-context";
 import gh = require("parse-github-url");
+import { requestHeaders, startStage } from "../../core/request-monitoring";
 
 const logger = createLogger("repository-public");
 
@@ -52,16 +53,20 @@ router.get(
         });
       }
 
+      res.attachment(`${repo.repoId}.zip`);
+      if (req.method === "HEAD") { res.end(); return; }
       await repo.countView();
 
       if (config.STREAMER_ENTRYPOINT) {
         // use the streamer service
         const token = await repo.getToken();
         const anonymizer = repo.generateAnonymizeTransformer("");
+        const headersDone = startStage("streamer_headers");
         res.attachment(`${repo.repoId}.zip`);
         const reqStream = got
           .stream(join(config.STREAMER_ENTRYPOINT, "api/download"), {
             method: "POST",
+            headers: requestHeaders(),
             json: {
               token: await githubTokenForStreamer(token, repo.model.source.repositoryName),
               repoFullName: repo.model.source.repositoryName,
@@ -76,6 +81,7 @@ router.get(
             },
           })
           .on("error", (err: Error & { response?: { statusCode?: number; body?: unknown } }) => {
+            headersDone(true);
             const upstreamStatus = err?.response?.statusCode;
             let upstreamBody: string | undefined;
             let errCode = "zip_not_available";
@@ -112,10 +118,11 @@ router.get(
               upstreamStatus,
               upstreamBody: upstreamBody?.slice(0, 500),
               url: config.STREAMER_ENTRYPOINT
-                ? join(config.STREAMER_ENTRYPOINT, "api/zip")
+                ? join(config.STREAMER_ENTRYPOINT, "api/download")
                 : undefined,
               err: serializeError(err),
             });
+            const streaming = res.headersSent;
             handleError(
               new AnonymousError(errCode, {
                 url: req.originalUrl,
@@ -124,7 +131,13 @@ router.get(
               }),
               res
             );
+            // Once ZIP bytes have been sent, handleError cannot send JSON.
+            // Terminate the truncated response so the client reports failure
+            // instead of waiting forever for the ZIP's central directory.
+            if (streaming && !res.writableEnded) res.destroy();
           });
+        reqStream.once("response", () => headersDone());
+        reqStream.once("close", () => headersDone());
         reqStream.pipe(res);
         res.on("close", () => {
           reqStream.destroy();
@@ -197,7 +210,7 @@ router.get(
     try {
       const repoId = repo.repoId;
       const results = await FileModel.aggregate([
-        { $match: { repoId, size: { $ne: null } } },
+        { $match: { repoId, treeGeneration: repo.model.treeGeneration || { $exists: false }, size: { $ne: null } } },
         { $project: { _id: 0, path: 1 } },
         { $group: { _id: "$path", count: { $sum: 1 } } },
       ]).exec();
@@ -251,45 +264,8 @@ router.get(
       if (!query || query.length < 2) {
         return res.json([]);
       }
-      const allFiles = await repo.anonymizedFiles({
-        includeSha: false,
-        recursive: true,
-      });
-      const q = query.toLowerCase();
+      res.json(await repo.searchFiles(query));
 
-      // Collect folder paths whose name segment matches the query
-      const matchingFolders = new Set<string>();
-      for (const f of allFiles) {
-        const segments = (f.path || "").split("/").filter(Boolean);
-        let accumulated = "";
-        for (const seg of segments) {
-          accumulated = accumulated ? `${accumulated}/${seg}` : seg;
-          if (seg.toLowerCase().includes(q)) {
-            matchingFolders.add(accumulated);
-          }
-        }
-      }
-
-      const matches = allFiles.filter((f) => {
-        // File name matches
-        if (f.name?.toLowerCase().includes(q)) return true;
-        // File is inside a matching folder
-        const fullPath = f.path ? `${f.path}/${f.name}` : f.name;
-        let found = false;
-        matchingFolders.forEach((folder) => {
-          if (fullPath?.startsWith(folder + "/") || fullPath === folder) found = true;
-        })
-        if (found) return true;
-        return false;
-      });
-
-      res.json(
-        matches.slice(0, 500).map((f) => ({
-          name: f.name,
-          path: f.path,
-          size: f.size,
-        }))
-      );
     } catch (error) {
       handleError(error, res, req);
     }
@@ -395,6 +371,11 @@ router.get(
         sourceCommitDate: repo.model.source.commitDate,
         settingsSavedAt: repo.model.settingsSavedAt,
         publishedAt: repo.model.publishedAt,
+        // Lets reviewers see how long the link stays available.
+        expirationDate:
+          repo.options.expirationMode !== "never"
+            ? repo.options.expirationDate || null
+            : null,
         isAdmin: user?.isAdmin === true,
         isOwner: user?.id == repo.model.owner,
         hasWebsite: !!repo.options.page && !!repo.options.pageSource,
@@ -407,6 +388,7 @@ router.get(
         hasSubmodules:
           (await FileModel.exists({
             repoId: repo.repoId,
+            treeGeneration: repo.model.treeGeneration || { $exists: false },
             name: ".gitmodules",
             path: "",
           })) != null,

src/server/routes/route-utils.ts

@@ -8,6 +8,7 @@ import { HTTPError } from "got";
 import { RepositoryStatus } from "../../core/types";
 import { createLogger, serializeError } from "../../core/logger";
 import { isDisabledAccount } from "./auth-utils";
+import { measureStage } from "../../core/request-monitoring";
 
 const logger = createLogger("route");
 
@@ -81,7 +82,8 @@ export async function getRepo(
     if (typeof req.params.repoId !== "string") {
       throw new AnonymousError("invalid_path", { httpStatus: 400 });
     }
-    const repo = await db.getRepository(req.params.repoId);
+    const repoId = req.params.repoId;
+    const repo = await measureStage("repository", () => db.getRepository(repoId));
     if (opt.nocheck !== true) {
       // redirect if the repository is expired
       if (

src/server/routes/user.ts

@@ -1,3 +1,5 @@
+import DashboardName from "../../core/model/dashboard-name";
+import { dashboardSummary } from "./dashboard-summary";
 import { revokeGrant } from "./github-app";
 import CredentialModel from "../../core/model/credentials/credentials.model";
 import * as express from "express";
@@ -13,6 +15,7 @@ import FileModel from "../../core/model/files/files.model";
 import { isConnected } from "../database";
 import { octokit } from "../../core/GitHubUtils";
 import { createLogger, serializeError } from "../../core/logger";
+import { projectNameKey, saveProjectName } from "./project-names";
 
 const logger = createLogger("user");
 
@@ -47,20 +50,22 @@ router.get("/", async (req, res) => {
   }
 });
 
+router.get("/dashboard", async (req, res) => {
+  try { res.json(await dashboardSummary(await getUser(req), req.query)); }
+  catch (error) { handleError(error, res, req); }
+});
+
 router.get("/quota", async (req, res) => {
   try {
     const user = await getUser(req);
-    const repositories = (await user.getRepositories()).filter(
-      (r) => r.owner.id === user.model.id
-    );
-    const ready = repositories.filter((r) => r.status == "ready");
+    const ready = await user.quotaRepositories();
 
     let totalStorage = 0;
     let totalFiles = 0;
     const uncachedIds: string[] = [];
     for (const r of ready) {
-      const cached = r.model.size;
-      if (cached && cached.file) {
+      const cached = r.size;
+      if (cached && (cached.file > 0 || r.sizeComputedAt) && cached.storage != null) {
         totalStorage += cached.storage;
         totalFiles += cached.file;
       } else {
@@ -71,7 +76,7 @@ router.get("/quota", async (req, res) => {
     if (uncachedIds.length) {
       const uncachedSet = new Set(uncachedIds);
       const agg = await FileModel.aggregate([
-        { $match: { repoId: { $in: uncachedIds } } },
+        { $match: { $or: ready.filter(r => uncachedSet.has(r.repoId)).map(r => ({ repoId: r.repoId, treeGeneration: r.treeGeneration || { $exists: false } })) } },
         {
           $group: {
             _id: "$repoId",
@@ -89,7 +94,8 @@ router.get("/quota", async (req, res) => {
         const size = byId.get(r.repoId) || { storage: 0, file: 0 };
         totalStorage += size.storage;
         totalFiles += size.file;
-        r.model.size = size;
+        r.size = size;
+        r.sizeComputedAt = new Date();
       }
       if (isConnected) {
         await Promise.all(
@@ -97,8 +103,9 @@ router.get("/quota", async (req, res) => {
             .filter((r) => uncachedSet.has(r.repoId))
             .map((r) =>
               AnonymizedRepositoryModel.updateOne(
-                { _id: r.model._id },
-                { $set: { size: r.model.size } }
+                { _id: r._id, status: "ready", treeGeneration: r.treeGeneration || { $exists: false },
+                  fileMetadataRevision: r.fileMetadataRevision || { $exists: false } },
+                { $set: { size: r.size, sizeComputedAt: r.sizeComputedAt } }
               ).exec()
             )
         );
@@ -124,6 +131,15 @@ router.get("/quota", async (req, res) => {
   }
 });
 
+router.post("/project-name", async (req, res) => {
+  try {
+    const user = await getUser(req);
+    res.json({ name: await saveProjectName(user, req.body) });
+  } catch (error) {
+    handleError(error, res, req);
+  }
+});
+
 router.get("/default", async (req, res) => {
   try {
     const user = await getUser(req);
@@ -197,6 +213,7 @@ router.delete("/", async (req, res) => {
     }
 
     await CredentialModel.deleteMany({ ownerId: user.model._id });
+    if (isConnected) await DashboardName.deleteMany({ owner: user.model._id });
 
     await UserModel.updateOne(
       { _id: user.model._id },
@@ -214,6 +231,7 @@ router.delete("/", async (req, res) => {
           externalIDs: "",
           photo: "",
           default: "",
+          projectNames: "",
         },
       }
     ).exec();
@@ -239,6 +257,7 @@ router.get(
         (await user.getRepositories()).map((x) => {
           const json = x.toJSON() as Record<string, unknown>;
           json.role = x.owner.id === user.model.id ? "owner" : "coauthor";
+          json.projectName = user.model.projectNames?.get(projectNameKey("repo", x.repoId));
           return json;
         })
       );
@@ -254,7 +273,7 @@ router.get(
       const user = await getUser(req);
       res.json(
         (await user.getGists()).map((x) => {
-          return x.toJSON();
+          return { ...x.toJSON(), projectName: user.model.projectNames?.get(projectNameKey("gist", x.gistId)) };
         })
       );
     } catch (error) {
@@ -269,7 +288,7 @@ router.get(
       const user = await getUser(req);
       res.json(
         (await user.getPullRequests()).map((x) => {
-          return x.toJSON();
+          return { ...x.toJSON(), projectName: user.model.projectNames?.get(projectNameKey("pr", x.pullRequestId)) };
         })
       );
     } catch (error) {

src/server/routes/webview.ts

@@ -5,8 +5,11 @@ import AnonymizedFile from "../../core/AnonymizedFile";
 import AnonymousError from "../../core/AnonymousError";
 import * as marked from "marked";
 import * as sanitizeHtml from "sanitize-html";
-import { streamToString } from "../../core/anonymize-utils";
+import { streamToString, isTextFile } from "../../core/anonymize-utils";
 import { IFile } from "../../core/model/files/files.types";
+import { fileETag } from "./file-etag";
+import { lookup } from "mime-types";
+import config from "../../config";
 
 function escapeHtml(str: string): string {
   return str
@@ -170,13 +173,28 @@ async function webView(
         object: f,
       });
     }
+    res.set("Cache-Control", "private, no-cache, must-revalidate");
+    const pageETag = fileETag(JSON.stringify(["web-v1", repo.model.source.commit, repo.model.treeGeneration, repo.model.contentCacheRevision,
+      await f.sha(), repo.model.source.repositoryName, repo.model.source.branch, config.APP_HOSTNAME, config.ANONYMIZATION_MASK]),
+      requestPath, repo.options);
+    res.set("ETag", pageETag);
+    if (req.fresh) { res.status(304).end(); return; }
+    if (req.method === "HEAD") {
+      const size = await f.size();
+      if (size != null && size > config.MAX_FILE_SIZE) throw new AnonymousError("file_too_big", { httpStatus: 413 });
+      const mime = lookup(requestPath);
+      res.type(f.extension() === "md" ? "text/html" : mime && !requestPath.endsWith(".ts") ? mime : isTextFile(requestPath) ? "text/plain" : "application/octet-stream");
+      res.end(); return;
+    }
     if (f.extension() == "md") {
       try {
         const content = await streamToString(await f.anonymizedContent());
-        const body = sanitizeHtml(marked.marked(content, { headerIds: false, mangle: false }), sanitizeOptions);
+        const body = sanitizeHtml(marked.marked.parse(content, { async: false }), sanitizeOptions);
         const html = `<!DOCTYPE html><html><head><title>Content</title></head><link rel="stylesheet" href="/css/all.min.css" /><body><div class="container p-3 file-content markdown-body">${body}</div></body></html>`;
         res.contentType("text/html").send(html);
       } catch {
+        // Raw Markdown must re-enter rendering when a transient failure clears.
+        res.set("ETag", fileETag(`web-raw:${pageETag}`, requestPath, repo.options));
         await f.send(res);
       }
     } else {

src/server/schedule.ts

@@ -1,8 +1,14 @@
+import { Model, Document } from "mongoose";
 import * as schedule from "node-schedule";
+import AnonymizedGistModel from "../core/model/anonymizedGists/anonymizedGists.model";
+import AnonymizedPullRequestModel from "../core/model/anonymizedPullRequests/anonymizedPullRequests.model";
+import Gist from "../core/Gist";
+import PullRequest from "../core/PullRequest";
 import Conference from "../core/Conference";
 import AnonymizedRepositoryModel from "../core/model/anonymizedRepositories/anonymizedRepositories.model";
 import ConferenceModel from "../core/model/conference/conferences.model";
 import Repository from "../core/Repository";
+import FileModel from "../core/model/files/files.model";
 import { createLogger, serializeError } from "../core/logger";
 import { RepositoryStatus } from "../core/types";
 import { computeAndStoreDailyStats } from "./dailyStatsSnapshot";
@@ -28,9 +34,12 @@ export function conferenceStatusCheck() {
 }
 
 export function repositoryStatusCheck() {
-  // check every 6 hours the status of the repositories
-  schedule.scheduleJob("0 */6 * * *", async () => {
-    await runRepositoryStatusCheck();
+  // Claim persisted expiration work every five minutes, without overlapping runs.
+  let running = false;
+  schedule.scheduleJob("*/5 * * * *", async () => {
+    if (running) return;
+    running = true;
+    try { await runRepositoryStatusCheck(); } finally { running = false; }
   });
 }
 
@@ -126,8 +135,7 @@ export async function runRepositoryStatusCheck(now = new Date()) {
       (async () => {
         const repo = new Repository(data);
         try {
-          await repo.resetSate();
-          await repo.updateStatus(RepositoryStatus.EXPIRED);
+          await repo.expire();
           logger.info("recovered expired repository cleanup", {
             repoId: repo.repoId,
           });
@@ -144,6 +152,62 @@ export async function runRepositoryStatusCheck(now = new Date()) {
     }
   }
   await flushBatch();
+  await expireContent(AnonymizedGistModel, data => new Gist(data), now);
+  await expireContent(AnonymizedPullRequestModel, data => new PullRequest(data), now);
+
+  const retiredCursor = AnonymizedRepositoryModel.find({ $or: [
+    { retiredTreeGenerations: { $exists: true } }, { retiredContentPrefixes: { $exists: true } },
+    { legacyContentCleanupPending: true, status: RepositoryStatus.READY },
+    { "stagedFileTrees.until": { $lte: now } },
+  ] }).cursor();
+  for await (const data of retiredCursor) {
+    batch.push(new Repository(data).cleanupRetiredFileTrees(now).catch(error => {
+      logger.error("retired tree cleanup failed", { ...serializeError(error), repoId: data.repoId });
+    }));
+    if (batch.length >= 10) await flushBatch();
+  }
+  await flushBatch();
+
+  const stagedCursor = FileModel.aggregate<{ _id: { repoId: string; generation: string } }>([
+    { $match: { treeStaged: true } }, { $group: { _id: { repoId: "$repoId", generation: "$treeGeneration" } } },
+  ]).cursor();
+  for await (const stage of stagedCursor) {
+    batch.push(Repository.cleanupStagedFileTree(stage._id.repoId, stage._id.generation, now).catch(error => {
+      logger.error("staged tree cleanup failed", { ...serializeError(error), repoId: stage._id.repoId });
+    }));
+    if (batch.length >= 10) await flushBatch();
+  }
+  await flushBatch();
+
+  const pendingCursor = FileModel.find({ metadataPending: true }).cursor();
+  for await (const file of pendingCursor) {
+    batch.push((async () => {
+      try {
+        const data = await AnonymizedRepositoryModel.findOne({ repoId: file.repoId }).exec();
+        if (!data || data.treeGeneration !== file.treeGeneration ||
+          (data.status && [RepositoryStatus.EXPIRING, RepositoryStatus.EXPIRED, RepositoryStatus.REMOVING, RepositoryStatus.REMOVED].includes(data.status))) {
+          await FileModel.deleteOne({ _id: file._id, metadataPending: true }).exec();
+          return;
+        }
+        await new Repository(data).completeRecoveredFileMetadata(file);
+      } catch (error) {
+        logger.error("recovered file invalidation failed", { ...serializeError(error), repoId: file.repoId });
+      }
+    })());
+    if (batch.length >= 10) await flushBatch();
+  }
+  await flushBatch();
+}
+async function expireContent<T extends Document>(model: Model<T>, create: (data: T) => Gist | PullRequest, now: Date) {
+    const cursor = model.find(contentMaintenanceQuery(now)).cursor();
+    for await (const data of cursor) {
+      try { await create(data).expire(); } catch (error) { logger.error("content expiration failed", serializeError(error)); }
+    }
+}
+
+export function contentMaintenanceQuery(now: Date) {
+  return { $or: [{ status: RepositoryStatus.EXPIRING },
+    { status: RepositoryStatus.READY, "options.expirationMode": { $ne: "never" }, "options.expirationDate": { $lte: now } }] };
 }
 
 export function dailyStatsSnapshot() {

src/server/service/review-consent-http.ts

@@ -186,6 +186,25 @@ export function createReviewConsentRouter(
     }
   });
   router.use(express.json({ limit: 12288, strict: true, inflate: false }));
+  // Uploads can outlive the session checked before body parsing. Revalidate
+  // before any provider call or consent write, not only before the response.
+  router.use(async (req, res, next) => {
+    try {
+      const actor = res.locals.consentActor as Context;
+      const fresh = await current(req);
+      if (res.destroyed || res.headersSent || res.locals.consentSignal.aborted)
+        return;
+      if (
+        !fresh ||
+        fresh.accountId !== actor.accountId ||
+        fresh.sessionId !== actor.sessionId
+      )
+        return reject(res, 401, "unauthorized");
+      next();
+    } catch {
+      reject(res, 503, "unavailable");
+    }
+  });
   router.post(["/preview", "/confirm"], async (req, res) => {
     const actor = res.locals.consentActor as Context;
     try {

src/streamer/index.ts

@@ -8,11 +8,17 @@ import config from "../config";
 import router from "./route";
 import { handleError } from "../server/routes/route-utils";
 import AnonymousError from "../core/AnonymousError";
-import { createLogger } from "../core/logger";
+import { createLogger, serializeError } from "../core/logger";
+import { connect } from "../server/database";
+import { startTemporaryStorageMaintenance } from "../core/temporary-storage";
+import { monitorRequests } from "../core/request-monitoring";
+import { startPerformanceMonitoring } from "../core/performance-monitoring";
+import { getAnonymizationPoolStats } from "../core/anonymization-pool";
 
 const logger = createLogger("streamer");
 
 const app = express();
+app.use(monitorRequests("streamer"));
 app.use(express.json());
 
 app.use(
@@ -43,7 +49,16 @@ app.all("/{*path}", (req, res) => {
     req
   );
 });
-app.listen(config.PORT, (error?: Error) => {
-  if (error) throw error;
-  logger.info("streamer started", { port: config.PORT });
+async function start() {
+  startPerformanceMonitoring("streamer", getAnonymizationPoolStats);
+  await connect({ appName: "Anonymous GitHub Streamer", maxPoolSize: 10, minPoolSize: 0 });
+  await startTemporaryStorageMaintenance();
+  app.listen(config.PORT, (error?: Error) => {
+    if (error) throw error;
+    logger.info("streamer started", { port: config.PORT });
+  });
+}
+void start().catch(error => {
+  logger.error("streamer startup failed", serializeError(error));
+  process.exit(1);
 });

src/streamer/route.ts

@@ -32,7 +32,7 @@ router.post(
           repoName: repoFullName[1],
           commit,
           getToken: () => token,
-          anonymizerOptions,
+          anonymizerOptions: { ...anonymizerOptions, repoId },
           contentOptions,
         },
         res
@@ -57,7 +57,9 @@ router.post("/", async (req: express.Request, res: express.Response) => {
   const commit = req.body.commit;
   const filePath: string = req.body.filePath;
   const anonymizerOptions = req.body.anonymizerOptions;
-  const anonymizer = new AnonymizeTransformer(anonymizerOptions);
+  const anonymizer = new AnonymizeTransformer({ ...anonymizerOptions, repoId,
+    cacheGeneration: typeof req.body.cacheGeneration === "string" ? req.body.cacheGeneration : anonymizerOptions?.cacheGeneration,
+  });
 
   // Defence in depth: the parent server validates filePath against
   // FileModel before calling us, but the streamer joins this directly
@@ -79,6 +81,8 @@ router.post("/", async (req: express.Request, res: express.Response) => {
     organization: repoFullName[0],
     repoName: repoFullName[1],
     commit: commit,
+    cacheGeneration: typeof req.body.cacheGeneration === "string" ? req.body.cacheGeneration : undefined,
+    cacheRevision: typeof req.body.cacheRevision === "string" ? req.body.cacheRevision : anonymizerOptions?.cacheRevision,
     getToken: () => token,
   });
   try {
@@ -87,6 +91,7 @@ router.post("/", async (req: express.Request, res: express.Response) => {
       repoId,
       () => ({ sha: fileSha, size: fileSize })
     );
+    if (res.destroyed || res.writableEnded) { content.destroy(); anonymizer.destroy(); return; }
     const mime = lookup(filePath);
     if (mime && !filePath.endsWith(".ts")) {
       res.contentType(mime);
@@ -122,6 +127,7 @@ router.post("/", async (req: express.Request, res: express.Response) => {
       .pipe(res)
       .on("error", handleStreamError)
       .on("close", () => {
+        anonymizer.destroy();
         if (!content.closed && !content.destroyed) {
           content.destroy();
         }