perf: index repository lookups and billing maintenance

src/core/Repository.ts

@@ -1,3 +1,4 @@
+import { REPOSITORY_ID_COLLATION } from "./repository-id";
 import { allowance, reserveRepository } from "./billing/quotas";
 import { FILE_TREE_BATCH_SIZE, FILE_LISTING_LIMIT, deleteRowsInBatches } from "./file-tree";
 import { SourceBase, sourceStorageSize } from "./source/Source";
@@ -422,7 +423,7 @@ export default class Repository {
 
   /** Indexed row markers also discover inserts that finish after a build lease is retired. */
   static async cleanupStagedFileTree(repoId: string, generation: string, now = new Date()) {
-    let current = await AnonymizedRepositoryModel.findOne({ repoId }).exec();
+    let current = await AnonymizedRepositoryModel.findOne({ repoId }, null, { collation: REPOSITORY_ID_COLLATION }).exec();
     if (current && current.treeGeneration !== generation &&
       current.stagedFileTrees?.some(stage => stage.generation === generation && stage.until > now)) return;
     if (current) {

src/core/billing/quotas.ts

@@ -1,3 +1,4 @@
+import { REPOSITORY_ID_COLLATION } from "../repository-id";
 import { randomUUID } from "crypto";
 import ConferenceModel from "../model/conference/conferences.model";
 import config from "../../config";
@@ -30,7 +31,8 @@ export async function reserveRepository(owner: string, repoId: string, bytes: nu
   // count toward the owner's capacity when creating new work.
   if (!await BillingQuota.exists({ _id: limits.scope })) {
     const existing = await RepositoryModel.find(limits.scope.startsWith("user:") ? { owner, status: { $nin: ["removed", "expired"] } }
-      : limits.scope.startsWith("conference:") ? { conference, status: { $nin: ["removed", "expired"] } } : { repoId, status: { $nin: ["removed", "expired"] } })
+      : limits.scope.startsWith("conference:") ? { conference, status: { $nin: ["removed", "expired"] } } : { repoId, status: { $nin: ["removed", "expired"] } }, null,
+      limits.scope.startsWith("repo:") ? { collation: REPOSITORY_ID_COLLATION } : {})
       .select("repoId size owner conference").lean();
     const funded = [];
     for (const repository of existing) {
@@ -42,7 +44,7 @@ export async function reserveRepository(owner: string, repoId: string, bytes: nu
   // Only confirmed terminal repositories release capacity. Unknown reservations
   // remain until reconciled by an operator, so a crash cannot oversell capacity.
   const ledger = await BillingQuota.findById(limits.scope).lean();
-  const terminal = await RepositoryModel.find({ repoId: { $in: ledger?.allocations.map(a => a.repoId) || [] }, status: "removed" }).select("repoId").lean();
+  const terminal = await RepositoryModel.find({ repoId: { $in: ledger?.allocations.map(a => a.repoId) || [] }, status: "removed" }, null, { collation: REPOSITORY_ID_COLLATION }).select("repoId").lean();
   if (terminal.length) await BillingQuota.updateOne({ _id: limits.scope }, { $pull: { allocations: { $or: terminal.map(r => ({
     repoId: r.repoId, reservationId: ledger?.allocations.find(a => a.repoId === r.repoId)?.reservationId || { $exists: false },
   })) } } });
@@ -81,7 +83,7 @@ export async function assertConferenceAdmission(owner: string, repoId: string, c
 export async function reserveFileExpansion(repoId: string, key: string, extraBytes: number) {
   if (!config.BILLING_ENABLED) return;
   if (!Number.isSafeInteger(extraBytes) || extraBytes < 0) throw billingError("billing_invalid_repository_size");
-  const repo = await RepositoryModel.findOne({ repoId }).select("owner conference billingManaged").lean();
+  const repo = await RepositoryModel.findOne({ repoId }, null, { collation: REPOSITORY_ID_COLLATION }).select("owner conference billingManaged").lean();
   if (!repo?.billingManaged) return;
   await reserveRepository(String(repo.owner), repoId, 0, repo.conference, { key, bytes: extraBytes });
 }

src/core/model/anonymizedRepositories/anonymizedRepositories.schema.ts

@@ -110,6 +110,10 @@ const AnonymizedRepositorySchema = new Schema({
   },
 });
 
+AnonymizedRepositorySchema.index({ billingNeverExpiresUntil: 1, status: 1 }, {
+  name: "billing_never_expiration",
+  partialFilterExpression: { "options.expirationMode": "never", billingNeverExpiresUntil: { $exists: true } },
+});
 AnonymizedRepositorySchema.index({ "source.repositoryName": 1 });
 AnonymizedRepositorySchema.index({ "coauthors.githubId": 1 });
 AnonymizedRepositorySchema.index({ retiredTreeGenerations: 1 }, { sparse: true });

src/core/model/files/files.schema.ts

@@ -4,7 +4,7 @@ export const UpstreamFileSchema = new Schema({ organization: String, repoName: S
 
 const FileSchema = new Schema({
   name: { type: String, index: true },
-  path: { type: String, index: true },
+  path: { type: String },
   repoId: { type: String, index: true },
   treeGeneration: String,
   treeStaged: Boolean,
@@ -19,7 +19,6 @@ const FileSchema = new Schema({
   },
 });
 
-FileSchema.index({ path: 1, repoId: 1 });
 FileSchema.index({ metadataPending: 1, repoId: 1, treeGeneration: 1 },
   { partialFilterExpression: { metadataPending: true } });
 FileSchema.index({ treeStaged: 1, repoId: 1, treeGeneration: 1 },

src/core/repository-id.ts

@@ -0,0 +1,2 @@
+/** Repository URLs and their unique Mongo index use case-insensitive IDs. */
+export const REPOSITORY_ID_COLLATION = { locale: "en", strength: 2 };

src/core/source/GitHubStream.ts

@@ -1,3 +1,4 @@
+import { REPOSITORY_ID_COLLATION } from "../repository-id";
 import { reserveFileExpansion } from "../billing/quotas";
 import { withOwnerAccessRecovery } from "../owner-notifications";
 import { ExpiringMap } from "../expiring-map";
@@ -362,7 +363,7 @@ export default class GitHubStream extends GitHubBase {
       // Retirement markers live inside the storage root and disappear when
       // expiration removes it. The persisted lifecycle also fences late fills.
       if (isConnected) {
-        const current = await AnonymizedRepositoryModel.findOne({ repoId })
+        const current = await AnonymizedRepositoryModel.findOne({ repoId }, null, { collation: REPOSITORY_ID_COLLATION })
           .select("status treeGeneration anonymizeDate contentCacheRevision").lean().exec();
         if (!current || !current.status || [RepositoryStatus.EXPIRING, RepositoryStatus.EXPIRED,
           RepositoryStatus.REMOVING, RepositoryStatus.REMOVED, RepositoryStatus.ARCHIVED].includes(current.status)
@@ -641,7 +642,7 @@ export default class GitHubStream extends GitHubBase {
   private async confirmOwnerAccess<T>(repoId: string, work: () => Promise<T>, confirmed?: (result: T) => boolean): Promise<T> {
     let repository;
     try {
-      repository = isConnected ? await AnonymizedRepositoryModel.findOne({ repoId })
+      repository = isConnected ? await AnonymizedRepositoryModel.findOne({ repoId }, null, { collation: REPOSITORY_ID_COLLATION })
         .select("owner source githubAccess").lean().exec() : null;
     } catch {
       // Email recovery bookkeeping must not prevent the upstream read.

src/core/transformed-cache.ts

@@ -1,3 +1,4 @@
+import { REPOSITORY_ID_COLLATION } from "./repository-id";
 import { observeCache } from "./cache-observability";
 import { promises as fs } from "fs";
 import { join } from "path";
@@ -18,7 +19,7 @@ interface CacheScope { repoId: string; generation?: string; revision?: string }
 
 async function scopeActive(scope?: CacheScope) {
   if (!scope || !isConnected) return true;
-  const current = await RepositoryModel.findOne({ repoId: scope.repoId })
+  const current = await RepositoryModel.findOne({ repoId: scope.repoId }, null, { collation: REPOSITORY_ID_COLLATION })
     .select("status treeGeneration anonymizeDate contentCacheRevision").lean().exec();
   return !!current?.status && ![RepositoryStatus.EXPIRING, RepositoryStatus.EXPIRED,
     RepositoryStatus.REMOVING, RepositoryStatus.REMOVED, RepositoryStatus.ARCHIVED].includes(current.status)

src/scripts/create-performance-indexes.ts

@@ -11,6 +11,10 @@ import Name from "../core/model/dashboard-name";
 async function main() {
   const uri = config.MONGODB_URI || `mongodb://${config.DB_USERNAME}:${config.DB_PASSWORD}@${config.DB_HOSTNAME}:27017/production`;
   await mongoose.connect(uri, { autoIndex: false, ...(config.MONGODB_URI ? {} : { authSource: "admin" }) });
+  await Repository.collection.createIndex({ billingNeverExpiresUntil: 1, status: 1 }, {
+    name: "billing_never_expiration",
+    partialFilterExpression: { "options.expirationMode": "never", billingNeverExpiresUntil: { $exists: true } },
+  });
   // Additive only. Never use syncIndexes here, which can drop deployed indexes.
   await Repository.collection.createIndex({ "coauthors.githubId": 1 });
   await File.collection.createIndex({ repoId: 1, treeGeneration: 1, path: 1, name: 1 });

src/scripts/retire-file-indexes.ts

@@ -0,0 +1,51 @@
+import "dotenv/config";
+import mongoose, { mongo } from "mongoose";
+import config from "../config";
+
+const candidates = [
+  { name: "path_1", key: { path: 1 } },
+  { name: "path_1_repoId_1", key: { path: 1, repoId: 1 } },
+  { name: "repoIDAndPath", key: { repoId: 1, path: 1 } },
+];
+const replacementKey = { repoId: 1, treeGeneration: 1, path: 1, name: 1 };
+
+/** Operator-controlled retirement: hide first, observe traffic, then drop. */
+export async function retireFileIndexes(db: mongo.Db, action: string) {
+  if (!["status", "hide", "restore", "drop"].includes(action)) throw Error("Use status, hide, restore or drop");
+  const collection = db.collection("files");
+  const indexes = await collection.listIndexes().toArray();
+  const selected = candidates.flatMap(candidate => {
+    const index = indexes.find(index => index.name === candidate.name);
+    if (!index) return [];
+    if (JSON.stringify(index.key) !== JSON.stringify(candidate.key) || index.unique || index.sparse ||
+        index.partialFilterExpression || index.collation || index.expireAfterSeconds !== undefined) {
+      throw Error(`Unexpected definition for ${candidate.name}; refusing retirement`);
+    }
+    return [index];
+  });
+  if (action === "hide" || action === "drop") {
+    if (!indexes.some(index => JSON.stringify(index.key) === JSON.stringify(replacementKey) && !index.hidden &&
+        !index.partialFilterExpression && !index.sparse && !index.collation && !index.buildUUID)) {
+      throw Error("The generation/path replacement index must be ready and visible");
+    }
+  }
+  // Validate every candidate before changing any of them.
+  if (action === "drop" && selected.some(index => !index.hidden)) throw Error("Hide all candidate indexes and observe traffic before dropping them");
+  for (const index of selected) {
+    if (action === "drop") await collection.dropIndex(index.name!);
+    else if (action === "hide" || action === "restore") {
+      await db.command({ collMod: "files", index: { name: index.name, hidden: action === "hide" } });
+    }
+  }
+  return selected.map(index => ({ name: index.name, action, previouslyHidden: !!index.hidden }));
+}
+
+async function main() {
+  const uri = config.MONGODB_URI || `mongodb://${encodeURIComponent(config.DB_USERNAME)}:${encodeURIComponent(config.DB_PASSWORD)}@${config.DB_HOSTNAME}:27017/production`;
+  await mongoose.connect(uri, { autoIndex: false, ...(config.MONGODB_URI ? {} : { authSource: "admin" }) });
+  console.log(await retireFileIndexes(mongoose.connection.db!, process.argv[2] || "status"));
+}
+if (require.main === module) {
+  main().catch(error => { console.error(error instanceof Error ? error.message : "Index retirement failed"); process.exitCode = 1; })
+    .finally(() => mongoose.disconnect());
+}

src/server/routes/github-app.ts

@@ -1,3 +1,4 @@
+import { REPOSITORY_ID_COLLATION } from "../../core/repository-id";
 import { notificationEmail, normalizeEmail, resetOwnerAccessAlerts } from "../../core/owner-notifications";
 import { saveRegistrationEmail } from "../../core/registration-email";
 import * as express from "express";
@@ -230,7 +231,7 @@ router.post("/connections/migrate", async (req, res) => {
     if (typeof id !== "string" || !["repository", "pull-request"].includes(type) || !["oauth", "github-app"].includes(connection)) throw appError("invalid_connection", 400);
     if (connection === "github-app" && (!config.GITHUB_APP_ENABLED || !config.GITHUB_APP_NEW_CONNECTIONS)) throw appError("github_app_disabled", 503);
     const isRepo = type === "repository";
-    const model = isRepo ? await RepositoryModel.findOne({ repoId: id, owner: user.id }) : await PullRequestModel.findOne({ pullRequestId: id, owner: user.id });
+    const model = isRepo ? await RepositoryModel.findOne({ repoId: id, owner: user.id }, null, { collation: REPOSITORY_ID_COLLATION }) : await PullRequestModel.findOne({ pullRequestId: id, owner: user.id });
     if (!model || ["removed", "archived"].includes(model.status || "")) throw appError("repo_not_found", 404);
     const stalePullRequest = !isRepo && model.status === "download" &&
       (!model.statusDate || model.statusDate.getTime() <= Date.now() - 5 * 60 * 1000);

src/server/schedule.ts

@@ -1,3 +1,4 @@
+import { REPOSITORY_ID_COLLATION } from "../core/repository-id";
 import { reconcileNeverExpiration } from "../core/billing/expiration";
 import { Model, Document } from "mongoose";
 import * as schedule from "node-schedule";
@@ -185,7 +186,7 @@ export async function runRepositoryStatusCheck(now = new Date()) {
   for await (const file of pendingCursor) {
     batch.push((async () => {
       try {
-        const data = await AnonymizedRepositoryModel.findOne({ repoId: file.repoId }).exec();
+        const data = await AnonymizedRepositoryModel.findOne({ repoId: file.repoId }, null, { collation: REPOSITORY_ID_COLLATION }).exec();
         if (!data || data.treeGeneration !== file.treeGeneration ||
           (data.status && [RepositoryStatus.EXPIRING, RepositoryStatus.EXPIRED, RepositoryStatus.REMOVING, RepositoryStatus.REMOVED].includes(data.status))) {
           await FileModel.deleteOne({ _id: file._id, metadataPending: true }).exec();