From 3999a71583bd7deaeee19b8386df9a7cf9e97bfb Mon Sep 17 00:00:00 2001 From: Codex Date: Wed, 11 Nov 2026 10:18:20 +0000 Subject: [PATCH] perf: index repository lookups and billing maintenance --- docs/mongo-index-retirement.md | 40 ++++++ src/core/Repository.ts | 3 +- src/core/billing/quotas.ts | 8 +- .../anonymizedRepositories.schema.ts | 4 + src/core/model/files/files.schema.ts | 3 +- src/core/repository-id.ts | 2 + src/core/source/GitHubStream.ts | 5 +- src/core/transformed-cache.ts | 3 +- src/scripts/create-performance-indexes.ts | 4 + src/scripts/retire-file-indexes.ts | 51 ++++++++ src/server/routes/github-app.ts | 3 +- src/server/schedule.ts | 3 +- test/billing-integration.test.js | 17 +++ test/mongo-index-audit.test.js | 123 ++++++++++++++++++ 14 files changed, 258 insertions(+), 11 deletions(-) create mode 100644 docs/mongo-index-retirement.md create mode 100644 src/core/repository-id.ts create mode 100644 src/scripts/retire-file-indexes.ts create mode 100644 test/mongo-index-audit.test.js diff --git a/docs/mongo-index-retirement.md b/docs/mongo-index-retirement.md new file mode 100644 index 0000000..e7ff8c6 --- /dev/null +++ b/docs/mongo-index-retirement.md @@ -0,0 +1,40 @@ +# Retire redundant file indexes + +The November 11 audit found no query use of `path_1`, `path_1_repoId_1`, or +`repoIDAndPath` since the Mongo restart on November 8. Together they occupied +3.17 GiB. This observation is a retirement candidate list, not proof that every +infrequent workload is covered. + +Deploy the updated application first so Mongoose no longer recreates the old +path indexes. The existing additive index migration does not drop indexes. +Then run these commands in the application environment: + +```sh +node build/scripts/retire-file-indexes.js status +node build/scripts/retire-file-indexes.js hide +``` + +Observe at least one full traffic/maintenance cycle, including ZIP reads, +legacy snapshots, path searches, tree cleanup and refreshes. Compare slow-query +plans and latency with the pre-hide baseline. Hidden indexes still consume +storage and incur write maintenance; hiding alone does not reclaim space. + +If queries regress, restore visibility immediately: + +```sh +node build/scripts/retire-file-indexes.js restore +``` + +Only after the observation period is satisfactory, reclaim the storage: + +```sh +node build/scripts/retire-file-indexes.js drop +``` + +The script accepts only the three audited index definitions, rejects unique, +TTL, partial, sparse or collated variants, requires the generation/path index +to be visible and ready, and refuses to drop candidates that are not hidden. +It never removes the unique or TTL indexes used by other collections. A dropped +index cannot be restored with `restore`; it must be rebuilt from its original +key definition. Keep the pre-hide baseline and observation results with the +operator's deployment record. diff --git a/src/core/Repository.ts b/src/core/Repository.ts index 8835774..e7aa8a5 100644 --- a/src/core/Repository.ts +++ b/src/core/Repository.ts @@ -1,3 +1,4 @@ +import { REPOSITORY_ID_COLLATION } from "./repository-id"; import { allowance, reserveRepository } from "./billing/quotas"; import { FILE_TREE_BATCH_SIZE, FILE_LISTING_LIMIT, deleteRowsInBatches } from "./file-tree"; import { SourceBase, sourceStorageSize } from "./source/Source"; @@ -422,7 +423,7 @@ export default class Repository { /** Indexed row markers also discover inserts that finish after a build lease is retired. */ static async cleanupStagedFileTree(repoId: string, generation: string, now = new Date()) { - let current = await AnonymizedRepositoryModel.findOne({ repoId }).exec(); + let current = await AnonymizedRepositoryModel.findOne({ repoId }, null, { collation: REPOSITORY_ID_COLLATION }).exec(); if (current && current.treeGeneration !== generation && current.stagedFileTrees?.some(stage => stage.generation === generation && stage.until > now)) return; if (current) { diff --git a/src/core/billing/quotas.ts b/src/core/billing/quotas.ts index c483c2f..8cb22dd 100644 --- a/src/core/billing/quotas.ts +++ b/src/core/billing/quotas.ts @@ -1,3 +1,4 @@ +import { REPOSITORY_ID_COLLATION } from "../repository-id"; import { randomUUID } from "crypto"; import ConferenceModel from "../model/conference/conferences.model"; import config from "../../config"; @@ -30,7 +31,8 @@ export async function reserveRepository(owner: string, repoId: string, bytes: nu // count toward the owner's capacity when creating new work. if (!await BillingQuota.exists({ _id: limits.scope })) { const existing = await RepositoryModel.find(limits.scope.startsWith("user:") ? { owner, status: { $nin: ["removed", "expired"] } } - : limits.scope.startsWith("conference:") ? { conference, status: { $nin: ["removed", "expired"] } } : { repoId, status: { $nin: ["removed", "expired"] } }) + : limits.scope.startsWith("conference:") ? { conference, status: { $nin: ["removed", "expired"] } } : { repoId, status: { $nin: ["removed", "expired"] } }, null, + limits.scope.startsWith("repo:") ? { collation: REPOSITORY_ID_COLLATION } : {}) .select("repoId size owner conference").lean(); const funded = []; for (const repository of existing) { @@ -42,7 +44,7 @@ export async function reserveRepository(owner: string, repoId: string, bytes: nu // Only confirmed terminal repositories release capacity. Unknown reservations // remain until reconciled by an operator, so a crash cannot oversell capacity. const ledger = await BillingQuota.findById(limits.scope).lean(); - const terminal = await RepositoryModel.find({ repoId: { $in: ledger?.allocations.map(a => a.repoId) || [] }, status: "removed" }).select("repoId").lean(); + const terminal = await RepositoryModel.find({ repoId: { $in: ledger?.allocations.map(a => a.repoId) || [] }, status: "removed" }, null, { collation: REPOSITORY_ID_COLLATION }).select("repoId").lean(); if (terminal.length) await BillingQuota.updateOne({ _id: limits.scope }, { $pull: { allocations: { $or: terminal.map(r => ({ repoId: r.repoId, reservationId: ledger?.allocations.find(a => a.repoId === r.repoId)?.reservationId || { $exists: false }, })) } } }); @@ -81,7 +83,7 @@ export async function assertConferenceAdmission(owner: string, repoId: string, c export async function reserveFileExpansion(repoId: string, key: string, extraBytes: number) { if (!config.BILLING_ENABLED) return; if (!Number.isSafeInteger(extraBytes) || extraBytes < 0) throw billingError("billing_invalid_repository_size"); - const repo = await RepositoryModel.findOne({ repoId }).select("owner conference billingManaged").lean(); + const repo = await RepositoryModel.findOne({ repoId }, null, { collation: REPOSITORY_ID_COLLATION }).select("owner conference billingManaged").lean(); if (!repo?.billingManaged) return; await reserveRepository(String(repo.owner), repoId, 0, repo.conference, { key, bytes: extraBytes }); } diff --git a/src/core/model/anonymizedRepositories/anonymizedRepositories.schema.ts b/src/core/model/anonymizedRepositories/anonymizedRepositories.schema.ts index c552c00..cb17663 100644 --- a/src/core/model/anonymizedRepositories/anonymizedRepositories.schema.ts +++ b/src/core/model/anonymizedRepositories/anonymizedRepositories.schema.ts @@ -110,6 +110,10 @@ const AnonymizedRepositorySchema = new Schema({ }, }); +AnonymizedRepositorySchema.index({ billingNeverExpiresUntil: 1, status: 1 }, { + name: "billing_never_expiration", + partialFilterExpression: { "options.expirationMode": "never", billingNeverExpiresUntil: { $exists: true } }, +}); AnonymizedRepositorySchema.index({ "source.repositoryName": 1 }); AnonymizedRepositorySchema.index({ "coauthors.githubId": 1 }); AnonymizedRepositorySchema.index({ retiredTreeGenerations: 1 }, { sparse: true }); diff --git a/src/core/model/files/files.schema.ts b/src/core/model/files/files.schema.ts index 2c2a01b..9979e52 100644 --- a/src/core/model/files/files.schema.ts +++ b/src/core/model/files/files.schema.ts @@ -4,7 +4,7 @@ export const UpstreamFileSchema = new Schema({ organization: String, repoName: S const FileSchema = new Schema({ name: { type: String, index: true }, - path: { type: String, index: true }, + path: { type: String }, repoId: { type: String, index: true }, treeGeneration: String, treeStaged: Boolean, @@ -19,7 +19,6 @@ const FileSchema = new Schema({ }, }); -FileSchema.index({ path: 1, repoId: 1 }); FileSchema.index({ metadataPending: 1, repoId: 1, treeGeneration: 1 }, { partialFilterExpression: { metadataPending: true } }); FileSchema.index({ treeStaged: 1, repoId: 1, treeGeneration: 1 }, diff --git a/src/core/repository-id.ts b/src/core/repository-id.ts new file mode 100644 index 0000000..233125b --- /dev/null +++ b/src/core/repository-id.ts @@ -0,0 +1,2 @@ +/** Repository URLs and their unique Mongo index use case-insensitive IDs. */ +export const REPOSITORY_ID_COLLATION = { locale: "en", strength: 2 }; diff --git a/src/core/source/GitHubStream.ts b/src/core/source/GitHubStream.ts index d7cc447..303e276 100644 --- a/src/core/source/GitHubStream.ts +++ b/src/core/source/GitHubStream.ts @@ -1,3 +1,4 @@ +import { REPOSITORY_ID_COLLATION } from "../repository-id"; import { reserveFileExpansion } from "../billing/quotas"; import { withOwnerAccessRecovery } from "../owner-notifications"; import { ExpiringMap } from "../expiring-map"; @@ -362,7 +363,7 @@ export default class GitHubStream extends GitHubBase { // Retirement markers live inside the storage root and disappear when // expiration removes it. The persisted lifecycle also fences late fills. if (isConnected) { - const current = await AnonymizedRepositoryModel.findOne({ repoId }) + const current = await AnonymizedRepositoryModel.findOne({ repoId }, null, { collation: REPOSITORY_ID_COLLATION }) .select("status treeGeneration anonymizeDate contentCacheRevision").lean().exec(); if (!current || !current.status || [RepositoryStatus.EXPIRING, RepositoryStatus.EXPIRED, RepositoryStatus.REMOVING, RepositoryStatus.REMOVED, RepositoryStatus.ARCHIVED].includes(current.status) @@ -641,7 +642,7 @@ export default class GitHubStream extends GitHubBase { private async confirmOwnerAccess(repoId: string, work: () => Promise, confirmed?: (result: T) => boolean): Promise { let repository; try { - repository = isConnected ? await AnonymizedRepositoryModel.findOne({ repoId }) + repository = isConnected ? await AnonymizedRepositoryModel.findOne({ repoId }, null, { collation: REPOSITORY_ID_COLLATION }) .select("owner source githubAccess").lean().exec() : null; } catch { // Email recovery bookkeeping must not prevent the upstream read. diff --git a/src/core/transformed-cache.ts b/src/core/transformed-cache.ts index 8f78469..473fa72 100644 --- a/src/core/transformed-cache.ts +++ b/src/core/transformed-cache.ts @@ -1,3 +1,4 @@ +import { REPOSITORY_ID_COLLATION } from "./repository-id"; import { observeCache } from "./cache-observability"; import { promises as fs } from "fs"; import { join } from "path"; @@ -18,7 +19,7 @@ interface CacheScope { repoId: string; generation?: string; revision?: string } async function scopeActive(scope?: CacheScope) { if (!scope || !isConnected) return true; - const current = await RepositoryModel.findOne({ repoId: scope.repoId }) + const current = await RepositoryModel.findOne({ repoId: scope.repoId }, null, { collation: REPOSITORY_ID_COLLATION }) .select("status treeGeneration anonymizeDate contentCacheRevision").lean().exec(); return !!current?.status && ![RepositoryStatus.EXPIRING, RepositoryStatus.EXPIRED, RepositoryStatus.REMOVING, RepositoryStatus.REMOVED, RepositoryStatus.ARCHIVED].includes(current.status) diff --git a/src/scripts/create-performance-indexes.ts b/src/scripts/create-performance-indexes.ts index 3a577b2..8ad5c3f 100644 --- a/src/scripts/create-performance-indexes.ts +++ b/src/scripts/create-performance-indexes.ts @@ -11,6 +11,10 @@ import Name from "../core/model/dashboard-name"; async function main() { const uri = config.MONGODB_URI || `mongodb://${config.DB_USERNAME}:${config.DB_PASSWORD}@${config.DB_HOSTNAME}:27017/production`; await mongoose.connect(uri, { autoIndex: false, ...(config.MONGODB_URI ? {} : { authSource: "admin" }) }); + await Repository.collection.createIndex({ billingNeverExpiresUntil: 1, status: 1 }, { + name: "billing_never_expiration", + partialFilterExpression: { "options.expirationMode": "never", billingNeverExpiresUntil: { $exists: true } }, + }); // Additive only. Never use syncIndexes here, which can drop deployed indexes. await Repository.collection.createIndex({ "coauthors.githubId": 1 }); await File.collection.createIndex({ repoId: 1, treeGeneration: 1, path: 1, name: 1 }); diff --git a/src/scripts/retire-file-indexes.ts b/src/scripts/retire-file-indexes.ts new file mode 100644 index 0000000..fdb247f --- /dev/null +++ b/src/scripts/retire-file-indexes.ts @@ -0,0 +1,51 @@ +import "dotenv/config"; +import mongoose, { mongo } from "mongoose"; +import config from "../config"; + +const candidates = [ + { name: "path_1", key: { path: 1 } }, + { name: "path_1_repoId_1", key: { path: 1, repoId: 1 } }, + { name: "repoIDAndPath", key: { repoId: 1, path: 1 } }, +]; +const replacementKey = { repoId: 1, treeGeneration: 1, path: 1, name: 1 }; + +/** Operator-controlled retirement: hide first, observe traffic, then drop. */ +export async function retireFileIndexes(db: mongo.Db, action: string) { + if (!["status", "hide", "restore", "drop"].includes(action)) throw Error("Use status, hide, restore or drop"); + const collection = db.collection("files"); + const indexes = await collection.listIndexes().toArray(); + const selected = candidates.flatMap(candidate => { + const index = indexes.find(index => index.name === candidate.name); + if (!index) return []; + if (JSON.stringify(index.key) !== JSON.stringify(candidate.key) || index.unique || index.sparse || + index.partialFilterExpression || index.collation || index.expireAfterSeconds !== undefined) { + throw Error(`Unexpected definition for ${candidate.name}; refusing retirement`); + } + return [index]; + }); + if (action === "hide" || action === "drop") { + if (!indexes.some(index => JSON.stringify(index.key) === JSON.stringify(replacementKey) && !index.hidden && + !index.partialFilterExpression && !index.sparse && !index.collation && !index.buildUUID)) { + throw Error("The generation/path replacement index must be ready and visible"); + } + } + // Validate every candidate before changing any of them. + if (action === "drop" && selected.some(index => !index.hidden)) throw Error("Hide all candidate indexes and observe traffic before dropping them"); + for (const index of selected) { + if (action === "drop") await collection.dropIndex(index.name!); + else if (action === "hide" || action === "restore") { + await db.command({ collMod: "files", index: { name: index.name, hidden: action === "hide" } }); + } + } + return selected.map(index => ({ name: index.name, action, previouslyHidden: !!index.hidden })); +} + +async function main() { + const uri = config.MONGODB_URI || `mongodb://${encodeURIComponent(config.DB_USERNAME)}:${encodeURIComponent(config.DB_PASSWORD)}@${config.DB_HOSTNAME}:27017/production`; + await mongoose.connect(uri, { autoIndex: false, ...(config.MONGODB_URI ? {} : { authSource: "admin" }) }); + console.log(await retireFileIndexes(mongoose.connection.db!, process.argv[2] || "status")); +} +if (require.main === module) { + main().catch(error => { console.error(error instanceof Error ? error.message : "Index retirement failed"); process.exitCode = 1; }) + .finally(() => mongoose.disconnect()); +} diff --git a/src/server/routes/github-app.ts b/src/server/routes/github-app.ts index 4c0a623..4bbd40c 100644 --- a/src/server/routes/github-app.ts +++ b/src/server/routes/github-app.ts @@ -1,3 +1,4 @@ +import { REPOSITORY_ID_COLLATION } from "../../core/repository-id"; import { notificationEmail, normalizeEmail, resetOwnerAccessAlerts } from "../../core/owner-notifications"; import { saveRegistrationEmail } from "../../core/registration-email"; import * as express from "express"; @@ -230,7 +231,7 @@ router.post("/connections/migrate", async (req, res) => { if (typeof id !== "string" || !["repository", "pull-request"].includes(type) || !["oauth", "github-app"].includes(connection)) throw appError("invalid_connection", 400); if (connection === "github-app" && (!config.GITHUB_APP_ENABLED || !config.GITHUB_APP_NEW_CONNECTIONS)) throw appError("github_app_disabled", 503); const isRepo = type === "repository"; - const model = isRepo ? await RepositoryModel.findOne({ repoId: id, owner: user.id }) : await PullRequestModel.findOne({ pullRequestId: id, owner: user.id }); + const model = isRepo ? await RepositoryModel.findOne({ repoId: id, owner: user.id }, null, { collation: REPOSITORY_ID_COLLATION }) : await PullRequestModel.findOne({ pullRequestId: id, owner: user.id }); if (!model || ["removed", "archived"].includes(model.status || "")) throw appError("repo_not_found", 404); const stalePullRequest = !isRepo && model.status === "download" && (!model.statusDate || model.statusDate.getTime() <= Date.now() - 5 * 60 * 1000); diff --git a/src/server/schedule.ts b/src/server/schedule.ts index af1f04d..6f77f86 100644 --- a/src/server/schedule.ts +++ b/src/server/schedule.ts @@ -1,3 +1,4 @@ +import { REPOSITORY_ID_COLLATION } from "../core/repository-id"; import { reconcileNeverExpiration } from "../core/billing/expiration"; import { Model, Document } from "mongoose"; import * as schedule from "node-schedule"; @@ -185,7 +186,7 @@ export async function runRepositoryStatusCheck(now = new Date()) { for await (const file of pendingCursor) { batch.push((async () => { try { - const data = await AnonymizedRepositoryModel.findOne({ repoId: file.repoId }).exec(); + const data = await AnonymizedRepositoryModel.findOne({ repoId: file.repoId }, null, { collation: REPOSITORY_ID_COLLATION }).exec(); if (!data || data.treeGeneration !== file.treeGeneration || (data.status && [RepositoryStatus.EXPIRING, RepositoryStatus.EXPIRED, RepositoryStatus.REMOVING, RepositoryStatus.REMOVED].includes(data.status))) { await FileModel.deleteOne({ _id: file._id, metadataPending: true }).exec(); diff --git a/test/billing-integration.test.js b/test/billing-integration.test.js index 96d6253..1bbae59 100644 --- a/test/billing-integration.test.js +++ b/test/billing-integration.test.js @@ -30,6 +30,23 @@ async function rejects(promise, message) { await BillingAccount.create({ _id: owner, customerId: "cus_owner" }); }); after(async () => { Object.assign(config, original); await mongoose.disconnect(); }); + it("indexes terminal quota lookups by repository ID rather than scanning removed repositories", async () => { + await Repo.insertMany(Array.from({ length: 1000 }, (_, i) => ({ repoId: `removed-${i}`, owner, status: "removed" }))); + await BillingQuota.create({ _id: `user:${owner}`, allocations: [{ repoId: "removed-999", reservationId: "old", bytes: 90 }] }); + const find = Repo.find; let terminalQuery; + Repo.find = function (...args) { + const query = find.apply(this, args); + if (args[0]?.repoId?.$in) terminalQuery = query; + return query; + }; + try { + await reserveRepository(owner, "new", 100); + expect((await BillingQuota.findById(`user:${owner}`)).allocations.map(a => a.repoId)).to.deep.equal(["new"]); + const plan = await terminalQuery.clone().explain("executionStats"); + expect(plan.executionStats.totalDocsExamined).to.equal(1); + expect(plan.executionStats.totalKeysExamined).to.equal(1); + } finally { Repo.find = find; } + }); it("atomically admits only three concurrent repositories", async () => { const results = await Promise.allSettled(Array.from({ length: 12 }, (_, i) => reserveRepository(owner, `repo-${i}`, 10))); expect(results.filter(r => r.status === "fulfilled")).to.have.length(3); diff --git a/test/mongo-index-audit.test.js b/test/mongo-index-audit.test.js new file mode 100644 index 0000000..9cfde81 --- /dev/null +++ b/test/mongo-index-audit.test.js @@ -0,0 +1,123 @@ +const { expect } = require("chai"); +require("ts-node/register/transpile-only"); +const mongoose = require("mongoose"); +const config = require("../src/config").default; +const db = require("../src/server/database"); +const Repo = require("../src/core/model/anonymizedRepositories/anonymizedRepositories.model").default; +const File = require("../src/core/model/files/files.model").default; +const { assertTransformScopeActive } = require("../src/core/transformed-cache"); +const GitHubStream = require("../src/core/source/GitHubStream").default; +const { reconcileNeverExpiration } = require("../src/core/billing/expiration"); +const { retireFileIndexes } = require("../src/scripts/retire-file-indexes"); +const uri = process.env.PERFORMANCE_MONGO_URI; +const owner = new mongoose.Types.ObjectId(); +const replacement = { repoId: 1, treeGeneration: 1, path: 1, name: 1 }; + +(uri ? describe : describe.skip)("Mongo index audit regressions", function () { + this.timeout(60000); + const originalBilling = config.BILLING_ENABLED; + before(async () => { + if (!/\/perf_test_[A-Za-z0-9_-]+(?:\?|$)/.test(uri)) throw Error("Use an isolated perf_test_ database"); + await mongoose.connect(uri); db.isConnected = true; + await Promise.all([Repo, File].map(m => m.createIndexes())); + }); + after(async () => { + config.BILLING_ENABLED = originalBilling; db.isConnected = false; + await mongoose.connection.dropDatabase(); await mongoose.disconnect(); + }); + beforeEach(async () => { await Repo.deleteMany({}); await File.deleteMany({}); }); + + it("uses the case-insensitive index for cache guards without weakening lifecycle or revision checks", async () => { + const when = new Date("2026-01-01"); + const repo = await Repo.create({ repoId: "Mixed-Case", owner, status: "ready", treeGeneration: "generation", + anonymizeDate: when, contentCacheRevision: "revision" }); + const options = { repoId: "mixed-case", cacheGeneration: `generation:${when.toISOString()}`, cacheRevision: "revision" }; + const find = Repo.findOne; let query; + Repo.findOne = function (...args) { query = find.apply(this, args); return query; }; + try { + await assertTransformScopeActive(options); + const plan = await query.clone().explain("executionStats"); + expect(plan.executionStats.totalDocsExamined).to.equal(1); + expect(plan.executionStats.totalKeysExamined).to.equal(1); + } finally { Repo.findOne = find; } + for (const patch of [{ cacheGeneration: "stale" }, { cacheRevision: "stale" }]) { + const error = await assertTransformScopeActive({ ...options, ...patch }).catch(e => e); + expect(error.message).to.equal("repository_changed"); + } + await Repo.updateOne({ _id: repo._id }, { $set: { status: "archived" } }); + const error = await assertTransformScopeActive(options).catch(e => e); + expect(error.message).to.equal("repository_changed"); + }); + + it("keeps raw-cache generation guards indexed and rejects a stale snapshot before upstream work", async () => { + await Repo.create({ repoId: "Mixed-Case", owner, status: "ready", treeGeneration: "new" }); + const find = Repo.findOne; let query; + Repo.findOne = function (...args) { query = find.apply(this, args); return query; }; + try { + const source = new GitHubStream({ cacheGeneration: "old:", getToken: () => { throw Error("unexpected upstream"); } }); + const error = await source.getFileContentCache("README.md", "mixed-case", () => ({ sha: "a".repeat(40) }), true).catch(e => e); + expect(error.message).to.equal("repository_changed"); + const plan = await query.clone().explain("executionStats"); + expect(plan.executionStats.totalDocsExamined).to.equal(1); + expect(plan.executionStats.totalKeysExamined).to.equal(1); + } finally { Repo.findOne = find; } + }); + + it("uses the partial billing index and excludes legacy never-expiring repositories", async () => { + config.BILLING_ENABLED = true; + await Repo.insertMany(Array.from({ length: 1000 }, (_, i) => ({ repoId: `legacy-${i}`, owner, + status: "ready", options: { expirationMode: "never" } }))); + await Repo.insertMany(Array.from({ length: 11 }, (_, i) => ({ repoId: `paid-${i}`, owner, + status: "ready", options: { expirationMode: "never" }, billingNeverExpiresUntil: new Date("2099-01-01") }))); + const find = Repo.find; let query; + Repo.find = function (...args) { query = find.apply(this, args); return query; }; + try { + await reconcileNeverExpiration(new Date("2026-01-01"), async () => { throw Error("unexpected reconciliation"); }); + const plan = await query.clone().explain("executionStats"); + expect(plan.executionStats.nReturned).to.equal(11); + expect(plan.executionStats.totalDocsExamined).to.equal(11); + expect(JSON.stringify(plan.queryPlanner.winningPlan)).to.include("billing_never_expiration"); + } finally { Repo.find = find; } + expect(await Repo.countDocuments({ billingNeverExpiresUntil: { $exists: false }, "options.expirationMode": "never" })).to.equal(1000); + }); + + it("hides, restores and only then drops the audited indexes while preserving file reads", async () => { + const coll = File.collection; + for (const [key, name] of [[{ path: 1 }, "path_1"], [{ path: 1, repoId: 1 }, "path_1_repoId_1"], [{ repoId: 1, path: 1 }, "repoIDAndPath"]]) { + await coll.createIndex(key, { name }); + } + await File.insertMany([{ repoId: "fixture", treeGeneration: "active", path: "src", name: "one", size: 1 }, + { repoId: "fixture", path: "legacy", name: "two", size: 2 }]); + const database = mongoose.connection.db; + const rejected = await retireFileIndexes(database, "drop").catch(e => e); + expect(rejected.message).to.include("Hide all"); + await retireFileIndexes(database, "hide"); + expect((await coll.indexes()).find(i => i.name === "path_1").hidden).to.equal(true); + for (const filter of [{ repoId: "fixture", treeGeneration: "active", path: "src" }, + { repoId: "fixture", treeGeneration: { $exists: false }, path: "legacy" }]) { + expect(await File.countDocuments(filter)).to.equal(1); + } + await retireFileIndexes(database, "restore"); + expect((await coll.indexes()).find(i => i.name === "path_1").hidden).not.to.equal(true); + await retireFileIndexes(database, "hide"); await retireFileIndexes(database, "drop"); + const indexes = await coll.indexes(); + expect(indexes.some(i => i.name === "path_1")).to.equal(false); + expect(indexes.some(i => JSON.stringify(i.key) === JSON.stringify(replacement))).to.equal(true); + }); + + it("refuses hiding when the replacement index is absent", async () => { + await File.collection.dropIndex("repoId_1_treeGeneration_1_path_1_name_1"); + try { + const error = await retireFileIndexes(mongoose.connection.db, "hide").catch(e => e); + expect(error.message).to.include("replacement index must be ready"); + } finally { await File.collection.createIndex(replacement); } + }); + + it("refuses retirement of an unexpectedly unique candidate", async () => { + await File.collection.createIndex({ path: 1 }, { name: "path_1", unique: true }); + try { + const error = await retireFileIndexes(mongoose.connection.db, "hide").catch(e => e); + expect(error.message).to.include("Unexpected definition"); + } finally { await File.collection.dropIndex("path_1"); } + }); +});