diff --git a/docs/mongo-index-retirement.md b/docs/mongo-index-retirement.md new file mode 100644 index 0000000..e7ff8c6 --- /dev/null +++ b/docs/mongo-index-retirement.md @@ -0,0 +1,40 @@ +# Retire redundant file indexes + +The November 11 audit found no query use of `path_1`, `path_1_repoId_1`, or +`repoIDAndPath` since the Mongo restart on November 8. Together they occupied +3.17 GiB. This observation is a retirement candidate list, not proof that every +infrequent workload is covered. + +Deploy the updated application first so Mongoose no longer recreates the old +path indexes. The existing additive index migration does not drop indexes. +Then run these commands in the application environment: + +```sh +node build/scripts/retire-file-indexes.js status +node build/scripts/retire-file-indexes.js hide +``` + +Observe at least one full traffic/maintenance cycle, including ZIP reads, +legacy snapshots, path searches, tree cleanup and refreshes. Compare slow-query +plans and latency with the pre-hide baseline. Hidden indexes still consume +storage and incur write maintenance; hiding alone does not reclaim space. + +If queries regress, restore visibility immediately: + +```sh +node build/scripts/retire-file-indexes.js restore +``` + +Only after the observation period is satisfactory, reclaim the storage: + +```sh +node build/scripts/retire-file-indexes.js drop +``` + +The script accepts only the three audited index definitions, rejects unique, +TTL, partial, sparse or collated variants, requires the generation/path index +to be visible and ready, and refuses to drop candidates that are not hidden. +It never removes the unique or TTL indexes used by other collections. A dropped +index cannot be restored with `restore`; it must be rebuilt from its original +key definition. Keep the pre-hide baseline and observation results with the +operator's deployment record. diff --git a/src/core/Repository.ts b/src/core/Repository.ts index 8835774..e7aa8a5 100644 --- a/src/core/Repository.ts +++ b/src/core/Repository.ts @@ -1,3 +1,4 @@ +import { REPOSITORY_ID_COLLATION } from "./repository-id"; import { allowance, reserveRepository } from "./billing/quotas"; import { FILE_TREE_BATCH_SIZE, FILE_LISTING_LIMIT, deleteRowsInBatches } from "./file-tree"; import { SourceBase, sourceStorageSize } from "./source/Source"; @@ -422,7 +423,7 @@ export default class Repository { /** Indexed row markers also discover inserts that finish after a build lease is retired. */ static async cleanupStagedFileTree(repoId: string, generation: string, now = new Date()) { - let current = await AnonymizedRepositoryModel.findOne({ repoId }).exec(); + let current = await AnonymizedRepositoryModel.findOne({ repoId }, null, { collation: REPOSITORY_ID_COLLATION }).exec(); if (current && current.treeGeneration !== generation && current.stagedFileTrees?.some(stage => stage.generation === generation && stage.until > now)) return; if (current) { diff --git a/src/core/billing/quotas.ts b/src/core/billing/quotas.ts index c483c2f..8cb22dd 100644 --- a/src/core/billing/quotas.ts +++ b/src/core/billing/quotas.ts @@ -1,3 +1,4 @@ +import { REPOSITORY_ID_COLLATION } from "../repository-id"; import { randomUUID } from "crypto"; import ConferenceModel from "../model/conference/conferences.model"; import config from "../../config"; @@ -30,7 +31,8 @@ export async function reserveRepository(owner: string, repoId: string, bytes: nu // count toward the owner's capacity when creating new work. if (!await BillingQuota.exists({ _id: limits.scope })) { const existing = await RepositoryModel.find(limits.scope.startsWith("user:") ? { owner, status: { $nin: ["removed", "expired"] } } - : limits.scope.startsWith("conference:") ? { conference, status: { $nin: ["removed", "expired"] } } : { repoId, status: { $nin: ["removed", "expired"] } }) + : limits.scope.startsWith("conference:") ? { conference, status: { $nin: ["removed", "expired"] } } : { repoId, status: { $nin: ["removed", "expired"] } }, null, + limits.scope.startsWith("repo:") ? { collation: REPOSITORY_ID_COLLATION } : {}) .select("repoId size owner conference").lean(); const funded = []; for (const repository of existing) { @@ -42,7 +44,7 @@ export async function reserveRepository(owner: string, repoId: string, bytes: nu // Only confirmed terminal repositories release capacity. Unknown reservations // remain until reconciled by an operator, so a crash cannot oversell capacity. const ledger = await BillingQuota.findById(limits.scope).lean(); - const terminal = await RepositoryModel.find({ repoId: { $in: ledger?.allocations.map(a => a.repoId) || [] }, status: "removed" }).select("repoId").lean(); + const terminal = await RepositoryModel.find({ repoId: { $in: ledger?.allocations.map(a => a.repoId) || [] }, status: "removed" }, null, { collation: REPOSITORY_ID_COLLATION }).select("repoId").lean(); if (terminal.length) await BillingQuota.updateOne({ _id: limits.scope }, { $pull: { allocations: { $or: terminal.map(r => ({ repoId: r.repoId, reservationId: ledger?.allocations.find(a => a.repoId === r.repoId)?.reservationId || { $exists: false }, })) } } }); @@ -81,7 +83,7 @@ export async function assertConferenceAdmission(owner: string, repoId: string, c export async function reserveFileExpansion(repoId: string, key: string, extraBytes: number) { if (!config.BILLING_ENABLED) return; if (!Number.isSafeInteger(extraBytes) || extraBytes < 0) throw billingError("billing_invalid_repository_size"); - const repo = await RepositoryModel.findOne({ repoId }).select("owner conference billingManaged").lean(); + const repo = await RepositoryModel.findOne({ repoId }, null, { collation: REPOSITORY_ID_COLLATION }).select("owner conference billingManaged").lean(); if (!repo?.billingManaged) return; await reserveRepository(String(repo.owner), repoId, 0, repo.conference, { key, bytes: extraBytes }); } diff --git a/src/core/folder-counts.ts b/src/core/folder-counts.ts index fd5d4c0..ee62754 100644 --- a/src/core/folder-counts.ts +++ b/src/core/folder-counts.ts @@ -22,7 +22,7 @@ export async function folderCounts(repo: Repository, requestedPath: unknown): Pr const prefix = originalPath ? originalPath + "/" : ""; const escapedPrefix = prefix.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); const childDepth = originalPath ? originalPath.split("/").length : 0; - const rows = await FileModel.aggregate<{ _id: string; count: number }>([ + const aggregate = () => FileModel.aggregate<{ _id: string; count: number }>([ { $match: { repoId: repo.repoId, treeGeneration: repo.model.treeGeneration || { $exists: false }, @@ -35,7 +35,19 @@ export async function folderCounts(repo: Repository, requestedPath: unknown): Pr { $arrayElemAt: [{ $split: ["$path", "/"] }, childDepth] }, ] } : { $arrayElemAt: [{ $split: ["$path", "/"] }, 0] }, count: { $sum: 1 } } }, { $limit: FILE_LISTING_LIMIT + 1 }, - ]).option({ maxTimeMS: 2000, allowDiskUse: true }).exec(); + ]).option({ maxTimeMS: 2000, allowDiskUse: true }); + let rows: { _id: string; count: number }[]; + try { + // The short planner trial can favor the size-first index despite its + // document fetches. Legacy snapshots still use the default planner. + rows = await aggregate().option(repo.model.treeGeneration ? { hint: "file_counts_covering" } : {}).exec(); + } catch (error) { + // During an additive rollout the index may not be ready yet. Retry only + // the missing-hint planning error, never timeouts or execution failures. + if (!repo.model.treeGeneration || !(error instanceof Error) || !("code" in error) || error.code !== 2 || + !/hint provided does not correspond to an existing index/.test(error.message || "")) throw error; + rows = await aggregate().exec(); + } if (rows.length > FILE_LISTING_LIMIT) throw new AnonymousError("file_listing_too_large", { httpStatus: 413 }); const counts: Record = Object.create(null); const compiled = compileTerms(repo.options.terms || []); diff --git a/src/core/model/anonymizedRepositories/anonymizedRepositories.schema.ts b/src/core/model/anonymizedRepositories/anonymizedRepositories.schema.ts index c552c00..cb17663 100644 --- a/src/core/model/anonymizedRepositories/anonymizedRepositories.schema.ts +++ b/src/core/model/anonymizedRepositories/anonymizedRepositories.schema.ts @@ -110,6 +110,10 @@ const AnonymizedRepositorySchema = new Schema({ }, }); +AnonymizedRepositorySchema.index({ billingNeverExpiresUntil: 1, status: 1 }, { + name: "billing_never_expiration", + partialFilterExpression: { "options.expirationMode": "never", billingNeverExpiresUntil: { $exists: true } }, +}); AnonymizedRepositorySchema.index({ "source.repositoryName": 1 }); AnonymizedRepositorySchema.index({ "coauthors.githubId": 1 }); AnonymizedRepositorySchema.index({ retiredTreeGenerations: 1 }, { sparse: true }); diff --git a/src/core/model/files/files.schema.ts b/src/core/model/files/files.schema.ts index 2c2a01b..9979e52 100644 --- a/src/core/model/files/files.schema.ts +++ b/src/core/model/files/files.schema.ts @@ -4,7 +4,7 @@ export const UpstreamFileSchema = new Schema({ organization: String, repoName: S const FileSchema = new Schema({ name: { type: String, index: true }, - path: { type: String, index: true }, + path: { type: String }, repoId: { type: String, index: true }, treeGeneration: String, treeStaged: Boolean, @@ -19,7 +19,6 @@ const FileSchema = new Schema({ }, }); -FileSchema.index({ path: 1, repoId: 1 }); FileSchema.index({ metadataPending: 1, repoId: 1, treeGeneration: 1 }, { partialFilterExpression: { metadataPending: true } }); FileSchema.index({ treeStaged: 1, repoId: 1, treeGeneration: 1 }, diff --git a/src/core/repository-id.ts b/src/core/repository-id.ts new file mode 100644 index 0000000..233125b --- /dev/null +++ b/src/core/repository-id.ts @@ -0,0 +1,2 @@ +/** Repository URLs and their unique Mongo index use case-insensitive IDs. */ +export const REPOSITORY_ID_COLLATION = { locale: "en", strength: 2 }; diff --git a/src/core/source/GitHubStream.ts b/src/core/source/GitHubStream.ts index d7cc447..303e276 100644 --- a/src/core/source/GitHubStream.ts +++ b/src/core/source/GitHubStream.ts @@ -1,3 +1,4 @@ +import { REPOSITORY_ID_COLLATION } from "../repository-id"; import { reserveFileExpansion } from "../billing/quotas"; import { withOwnerAccessRecovery } from "../owner-notifications"; import { ExpiringMap } from "../expiring-map"; @@ -362,7 +363,7 @@ export default class GitHubStream extends GitHubBase { // Retirement markers live inside the storage root and disappear when // expiration removes it. The persisted lifecycle also fences late fills. if (isConnected) { - const current = await AnonymizedRepositoryModel.findOne({ repoId }) + const current = await AnonymizedRepositoryModel.findOne({ repoId }, null, { collation: REPOSITORY_ID_COLLATION }) .select("status treeGeneration anonymizeDate contentCacheRevision").lean().exec(); if (!current || !current.status || [RepositoryStatus.EXPIRING, RepositoryStatus.EXPIRED, RepositoryStatus.REMOVING, RepositoryStatus.REMOVED, RepositoryStatus.ARCHIVED].includes(current.status) @@ -641,7 +642,7 @@ export default class GitHubStream extends GitHubBase { private async confirmOwnerAccess(repoId: string, work: () => Promise, confirmed?: (result: T) => boolean): Promise { let repository; try { - repository = isConnected ? await AnonymizedRepositoryModel.findOne({ repoId }) + repository = isConnected ? await AnonymizedRepositoryModel.findOne({ repoId }, null, { collation: REPOSITORY_ID_COLLATION }) .select("owner source githubAccess").lean().exec() : null; } catch { // Email recovery bookkeeping must not prevent the upstream read. diff --git a/src/core/transformed-cache.ts b/src/core/transformed-cache.ts index 8f78469..473fa72 100644 --- a/src/core/transformed-cache.ts +++ b/src/core/transformed-cache.ts @@ -1,3 +1,4 @@ +import { REPOSITORY_ID_COLLATION } from "./repository-id"; import { observeCache } from "./cache-observability"; import { promises as fs } from "fs"; import { join } from "path"; @@ -18,7 +19,7 @@ interface CacheScope { repoId: string; generation?: string; revision?: string } async function scopeActive(scope?: CacheScope) { if (!scope || !isConnected) return true; - const current = await RepositoryModel.findOne({ repoId: scope.repoId }) + const current = await RepositoryModel.findOne({ repoId: scope.repoId }, null, { collation: REPOSITORY_ID_COLLATION }) .select("status treeGeneration anonymizeDate contentCacheRevision").lean().exec(); return !!current?.status && ![RepositoryStatus.EXPIRING, RepositoryStatus.EXPIRED, RepositoryStatus.REMOVING, RepositoryStatus.REMOVED, RepositoryStatus.ARCHIVED].includes(current.status) diff --git a/src/scripts/create-performance-indexes.ts b/src/scripts/create-performance-indexes.ts index 3a577b2..8ad5c3f 100644 --- a/src/scripts/create-performance-indexes.ts +++ b/src/scripts/create-performance-indexes.ts @@ -11,6 +11,10 @@ import Name from "../core/model/dashboard-name"; async function main() { const uri = config.MONGODB_URI || `mongodb://${config.DB_USERNAME}:${config.DB_PASSWORD}@${config.DB_HOSTNAME}:27017/production`; await mongoose.connect(uri, { autoIndex: false, ...(config.MONGODB_URI ? {} : { authSource: "admin" }) }); + await Repository.collection.createIndex({ billingNeverExpiresUntil: 1, status: 1 }, { + name: "billing_never_expiration", + partialFilterExpression: { "options.expirationMode": "never", billingNeverExpiresUntil: { $exists: true } }, + }); // Additive only. Never use syncIndexes here, which can drop deployed indexes. await Repository.collection.createIndex({ "coauthors.githubId": 1 }); await File.collection.createIndex({ repoId: 1, treeGeneration: 1, path: 1, name: 1 }); diff --git a/src/scripts/retire-file-indexes.ts b/src/scripts/retire-file-indexes.ts new file mode 100644 index 0000000..fdb247f --- /dev/null +++ b/src/scripts/retire-file-indexes.ts @@ -0,0 +1,51 @@ +import "dotenv/config"; +import mongoose, { mongo } from "mongoose"; +import config from "../config"; + +const candidates = [ + { name: "path_1", key: { path: 1 } }, + { name: "path_1_repoId_1", key: { path: 1, repoId: 1 } }, + { name: "repoIDAndPath", key: { repoId: 1, path: 1 } }, +]; +const replacementKey = { repoId: 1, treeGeneration: 1, path: 1, name: 1 }; + +/** Operator-controlled retirement: hide first, observe traffic, then drop. */ +export async function retireFileIndexes(db: mongo.Db, action: string) { + if (!["status", "hide", "restore", "drop"].includes(action)) throw Error("Use status, hide, restore or drop"); + const collection = db.collection("files"); + const indexes = await collection.listIndexes().toArray(); + const selected = candidates.flatMap(candidate => { + const index = indexes.find(index => index.name === candidate.name); + if (!index) return []; + if (JSON.stringify(index.key) !== JSON.stringify(candidate.key) || index.unique || index.sparse || + index.partialFilterExpression || index.collation || index.expireAfterSeconds !== undefined) { + throw Error(`Unexpected definition for ${candidate.name}; refusing retirement`); + } + return [index]; + }); + if (action === "hide" || action === "drop") { + if (!indexes.some(index => JSON.stringify(index.key) === JSON.stringify(replacementKey) && !index.hidden && + !index.partialFilterExpression && !index.sparse && !index.collation && !index.buildUUID)) { + throw Error("The generation/path replacement index must be ready and visible"); + } + } + // Validate every candidate before changing any of them. + if (action === "drop" && selected.some(index => !index.hidden)) throw Error("Hide all candidate indexes and observe traffic before dropping them"); + for (const index of selected) { + if (action === "drop") await collection.dropIndex(index.name!); + else if (action === "hide" || action === "restore") { + await db.command({ collMod: "files", index: { name: index.name, hidden: action === "hide" } }); + } + } + return selected.map(index => ({ name: index.name, action, previouslyHidden: !!index.hidden })); +} + +async function main() { + const uri = config.MONGODB_URI || `mongodb://${encodeURIComponent(config.DB_USERNAME)}:${encodeURIComponent(config.DB_PASSWORD)}@${config.DB_HOSTNAME}:27017/production`; + await mongoose.connect(uri, { autoIndex: false, ...(config.MONGODB_URI ? {} : { authSource: "admin" }) }); + console.log(await retireFileIndexes(mongoose.connection.db!, process.argv[2] || "status")); +} +if (require.main === module) { + main().catch(error => { console.error(error instanceof Error ? error.message : "Index retirement failed"); process.exitCode = 1; }) + .finally(() => mongoose.disconnect()); +} diff --git a/src/server/routes/github-app.ts b/src/server/routes/github-app.ts index 4c0a623..4bbd40c 100644 --- a/src/server/routes/github-app.ts +++ b/src/server/routes/github-app.ts @@ -1,3 +1,4 @@ +import { REPOSITORY_ID_COLLATION } from "../../core/repository-id"; import { notificationEmail, normalizeEmail, resetOwnerAccessAlerts } from "../../core/owner-notifications"; import { saveRegistrationEmail } from "../../core/registration-email"; import * as express from "express"; @@ -230,7 +231,7 @@ router.post("/connections/migrate", async (req, res) => { if (typeof id !== "string" || !["repository", "pull-request"].includes(type) || !["oauth", "github-app"].includes(connection)) throw appError("invalid_connection", 400); if (connection === "github-app" && (!config.GITHUB_APP_ENABLED || !config.GITHUB_APP_NEW_CONNECTIONS)) throw appError("github_app_disabled", 503); const isRepo = type === "repository"; - const model = isRepo ? await RepositoryModel.findOne({ repoId: id, owner: user.id }) : await PullRequestModel.findOne({ pullRequestId: id, owner: user.id }); + const model = isRepo ? await RepositoryModel.findOne({ repoId: id, owner: user.id }, null, { collation: REPOSITORY_ID_COLLATION }) : await PullRequestModel.findOne({ pullRequestId: id, owner: user.id }); if (!model || ["removed", "archived"].includes(model.status || "")) throw appError("repo_not_found", 404); const stalePullRequest = !isRepo && model.status === "download" && (!model.statusDate || model.statusDate.getTime() <= Date.now() - 5 * 60 * 1000); diff --git a/src/server/schedule.ts b/src/server/schedule.ts index af1f04d..6f77f86 100644 --- a/src/server/schedule.ts +++ b/src/server/schedule.ts @@ -1,3 +1,4 @@ +import { REPOSITORY_ID_COLLATION } from "../core/repository-id"; import { reconcileNeverExpiration } from "../core/billing/expiration"; import { Model, Document } from "mongoose"; import * as schedule from "node-schedule"; @@ -185,7 +186,7 @@ export async function runRepositoryStatusCheck(now = new Date()) { for await (const file of pendingCursor) { batch.push((async () => { try { - const data = await AnonymizedRepositoryModel.findOne({ repoId: file.repoId }).exec(); + const data = await AnonymizedRepositoryModel.findOne({ repoId: file.repoId }, null, { collation: REPOSITORY_ID_COLLATION }).exec(); if (!data || data.treeGeneration !== file.treeGeneration || (data.status && [RepositoryStatus.EXPIRING, RepositoryStatus.EXPIRED, RepositoryStatus.REMOVING, RepositoryStatus.REMOVED].includes(data.status))) { await FileModel.deleteOne({ _id: file._id, metadataPending: true }).exec(); diff --git a/test/billing-integration.test.js b/test/billing-integration.test.js index 96d6253..1bbae59 100644 --- a/test/billing-integration.test.js +++ b/test/billing-integration.test.js @@ -30,6 +30,23 @@ async function rejects(promise, message) { await BillingAccount.create({ _id: owner, customerId: "cus_owner" }); }); after(async () => { Object.assign(config, original); await mongoose.disconnect(); }); + it("indexes terminal quota lookups by repository ID rather than scanning removed repositories", async () => { + await Repo.insertMany(Array.from({ length: 1000 }, (_, i) => ({ repoId: `removed-${i}`, owner, status: "removed" }))); + await BillingQuota.create({ _id: `user:${owner}`, allocations: [{ repoId: "removed-999", reservationId: "old", bytes: 90 }] }); + const find = Repo.find; let terminalQuery; + Repo.find = function (...args) { + const query = find.apply(this, args); + if (args[0]?.repoId?.$in) terminalQuery = query; + return query; + }; + try { + await reserveRepository(owner, "new", 100); + expect((await BillingQuota.findById(`user:${owner}`)).allocations.map(a => a.repoId)).to.deep.equal(["new"]); + const plan = await terminalQuery.clone().explain("executionStats"); + expect(plan.executionStats.totalDocsExamined).to.equal(1); + expect(plan.executionStats.totalKeysExamined).to.equal(1); + } finally { Repo.find = find; } + }); it("atomically admits only three concurrent repositories", async () => { const results = await Promise.allSettled(Array.from({ length: 12 }, (_, i) => reserveRepository(owner, `repo-${i}`, 10))); expect(results.filter(r => r.status === "fulfilled")).to.have.length(3); diff --git a/test/folder-counts.test.js b/test/folder-counts.test.js index 0a1d90d..52a212c 100644 --- a/test/folder-counts.test.js +++ b/test/folder-counts.test.js @@ -17,6 +17,22 @@ describe("folder count request validation", () => { expect(error.message).to.equal("invalid_path"); expect(error.httpStatus).to.equal(400); }); } + for (const error of [Object.assign(Error("operation exceeded time limit"), { code: 50 }), + Object.assign(Error("unrelated bad value"), { code: 2 })]) { + it(`does not retry ${error.message}`, async () => { + const original = File.aggregate; let calls = 0; + File.aggregate = () => { + calls++; + return { option() { return this; }, exec: async () => { throw error; } }; + }; + try { + const caught = await folderCounts({ repoId: "fixture", model: { treeGeneration: "active" } }, "") + .then(() => null, error => error); + expect(caught).to.equal(error); expect(calls).to.equal(1); + } finally { File.aggregate = original; } + }); + } + }); (uri ? describe : describe.skip)("scoped folder counts with MongoDB", function () { @@ -50,17 +66,31 @@ describe("folder count request validation", () => { await insert(Array.from({ length: 2000 }, (_, i) => ({ name: `file${i}`, path: `src/folder${i % 20}`, size: i % 2 }))); await insert([{ name: "excluded-directory", path: "" }, { name: "old", path: "old", size: 1, treeGeneration: "retired" }]); - let pipeline; + let pipeline, query; const aggregate = File.aggregate; - File.aggregate = function (stages, ...args) { pipeline = stages; return aggregate.call(this, stages, ...args); }; + File.aggregate = function (stages, ...args) { + pipeline = stages; query = aggregate.call(this, stages, ...args); return query; + }; try { expect(await folderCounts(repo, "")).to.deep.equal({ "": 2000, src: 2000 }); } finally { File.aggregate = aggregate; } - const explained = await File.aggregate(pipeline).option({ hint: "file_counts_covering" }).explain("executionStats"); + expect(query.options.hint).to.equal("file_counts_covering"); + const explained = await File.aggregate(pipeline).option(query.options).explain("executionStats"); const cursor = explained.stages?.find(stage => stage.$cursor)?.$cursor || explained; expect(cursor.executionStats.totalDocsExamined).to.equal(0); expect(cursor.executionStats.totalKeysExamined).to.be.at.least(2000); }); + it("falls back before the covering index exists and uses it after creation", async () => { + await insert([{ name: "one", path: "src", size: 1 }]); + await File.collection.dropIndex("file_counts_covering"); + try { + expect(await folderCounts(repo, "")).to.deep.equal({ "": 1, src: 1 }); + } finally { + await File.collection.createIndex({ repoId: 1, treeGeneration: 1, path: 1, size: 1 }, { name: "file_counts_covering" }); + } + expect(await folderCounts(repo, "")).to.deep.equal({ "": 1, src: 1 }); + }); + it("preserves counts for legacy repositories without a generation", async () => { repo.model.treeGeneration = undefined; await repo.model.save(); await File.collection.insertMany([ diff --git a/test/mongo-index-audit.test.js b/test/mongo-index-audit.test.js new file mode 100644 index 0000000..9cfde81 --- /dev/null +++ b/test/mongo-index-audit.test.js @@ -0,0 +1,123 @@ +const { expect } = require("chai"); +require("ts-node/register/transpile-only"); +const mongoose = require("mongoose"); +const config = require("../src/config").default; +const db = require("../src/server/database"); +const Repo = require("../src/core/model/anonymizedRepositories/anonymizedRepositories.model").default; +const File = require("../src/core/model/files/files.model").default; +const { assertTransformScopeActive } = require("../src/core/transformed-cache"); +const GitHubStream = require("../src/core/source/GitHubStream").default; +const { reconcileNeverExpiration } = require("../src/core/billing/expiration"); +const { retireFileIndexes } = require("../src/scripts/retire-file-indexes"); +const uri = process.env.PERFORMANCE_MONGO_URI; +const owner = new mongoose.Types.ObjectId(); +const replacement = { repoId: 1, treeGeneration: 1, path: 1, name: 1 }; + +(uri ? describe : describe.skip)("Mongo index audit regressions", function () { + this.timeout(60000); + const originalBilling = config.BILLING_ENABLED; + before(async () => { + if (!/\/perf_test_[A-Za-z0-9_-]+(?:\?|$)/.test(uri)) throw Error("Use an isolated perf_test_ database"); + await mongoose.connect(uri); db.isConnected = true; + await Promise.all([Repo, File].map(m => m.createIndexes())); + }); + after(async () => { + config.BILLING_ENABLED = originalBilling; db.isConnected = false; + await mongoose.connection.dropDatabase(); await mongoose.disconnect(); + }); + beforeEach(async () => { await Repo.deleteMany({}); await File.deleteMany({}); }); + + it("uses the case-insensitive index for cache guards without weakening lifecycle or revision checks", async () => { + const when = new Date("2026-01-01"); + const repo = await Repo.create({ repoId: "Mixed-Case", owner, status: "ready", treeGeneration: "generation", + anonymizeDate: when, contentCacheRevision: "revision" }); + const options = { repoId: "mixed-case", cacheGeneration: `generation:${when.toISOString()}`, cacheRevision: "revision" }; + const find = Repo.findOne; let query; + Repo.findOne = function (...args) { query = find.apply(this, args); return query; }; + try { + await assertTransformScopeActive(options); + const plan = await query.clone().explain("executionStats"); + expect(plan.executionStats.totalDocsExamined).to.equal(1); + expect(plan.executionStats.totalKeysExamined).to.equal(1); + } finally { Repo.findOne = find; } + for (const patch of [{ cacheGeneration: "stale" }, { cacheRevision: "stale" }]) { + const error = await assertTransformScopeActive({ ...options, ...patch }).catch(e => e); + expect(error.message).to.equal("repository_changed"); + } + await Repo.updateOne({ _id: repo._id }, { $set: { status: "archived" } }); + const error = await assertTransformScopeActive(options).catch(e => e); + expect(error.message).to.equal("repository_changed"); + }); + + it("keeps raw-cache generation guards indexed and rejects a stale snapshot before upstream work", async () => { + await Repo.create({ repoId: "Mixed-Case", owner, status: "ready", treeGeneration: "new" }); + const find = Repo.findOne; let query; + Repo.findOne = function (...args) { query = find.apply(this, args); return query; }; + try { + const source = new GitHubStream({ cacheGeneration: "old:", getToken: () => { throw Error("unexpected upstream"); } }); + const error = await source.getFileContentCache("README.md", "mixed-case", () => ({ sha: "a".repeat(40) }), true).catch(e => e); + expect(error.message).to.equal("repository_changed"); + const plan = await query.clone().explain("executionStats"); + expect(plan.executionStats.totalDocsExamined).to.equal(1); + expect(plan.executionStats.totalKeysExamined).to.equal(1); + } finally { Repo.findOne = find; } + }); + + it("uses the partial billing index and excludes legacy never-expiring repositories", async () => { + config.BILLING_ENABLED = true; + await Repo.insertMany(Array.from({ length: 1000 }, (_, i) => ({ repoId: `legacy-${i}`, owner, + status: "ready", options: { expirationMode: "never" } }))); + await Repo.insertMany(Array.from({ length: 11 }, (_, i) => ({ repoId: `paid-${i}`, owner, + status: "ready", options: { expirationMode: "never" }, billingNeverExpiresUntil: new Date("2099-01-01") }))); + const find = Repo.find; let query; + Repo.find = function (...args) { query = find.apply(this, args); return query; }; + try { + await reconcileNeverExpiration(new Date("2026-01-01"), async () => { throw Error("unexpected reconciliation"); }); + const plan = await query.clone().explain("executionStats"); + expect(plan.executionStats.nReturned).to.equal(11); + expect(plan.executionStats.totalDocsExamined).to.equal(11); + expect(JSON.stringify(plan.queryPlanner.winningPlan)).to.include("billing_never_expiration"); + } finally { Repo.find = find; } + expect(await Repo.countDocuments({ billingNeverExpiresUntil: { $exists: false }, "options.expirationMode": "never" })).to.equal(1000); + }); + + it("hides, restores and only then drops the audited indexes while preserving file reads", async () => { + const coll = File.collection; + for (const [key, name] of [[{ path: 1 }, "path_1"], [{ path: 1, repoId: 1 }, "path_1_repoId_1"], [{ repoId: 1, path: 1 }, "repoIDAndPath"]]) { + await coll.createIndex(key, { name }); + } + await File.insertMany([{ repoId: "fixture", treeGeneration: "active", path: "src", name: "one", size: 1 }, + { repoId: "fixture", path: "legacy", name: "two", size: 2 }]); + const database = mongoose.connection.db; + const rejected = await retireFileIndexes(database, "drop").catch(e => e); + expect(rejected.message).to.include("Hide all"); + await retireFileIndexes(database, "hide"); + expect((await coll.indexes()).find(i => i.name === "path_1").hidden).to.equal(true); + for (const filter of [{ repoId: "fixture", treeGeneration: "active", path: "src" }, + { repoId: "fixture", treeGeneration: { $exists: false }, path: "legacy" }]) { + expect(await File.countDocuments(filter)).to.equal(1); + } + await retireFileIndexes(database, "restore"); + expect((await coll.indexes()).find(i => i.name === "path_1").hidden).not.to.equal(true); + await retireFileIndexes(database, "hide"); await retireFileIndexes(database, "drop"); + const indexes = await coll.indexes(); + expect(indexes.some(i => i.name === "path_1")).to.equal(false); + expect(indexes.some(i => JSON.stringify(i.key) === JSON.stringify(replacement))).to.equal(true); + }); + + it("refuses hiding when the replacement index is absent", async () => { + await File.collection.dropIndex("repoId_1_treeGeneration_1_path_1_name_1"); + try { + const error = await retireFileIndexes(mongoose.connection.db, "hide").catch(e => e); + expect(error.message).to.include("replacement index must be ready"); + } finally { await File.collection.createIndex(replacement); } + }); + + it("refuses retirement of an unexpectedly unique candidate", async () => { + await File.collection.createIndex({ path: 1 }, { name: "path_1", unique: true }); + try { + const error = await retireFileIndexes(mongoose.connection.db, "hide").catch(e => e); + expect(error.message).to.include("Unexpected definition"); + } finally { await File.collection.dropIndex("path_1"); } + }); +});