perf: index repository lookups and billing maintenance

This commit is contained in:
Codex committed 2026-11-11 10:18:20 +00:00
1 parent 6a230f73b9
commit 3999a71583
14 files changed
+258 -11

No files matched your search

+2 -1
View File
@@ -1,3 +1,4 @@
import { REPOSITORY_ID_COLLATION } from "./repository-id";
import { allowance, reserveRepository } from "./billing/quotas";
import { FILE_TREE_BATCH_SIZE, FILE_LISTING_LIMIT, deleteRowsInBatches } from "./file-tree";
import { SourceBase, sourceStorageSize } from "./source/Source";
@@ -422,7 +423,7 @@ export default class Repository {
/** Indexed row markers also discover inserts that finish after a build lease is retired. */
static async cleanupStagedFileTree(repoId: string, generation: string, now = new Date()) {
let current = await AnonymizedRepositoryModel.findOne({ repoId }).exec();
let current = await AnonymizedRepositoryModel.findOne({ repoId }, null, { collation: REPOSITORY_ID_COLLATION }).exec();
if (current && current.treeGeneration !== generation &&
current.stagedFileTrees?.some(stage => stage.generation === generation && stage.until > now)) return;
if (current) {
+5 -3
View File
@@ -1,3 +1,4 @@
import { REPOSITORY_ID_COLLATION } from "../repository-id";
import { randomUUID } from "crypto";
import ConferenceModel from "../model/conference/conferences.model";
import config from "../../config";
@@ -30,7 +31,8 @@ export async function reserveRepository(owner: string, repoId: string, bytes: nu
// count toward the owner's capacity when creating new work.
if (!await BillingQuota.exists({ _id: limits.scope })) {
const existing = await RepositoryModel.find(limits.scope.startsWith("user:") ? { owner, status: { $nin: ["removed", "expired"] } }
: limits.scope.startsWith("conference:") ? { conference, status: { $nin: ["removed", "expired"] } } : { repoId, status: { $nin: ["removed", "expired"] } })
: limits.scope.startsWith("conference:") ? { conference, status: { $nin: ["removed", "expired"] } } : { repoId, status: { $nin: ["removed", "expired"] } }, null,
limits.scope.startsWith("repo:") ? { collation: REPOSITORY_ID_COLLATION } : {})
.select("repoId size owner conference").lean();
const funded = [];
for (const repository of existing) {
@@ -42,7 +44,7 @@ export async function reserveRepository(owner: string, repoId: string, bytes: nu
// Only confirmed terminal repositories release capacity. Unknown reservations
// remain until reconciled by an operator, so a crash cannot oversell capacity.
const ledger = await BillingQuota.findById(limits.scope).lean();
const terminal = await RepositoryModel.find({ repoId: { $in: ledger?.allocations.map(a => a.repoId) || [] }, status: "removed" }).select("repoId").lean();
const terminal = await RepositoryModel.find({ repoId: { $in: ledger?.allocations.map(a => a.repoId) || [] }, status: "removed" }, null, { collation: REPOSITORY_ID_COLLATION }).select("repoId").lean();
if (terminal.length) await BillingQuota.updateOne({ _id: limits.scope }, { $pull: { allocations: { $or: terminal.map(r => ({
repoId: r.repoId, reservationId: ledger?.allocations.find(a => a.repoId === r.repoId)?.reservationId || { $exists: false },
})) } } });
@@ -81,7 +83,7 @@ export async function assertConferenceAdmission(owner: string, repoId: string, c
export async function reserveFileExpansion(repoId: string, key: string, extraBytes: number) {
if (!config.BILLING_ENABLED) return;
if (!Number.isSafeInteger(extraBytes) || extraBytes < 0) throw billingError("billing_invalid_repository_size");
const repo = await RepositoryModel.findOne({ repoId }).select("owner conference billingManaged").lean();
const repo = await RepositoryModel.findOne({ repoId }, null, { collation: REPOSITORY_ID_COLLATION }).select("owner conference billingManaged").lean();
if (!repo?.billingManaged) return;
await reserveRepository(String(repo.owner), repoId, 0, repo.conference, { key, bytes: extraBytes });
}
@@ -110,6 +110,10 @@ const AnonymizedRepositorySchema = new Schema({
},
});
AnonymizedRepositorySchema.index({ billingNeverExpiresUntil: 1, status: 1 }, {
name: "billing_never_expiration",
partialFilterExpression: { "options.expirationMode": "never", billingNeverExpiresUntil: { $exists: true } },
});
AnonymizedRepositorySchema.index({ "source.repositoryName": 1 });
AnonymizedRepositorySchema.index({ "coauthors.githubId": 1 });
AnonymizedRepositorySchema.index({ retiredTreeGenerations: 1 }, { sparse: true });
+1 -2
View File
@@ -4,7 +4,7 @@ export const UpstreamFileSchema = new Schema({ organization: String, repoName: S
const FileSchema = new Schema({
name: { type: String, index: true },
path: { type: String, index: true },
path: { type: String },
repoId: { type: String, index: true },
treeGeneration: String,
treeStaged: Boolean,
@@ -19,7 +19,6 @@ const FileSchema = new Schema({
},
});
FileSchema.index({ path: 1, repoId: 1 });
FileSchema.index({ metadataPending: 1, repoId: 1, treeGeneration: 1 },
{ partialFilterExpression: { metadataPending: true } });
FileSchema.index({ treeStaged: 1, repoId: 1, treeGeneration: 1 },
+2
View File
@@ -0,0 +1,2 @@
/** Repository URLs and their unique Mongo index use case-insensitive IDs. */
export const REPOSITORY_ID_COLLATION = { locale: "en", strength: 2 };
+3 -2
View File
@@ -1,3 +1,4 @@
import { REPOSITORY_ID_COLLATION } from "../repository-id";
import { reserveFileExpansion } from "../billing/quotas";
import { withOwnerAccessRecovery } from "../owner-notifications";
import { ExpiringMap } from "../expiring-map";
@@ -362,7 +363,7 @@ export default class GitHubStream extends GitHubBase {
// Retirement markers live inside the storage root and disappear when
// expiration removes it. The persisted lifecycle also fences late fills.
if (isConnected) {
const current = await AnonymizedRepositoryModel.findOne({ repoId })
const current = await AnonymizedRepositoryModel.findOne({ repoId }, null, { collation: REPOSITORY_ID_COLLATION })
.select("status treeGeneration anonymizeDate contentCacheRevision").lean().exec();
if (!current || !current.status || [RepositoryStatus.EXPIRING, RepositoryStatus.EXPIRED,
RepositoryStatus.REMOVING, RepositoryStatus.REMOVED, RepositoryStatus.ARCHIVED].includes(current.status)
@@ -641,7 +642,7 @@ export default class GitHubStream extends GitHubBase {
private async confirmOwnerAccess<T>(repoId: string, work: () => Promise<T>, confirmed?: (result: T) => boolean): Promise<T> {
let repository;
try {
repository = isConnected ? await AnonymizedRepositoryModel.findOne({ repoId })
repository = isConnected ? await AnonymizedRepositoryModel.findOne({ repoId }, null, { collation: REPOSITORY_ID_COLLATION })
.select("owner source githubAccess").lean().exec() : null;
} catch {
// Email recovery bookkeeping must not prevent the upstream read.
+2 -1
View File
@@ -1,3 +1,4 @@
import { REPOSITORY_ID_COLLATION } from "./repository-id";
import { observeCache } from "./cache-observability";
import { promises as fs } from "fs";
import { join } from "path";
@@ -18,7 +19,7 @@ interface CacheScope { repoId: string; generation?: string; revision?: string }
async function scopeActive(scope?: CacheScope) {
if (!scope || !isConnected) return true;
const current = await RepositoryModel.findOne({ repoId: scope.repoId })
const current = await RepositoryModel.findOne({ repoId: scope.repoId }, null, { collation: REPOSITORY_ID_COLLATION })
.select("status treeGeneration anonymizeDate contentCacheRevision").lean().exec();
return !!current?.status && ![RepositoryStatus.EXPIRING, RepositoryStatus.EXPIRED,
RepositoryStatus.REMOVING, RepositoryStatus.REMOVED, RepositoryStatus.ARCHIVED].includes(current.status)