mirror of
https://github.com/tdurieux/anonymous_github.git
synced 2026-11-18 01:54:23 +01:00
perf: index repository lookups and billing maintenance
This commit is contained in:
1 parent
6a230f73b9
commit
3999a71583
14 files changed
+258
-11
No files matched your search
@@ -1,3 +1,4 @@
|
||||
import { REPOSITORY_ID_COLLATION } from "./repository-id";
|
||||
import { allowance, reserveRepository } from "./billing/quotas";
|
||||
import { FILE_TREE_BATCH_SIZE, FILE_LISTING_LIMIT, deleteRowsInBatches } from "./file-tree";
|
||||
import { SourceBase, sourceStorageSize } from "./source/Source";
|
||||
@@ -422,7 +423,7 @@ export default class Repository {
|
||||
|
||||
/** Indexed row markers also discover inserts that finish after a build lease is retired. */
|
||||
static async cleanupStagedFileTree(repoId: string, generation: string, now = new Date()) {
|
||||
let current = await AnonymizedRepositoryModel.findOne({ repoId }).exec();
|
||||
let current = await AnonymizedRepositoryModel.findOne({ repoId }, null, { collation: REPOSITORY_ID_COLLATION }).exec();
|
||||
if (current && current.treeGeneration !== generation &&
|
||||
current.stagedFileTrees?.some(stage => stage.generation === generation && stage.until > now)) return;
|
||||
if (current) {
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { REPOSITORY_ID_COLLATION } from "../repository-id";
|
||||
import { randomUUID } from "crypto";
|
||||
import ConferenceModel from "../model/conference/conferences.model";
|
||||
import config from "../../config";
|
||||
@@ -30,7 +31,8 @@ export async function reserveRepository(owner: string, repoId: string, bytes: nu
|
||||
// count toward the owner's capacity when creating new work.
|
||||
if (!await BillingQuota.exists({ _id: limits.scope })) {
|
||||
const existing = await RepositoryModel.find(limits.scope.startsWith("user:") ? { owner, status: { $nin: ["removed", "expired"] } }
|
||||
: limits.scope.startsWith("conference:") ? { conference, status: { $nin: ["removed", "expired"] } } : { repoId, status: { $nin: ["removed", "expired"] } })
|
||||
: limits.scope.startsWith("conference:") ? { conference, status: { $nin: ["removed", "expired"] } } : { repoId, status: { $nin: ["removed", "expired"] } }, null,
|
||||
limits.scope.startsWith("repo:") ? { collation: REPOSITORY_ID_COLLATION } : {})
|
||||
.select("repoId size owner conference").lean();
|
||||
const funded = [];
|
||||
for (const repository of existing) {
|
||||
@@ -42,7 +44,7 @@ export async function reserveRepository(owner: string, repoId: string, bytes: nu
|
||||
// Only confirmed terminal repositories release capacity. Unknown reservations
|
||||
// remain until reconciled by an operator, so a crash cannot oversell capacity.
|
||||
const ledger = await BillingQuota.findById(limits.scope).lean();
|
||||
const terminal = await RepositoryModel.find({ repoId: { $in: ledger?.allocations.map(a => a.repoId) || [] }, status: "removed" }).select("repoId").lean();
|
||||
const terminal = await RepositoryModel.find({ repoId: { $in: ledger?.allocations.map(a => a.repoId) || [] }, status: "removed" }, null, { collation: REPOSITORY_ID_COLLATION }).select("repoId").lean();
|
||||
if (terminal.length) await BillingQuota.updateOne({ _id: limits.scope }, { $pull: { allocations: { $or: terminal.map(r => ({
|
||||
repoId: r.repoId, reservationId: ledger?.allocations.find(a => a.repoId === r.repoId)?.reservationId || { $exists: false },
|
||||
})) } } });
|
||||
@@ -81,7 +83,7 @@ export async function assertConferenceAdmission(owner: string, repoId: string, c
|
||||
export async function reserveFileExpansion(repoId: string, key: string, extraBytes: number) {
|
||||
if (!config.BILLING_ENABLED) return;
|
||||
if (!Number.isSafeInteger(extraBytes) || extraBytes < 0) throw billingError("billing_invalid_repository_size");
|
||||
const repo = await RepositoryModel.findOne({ repoId }).select("owner conference billingManaged").lean();
|
||||
const repo = await RepositoryModel.findOne({ repoId }, null, { collation: REPOSITORY_ID_COLLATION }).select("owner conference billingManaged").lean();
|
||||
if (!repo?.billingManaged) return;
|
||||
await reserveRepository(String(repo.owner), repoId, 0, repo.conference, { key, bytes: extraBytes });
|
||||
}
|
||||
@@ -110,6 +110,10 @@ const AnonymizedRepositorySchema = new Schema({
|
||||
},
|
||||
});
|
||||
|
||||
AnonymizedRepositorySchema.index({ billingNeverExpiresUntil: 1, status: 1 }, {
|
||||
name: "billing_never_expiration",
|
||||
partialFilterExpression: { "options.expirationMode": "never", billingNeverExpiresUntil: { $exists: true } },
|
||||
});
|
||||
AnonymizedRepositorySchema.index({ "source.repositoryName": 1 });
|
||||
AnonymizedRepositorySchema.index({ "coauthors.githubId": 1 });
|
||||
AnonymizedRepositorySchema.index({ retiredTreeGenerations: 1 }, { sparse: true });
|
||||
|
||||
@@ -4,7 +4,7 @@ export const UpstreamFileSchema = new Schema({ organization: String, repoName: S
|
||||
|
||||
const FileSchema = new Schema({
|
||||
name: { type: String, index: true },
|
||||
path: { type: String, index: true },
|
||||
path: { type: String },
|
||||
repoId: { type: String, index: true },
|
||||
treeGeneration: String,
|
||||
treeStaged: Boolean,
|
||||
@@ -19,7 +19,6 @@ const FileSchema = new Schema({
|
||||
},
|
||||
});
|
||||
|
||||
FileSchema.index({ path: 1, repoId: 1 });
|
||||
FileSchema.index({ metadataPending: 1, repoId: 1, treeGeneration: 1 },
|
||||
{ partialFilterExpression: { metadataPending: true } });
|
||||
FileSchema.index({ treeStaged: 1, repoId: 1, treeGeneration: 1 },
|
||||
|
||||
@@ -0,0 +1,2 @@
|
||||
/** Repository URLs and their unique Mongo index use case-insensitive IDs. */
|
||||
export const REPOSITORY_ID_COLLATION = { locale: "en", strength: 2 };
|
||||
@@ -1,3 +1,4 @@
|
||||
import { REPOSITORY_ID_COLLATION } from "../repository-id";
|
||||
import { reserveFileExpansion } from "../billing/quotas";
|
||||
import { withOwnerAccessRecovery } from "../owner-notifications";
|
||||
import { ExpiringMap } from "../expiring-map";
|
||||
@@ -362,7 +363,7 @@ export default class GitHubStream extends GitHubBase {
|
||||
// Retirement markers live inside the storage root and disappear when
|
||||
// expiration removes it. The persisted lifecycle also fences late fills.
|
||||
if (isConnected) {
|
||||
const current = await AnonymizedRepositoryModel.findOne({ repoId })
|
||||
const current = await AnonymizedRepositoryModel.findOne({ repoId }, null, { collation: REPOSITORY_ID_COLLATION })
|
||||
.select("status treeGeneration anonymizeDate contentCacheRevision").lean().exec();
|
||||
if (!current || !current.status || [RepositoryStatus.EXPIRING, RepositoryStatus.EXPIRED,
|
||||
RepositoryStatus.REMOVING, RepositoryStatus.REMOVED, RepositoryStatus.ARCHIVED].includes(current.status)
|
||||
@@ -641,7 +642,7 @@ export default class GitHubStream extends GitHubBase {
|
||||
private async confirmOwnerAccess<T>(repoId: string, work: () => Promise<T>, confirmed?: (result: T) => boolean): Promise<T> {
|
||||
let repository;
|
||||
try {
|
||||
repository = isConnected ? await AnonymizedRepositoryModel.findOne({ repoId })
|
||||
repository = isConnected ? await AnonymizedRepositoryModel.findOne({ repoId }, null, { collation: REPOSITORY_ID_COLLATION })
|
||||
.select("owner source githubAccess").lean().exec() : null;
|
||||
} catch {
|
||||
// Email recovery bookkeeping must not prevent the upstream read.
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { REPOSITORY_ID_COLLATION } from "./repository-id";
|
||||
import { observeCache } from "./cache-observability";
|
||||
import { promises as fs } from "fs";
|
||||
import { join } from "path";
|
||||
@@ -18,7 +19,7 @@ interface CacheScope { repoId: string; generation?: string; revision?: string }
|
||||
|
||||
async function scopeActive(scope?: CacheScope) {
|
||||
if (!scope || !isConnected) return true;
|
||||
const current = await RepositoryModel.findOne({ repoId: scope.repoId })
|
||||
const current = await RepositoryModel.findOne({ repoId: scope.repoId }, null, { collation: REPOSITORY_ID_COLLATION })
|
||||
.select("status treeGeneration anonymizeDate contentCacheRevision").lean().exec();
|
||||
return !!current?.status && ![RepositoryStatus.EXPIRING, RepositoryStatus.EXPIRED,
|
||||
RepositoryStatus.REMOVING, RepositoryStatus.REMOVED, RepositoryStatus.ARCHIVED].includes(current.status)
|
||||
|
||||
Reference in new issue
Block a user