merge: preserve billing admission with bounded tree preparation

This commit is contained in:
Thomas Durieux committed 2026-11-03 08:33:12 +00:00
commit 860169f27b
20 files changed
+801 -103

No files matched your search

+20 -13
View File
@@ -1,3 +1,5 @@
import Repository from "../../core/Repository";
import { sourceStorageSize } from "../../core/source/Source";
import { neverExpirationDeadline, validateBillingExpiration } from "../../core/billing/expiration";
import config from "../../config";
import { assertConferenceAdmission, reserveRepository } from "../../core/billing/quotas";
@@ -535,19 +537,22 @@ router.post(
const sourceChanged = reconnecting || hasRepositorySourceChanged(repo.model, repoUpdate);
const previousAccessRevision = repo.model.githubAccess?.revision;
// Keep proposed settings separate until cleanup passes the persisted fences.
const proposed = new AnonymizedRepositoryModel(repo.model.toObject());
if (repo.model.billingManaged) validateBillingExpiration(repoUpdate.options);
if (config.BILLING_ENABLED && repoUpdate.options.expirationMode === "never" &&
(repo.model.options.expirationMode !== "never" || repo.model.billingNeverExpiresUntil)) {
repo.model.billingNeverExpiresUntil = await neverExpirationDeadline(String(repo.model.owner),
proposed.billingNeverExpiresUntil = await neverExpirationDeadline(String(repo.model.owner),
repo.model.options.expirationMode === "never" ? repo.model.billingNeverExpiresUntil : undefined);
} else if (repoUpdate.options.expirationMode !== "never") {
repo.model.billingNeverExpiresUntil = undefined;
proposed.billingNeverExpiresUntil = undefined;
}
updateRepoModel(repo.model, repoUpdate);
const reactivating = shouldReactivateInactiveRepository(repo.model);
updateRepoModel(proposed, repoUpdate);
const reactivating = shouldReactivateInactiveRepository(proposed);
if (reactivating) {
repo.model.anonymizeDate = new Date();
proposed.anonymizeDate = new Date();
}
const needsAdmission = sourceChanged || reactivating || repoUpdate.conference !== repo.model.conference;
@@ -606,20 +611,22 @@ router.post(
const commit = await repository.getCommitInfo(repoUpdate.source.commit, {
accessToken: sourceAccess.token,
});
if (reconnecting) repo.model.source.commitDate = commit.commit.committer?.date
if (reconnecting) proposed.source.commitDate = commit.commit.committer?.date
? new Date(commit.commit.committer.date) : undefined;
repo.model.githubAccess = { ...sourceAccess.binding, revision: randomUUID() };
repo.model.source.repositoryId = repository.model.id;
repo.model.source.repositoryName =
proposed.githubAccess = { ...sourceAccess.binding, revision: randomUUID() };
proposed.source.repositoryId = repository.model.id;
proposed.source.repositoryName =
repository.fullName || repoUpdate.fullName;
repo.model.anonymizeDate = new Date();
proposed.anonymizeDate = new Date();
if (repo.model.billingManaged) {
const files = await repo.source.getFiles();
await reserveRepository(String(repo.model.owner), repo.repoId,
files.reduce((sum, file) => sum + (file.size || 0), 0), repoUpdate.conference);
const storageBytes = await sourceStorageSize(new Repository(proposed).source);
await reserveRepository(String(repo.model.owner), repo.repoId, storageBytes, repoUpdate.conference);
}
await repo.remove({ accessRevision: previousAccessRevision, replacement: true });
}
repo.model.set({ options: proposed.options, source: proposed.source,
githubAccess: proposed.githubAccess, anonymizeDate: proposed.anonymizeDate,
billingNeverExpiresUntil: proposed.billingNeverExpiresUntil });
const removeRepoFromConference = async (conferenceID: string) => {
const conf = await ConferenceModel.findOne({
+15 -5
View File
@@ -1,3 +1,4 @@
import { FILE_LISTING_LIMIT } from "../../core/file-tree";
import { repositoryFailureStatus } from "../../core/repository-failure-status";
import { anonymizePath } from "../../core/anonymize-utils";
import * as express from "express";
@@ -218,21 +219,30 @@ router.get(
{ $match: { repoId, treeGeneration: repo.model.treeGeneration || { $exists: false }, size: { $ne: null } } },
{ $project: { _id: 0, path: 1 } },
{ $group: { _id: "$path", count: { $sum: 1 } } },
{ $limit: FILE_LISTING_LIMIT + 1 },
]).exec();
if (results.length > FILE_LISTING_LIMIT) throw new AnonymousError("file_listing_too_large", { httpStatus: 413 });
const directCounts: Record<string, number> = {};
const directCounts: Record<string, number> = Object.create(null);
for (const r of results) {
directCounts[r._id ?? ""] = r.count;
}
const folderCounts: Record<string, number> = {};
const folderCounts: Record<string, number> = Object.create(null);
let folders = 0;
const addCount = (folder: string, count: number) => {
if (!Object.hasOwn(folderCounts, folder) && ++folders > FILE_LISTING_LIMIT) {
throw new AnonymousError("file_listing_too_large", { httpStatus: 413 });
}
folderCounts[folder] = (folderCounts[folder] || 0) + count;
};
for (const [folder, count] of Object.entries(directCounts)) {
let p = folder;
folderCounts[p] = (folderCounts[p] || 0) + count;
addCount(p, count);
while (p) {
const idx = p.lastIndexOf("/");
p = idx >= 0 ? p.substring(0, idx) : "";
folderCounts[p] = (folderCounts[p] || 0) + count;
addCount(p, count);
if (!p) break;
}
}
@@ -243,7 +253,7 @@ router.get(
"../../core/anonymize-utils"
);
const compiled = compileTerms(terms);
const anonymized: Record<string, number> = {};
const anonymized: Record<string, number> = Object.create(null);
for (const [folder, count] of Object.entries(folderCounts)) {
const anonFolder = anonymizePathCompiled(folder, compiled);
anonymized[anonFolder] = (anonymized[anonFolder] || 0) + count;
+1 -1
View File
@@ -158,7 +158,7 @@ export async function runRepositoryStatusCheck(now = new Date()) {
await expireContent(AnonymizedPullRequestModel, data => new PullRequest(data), now);
const retiredCursor = AnonymizedRepositoryModel.find({ $or: [
{ retiredTreeGenerations: { $exists: true } }, { retiredContentPrefixes: { $exists: true } },
{ retiredTreeGenerations: { $exists: true } }, { retiredPathIndexKeys: { $exists: true } }, { retiredContentPrefixes: { $exists: true } },
{ legacyContentCleanupPending: true, status: RepositoryStatus.READY },
{ "stagedFileTrees.until": { $lte: now } },
] }).cursor();