mirror of
https://github.com/tdurieux/anonymous_github.git
synced 2026-11-18 01:54:23 +01:00
merge: preserve billing admission with bounded tree preparation
This commit is contained in:
commit
860169f27b
20 files changed
+801
-103
No files matched your search
@@ -1,3 +1,5 @@
|
||||
import Repository from "../../core/Repository";
|
||||
import { sourceStorageSize } from "../../core/source/Source";
|
||||
import { neverExpirationDeadline, validateBillingExpiration } from "../../core/billing/expiration";
|
||||
import config from "../../config";
|
||||
import { assertConferenceAdmission, reserveRepository } from "../../core/billing/quotas";
|
||||
@@ -535,19 +537,22 @@ router.post(
|
||||
const sourceChanged = reconnecting || hasRepositorySourceChanged(repo.model, repoUpdate);
|
||||
const previousAccessRevision = repo.model.githubAccess?.revision;
|
||||
|
||||
// Keep proposed settings separate until cleanup passes the persisted fences.
|
||||
const proposed = new AnonymizedRepositoryModel(repo.model.toObject());
|
||||
if (repo.model.billingManaged) validateBillingExpiration(repoUpdate.options);
|
||||
if (config.BILLING_ENABLED && repoUpdate.options.expirationMode === "never" &&
|
||||
(repo.model.options.expirationMode !== "never" || repo.model.billingNeverExpiresUntil)) {
|
||||
repo.model.billingNeverExpiresUntil = await neverExpirationDeadline(String(repo.model.owner),
|
||||
proposed.billingNeverExpiresUntil = await neverExpirationDeadline(String(repo.model.owner),
|
||||
repo.model.options.expirationMode === "never" ? repo.model.billingNeverExpiresUntil : undefined);
|
||||
} else if (repoUpdate.options.expirationMode !== "never") {
|
||||
repo.model.billingNeverExpiresUntil = undefined;
|
||||
proposed.billingNeverExpiresUntil = undefined;
|
||||
}
|
||||
updateRepoModel(repo.model, repoUpdate);
|
||||
const reactivating = shouldReactivateInactiveRepository(repo.model);
|
||||
updateRepoModel(proposed, repoUpdate);
|
||||
const reactivating = shouldReactivateInactiveRepository(proposed);
|
||||
|
||||
|
||||
if (reactivating) {
|
||||
repo.model.anonymizeDate = new Date();
|
||||
proposed.anonymizeDate = new Date();
|
||||
}
|
||||
|
||||
const needsAdmission = sourceChanged || reactivating || repoUpdate.conference !== repo.model.conference;
|
||||
@@ -606,20 +611,22 @@ router.post(
|
||||
const commit = await repository.getCommitInfo(repoUpdate.source.commit, {
|
||||
accessToken: sourceAccess.token,
|
||||
});
|
||||
if (reconnecting) repo.model.source.commitDate = commit.commit.committer?.date
|
||||
if (reconnecting) proposed.source.commitDate = commit.commit.committer?.date
|
||||
? new Date(commit.commit.committer.date) : undefined;
|
||||
repo.model.githubAccess = { ...sourceAccess.binding, revision: randomUUID() };
|
||||
repo.model.source.repositoryId = repository.model.id;
|
||||
repo.model.source.repositoryName =
|
||||
proposed.githubAccess = { ...sourceAccess.binding, revision: randomUUID() };
|
||||
proposed.source.repositoryId = repository.model.id;
|
||||
proposed.source.repositoryName =
|
||||
repository.fullName || repoUpdate.fullName;
|
||||
repo.model.anonymizeDate = new Date();
|
||||
proposed.anonymizeDate = new Date();
|
||||
if (repo.model.billingManaged) {
|
||||
const files = await repo.source.getFiles();
|
||||
await reserveRepository(String(repo.model.owner), repo.repoId,
|
||||
files.reduce((sum, file) => sum + (file.size || 0), 0), repoUpdate.conference);
|
||||
const storageBytes = await sourceStorageSize(new Repository(proposed).source);
|
||||
await reserveRepository(String(repo.model.owner), repo.repoId, storageBytes, repoUpdate.conference);
|
||||
}
|
||||
await repo.remove({ accessRevision: previousAccessRevision, replacement: true });
|
||||
}
|
||||
repo.model.set({ options: proposed.options, source: proposed.source,
|
||||
githubAccess: proposed.githubAccess, anonymizeDate: proposed.anonymizeDate,
|
||||
billingNeverExpiresUntil: proposed.billingNeverExpiresUntil });
|
||||
|
||||
const removeRepoFromConference = async (conferenceID: string) => {
|
||||
const conf = await ConferenceModel.findOne({
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { FILE_LISTING_LIMIT } from "../../core/file-tree";
|
||||
import { repositoryFailureStatus } from "../../core/repository-failure-status";
|
||||
import { anonymizePath } from "../../core/anonymize-utils";
|
||||
import * as express from "express";
|
||||
@@ -218,21 +219,30 @@ router.get(
|
||||
{ $match: { repoId, treeGeneration: repo.model.treeGeneration || { $exists: false }, size: { $ne: null } } },
|
||||
{ $project: { _id: 0, path: 1 } },
|
||||
{ $group: { _id: "$path", count: { $sum: 1 } } },
|
||||
{ $limit: FILE_LISTING_LIMIT + 1 },
|
||||
]).exec();
|
||||
if (results.length > FILE_LISTING_LIMIT) throw new AnonymousError("file_listing_too_large", { httpStatus: 413 });
|
||||
|
||||
const directCounts: Record<string, number> = {};
|
||||
const directCounts: Record<string, number> = Object.create(null);
|
||||
for (const r of results) {
|
||||
directCounts[r._id ?? ""] = r.count;
|
||||
}
|
||||
|
||||
const folderCounts: Record<string, number> = {};
|
||||
const folderCounts: Record<string, number> = Object.create(null);
|
||||
let folders = 0;
|
||||
const addCount = (folder: string, count: number) => {
|
||||
if (!Object.hasOwn(folderCounts, folder) && ++folders > FILE_LISTING_LIMIT) {
|
||||
throw new AnonymousError("file_listing_too_large", { httpStatus: 413 });
|
||||
}
|
||||
folderCounts[folder] = (folderCounts[folder] || 0) + count;
|
||||
};
|
||||
for (const [folder, count] of Object.entries(directCounts)) {
|
||||
let p = folder;
|
||||
folderCounts[p] = (folderCounts[p] || 0) + count;
|
||||
addCount(p, count);
|
||||
while (p) {
|
||||
const idx = p.lastIndexOf("/");
|
||||
p = idx >= 0 ? p.substring(0, idx) : "";
|
||||
folderCounts[p] = (folderCounts[p] || 0) + count;
|
||||
addCount(p, count);
|
||||
if (!p) break;
|
||||
}
|
||||
}
|
||||
@@ -243,7 +253,7 @@ router.get(
|
||||
"../../core/anonymize-utils"
|
||||
);
|
||||
const compiled = compileTerms(terms);
|
||||
const anonymized: Record<string, number> = {};
|
||||
const anonymized: Record<string, number> = Object.create(null);
|
||||
for (const [folder, count] of Object.entries(folderCounts)) {
|
||||
const anonFolder = anonymizePathCompiled(folder, compiled);
|
||||
anonymized[anonFolder] = (anonymized[anonFolder] || 0) + count;
|
||||
|
||||
@@ -158,7 +158,7 @@ export async function runRepositoryStatusCheck(now = new Date()) {
|
||||
await expireContent(AnonymizedPullRequestModel, data => new PullRequest(data), now);
|
||||
|
||||
const retiredCursor = AnonymizedRepositoryModel.find({ $or: [
|
||||
{ retiredTreeGenerations: { $exists: true } }, { retiredContentPrefixes: { $exists: true } },
|
||||
{ retiredTreeGenerations: { $exists: true } }, { retiredPathIndexKeys: { $exists: true } }, { retiredContentPrefixes: { $exists: true } },
|
||||
{ legacyContentCleanupPending: true, status: RepositoryStatus.READY },
|
||||
{ "stagedFileTrees.until": { $lte: now } },
|
||||
] }).cursor();
|
||||
|
||||
Reference in new issue
Block a user