diff --git a/docs/paid-hosting.md b/docs/paid-hosting.md index eee2f39..b215e38 100644 --- a/docs/paid-hosting.md +++ b/docs/paid-hosting.md @@ -253,3 +253,13 @@ The donation entry remains available. Local browser checks used simulated payment responses for the final layout. The real Stripe sandbox checks and their limitations are documented in [stripe-sandbox-validation.md](stripe-sandbox-validation.md). + +### Resolved source-file storage + +Git LFS pointer sizes do not represent cached object sizes. Source-cache fills reserve +bytes beyond the tree size before writing them, in blocks of at most 1 MiB. The ledger +keeps per-path high-water marks so retries and concurrent fills do not charge twice. +Reservations remain conservative across generations and failed fills and require +operator reconciliation to release unused capacity. This also covers public raw +downloads and expanded submodule content. Existing cached content is not retroactively +measured by this write-time guard. diff --git a/src/core/User.ts b/src/core/User.ts index 89cf446..995af3b 100644 --- a/src/core/User.ts +++ b/src/core/User.ts @@ -218,7 +218,7 @@ export default class User { return AnonymizedRepositoryModel.find({ owner: this.id, status: "ready", $or: [{ "options.expirationMode": "never" }, { "options.expirationDate": { $exists: false } }, { "options.expirationDate": null }, { "options.expirationDate": { $gt: new Date() } }] }) - .select("repoId size sizeComputedAt treeGeneration fileMetadataRevision").lean().exec(); + .select("repoId conference size sizeComputedAt treeGeneration fileMetadataRevision").lean().exec(); } get model() { diff --git a/src/core/billing/models.ts b/src/core/billing/models.ts index 05b8b6b..9095c55 100644 --- a/src/core/billing/models.ts +++ b/src/core/billing/models.ts @@ -16,7 +16,8 @@ export const BillingGrant = model("billingGrant", new Schema({ })); export const BillingQuota = model("billingQuota", new Schema({ _id: String, - allocations: [{ _id: false, repoId: String, bytes: Number, createdAt: Date }], + allocations: [{ _id: false, repoId: String, bytes: Number, baseBytes: Number, + expansions: [{ _id: false, key: String, bytes: Number }], createdAt: Date }], })); // A short lease serializes Stripe checkout creation across API instances. export const BillingLock = model("billingLock", new Schema({ _id: String, token: String, until: Date })); diff --git a/src/core/billing/quotas.ts b/src/core/billing/quotas.ts index 7e463c5..9c811b1 100644 --- a/src/core/billing/quotas.ts +++ b/src/core/billing/quotas.ts @@ -21,7 +21,7 @@ export async function allowance(owner: string, repoId?: string, conference?: str export function allocationExpression(repoId: string) { return { $filter: { input: "$allocations", as: "allocation", cond: { $ne: ["$$allocation.repoId", { $literal: repoId }] } } }; } -export async function reserveRepository(owner: string, repoId: string, bytes: number, conference?: string) { +export async function reserveRepository(owner: string, repoId: string, bytes: number, conference?: string, expansion?: { key: string; bytes: number }) { if (!config.BILLING_ENABLED) return; if (!Number.isSafeInteger(bytes) || bytes < 0) throw billingError("billing_invalid_repository_size"); const limits = await allowance(owner, repoId, conference); @@ -46,11 +46,22 @@ export async function reserveRepository(owner: string, repoId: string, bytes: nu const others = allocationExpression(repoId); // Never shrink a reservation during refresh. Concurrent generations and // failed builds cannot free bytes still needed by the published generation. - const reservedBytes = { $max: [bytes, { $sum: { $map: { input: { $filter: { input: "$allocations", as: "a", cond: { $eq: ["$$a.repoId", { $literal: repoId }] } } }, as: "a", in: "$$a.bytes" } } }] }; + const previous = { $arrayElemAt: [{ $filter: { input: "$allocations", as: "a", cond: { $eq: ["$$a.repoId", { $literal: repoId }] } } }, 0] }; + const oldExpansions = { $ifNull: [{ $getField: { field: "expansions", input: previous } }, []] }; + const sumExpansions = (input: object) => ({ $sum: { $map: { input, as: "e", in: "$$e.bytes" } } }); + const expansions = expansion ? { $concatArrays: [ + { $filter: { input: oldExpansions, as: "e", cond: { $ne: ["$$e.key", { $literal: expansion.key }] } } }, + [{ key: { $literal: expansion.key }, bytes: { $max: [expansion.bytes, { $sum: { $map: { + input: { $filter: { input: oldExpansions, as: "e", cond: { $eq: ["$$e.key", { $literal: expansion.key }] } } }, as: "e", in: "$$e.bytes", + } } }] } }], + ] } : oldExpansions; + const baseBytes = { $max: [bytes, { $ifNull: [{ $getField: { field: "baseBytes", input: previous } }, + { $subtract: [{ $ifNull: [{ $getField: { field: "bytes", input: previous } }, 0] }, sumExpansions(oldExpansions)] }] }] }; + const reservedBytes = { $add: [baseBytes, sumExpansions(expansions)] }; const result = await BillingQuota.updateOne({ _id: limits.scope, $expr: { $and: [ { $lt: [{ $size: others }, limits.repositories] }, { $lte: [{ $add: [{ $sum: { $map: { input: others, as: "a", in: "$$a.bytes" } } }, reservedBytes] }, limits.storage] }, - ] } }, [{ $set: { allocations: { $concatArrays: [others, [{ repoId: { $literal: repoId }, bytes: reservedBytes, createdAt: new Date() }]] } } }]); + ] } }, [{ $set: { allocations: { $concatArrays: [others, [{ repoId: { $literal: repoId }, bytes: reservedBytes, baseBytes, expansions, createdAt: new Date() }]] } } }]); if (!result.matchedCount) throw billingError("quota_exceeded", 402); } @@ -60,3 +71,14 @@ export async function assertConferenceAdmission(owner: string, repoId: string, c if (!conf || conf.status !== "ready" || conf.startDate > new Date() || conf.endDate <= new Date()) throw billingError("conf_not_activated"); if (conf.plan.planID === "conference" && (await allowance(owner, repoId, conference)).scope !== `conference:${conference}`) throw billingError("conference_payment_required", 402); } + +// Source trees report LFS pointer sizes. Charge resolved bytes before allowing +// them into the cache. Per-path high-water marks make retries/concurrent fills +// idempotent and conservatively retain capacity across generations. +export async function reserveFileExpansion(repoId: string, key: string, extraBytes: number) { + if (!config.BILLING_ENABLED) return; + if (!Number.isSafeInteger(extraBytes) || extraBytes < 0) throw billingError("billing_invalid_repository_size"); + const repo = await RepositoryModel.findOne({ repoId }).select("owner conference billingManaged").lean(); + if (!repo?.billingManaged) return; + await reserveRepository(String(repo.owner), repoId, 0, repo.conference, { key, bytes: extraBytes }); +} diff --git a/src/core/source/GitHubStream.ts b/src/core/source/GitHubStream.ts index 122e8c4..9a23820 100644 --- a/src/core/source/GitHubStream.ts +++ b/src/core/source/GitHubStream.ts @@ -1,3 +1,4 @@ +import { reserveFileExpansion } from "../billing/quotas"; import { withOwnerAccessRecovery } from "../owner-notifications"; import { ExpiringMap } from "../expiring-map"; import { contentGenerationPrefix, contentRetirementMarker } from "../content-generation"; @@ -409,9 +410,16 @@ export default class GitHubStream extends GitHubBase { const content = await origin.downloadWithFallback(token, expected.sha, sourcePath) .catch(error => { throw downloadError(error, filePath); }); let count = 0; + let admitted = expected.size || 0; + const expansionKey = createHash("sha256").update(filePath).digest("hex"); const limited = new stream.Transform({ transform(chunk, encoding, callback) { count += chunk.length; - callback(count > config.MAX_FILE_SIZE ? new AnonymousError("file_too_big", { httpStatus: 413 }) : null, chunk); + if (count > config.MAX_FILE_SIZE) return callback(new AnonymousError("file_too_big", { httpStatus: 413 })); + if (!config.BILLING_ENABLED || count <= admitted) return callback(null, chunk); + // Reserve in bounded blocks to avoid a database write for every stream chunk. + admitted = Math.min(config.MAX_FILE_SIZE, Math.ceil(count / (1024 ** 2)) * 1024 ** 2); + reserveFileExpansion(repoId, expansionKey, admitted - (expected.size || 0)) + .then(() => callback(null, chunk), error => callback(error)); } }); content.on("error", error => limited.destroy(downloadError(error, filePath))); limited.once("close", () => content.destroy()); diff --git a/src/server/routes/repository-private.ts b/src/server/routes/repository-private.ts index e46cc53..506ef1a 100644 --- a/src/server/routes/repository-private.ts +++ b/src/server/routes/repository-private.ts @@ -547,8 +547,11 @@ router.post( repo.model.anonymizeDate = new Date(); } - await assertConferenceAdmission(String(repo.model.owner), repo.repoId, repoUpdate.conference); - if (repo.model.billingManaged) await reserveRepository(String(repo.model.owner), repo.repoId, repo.model.size?.storage || 0, repoUpdate.conference); + const needsAdmission = sourceChanged || reactivating || repoUpdate.conference !== repo.model.conference; + if (needsAdmission) { + await assertConferenceAdmission(String(repo.model.owner), repo.repoId, repoUpdate.conference); + if (repo.model.billingManaged && !sourceChanged) await reserveRepository(String(repo.model.owner), repo.repoId, repo.model.size?.storage || 0, repoUpdate.conference); + } if (sourceChanged) { const parsedRepository = gh(repoUpdate.fullName); @@ -591,6 +594,11 @@ router.post( repo.model.source.repositoryName = repository.fullName || repoUpdate.fullName; repo.model.anonymizeDate = new Date(); + if (repo.model.billingManaged) { + const files = await repo.source.getFiles(); + await reserveRepository(String(repo.model.owner), repo.repoId, + files.reduce((sum, file) => sum + (file.size || 0), 0), repoUpdate.conference); + } await repo.remove({ accessRevision: previousAccessRevision }); } diff --git a/src/server/routes/user.ts b/src/server/routes/user.ts index 214b72f..c163ef6 100644 --- a/src/server/routes/user.ts +++ b/src/server/routes/user.ts @@ -73,7 +73,15 @@ router.get("/dashboard", async (req, res) => { router.get("/quota", async (req, res) => { try { const user = await getUser(req); - const ready = await user.quotaRepositories(); + let ready = await user.quotaRepositories(); + const limits = config.BILLING_ENABLED ? await allowance(user.id) : null; + if (limits) { + const funded = []; + for (const repository of ready) { + if ((await allowance(user.id, repository.repoId, repository.conference)).scope === limits.scope) funded.push(repository); + } + ready = funded; + } let totalStorage = 0; let totalFiles = 0; @@ -127,7 +135,6 @@ router.get("/quota", async (req, res) => { } } - const limits = config.BILLING_ENABLED ? await allowance(user.id) : null; const reserved = limits ? await BillingQuota.findById(limits.scope).lean() : null; res.json({ storage: { diff --git a/test/billing-integration.test.js b/test/billing-integration.test.js index 00ee449..ae21b68 100644 --- a/test/billing-integration.test.js +++ b/test/billing-integration.test.js @@ -54,6 +54,37 @@ async function rejects(promise, message) { const ledger = await BillingQuota.findById(`user:${owner}`); expect(ledger.allocations.map(a => a.repoId)).to.deep.equal(["personal"]); }); + it("accounts for concurrent resolved-file growth without double charging retries", async () => { + const { reserveFileExpansion } = require("../src/core/billing/quotas"); + await Repo.create({ repoId: "lfs", owner, billingManaged: true, status: "ready", size: { storage: 10 } }); + await reserveRepository(owner, "lfs", 10); + await Promise.all(Array.from({ length: 8 }, () => reserveFileExpansion("lfs", "file-a", 40))); + expect((await BillingQuota.findById(`user:${owner}`)).allocations[0].bytes).to.equal(50); + await reserveFileExpansion("lfs", "file-b", 40); + await rejects(reserveFileExpansion("lfs", "file-c", 20), "quota_exceeded"); + await reserveRepository(owner, "lfs", 15); + expect((await BillingQuota.findById(`user:${owner}`)).allocations[0].bytes).to.equal(95); + await reserveFileExpansion("lfs", "file-a", 1); + expect((await BillingQuota.findById(`user:${owner}`)).allocations[0].bytes).to.equal(95); + }); + it("shows only personally funded repositories before a user ledger exists", async () => { + const utils = require("../src/server/routes/route-utils"); + const originalUser = utils.getUser; + await BillingGrant.create({ _id: "conf-usage", owner, scope: "conference:funded", status: "paid", until: future(), planId: "conference" }); + utils.getUser = async () => ({ id: owner, quotaRepositories: async () => [ + { repoId: "conference-work", conference: "funded", size: { storage: 90, file: 1 } }, + { repoId: "personal-work", size: { storage: 10, file: 1 } }, + ] }); + try { + const router = require("../src/server/routes/user").default; + const handler = router.stack.find(layer => layer.route?.path === "/quota").route.stack[0].handle; + let body; + await handler({}, { json: value => { body = value; } }); + expect(body.repository.used).to.equal(1); + expect(body.storage.used).to.equal(10); + expect(body.file.used).to.equal(1); + } finally { utils.getUser = originalUser; } + }); it("counts legacy repositories and releases removed repositories", async () => { await Repo.create(["one", "two", "three"].map(repoId => ({ repoId, owner, status: "ready", size: { storage: 10 } }))); await rejects(reserveRepository(owner, "four", 1), "quota_exceeded"); diff --git a/test/production-failure-fixes.test.js b/test/production-failure-fixes.test.js index e10eda9..48846ba 100644 --- a/test/production-failure-fixes.test.js +++ b/test/production-failure-fixes.test.js @@ -88,6 +88,23 @@ describe("production failure fixes", function () { source.downloadWithFallback = async () => { attempts++; throw Object.assign(Error("forbidden"), { response: { statusCode: 403 } }); }; now += 30001; await read().catch(() => {}); await read().catch(() => {}); expect(attempts).to.equal(6); }); + it("blocks resolved LFS bytes before they reach the source cache when capacity is full", async () => { + const config = require("../src/config").default; + const quotas = require("../src/core/billing/quotas"); + stub(config, "BILLING_ENABLED", true); + stub(storage, "fileInfo", async () => { throw Object.assign(Error("missing"), { code: "ENOENT" }); }); + let written = 0, requested; + stub(storage, "write", async (_repo, _path, input) => { for await (const chunk of input) written += chunk.length; }); + stub(quotas, "reserveFileExpansion", async (_id, _key, bytes) => { requested = bytes; throw Error("quota_exceeded"); }); + const source = new GitHubStream({ repoId: "lfs-quota", organization: "owner", repoName: "repo", commit: "first", getToken: () => "token" }); + source.downloadWithFallback = async () => require("node:stream").Readable.from([Buffer.alloc(1024)]); + let error; + try { await source.getFileContentCache("large.bin", source.data.repoId, () => ({ sha: "sha", size: 130 })); } + catch (caught) { error = caught; } + expect(error?.message).to.equal("quota_exceeded"); + expect(requested).to.equal(1024 ** 2 - 130); + expect(written).to.equal(0); + }); it("does not cache a storage 404 as a missing upstream file", async () => { let attempts = 0; stub(storage, "fileInfo", async () => { throw Object.assign(Error("missing"), { code: "ENOENT" }); }); diff --git a/test/repository-activity.test.js b/test/repository-activity.test.js index 55a9710..787418d 100644 --- a/test/repository-activity.test.js +++ b/test/repository-activity.test.js @@ -66,6 +66,43 @@ describe("repository activity dates", () => { expect(saved.source.commit).to.equal("abc123"); expect(saved).not.to.have.property("publishedAt"); }); + it("allows privacy-only saves after hosting capacity or conference admission ends", async () => { + const repo = repository(); auth(repo); + stub(require("../src/config").default, "BILLING_ENABLED", true); + const quotas = require("../src/core/billing/quotas"); + repo.model.billingManaged = true; + repo.model.conference = "ended-conf"; + repo.model.options.expirationMode = "remove"; + stub(quotas, "assertConferenceAdmission", async () => { throw Error("must not check conference"); }); + stub(quotas, "reserveRepository", async () => { throw Error("must not reserve"); }); + let saved; + stub(Model, "updateOne", (_filter, update) => ({ exec: async () => { saved = update.$set; return { matchedCount: 1 }; } })); + await handler("repository-private", "/:repoId/", "post")({ body: { + repoId: repo.repoId, fullName: "owner/repo", conference: "ended-conf", source: { commit: "abc123", branch: "main" }, + terms: ["private"], options: { expirationMode: "remove", expirationDate: "2027-01-01", image: false, pdf: false, update: false }, + } }, { json: () => {} }); + expect(saved.options).to.include({ image: false, pdf: false, update: false }); + }); + it("rejects an oversized edited source before removing the published files", async () => { + const repo = repository(); auth(repo); repo.model.billingManaged = true; + stub(require("../src/config").default, "BILLING_ENABLED", true); + const quotas = require("../src/core/billing/quotas"); + const access = require("../src/core/github-app"); + const github = require("../src/core/source/GitHubRepository"); + const Stream = require("../src/core/source/GitHubStream").default; + stub(quotas, "assertConferenceAdmission", async () => {}); + stub(access, "selectRepositoryAccess", async () => ({ token: "test", binding: { kind: "oauth" } })); + stub(github, "getRepositoryFromGitHub", async () => ({ model: { id: "gh_1" }, fullName: "owner/repo", getCommitInfo: async () => ({ commit: {} }) })); + stub(Stream.prototype, "getFiles", async function () { expect(this.data.commit).to.equal("abcdef456"); return [{ size: 500 }]; }); + stub(quotas, "reserveRepository", async (_owner, _repo, bytes) => { expect(bytes).to.equal(500); throw Error("quota_exceeded"); }); + repo.remove = async () => { throw Error("must preserve snapshot"); }; + let error; + try { await handler("repository-private", "/:repoId/", "post")({ body: { + repoId: repo.repoId, fullName: "owner/repo", source: { commit: "abcdef456", branch: "main" }, + terms: [], options: { expirationMode: "never" }, + } }, { json: () => {} }); } catch (caught) { error = caught; } + expect(error?.message).to.equal("quota_exceeded"); + }); for (const mode of ["never", "remove"]) { it(`persists the billing deadline when settings change to ${mode}`, async () => { const config = require("../src/config").default;