fix: admit replacement sources and account for resolved storage

docs/paid-hosting.md

@@ -253,3 +253,13 @@ The donation entry remains available.
 Local browser checks used simulated payment responses for the final layout. The real
 Stripe sandbox checks and their limitations are documented in
 [stripe-sandbox-validation.md](stripe-sandbox-validation.md).
+
+### Resolved source-file storage
+
+Git LFS pointer sizes do not represent cached object sizes. Source-cache fills reserve
+bytes beyond the tree size before writing them, in blocks of at most 1 MiB. The ledger
+keeps per-path high-water marks so retries and concurrent fills do not charge twice.
+Reservations remain conservative across generations and failed fills and require
+operator reconciliation to release unused capacity. This also covers public raw
+downloads and expanded submodule content. Existing cached content is not retroactively
+measured by this write-time guard.

src/core/User.ts

@@ -218,7 +218,7 @@ export default class User {
     return AnonymizedRepositoryModel.find({ owner: this.id, status: "ready",
       $or: [{ "options.expirationMode": "never" }, { "options.expirationDate": { $exists: false } },
         { "options.expirationDate": null }, { "options.expirationDate": { $gt: new Date() } }] })
-      .select("repoId size sizeComputedAt treeGeneration fileMetadataRevision").lean().exec();
+      .select("repoId conference size sizeComputedAt treeGeneration fileMetadataRevision").lean().exec();
   }
 
   get model() {

src/core/billing/models.ts

@@ -16,7 +16,8 @@ export const BillingGrant = model("billingGrant", new Schema({
 }));
 export const BillingQuota = model("billingQuota", new Schema({
   _id: String,
-  allocations: [{ _id: false, repoId: String, bytes: Number, createdAt: Date }],
+  allocations: [{ _id: false, repoId: String, bytes: Number, baseBytes: Number,
+    expansions: [{ _id: false, key: String, bytes: Number }], createdAt: Date }],
 }));
 // A short lease serializes Stripe checkout creation across API instances.
 export const BillingLock = model("billingLock", new Schema({ _id: String, token: String, until: Date }));

src/core/billing/quotas.ts

@@ -21,7 +21,7 @@ export async function allowance(owner: string, repoId?: string, conference?: str
 export function allocationExpression(repoId: string) {
   return { $filter: { input: "$allocations", as: "allocation", cond: { $ne: ["$$allocation.repoId", { $literal: repoId }] } } };
 }
-export async function reserveRepository(owner: string, repoId: string, bytes: number, conference?: string) {
+export async function reserveRepository(owner: string, repoId: string, bytes: number, conference?: string, expansion?: { key: string; bytes: number }) {
   if (!config.BILLING_ENABLED) return;
   if (!Number.isSafeInteger(bytes) || bytes < 0) throw billingError("billing_invalid_repository_size");
   const limits = await allowance(owner, repoId, conference);
@@ -46,11 +46,22 @@ export async function reserveRepository(owner: string, repoId: string, bytes: nu
   const others = allocationExpression(repoId);
   // Never shrink a reservation during refresh. Concurrent generations and
   // failed builds cannot free bytes still needed by the published generation.
-  const reservedBytes = { $max: [bytes, { $sum: { $map: { input: { $filter: { input: "$allocations", as: "a", cond: { $eq: ["$$a.repoId", { $literal: repoId }] } } }, as: "a", in: "$$a.bytes" } } }] };
+  const previous = { $arrayElemAt: [{ $filter: { input: "$allocations", as: "a", cond: { $eq: ["$$a.repoId", { $literal: repoId }] } } }, 0] };
+  const oldExpansions = { $ifNull: [{ $getField: { field: "expansions", input: previous } }, []] };
+  const sumExpansions = (input: object) => ({ $sum: { $map: { input, as: "e", in: "$$e.bytes" } } });
+  const expansions = expansion ? { $concatArrays: [
+    { $filter: { input: oldExpansions, as: "e", cond: { $ne: ["$$e.key", { $literal: expansion.key }] } } },
+    [{ key: { $literal: expansion.key }, bytes: { $max: [expansion.bytes, { $sum: { $map: {
+      input: { $filter: { input: oldExpansions, as: "e", cond: { $eq: ["$$e.key", { $literal: expansion.key }] } } }, as: "e", in: "$$e.bytes",
+    } } }] } }],
+  ] } : oldExpansions;
+  const baseBytes = { $max: [bytes, { $ifNull: [{ $getField: { field: "baseBytes", input: previous } },
+    { $subtract: [{ $ifNull: [{ $getField: { field: "bytes", input: previous } }, 0] }, sumExpansions(oldExpansions)] }] }] };
+  const reservedBytes = { $add: [baseBytes, sumExpansions(expansions)] };
   const result = await BillingQuota.updateOne({ _id: limits.scope, $expr: { $and: [
     { $lt: [{ $size: others }, limits.repositories] },
     { $lte: [{ $add: [{ $sum: { $map: { input: others, as: "a", in: "$$a.bytes" } } }, reservedBytes] }, limits.storage] },
-  ] } }, [{ $set: { allocations: { $concatArrays: [others, [{ repoId: { $literal: repoId }, bytes: reservedBytes, createdAt: new Date() }]] } } }]);
+  ] } }, [{ $set: { allocations: { $concatArrays: [others, [{ repoId: { $literal: repoId }, bytes: reservedBytes, baseBytes, expansions, createdAt: new Date() }]] } } }]);
   if (!result.matchedCount) throw billingError("quota_exceeded", 402);
 }
 
@@ -60,3 +71,14 @@ export async function assertConferenceAdmission(owner: string, repoId: string, c
   if (!conf || conf.status !== "ready" || conf.startDate > new Date() || conf.endDate <= new Date()) throw billingError("conf_not_activated");
   if (conf.plan.planID === "conference" && (await allowance(owner, repoId, conference)).scope !== `conference:${conference}`) throw billingError("conference_payment_required", 402);
 }
+
+// Source trees report LFS pointer sizes. Charge resolved bytes before allowing
+// them into the cache. Per-path high-water marks make retries/concurrent fills
+// idempotent and conservatively retain capacity across generations.
+export async function reserveFileExpansion(repoId: string, key: string, extraBytes: number) {
+  if (!config.BILLING_ENABLED) return;
+  if (!Number.isSafeInteger(extraBytes) || extraBytes < 0) throw billingError("billing_invalid_repository_size");
+  const repo = await RepositoryModel.findOne({ repoId }).select("owner conference billingManaged").lean();
+  if (!repo?.billingManaged) return;
+  await reserveRepository(String(repo.owner), repoId, 0, repo.conference, { key, bytes: extraBytes });
+}

src/core/source/GitHubStream.ts

@@ -1,3 +1,4 @@
+import { reserveFileExpansion } from "../billing/quotas";
 import { withOwnerAccessRecovery } from "../owner-notifications";
 import { ExpiringMap } from "../expiring-map";
 import { contentGenerationPrefix, contentRetirementMarker } from "../content-generation";
@@ -409,9 +410,16 @@ export default class GitHubStream extends GitHubBase {
             const content = await origin.downloadWithFallback(token, expected.sha, sourcePath)
               .catch(error => { throw downloadError(error, filePath); });
             let count = 0;
+            let admitted = expected.size || 0;
+            const expansionKey = createHash("sha256").update(filePath).digest("hex");
             const limited = new stream.Transform({ transform(chunk, encoding, callback) {
               count += chunk.length;
-              callback(count > config.MAX_FILE_SIZE ? new AnonymousError("file_too_big", { httpStatus: 413 }) : null, chunk);
+              if (count > config.MAX_FILE_SIZE) return callback(new AnonymousError("file_too_big", { httpStatus: 413 }));
+              if (!config.BILLING_ENABLED || count <= admitted) return callback(null, chunk);
+              // Reserve in bounded blocks to avoid a database write for every stream chunk.
+              admitted = Math.min(config.MAX_FILE_SIZE, Math.ceil(count / (1024 ** 2)) * 1024 ** 2);
+              reserveFileExpansion(repoId, expansionKey, admitted - (expected.size || 0))
+                .then(() => callback(null, chunk), error => callback(error));
             } });
             content.on("error", error => limited.destroy(downloadError(error, filePath)));
             limited.once("close", () => content.destroy());

src/server/routes/repository-private.ts

@@ -547,8 +547,11 @@ router.post(
         repo.model.anonymizeDate = new Date();
       }
 
-      await assertConferenceAdmission(String(repo.model.owner), repo.repoId, repoUpdate.conference);
-      if (repo.model.billingManaged) await reserveRepository(String(repo.model.owner), repo.repoId, repo.model.size?.storage || 0, repoUpdate.conference);
+      const needsAdmission = sourceChanged || reactivating || repoUpdate.conference !== repo.model.conference;
+      if (needsAdmission) {
+        await assertConferenceAdmission(String(repo.model.owner), repo.repoId, repoUpdate.conference);
+        if (repo.model.billingManaged && !sourceChanged) await reserveRepository(String(repo.model.owner), repo.repoId, repo.model.size?.storage || 0, repoUpdate.conference);
+      }
 
       if (sourceChanged) {
         const parsedRepository = gh(repoUpdate.fullName);
@@ -591,6 +594,11 @@ router.post(
         repo.model.source.repositoryName =
           repository.fullName || repoUpdate.fullName;
         repo.model.anonymizeDate = new Date();
+        if (repo.model.billingManaged) {
+          const files = await repo.source.getFiles();
+          await reserveRepository(String(repo.model.owner), repo.repoId,
+            files.reduce((sum, file) => sum + (file.size || 0), 0), repoUpdate.conference);
+        }
         await repo.remove({ accessRevision: previousAccessRevision });
       }
 

src/server/routes/user.ts

@@ -73,7 +73,15 @@ router.get("/dashboard", async (req, res) => {
 router.get("/quota", async (req, res) => {
   try {
     const user = await getUser(req);
-    const ready = await user.quotaRepositories();
+    let ready = await user.quotaRepositories();
+    const limits = config.BILLING_ENABLED ? await allowance(user.id) : null;
+    if (limits) {
+      const funded = [];
+      for (const repository of ready) {
+        if ((await allowance(user.id, repository.repoId, repository.conference)).scope === limits.scope) funded.push(repository);
+      }
+      ready = funded;
+    }
 
     let totalStorage = 0;
     let totalFiles = 0;
@@ -127,7 +135,6 @@ router.get("/quota", async (req, res) => {
       }
     }
 
-    const limits = config.BILLING_ENABLED ? await allowance(user.id) : null;
     const reserved = limits ? await BillingQuota.findById(limits.scope).lean() : null;
     res.json({
       storage: {

test/billing-integration.test.js

@@ -54,6 +54,37 @@ async function rejects(promise, message) {
     const ledger = await BillingQuota.findById(`user:${owner}`);
     expect(ledger.allocations.map(a => a.repoId)).to.deep.equal(["personal"]);
   });
+  it("accounts for concurrent resolved-file growth without double charging retries", async () => {
+    const { reserveFileExpansion } = require("../src/core/billing/quotas");
+    await Repo.create({ repoId: "lfs", owner, billingManaged: true, status: "ready", size: { storage: 10 } });
+    await reserveRepository(owner, "lfs", 10);
+    await Promise.all(Array.from({ length: 8 }, () => reserveFileExpansion("lfs", "file-a", 40)));
+    expect((await BillingQuota.findById(`user:${owner}`)).allocations[0].bytes).to.equal(50);
+    await reserveFileExpansion("lfs", "file-b", 40);
+    await rejects(reserveFileExpansion("lfs", "file-c", 20), "quota_exceeded");
+    await reserveRepository(owner, "lfs", 15);
+    expect((await BillingQuota.findById(`user:${owner}`)).allocations[0].bytes).to.equal(95);
+    await reserveFileExpansion("lfs", "file-a", 1);
+    expect((await BillingQuota.findById(`user:${owner}`)).allocations[0].bytes).to.equal(95);
+  });
+  it("shows only personally funded repositories before a user ledger exists", async () => {
+    const utils = require("../src/server/routes/route-utils");
+    const originalUser = utils.getUser;
+    await BillingGrant.create({ _id: "conf-usage", owner, scope: "conference:funded", status: "paid", until: future(), planId: "conference" });
+    utils.getUser = async () => ({ id: owner, quotaRepositories: async () => [
+      { repoId: "conference-work", conference: "funded", size: { storage: 90, file: 1 } },
+      { repoId: "personal-work", size: { storage: 10, file: 1 } },
+    ] });
+    try {
+      const router = require("../src/server/routes/user").default;
+      const handler = router.stack.find(layer => layer.route?.path === "/quota").route.stack[0].handle;
+      let body;
+      await handler({}, { json: value => { body = value; } });
+      expect(body.repository.used).to.equal(1);
+      expect(body.storage.used).to.equal(10);
+      expect(body.file.used).to.equal(1);
+    } finally { utils.getUser = originalUser; }
+  });
   it("counts legacy repositories and releases removed repositories", async () => {
     await Repo.create(["one", "two", "three"].map(repoId => ({ repoId, owner, status: "ready", size: { storage: 10 } })));
     await rejects(reserveRepository(owner, "four", 1), "quota_exceeded");

test/production-failure-fixes.test.js

@@ -88,6 +88,23 @@ describe("production failure fixes", function () {
     source.downloadWithFallback = async () => { attempts++; throw Object.assign(Error("forbidden"), { response: { statusCode: 403 } }); };
     now += 30001; await read().catch(() => {}); await read().catch(() => {}); expect(attempts).to.equal(6);
   });
+  it("blocks resolved LFS bytes before they reach the source cache when capacity is full", async () => {
+    const config = require("../src/config").default;
+    const quotas = require("../src/core/billing/quotas");
+    stub(config, "BILLING_ENABLED", true);
+    stub(storage, "fileInfo", async () => { throw Object.assign(Error("missing"), { code: "ENOENT" }); });
+    let written = 0, requested;
+    stub(storage, "write", async (_repo, _path, input) => { for await (const chunk of input) written += chunk.length; });
+    stub(quotas, "reserveFileExpansion", async (_id, _key, bytes) => { requested = bytes; throw Error("quota_exceeded"); });
+    const source = new GitHubStream({ repoId: "lfs-quota", organization: "owner", repoName: "repo", commit: "first", getToken: () => "token" });
+    source.downloadWithFallback = async () => require("node:stream").Readable.from([Buffer.alloc(1024)]);
+    let error;
+    try { await source.getFileContentCache("large.bin", source.data.repoId, () => ({ sha: "sha", size: 130 })); }
+    catch (caught) { error = caught; }
+    expect(error?.message).to.equal("quota_exceeded");
+    expect(requested).to.equal(1024 ** 2 - 130);
+    expect(written).to.equal(0);
+  });
   it("does not cache a storage 404 as a missing upstream file", async () => {
     let attempts = 0;
     stub(storage, "fileInfo", async () => { throw Object.assign(Error("missing"), { code: "ENOENT" }); });

test/repository-activity.test.js

@@ -66,6 +66,43 @@ describe("repository activity dates", () => {
     expect(saved.source.commit).to.equal("abc123");
     expect(saved).not.to.have.property("publishedAt");
   });
+  it("allows privacy-only saves after hosting capacity or conference admission ends", async () => {
+    const repo = repository(); auth(repo);
+    stub(require("../src/config").default, "BILLING_ENABLED", true);
+    const quotas = require("../src/core/billing/quotas");
+    repo.model.billingManaged = true;
+    repo.model.conference = "ended-conf";
+    repo.model.options.expirationMode = "remove";
+    stub(quotas, "assertConferenceAdmission", async () => { throw Error("must not check conference"); });
+    stub(quotas, "reserveRepository", async () => { throw Error("must not reserve"); });
+    let saved;
+    stub(Model, "updateOne", (_filter, update) => ({ exec: async () => { saved = update.$set; return { matchedCount: 1 }; } }));
+    await handler("repository-private", "/:repoId/", "post")({ body: {
+      repoId: repo.repoId, fullName: "owner/repo", conference: "ended-conf", source: { commit: "abc123", branch: "main" },
+      terms: ["private"], options: { expirationMode: "remove", expirationDate: "2027-01-01", image: false, pdf: false, update: false },
+    } }, { json: () => {} });
+    expect(saved.options).to.include({ image: false, pdf: false, update: false });
+  });
+  it("rejects an oversized edited source before removing the published files", async () => {
+    const repo = repository(); auth(repo); repo.model.billingManaged = true;
+    stub(require("../src/config").default, "BILLING_ENABLED", true);
+    const quotas = require("../src/core/billing/quotas");
+    const access = require("../src/core/github-app");
+    const github = require("../src/core/source/GitHubRepository");
+    const Stream = require("../src/core/source/GitHubStream").default;
+    stub(quotas, "assertConferenceAdmission", async () => {});
+    stub(access, "selectRepositoryAccess", async () => ({ token: "test", binding: { kind: "oauth" } }));
+    stub(github, "getRepositoryFromGitHub", async () => ({ model: { id: "gh_1" }, fullName: "owner/repo", getCommitInfo: async () => ({ commit: {} }) }));
+    stub(Stream.prototype, "getFiles", async function () { expect(this.data.commit).to.equal("abcdef456"); return [{ size: 500 }]; });
+    stub(quotas, "reserveRepository", async (_owner, _repo, bytes) => { expect(bytes).to.equal(500); throw Error("quota_exceeded"); });
+    repo.remove = async () => { throw Error("must preserve snapshot"); };
+    let error;
+    try { await handler("repository-private", "/:repoId/", "post")({ body: {
+      repoId: repo.repoId, fullName: "owner/repo", source: { commit: "abcdef456", branch: "main" },
+      terms: [], options: { expirationMode: "never" },
+    } }, { json: () => {} }); } catch (caught) { error = caught; }
+    expect(error?.message).to.equal("quota_exceeded");
+  });
   for (const mode of ["never", "remove"]) {
     it(`persists the billing deadline when settings change to ${mode}`, async () => {
       const config = require("../src/config").default;