diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9a5ad22..5a648ad 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -40,6 +40,7 @@ jobs: --health-cmd "redis-cli ping" --health-interval 5s --health-timeout 3s --health-retries 10 env: + BILLING_TEST_MONGODB_URI: mongodb://127.0.0.1:27017/billing_test_ci TEST_MONGODB_URI: mongodb://127.0.0.1:27017/dependency_test PERFORMANCE_MONGO_URI: mongodb://127.0.0.1:27017/perf_test_ci PERFORMANCE_REDIS_PORT: 6379 diff --git a/docs/paid-hosting.md b/docs/paid-hosting.md index 668ea9a..3ce64f4 100644 --- a/docs/paid-hosting.md +++ b/docs/paid-hosting.md @@ -261,8 +261,9 @@ bytes beyond the tree size before writing them, in blocks of at most 1 MiB. The keeps per-path high-water marks so retries and concurrent fills do not charge twice. Reservations remain conservative across generations and failed fills and require operator reconciliation to release unused capacity. This also covers public raw -downloads and expanded submodule content. Existing cached content is not retroactively -measured by this write-time guard. +downloads and expanded submodule content. Shared-cache hits and legacy content promoted +to the shared cache also reserve their resolved size before serving. Legacy cache hits +outside the shared cache are not retroactively measured by this write-time guard. Accepted one-time prices are stored in `billingprices` when checkout starts and when existing purchases are fulfilled. Hiding an offer by removing its configured price diff --git a/src/core/source/GitHubStream.ts b/src/core/source/GitHubStream.ts index f9e2859..1817c13 100644 --- a/src/core/source/GitHubStream.ts +++ b/src/core/source/GitHubStream.ts @@ -392,6 +392,14 @@ export default class GitHubStream extends GitHubBase { await assertActive(); if (expected.size != null && expected.size > config.MAX_FILE_SIZE) throw new AnonymousError("file_too_big", { httpStatus: 413 }); let legacy = false, allowLegacy = true; + let admitted = expected.size || 0; + const expansionKey = createHash("sha256").update(filePath).digest("hex"); + const admitResolvedBytes = async (bytes: number) => { + if (!config.BILLING_ENABLED || bytes <= admitted) return; + const next = Math.min(config.MAX_FILE_SIZE, Math.ceil(bytes / (1024 ** 2)) * 1024 ** 2); + await reserveFileExpansion(repoId, expansionKey, next - (expected.size || 0)); + admitted = next; + }; const download = async (checkMinimum = false) => { const parentToken = await this.data.getToken(); const token = expected.upstream ? "" : parentToken; @@ -424,6 +432,7 @@ export default class GitHubStream extends GitHubBase { for await (const chunk of input) { bytes += chunk.length; if (bytes > config.MAX_FILE_SIZE) throw new AnonymousError("file_too_big", { httpStatus: 413 }); + await admitResolvedBytes(bytes); yield chunk; } if (checkMinimum && expected.size != null && expected.size > 0 && bytes < expected.size) @@ -435,7 +444,12 @@ export default class GitHubStream extends GitHubBase { return limited; }; const produce = async () => { - if (allowLegacy && await cached()) { legacy = true; observeCache("source_legacy_promotion"); return storage.read(repoId, cachePath); } + if (allowLegacy && await cached()) { + legacy = true; + const info = await storage.fileInfo(repoId, cachePath); + await admitResolvedBytes(info.size || 0); + observeCache("source_legacy_promotion"); return storage.read(repoId, cachePath); + } legacy = false; return download(); }; const verify = async (proof: BlobProof) => { @@ -463,7 +477,10 @@ export default class GitHubStream extends GitHubBase { allowLegacy = false; content = await shared.getOrFill(`git-${expected.sha}`, produce, verify, expected.size); } - try { await assertActive(); } catch (error) { content.destroy(); throw error; } + try { + await assertActive(); + await admitResolvedBytes(content.cachedBlob?.record.size || 0); + } catch (error) { content.destroy(); throw error; } return content; } catch (error) { if (!(error instanceof CacheAdmissionError)) throw error; diff --git a/test/shared-cache-http.test.js b/test/shared-cache-http.test.js index cb30da5..6d0bb66 100644 --- a/test/shared-cache-http.test.js +++ b/test/shared-cache-http.test.js @@ -56,6 +56,34 @@ describe("shared cache HTTP and telemetry integration", function () { await new Promise(resolve => server.close(resolve)); closeBlobCaches(); closeCacheMetrics(); Object.assign(config, old); await fs.rm(root, { recursive: true, force: true }); }); + for (const mode of ["fill", "hit", "bypass"]) { + it(`enforces resolved-byte billing admission on shared source ${mode}`, async () => { + const quotas = require("../src/core/billing/quotas"); + const { billingError } = require("../src/core/billing/config"); + const reserve = quotas.reserveFileExpansion; + const payload = Buffer.alloc(2048, 65); + const source = new GitHubStream({ repoId: "billed-project", organization: "org", repoName: "repo", commit: "commit", getToken: async () => "" }); + let downloads = 0, reservations = 0; + source.downloadWithFallback = async () => { downloads++; return Readable.from(payload); }; + let cache = sharedBlobCache(); + if (mode === "hit") await consume(await cache.getOrFill(`git-${sha(payload)}`, async () => Readable.from(payload))); + if (mode === "bypass") { closeBlobCaches(); config.SHARED_CACHE_MAX_FILE_BYTES = 1; cache = sharedBlobCache(); } + config.BILLING_ENABLED = true; + quotas.reserveFileExpansion = async (repoId, key, bytes) => { + reservations++; + assert.equal(repoId, "billed-project"); + assert.equal(key, createHash("sha256").update("large.bin").digest("hex")); + assert.equal(bytes, 1024 ** 2 - 130); + throw billingError("quota_exceeded", 402); + }; + try { + await assert.rejects(async () => consume(await source.getFileContentCache("large.bin", "billed-project", () => ({ sha: sha(payload), size: 130 }))), /quota_exceeded/); + assert.equal(reservations, 1); + assert.equal(downloads, mode === "hit" ? 0 : 1); + if (mode !== "hit") assert.equal(await cache.lookup(`git-${sha(payload)}`), undefined); + } finally { quotas.reserveFileExpansion = reserve; } + }); + } const headers = { authorization: "test-access" }; it("shares source content across projects without bypassing authorization", async () => { assert.equal((await request(port, "/a/plain")).status, 403); assert.equal(fills, 0);