diff --git a/src/core/Repository.ts b/src/core/Repository.ts index 63424ed..f61bfef 100644 --- a/src/core/Repository.ts +++ b/src/core/Repository.ts @@ -213,7 +213,9 @@ export default class Repository { ? new RegExp(`^${escapedPath}`) : undefined; if (opt.recursive === false) { - pathQuery = escapedPath ? new RegExp(`^${escapedPath}$`) : ""; + // Equality bounds the path index to this directory. An anchored regex + // still scans its descendants before filtering out their paths. + pathQuery = opt.path || ""; } const query: FilterQuery = { diff --git a/src/core/repository-failure-status.ts b/src/core/repository-failure-status.ts index 3e78184..0ee5588 100644 --- a/src/core/repository-failure-status.ts +++ b/src/core/repository-failure-status.ts @@ -1,5 +1,10 @@ +export function isQuotaFailure(code?: string): boolean { + return code === "quota_exceeded"; +} + /** Persisted repository failures need the same client status as the original error. */ export function repositoryFailureStatus(code?: string): number { + if (isQuotaFailure(code)) return 402; switch (code) { case "repo_not_found": case "branch_not_found": case "commit_not_found": return 404; case "repo_not_accessible": case "repository_not_accessible": return 403; diff --git a/src/core/source/GitHubStream.ts b/src/core/source/GitHubStream.ts index 303e276..42d47e0 100644 --- a/src/core/source/GitHubStream.ts +++ b/src/core/source/GitHubStream.ts @@ -139,10 +139,14 @@ export default class GitHubStream extends GitHubBase { this.data.commit || "HEAD", filePath ); - logger.debug("downloading via raw URL (LFS)", { url }); + logger.debug("downloading via raw URL (LFS)"); const assertDownloadUrl = (value: URL) => { if (value.protocol !== "https:" || value.username || value.password || value.port || !["raw.githubusercontent.com", "media.githubusercontent.com"].includes(value.hostname)) { + logger.warn("rejected upstream download destination", { + hostname: value.hostname, protocol: value.protocol, + reason: value.protocol !== "https:" ? "protocol" : value.username || value.password ? "credentials" : value.port ? "port" : "host", + }); throw new AnonymousError("upstream_error", { httpStatus: 502 }); } }; diff --git a/src/queue/index.ts b/src/queue/index.ts index 76e7bbb..738ca21 100644 --- a/src/queue/index.ts +++ b/src/queue/index.ts @@ -1,3 +1,4 @@ +import { isQuotaFailure } from "../core/repository-failure-status"; import UserModel from "../core/model/users/users.model"; import { JobsOptions, Queue, Worker } from "bullmq"; import config from "../config"; @@ -313,13 +314,13 @@ export function startWorker() { downloadWorker.on("failed", async (job, err) => { if (job) recordMetric("download", "failed", Date.now() - (job.processedOn || job.timestamp)); const repoId = job?.data?.repoId; - logger.error("download failed", { + logger[isQuotaFailure(err?.message) ? "warn" : "error"]("download failed", { ...serializeError(err), repoId, }); if (!repoId) return; if (job && typeof job.attemptsMade === "number" && job.opts?.attempts) { - if (job.attemptsMade < job.opts.attempts) return; + if (job.attemptsMade < job.opts.attempts && err?.name !== "UnrecoverableError") return; } await markErrorIfInFlight(repoId, err?.message || "preparation_failed"); }); diff --git a/src/queue/processes/downloadRepository.ts b/src/queue/processes/downloadRepository.ts index 85e36ee..49ac4a9 100644 --- a/src/queue/processes/downloadRepository.ts +++ b/src/queue/processes/downloadRepository.ts @@ -1,5 +1,6 @@ import { githubQuotaKey } from "../../core/github-token-context"; -import { SandboxedJob } from "bullmq"; +import { isQuotaFailure } from "../../core/repository-failure-status"; +import { SandboxedJob, UnrecoverableError } from "bullmq"; import { config } from "dotenv"; config(); import { notifyOwnerAccessProblem } from "../../core/owner-notifications"; @@ -144,7 +145,8 @@ export default async function (job: SandboxedJob) { throw error; } await notifyOwnerAccessProblem(repo.owner.id, error, { kind: "repository", id: String(repo.model._id) }); - logger.error("finished with error", { + const quotaDenied = error instanceof Error && isQuotaFailure(error.message); + logger[quotaDenied ? "warn" : "error"]("finished with error", { ...serializeError(error), repoId: job.data.repoId, url: `/api/repo/${job.data.repoId}`, @@ -161,6 +163,9 @@ export default async function (job: SandboxedJob) { url: `/api/repo/${job.data.repoId}`, }); } + // Permanent for this attempt, not for the repository: owner refresh + // performs admission again after capacity or the plan changes. + if (quotaDenied) throw new UnrecoverableError("quota_exceeded"); throw error; } finally { clearInterval(statusInterval); diff --git a/src/server/routes/github-app.ts b/src/server/routes/github-app.ts index 4bbd40c..bba1700 100644 --- a/src/server/routes/github-app.ts +++ b/src/server/routes/github-app.ts @@ -1,3 +1,4 @@ +import { createLogger } from "../../core/logger"; import { REPOSITORY_ID_COLLATION } from "../../core/repository-id"; import { notificationEmail, normalizeEmail, resetOwnerAccessAlerts } from "../../core/owner-notifications"; import { saveRegistrationEmail } from "../../core/registration-email"; @@ -14,7 +15,7 @@ import GistModel from "../../core/model/anonymizedGists/anonymizedGists.model"; import { getCredentialToken } from "../../core/credentials"; import { APP_PROVIDER, appError, appUserToken, AppInstallation, clearAppTokenCache, exchangeAppToken, githubRequest, GitHubRepositoryInfo, reconcileAppGrant, reconcileInstallation, installationURL, saveAppGrant, selectRepositoryAccess, userInstallations } from "../../core/github-app"; -import { getUser, handleError } from "./route-utils"; +import { explicitErrorStatus, getUser, handleError } from "./route-utils"; import { isDisabledAccount, safeAuthReturnTo } from "./auth-utils"; type Flow = { state: string; expires: number; ownerId?: string; returnTo: string; repository?: string; install?: boolean }; @@ -304,16 +305,22 @@ export function validWebhookSignature(body: Buffer, signature: unknown, secret: const expected = createHmac("sha256", secret).update(body).digest(); return timingSafeEqual(expected, Buffer.from(signature.slice(7), "hex")); } +const webhookLogger = createLogger("github-app-webhook"); export const githubAppWebhook = express.Router(); githubAppWebhook.post("/", express.raw({ type: "application/json", limit: "2mb" }), async (req, res) => { if (!config.GITHUB_APP_ENABLED || !Buffer.isBuffer(req.body) || !validWebhookSignature(req.body, req.headers["x-hub-signature-256"], config.GITHUB_APP_WEBHOOK_SECRET)) { return res.status(401).json({ error: "invalid_webhook_signature" }); } + let phase = "parse"; + let action: string | undefined; + const event = req.headers["x-github-event"]; try { const body = JSON.parse(req.body.toString("utf8")); - const event = req.headers["x-github-event"]; + action = typeof body.action === "string" && /^[a-z_]{1,64}$/.test(body.action) ? body.action : undefined; if (event === "github_app_authorization" && body.action === "revoked" && body.sender?.id) { + phase = "find_grant_owners"; const users = await UserModel.find({ "externalIDs.github": String(body.sender.id) }).select("_id").lean(); + phase = "reconcile_grant"; for (const user of users) await reconcileAppGrant(String(user._id)); } if (["installation", "installation_repositories"].includes(String(event)) && Number.isSafeInteger(body.installation?.id) && String(body.installation.app_id) === config.GITHUB_APP_ID) { @@ -322,11 +329,25 @@ githubAppWebhook.post("/", express.raw({ type: "application/json", limit: "2mb" // to reactivate an installation. Duplicate events are safe to replay. const filter = { appId: config.GITHUB_APP_ID, installationId: installation.id }; const revision = randomUUID(); + phase = "persist_installation_block"; await InstallationModel.updateOne(filter, { $set: { blocked: true, revision, reconciliationPending: body.action !== "deleted" } }, { upsert: true }); clearAppTokenCache(); + phase = "reconcile_installation"; if (body.action !== "deleted") await reconcileInstallation(installation.id, revision); } return res.status(204).end(); - } catch { return res.status(503).json({ error: "webhook_processing_failed" }); } + } catch (error) { + const delivery = req.headers["x-github-delivery"]; + // Never log payloads, signatures, upstream bodies or URLs (which may contain tokens). + webhookLogger.error("webhook processing failed", { + phase, action, + event: typeof event === "string" && /^[a-z_]{1,64}$/.test(event) ? event : undefined, + deliveryId: typeof delivery === "string" && /^[a-f0-9-]{1,64}$/i.test(delivery) ? delivery : undefined, + requestId: res.getHeader("X-Request-Id"), + upstreamStatus: explicitErrorStatus(error), + errorClass: error instanceof Error ? error.constructor.name : "UnknownError", + }); + return res.status(503).json({ error: "webhook_processing_failed" }); + } }); diff --git a/src/server/routes/repository-public.ts b/src/server/routes/repository-public.ts index 375db07..381ab52 100644 --- a/src/server/routes/repository-public.ts +++ b/src/server/routes/repository-public.ts @@ -1,7 +1,7 @@ import { hasVisibleSubmodules } from "../../core/file-submodules"; import { FILE_LISTING_LIMIT } from "../../core/file-tree"; import { folderCounts as scopedFolderCounts } from "../../core/folder-counts"; -import { repositoryFailureStatus } from "../../core/repository-failure-status"; +import { isQuotaFailure, repositoryFailureStatus } from "../../core/repository-failure-status"; import { anonymizePath } from "../../core/anonymize-utils"; import * as express from "express"; import config from "../../config"; @@ -338,6 +338,7 @@ router.get( if (repo.status != "ready") { if ( repo.status != RepositoryStatus.QUEUE && + !isQuotaFailure(repo.model.statusMessage) && repo.statusActivityDate < fiveMinuteAgo ) { await repo.updateStatus(RepositoryStatus.PREPARING); diff --git a/src/server/routes/route-utils.ts b/src/server/routes/route-utils.ts index 3355176..9010635 100644 --- a/src/server/routes/route-utils.ts +++ b/src/server/routes/route-utils.ts @@ -176,6 +176,14 @@ function ensureUrl( if (origin) payload.url = origin; } +/** Use the same validated upstream status for response mapping and log severity. */ +export function explicitErrorStatus(error: unknown): number | undefined { + const value = error as { httpStatus?: number; status?: number; response?: { statusCode?: number }; + $metadata?: { httpStatusCode?: number } } | null; + return [value?.httpStatus, value?.status, value?.response?.statusCode, value?.$metadata?.httpStatusCode] + .find(status => Number.isInteger(status) && status! >= 400 && status! <= 599); +} + // eslint-disable-next-line @typescript-eslint/no-explicit-any function printError(error: any, req?: express.Request) { if (error instanceof AnonymousError) { @@ -189,7 +197,7 @@ function printError(error: any, req?: express.Request) { // 4xx are expected client errors (not_found, expired, not_connected) — // route them to warn so the admin Errors page can split server faults // (5xx) from client misuse (4xx) cleanly. - const status = error.httpStatus; + const status = explicitErrorStatus(error); if (typeof status === "number" && status >= 400 && status < 500) { logger.warn(summary, payload); } else { @@ -198,7 +206,8 @@ function printError(error: any, req?: express.Request) { } else if (error instanceof HTTPError) { const payload: Record = serializeError(error); ensureUrl(payload, req); - logger.error(error.code || error.name || "HTTPError", payload); + const status = explicitErrorStatus(error); + logger[status && status < 500 ? "warn" : "error"](error.code || error.name || "HTTPError", payload); } else { // Unhandled errors: use the error class name (SyntaxError, TypeError, // RangeError, ...) as the summary so the admin page shows @@ -208,7 +217,7 @@ function printError(error: any, req?: express.Request) { typeof serialized.status !== "number" && typeof serialized.httpStatus !== "number" ) { - serialized.httpStatus = 500; + serialized.httpStatus = explicitErrorStatus(error) ?? 500; } ensureUrl(serialized, req); const summary = @@ -216,7 +225,8 @@ function printError(error: any, req?: express.Request) { ((error as { name?: string }).name || (error as { message?: string }).message)) || "UnhandledError"; - logger.error(summary, serialized); + const status = explicitErrorStatus(error); + logger[status && status < 500 ? "warn" : "error"](summary, serialized); } } @@ -235,8 +245,7 @@ export function handleError( errorCode = String(error); } let status = 500; - const explicitStatus = [error?.httpStatus, error?.status, error?.response?.statusCode, - error?.$metadata?.httpStatusCode].find(value => Number.isInteger(value) && value >= 400 && value <= 599); + const explicitStatus = explicitErrorStatus(error); if (explicitStatus !== undefined) { status = explicitStatus; } else if ( diff --git a/src/server/routes/user.ts b/src/server/routes/user.ts index c163ef6..e25d331 100644 --- a/src/server/routes/user.ts +++ b/src/server/routes/user.ts @@ -1,3 +1,4 @@ +import AnonymousError from "../../core/AnonymousError"; import { BillingQuota, BillingDraft } from "../../core/billing/models"; import { cancelAccountBilling } from "../../core/billing/stripe"; import { allowance } from "../../core/billing/quotas"; @@ -9,7 +10,7 @@ import CredentialModel from "../../core/model/credentials/credentials.model"; import * as express from "express"; import config from "../../config"; import { ensureAuthenticated } from "./connection"; -import { handleError, getUser, isOwnerOrAdmin } from "./route-utils"; +import { explicitErrorStatus, handleError, getUser, isOwnerOrAdmin } from "./route-utils"; import UserModel from "../../core/model/users/users.model"; import AnonymizedRepositoryModel from "../../core/model/anonymizedRepositories/anonymizedRepositories.model"; import AnonymizedPullRequestModel from "../../core/model/anonymizedPullRequests/anonymizedPullRequests.model"; @@ -352,9 +353,14 @@ router.get( } ); -async function getAllRepositories(user: User, force: boolean) { - const repos = await user.getGitHubRepositories({ - force, +export async function getAllRepositories(user: User, force: boolean) { + const repos = await user.getGitHubRepositories({ force }).catch(error => { + // Normalize confirmed GitHub credential denial only at this provider boundary. + // Preserve App-specific reconnect errors and all permission/rate-limit failures. + if (!(error instanceof AnonymousError) && explicitErrorStatus(error) === 401) { + throw new AnonymousError("token_expired", { httpStatus: 401, cause: error }); + } + throw error; }); return repos.map((x) => { return { diff --git a/test/credential-webhook-diagnostics.test.js b/test/credential-webhook-diagnostics.test.js new file mode 100644 index 0000000..3fe6bab --- /dev/null +++ b/test/credential-webhook-diagnostics.test.js @@ -0,0 +1,67 @@ +const { expect } = require("chai"); +const { createHmac } = require("crypto"); +require("ts-node/register/transpile-only"); +const { handleError } = require("../src/server/routes/route-utils"); +const { getAllRepositories } = require("../src/server/routes/user"); +const { githubAppWebhook } = require("../src/server/routes/github-app"); +const config = require("../src/config").default; +const Installation = require("../src/core/model/github-installation").default; +const AnonymousError = require("../src/core/AnonymousError").default; + +function response() { + return { headersSent: false, status(code) { this.code = code; return this; }, + json(body) { this.body = body; return this; }, getHeader() { return "request-id"; } }; +} + +describe("credential and webhook failure diagnostics", () => { + const restores = []; + function stub(object, key, value) { const original = object[key]; object[key] = value; restores.push(() => { object[key] = original; }); } + afterEach(() => restores.splice(0).reverse().forEach(restore => restore())); + it("logs generic upstream 401 as warning while retaining server errors", () => { + const warnings = [], errors = []; + stub(console, "warn", (...args) => warnings.push(args)); + stub(console, "error", (...args) => errors.push(args)); + handleError(Object.assign(new Error("denied"), { status: 401 }), response()); + handleError(Object.assign(new Error("failed"), { response: { statusCode: 503 } }), response()); + expect(warnings).to.have.length(1); expect(errors).to.have.length(1); + expect(errors.flat().join(" ")).to.include('"httpStatus":503'); + }); + it("logs got HTTP401 as warning with its actual status", () => { + const warnings = []; stub(console, "warn", (...args) => warnings.push(args)); + const failure = Object.assign(new Error("Unauthorized"), { response: { statusCode: 401 } }); + Object.setPrototypeOf(failure, require("got").HTTPError.prototype); + const res = response(); handleError(failure, res); + expect(warnings).to.have.length(1); expect(res.code).to.equal(401); + }); + for (const status of [401, 403, 429, 503]) it(`normalizes only confirmed discovery 401 (${status})`, async () => { + const failure = Object.assign(new Error("private provider details"), { status }); + let received; + try { await getAllRepositories({ getGitHubRepositories: async () => { throw failure; } }, true); } + catch (error) { received = error; } + if (status === 401) { + expect(received).to.be.instanceOf(AnonymousError); + expect(received.message).to.equal("token_expired"); expect(received.httpStatus).to.equal(401); + } else expect(received).to.equal(failure); + }); + it("preserves App-specific reconnect errors", async () => { + const failure = new AnonymousError("github_app_reconnect_required", { httpStatus: 401 }); + try { await getAllRepositories({ getGitHubRepositories: async () => { throw failure; } }, true); throw Error("expected denial"); } + catch (error) { expect(error).to.equal(failure); } + }); + it("logs signed webhook persistence failures without secrets and still returns 503", async () => { + stub(config, "GITHUB_APP_ENABLED", true); stub(config, "GITHUB_APP_ID", "123"); + stub(config, "GITHUB_APP_WEBHOOK_SECRET", "test-secret"); + const messages = []; stub(console, "error", (...args) => messages.push(args)); + stub(Installation, "updateOne", async () => { throw new Error("secret-token https://private/?token=secret"); }); + const body = Buffer.from(JSON.stringify({ action: "added", installation: { id: 4, app_id: 123 }, private: "private-body" })); + const req = { body, headers: { "x-github-event": "installation_repositories", "x-github-delivery": "abcd-1234", + "x-hub-signature-256": "sha256=" + createHmac("sha256", "test-secret").update(body).digest("hex") } }; + const handler = githubAppWebhook.stack.find(layer => layer.route).route.stack.at(-1).handle; + const res = response(); await handler(req, res); + expect(res.code).to.equal(503); expect(res.body.error).to.equal("webhook_processing_failed"); + const text = JSON.stringify(messages); + expect(text).to.include("persist_installation_block").and.include("abcd-1234"); + expect(text).not.to.include("secret-token"); expect(text).not.to.include("private-body"); expect(text).not.to.include("https://"); + expect(text).not.to.include(req.headers["x-hub-signature-256"]); + }); +}); diff --git a/test/fixtures/quota-download.worker.cjs b/test/fixtures/quota-download.worker.cjs new file mode 100644 index 0000000..e274efc --- /dev/null +++ b/test/fixtures/quota-download.worker.cjs @@ -0,0 +1,20 @@ +/* global require, module */ +require("ts-node/register/transpile-only"); +const db = require("../../src/server/database"); +const gh = require("../../src/core/GitHubUtils"); +const notifications = require("../../src/core/owner-notifications"); +const AnonymousError = require("../../src/core/AnonymousError").default; +const processDownload = require("../../src/queue/processes/downloadRepository").default; +db.connect = async () => {}; +gh.getToken = async () => "test-token"; +gh.getRedisGateResetAt = async () => 0; +notifications.notifyOwnerAccessProblem = async () => {}; +module.exports = async job => { + db.getRepository = async () => ({ + status: "preparing", owner: { id: "owner" }, model: {}, + async updateStatus(status) { this.status = status; }, + async resetSate() {}, + async anonymize() { throw new AnonymousError(job.data.failure, { httpStatus: job.data.failure === "quota_exceeded" ? 402 : 502 }); }, + }); + return processDownload(job); +}; diff --git a/test/folder-file-security.test.js b/test/folder-file-security.test.js index bf2b7e5..e6dee48 100644 --- a/test/folder-file-security.test.js +++ b/test/folder-file-security.test.js @@ -51,13 +51,18 @@ describe("folder file endpoint disclosure", function () { } finally { await close(server); } }); it("blocks a real HTTP redirect to a GitHub tree page before fetching its HTML", async () => { - const server = await listen(http.createServer((_req, res) => { res.writeHead(302, { location: "https://github.com/example/private-repo/tree/main/folder" }); res.end(); })); + const warnings = []; + stub(console, "warn", (...args) => warnings.push(args.join(" "))); + const server = await listen(http.createServer((_req, res) => { res.writeHead(302, { location: "https://github.com/example/private-repo/tree/main/folder?token=secret-query" }); res.end(); })); const original = got.stream; stub(got, "stream", (_url, options) => original(`http://127.0.0.1:${server.address().port}/raw`, options)); const source = new GitHubStream({ repoId: "test", organization: "example", repoName: "repo", commit: "main", getToken: () => "" }); try { const error = await collect(await source.downloadWithFallback("", "tree-sha", "folder")).catch(error => error); expect(error.message).to.equal("upstream_error"); + expect(warnings.join(" ")).to.include("github.com").and.include("host"); + expect(warnings.join(" ")).not.to.include("private-repo"); + expect(warnings.join(" ")).not.to.include("secret-query"); } finally { await close(server); } }); it("rejects a successful web-page response before emitting any bytes", async () => { diff --git a/test/large-repository-tree.test.js b/test/large-repository-tree.test.js index 8d6e379..91877ae 100644 --- a/test/large-repository-tree.test.js +++ b/test/large-repository-tree.test.js @@ -451,6 +451,43 @@ const uri = process.env.PERFORMANCE_MONGO_URI; expect(files[0]).not.to.be.instanceOf(mongoose.Document); }); + it("bounds a direct directory listing to its immediate children in the path index", async () => { + const repo = new Repository(await repository({ status: "ready", treeGeneration: "old" })); + await rows(12000, i => ({ path: `src/deep/${i}` })); + await rows(9, i => ({ path: "src", name: `direct${i}.txt` })); + const find = File.find; let query; + File.find = function (...args) { query = find.apply(this, args); return query; }; + try { + const files = await repo.files({ recursive: false, path: "src" }); + expect(files).to.have.length(9); + const plan = await query.clone().explain("executionStats"); + expect(plan.executionStats.totalKeysExamined).to.be.at.most(10); + expect(plan.executionStats.totalDocsExamined).to.equal(9); + } finally { File.find = find; } + }); + + for (const generation of ["old", undefined]) { + it(`uses literal case-sensitive directory paths for ${generation || "legacy"} listings`, async () => { + const repo = new Repository(await repository({ status: "ready", treeGeneration: generation })); + const paths = ["", "猫.[x] +$", "Src", "src", "src\n", "src/deep"]; + await File.collection.insertMany(paths.flatMap((directory, i) => [ + { repoId: "large", ...(generation ? { treeGeneration: generation } : {}), + path: directory, name: `wanted${i}`, size: 1, sha: "b".repeat(40) }, + { repoId: "large", treeGeneration: "staged", path: directory, name: "hidden", size: 1 }, + ])); + // Explicit null is not a legacy snapshot, even though Mongo indexes + // missing and null fields alike. + await File.collection.insertOne({ repoId: "large", treeGeneration: null, path: "src", name: "null-generation", size: 1 }); + for (const [i, directory] of paths.entries()) { + const files = await repo.files({ recursive: false, path: directory }); + expect(files.map(file => file.name)).to.deep.equal([`wanted${i}`]); + } + expect((await repo.files({ recursive: false })).map(file => file.name)).to.deep.equal(["wanted0"]); + expect((await repo.files({ recursive: true, path: "src" })).map(file => file.name).sort()) + .to.deep.equal(["wanted3", "wanted4", "wanted5"]); + }); + } + it("preserves rows that stop matching the deletion filter after selection", async () => { await rows(2); let checks = 0; diff --git a/test/quota-job-failures.test.js b/test/quota-job-failures.test.js new file mode 100644 index 0000000..4843ce8 --- /dev/null +++ b/test/quota-job-failures.test.js @@ -0,0 +1,68 @@ +const { expect } = require("chai"); +require("ts-node/register/transpile-only"); +const db = require("../src/server/database"); +const gh = require("../src/core/GitHubUtils"); +const notifications = require("../src/core/owner-notifications"); +const utils = require("../src/server/routes/route-utils"); +const queue = require("../src/queue"); +const AnonymousError = require("../src/core/AnonymousError").default; +const { repositoryFailureStatus } = require("../src/core/repository-failure-status"); +const { UnrecoverableError } = require("bullmq"); + +describe("quota job failures", () => { + const restores = []; + function stub(object, key, value) { + const previous = object[key]; object[key] = value; restores.push(() => { object[key] = previous; }); + } + afterEach(() => { while (restores.length) restores.pop()(); }); + function repository() { + return { repoId: "quota-test", status: "error", statusActivityDate: new Date(0), + owner: { id: "owner" }, options: {}, model: { owner: "owner", statusMessage: "quota_exceeded" }, + assertNotArchived() {}, + async updateStatus(status, message) { this.status = status; this.model.statusMessage = message; }, + async resetSate() {}, + }; + } + it("maps stored quota errors to 402, leaving unknown failures as 500", () => { + expect(repositoryFailureStatus("quota_exceeded")).to.equal(402); + expect(repositoryFailureStatus("unexpected")).to.equal(500); + }); + it("does not let public reads retry an old quota denial", async () => { + const repo = repository(); let enqueues = 0; let writes = 0; + stub(utils, "getRepo", async () => repo); + stub(utils, "getUser", async () => { throw Error("not_connected"); }); + stub(queue, "downloadQueue", { add: async () => { enqueues++; } }); + stub(repo, "updateStatus", async () => { writes++; }); + const route = require("../src/server/routes/repository-public").default.stack.find(l => l.route?.path === "/:repoId/options").route.stack[0].handle; + for (let i = 0; i < 2; i++) { + let status, body; + await route({}, { header() {}, status(value) { status = value; return this; }, json(value) { body = value; } }); + expect(status).to.equal(402); expect(body).to.deep.equal({ error: "quota_exceeded" }); + } + expect(enqueues).to.equal(0); expect(writes).to.equal(0); + }); + for (const code of ["quota_exceeded", "github_unavailable"]) { + it(`classifies ${code} at the worker boundary and preserves its status message`, async () => { + const repo = repository(); const failure = new AnonymousError(code, { httpStatus: code === "quota_exceeded" ? 402 : 502 }); + stub(db, "connect", async () => {}); stub(db, "getRepository", async () => repo); + stub(gh, "getToken", async () => "test-token"); stub(gh, "getRedisGateResetAt", async () => 0); + stub(notifications, "notifyOwnerAccessProblem", async () => {}); + repo.anonymize = async () => { throw failure; }; + const error = await require("../src/queue/processes/downloadRepository").default({ data: { repoId: repo.repoId }, updateProgress() {} }).catch(error => error); + expect(repo.status).to.equal("error"); expect(repo.model.statusMessage).to.equal(code); + if (code === "quota_exceeded") expect(error).to.be.instanceOf(UnrecoverableError); + else expect(error).to.equal(failure); + }); + } + it("allows an authenticated owner to explicitly retry quota admission", async () => { + const repo = repository(); let refreshes = 0; + repo.refresh = async () => { refreshes++; }; + stub(utils, "getRepo", async () => repo); + stub(utils, "getUser", async () => ({ id: "owner", model: { id: "owner" } })); + stub(notifications, "resetOwnerAccessAlerts", async () => {}); + const route = require("../src/server/routes/repository-private").default.stack.find(l => l.route?.path === "/:repoId/refresh").route.stack[0].handle; + let body; + await route({}, { json(value) { body = value; } }); + expect(refreshes).to.equal(1); expect(body).to.deep.equal({ status: "error" }); + }); +}); diff --git a/test/quota-job-redis-integration.test.js b/test/quota-job-redis-integration.test.js new file mode 100644 index 0000000..bb926b3 --- /dev/null +++ b/test/quota-job-redis-integration.test.js @@ -0,0 +1,40 @@ +const { expect } = require("chai"); +const { Queue, QueueEvents, Worker } = require("bullmq"); +const { randomUUID } = require("node:crypto"); +const path = require("node:path"); +const port = Number(process.env.PERFORMANCE_REDIS_PORT); + +(port ? describe : describe.skip)("sandboxed quota retry policy", function () { + this.timeout(30000); + let queue, events, worker; + before(async () => { + const name = `quota-test-${randomUUID()}`; + const connection = { host: "127.0.0.1", port }; + queue = new Queue(name, { connection }); + events = new QueueEvents(name, { connection }); + await events.waitUntilReady(); + worker = new Worker(name, path.resolve(__dirname, "fixtures/quota-download.worker.cjs"), { connection }); + await worker.waitUntilReady(); + }); + after(async () => { + await worker?.close(); + if (queue) await queue.obliterate({ force: true }); + await events?.close(); await queue?.close(); + }); + for (const [failure, attempts] of [["quota_exceeded", 1], ["github_unavailable", 3]]) { + it(`${failure} executes ${attempts} time(s) across the sandbox boundary`, async () => { + let executions = 0; + const active = () => { executions++; }; + worker.on("active", active); + try { + const job = await queue.add("download", { repoId: failure, failure }, { attempts: 3 }); + const error = await job.waitUntilFinished(events, 20000).catch(error => error); + expect(error).to.be.instanceOf(Error); expect(error.message).to.equal(failure); + const failed = await queue.getJob(job.id); + expect(await failed.getState()).to.equal("failed"); + expect(failed.attemptsMade).to.equal(attempts); + expect(executions).to.equal(attempts); + } finally { worker.off("active", active); } + }); + } +});