Merge pull request #955 from tdurieux/fix/remaining-production-findings

fix: stop quota retry loops, narrow folder queries and diagnose upstream failures
This commit is contained in:
Thomas Durieux authored and GitHub committed 2026-11-14 18:22:19 +01:00
commit 88d0d0cd12
15 files changed
+312 -21

No files matched your search

+24 -3
View File
@@ -1,3 +1,4 @@
import { createLogger } from "../../core/logger";
import { REPOSITORY_ID_COLLATION } from "../../core/repository-id";
import { notificationEmail, normalizeEmail, resetOwnerAccessAlerts } from "../../core/owner-notifications";
import { saveRegistrationEmail } from "../../core/registration-email";
@@ -14,7 +15,7 @@ import GistModel from "../../core/model/anonymizedGists/anonymizedGists.model";
import { getCredentialToken } from "../../core/credentials";
import { APP_PROVIDER, appError, appUserToken, AppInstallation, clearAppTokenCache, exchangeAppToken,
githubRequest, GitHubRepositoryInfo, reconcileAppGrant, reconcileInstallation, installationURL, saveAppGrant, selectRepositoryAccess, userInstallations } from "../../core/github-app";
import { getUser, handleError } from "./route-utils";
import { explicitErrorStatus, getUser, handleError } from "./route-utils";
import { isDisabledAccount, safeAuthReturnTo } from "./auth-utils";
type Flow = { state: string; expires: number; ownerId?: string; returnTo: string; repository?: string; install?: boolean };
@@ -304,16 +305,22 @@ export function validWebhookSignature(body: Buffer, signature: unknown, secret:
const expected = createHmac("sha256", secret).update(body).digest();
return timingSafeEqual(expected, Buffer.from(signature.slice(7), "hex"));
}
const webhookLogger = createLogger("github-app-webhook");
export const githubAppWebhook = express.Router();
githubAppWebhook.post("/", express.raw({ type: "application/json", limit: "2mb" }), async (req, res) => {
if (!config.GITHUB_APP_ENABLED || !Buffer.isBuffer(req.body) || !validWebhookSignature(req.body, req.headers["x-hub-signature-256"], config.GITHUB_APP_WEBHOOK_SECRET)) {
return res.status(401).json({ error: "invalid_webhook_signature" });
}
let phase = "parse";
let action: string | undefined;
const event = req.headers["x-github-event"];
try {
const body = JSON.parse(req.body.toString("utf8"));
const event = req.headers["x-github-event"];
action = typeof body.action === "string" && /^[a-z_]{1,64}$/.test(body.action) ? body.action : undefined;
if (event === "github_app_authorization" && body.action === "revoked" && body.sender?.id) {
phase = "find_grant_owners";
const users = await UserModel.find({ "externalIDs.github": String(body.sender.id) }).select("_id").lean();
phase = "reconcile_grant";
for (const user of users) await reconcileAppGrant(String(user._id));
}
if (["installation", "installation_repositories"].includes(String(event)) && Number.isSafeInteger(body.installation?.id) && String(body.installation.app_id) === config.GITHUB_APP_ID) {
@@ -322,11 +329,25 @@ githubAppWebhook.post("/", express.raw({ type: "application/json", limit: "2mb"
// to reactivate an installation. Duplicate events are safe to replay.
const filter = { appId: config.GITHUB_APP_ID, installationId: installation.id };
const revision = randomUUID();
phase = "persist_installation_block";
await InstallationModel.updateOne(filter, { $set: { blocked: true, revision,
reconciliationPending: body.action !== "deleted" } }, { upsert: true });
clearAppTokenCache();
phase = "reconcile_installation";
if (body.action !== "deleted") await reconcileInstallation(installation.id, revision);
}
return res.status(204).end();
} catch { return res.status(503).json({ error: "webhook_processing_failed" }); }
} catch (error) {
const delivery = req.headers["x-github-delivery"];
// Never log payloads, signatures, upstream bodies or URLs (which may contain tokens).
webhookLogger.error("webhook processing failed", {
phase, action,
event: typeof event === "string" && /^[a-z_]{1,64}$/.test(event) ? event : undefined,
deliveryId: typeof delivery === "string" && /^[a-f0-9-]{1,64}$/i.test(delivery) ? delivery : undefined,
requestId: res.getHeader("X-Request-Id"),
upstreamStatus: explicitErrorStatus(error),
errorClass: error instanceof Error ? error.constructor.name : "UnknownError",
});
return res.status(503).json({ error: "webhook_processing_failed" });
}
});
+2 -1
View File
@@ -1,7 +1,7 @@
import { hasVisibleSubmodules } from "../../core/file-submodules";
import { FILE_LISTING_LIMIT } from "../../core/file-tree";
import { folderCounts as scopedFolderCounts } from "../../core/folder-counts";
import { repositoryFailureStatus } from "../../core/repository-failure-status";
import { isQuotaFailure, repositoryFailureStatus } from "../../core/repository-failure-status";
import { anonymizePath } from "../../core/anonymize-utils";
import * as express from "express";
import config from "../../config";
@@ -338,6 +338,7 @@ router.get(
if (repo.status != "ready") {
if (
repo.status != RepositoryStatus.QUEUE &&
!isQuotaFailure(repo.model.statusMessage) &&
repo.statusActivityDate < fiveMinuteAgo
) {
await repo.updateStatus(RepositoryStatus.PREPARING);
+15 -6
View File
@@ -176,6 +176,14 @@ function ensureUrl(
if (origin) payload.url = origin;
}
/** Use the same validated upstream status for response mapping and log severity. */
export function explicitErrorStatus(error: unknown): number | undefined {
const value = error as { httpStatus?: number; status?: number; response?: { statusCode?: number };
$metadata?: { httpStatusCode?: number } } | null;
return [value?.httpStatus, value?.status, value?.response?.statusCode, value?.$metadata?.httpStatusCode]
.find(status => Number.isInteger(status) && status! >= 400 && status! <= 599);
}
// eslint-disable-next-line @typescript-eslint/no-explicit-any
function printError(error: any, req?: express.Request) {
if (error instanceof AnonymousError) {
@@ -189,7 +197,7 @@ function printError(error: any, req?: express.Request) {
// 4xx are expected client errors (not_found, expired, not_connected) —
// route them to warn so the admin Errors page can split server faults
// (5xx) from client misuse (4xx) cleanly.
const status = error.httpStatus;
const status = explicitErrorStatus(error);
if (typeof status === "number" && status >= 400 && status < 500) {
logger.warn(summary, payload);
} else {
@@ -198,7 +206,8 @@ function printError(error: any, req?: express.Request) {
} else if (error instanceof HTTPError) {
const payload: Record<string, unknown> = serializeError(error);
ensureUrl(payload, req);
logger.error(error.code || error.name || "HTTPError", payload);
const status = explicitErrorStatus(error);
logger[status && status < 500 ? "warn" : "error"](error.code || error.name || "HTTPError", payload);
} else {
// Unhandled errors: use the error class name (SyntaxError, TypeError,
// RangeError, ...) as the summary so the admin page shows
@@ -208,7 +217,7 @@ function printError(error: any, req?: express.Request) {
typeof serialized.status !== "number" &&
typeof serialized.httpStatus !== "number"
) {
serialized.httpStatus = 500;
serialized.httpStatus = explicitErrorStatus(error) ?? 500;
}
ensureUrl(serialized, req);
const summary =
@@ -216,7 +225,8 @@ function printError(error: any, req?: express.Request) {
((error as { name?: string }).name ||
(error as { message?: string }).message)) ||
"UnhandledError";
logger.error(summary, serialized);
const status = explicitErrorStatus(error);
logger[status && status < 500 ? "warn" : "error"](summary, serialized);
}
}
@@ -235,8 +245,7 @@ export function handleError(
errorCode = String(error);
}
let status = 500;
const explicitStatus = [error?.httpStatus, error?.status, error?.response?.statusCode,
error?.$metadata?.httpStatusCode].find(value => Number.isInteger(value) && value >= 400 && value <= 599);
const explicitStatus = explicitErrorStatus(error);
if (explicitStatus !== undefined) {
status = explicitStatus;
} else if (
+10 -4
View File
@@ -1,3 +1,4 @@
import AnonymousError from "../../core/AnonymousError";
import { BillingQuota, BillingDraft } from "../../core/billing/models";
import { cancelAccountBilling } from "../../core/billing/stripe";
import { allowance } from "../../core/billing/quotas";
@@ -9,7 +10,7 @@ import CredentialModel from "../../core/model/credentials/credentials.model";
import * as express from "express";
import config from "../../config";
import { ensureAuthenticated } from "./connection";
import { handleError, getUser, isOwnerOrAdmin } from "./route-utils";
import { explicitErrorStatus, handleError, getUser, isOwnerOrAdmin } from "./route-utils";
import UserModel from "../../core/model/users/users.model";
import AnonymizedRepositoryModel from "../../core/model/anonymizedRepositories/anonymizedRepositories.model";
import AnonymizedPullRequestModel from "../../core/model/anonymizedPullRequests/anonymizedPullRequests.model";
@@ -352,9 +353,14 @@ router.get(
}
);
async function getAllRepositories(user: User, force: boolean) {
const repos = await user.getGitHubRepositories({
force,
export async function getAllRepositories(user: User, force: boolean) {
const repos = await user.getGitHubRepositories({ force }).catch(error => {
// Normalize confirmed GitHub credential denial only at this provider boundary.
// Preserve App-specific reconnect errors and all permission/rate-limit failures.
if (!(error instanceof AnonymousError) && explicitErrorStatus(error) === 401) {
throw new AnonymousError("token_expired", { httpStatus: 401, cause: error });
}
throw error;
});
return repos.map((x) => {
return {