Merge pull request #956 from tdurieux/t3/oss-logging-infrastructure

feat: add private structured logging and investigation dashboards
This commit is contained in:
Thomas Durieux authored and GitHub committed 2026-11-14 18:22:32 +01:00
commit 8e57cc3d0e
40 files changed
+1835 -75

No files matched your search

+2 -1
View File
@@ -1,3 +1,4 @@
import { jobTrace } from "./log-context";
import { REPOSITORY_ID_COLLATION } from "./repository-id";
import { reserveRepository } from "./billing/quotas";
import { hasAutomaticUpdates } from "./billing/expiration";
@@ -856,7 +857,7 @@ export default class Repository {
const current = await AnonymizedRepositoryModel.exists({ _id: this.model._id, ...this.refreshFilter() });
if (!current) throw new AnonymousError("invalid_status", { httpStatus: 409 });
}
await downloadQueue.add(this.repoId, { repoId: this.repoId }, {
await downloadQueue.add(this.repoId, { repoId: this.repoId, ...jobTrace() }, {
jobId: `repo-${this.repoId}`,
attempts: 3,
});
+11
View File
@@ -0,0 +1,11 @@
import { AsyncLocalStorage } from "async_hooks";
import { randomUUID } from "crypto";
export interface LogContext { requestId?: string; jobId?: string; attempt?: number; service?: string }
export const logContext = new AsyncLocalStorage<LogContext>();
export const validTraceId = (value: unknown): value is string => typeof value === "string" &&
/^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$/.test(value);
export function jobTrace() {
const requestId = logContext.getStore()?.requestId;
return { logJobId: randomUUID(), ...(requestId ? { requestId } : {}) };
}
+15
View File
@@ -0,0 +1,15 @@
import { readFile } from "fs/promises";
/** Container usage includes sandbox workers and page cache; process RSS does not. */
export async function logMemory() {
const rss = process.memoryUsage().rss;
for (const [usage, limit] of [
["/sys/fs/cgroup/memory.current", "/sys/fs/cgroup/memory.max"],
["/sys/fs/cgroup/memory/memory.usage_in_bytes", "/sys/fs/cgroup/memory/memory.limit_in_bytes"],
]) {
const [used, max] = await Promise.all([usage, limit].map(path => readFile(path, "utf8").then(Number).catch(() => NaN)));
if (Number.isFinite(used) && max > 0 && max < Number.MAX_SAFE_INTEGER) {
return { rss, memoryLimitBytes: max, memoryRatio: used / max };
}
}
return { rss };
}
+72
View File
@@ -0,0 +1,72 @@
import { LOG_MODULES, ERROR_CODES, LOG_EVENTS } from "./log-schema";
import { randomUUID } from "crypto";
import { logContext, validTraceId } from "./log-context";
// No arbitrary strings: upstream errors, paths and even innocuous-looking messages
// can contain a private repository name. Add diagnostics here deliberately.
const instance = randomUUID();
const allowed = (values: readonly string[], value: unknown): value is string => typeof value === "string" && values.includes(value);
const events: Record<string, string> = {
...LOG_EVENTS,
request: "request_completed", "slow request": "request_slow", "request interrupted": "request_cancelled",
"process fatal error": "process_fatal_error", "process started": "process_started",
"process heartbeat": "process_heartbeat", "job started": "job_started", "job completed": "job_completed",
"job failed": "job_failed", "job stalled": "job_stalled", "queue heartbeat": "queue_heartbeat",
};
const errorCodes = new Set([
...ERROR_CODES,
"token_refresh_failed", "quota_exceeded", "github_rate_limit_exceeded", "github_app_reconnect_required", "github_token_invalid",
"repository_expired", "repository_not_ready", "repository_not_found", "repository_job_cancelled",
"ECONNRESET", "ECONNREFUSED", "ETIMEDOUT", "ENOTFOUND", "EPIPE", "ABORT_ERR",
"upstream_error", "upstream_timeout", "invalid_signature", "webhook_processing_failed",
]);
const numbers = ["status", "httpStatus", "ms", "firstByteMs", "responseMs", "bytes", "bytesPerSecond",
"attempt", "rss", "memoryLimitBytes", "memoryRatio", "uptime", "activeRequests", "waiting", "active", "delayed", "failed",
"oldestWaitingMs", "droppedLogs", "loopP95Ms", "upstreamStatus"];
const stages = ["repository", "credential_resolution", "upstream", "cache_lookup", "cache_fill", "worker_wait", "anonymize", "streamer_headers"];
export function logRecord(level: string, module: string, args: unknown[]) {
const context = logContext.getStore() || {};
const details: Record<string, unknown> = {};
const keys = [...numbers, "requestId", "jobId", "code", "message", "outcome", "method", "route", "queue",
"stages", "reason", "phase", "event", "deliveryId", "hostname", "service"];
// Do not traverse arbitrary objects or invoke error getters while logging.
for (const arg of args.slice(0, 8)) if (arg && typeof arg === "object") {
for (const key of keys) {
try {
const property = Object.getOwnPropertyDescriptor(arg, key);
if (property && "value" in property) details[key] = property.value;
} catch { /* A hostile Proxy must not make logging throw. */ }
}
}
const first = typeof args[0] === "string" && args[0].length < 256 ? args[0] : "";
const event = (Object.hasOwn(events, first) ? events[first] : undefined) || (module.includes("webhook") ? "webhook_log" : "application_log");
const record: Record<string, unknown> = { timestamp: new Date().toISOString(), severity: level,
service: context.service || (allowed(["api", "streamer", "worker"], details.service) ? details.service : undefined) || (process.env.SERVICE_NAME?.toLowerCase() === "streamer" ? "streamer" : "api"),
instance, release: /^[a-f0-9]{7,40}$/.test(process.env.APP_RELEASE || "") ? process.env.APP_RELEASE : "unknown",
event, message: event, module: LOG_MODULES.includes(module) ? module : "application" };
for (const key of ["requestId", "jobId"] as const) {
const value = details[key] || context[key];
if (validTraceId(value)) record[key] = value;
}
if (Number.isSafeInteger(context.attempt)) record.attempt = context.attempt;
for (const key of numbers) if (typeof details[key] === "number" && Number.isFinite(details[key])) record[key] = details[key];
const code = [details.code, details.message].find(value => typeof value === "string" && errorCodes.has(value));
if (code) record.errorCode = code;
if (allowed(["protocol", "credentials", "port", "host"], details.reason)) record.reason = details.reason;
if (allowed(["parse", "find_grant_owners", "reconcile_grant", "persist_installation_block", "reconcile_installation"], details.phase)) record.phase = details.phase;
if (allowed(["github_app_authorization", "installation", "installation_repositories"], details.event)) record.webhookEvent = details.event;
if (validTraceId(details.deliveryId)) record.deliveryId = details.deliveryId;
if (allowed(["github.com", "api.github.com", "codeload.github.com", "raw.githubusercontent.com", "media.githubusercontent.com"], details.hostname)) record.upstreamHost = details.hostname;
else if (details.hostname) record.upstreamHost = "other";
if (allowed(["completed", "interrupted"], details.outcome)) record.outcome = details.outcome;
if (allowed(["GET", "HEAD", "POST", "PUT", "PATCH", "DELETE", "OPTIONS", "OTHER"], details.method)) record.method = details.method;
// Route labels are generated by request-monitoring, never from URLs in arbitrary errors.
if (module === "requests" && typeof details.route === "string") record.route = details.route;
if (allowed(["download", "remove", "cache"], details.queue)) record.queue = details.queue;
if (details.stages && typeof details.stages === "object") for (const key of stages) {
let value: unknown;
try { value = Object.getOwnPropertyDescriptor(details.stages, key)?.value; } catch { continue; }
if (typeof value === "number" && Number.isFinite(value)) record[`stage_${key}_ms`] = value;
}
return record;
}
+121
View File
@@ -0,0 +1,121 @@
// Reviewed literal diagnostics. Never extend this list from runtime data.
export const LOG_MODULES = ["admin", "anonymized-file", "auth", "billing", "dailyStats", "file-tree", "fs", "gh-stream", "gist", "github", "github-app-webhook", "owner-notifications", "process", "pull-request", "queue", "queue:cache", "queue:download", "queue:remove", "repository", "repository-public", "requests", "route", "route:repo", "s3", "schedule", "server", "streamer", "temporary-storage", "token-auth", "user", "zip-stream"];
export const ERROR_CODES = ["billing_address_missing", "billing_city_missing", "billing_country_missing", "billing_email_missing", "billing_missing", "billing_name_missing", "billing_zip_missing", "branch_not_found", "branch_not_specified", "cache_busy", "cache_miss", "cannot_coauthor_self", "commit_not_specified", "conf_end_date_invalid", "conf_end_date_missing", "conf_id_missing", "conf_id_used", "conf_name_missing", "conf_not_activated", "conf_not_found", "conf_start_date_invalid", "conf_start_date_missing", "conference_not_found", "connection_changed", "download_not_enabled", "expiration_limit_reached", "file_listing_too_large", "file_not_accessible", "file_not_found", "file_not_supported", "file_too_big", "folder_not_supported", "gistId_already_used", "gistId_not_specified", "gist_expired", "gist_not_found", "gist_not_ready", "github_app_access_required", "github_identity_mismatch", "github_rate_limit_exceeded", "github_user_not_found", "invalid_commit_format", "invalid_cursor", "invalid_file_path", "invalid_gistId", "invalid_path", "invalid_plan", "invalid_preserve_files_format", "invalid_project_name", "invalid_pullRequestId", "invalid_repo", "invalid_repoId", "invalid_request", "invalid_settings", "invalid_sort", "invalid_status", "invalid_terms_format", "invalid_token_id", "invalid_token_options", "is_removed", "not_authorized", "not_connected", "options_not_provided", "page_not_activated", "page_not_supported_on_different_branch", "path_not_defined", "path_not_specified", "pullRequestId_already_used", "pullRequestId_is_not_a_number", "pullRequestId_not_specified", "pull_request_expired", "pull_request_not_found", "pull_request_not_ready", "rate_limited", "readme_not_available", "repoId_already_used", "repoId_not_defined", "repoUrl_not_defined", "repo_access_limited", "repo_empty", "repo_not_accessible", "repo_not_found", "repo_saml_enforcement", "repository_archived", "repository_changed", "repository_expired", "repository_job_cancelled", "repository_not_ready", "repository_not_specified", "repository_tree_incomplete", "repository_tree_too_large", "s3_config_not_provided", "source_not_provided", "storage_delete_failed", "storage_write_size_mismatch", "submodule_not_available", "terms_not_specified", "token_expired", "token_limit_reached", "unable_to_connect_user", "unable_to_download", "upstream_error", "user_not_found", "username_not_defined"];
export const LOG_EVENTS: Record<string, string> = {
"PR comments 404, continuing without them": "pr_comments_404_continuing_without_them",
"S3 body stream error": "s3_body_stream_error",
"account removed": "account_removed",
"billing maintenance failed": "billing_maintenance_failed",
"billing reconciliation deferred": "billing_reconciliation_deferred",
"billing requires attention": "billing_requires_attention",
"billing worker failed": "billing_worker_failed",
"blob API failed, falling back to raw URL": "blob_api_failed_falling_back_to_raw_url",
"branch not found": "branch_not_found",
"cache removal failed": "cache_removal_failed",
"cache removed": "cache_removed",
"checking repository status and unused repositories": "checking_repository_status_and_unused_repositories",
"cleared lock for active job": "cleared_lock_for_active_job",
"conference expire failed": "conference_expire_failed",
"content expiration failed": "content_expiration_failed",
"daily stats snapshot failed": "daily_stats_snapshot_failed",
"daily stats snapshot stored": "daily_stats_snapshot_stored",
"download completed": "download_completed",
"download start": "download_start",
"downloadFile failed": "downloadfile_failed",
"downloaded": "downloaded",
"downloading file": "downloading_file",
"downloading gist": "downloading_gist",
"downloading pull request": "downloading_pull_request",
"downloading via raw URL (LFS)": "downloading_via_raw_url_lfs",
"duplicate Stripe customer requires review": "duplicate_stripe_customer_requires_review",
"ensureTodaySnapshot failed": "ensuretodaysnapshot_failed",
"error log redis connect failed": "error_log_redis_connect_failed",
"expired repository cleanup failed": "expired_repository_cleanup_failed",
"failed removal recovery failed": "failed_removal_recovery_failed",
"failed to persist ERROR status": "failed_to_persist_error_status",
"failed to record repository removal error": "failed_to_record_repository_removal_error",
"getToken": "gettoken",
"github primary rate limit hit": "github_primary_rate_limit_hit",
"github secondary rate limit hit": "github_secondary_rate_limit_hit",
"grant revocation failed": "grant_revocation_failed",
"initial repository status check failed": "initial_repository_status_check_failed",
"job completed": "job_completed",
"job stalled": "job_stalled",
"job started": "job_started",
"lastUsedAt update failed": "lastusedat_update_failed",
"login-token failed": "login_token_failed",
"login-token req.login failed": "login_token_req_login_failed",
"logout after account removal failed": "logout_after_account_removal_failed",
"logout failed": "logout_failed",
"lookup failed": "lookup_failed",
"markErrorIfInFlight failed": "markerrorifinflight_failed",
"markErrorIfRemoving failed": "markerrorifremoving_failed",
"metadata cleanup complete": "metadata_cleanup_complete",
"metadata cleanup deferred": "metadata_cleanup_deferred",
"owner access alert reset unavailable": "owner_access_alert_reset_unavailable",
"owner access email accepted": "owner_access_email_accepted",
"owner access email audit unavailable": "owner_access_email_audit_unavailable",
"owner access email claim cleanup unavailable": "owner_access_email_claim_cleanup_unavailable",
"owner access email deferred recovery unavailable": "owner_access_email_deferred_recovery_unavailable",
"owner access email delivery failed": "owner_access_email_delivery_failed",
"owner access email preparation cleanup unavailable": "owner_access_email_preparation_cleanup_unavailable",
"owner access email recovery recorded": "owner_access_email_recovery_recorded",
"owner access email recovery unavailable": "owner_access_email_recovery_unavailable",
"owner access email result reconciliation required": "owner_access_email_result_reconciliation_required",
"owner access email unavailable": "owner_access_email_unavailable",
"process fatal error": "process_fatal_error",
"process heartbeat": "process_heartbeat",
"process started": "process_started",
"progress": "progress",
"queue heartbeat": "queue_heartbeat",
"queued for download": "queued_for_download",
"rate limit gate active, delaying job before work": "rate_limit_gate_active_delaying_job_before_work",
"rate limit gate set": "rate_limit_gate_set",
"rate-limited, delaying job": "rate_limited_delaying_job",
"recover failed": "recover_failed",
"recoverStuckPreparing failed": "recoverstuckpreparing_failed",
"recoverStuckRemoving failed": "recoverstuckremoving_failed",
"recovered expired repository cleanup": "recovered_expired_repository_cleanup",
"recovered file from truncated tree": "recovered_file_from_truncated_tree",
"recovered file invalidation failed": "recovered_file_invalidation_failed",
"recovered stuck repo": "recovered_stuck_repo",
"redis client error": "redis_client_error",
"redis rate limit gate written": "redis_rate_limit_gate_written",
"rejected upstream download destination": "rejected_upstream_download_destination",
"removal failed": "removal_failed",
"removal recovery failed": "removal_recovery_failed",
"removeCache save failed": "removecache_save_failed",
"removed cache for unused repository": "removed_cache_for_unused_repository",
"removing cache": "removing_cache",
"removing repository": "removing_repository",
"repo claimed": "repo_claimed",
"repository cache removal failed": "repository_cache_removal_failed",
"repository expiration failed": "repository_expiration_failed",
"repository expired": "repository_expired",
"repository removal failed": "repository_removal_failed",
"repository removed": "repository_removed",
"requeued failed removal": "requeued_failed_removal",
"requeued interrupted removal": "requeued_interrupted_removal",
"reset": "reset",
"retired tree cleanup deferred": "retired_tree_cleanup_deferred",
"retired tree cleanup failed": "retired_tree_cleanup_failed",
"running daily stats snapshot": "running_daily_stats_snapshot",
"send failed": "send_failed",
"server started": "server_started",
"source stream failed; cache write discarded": "source_stream_failed_cache_write_discarded",
"staged tree cleanup failed": "staged_tree_cleanup_failed",
"staged tree completion deferred": "staged_tree_completion_deferred",
"streamer fetch failed": "streamer_fetch_failed",
"streamer started": "streamer_started",
"streamer startup failed": "streamer_startup_failed",
"streamer zip fetch failed": "streamer_zip_fetch_failed",
"temporary storage cleanup deferred": "temporary_storage_cleanup_deferred",
"token refresh failed; backing off": "token_refresh_failed_backing_off",
"up to date": "up_to_date",
"update queued": "update_queued",
"verify failed": "verify_failed",
"waiting for rate limit gate": "waiting_for_rate_limit_gate",
"webhook processing failed": "webhook_processing_failed",
"write failed": "write_failed",
"zip stream failed": "zip_stream_failed"
};
+33 -33
View File
@@ -1,3 +1,4 @@
import { logRecord } from "./log-record";
import { redactSecrets } from "./redact-secrets";
import { createClient, RedisClientType } from "redis";
import config from "../config";
@@ -40,16 +41,6 @@ function resolveThreshold(): number {
const threshold = resolveThreshold();
function formatArg(a: unknown): string {
if (typeof a === "string") return a;
if (a instanceof Error) return JSON.stringify(serializeError(a));
try {
return JSON.stringify(a);
} catch {
return String(a);
}
}
let redisClient: RedisClientType | null = null;
let redisDisabled = false;
@@ -58,6 +49,8 @@ function getRedis(): RedisClientType | null {
if (redisClient) return redisClient;
try {
redisClient = createClient({
disableOfflineQueue: true,
commandsQueueMaxLength: 128,
socket: {
host: config.REDIS_HOSTNAME,
port: config.REDIS_PORT,
@@ -92,6 +85,13 @@ function getRedis(): RedisClientType | null {
// Redis is back. Read by /admin/errors/stats so the admin page surfaces
// "you're losing logs" instead of silently rendering an empty table.
let droppedInProcess = 0;
let droppedOutput = 0;
let outputBroken = false;
let errorOutputBroken = false;
// A closed log reader must not become an uncaught EPIPE on the serving process.
process.stdout.on("error", () => { outputBroken = true; droppedOutput++; });
process.stderr.on("error", () => { errorOutputBroken = true; droppedOutput++; });
export function getDroppedOutput() { return droppedOutput; }
export function getInProcessDropped(): number {
return droppedInProcess;
}
@@ -192,8 +192,9 @@ function persistError(entry: {
message: string;
raw: unknown[];
}) {
if (process.env.LOG_REDIS_MIRROR === "off") return;
const client = getRedis();
if (!client || !client.isOpen) {
if (!client || !client.isReady) {
droppedInProcess++;
return;
}
@@ -204,12 +205,12 @@ function persistError(entry: {
) as Record<string, unknown> | undefined;
const bucket = bucketFor(detail, entry.level);
const code =
(detail && typeof detail.message === "string"
? (detail.message as string)
: "") ||
(detail && typeof detail.code === "string"
? (detail.code as string)
: "") ||
(detail && typeof detail.message === "string"
? (detail.message as string)
: "") ||
"_";
// Routine client misuse (401/403/404 and the allowlisted "expected" 4xx
// codes below) drowns out real errors in the recent-entries ring. Keep the
@@ -252,29 +253,26 @@ function persistError(entry: {
function emit(level: Level, module: string, args: unknown[]) {
if (LEVEL_ORDER[level] < threshold) return;
args = args.map(arg => redactSecrets(arg instanceof Error ? serializeError(arg) : arg));
const ts = new Date().toISOString();
const formatted = args.map(formatArg);
const line = `${ts} ${level.toUpperCase()} [${module}] ${formatted.join(" ")}`;
const sink =
level === "error"
? console.error
: level === "warn"
? console.warn
: level === "debug"
? console.debug
: console.log;
sink(line);
const record = logRecord(level, module, args);
// stdout is a pipe in Docker. Respect backpressure without awaiting it or
// building an unbounded userland queue. The next heartbeat reports losses.
const output = outputBroken ? process.stderr : process.stdout;
if (!(outputBroken && errorOutputBroken) && output.writableLength < 256 * 1024) {
try { output.write(JSON.stringify(record) + "\n"); }
catch {
if (output === process.stdout) outputBroken = true;
else errorOutputBroken = true;
droppedOutput++;
}
} else { droppedOutput++; }
if (level === "error" || level === "warn") {
persistError({
ts,
level,
module,
message: typeof args[0] === "string" ? args[0] : "",
raw: args.map((a) => {
if (a instanceof Error) return serializeError(a);
return a;
}),
message: String(record.errorCode || record.event),
raw: [{ ...record, message: record.errorCode || record.event, code: record.errorCode || record.event }],
});
}
}
@@ -302,10 +300,12 @@ type ErrorLike = {
url?: string | (() => string | undefined);
};
export function serializeError(err: unknown): Record<string, unknown> {
export function serializeError(err: unknown, seen = new WeakSet<object>(), depth = 0): Record<string, unknown> {
if (err == null) return { value: err };
if (typeof err !== "object") return { value: String(err) };
if (seen.has(err) || depth >= 4) return { message: "error_details_truncated" };
seen.add(err);
const e = err as ErrorLike;
const out: Record<string, unknown> = {};
if (e.name) out.name = e.name;
@@ -340,7 +340,7 @@ export function serializeError(err: unknown): Record<string, unknown> {
/* ignore */
}
}
if (e.cause) out.cause = serializeError(e.cause);
if (e.cause) out.cause = serializeError(e.cause, seen, depth + 1);
// Only include the stack when there's nothing else useful — avoids dumping
// a stack for handled HTTP errors but keeps debuggability for plain Errors.
+9 -2
View File
@@ -1,4 +1,5 @@
import { createLogger, serializeError } from "./logger";
import { logMemory } from "./log-memory";
import { createLogger, serializeError, getDroppedOutput } from "./logger";
/** Observe fatal errors without overriding Node's default exit behavior. */
export function installFatalErrorLogging(service: "api" | "streamer") {
@@ -6,6 +7,12 @@ export function installFatalErrorLogging(service: "api" | "streamer") {
const report = (error: Error, origin: string) => logger.error("process fatal error", {
...serializeError(error), code: "process_fatal_error", service, origin,
});
logger.info("process started", { service });
const heartbeat = setInterval(() => {
void logMemory().then(memory => logger.info("process heartbeat", {
service, uptime: process.uptime(), ...memory, droppedLogs: getDroppedOutput() }));
}, 30000);
heartbeat.unref();
process.on("uncaughtExceptionMonitor", report);
return () => process.removeListener("uncaughtExceptionMonitor", report);
return () => { clearInterval(heartbeat); process.removeListener("uncaughtExceptionMonitor", report); };
}
+2 -1
View File
@@ -1,3 +1,4 @@
import { logContext } from "./log-context";
import { AsyncLocalStorage } from "async_hooks";
import { randomUUID } from "crypto";
import { performance } from "perf_hooks";
@@ -164,6 +165,6 @@ export function monitorRequests(service: Service, metrics = requestMetrics,
else if (!route.includes("healthcheck")) log.info("request", detail);
};
res.once("finish", done); res.once("close", done);
context.run(trace, next);
context.run(trace, () => logContext.run({ requestId: id, service }, next));
};
}
+20 -1
View File
@@ -1,3 +1,4 @@
import { jobTrace } from "../core/log-context";
import { isQuotaFailure } from "../core/repository-failure-status";
import UserModel from "../core/model/users/users.model";
import { JobsOptions, Queue, Worker } from "bullmq";
@@ -16,6 +17,8 @@ const logger = createLogger("queue");
// ERR_BUFFER_OUT_OF_BOUNDS on long term lists / large nested fields.
export interface RepoJobData {
repoId: string;
requestId?: string;
logJobId?: string;
}
const IN_FLIGHT_STATUSES: RepositoryStatus[] = [
@@ -139,7 +142,7 @@ export async function addRemovalJob(
if (LIVE_JOB_STATES.has(state)) return false;
await existing.remove().catch(() => undefined);
}
await queue.add(repoId, { repoId }, { ...REMOVAL_JOB_OPTIONS, jobId });
await queue.add(repoId, { repoId, ...jobTrace() }, { ...REMOVAL_JOB_OPTIONS, jobId });
return true;
}
@@ -302,6 +305,22 @@ export function startWorker() {
autorun: true,
}
);
for (const [queue, worker] of [["cache", cacheWorker], ["remove", removeWorker], ["download", downloadWorker]] as const) {
worker.on("stalled", () => logger.error("job stalled", { queue }));
}
let sampling = false;
const timer = setInterval(async () => {
if (sampling) return;
sampling = true;
try {
for (const [queue, client] of [["cache", cacheQueue], ["remove", removeQueue], ["download", downloadQueue]] as const) {
const counts = await client.getJobCounts("waiting", "active", "delayed", "failed");
const oldest = (await client.getJobs(["waiting"], 0, 0, true))[0];
logger.info("queue heartbeat", { queue, ...counts, oldestWaitingMs: oldest ? Date.now() - oldest.timestamp : 0 });
}
} catch { /* queue availability is reported through missing heartbeat alerts */ }
finally { sampling = false; }
}, 30000); timer.unref();
if (!downloadWorker.isRunning()) downloadWorker.run();
downloadWorker.on("active", async (job) => {
+28
View File
@@ -0,0 +1,28 @@
import { logMemory } from "../core/log-memory";
import { randomUUID } from "crypto";
import { SandboxedJob } from "bullmq";
import { logContext, validTraceId } from "../core/log-context";
import { createLogger, getDroppedOutput } from "../core/logger";
import type { RepoJobData } from "./index";
const logger = createLogger("queue");
export function loggedJob(queue: "download" | "remove" | "cache", work: (job: SandboxedJob<RepoJobData, void>) => Promise<void>) {
return (job: SandboxedJob<RepoJobData, void>) => logContext.run({
service: "worker", jobId: validTraceId(job.data.logJobId) ? job.data.logJobId : randomUUID(),
requestId: validTraceId(job.data.requestId) ? job.data.requestId : undefined,
attempt: job.attemptsMade + 1,
}, async () => {
logger.info("job started", { queue });
const started = Date.now();
const heartbeat = setInterval(() => {
void logMemory().then(memory => logger.info("process heartbeat", { queue, ...memory, uptime: process.uptime(), droppedLogs: getDroppedOutput() }));
}, 30000); heartbeat.unref();
try {
await work(job);
logger.info("job completed", { queue, ms: Date.now() - started, droppedLogs: getDroppedOutput() });
} catch (error) {
const expected = error instanceof Error && ["DelayedError", "RateLimitDelayError"].includes(error.name);
logger[expected ? "info" : "error"]("job failed", { queue, ms: Date.now() - started, droppedLogs: getDroppedOutput() });
throw error;
} finally { clearInterval(heartbeat); }
});
}
+3 -2
View File
@@ -1,3 +1,4 @@
import { loggedJob } from "../log-job";
import { githubQuotaKey } from "../../core/github-token-context";
import { isQuotaFailure } from "../../core/repository-failure-status";
import { SandboxedJob, UnrecoverableError } from "bullmq";
@@ -13,7 +14,7 @@ import { DelayedError } from "bullmq";
const logger = createLogger("queue:download");
export default async function (job: SandboxedJob<RepoJobData, void>) {
export default loggedJob("download", async function (job: SandboxedJob<RepoJobData, void>) {
const {
connect,
getRepository,
@@ -170,4 +171,4 @@ export default async function (job: SandboxedJob<RepoJobData, void>) {
} finally {
clearInterval(statusInterval);
}
}
});
+2 -1
View File
@@ -1,3 +1,4 @@
import { loggedJob } from "../log-job";
import { SandboxedJob } from "bullmq";
import { getRepository as getRepositoryImport } from "../../server/database";
import { RepoJobData } from "../index";
@@ -42,4 +43,4 @@ export function createRemoveCacheProcessor(database?: Database) {
) => processRemoveCache(job, database);
}
export default createRemoveCacheProcessor();
export default loggedJob("cache", createRemoveCacheProcessor());
+2 -1
View File
@@ -1,3 +1,4 @@
import { loggedJob } from "../log-job";
import { SandboxedJob } from "bullmq";
import { getRepository as getRepositoryImport } from "../../server/database";
import { RepositoryStatus } from "../../core/types";
@@ -61,4 +62,4 @@ export function createRemoveRepositoryProcessor(database?: Database) {
) => processRemoveRepository(job, database);
}
export default createRemoveRepositoryProcessor();
export default loggedJob("remove", createRemoveRepositoryProcessor());
+2 -1
View File
@@ -1,3 +1,4 @@
import { jobTrace } from "../../core/log-context";
import { cacheMetricsReport } from "../../core/cache-observability";
import { sharedBlobCache, resultBlobCache } from "../../core/blob-cache";
import { AsyncCache } from "../../core/async-cache";
@@ -985,7 +986,7 @@ router.delete(
const repo = await getRepo(req, res, { nocheck: true });
if (!repo) return;
try {
await cacheQueue.add(repo.repoId, { repoId: repo.repoId }, { jobId: `repo-${repo.repoId}` });
await cacheQueue.add(repo.repoId, { repoId: repo.repoId, ...jobTrace() }, { jobId: `repo-${repo.repoId}` });
return res.json({ status: repo.status });
} catch (error) {
handleError(error, res, req);
+3 -2
View File
@@ -1,3 +1,4 @@
import { jobTrace } from "../../core/log-context";
import Repository from "../../core/Repository";
import { sourceStorageSize } from "../../core/source/Source";
import { maximumExpiration, neverExpirationDeadline, validateBillingExpiration } from "../../core/billing/expiration";
@@ -42,7 +43,7 @@ router.use(ensureAuthenticated);
async function enqueueSavedSnapshot(model: IAnonymizedRepositoryDocument) {
try {
await downloadQueue.add(model.repoId, { repoId: model.repoId }, {
await downloadQueue.add(model.repoId, { repoId: model.repoId, ...jobTrace() }, {
jobId: `repo-${model.repoId}`, attempts: 3,
});
} catch (error) {
@@ -878,7 +879,7 @@ router.post("/", async (req, res) => {
}
res.send({ status: repo.status });
downloadQueue.add(repo.repoId, { repoId: repo.repoId }, {
downloadQueue.add(repo.repoId, { repoId: repo.repoId, ...jobTrace() }, {
jobId: `repo-${repo.repoId}`,
attempts: 3,
});
+2 -1
View File
@@ -1,3 +1,4 @@
import { jobTrace } from "../../core/log-context";
import { hasVisibleSubmodules } from "../../core/file-submodules";
import { FILE_LISTING_LIMIT } from "../../core/file-tree";
import { folderCounts as scopedFolderCounts } from "../../core/folder-counts";
@@ -342,7 +343,7 @@ router.get(
repo.statusActivityDate < fiveMinuteAgo
) {
await repo.updateStatus(RepositoryStatus.PREPARING);
await downloadQueue.add(repo.repoId, { repoId: repo.repoId }, {
await downloadQueue.add(repo.repoId, { repoId: repo.repoId, ...jobTrace() }, {
jobId: `repo-${repo.repoId}`,
attempts: 3,
});