mirror of
https://github.com/tdurieux/anonymous_github.git
synced 2026-11-18 10:04:17 +01:00
Merge pull request #956 from tdurieux/t3/oss-logging-infrastructure
feat: add private structured logging and investigation dashboards
This commit is contained in:
40 files changed
+1835
-75
No files matched your search
@@ -1,3 +1,4 @@
|
||||
import { jobTrace } from "./log-context";
|
||||
import { REPOSITORY_ID_COLLATION } from "./repository-id";
|
||||
import { reserveRepository } from "./billing/quotas";
|
||||
import { hasAutomaticUpdates } from "./billing/expiration";
|
||||
@@ -856,7 +857,7 @@ export default class Repository {
|
||||
const current = await AnonymizedRepositoryModel.exists({ _id: this.model._id, ...this.refreshFilter() });
|
||||
if (!current) throw new AnonymousError("invalid_status", { httpStatus: 409 });
|
||||
}
|
||||
await downloadQueue.add(this.repoId, { repoId: this.repoId }, {
|
||||
await downloadQueue.add(this.repoId, { repoId: this.repoId, ...jobTrace() }, {
|
||||
jobId: `repo-${this.repoId}`,
|
||||
attempts: 3,
|
||||
});
|
||||
|
||||
@@ -0,0 +1,11 @@
|
||||
import { AsyncLocalStorage } from "async_hooks";
|
||||
import { randomUUID } from "crypto";
|
||||
|
||||
export interface LogContext { requestId?: string; jobId?: string; attempt?: number; service?: string }
|
||||
export const logContext = new AsyncLocalStorage<LogContext>();
|
||||
export const validTraceId = (value: unknown): value is string => typeof value === "string" &&
|
||||
/^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$/.test(value);
|
||||
export function jobTrace() {
|
||||
const requestId = logContext.getStore()?.requestId;
|
||||
return { logJobId: randomUUID(), ...(requestId ? { requestId } : {}) };
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
import { readFile } from "fs/promises";
|
||||
/** Container usage includes sandbox workers and page cache; process RSS does not. */
|
||||
export async function logMemory() {
|
||||
const rss = process.memoryUsage().rss;
|
||||
for (const [usage, limit] of [
|
||||
["/sys/fs/cgroup/memory.current", "/sys/fs/cgroup/memory.max"],
|
||||
["/sys/fs/cgroup/memory/memory.usage_in_bytes", "/sys/fs/cgroup/memory/memory.limit_in_bytes"],
|
||||
]) {
|
||||
const [used, max] = await Promise.all([usage, limit].map(path => readFile(path, "utf8").then(Number).catch(() => NaN)));
|
||||
if (Number.isFinite(used) && max > 0 && max < Number.MAX_SAFE_INTEGER) {
|
||||
return { rss, memoryLimitBytes: max, memoryRatio: used / max };
|
||||
}
|
||||
}
|
||||
return { rss };
|
||||
}
|
||||
@@ -0,0 +1,72 @@
|
||||
import { LOG_MODULES, ERROR_CODES, LOG_EVENTS } from "./log-schema";
|
||||
import { randomUUID } from "crypto";
|
||||
import { logContext, validTraceId } from "./log-context";
|
||||
|
||||
// No arbitrary strings: upstream errors, paths and even innocuous-looking messages
|
||||
// can contain a private repository name. Add diagnostics here deliberately.
|
||||
const instance = randomUUID();
|
||||
const allowed = (values: readonly string[], value: unknown): value is string => typeof value === "string" && values.includes(value);
|
||||
const events: Record<string, string> = {
|
||||
...LOG_EVENTS,
|
||||
request: "request_completed", "slow request": "request_slow", "request interrupted": "request_cancelled",
|
||||
"process fatal error": "process_fatal_error", "process started": "process_started",
|
||||
"process heartbeat": "process_heartbeat", "job started": "job_started", "job completed": "job_completed",
|
||||
"job failed": "job_failed", "job stalled": "job_stalled", "queue heartbeat": "queue_heartbeat",
|
||||
};
|
||||
const errorCodes = new Set([
|
||||
...ERROR_CODES,
|
||||
"token_refresh_failed", "quota_exceeded", "github_rate_limit_exceeded", "github_app_reconnect_required", "github_token_invalid",
|
||||
"repository_expired", "repository_not_ready", "repository_not_found", "repository_job_cancelled",
|
||||
"ECONNRESET", "ECONNREFUSED", "ETIMEDOUT", "ENOTFOUND", "EPIPE", "ABORT_ERR",
|
||||
"upstream_error", "upstream_timeout", "invalid_signature", "webhook_processing_failed",
|
||||
]);
|
||||
const numbers = ["status", "httpStatus", "ms", "firstByteMs", "responseMs", "bytes", "bytesPerSecond",
|
||||
"attempt", "rss", "memoryLimitBytes", "memoryRatio", "uptime", "activeRequests", "waiting", "active", "delayed", "failed",
|
||||
"oldestWaitingMs", "droppedLogs", "loopP95Ms", "upstreamStatus"];
|
||||
const stages = ["repository", "credential_resolution", "upstream", "cache_lookup", "cache_fill", "worker_wait", "anonymize", "streamer_headers"];
|
||||
export function logRecord(level: string, module: string, args: unknown[]) {
|
||||
const context = logContext.getStore() || {};
|
||||
const details: Record<string, unknown> = {};
|
||||
const keys = [...numbers, "requestId", "jobId", "code", "message", "outcome", "method", "route", "queue",
|
||||
"stages", "reason", "phase", "event", "deliveryId", "hostname", "service"];
|
||||
// Do not traverse arbitrary objects or invoke error getters while logging.
|
||||
for (const arg of args.slice(0, 8)) if (arg && typeof arg === "object") {
|
||||
for (const key of keys) {
|
||||
try {
|
||||
const property = Object.getOwnPropertyDescriptor(arg, key);
|
||||
if (property && "value" in property) details[key] = property.value;
|
||||
} catch { /* A hostile Proxy must not make logging throw. */ }
|
||||
}
|
||||
}
|
||||
const first = typeof args[0] === "string" && args[0].length < 256 ? args[0] : "";
|
||||
const event = (Object.hasOwn(events, first) ? events[first] : undefined) || (module.includes("webhook") ? "webhook_log" : "application_log");
|
||||
const record: Record<string, unknown> = { timestamp: new Date().toISOString(), severity: level,
|
||||
service: context.service || (allowed(["api", "streamer", "worker"], details.service) ? details.service : undefined) || (process.env.SERVICE_NAME?.toLowerCase() === "streamer" ? "streamer" : "api"),
|
||||
instance, release: /^[a-f0-9]{7,40}$/.test(process.env.APP_RELEASE || "") ? process.env.APP_RELEASE : "unknown",
|
||||
event, message: event, module: LOG_MODULES.includes(module) ? module : "application" };
|
||||
for (const key of ["requestId", "jobId"] as const) {
|
||||
const value = details[key] || context[key];
|
||||
if (validTraceId(value)) record[key] = value;
|
||||
}
|
||||
if (Number.isSafeInteger(context.attempt)) record.attempt = context.attempt;
|
||||
for (const key of numbers) if (typeof details[key] === "number" && Number.isFinite(details[key])) record[key] = details[key];
|
||||
const code = [details.code, details.message].find(value => typeof value === "string" && errorCodes.has(value));
|
||||
if (code) record.errorCode = code;
|
||||
if (allowed(["protocol", "credentials", "port", "host"], details.reason)) record.reason = details.reason;
|
||||
if (allowed(["parse", "find_grant_owners", "reconcile_grant", "persist_installation_block", "reconcile_installation"], details.phase)) record.phase = details.phase;
|
||||
if (allowed(["github_app_authorization", "installation", "installation_repositories"], details.event)) record.webhookEvent = details.event;
|
||||
if (validTraceId(details.deliveryId)) record.deliveryId = details.deliveryId;
|
||||
if (allowed(["github.com", "api.github.com", "codeload.github.com", "raw.githubusercontent.com", "media.githubusercontent.com"], details.hostname)) record.upstreamHost = details.hostname;
|
||||
else if (details.hostname) record.upstreamHost = "other";
|
||||
if (allowed(["completed", "interrupted"], details.outcome)) record.outcome = details.outcome;
|
||||
if (allowed(["GET", "HEAD", "POST", "PUT", "PATCH", "DELETE", "OPTIONS", "OTHER"], details.method)) record.method = details.method;
|
||||
// Route labels are generated by request-monitoring, never from URLs in arbitrary errors.
|
||||
if (module === "requests" && typeof details.route === "string") record.route = details.route;
|
||||
if (allowed(["download", "remove", "cache"], details.queue)) record.queue = details.queue;
|
||||
if (details.stages && typeof details.stages === "object") for (const key of stages) {
|
||||
let value: unknown;
|
||||
try { value = Object.getOwnPropertyDescriptor(details.stages, key)?.value; } catch { continue; }
|
||||
if (typeof value === "number" && Number.isFinite(value)) record[`stage_${key}_ms`] = value;
|
||||
}
|
||||
return record;
|
||||
}
|
||||
@@ -0,0 +1,121 @@
|
||||
// Reviewed literal diagnostics. Never extend this list from runtime data.
|
||||
export const LOG_MODULES = ["admin", "anonymized-file", "auth", "billing", "dailyStats", "file-tree", "fs", "gh-stream", "gist", "github", "github-app-webhook", "owner-notifications", "process", "pull-request", "queue", "queue:cache", "queue:download", "queue:remove", "repository", "repository-public", "requests", "route", "route:repo", "s3", "schedule", "server", "streamer", "temporary-storage", "token-auth", "user", "zip-stream"];
|
||||
export const ERROR_CODES = ["billing_address_missing", "billing_city_missing", "billing_country_missing", "billing_email_missing", "billing_missing", "billing_name_missing", "billing_zip_missing", "branch_not_found", "branch_not_specified", "cache_busy", "cache_miss", "cannot_coauthor_self", "commit_not_specified", "conf_end_date_invalid", "conf_end_date_missing", "conf_id_missing", "conf_id_used", "conf_name_missing", "conf_not_activated", "conf_not_found", "conf_start_date_invalid", "conf_start_date_missing", "conference_not_found", "connection_changed", "download_not_enabled", "expiration_limit_reached", "file_listing_too_large", "file_not_accessible", "file_not_found", "file_not_supported", "file_too_big", "folder_not_supported", "gistId_already_used", "gistId_not_specified", "gist_expired", "gist_not_found", "gist_not_ready", "github_app_access_required", "github_identity_mismatch", "github_rate_limit_exceeded", "github_user_not_found", "invalid_commit_format", "invalid_cursor", "invalid_file_path", "invalid_gistId", "invalid_path", "invalid_plan", "invalid_preserve_files_format", "invalid_project_name", "invalid_pullRequestId", "invalid_repo", "invalid_repoId", "invalid_request", "invalid_settings", "invalid_sort", "invalid_status", "invalid_terms_format", "invalid_token_id", "invalid_token_options", "is_removed", "not_authorized", "not_connected", "options_not_provided", "page_not_activated", "page_not_supported_on_different_branch", "path_not_defined", "path_not_specified", "pullRequestId_already_used", "pullRequestId_is_not_a_number", "pullRequestId_not_specified", "pull_request_expired", "pull_request_not_found", "pull_request_not_ready", "rate_limited", "readme_not_available", "repoId_already_used", "repoId_not_defined", "repoUrl_not_defined", "repo_access_limited", "repo_empty", "repo_not_accessible", "repo_not_found", "repo_saml_enforcement", "repository_archived", "repository_changed", "repository_expired", "repository_job_cancelled", "repository_not_ready", "repository_not_specified", "repository_tree_incomplete", "repository_tree_too_large", "s3_config_not_provided", "source_not_provided", "storage_delete_failed", "storage_write_size_mismatch", "submodule_not_available", "terms_not_specified", "token_expired", "token_limit_reached", "unable_to_connect_user", "unable_to_download", "upstream_error", "user_not_found", "username_not_defined"];
|
||||
export const LOG_EVENTS: Record<string, string> = {
|
||||
"PR comments 404, continuing without them": "pr_comments_404_continuing_without_them",
|
||||
"S3 body stream error": "s3_body_stream_error",
|
||||
"account removed": "account_removed",
|
||||
"billing maintenance failed": "billing_maintenance_failed",
|
||||
"billing reconciliation deferred": "billing_reconciliation_deferred",
|
||||
"billing requires attention": "billing_requires_attention",
|
||||
"billing worker failed": "billing_worker_failed",
|
||||
"blob API failed, falling back to raw URL": "blob_api_failed_falling_back_to_raw_url",
|
||||
"branch not found": "branch_not_found",
|
||||
"cache removal failed": "cache_removal_failed",
|
||||
"cache removed": "cache_removed",
|
||||
"checking repository status and unused repositories": "checking_repository_status_and_unused_repositories",
|
||||
"cleared lock for active job": "cleared_lock_for_active_job",
|
||||
"conference expire failed": "conference_expire_failed",
|
||||
"content expiration failed": "content_expiration_failed",
|
||||
"daily stats snapshot failed": "daily_stats_snapshot_failed",
|
||||
"daily stats snapshot stored": "daily_stats_snapshot_stored",
|
||||
"download completed": "download_completed",
|
||||
"download start": "download_start",
|
||||
"downloadFile failed": "downloadfile_failed",
|
||||
"downloaded": "downloaded",
|
||||
"downloading file": "downloading_file",
|
||||
"downloading gist": "downloading_gist",
|
||||
"downloading pull request": "downloading_pull_request",
|
||||
"downloading via raw URL (LFS)": "downloading_via_raw_url_lfs",
|
||||
"duplicate Stripe customer requires review": "duplicate_stripe_customer_requires_review",
|
||||
"ensureTodaySnapshot failed": "ensuretodaysnapshot_failed",
|
||||
"error log redis connect failed": "error_log_redis_connect_failed",
|
||||
"expired repository cleanup failed": "expired_repository_cleanup_failed",
|
||||
"failed removal recovery failed": "failed_removal_recovery_failed",
|
||||
"failed to persist ERROR status": "failed_to_persist_error_status",
|
||||
"failed to record repository removal error": "failed_to_record_repository_removal_error",
|
||||
"getToken": "gettoken",
|
||||
"github primary rate limit hit": "github_primary_rate_limit_hit",
|
||||
"github secondary rate limit hit": "github_secondary_rate_limit_hit",
|
||||
"grant revocation failed": "grant_revocation_failed",
|
||||
"initial repository status check failed": "initial_repository_status_check_failed",
|
||||
"job completed": "job_completed",
|
||||
"job stalled": "job_stalled",
|
||||
"job started": "job_started",
|
||||
"lastUsedAt update failed": "lastusedat_update_failed",
|
||||
"login-token failed": "login_token_failed",
|
||||
"login-token req.login failed": "login_token_req_login_failed",
|
||||
"logout after account removal failed": "logout_after_account_removal_failed",
|
||||
"logout failed": "logout_failed",
|
||||
"lookup failed": "lookup_failed",
|
||||
"markErrorIfInFlight failed": "markerrorifinflight_failed",
|
||||
"markErrorIfRemoving failed": "markerrorifremoving_failed",
|
||||
"metadata cleanup complete": "metadata_cleanup_complete",
|
||||
"metadata cleanup deferred": "metadata_cleanup_deferred",
|
||||
"owner access alert reset unavailable": "owner_access_alert_reset_unavailable",
|
||||
"owner access email accepted": "owner_access_email_accepted",
|
||||
"owner access email audit unavailable": "owner_access_email_audit_unavailable",
|
||||
"owner access email claim cleanup unavailable": "owner_access_email_claim_cleanup_unavailable",
|
||||
"owner access email deferred recovery unavailable": "owner_access_email_deferred_recovery_unavailable",
|
||||
"owner access email delivery failed": "owner_access_email_delivery_failed",
|
||||
"owner access email preparation cleanup unavailable": "owner_access_email_preparation_cleanup_unavailable",
|
||||
"owner access email recovery recorded": "owner_access_email_recovery_recorded",
|
||||
"owner access email recovery unavailable": "owner_access_email_recovery_unavailable",
|
||||
"owner access email result reconciliation required": "owner_access_email_result_reconciliation_required",
|
||||
"owner access email unavailable": "owner_access_email_unavailable",
|
||||
"process fatal error": "process_fatal_error",
|
||||
"process heartbeat": "process_heartbeat",
|
||||
"process started": "process_started",
|
||||
"progress": "progress",
|
||||
"queue heartbeat": "queue_heartbeat",
|
||||
"queued for download": "queued_for_download",
|
||||
"rate limit gate active, delaying job before work": "rate_limit_gate_active_delaying_job_before_work",
|
||||
"rate limit gate set": "rate_limit_gate_set",
|
||||
"rate-limited, delaying job": "rate_limited_delaying_job",
|
||||
"recover failed": "recover_failed",
|
||||
"recoverStuckPreparing failed": "recoverstuckpreparing_failed",
|
||||
"recoverStuckRemoving failed": "recoverstuckremoving_failed",
|
||||
"recovered expired repository cleanup": "recovered_expired_repository_cleanup",
|
||||
"recovered file from truncated tree": "recovered_file_from_truncated_tree",
|
||||
"recovered file invalidation failed": "recovered_file_invalidation_failed",
|
||||
"recovered stuck repo": "recovered_stuck_repo",
|
||||
"redis client error": "redis_client_error",
|
||||
"redis rate limit gate written": "redis_rate_limit_gate_written",
|
||||
"rejected upstream download destination": "rejected_upstream_download_destination",
|
||||
"removal failed": "removal_failed",
|
||||
"removal recovery failed": "removal_recovery_failed",
|
||||
"removeCache save failed": "removecache_save_failed",
|
||||
"removed cache for unused repository": "removed_cache_for_unused_repository",
|
||||
"removing cache": "removing_cache",
|
||||
"removing repository": "removing_repository",
|
||||
"repo claimed": "repo_claimed",
|
||||
"repository cache removal failed": "repository_cache_removal_failed",
|
||||
"repository expiration failed": "repository_expiration_failed",
|
||||
"repository expired": "repository_expired",
|
||||
"repository removal failed": "repository_removal_failed",
|
||||
"repository removed": "repository_removed",
|
||||
"requeued failed removal": "requeued_failed_removal",
|
||||
"requeued interrupted removal": "requeued_interrupted_removal",
|
||||
"reset": "reset",
|
||||
"retired tree cleanup deferred": "retired_tree_cleanup_deferred",
|
||||
"retired tree cleanup failed": "retired_tree_cleanup_failed",
|
||||
"running daily stats snapshot": "running_daily_stats_snapshot",
|
||||
"send failed": "send_failed",
|
||||
"server started": "server_started",
|
||||
"source stream failed; cache write discarded": "source_stream_failed_cache_write_discarded",
|
||||
"staged tree cleanup failed": "staged_tree_cleanup_failed",
|
||||
"staged tree completion deferred": "staged_tree_completion_deferred",
|
||||
"streamer fetch failed": "streamer_fetch_failed",
|
||||
"streamer started": "streamer_started",
|
||||
"streamer startup failed": "streamer_startup_failed",
|
||||
"streamer zip fetch failed": "streamer_zip_fetch_failed",
|
||||
"temporary storage cleanup deferred": "temporary_storage_cleanup_deferred",
|
||||
"token refresh failed; backing off": "token_refresh_failed_backing_off",
|
||||
"up to date": "up_to_date",
|
||||
"update queued": "update_queued",
|
||||
"verify failed": "verify_failed",
|
||||
"waiting for rate limit gate": "waiting_for_rate_limit_gate",
|
||||
"webhook processing failed": "webhook_processing_failed",
|
||||
"write failed": "write_failed",
|
||||
"zip stream failed": "zip_stream_failed"
|
||||
};
|
||||
+33
-33
@@ -1,3 +1,4 @@
|
||||
import { logRecord } from "./log-record";
|
||||
import { redactSecrets } from "./redact-secrets";
|
||||
import { createClient, RedisClientType } from "redis";
|
||||
import config from "../config";
|
||||
@@ -40,16 +41,6 @@ function resolveThreshold(): number {
|
||||
|
||||
const threshold = resolveThreshold();
|
||||
|
||||
function formatArg(a: unknown): string {
|
||||
if (typeof a === "string") return a;
|
||||
if (a instanceof Error) return JSON.stringify(serializeError(a));
|
||||
try {
|
||||
return JSON.stringify(a);
|
||||
} catch {
|
||||
return String(a);
|
||||
}
|
||||
}
|
||||
|
||||
let redisClient: RedisClientType | null = null;
|
||||
let redisDisabled = false;
|
||||
|
||||
@@ -58,6 +49,8 @@ function getRedis(): RedisClientType | null {
|
||||
if (redisClient) return redisClient;
|
||||
try {
|
||||
redisClient = createClient({
|
||||
disableOfflineQueue: true,
|
||||
commandsQueueMaxLength: 128,
|
||||
socket: {
|
||||
host: config.REDIS_HOSTNAME,
|
||||
port: config.REDIS_PORT,
|
||||
@@ -92,6 +85,13 @@ function getRedis(): RedisClientType | null {
|
||||
// Redis is back. Read by /admin/errors/stats so the admin page surfaces
|
||||
// "you're losing logs" instead of silently rendering an empty table.
|
||||
let droppedInProcess = 0;
|
||||
let droppedOutput = 0;
|
||||
let outputBroken = false;
|
||||
let errorOutputBroken = false;
|
||||
// A closed log reader must not become an uncaught EPIPE on the serving process.
|
||||
process.stdout.on("error", () => { outputBroken = true; droppedOutput++; });
|
||||
process.stderr.on("error", () => { errorOutputBroken = true; droppedOutput++; });
|
||||
export function getDroppedOutput() { return droppedOutput; }
|
||||
export function getInProcessDropped(): number {
|
||||
return droppedInProcess;
|
||||
}
|
||||
@@ -192,8 +192,9 @@ function persistError(entry: {
|
||||
message: string;
|
||||
raw: unknown[];
|
||||
}) {
|
||||
if (process.env.LOG_REDIS_MIRROR === "off") return;
|
||||
const client = getRedis();
|
||||
if (!client || !client.isOpen) {
|
||||
if (!client || !client.isReady) {
|
||||
droppedInProcess++;
|
||||
return;
|
||||
}
|
||||
@@ -204,12 +205,12 @@ function persistError(entry: {
|
||||
) as Record<string, unknown> | undefined;
|
||||
const bucket = bucketFor(detail, entry.level);
|
||||
const code =
|
||||
(detail && typeof detail.message === "string"
|
||||
? (detail.message as string)
|
||||
: "") ||
|
||||
(detail && typeof detail.code === "string"
|
||||
? (detail.code as string)
|
||||
: "") ||
|
||||
(detail && typeof detail.message === "string"
|
||||
? (detail.message as string)
|
||||
: "") ||
|
||||
"_";
|
||||
// Routine client misuse (401/403/404 and the allowlisted "expected" 4xx
|
||||
// codes below) drowns out real errors in the recent-entries ring. Keep the
|
||||
@@ -252,29 +253,26 @@ function persistError(entry: {
|
||||
|
||||
function emit(level: Level, module: string, args: unknown[]) {
|
||||
if (LEVEL_ORDER[level] < threshold) return;
|
||||
args = args.map(arg => redactSecrets(arg instanceof Error ? serializeError(arg) : arg));
|
||||
const ts = new Date().toISOString();
|
||||
const formatted = args.map(formatArg);
|
||||
const line = `${ts} ${level.toUpperCase()} [${module}] ${formatted.join(" ")}`;
|
||||
const sink =
|
||||
level === "error"
|
||||
? console.error
|
||||
: level === "warn"
|
||||
? console.warn
|
||||
: level === "debug"
|
||||
? console.debug
|
||||
: console.log;
|
||||
sink(line);
|
||||
const record = logRecord(level, module, args);
|
||||
// stdout is a pipe in Docker. Respect backpressure without awaiting it or
|
||||
// building an unbounded userland queue. The next heartbeat reports losses.
|
||||
const output = outputBroken ? process.stderr : process.stdout;
|
||||
if (!(outputBroken && errorOutputBroken) && output.writableLength < 256 * 1024) {
|
||||
try { output.write(JSON.stringify(record) + "\n"); }
|
||||
catch {
|
||||
if (output === process.stdout) outputBroken = true;
|
||||
else errorOutputBroken = true;
|
||||
droppedOutput++;
|
||||
}
|
||||
} else { droppedOutput++; }
|
||||
if (level === "error" || level === "warn") {
|
||||
persistError({
|
||||
ts,
|
||||
level,
|
||||
module,
|
||||
message: typeof args[0] === "string" ? args[0] : "",
|
||||
raw: args.map((a) => {
|
||||
if (a instanceof Error) return serializeError(a);
|
||||
return a;
|
||||
}),
|
||||
message: String(record.errorCode || record.event),
|
||||
raw: [{ ...record, message: record.errorCode || record.event, code: record.errorCode || record.event }],
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -302,10 +300,12 @@ type ErrorLike = {
|
||||
url?: string | (() => string | undefined);
|
||||
};
|
||||
|
||||
export function serializeError(err: unknown): Record<string, unknown> {
|
||||
export function serializeError(err: unknown, seen = new WeakSet<object>(), depth = 0): Record<string, unknown> {
|
||||
if (err == null) return { value: err };
|
||||
if (typeof err !== "object") return { value: String(err) };
|
||||
|
||||
if (seen.has(err) || depth >= 4) return { message: "error_details_truncated" };
|
||||
seen.add(err);
|
||||
const e = err as ErrorLike;
|
||||
const out: Record<string, unknown> = {};
|
||||
if (e.name) out.name = e.name;
|
||||
@@ -340,7 +340,7 @@ export function serializeError(err: unknown): Record<string, unknown> {
|
||||
/* ignore */
|
||||
}
|
||||
}
|
||||
if (e.cause) out.cause = serializeError(e.cause);
|
||||
if (e.cause) out.cause = serializeError(e.cause, seen, depth + 1);
|
||||
|
||||
// Only include the stack when there's nothing else useful — avoids dumping
|
||||
// a stack for handled HTTP errors but keeps debuggability for plain Errors.
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { createLogger, serializeError } from "./logger";
|
||||
import { logMemory } from "./log-memory";
|
||||
import { createLogger, serializeError, getDroppedOutput } from "./logger";
|
||||
|
||||
/** Observe fatal errors without overriding Node's default exit behavior. */
|
||||
export function installFatalErrorLogging(service: "api" | "streamer") {
|
||||
@@ -6,6 +7,12 @@ export function installFatalErrorLogging(service: "api" | "streamer") {
|
||||
const report = (error: Error, origin: string) => logger.error("process fatal error", {
|
||||
...serializeError(error), code: "process_fatal_error", service, origin,
|
||||
});
|
||||
logger.info("process started", { service });
|
||||
const heartbeat = setInterval(() => {
|
||||
void logMemory().then(memory => logger.info("process heartbeat", {
|
||||
service, uptime: process.uptime(), ...memory, droppedLogs: getDroppedOutput() }));
|
||||
}, 30000);
|
||||
heartbeat.unref();
|
||||
process.on("uncaughtExceptionMonitor", report);
|
||||
return () => process.removeListener("uncaughtExceptionMonitor", report);
|
||||
return () => { clearInterval(heartbeat); process.removeListener("uncaughtExceptionMonitor", report); };
|
||||
}
|
||||
@@ -1,3 +1,4 @@
|
||||
import { logContext } from "./log-context";
|
||||
import { AsyncLocalStorage } from "async_hooks";
|
||||
import { randomUUID } from "crypto";
|
||||
import { performance } from "perf_hooks";
|
||||
@@ -164,6 +165,6 @@ export function monitorRequests(service: Service, metrics = requestMetrics,
|
||||
else if (!route.includes("healthcheck")) log.info("request", detail);
|
||||
};
|
||||
res.once("finish", done); res.once("close", done);
|
||||
context.run(trace, next);
|
||||
context.run(trace, () => logContext.run({ requestId: id, service }, next));
|
||||
};
|
||||
}
|
||||
Reference in new issue
Block a user