feat: retain bounded failed-request reproduction context

This commit is contained in:
Thomas Durieux committed 2026-11-14 21:44:48 +00:00
1 parent 3a1d013b11
commit 5dbf3e7829
22 files changed
+198 -67

No files matched your search

+1 -1
View File
@@ -1,7 +1,7 @@
import { AsyncLocalStorage } from "async_hooks";
import { randomUUID } from "crypto";
export interface LogContext { requestId?: string; jobId?: string; attempt?: number; service?: string; route?: string; method?: string }
export interface LogContext { requestId?: string; jobId?: string; attempt?: number; service?: string; route?: string; method?: string; requestPath?: string }
export const logContext = new AsyncLocalStorage<LogContext>();
export const validTraceId = (value: unknown): value is string => typeof value === "string" &&
/^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$/.test(value);
+10
View File
@@ -1034,6 +1034,16 @@ export const LOG_SOURCE_FUNCTIONS: Record<string, readonly string[]> = {
"allowed",
"logRecord"
],
"src/core/log-reproduction.ts": [
"<anonymous>",
"requestReproduction",
"safeReproductionPath"
],
"build/core/log-reproduction.js": [
"<anonymous>",
"requestReproduction",
"safeReproductionPath"
],
"src/core/log-schema.ts": [
"<anonymous>"
],
+2 -2
View File
@@ -33,7 +33,7 @@ export function safeDiagnosticFields(value: unknown): Record<string, string> {
const stack = data(value,key);
if (typeof stack !== "string" || stack.length > 1024) continue;
const frames = stack.split("\n").slice(0,8).flatMap(line => {
const match = /^at ([A-Za-z_$][A-Za-z0-9_$]*|<anonymous>) \(((?:src|build)\/[A-Za-z0-9_/-]+\.(?:ts|js)):([1-9]\d{0,5}):([1-9]\d{0,4})\)$/.exec(line);
const match = /^at ([A-Za-z_$][A-Za-z0-9_$]*|<anonymous>) \(((?:src|build)\/[A-Za-z0-9_./-]+\.(?:ts|js)):([1-9]\d{0,5}):([1-9]\d{0,4})\)$/.exec(line);
const safe = match && frame(match[2],match[1],match[3],match[4]);
return safe ? [safe] : [];
});
@@ -54,7 +54,7 @@ export function errorDiagnostics(error: unknown): Record<string,string> {
const stack = data(current,"stack");
const frames: string[] = [];
if(typeof stack === "string") for(const line of stack.slice(0,16384).split("\n").slice(1,33)) {
const match = /^\s*at (?:(?:async )?([^()]+) \()?([^()\s]+):(\d+):(\d+)\)?$/.exec(line);
const match = /^\s*at (?:async )?(?:([^()]+) \()?([^()\s]+):(\d+):(\d+)\)?$/.exec(line);
if(!match) continue;
const local = relative(root,match[2]).split(sep).join("/");
if(local.startsWith("../") || local.includes(":")) continue;
+7 -1
View File
@@ -1,3 +1,4 @@
import { requestReproduction, safeReproductionPath } from "./log-reproduction";
import { safeDiagnosticFields, errorDiagnostics } from "./log-diagnostics";
import { LOG_ROUTES } from "./log-diagnostic-schema";
import { LOG_MODULES, ERROR_CODES, LOG_EVENTS } from "./log-schema";
@@ -30,7 +31,7 @@ export function logRecord(level: string, module: string, args: unknown[]) {
const context = logContext.getStore() || {};
const details: Record<string, unknown> = {};
const keys = [...numbers, "requestId", "jobId", "code", "message", "outcome", "method", "route", "queue",
"errorType", "errorStack", "causeType", "causeStack", "stages", "reason", "phase", "event", "deliveryId", "hostname", "service"];
"errorType", "errorStack", "causeType", "causeStack", "requestPath", "stages", "reason", "phase", "event", "deliveryId", "hostname", "service"];
// Do not traverse arbitrary objects or invoke error getters while logging.
for (const arg of args.slice(0, 8)) if (arg && typeof arg === "object") {
for (const key of keys) {
@@ -70,6 +71,11 @@ export function logRecord(level: string, module: string, args: unknown[]) {
if (typeof route === "string" && LOG_ROUTES.includes(route)) record.route = route;
if (context.method && allowed(["GET", "HEAD", "POST", "PUT", "PATCH", "DELETE", "OPTIONS", "OTHER"], context.method)) record.method = context.method;
Object.assign(record, safeDiagnosticFields(details));
// Reproduction data is restricted to failures, not successful traffic/metrics.
if (["error", "fatal"].includes(level) || Number(details.status || details.httpStatus) >= 400) {
const requestPath = safeReproductionPath(context.requestPath || details.requestPath);
if (requestPath) Object.assign(record, requestReproduction(requestPath));
}
if (allowed(["download", "remove", "cache"], details.queue)) record.queue = details.queue;
if (details.stages && typeof details.stages === "object") for (const key of stages) {
let value: unknown;
+19
View File
@@ -0,0 +1,19 @@
// Explicit admin diagnostics: public anonymous repository ID and request pathname only.
// Never collect upstream owner/repository URLs, headers, query strings or bodies.
const fileRequest = /^\/api\/repo\/([A-Za-z0-9_-]{3,200})\/file\/([A-Za-z0-9._~!$&'()+,;=@%/-]+)$/;
export function safeReproductionPath(value: unknown): string | undefined {
if (typeof value !== "string" || value.length > 2048) return undefined;
const match = fileRequest.exec(value);
if (!match || /%(?![a-fA-F0-9]{2})/.test(value)) return undefined;
try {
const path = decodeURIComponent(match[2]);
// Reject controls, traversal, URL syntax, encoded delimiters and double encoding.
// eslint-disable-next-line no-control-regex -- reject control characters from stored paths
if (/[\x00-\x1f\x7f-\x9f\u2028-\u202e\u2066-\u2069\ufffd\\:?#%]/.test(path) || path.startsWith("/") || path.split("/").some(part => part === "." || part === "..")) return undefined;
} catch { return undefined; }
return value;
}
export function requestReproduction(url: string) {
const requestPath = safeReproductionPath(url.split("?", 1)[0]);
return requestPath ? { requestPath, repoId: fileRequest.exec(requestPath)![1] } : {};
}
+15
View File
@@ -149,6 +149,21 @@ function clampPayload(entry: {
s = JSON.stringify(entry);
if (s.length <= MAX_PAYLOAD_BYTES) return s;
}
// Keep reproduction/correlation when bounded paths plus diagnostic stacks exceed
// the mirror limit. Reduce frame depth and drop secondary measurements first.
if (first && typeof first === "object" && !Array.isArray(first)) {
const diagnostic = first as Record<string, unknown>;
const compact: Record<string, unknown> = {};
for (const key of ["timestamp", "severity", "service", "event", "module", "release", "requestId", "jobId", "method", "route", "requestPath", "repoId", "httpStatus", "status", "errorCode", "errorType", "causeType", "code", "message"]) {
if (diagnostic[key] !== undefined) compact[key] = diagnostic[key];
}
for (const key of ["errorStack", "causeStack"]) {
if (typeof diagnostic[key] === "string") compact[key] = diagnostic[key].split("\n")[0];
}
entry.raw = [compact];
s = JSON.stringify(entry);
if (s.length <= MAX_PAYLOAD_BYTES) return s;
}
// Step 3: nothing left to trim — replace with a placeholder so the entry
// still shows up in the list.
entry.raw = [{ truncated: true, originalBytes: s.length }];
+4 -2
View File
@@ -1,3 +1,4 @@
import { requestReproduction } from "./log-reproduction";
import { logContext } from "./log-context";
import { AsyncLocalStorage } from "async_hooks";
import { randomUUID } from "crypto";
@@ -116,6 +117,7 @@ export function monitorRequests(service: Service, metrics = requestMetrics,
: req.originalUrl.startsWith("/w/") ? "website" : "file";
const trace = { id, service, traffic, stages: {} };
const route = requestRoute(req.originalUrl, service);
const reproduction = service === "api" ? requestReproduction(req.originalUrl) : {};
const method = methods.has(req.method) ? req.method : "OTHER";
const started = performance.now();
let firstByteMs: number | undefined;
@@ -156,7 +158,7 @@ export function monitorRequests(service: Service, metrics = requestMetrics,
metrics.observe("request", route, method, ms, { bytes, aborted, error: res.statusCode >= 500, slow });
if (firstByteMs != null) metrics.observe("first_byte", route, method, firstByteMs);
const responseMs = firstByteMs == null ? null : Math.max(0, ms - firstByteMs);
const detail = { requestId: id, service, method, route, status: res.statusCode,
const detail = { ...(res.statusCode >= 400 ? reproduction : {}), requestId: id, service, method, route, status: res.statusCode,
outcome: aborted ? "interrupted" : "completed", ms: Math.round(ms), firstByteMs: firstByteMs == null ? null : Math.round(firstByteMs),
responseMs: responseMs == null ? null : Math.round(responseMs),
bytesPerSecond: responseMs != null && responseMs >= 1 && bytes > 0 ? Math.round(bytes * 1000 / responseMs) : null,
@@ -165,6 +167,6 @@ export function monitorRequests(service: Service, metrics = requestMetrics,
else if (!route.includes("healthcheck")) log.info("request", detail);
};
res.once("finish", done); res.once("close", done);
context.run(trace, () => logContext.run({ requestId: id, service, route, method }, next));
context.run(trace, () => logContext.run({ ...reproduction, requestId: id, service, route, method }, next));
};
}