mirror of
https://github.com/tdurieux/anonymous_github.git
synced 2026-11-18 01:54:23 +01:00
Merge remote-tracking branch 'origin/main' into integrate/source-errors
# Conflicts: # public/asset-manifest.json # public/css/all.min.css # public/script/vendor.min.js
This commit is contained in:
commit
bc44879b43
24 files changed
+3370
-73
No files matched your search
@@ -1,7 +1,7 @@
|
||||
import { AsyncLocalStorage } from "async_hooks";
|
||||
import { randomUUID } from "crypto";
|
||||
|
||||
export interface LogContext { requestId?: string; jobId?: string; attempt?: number; service?: string }
|
||||
export interface LogContext { requestId?: string; jobId?: string; attempt?: number; service?: string; route?: string; method?: string; requestPath?: string }
|
||||
export const logContext = new AsyncLocalStorage<LogContext>();
|
||||
export const validTraceId = (value: unknown): value is string => typeof value === "string" &&
|
||||
/^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$/.test(value);
|
||||
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,74 @@
|
||||
import { resolve, relative, sep } from "path";
|
||||
import { LOG_ERROR_TYPES, LOG_SOURCE_FUNCTIONS } from "./log-diagnostic-schema";
|
||||
|
||||
// Node 24 exposes Error.stack through a shared native accessor; never call user getters.
|
||||
const nativeStackGetter = Object.getOwnPropertyDescriptor(new Error(), "stack")?.get;
|
||||
const root = resolve(__dirname, "..");
|
||||
function data(value: unknown, key: string, inherited = false): unknown {
|
||||
try {
|
||||
let object = value;
|
||||
for (let depth = 0; object && typeof object === "object" && depth < (inherited ? 4 : 1); depth++) {
|
||||
const descriptor = Object.getOwnPropertyDescriptor(object, key);
|
||||
if (descriptor) {
|
||||
if ("value" in descriptor) return descriptor.value;
|
||||
if (key === "stack" && nativeStackGetter && descriptor.get === nativeStackGetter) return nativeStackGetter.call(object);
|
||||
return undefined;
|
||||
}
|
||||
object = Object.getPrototypeOf(object);
|
||||
}
|
||||
} catch { /* Logging must not invoke hostile getters or fail with a Proxy. */ }
|
||||
}
|
||||
function frame(file: string, fn: string, line: string, column: string): string | null {
|
||||
if (!LOG_SOURCE_FUNCTIONS[file]?.includes(fn) || !/^[1-9]\d{0,5}$/.test(line) || !/^[1-9]\d{0,4}$/.test(column)) return null;
|
||||
return `at ${fn} (${file}:${line}:${column})`;
|
||||
}
|
||||
/** Validate already-projected diagnostics again at every storage/API boundary. */
|
||||
export function safeDiagnosticFields(value: unknown): Record<string, string> {
|
||||
const out: Record<string,string> = {};
|
||||
for (const key of ["errorType", "causeType"]) {
|
||||
const type = data(value,key);
|
||||
if (typeof type === "string" && LOG_ERROR_TYPES.includes(type)) out[key] = type;
|
||||
}
|
||||
for (const key of ["errorStack", "causeStack"]) {
|
||||
const stack = data(value,key);
|
||||
if (typeof stack !== "string" || stack.length > 1024) continue;
|
||||
const frames = stack.split("\n").slice(0,8).flatMap(line => {
|
||||
const match = /^at ([A-Za-z_$][A-Za-z0-9_$]*|<anonymous>) \(((?:src|build)\/[A-Za-z0-9_./-]+\.(?:ts|js)):([1-9]\d{0,5}):([1-9]\d{0,4})\)$/.exec(line);
|
||||
const safe = match && frame(match[2],match[1],match[3],match[4]);
|
||||
return safe ? [safe] : [];
|
||||
});
|
||||
if (frames.length) out[key] = frames.join("\n");
|
||||
}
|
||||
return out;
|
||||
}
|
||||
/** Discard exception messages, absolute roots, external frames and unknown function names. */
|
||||
export function errorDiagnostics(error: unknown): Record<string,string> {
|
||||
if (!error || typeof error !== "object") return {};
|
||||
const out: Record<string,string> = {};
|
||||
const seen = new Set<unknown>();
|
||||
let current: unknown = error;
|
||||
for(let depth=0; current && typeof current === "object" && depth<3 && !seen.has(current); depth++) {
|
||||
seen.add(current);
|
||||
const name = data(current,"name",true);
|
||||
const type = typeof name === "string" && LOG_ERROR_TYPES.includes(name) ? name : "UnknownError";
|
||||
const stack = data(current,"stack");
|
||||
const frames: string[] = [];
|
||||
if(typeof stack === "string") for(const line of stack.slice(0,16384).split("\n").slice(1,33)) {
|
||||
const match = /^\s*at (?:async )?(?:([^()]+) \()?([^()\s]+):(\d+):(\d+)\)?$/.exec(line);
|
||||
if(!match) continue;
|
||||
const local = relative(root,match[2]).split(sep).join("/");
|
||||
if(local.startsWith("../") || local.includes(":")) continue;
|
||||
const file = (root.endsWith(sep+"build") ? "build/" : "src/")+local;
|
||||
const candidate = (match[1] || "").trim().split(".").at(-1) || "<anonymous>";
|
||||
const fn = LOG_SOURCE_FUNCTIONS[file]?.includes(candidate) ? candidate : "<anonymous>";
|
||||
const safe = frame(file,fn,match[3],match[4]);
|
||||
if(safe && frames.join("\n").length + safe.length + 1 <= 1024) frames.push(safe);
|
||||
if(frames.length===8) break;
|
||||
}
|
||||
if (depth) delete out.causeStack;
|
||||
out[depth ? "causeType" : "errorType"] = type;
|
||||
if(frames.length) out[depth ? "causeStack" : "errorStack"] = frames.join("\n");
|
||||
current = data(current,"cause");
|
||||
}
|
||||
return out;
|
||||
}
|
||||
+17
-3
@@ -1,3 +1,6 @@
|
||||
import { requestReproduction, safeReproductionPath } from "./log-reproduction";
|
||||
import { safeDiagnosticFields, errorDiagnostics } from "./log-diagnostics";
|
||||
import { LOG_ROUTES } from "./log-diagnostic-schema";
|
||||
import { LOG_MODULES, ERROR_CODES, LOG_EVENTS } from "./log-schema";
|
||||
import { randomUUID } from "crypto";
|
||||
import { logContext, validTraceId } from "./log-context";
|
||||
@@ -28,7 +31,7 @@ export function logRecord(level: string, module: string, args: unknown[]) {
|
||||
const context = logContext.getStore() || {};
|
||||
const details: Record<string, unknown> = {};
|
||||
const keys = [...numbers, "requestId", "jobId", "code", "message", "outcome", "method", "route", "queue",
|
||||
"stages", "reason", "phase", "event", "deliveryId", "hostname", "service"];
|
||||
"errorType", "errorStack", "causeType", "causeStack", "requestPath", "stages", "reason", "phase", "event", "deliveryId", "hostname", "service"];
|
||||
// Do not traverse arbitrary objects or invoke error getters while logging.
|
||||
for (const arg of args.slice(0, 8)) if (arg && typeof arg === "object") {
|
||||
for (const key of keys) {
|
||||
@@ -38,6 +41,9 @@ export function logRecord(level: string, module: string, args: unknown[]) {
|
||||
} catch { /* A hostile Proxy must not make logging throw. */ }
|
||||
}
|
||||
}
|
||||
for (const arg of args.slice(0,8)) {
|
||||
try { if (arg instanceof Error) Object.assign(details, errorDiagnostics(arg)); } catch { /* hostile Proxy */ }
|
||||
}
|
||||
const first = typeof args[0] === "string" && args[0].length < 256 ? args[0] : "";
|
||||
const event = (Object.hasOwn(events, first) ? events[first] : undefined) || (module.includes("webhook") ? "webhook_log" : "application_log");
|
||||
const record: Record<string, unknown> = { timestamp: new Date().toISOString(), severity: level,
|
||||
@@ -60,8 +66,16 @@ export function logRecord(level: string, module: string, args: unknown[]) {
|
||||
else if (details.hostname) record.upstreamHost = "other";
|
||||
if (allowed(["completed", "interrupted"], details.outcome)) record.outcome = details.outcome;
|
||||
if (allowed(["GET", "HEAD", "POST", "PUT", "PATCH", "DELETE", "OPTIONS", "OTHER"], details.method)) record.method = details.method;
|
||||
// Route labels are generated by request-monitoring, never from URLs in arbitrary errors.
|
||||
if (module === "requests" && typeof details.route === "string") record.route = details.route;
|
||||
// Only fixed route labels survive; never raw URLs or request parameters.
|
||||
const route = context.route || details.route;
|
||||
if (typeof route === "string" && LOG_ROUTES.includes(route)) record.route = route;
|
||||
if (context.method && allowed(["GET", "HEAD", "POST", "PUT", "PATCH", "DELETE", "OPTIONS", "OTHER"], context.method)) record.method = context.method;
|
||||
Object.assign(record, safeDiagnosticFields(details));
|
||||
// Reproduction data is restricted to failures, not successful traffic/metrics.
|
||||
if (["error", "fatal"].includes(level) || Number(details.status || details.httpStatus) >= 400) {
|
||||
const requestPath = safeReproductionPath(context.requestPath || details.requestPath);
|
||||
if (requestPath) Object.assign(record, requestReproduction(requestPath));
|
||||
}
|
||||
if (allowed(["download", "remove", "cache"], details.queue)) record.queue = details.queue;
|
||||
if (details.stages && typeof details.stages === "object") for (const key of stages) {
|
||||
let value: unknown;
|
||||
|
||||
@@ -0,0 +1,19 @@
|
||||
// Explicit admin diagnostics: public anonymous repository ID and request pathname only.
|
||||
// Never collect upstream owner/repository URLs, headers, query strings or bodies.
|
||||
const fileRequest = /^\/api\/repo\/([A-Za-z0-9_-]{3,200})\/file\/([A-Za-z0-9._~!$&'()+,;=@%/-]+)$/;
|
||||
export function safeReproductionPath(value: unknown): string | undefined {
|
||||
if (typeof value !== "string" || value.length > 2048) return undefined;
|
||||
const match = fileRequest.exec(value);
|
||||
if (!match || /%(?![a-fA-F0-9]{2})/.test(value)) return undefined;
|
||||
try {
|
||||
const path = decodeURIComponent(match[2]);
|
||||
// Reject controls, traversal, URL syntax, encoded delimiters and double encoding.
|
||||
// eslint-disable-next-line no-control-regex -- reject control characters from stored paths
|
||||
if (/[\x00-\x1f\x7f-\x9f\u2028-\u202e\u2066-\u2069\ufffd\\:?#%]/.test(path) || path.startsWith("/") || path.split("/").some(part => part === "." || part === "..")) return undefined;
|
||||
} catch { return undefined; }
|
||||
return value;
|
||||
}
|
||||
export function requestReproduction(url: string) {
|
||||
const requestPath = safeReproductionPath(url.split("?", 1)[0]);
|
||||
return requestPath ? { requestPath, repoId: fileRequest.exec(requestPath)![1] } : {};
|
||||
}
|
||||
+17
-1
@@ -1,3 +1,4 @@
|
||||
import { errorDiagnostics } from "./log-diagnostics";
|
||||
import { logRecord } from "./log-record";
|
||||
import { redactSecrets } from "./redact-secrets";
|
||||
import { createClient, RedisClientType } from "redis";
|
||||
@@ -148,6 +149,21 @@ function clampPayload(entry: {
|
||||
s = JSON.stringify(entry);
|
||||
if (s.length <= MAX_PAYLOAD_BYTES) return s;
|
||||
}
|
||||
// Keep reproduction/correlation when bounded paths plus diagnostic stacks exceed
|
||||
// the mirror limit. Reduce frame depth and drop secondary measurements first.
|
||||
if (first && typeof first === "object" && !Array.isArray(first)) {
|
||||
const diagnostic = first as Record<string, unknown>;
|
||||
const compact: Record<string, unknown> = {};
|
||||
for (const key of ["timestamp", "severity", "service", "event", "module", "release", "requestId", "jobId", "method", "route", "requestPath", "repoId", "httpStatus", "status", "errorCode", "errorType", "causeType", "code", "message"]) {
|
||||
if (diagnostic[key] !== undefined) compact[key] = diagnostic[key];
|
||||
}
|
||||
for (const key of ["errorStack", "causeStack"]) {
|
||||
if (typeof diagnostic[key] === "string") compact[key] = diagnostic[key].split("\n")[0];
|
||||
}
|
||||
entry.raw = [compact];
|
||||
s = JSON.stringify(entry);
|
||||
if (s.length <= MAX_PAYLOAD_BYTES) return s;
|
||||
}
|
||||
// Step 3: nothing left to trim — replace with a placeholder so the entry
|
||||
// still shows up in the list.
|
||||
entry.raw = [{ truncated: true, originalBytes: s.length }];
|
||||
@@ -307,7 +323,7 @@ export function serializeError(err: unknown, seen = new WeakSet<object>(), depth
|
||||
if (seen.has(err) || depth >= 4) return { message: "error_details_truncated" };
|
||||
seen.add(err);
|
||||
const e = err as ErrorLike;
|
||||
const out: Record<string, unknown> = {};
|
||||
const out: Record<string, unknown> = errorDiagnostics(err);
|
||||
if (e.name) out.name = e.name;
|
||||
if (e.message) out.message = e.message;
|
||||
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { requestReproduction } from "./log-reproduction";
|
||||
import { logContext } from "./log-context";
|
||||
import { AsyncLocalStorage } from "async_hooks";
|
||||
import { randomUUID } from "crypto";
|
||||
@@ -116,6 +117,7 @@ export function monitorRequests(service: Service, metrics = requestMetrics,
|
||||
: req.originalUrl.startsWith("/w/") ? "website" : "file";
|
||||
const trace = { id, service, traffic, stages: {} };
|
||||
const route = requestRoute(req.originalUrl, service);
|
||||
const reproduction = service === "api" ? requestReproduction(req.originalUrl) : {};
|
||||
const method = methods.has(req.method) ? req.method : "OTHER";
|
||||
const started = performance.now();
|
||||
let firstByteMs: number | undefined;
|
||||
@@ -156,7 +158,7 @@ export function monitorRequests(service: Service, metrics = requestMetrics,
|
||||
metrics.observe("request", route, method, ms, { bytes, aborted, error: res.statusCode >= 500, slow });
|
||||
if (firstByteMs != null) metrics.observe("first_byte", route, method, firstByteMs);
|
||||
const responseMs = firstByteMs == null ? null : Math.max(0, ms - firstByteMs);
|
||||
const detail = { requestId: id, service, method, route, status: res.statusCode,
|
||||
const detail = { ...(res.statusCode >= 400 ? reproduction : {}), requestId: id, service, method, route, status: res.statusCode,
|
||||
outcome: aborted ? "interrupted" : "completed", ms: Math.round(ms), firstByteMs: firstByteMs == null ? null : Math.round(firstByteMs),
|
||||
responseMs: responseMs == null ? null : Math.round(responseMs),
|
||||
bytesPerSecond: responseMs != null && responseMs >= 1 && bytes > 0 ? Math.round(bytes * 1000 / responseMs) : null,
|
||||
@@ -165,6 +167,6 @@ export function monitorRequests(service: Service, metrics = requestMetrics,
|
||||
else if (!route.includes("healthcheck")) log.info("request", detail);
|
||||
};
|
||||
res.once("finish", done); res.once("close", done);
|
||||
context.run(trace, () => logContext.run({ requestId: id, service }, next));
|
||||
context.run(trace, () => logContext.run({ ...reproduction, requestId: id, service, route, method }, next));
|
||||
};
|
||||
}
|
||||
@@ -53,10 +53,10 @@ function safeRow(input: Record<string, unknown>): Row | null {
|
||||
const numeric = ["status", "httpStatus", "ms", "firstByteMs", "responseMs", "attempt", "rss", "memoryRatio", "droppedLogs", "oldestWaitingMs", "waiting"];
|
||||
const data: Record<string, unknown> = { ...input, code: input.errorCode };
|
||||
for (const key of numeric) if (typeof input[key] === "string" && /^-?\d+(?:\.\d+)?$/.test(input[key] as string)) data[key] = Number(input[key]);
|
||||
// Routes and arbitrary module text never pass through this projection.
|
||||
const projected = logContext.run({}, () => logRecord(["error", "warn", "info", "debug"].includes(String(input.severity)) ? String(input.severity) : "info", String(input.module || "application"), ["", { ...data, route: undefined }]));
|
||||
// Revalidate finite route and diagnostic allowlists; never pass raw strings.
|
||||
const projected = logContext.run({}, () => logRecord(["error", "warn", "info", "debug"].includes(String(input.severity)) ? String(input.severity) : "info", String(input.module || "application"), ["", data]));
|
||||
const result: Row = { timestamp: /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d{1,9})?Z$/.test(time) ? time : new Date(time).toISOString(), severity: String(projected.severity), service: services.includes(String(input.service)) ? String(input.service) : "unknown", event: events.has(String(input.event)) ? String(input.event) : "application_log", module: String(projected.module), release: /^(?:[a-f0-9]{7,40}|unknown)$/.test(String(input.release)) ? String(input.release) : "unknown" };
|
||||
for (const key of [...numeric, "errorCode", "requestId", "jobId", "queue"]) if (typeof projected[key] === "string" || typeof projected[key] === "number") result[key] = projected[key] as string | number;
|
||||
for (const key of [...numeric, "errorCode", "requestId", "jobId", "queue", "route", "method", "errorType", "errorStack", "causeType", "causeStack", "repoId", "requestPath"]) if (typeof projected[key] === "string" || typeof projected[key] === "number") result[key] = projected[key] as string | number;
|
||||
if (validTraceId(input.instance)) result.instance = input.instance;
|
||||
return result;
|
||||
}
|
||||
@@ -64,7 +64,7 @@ export function safeRedisLog(entry: Record<string, unknown>) {
|
||||
const detail = Array.isArray(entry.raw) ? entry.raw.find(value => value && typeof value === "object" && !Array.isArray(value)) || {} : {};
|
||||
const row = safeRow({ ...detail, _time: entry.ts, severity: entry.level, module: entry.module,
|
||||
errorCode: detail.errorCode || detail.code || detail.message, event: detail.event || entry.message });
|
||||
return row ? { ts: row.timestamp, level: row.severity, module: row.module, message: row.errorCode || row.event, raw: [row] } : null;
|
||||
return row ? { ts: row.timestamp, level: row.severity, module: row.module, message: row.errorCode || row.errorType || row.event, raw: [row] } : null;
|
||||
}
|
||||
async function rows(base: string, query: string, limit: number, signal: AbortSignal) {
|
||||
const text = await read(base + "/select/logsql/query", signal, new URLSearchParams({ query, limit: String(limit) }));
|
||||
@@ -97,7 +97,7 @@ function timestampKey(value: string | number) {
|
||||
function group(records: Row[]) {
|
||||
const groups = new Map<string, { fingerprint: string; count: number; first: string; last: string; sample: Row }>();
|
||||
for (const row of records) {
|
||||
const key = JSON.stringify([row.service, row.module, row.event, row.errorCode || "", row.severity, row.status || row.httpStatus || ""]);
|
||||
const key = JSON.stringify([row.service, row.module, row.event, row.errorCode || "", row.severity, row.status || row.httpStatus || "", row.errorType || "", String(row.errorStack || "").split("\n")[0]]);
|
||||
const fingerprint = createHash("sha256").update(key).digest("hex").slice(0, 24);
|
||||
const existing = groups.get(fingerprint);
|
||||
if (existing) { existing.count++; existing.first = timestampKey(existing.first) < timestampKey(row.timestamp) ? existing.first : String(row.timestamp); existing.last = timestampKey(existing.last) > timestampKey(row.timestamp) ? existing.last : String(row.timestamp); }
|
||||
|
||||
Reference in new issue
Block a user