mirror of
https://github.com/tdurieux/anonymous_github.git
synced 2026-11-18 01:54:23 +01:00
feat(admin): investigate correlated logs from the Errors page
This commit is contained in:
1 parent
f084f5cbfc
commit
8b928a4c82
14 files changed
+756
-67
No files matched your search
@@ -0,0 +1,168 @@
|
||||
import { createHash } from "crypto";
|
||||
import { logRecord } from "../core/log-record";
|
||||
import { logContext, validTraceId } from "../core/log-context";
|
||||
import { LOG_EVENTS } from "../core/log-schema";
|
||||
|
||||
const services = ["api", "streamer", "worker", "nginx", "mongodb", "redis", "backup"];
|
||||
const events = new Set([...Object.values(LOG_EVENTS), "application_log", "webhook_log", "request_completed", "request_slow", "request_cancelled", "process_fatal_error", "process_started", "process_heartbeat", "job_started", "job_completed", "job_failed", "job_stalled", "queue_heartbeat", "edge_request", "infrastructure_log"]);
|
||||
const windows: Record<string, number> = { "15m": 15, "1h": 60, "6h": 360, "24h": 1440 };
|
||||
let active = 0;
|
||||
export class InvalidLogQuery extends Error {}
|
||||
export function logOptions(query: Record<string, unknown>, correlation = false) {
|
||||
const permitted = correlation ? ["window", "kind", "id"] : ["window", "service", "release"];
|
||||
if (Object.keys(query).some(key => !permitted.includes(key)) || Object.values(query).some(value => typeof value !== "string")) throw new InvalidLogQuery();
|
||||
const window = (query.window as string) || "1h";
|
||||
if (!Object.hasOwn(windows, window)) throw new InvalidLogQuery();
|
||||
const service = (query.service as string) || "";
|
||||
const release = (query.release as string) || "";
|
||||
if (service && !services.includes(service)) throw new InvalidLogQuery();
|
||||
if (release && !/^(?:[a-f0-9]{7,40}|unknown)$/.test(release)) throw new InvalidLogQuery();
|
||||
if (correlation && (!validTraceId(query.id) || !["requestId", "jobId"].includes(String(query.kind)))) throw new InvalidLogQuery();
|
||||
const end = new Date().toISOString();
|
||||
const start = new Date(Date.now() - windows[window] * 60000).toISOString();
|
||||
return { window, start, end, service, release, id: query.id as string, kind: query.kind as "requestId" | "jobId" };
|
||||
}
|
||||
type Options = ReturnType<typeof logOptions>;
|
||||
type Row = Record<string, string | number>;
|
||||
function endpoint(value: string | undefined) {
|
||||
if (!value) return null;
|
||||
const url = new URL(value);
|
||||
if (!["http:", "https:"].includes(url.protocol) || url.username || url.password || url.search || url.hash || url.pathname !== "/") throw new Error("invalid_operator_endpoint");
|
||||
return url.origin;
|
||||
}
|
||||
// A total response byte cap applies even if Content-Length is absent or dishonest.
|
||||
async function read(url: string, signal: AbortSignal, body?: URLSearchParams) {
|
||||
const response = await fetch(url, { method: body ? "POST" : "GET", body, signal, redirect: "error", headers: body ? { "Content-Type": "application/x-www-form-urlencoded" } : undefined });
|
||||
if (!response.ok || !response.body) throw new Error("logging_unavailable");
|
||||
const reader = response.body.getReader();
|
||||
const chunks: Uint8Array[] = []; let bytes = 0;
|
||||
try {
|
||||
for (;;) {
|
||||
const part = await reader.read(); if (part.done) break;
|
||||
bytes += part.value.byteLength;
|
||||
if (bytes > 2 * 1024 * 1024) throw new Error("logging_response_limit");
|
||||
chunks.push(part.value);
|
||||
}
|
||||
return Buffer.concat(chunks).toString("utf8");
|
||||
} finally { await reader.cancel().catch(() => undefined); }
|
||||
}
|
||||
function safeRow(input: Record<string, unknown>): Row | null {
|
||||
const time = typeof input._time === "string" ? input._time : input.timestamp;
|
||||
if (typeof time !== "string" || !Number.isFinite(Date.parse(time))) return null;
|
||||
// Reuse the application's reviewed projection. Never return raw backend data.
|
||||
const numeric = ["status", "httpStatus", "ms", "firstByteMs", "responseMs", "attempt", "rss", "memoryRatio", "droppedLogs", "oldestWaitingMs", "waiting"];
|
||||
const data: Record<string, unknown> = { ...input, code: input.errorCode };
|
||||
for (const key of numeric) if (typeof input[key] === "string" && /^-?\d+(?:\.\d+)?$/.test(input[key] as string)) data[key] = Number(input[key]);
|
||||
// Routes and arbitrary module text never pass through this projection.
|
||||
const projected = logContext.run({}, () => logRecord(["error", "warn", "info", "debug"].includes(String(input.severity)) ? String(input.severity) : "info", String(input.module || "application"), ["", { ...data, route: undefined }]));
|
||||
const result: Row = { timestamp: /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d{1,9})?Z$/.test(time) ? time : new Date(time).toISOString(), severity: String(projected.severity), service: services.includes(String(input.service)) ? String(input.service) : "unknown", event: events.has(String(input.event)) ? String(input.event) : "application_log", module: String(projected.module), release: /^(?:[a-f0-9]{7,40}|unknown)$/.test(String(input.release)) ? String(input.release) : "unknown" };
|
||||
for (const key of [...numeric, "errorCode", "requestId", "jobId", "queue"]) if (typeof projected[key] === "string" || typeof projected[key] === "number") result[key] = projected[key] as string | number;
|
||||
if (validTraceId(input.instance)) result.instance = input.instance;
|
||||
return result;
|
||||
}
|
||||
export function safeRedisLog(entry: Record<string, unknown>) {
|
||||
const detail = Array.isArray(entry.raw) ? entry.raw.find(value => value && typeof value === "object" && !Array.isArray(value)) || {} : {};
|
||||
const row = safeRow({ ...detail, _time: entry.ts, severity: entry.level, module: entry.module,
|
||||
errorCode: detail.errorCode || detail.code || detail.message, event: detail.event || entry.message });
|
||||
return row ? { ts: row.timestamp, level: row.severity, module: row.module, message: row.errorCode || row.event, raw: [row] } : null;
|
||||
}
|
||||
async function rows(base: string, query: string, limit: number, signal: AbortSignal) {
|
||||
const text = await read(base + "/select/logsql/query", signal, new URLSearchParams({ query, limit: String(limit) }));
|
||||
const lines = text.split("\n").filter(Boolean);
|
||||
let rejected = 0;
|
||||
const records = lines.slice(0, limit).flatMap(line => {
|
||||
try { const row = safeRow(JSON.parse(line)); if (row) return [row]; } catch { /* Malformed records are not raw UI diagnostics. */ }
|
||||
rejected++; return [];
|
||||
});
|
||||
return { records, partial: lines.length >= limit || rejected > 0 };
|
||||
}
|
||||
const range = (o: Options) => `_time:[${o.start}, ${o.end}]`;
|
||||
function grafana(o: Options) {
|
||||
try {
|
||||
const value = process.env.ADMIN_LOG_GRAFANA_URL;
|
||||
if (!value) return null;
|
||||
const url = new URL(value);
|
||||
if (!["http:", "https:"].includes(url.protocol) || url.username || url.password) return null;
|
||||
url.search = ""; url.hash = "";
|
||||
url.searchParams.set("from", String(Date.parse(o.start))); url.searchParams.set("to", String(Date.parse(o.end)));
|
||||
if (o.id) url.searchParams.set(o.kind === "jobId" ? "var-job_id" : "var-request_id", o.id);
|
||||
return url.toString();
|
||||
} catch { return null; }
|
||||
}
|
||||
// ISO Z timestamps may mix seconds, milliseconds and nanoseconds. Pad only the
|
||||
// comparison key; keep original precision in the displayed event.
|
||||
function timestampKey(value: string | number) {
|
||||
return String(value).replace(/(?:\.(\d+))?Z$/, (_match, fraction = "") => `.${fraction.padEnd(9, "0")}Z`);
|
||||
}
|
||||
function group(records: Row[]) {
|
||||
const groups = new Map<string, { fingerprint: string; count: number; first: string; last: string; sample: Row }>();
|
||||
for (const row of records) {
|
||||
const key = JSON.stringify([row.service, row.module, row.event, row.errorCode || "", row.severity, row.status || row.httpStatus || ""]);
|
||||
const fingerprint = createHash("sha256").update(key).digest("hex").slice(0, 24);
|
||||
const existing = groups.get(fingerprint);
|
||||
if (existing) { existing.count++; existing.first = timestampKey(existing.first) < timestampKey(row.timestamp) ? existing.first : String(row.timestamp); existing.last = timestampKey(existing.last) > timestampKey(row.timestamp) ? existing.last : String(row.timestamp); }
|
||||
else groups.set(fingerprint, { fingerprint, count: 1, first: String(row.timestamp), last: String(row.timestamp), sample: row });
|
||||
}
|
||||
return [...groups.values()].sort((a, b) => timestampKey(b.last).localeCompare(timestampKey(a.last)));
|
||||
}
|
||||
async function collector(signal: AbortSignal) {
|
||||
try {
|
||||
const base = endpoint(process.env.ADMIN_LOG_VECTOR_URL);
|
||||
if (!base) return { state: "disabled", dropped: null };
|
||||
const metrics = await read(base + "/metrics", signal);
|
||||
let dropped = 0; let observed = false;
|
||||
for (const line of metrics.split("\n")) {
|
||||
if (/^vector_buffer_max_size_/.test(line)) observed = true;
|
||||
if (/^vector_buffer_discarded_events_total[ {]/.test(line) || (/^vector_component_discarded_events_total[ {]/.test(line) && !line.includes('intentional="true"'))) {
|
||||
const value = Number(line.split(/\s+/)[1]); if (Number.isFinite(value)) dropped += value;
|
||||
}
|
||||
}
|
||||
return { state: observed ? "reachable" : "unknown", dropped: observed ? dropped : null };
|
||||
} catch { return { state: "unreachable", dropped: null }; }
|
||||
}
|
||||
export async function investigateLogs(options: Options, correlation = false) {
|
||||
const common = { source: "victorialogs", window: options.window, start: options.start, end: options.end, redisCollecting: process.env.LOG_REDIS_MIRROR !== "off", grafanaUrl: grafana(options) };
|
||||
let base: string | null;
|
||||
try { base = endpoint(process.env.ADMIN_LOG_URL); } catch { return { ...common, state: "misconfigured", groups: [], events: [] }; }
|
||||
if (!base) return { ...common, state: "disabled", groups: [], events: [] };
|
||||
if (active >= 2) return { ...common, state: "busy", groups: [], events: [] };
|
||||
active++;
|
||||
const controller = new AbortController();
|
||||
const timer = setTimeout(() => controller.abort(), 4000); timer.unref();
|
||||
try {
|
||||
if (correlation) {
|
||||
const first = await rows(base, `${range(options)} ${options.kind}:"${options.id}" | sort by (_time)`, 201, controller.signal);
|
||||
let records = first.records; let partial = first.partial;
|
||||
if (options.kind === "jobId") {
|
||||
const allIds = [...new Set(records.map(row => row.requestId).filter(validTraceId))];
|
||||
const ids = allIds.slice(0, 4);
|
||||
partial ||= allIds.length > 4;
|
||||
if (ids.length) {
|
||||
const linked = await rows(base, `${range(options)} (${ids.map(id => `requestId:"${id}"`).join(" OR ")}) NOT jobId:"${options.id}" | sort by (_time)`, 201, controller.signal);
|
||||
records = [...records, ...linked.records]; partial ||= linked.partial;
|
||||
}
|
||||
}
|
||||
// Queries are disjoint by jobId; preserve legitimate identical events.
|
||||
const unique = records.sort((a, b) => timestampKey(a.timestamp).localeCompare(timestampKey(b.timestamp)));
|
||||
return { ...common, state: partial || unique.length > 200 ? "partial" : unique.length ? "ready" : "empty", events: unique.slice(0, 200), limit: 200 };
|
||||
}
|
||||
const filter = `${range(options)} (severity:error OR severity:warn)${options.service ? ` service:${options.service}` : ""}${options.release ? ` release:${options.release}` : ""}`;
|
||||
const [errors, heartbeat, drops] = await Promise.all([
|
||||
rows(base, `${filter} | sort by (_time) desc`, 1001, controller.signal),
|
||||
rows(base, '_time:5m (event:process_heartbeat OR event:queue_heartbeat OR event:job_completed OR event:job_failed) | sort by (_time) desc', 501, controller.signal),
|
||||
collector(controller.signal),
|
||||
]);
|
||||
const latest = new Map<string, Row>();
|
||||
for (const row of heartbeat.records) {
|
||||
const key = `${row.service}:${row.instance || ""}:${row.queue || ""}`;
|
||||
if (!latest.has(key)) latest.set(key, row);
|
||||
}
|
||||
const recent = heartbeat.records.filter(row => Date.parse(String(row.timestamp)) > Date.now() - 90000);
|
||||
const streamers = new Set(recent.filter(row => row.service === "streamer" && row.event === "process_heartbeat").map(row => row.instance).filter(Boolean)).size;
|
||||
const queues = new Set(recent.filter(row => row.event === "queue_heartbeat").map(row => row.queue).filter(Boolean)).size;
|
||||
const missing = [...(!recent.some(row => row.service === "api" && row.event === "process_heartbeat") ? ["API"] : []), ...(streamers < 4 ? ["streamers"] : []), ...(queues < 3 ? ["queues"] : [])];
|
||||
const records = errors.records.slice(0, 1000);
|
||||
return { ...common, state: errors.partial || heartbeat.partial ? "partial" : missing.length ? "stale" : records.length ? "ready" : "empty", groups: group(records), captured: records.length, limit: 1000, telemetry: { missing, streamers, queues, partial: heartbeat.partial, instances: [...latest.values()].map(row => ({ service: row.service, instance: row.instance, last: row.timestamp, dropped: row.droppedLogs ?? null })), collector: drops } };
|
||||
} catch { return { ...common, state: "unreachable", groups: [], events: [] }; }
|
||||
finally { controller.abort(); clearTimeout(timer); active--; }
|
||||
}
|
||||
@@ -1,3 +1,4 @@
|
||||
import { investigateLogs, logOptions, safeRedisLog } from "../admin-log-investigation";
|
||||
import { jobTrace } from "../../core/log-context";
|
||||
import { cacheMetricsReport } from "../../core/cache-observability";
|
||||
import { sharedBlobCache, resultBlobCache } from "../../core/blob-cache";
|
||||
@@ -52,7 +53,11 @@ async function getErrorLogClient(): Promise<RedisClientType | null> {
|
||||
socket: {
|
||||
host: config.REDIS_HOSTNAME,
|
||||
port: config.REDIS_PORT,
|
||||
connectTimeout: 1500,
|
||||
reconnectStrategy: false,
|
||||
},
|
||||
disableOfflineQueue: true,
|
||||
commandsQueueMaxLength: 32,
|
||||
}) as RedisClientType;
|
||||
errorLogClient.on("error", () => undefined);
|
||||
await errorLogClient.connect();
|
||||
@@ -84,6 +89,17 @@ router.use(
|
||||
}
|
||||
);
|
||||
|
||||
// These routes inherit both authentication and admin authorization above.
|
||||
for (const [path, correlation] of [["/errors/investigation", false], ["/errors/correlation", true]] as const) {
|
||||
router.get(path, async (req, res) => {
|
||||
res.setHeader("Cache-Control", "no-store");
|
||||
let options;
|
||||
try { options = logOptions(req.query as Record<string, unknown>, correlation); }
|
||||
catch { return res.status(400).json({ error: "invalid_request" }); }
|
||||
return res.json(await investigateLogs(options, correlation));
|
||||
});
|
||||
}
|
||||
|
||||
router.use("/tokens", adminTokensRouter);
|
||||
router.use("/billing", adminBillingRouter);
|
||||
|
||||
@@ -450,6 +466,7 @@ router.get("/errors", async (req, res) => {
|
||||
total: 0,
|
||||
max: ERROR_LOG_MAX,
|
||||
available: false,
|
||||
redisCollecting: process.env.LOG_REDIS_MIRROR !== "off",
|
||||
});
|
||||
}
|
||||
const offset = Math.max(0, parseInt(String(req.query.offset || "0"), 10) || 0);
|
||||
@@ -464,18 +481,21 @@ router.get("/errors", async (req, res) => {
|
||||
]);
|
||||
const entries = raw.map((s) => {
|
||||
try {
|
||||
return JSON.parse(s);
|
||||
return safeRedisLog(JSON.parse(s));
|
||||
} catch {
|
||||
return { ts: null, module: null, message: s, raw: [] };
|
||||
return null;
|
||||
}
|
||||
});
|
||||
}).filter(Boolean);
|
||||
res.json({
|
||||
entries,
|
||||
offset,
|
||||
limit,
|
||||
total,
|
||||
nextOffset: offset + raw.length,
|
||||
redactedEntries: raw.length - entries.length,
|
||||
max: ERROR_LOG_MAX,
|
||||
available: true,
|
||||
redisCollecting: process.env.LOG_REDIS_MIRROR !== "off",
|
||||
});
|
||||
} catch (error) {
|
||||
handleError(error, res, req);
|
||||
|
||||
Reference in new issue
Block a user