fix: explain interrupted requests and account for overlapping latency

This commit is contained in:
Thomas Durieux committed 2026-11-15 13:31:16 +00:00
1 parent 1e617d2df0
commit d2e3954d21
18 files changed
+394 -33

No files matched your search

+2 -1
View File
@@ -1,3 +1,4 @@
import { recordUpstreamFailure } from "./request-evidence";
import { rejectStreamerRedirects } from "./streamer-response";
import { isResponseCancellation } from "./response-cancellation";
import { canReadCachedRepository } from "./cached-repository-access";
@@ -585,7 +586,7 @@ export default class AnonymizedFile {
reject(error);
}
content
.on("error", handleStreamError)
.on("error", error => { if (!res.destroyed) recordUpstreamFailure(res, error); handleStreamError(error); })
.pipe(anonymizer)
.on("error", handleStreamError)
.pipe(res)
+18
View File
@@ -0,0 +1,18 @@
import mongoose from "mongoose";
import { measureStage } from "./request-monitoring";
let installed = false;
/** Measure Mongoose query/aggregate execution, including pool/buffering wait.
* No query text, model names, results or parameters enter the logging context.
* Direct driver operations, document saves and bulk writes are not covered.
*/
export function installDatabaseTiming() {
if (installed) return;
installed = true;
for (const prototype of [mongoose.Query.prototype, mongoose.Aggregate.prototype]) {
const original = prototype.exec;
prototype.exec = function (this: unknown, ...args: unknown[]) {
return measureStage("database", () => Reflect.apply(original, this, args));
} as typeof original;
}
}
+1 -1
View File
@@ -1,7 +1,7 @@
import { AsyncLocalStorage } from "async_hooks";
import { randomUUID } from "crypto";
export interface LogContext { requestId?: string; jobId?: string; attempt?: number; service?: string; route?: string; method?: string; requestPath?: string }
export interface LogContext { requestId?: string; hopId?: string; parentHopId?: string; jobId?: string; attempt?: number; service?: string; route?: string; method?: string; requestPath?: string }
export const logContext = new AsyncLocalStorage<LogContext>();
export const validTraceId = (value: unknown): value is string => typeof value === "string" &&
/^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$/.test(value);
+45 -1
View File
@@ -858,6 +858,14 @@ export const LOG_SOURCE_FUNCTIONS: Record<string, readonly string[]> = {
"replaceCredential",
"setCredential"
],
"src/core/database-timing.ts": [
"<anonymous>",
"installDatabaseTiming"
],
"build/core/database-timing.js": [
"<anonymous>",
"installDatabaseTiming"
],
"src/core/email-history.ts": [
"<anonymous>",
"complete",
@@ -960,6 +968,14 @@ export const LOG_SOURCE_FUNCTIONS: Record<string, readonly string[]> = {
"<anonymous>",
"isGitHubRateLimitError"
],
"src/core/github-repository-picker.ts": [
"<anonymous>",
"repositoryPickerPage"
],
"build/core/github-repository-picker.js": [
"<anonymous>",
"repositoryPickerPage"
],
"src/core/github-token-context.ts": [
"<anonymous>",
"githubQuotaKey",
@@ -1482,34 +1498,54 @@ export const LOG_SOURCE_FUNCTIONS: Record<string, readonly string[]> = {
"build/core/repository-id.js": [
"<anonymous>"
],
"src/core/request-evidence.ts": [
"<anonymous>",
"expectedResponseBytes",
"recordResponseFailure",
"recordUpstreamFailure",
"responseEvidence"
],
"build/core/request-evidence.js": [
"<anonymous>",
"expectedResponseBytes",
"recordResponseFailure",
"recordUpstreamFailure",
"responseEvidence"
],
"src/core/request-monitoring.ts": [
"<anonymous>",
"activeRequestCount",
"advance",
"cacheTraffic",
"count",
"done",
"drain",
"finish",
"measureStage",
"monitorRequests",
"observe",
"recordCacheHit",
"requestHeaders",
"requestRoute",
"start",
"startStage"
],
"build/core/request-monitoring.js": [
"<anonymous>",
"activeRequestCount",
"advance",
"cacheTraffic",
"count",
"done",
"drain",
"finish",
"measureStage",
"monitorRequests",
"observe",
"recordCacheHit",
"requestHeaders",
"requestRoute",
"start",
"startStage"
],
"src/core/response-cancellation.ts": [
@@ -2104,6 +2140,14 @@ export const LOG_SOURCE_FUNCTIONS: Record<string, readonly string[]> = {
"<anonymous>",
"main"
],
"src/scripts/check-github-app-key.ts": [
"<anonymous>",
"checkGitHubAppKey"
],
"build/scripts/check-github-app-key.js": [
"<anonymous>",
"checkGitHubAppKey"
],
"src/scripts/create-performance-indexes.ts": [
"<anonymous>",
"main"
@@ -2860,4 +2904,4 @@ export const LOG_SOURCE_FUNCTIONS: Record<string, readonly string[]> = {
]
};
export const LOG_ERROR_TYPES = ["Error","TypeError","RangeError","ReferenceError","SyntaxError","URIError","EvalError","AggregateError","AnonymousError","HTTPError","RequestError","MongoServerError","MongoNetworkError","MongoServerSelectionError","ValidationError","TimeoutError","AbortError","UnknownError"];
export const LOG_ROUTES = ["/","other","/dashboard","/anonymize","/healthcheck","/api/healthcheck","/api","/api/download","/api/stat","/api/stat/history","/api/options","/api/message","/api/user","/api/user/quota","/api/user/dashboard","/api/repo","/api/anonymize-preview","/api/admin/overview","/api/admin/performance","/api/admin/errors","/api/admin/errors/stats","/api/admin/queues","/api/admin/queues/metrics","/github/app/webhook","/api/repo/:repoId","/api/repo/:repoId/file/:path","/api/repo/:repoId/files","/api/repo/:repoId/files/counts","/api/repo/:repoId/files/search","/api/repo/:repoId/zip","/api/repo/:repoId/options","/api/repo/:repoId/refresh","/api/repo/:repoId/coauthors","/api/repo/:repoId/extend","/api/repo/:owner/:repo/readme","/api/repo/:owner/:repo/branches","/w/:repoId/:path","/r/:repoId/:path","/api/pr/:id/:path","/api/gist/:id/:path","/assets/:path","/github/:action","/api/admin/:action","/api/user/:action","/api/repo/:id/file/:path","/api/repo/:id/files/counts","/api/repo/:id/files/search","/api/repo/:id/files","/api/repo/:id/zip","/api/repo/:id","/viewer/:id/:path"];
export const LOG_ROUTES = ["/","other","/dashboard","/anonymize","/healthcheck","/api/healthcheck","/api","/api/download","/api/stat","/api/stat/history","/api/options","/api/message","/api/user","/api/user/quota","/api/user/dashboard","/api/repo","/api/anonymize-preview","/api/admin/overview","/api/admin/performance","/api/admin/errors","/api/admin/errors/stats","/api/admin/queues","/api/admin/queues/metrics","/github/app/webhook","/api/repo/:repoId","/api/repo/:repoId/file/:path","/api/repo/:repoId/files","/api/repo/:repoId/files/counts","/api/repo/:repoId/files/search","/api/repo/:repoId/zip","/api/repo/:repoId/options","/api/repo/:repoId/refresh","/api/repo/:repoId/coauthors","/api/repo/:repoId/extend","/api/repo/:owner/:repo","/api/repo/:owner/:repo/tree","/api/repo/:owner/:repo/file","/api/repo/:owner/:repo/readme","/api/repo/:owner/:repo/branches","/w/:repoId/:path","/r/:repoId/:path","/api/pr/:id/:path","/api/gist/:id/:path","/assets/:path","/github/:action","/api/admin/:action","/api/user/:action","/api/repo/:id/file/:path","/api/repo/:id/files/counts","/api/repo/:id/files/search","/api/repo/:id/files","/api/repo/:id/zip","/api/repo/:id","/viewer/:id/:path"];
+19 -7
View File
@@ -23,14 +23,14 @@ const errorCodes = new Set([
"ECONNRESET", "ECONNREFUSED", "ETIMEDOUT", "ENOTFOUND", "EPIPE", "ABORT_ERR",
"upstream_error", "upstream_timeout", "invalid_signature", "webhook_processing_failed",
]);
const numbers = ["status", "httpStatus", "ms", "firstByteMs", "responseMs", "bytes", "bytesPerSecond",
export const LOG_NUMBERS = ["expectedBytes", "measuredMs", "unattributedMs", "overlapMs","status", "httpStatus", "ms", "firstByteMs", "responseMs", "bytes", "bytesPerSecond",
"attempt", "rss", "memoryLimitBytes", "memoryRatio", "uptime", "activeRequests", "waiting", "active", "delayed", "failed",
"oldestWaitingMs", "droppedLogs", "loopP95Ms", "upstreamStatus"];
const stages = ["repository", "credential_resolution", "upstream", "cache_lookup", "cache_fill", "worker_wait", "anonymize", "streamer_headers"];
export const LOG_STAGES = ["transfer", "database","repository", "credential_resolution", "upstream", "cache_lookup", "cache_fill", "worker_wait", "anonymize", "streamer_headers"];
export function logRecord(level: string, module: string, args: unknown[]) {
const context = logContext.getStore() || {};
const details: Record<string, unknown> = {};
const keys = [...numbers, "requestId", "jobId", "code", "message", "outcome", "method", "route", "queue",
const keys = [...LOG_NUMBERS, "requestId", "jobId", "hopId", "parentHopId", "cancellationOrigin", "cancellationEvidence", "endpointKind", "stageCounts", "stageOpen", "code", "message", "outcome", "method", "route", "queue",
"errorType", "errorStack", "causeType", "causeStack", "requestPath", "stages", "reason", "phase", "event", "deliveryId", "hostname", "service"];
// Do not traverse arbitrary objects or invoke error getters while logging.
for (const arg of args.slice(0, 8)) if (arg && typeof arg === "object") {
@@ -50,12 +50,12 @@ export function logRecord(level: string, module: string, args: unknown[]) {
service: context.service || (allowed(["api", "streamer", "worker"], details.service) ? details.service : undefined) || (process.env.SERVICE_NAME?.toLowerCase() === "streamer" ? "streamer" : "api"),
instance, release: /^[a-f0-9]{7,40}$/.test(process.env.APP_RELEASE || "") ? process.env.APP_RELEASE : "unknown",
event, message: event, module: LOG_MODULES.includes(module) ? module : "application" };
for (const key of ["requestId", "jobId"] as const) {
for (const key of ["requestId", "jobId", "hopId", "parentHopId"] as const) {
const value = details[key] || context[key];
if (validTraceId(value)) record[key] = value;
}
if (Number.isSafeInteger(context.attempt)) record.attempt = context.attempt;
for (const key of numbers) if (typeof details[key] === "number" && Number.isFinite(details[key])) record[key] = details[key];
for (const key of LOG_NUMBERS) if (typeof details[key] === "number" && Number.isFinite(details[key])) record[key] = details[key];
const code = [details.code, details.message].find(value => typeof value === "string" && errorCodes.has(value));
if (code) record.errorCode = code;
if (allowed(["protocol", "credentials", "port", "host"], details.reason)) record.reason = details.reason;
@@ -64,21 +64,33 @@ export function logRecord(level: string, module: string, args: unknown[]) {
if (validTraceId(details.deliveryId)) record.deliveryId = details.deliveryId;
if (allowed(["github.com", "api.github.com", "codeload.github.com", "raw.githubusercontent.com", "media.githubusercontent.com"], details.hostname)) record.upstreamHost = details.hostname;
else if (details.hostname) record.upstreamHost = "other";
for (const [key, values] of Object.entries({ cancellationOrigin: ["unknown", "api", "streamer", "timeout", "upstream"],
cancellationEvidence: ["response_close", "request_aborted", "upstream_error", "timeout", "local_abort"], endpointKind: ["public", "internal"] })) {
if (allowed(values, details[key])) record[key] = details[key];
}
if (allowed(["completed", "interrupted"], details.outcome)) record.outcome = details.outcome;
if (allowed(["GET", "HEAD", "POST", "PUT", "PATCH", "DELETE", "OPTIONS", "OTHER"], details.method)) record.method = details.method;
// Only fixed route labels survive; never raw URLs or request parameters.
const route = context.route || details.route;
if (typeof route === "string" && LOG_ROUTES.includes(route)) record.route = route;
if (typeof route === "string" && LOG_ROUTES.includes(route)) record.route = route.replace(/^\/api\/repo\/:id(?=\/|$)/, "/api/repo/:repoId");
if (context.method && allowed(["GET", "HEAD", "POST", "PUT", "PATCH", "DELETE", "OPTIONS", "OTHER"], context.method)) record.method = context.method;
Object.assign(record, safeDiagnosticFields(details));
// Anonymous request paths are diagnostic fields, never metric labels.
const requestPath = safeReproductionPath(context.requestPath || details.requestPath, record.method);
if (requestPath) Object.assign(record, requestReproduction(requestPath, record.method));
if (allowed(["download", "remove", "cache"], details.queue)) record.queue = details.queue;
if (details.stages && typeof details.stages === "object") for (const key of stages) {
if (details.stages && typeof details.stages === "object") for (const key of LOG_STAGES) {
let value: unknown;
try { value = Object.getOwnPropertyDescriptor(details.stages, key)?.value; } catch { continue; }
if (typeof value === "number" && Number.isFinite(value)) record[`stage_${key}_ms`] = value;
}
for (const [field, suffix] of [["stageCounts", "count"], ["stageOpen", "open"]]) {
if (!details[field] || typeof details[field] !== "object") continue;
for (const stage of LOG_STAGES) {
let value: unknown;
try { value = Object.getOwnPropertyDescriptor(details[field], stage)?.value; } catch { continue; }
if (typeof value === "number" && Number.isSafeInteger(value) && value >= 0) record[`stage_${stage}_${suffix}`] = value;
}
}
return record;
}
+22
View File
@@ -0,0 +1,22 @@
export type FailureOrigin = "unknown" | "api" | "streamer" | "timeout" | "upstream";
export type FailureEvidence = "response_close" | "request_aborted" | "upstream_error" | "timeout" | "local_abort";
const evidence = new WeakMap<object, { cancellationOrigin: FailureOrigin; cancellationEvidence: FailureEvidence }>();
/** Record at the causal boundary, before destroying the response. First evidence wins. */
export function recordResponseFailure(response: object, origin: FailureOrigin, reason: FailureEvidence) {
if (!evidence.has(response)) evidence.set(response, { cancellationOrigin: origin, cancellationEvidence: reason });
}
export function recordUpstreamFailure(response: object, error: unknown, origin: "streamer" | "upstream" = "upstream") {
const code = error && typeof error === "object" ? Object.getOwnPropertyDescriptor(error, "code")?.value : undefined;
recordResponseFailure(response, code === "ETIMEDOUT" ? "timeout" : origin, code === "ETIMEDOUT" ? "timeout" : "upstream_error");
}
export function responseEvidence(response: object, aborted = false) {
return evidence.get(response) || { cancellationOrigin: "unknown", cancellationEvidence: aborted ? "request_aborted" : "response_close" };
}
/** Only output representation length is comparable to response writes. */
export function expectedResponseBytes(method: string, status: number, length: unknown): number | undefined {
if (method === "HEAD" || status === 204 || status === 304) return 0;
if ((typeof length !== "string" && typeof length !== "number") || !/^\d+$/.test(String(length))) return undefined;
const value = Number(length);
return Number.isSafeInteger(value) && value >= 0 ? value : undefined;
}
+69 -13
View File
@@ -1,5 +1,6 @@
import { responseEvidence, expectedResponseBytes } from "./request-evidence";
import { requestReproduction } from "./log-reproduction";
import { logContext } from "./log-context";
import { logContext, validTraceId } from "./log-context";
import { AsyncLocalStorage } from "async_hooks";
import { randomUUID } from "crypto";
import { performance } from "perf_hooks";
@@ -8,13 +9,55 @@ import { createLogger } from "./logger";
export const LATENCY_BUCKETS = [10, 50, 100, 250, 500, 1000, 2500, 5000, 10000, 30000, 60000];
export type Service = "api" | "streamer";
export type Stage = "repository" | "credential_resolution" | "upstream" | "cache_lookup" | "cache_fill" | "worker_wait" | "anonymize" | "streamer_headers";
export type Stage = "transfer" | "database" | "repository" | "credential_resolution" | "upstream" | "cache_lookup" | "cache_fill" | "worker_wait" | "anonymize" | "streamer_headers";
export interface Metric {
metric: string; route: string; method: string; count: number; sumMs: number;
bytes: number; aborted: number; errors: number; slow: number; buckets: number[];
}
const logger = createLogger("requests");
const context = new AsyncLocalStorage<{ id: string; service: Service; traffic?: string; stages: Partial<Record<Stage, number>> }>();
interface Trace { id: string; hopId: string; service: Service; traffic?: string; timing: RequestTiming }
const context = new AsyncLocalStorage<Trace>();
/** Constant storage: integrate active spans instead of retaining a per-call timeline. */
export class RequestTiming {
stages: Partial<Record<Stage, number>> = {};
counts: Partial<Record<Stage, number>> = {};
open: Partial<Record<Stage, number>> = {};
private last: number;
private covered = 0;
private concurrent = 0;
private ended = false;
constructor(private started = performance.now(), private now = () => performance.now()) { this.last = started; }
private advance(at: number) {
const elapsed = Math.max(0, at - this.last);
let active = 0;
for (const stage of Object.keys(this.open) as Stage[]) {
const count = this.open[stage] || 0;
this.stages[stage] = (this.stages[stage] || 0) + elapsed * count;
active += count;
}
if (active) this.covered += elapsed;
this.concurrent += elapsed * Math.max(0, active - 1);
this.last = at;
}
start(stage: Stage) {
if (this.ended) return () => undefined;
this.advance(this.now());
this.counts[stage] = (this.counts[stage] || 0) + 1;
this.open[stage] = (this.open[stage] || 0) + 1;
let done = false;
return () => {
if (done || this.ended) return;
done = true; this.advance(this.now()); this.open[stage]!--;
};
}
finish(at = this.now()) {
if (!this.ended) { this.advance(at); this.ended = true; }
return { stages: { ...this.stages }, stageCounts: { ...this.counts }, stageOpen: { ...this.open },
measuredMs: this.covered, unattributedMs: Math.max(0, this.last - this.started - this.covered), overlapMs: this.concurrent };
}
}
const methods = new Set(["GET", "HEAD", "POST", "PUT", "PATCH", "DELETE", "OPTIONS"]);
const staticRoutes = new Set([
"/", "/dashboard", "/anonymize", "/healthcheck", "/api/healthcheck", "/api/stat", "/api/stat/history",
@@ -24,11 +67,16 @@ const staticRoutes = new Set([
]);
/** Fixed labels: never use repository IDs, file names or query strings as metric keys. */
export function requestRoute(url: string, service: Service): string {
export function requestRoute(url: string, service: Service, method = "GET"): string {
const path = url.split("?", 1)[0].replace(/\/+$/, "") || "/";
if (service === "streamer") return ["/api/download", "/api", "/healthcheck"].includes(path) ? path : "other";
if (staticRoutes.has(path)) return path;
if (/^\/api\/repo\//.test(path)) {
if (["GET", "HEAD"].includes(method)) {
const source = path.match(/^\/api\/repo\/[^/]+\/[^/]+\/(readme|branches|tree|file)$/);
if (source && !/^\/api\/repo\/[^/]+\/file\//.test(path)) return `/api/repo/:owner/:repo/${source[1]}`;
if (/^\/api\/repo\/[^/]+\/[^/]+$/.test(path) && !/\/(file|files|zip|options)$/.test(path)) return "/api/repo/:owner/:repo";
}
const match = path.match(/^\/api\/repo\/[^/]+\/(file|files|zip|options|refresh|coauthors|extend)(?:\/|$)/);
if (match) {
const subroute = match[1] === "files" && /\/(counts|search)$/.exec(path);
@@ -79,7 +127,7 @@ let activeRequests = 0;
export function activeRequestCount() { return activeRequests; }
export function requestHeaders(): Record<string, string> {
const id = context.getStore()?.id;
return id ? { "x-request-id": id, "x-cache-traffic": cacheTraffic() } : {};
return id ? { "x-request-id": id, "x-cache-traffic": cacheTraffic(), "x-parent-hop-id": context.getStore()!.hopId } : {};
}
export function cacheTraffic() { return context.getStore()?.traffic || "background"; }
export function recordCacheHit(cache: "source" | "transformed", hit: boolean) {
@@ -90,12 +138,13 @@ export function recordCacheHit(cache: "source" | "transformed", hit: boolean) {
export function startStage(stage: Stage) {
const started = performance.now();
const trace = context.getStore();
const stop = trace?.timing.start(stage);
let ended = false;
return (error = false) => {
if (ended) return; ended = true;
if (!trace) return;
const ms = performance.now() - started;
trace.stages[stage] = (trace.stages[stage] || 0) + ms;
stop?.();
requestMetrics.observe(stage, "all", "all", ms, { error });
};
}
@@ -109,25 +158,29 @@ export function monitorRequests(service: Service, metrics = requestMetrics,
log: Pick<typeof logger, "info" | "warn"> = logger): RequestHandler {
return (req, res, next) => {
const forwarded = service === "streamer" ? req.get("x-request-id") : undefined;
const id = forwarded && /^[a-f0-9-]{36}$/.test(forwarded) ? forwarded : randomUUID();
const id = validTraceId(forwarded) ? forwarded : randomUUID();
const forwardedTraffic = service === "streamer" ? req.get("x-cache-traffic") : undefined;
const traffic = forwardedTraffic && ["file", "website", "zip", "monitor", "background"].includes(forwardedTraffic)
? forwardedTraffic : /uptime|healthcheck|monitor/i.test(req.get("user-agent") || "") ? "monitor"
: /\/zip(?:[/?]|$)|\/download(?:[/?]|$)/.test(req.originalUrl) ? "zip"
: req.originalUrl.startsWith("/w/") ? "website" : "file";
const trace = { id, service, traffic, stages: {} };
const route = requestRoute(req.originalUrl, service);
const started = performance.now();
const hopId = randomUUID();
const parent = service === "streamer" ? req.get("x-parent-hop-id") : undefined;
const parentHopId = validTraceId(parent) ? parent : undefined;
const trace = { id, hopId, service, traffic, timing: new RequestTiming(started) };
const route = requestRoute(req.originalUrl, service, req.method);
const reproduction = service === "api" ? requestReproduction(req.originalUrl, req.method) : {};
const method = methods.has(req.method) ? req.method : "OTHER";
const started = performance.now();
let firstByteMs: number | undefined;
let transferDone: (() => void) | undefined;
let bytes = 0;
let ended = false;
activeRequests++;
res.setHeader("X-Request-ID", id);
const writeHead = res.writeHead;
res.writeHead = function (this: typeof res, ...args: unknown[]): typeof res {
firstByteMs ??= performance.now() - started;
if (firstByteMs == null) { firstByteMs = performance.now() - started; transferDone = trace.timing.start("transfer"); }
return Reflect.apply(writeHead, this, args);
};
const count = (chunk: unknown, encoding: unknown) => {
@@ -151,6 +204,7 @@ export function monitorRequests(service: Service, metrics = requestMetrics,
if (ended) return; ended = true;
activeRequests--;
res.removeListener("finish", done); res.removeListener("close", done);
transferDone?.();
const ms = performance.now() - started;
const aborted = !res.writableFinished;
const transfer = /\/(file|zip)(?:\/|$)/.test(route) || route.startsWith("/w/") || service === "streamer";
@@ -162,11 +216,13 @@ export function monitorRequests(service: Service, metrics = requestMetrics,
outcome: aborted ? "interrupted" : "completed", ms: Math.round(ms), firstByteMs: firstByteMs == null ? null : Math.round(firstByteMs),
responseMs: responseMs == null ? null : Math.round(responseMs),
bytesPerSecond: responseMs != null && responseMs >= 1 && bytes > 0 ? Math.round(bytes * 1000 / responseMs) : null,
bytes, stages: trace.stages };
bytes, expectedBytes: expectedResponseBytes(req.method, res.statusCode, res.getHeader("content-length")),
endpointKind: service === "streamer" ? "internal" : "public", hopId, parentHopId,
...(aborted ? responseEvidence(res, req.aborted) : {}), ...trace.timing.finish(started + ms) };
if (slow || aborted) log.warn(aborted ? "request interrupted" : "slow request", detail);
else if (!route.includes("healthcheck")) log.info("request", detail);
};
res.once("finish", done); res.once("close", done);
context.run(trace, () => logContext.run({ ...reproduction, requestId: id, service, route, method }, next));
context.run(trace, () => logContext.run({ ...reproduction, requestId: id, hopId, parentHopId, service, route, method }, next));
};
}
+2 -1
View File
@@ -1,3 +1,4 @@
import { recordUpstreamFailure } from "./request-evidence";
import { PassThrough, Readable, Writable } from "stream";
/** Cancel upstream on every response terminal event, including a pre-header close. */
@@ -19,7 +20,7 @@ export function streamResponse(source: Readable, response: Writable,
};
const done = () => { cleanup(); resolve(); };
const failed = (error: Error) => { cleanup(); reject(error); };
const upstreamFailed = (error: Error) => { onError(error); done(); };
const upstreamFailed = (error: Error) => { if (!response.destroyed) recordUpstreamFailure(response, error, "streamer"); onError(error); done(); };
response.once("close", done);
response.once("finish", done);
response.once("error", failed);
+3 -1
View File
@@ -1,3 +1,4 @@
import { recordUpstreamFailure, recordResponseFailure } from "./request-evidence";
import got from "got";
import config from "../config";
import { MAX_SUBMODULE_ENTRIES, SubmoduleMount, safeRepositoryPath, validateMounts } from "./source/github-submodules";
@@ -169,6 +170,7 @@ export async function streamAnonymizedZip(
};
const fail = (error: Error) => {
if (stopped) return;
recordResponseFailure(res, "unknown", "local_abort");
logger.error("zip stream failed", serializeError(error));
stop();
const destroyable = res as unknown as {
@@ -197,7 +199,7 @@ export async function streamAnonymizedZip(
input.once("close", () => upstreamDone(!input.readableEnded));
parser = Parse() as Transform;
downloadStream
.on("error", fail)
.on("error", error => { recordUpstreamFailure(res, error); fail(error); })
.pipe(parser)
.on("entry", (entry: Readable & { type: string; path: string; autodrain: () => void }) => {
if (stopped) {
+6 -3
View File
@@ -1,5 +1,5 @@
import { createHash } from "crypto";
import { logRecord } from "../core/log-record";
import { logRecord, LOG_NUMBERS, LOG_STAGES } from "../core/log-record";
import { logContext, validTraceId } from "../core/log-context";
import { LOG_EVENTS } from "../core/log-schema";
@@ -50,13 +50,16 @@ function safeRow(input: Record<string, unknown>): Row | null {
const time = typeof input._time === "string" ? input._time : input.timestamp;
if (typeof time !== "string" || !Number.isFinite(Date.parse(time))) return null;
// Reuse the application's reviewed projection. Never return raw backend data.
const numeric = ["status", "httpStatus", "ms", "firstByteMs", "responseMs", "attempt", "rss", "memoryRatio", "droppedLogs", "oldestWaitingMs", "waiting"];
const numeric = [...LOG_NUMBERS, ...LOG_STAGES.flatMap(stage => [`stage_${stage}_ms`, `stage_${stage}_count`, `stage_${stage}_open`])];
const data: Record<string, unknown> = { ...input, code: input.errorCode };
for (const key of numeric) if (typeof input[key] === "string" && /^-?\d+(?:\.\d+)?$/.test(input[key] as string)) data[key] = Number(input[key]);
for (const [field, suffix] of [["stages", "ms"], ["stageCounts", "count"], ["stageOpen", "open"]]) {
data[field] = Object.fromEntries(LOG_STAGES.map(stage => [stage, data[`stage_${stage}_${suffix}`]]));
}
// Revalidate finite route and diagnostic allowlists; never pass raw strings.
const projected = logContext.run({}, () => logRecord(["error", "warn", "info", "debug"].includes(String(input.severity)) ? String(input.severity) : "info", String(input.module || "application"), ["", data]));
const result: Row = { timestamp: /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d{1,9})?Z$/.test(time) ? time : new Date(time).toISOString(), severity: String(projected.severity), service: services.includes(String(input.service)) ? String(input.service) : "unknown", event: events.has(String(input.event)) ? String(input.event) : "application_log", module: String(projected.module), release: /^(?:[a-f0-9]{7,40}|unknown)$/.test(String(input.release)) ? String(input.release) : "unknown" };
for (const key of [...numeric, "errorCode", "requestId", "jobId", "queue", "route", "method", "errorType", "errorStack", "causeType", "causeStack", "repoId", "requestPath"]) if (typeof projected[key] === "string" || typeof projected[key] === "number") result[key] = projected[key] as string | number;
for (const key of [...numeric, "outcome", "endpointKind", "cancellationOrigin", "cancellationEvidence", "hopId", "parentHopId", "errorCode", "requestId", "jobId", "queue", "route", "method", "errorType", "errorStack", "causeType", "causeStack", "repoId", "requestPath"]) if (typeof projected[key] === "string" || typeof projected[key] === "number") result[key] = projected[key] as string | number;
if (validTraceId(input.instance)) result.instance = input.instance;
return result;
}
+2
View File
@@ -1,3 +1,4 @@
import { installDatabaseTiming } from "../core/database-timing";
import InstallationModel from "../core/model/github-installation";
import { credentialCipher } from "../core/credentials";
import CredentialModel from "../core/model/credentials/credentials.model";
@@ -24,6 +25,7 @@ export const database = mongoose.connection;
export let isConnected = false;
export async function connect(overrides: Pick<ConnectOptions, "appName" | "maxPoolSize" | "minPoolSize"> = {}) {
installDatabaseTiming();
credentialCipher(); // Refuse to serve persisted credentials without a valid keyring.
mongoose.set("strictQuery", false);
const options: ConnectOptions = {
+2
View File
@@ -1,3 +1,4 @@
import { recordUpstreamFailure } from "../../core/request-evidence";
import { jobTrace } from "../../core/log-context";
import { hasVisibleSubmodules } from "../../core/file-submodules";
import { FILE_LISTING_LIMIT } from "../../core/file-tree";
@@ -89,6 +90,7 @@ router.get(
},
})
.on("error", (err: Error & { response?: { statusCode?: number; body?: unknown } }) => {
if (!res.destroyed) recordUpstreamFailure(res, err, "streamer");
headersDone(true);
const upstreamStatus = err?.response?.statusCode;
let upstreamBody: string | undefined;
+2 -1
View File
@@ -1,3 +1,4 @@
import { recordUpstreamFailure } from "../core/request-evidence";
import { sendSharedContent } from "../core/cached-response";
import * as express from "express";
import GitHubStream from "../core/source/GitHubStream";
@@ -125,7 +126,7 @@ router.post("/", async (req: express.Request, res: express.Response) => {
anonymizer.destroy();
}
content
.on("error", handleStreamError)
.on("error", error => { if (!res.destroyed) recordUpstreamFailure(res, error); handleStreamError(error); })
.pipe(anonymizer)
.on("error", handleStreamError)
.pipe(res)