Merge pull request #970 from tdurieux/feat/operational-logging-evidence

fix: explain interrupted requests and account for overlapping latency
This commit is contained in:
Thomas Durieux authored and GitHub committed 2026-11-15 16:27:40 +01:00
commit bc7afdd906
21 files changed
+467 -38

No files matched your search

+53
View File
@@ -359,3 +359,56 @@ Application requests to the supported anonymous repository endpoints retain thei
For the focused restricted-file/rotation regression, run
`LOGGING_ACCESS_ONLY=1 python3 scripts/test-logging.py`. This uses synthetic
`0640` logs in an isolated Docker project and preserves capability restrictions.
## Request evidence contract
`outcome=completed` means Node emitted response `finish`. It does not prove that
an end user received the body. A 200 can have `outcome=interrupted`. Count public
API outcomes once; correlated streamer hops describe the same operation and must
not be added to public failure counts. `endpointKind` distinguishes public API
from internal streamer requests. Each middleware invocation has a fresh `hopId`;
`parentHopId` links streamer calls to the API hop, including repeated cache probes.
Public caller correlation headers are ignored. Internal IDs are diagnostics, not
proof of caller identity. Existing job UUIDs and attempt numbers remain separate.
For interrupted responses, `cancellationOrigin` and `cancellationEvidence` retain
the first observed causal boundary. Explicit upstream errors identify the failing
upstream hop; ETIMEDOUT identifies a timeout. A request abort or response close
alone has origin `unknown`: the peer may be a proxy, API or browser, and a close
can also follow server teardown. No client-action claim is inferred from 200,
socket close, or lack of error logs. ZIP processing failures with no identified
upstream retain unknown origin with local-abort evidence. Historical records lack
this evidence and cannot be reclassified reliably.
`bytes` counts encoded body bytes passed to Node response writes, including
buffered writes; it is not an acknowledgement from the peer. `expectedBytes` is
only the response Content-Length, absent when unknown. Source file size is not
comparable after anonymization/compression. HEAD, 204 and 304 have zero expected
body bytes. `firstByteMs` is the existing header-write boundary, not a measured
network first byte. `responseMs` and `stage_transfer_ms` cover header write through
finish/close, including backpressure and waits for content.
Named stage durations are inclusive sums of instrumented spans, clipped at the
response terminal event. Repeated and parallel spans add to their stage totals.
`stage_<name>_count` counts instrumented calls, not hidden library retries;
`stage_<name>_open` counts spans still running at termination. Uninstrumented
library retries/backoff stay inside their enclosing span and cannot be separated.
`measuredMs` is the union of all spans, `overlapMs` is the excess of their summed
durations over that union, and `unattributedMs` is request time outside any span.
Do not add transfer, credential, database, cache, upstream or anonymization totals
to estimate request duration. Millisecond rounding of legacy request totals may
cause sub-millisecond differences. Storage is fixed per stage, not per call.
`database` measures Mongoose Query/Aggregate execution after database setup,
including pool/buffering wait and conversion. It excludes document saves, bulk
writes and direct driver operations. `repository` remains the application
repository lookup, which can enclose database time. Credential resolution can
enclose database and GitHub work. GitHub `upstream` spans cover the existing
Octokit request wrapper or explicitly instrumented stream intervals. Cache lookup,
fill, worker admission, anonymization and streamer-header spans retain their
existing call boundaries. Missing stages mean unavailable instrumentation, never
zero cost. Unattributed time is visible; it is not evidence of a stall.
The admin API and Redis projection preserve these allowlisted fields. Collector
and application versions must be updated together. No ordinary request depends
on the collector, Grafana or VictoriaLogs being available.
+7 -4
View File
@@ -46,18 +46,21 @@ if service == "mongodb" {
if includes(["github_app_authorization", "installation", "installation_repositories"], record.webhookEvent) { out.webhookEvent = record.webhookEvent }
if includes(["github.com", "api.github.com", "codeload.github.com", "raw.githubusercontent.com", "media.githubusercontent.com", "other"], record.upstreamHost) { out.upstreamHost = record.upstreamHost }
if includes(["download", "remove", "cache"], record.queue) { out.queue = record.queue }
if includes(["unknown", "api", "streamer", "timeout", "upstream"], record.cancellationOrigin) { out.cancellationOrigin = record.cancellationOrigin }
if includes(["response_close", "request_aborted", "upstream_error", "timeout", "local_abort"], record.cancellationEvidence) { out.cancellationEvidence = record.cancellationEvidence }
if includes(["public", "internal"], record.endpointKind) { out.endpointKind = record.endpointKind }
if includes(["completed", "interrupted"], record.outcome) { out.outcome = record.outcome }
}
for_each(["requestId", "jobId", "deliveryId"]) -> |_index, key| {
for_each(["requestId", "jobId", "deliveryId", "hopId", "parentHopId"]) -> |_index, key| {
value = get(record, [key]) ?? null
if is_string(value) && match(string!(value), r'^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$') { out = set!(out, [key], value) }
}
for_each(["status", "httpStatus", "ms", "firstByteMs", "responseMs", "bytes", "bytesPerSecond", "attempt", "rss", "memoryLimitBytes", "memoryRatio", "uptime", "activeRequests", "waiting", "active", "delayed", "failed", "oldestWaitingMs", "droppedLogs", "loopP95Ms", "upstreamStatus", "stage_repository_ms", "stage_credential_resolution_ms", "stage_upstream_ms", "stage_cache_lookup_ms", "stage_cache_fill_ms", "stage_worker_wait_ms", "stage_anonymize_ms", "stage_streamer_headers_ms"]) -> |_index, key| {
for_each(["stage_transfer_ms", "stage_transfer_count", "stage_transfer_open", "stage_database_ms", "stage_database_count", "stage_database_open", "expectedBytes", "measuredMs", "unattributedMs", "overlapMs", "stage_repository_count", "stage_repository_open", "stage_credential_resolution_count", "stage_credential_resolution_open", "stage_upstream_count", "stage_upstream_open", "stage_cache_lookup_count", "stage_cache_lookup_open", "stage_cache_fill_count", "stage_cache_fill_open", "stage_worker_wait_count", "stage_worker_wait_open", "stage_anonymize_count", "stage_anonymize_open", "stage_streamer_headers_count", "stage_streamer_headers_open", "status", "httpStatus", "ms", "firstByteMs", "responseMs", "bytes", "bytesPerSecond", "attempt", "rss", "memoryLimitBytes", "memoryRatio", "uptime", "activeRequests", "waiting", "active", "delayed", "failed", "oldestWaitingMs", "droppedLogs", "loopP95Ms", "upstreamStatus", "stage_repository_ms", "stage_credential_resolution_ms", "stage_upstream_ms", "stage_cache_lookup_ms", "stage_cache_fill_ms", "stage_worker_wait_ms", "stage_anonymize_ms", "stage_streamer_headers_ms"]) -> |_index, key| {
value = get(record, [key]) ?? null
if is_integer(value) || is_float(value) { out = set!(out, [key], value) }
}
if includes(["GET", "HEAD", "POST", "PUT", "PATCH", "DELETE", "OPTIONS", "OTHER"], record.method) { out.method = record.method }
if includes(["/", "other", "/dashboard", "/anonymize", "/healthcheck", "/api/healthcheck", "/api", "/api/download", "/api/stat", "/api/stat/history", "/api/options", "/api/message", "/api/user", "/api/user/quota", "/api/user/dashboard", "/api/repo", "/api/anonymize-preview", "/api/admin/overview", "/api/admin/performance", "/api/admin/errors", "/api/admin/errors/stats", "/api/admin/queues", "/api/admin/queues/metrics", "/github/app/webhook", "/api/repo/:repoId", "/api/repo/:repoId/file/:path", "/api/repo/:repoId/files", "/api/repo/:repoId/files/counts", "/api/repo/:repoId/files/search", "/api/repo/:repoId/zip", "/api/repo/:repoId/options", "/api/repo/:repoId/refresh", "/api/repo/:repoId/coauthors", "/api/repo/:repoId/extend", "/api/repo/:owner/:repo/readme", "/api/repo/:owner/:repo/branches", "/w/:repoId/:path", "/r/:repoId/:path", "/api/pr/:id/:path", "/api/gist/:id/:path", "/assets/:path", "/github/:action", "/api/admin/:action", "/api/user/:action", "/api/repo/:id/file/:path", "/api/repo/:id/files/counts", "/api/repo/:id/files/search", "/api/repo/:id/files", "/api/repo/:id/zip", "/api/repo/:id", "/viewer/:id/:path"], record.route) { out.route = record.route }
if includes(["/", "other", "/dashboard", "/anonymize", "/healthcheck", "/api/healthcheck", "/api", "/api/download", "/api/stat", "/api/stat/history", "/api/options", "/api/message", "/api/user", "/api/user/quota", "/api/user/dashboard", "/api/repo", "/api/anonymize-preview", "/api/admin/overview", "/api/admin/performance", "/api/admin/errors", "/api/admin/errors/stats", "/api/admin/queues", "/api/admin/queues/metrics", "/github/app/webhook", "/api/repo/:repoId", "/api/repo/:repoId/file/:path", "/api/repo/:repoId/files", "/api/repo/:repoId/files/counts", "/api/repo/:repoId/files/search", "/api/repo/:repoId/zip", "/api/repo/:repoId/options", "/api/repo/:repoId/refresh", "/api/repo/:repoId/coauthors", "/api/repo/:repoId/extend", "/api/repo/:owner/:repo", "/api/repo/:owner/:repo/tree", "/api/repo/:owner/:repo/file", "/api/repo/:owner/:repo/readme", "/api/repo/:owner/:repo/branches", "/w/:repoId/:path", "/r/:repoId/:path", "/api/pr/:id/:path", "/api/gist/:id/:path", "/assets/:path", "/github/:action", "/api/admin/:action", "/api/user/:action", "/api/repo/:id/file/:path", "/api/repo/:id/files/counts", "/api/repo/:id/files/search", "/api/repo/:id/files", "/api/repo/:id/zip", "/api/repo/:id", "/viewer/:id/:path"], record.route) { out.route = record.route }
# VictoriaLogs consumes its message field; keep event independently searchable.
# Anonymous application request context at every outcome, never query strings.
{
@@ -72,7 +75,7 @@ if includes(["/", "other", "/dashboard", "/anonymize", "/healthcheck", "/api/hea
}
}
# BEGIN GENERATED DIAGNOSTICS
source_functions = {"src/cli/index.ts":["<anonymous>","generateRandomFileName","main"],"build/cli/index.js":["<anonymous>","generateRandomFileName","main"],"src/config.ts":["<anonymous>"],"build/config.js":["<anonymous>"],"src/core/AnonymizedFile.ts":["<anonymous>","anonymizedContent","assertContentFile","cancel","content","defaultMaskCandidateRegex","escapeRegex","extension","forwardHeaders","getFileInfo","handleStreamError","isFileSupported","isImage","originalPath","probe","recoverTruncatedFile","send","sha","size","streamerErrorToAnonymous","visibleFile"],"build/core/AnonymizedFile.js":["<anonymous>","anonymizedContent","assertContentFile","cancel","content","defaultMaskCandidateRegex","escapeRegex","extension","forwardHeaders","getFileInfo","handleStreamError","isFileSupported","isImage","originalPath","probe","recoverTruncatedFile","send","sha","size","streamerErrorToAnonymous","visibleFile"],"src/core/AnonymousError.ts":["<anonymous>","detail","toString","url"],"build/core/AnonymousError.js":["<anonymous>","detail","toString","url"],"src/core/Conference.ts":["<anonymous>","expire","isExpired","remove","repositories","toJSON","updateStatus"],"build/core/Conference.js":["<anonymous>","expire","isExpired","remove","repositories","toJSON","updateStatus"],"src/core/Gist.ts":["<anonymous>","anonymize","check","conference","content","countView","download","downloadContent","downloadWithAccess","expire","getAccess","getToken","markExpired","remove","resetSate","toJSON","updateIfNeeded","updateStatus"],"build/core/Gist.js":["<anonymous>","anonymize","check","conference","content","countView","download","downloadContent","downloadWithAccess","expire","getAccess","getToken","markExpired","remove","resetSate","toJSON","updateIfNeeded","updateStatus"],"src/core/GitHubUtils.ts":["<anonymous>","abortableGateLookup","cancelled","checkToken","ensureRedisGateClient","getRedisGateResetAt","getToken","getTokenGateResetAt","octokit","rateLimitDetail","resolveRepositoryToken","run","setRedisGate","setRedisGateFromWorker","setTokenGate","waitForTokenGate"],"build/core/GitHubUtils.js":["<anonymous>","abortableGateLookup","cancelled","checkToken","ensureRedisGateClient","getRedisGateResetAt","getToken","getTokenGateResetAt","octokit","rateLimitDetail","resolveRepositoryToken","run","setRedisGate","setRedisGateFromWorker","setTokenGate","waitForTokenGate"],"src/core/PullRequest.ts":["<anonymous>","anonymize","check","conference","content","countView","download","downloadContent","expire","getToken","markExpired","remove","resetSate","toJSON","updateIfNeeded","updateStatus"],"build/core/PullRequest.js":["<anonymous>","anonymize","check","conference","content","countView","download","downloadContent","expire","getToken","markExpired","remove","resetSate","toJSON","updateIfNeeded","updateStatus"],"src/core/Repository.ts":["<anonymous>","anonymize","anonymizeTreeRecursive","anonymizedFiles","assertNotArchived","buildPathIndex","check","checkRefreshWrite","cleanupRetiredFileTrees","cleanupStagedFileTree","clearRows","completeRecoveredFileMetadata","computeSize","conference","countView","current","ensureFileTree","expire","fileSnapshotFilter","files","findAnonymizedPath","generateAnonymizeTransformer","getToken","hasEmptyTree","indexPaths","insertBatch","invalidateFileMetadata","isReady","markCachePresent","markExpired","pathKey","readFiles","refresh","refreshFilter","refreshIfNeeded","remove","removeCache","resetSate","searchFiles","snapshotValue","submoduleMounts","toJSON","updateIfNeeded","updateProgressMessage","updateRetiringPathIndex","updateStatus","zip"],"build/core/Repository.js":["<anonymous>","anonymize","anonymizeTreeRecursive","anonymizedFiles","assertNotArchived","buildPathIndex","check","checkRefreshWrite","cleanupRetiredFileTrees","cleanupStagedFileTree","clearRows","completeRecoveredFileMetadata","computeSize","conference","countView","current","ensureFileTree","expire","fileSnapshotFilter","files","findAnonymizedPath","generateAnonymizeTransformer","getToken","hasEmptyTree","indexPaths","insertBatch","invalidateFileMetadata","isReady","markCachePresent","markExpired","pathKey","readFiles","refresh","refreshFilter","refreshIfNeeded","remove","removeCache","resetSate","searchFiles","snapshotValue","submoduleMounts","toJSON","updateIfNeeded","updateProgressMessage","updateRetiringPathIndex","updateStatus","zip"],"src/core/User.ts":["<anonymous>","getAccessToken","getGists","getGitHubRepositories","getPullRequests","getRepositories","markListExpired","quotaRepositories","repositoryMembership","toJSON"],"build/core/User.js":["<anonymous>","getAccessToken","getGists","getGitHubRepositories","getPullRequests","getRepositories","markListExpired","quotaRepositories","repositoryMembership","toJSON"],"src/core/anonymization-pool.ts":["<anonymous>","anonymizeOnWorker","cancel","exited","failed","finish","getAnonymizationPoolStats","message","pump","release","releaseSpool","reserveSpool"],"build/core/anonymization-pool.js":["Line truncated
source_functions = {"src/cli/index.ts":["<anonymous>","generateRandomFileName","main"],"build/cli/index.js":["<anonymous>","generateRandomFileName","main"],"src/config.ts":["<anonymous>"],"build/config.js":["<anonymous>"],"src/core/AnonymizedFile.ts":["<anonymous>","anonymizedContent","assertContentFile","cancel","content","defaultMaskCandidateRegex","escapeRegex","extension","forwardHeaders","getFileInfo","handleStreamError","isFileSupported","isImage","originalPath","probe","recoverTruncatedFile","send","sha","size","streamerErrorToAnonymous","visibleFile"],"build/core/AnonymizedFile.js":["<anonymous>","anonymizedContent","assertContentFile","cancel","content","defaultMaskCandidateRegex","escapeRegex","extension","forwardHeaders","getFileInfo","handleStreamError","isFileSupported","isImage","originalPath","probe","recoverTruncatedFile","send","sha","size","streamerErrorToAnonymous","visibleFile"],"src/core/AnonymousError.ts":["<anonymous>","detail","toString","url"],"build/core/AnonymousError.js":["<anonymous>","detail","toString","url"],"src/core/Conference.ts":["<anonymous>","expire","isExpired","remove","repositories","toJSON","updateStatus"],"build/core/Conference.js":["<anonymous>","expire","isExpired","remove","repositories","toJSON","updateStatus"],"src/core/Gist.ts":["<anonymous>","anonymize","check","conference","content","countView","download","downloadContent","downloadWithAccess","expire","getAccess","getToken","markExpired","remove","resetSate","toJSON","updateIfNeeded","updateStatus"],"build/core/Gist.js":["<anonymous>","anonymize","check","conference","content","countView","download","downloadContent","downloadWithAccess","expire","getAccess","getToken","markExpired","remove","resetSate","toJSON","updateIfNeeded","updateStatus"],"src/core/GitHubUtils.ts":["<anonymous>","abortableGateLookup","cancelled","checkToken","ensureRedisGateClient","getRedisGateResetAt","getToken","getTokenGateResetAt","octokit","rateLimitDetail","resolveRepositoryToken","run","setRedisGate","setRedisGateFromWorker","setTokenGate","waitForTokenGate"],"build/core/GitHubUtils.js":["<anonymous>","abortableGateLookup","cancelled","checkToken","ensureRedisGateClient","getRedisGateResetAt","getToken","getTokenGateResetAt","octokit","rateLimitDetail","resolveRepositoryToken","run","setRedisGate","setRedisGateFromWorker","setTokenGate","waitForTokenGate"],"src/core/PullRequest.ts":["<anonymous>","anonymize","check","conference","content","countView","download","downloadContent","expire","getToken","markExpired","remove","resetSate","toJSON","updateIfNeeded","updateStatus"],"build/core/PullRequest.js":["<anonymous>","anonymize","check","conference","content","countView","download","downloadContent","expire","getToken","markExpired","remove","resetSate","toJSON","updateIfNeeded","updateStatus"],"src/core/Repository.ts":["<anonymous>","anonymize","anonymizeTreeRecursive","anonymizedFiles","assertNotArchived","buildPathIndex","check","checkRefreshWrite","cleanupRetiredFileTrees","cleanupStagedFileTree","clearRows","completeRecoveredFileMetadata","computeSize","conference","countView","current","ensureFileTree","expire","fileSnapshotFilter","files","findAnonymizedPath","generateAnonymizeTransformer","getToken","hasEmptyTree","indexPaths","insertBatch","invalidateFileMetadata","isReady","markCachePresent","markExpired","pathKey","readFiles","refresh","refreshFilter","refreshIfNeeded","remove","removeCache","resetSate","searchFiles","snapshotValue","submoduleMounts","toJSON","updateIfNeeded","updateProgressMessage","updateRetiringPathIndex","updateStatus","zip"],"build/core/Repository.js":["<anonymous>","anonymize","anonymizeTreeRecursive","anonymizedFiles","assertNotArchived","buildPathIndex","check","checkRefreshWrite","cleanupRetiredFileTrees","cleanupStagedFileTree","clearRows","completeRecoveredFileMetadata","computeSize","conference","countView","current","ensureFileTree","expire","fileSnapshotFilter","files","findAnonymizedPath","generateAnonymizeTransformer","getToken","hasEmptyTree","indexPaths","insertBatch","invalidateFileMetadata","isReady","markCachePresent","markExpired","pathKey","readFiles","refresh","refreshFilter","refreshIfNeeded","remove","removeCache","resetSate","searchFiles","snapshotValue","submoduleMounts","toJSON","updateIfNeeded","updateProgressMessage","updateRetiringPathIndex","updateStatus","zip"],"src/core/User.ts":["<anonymous>","getAccessToken","getGists","getGitHubRepositories","getPullRequests","getRepositories","markListExpired","quotaRepositories","repositoryMembership","toJSON"],"build/core/User.js":["<anonymous>","getAccessToken","getGists","getGitHubRepositories","getPullRequests","getRepositories","markListExpired","quotaRepositories","repositoryMembership","toJSON"],"src/core/anonymization-pool.ts":["<anonymous>","anonymizeOnWorker","cancel","exited","failed","finish","getAnonymizationPoolStats","message","pump","release","releaseSpool","reserveSpool"],"build/core/anonymization-pool.js":["Line truncated
for_each(["errorType", "causeType"]) -> |_index, key| {
value = get(record, [key]) ?? null
if includes(["Error","TypeError","RangeError","ReferenceError","SyntaxError","URIError","EvalError","AggregateError","AnonymousError","HTTPError","RequestError","MongoServerError","MongoNetworkError","MongoServerSelectionError","ValidationError","TimeoutError","AbortError","UnknownError"], value) { out = set!(out, [key], value) }
+3 -2
View File
@@ -1,3 +1,4 @@
import { recordResponseFailure } from "./request-evidence";
import { rejectStreamerRedirects } from "./streamer-response";
import { isResponseCancellation } from "./response-cancellation";
import { canReadCachedRepository } from "./cached-repository-access";
@@ -585,9 +586,9 @@ export default class AnonymizedFile {
reject(error);
}
content
.on("error", handleStreamError)
.on("error", error => { if (!res.destroyed) recordResponseFailure(res, "unknown", "local_abort"); handleStreamError(error); })
.pipe(anonymizer)
.on("error", handleStreamError)
.on("error", error => { if (!res.destroyed) recordResponseFailure(res, "api", "local_abort"); handleStreamError(error); })
.pipe(res)
.on("error", handleStreamError)
.on("finish", () => {
+7 -1
View File
@@ -1,3 +1,4 @@
import { recordResponseFailure } from "./request-evidence";
import { createGzip } from "zlib";
import { createHash } from "crypto";
import { Readable, PassThrough } from "stream";
@@ -49,9 +50,14 @@ async function sendCached(content: Readable, res: Response,
if (isText && !res.getHeader("Content-Type")) res.type("text/plain");
if (isText) res.setHeader("Accept-Ranges", "none");
else res.removeHeader("Accept-Ranges");
// Install evidence before pipeline's listeners destroy the response.
const localFailure = () => { if (!res.destroyed) recordResponseFailure(res, "unknown", "local_abort"); };
stream.once("error", localFailure);
if (record.encoding === "identity" && !res.req.acceptsEncodings("identity")) {
res.setHeader("Content-Encoding", "gzip"); res.removeHeader("Content-Length");
await pipeline(stream, createGzip(), res);
const gzip = createGzip();
gzip.once("error", localFailure);
await pipeline(stream, gzip, res);
} else await pipeline(stream, res);
return true;
};
+18
View File
@@ -0,0 +1,18 @@
import mongoose from "mongoose";
import { measureStage } from "./request-monitoring";
let installed = false;
/** Measure Mongoose query/aggregate execution, including pool/buffering wait.
* No query text, model names, results or parameters enter the logging context.
* Direct driver operations, document saves and bulk writes are not covered.
*/
export function installDatabaseTiming() {
if (installed) return;
installed = true;
for (const prototype of [mongoose.Query.prototype, mongoose.Aggregate.prototype]) {
const original = prototype.exec;
prototype.exec = function (this: unknown, ...args: unknown[]) {
return measureStage("database", () => Reflect.apply(original, this, args));
} as typeof original;
}
}
+1 -1
View File
@@ -1,7 +1,7 @@
import { AsyncLocalStorage } from "async_hooks";
import { randomUUID } from "crypto";
export interface LogContext { requestId?: string; jobId?: string; attempt?: number; service?: string; route?: string; method?: string; requestPath?: string }
export interface LogContext { requestId?: string; hopId?: string; parentHopId?: string; jobId?: string; attempt?: number; service?: string; route?: string; method?: string; requestPath?: string }
export const logContext = new AsyncLocalStorage<LogContext>();
export const validTraceId = (value: unknown): value is string => typeof value === "string" &&
/^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$/.test(value);
+47 -1
View File
@@ -785,6 +785,7 @@ export const LOG_SOURCE_FUNCTIONS: Record<string, readonly string[]> = {
"abort",
"assertActive",
"decisionKey",
"localFailure",
"send",
"sendCached",
"sendSharedContent"
@@ -794,6 +795,7 @@ export const LOG_SOURCE_FUNCTIONS: Record<string, readonly string[]> = {
"abort",
"assertActive",
"decisionKey",
"localFailure",
"send",
"sendCached",
"sendSharedContent"
@@ -858,6 +860,14 @@ export const LOG_SOURCE_FUNCTIONS: Record<string, readonly string[]> = {
"replaceCredential",
"setCredential"
],
"src/core/database-timing.ts": [
"<anonymous>",
"installDatabaseTiming"
],
"build/core/database-timing.js": [
"<anonymous>",
"installDatabaseTiming"
],
"src/core/email-history.ts": [
"<anonymous>",
"complete",
@@ -960,6 +970,14 @@ export const LOG_SOURCE_FUNCTIONS: Record<string, readonly string[]> = {
"<anonymous>",
"isGitHubRateLimitError"
],
"src/core/github-repository-picker.ts": [
"<anonymous>",
"repositoryPickerPage"
],
"build/core/github-repository-picker.js": [
"<anonymous>",
"repositoryPickerPage"
],
"src/core/github-token-context.ts": [
"<anonymous>",
"githubQuotaKey",
@@ -1482,34 +1500,54 @@ export const LOG_SOURCE_FUNCTIONS: Record<string, readonly string[]> = {
"build/core/repository-id.js": [
"<anonymous>"
],
"src/core/request-evidence.ts": [
"<anonymous>",
"expectedResponseBytes",
"recordResponseFailure",
"recordUpstreamFailure",
"responseEvidence"
],
"build/core/request-evidence.js": [
"<anonymous>",
"expectedResponseBytes",
"recordResponseFailure",
"recordUpstreamFailure",
"responseEvidence"
],
"src/core/request-monitoring.ts": [
"<anonymous>",
"activeRequestCount",
"advance",
"cacheTraffic",
"count",
"done",
"drain",
"finish",
"measureStage",
"monitorRequests",
"observe",
"recordCacheHit",
"requestHeaders",
"requestRoute",
"start",
"startStage"
],
"build/core/request-monitoring.js": [
"<anonymous>",
"activeRequestCount",
"advance",
"cacheTraffic",
"count",
"done",
"drain",
"finish",
"measureStage",
"monitorRequests",
"observe",
"recordCacheHit",
"requestHeaders",
"requestRoute",
"start",
"startStage"
],
"src/core/response-cancellation.ts": [
@@ -2104,6 +2142,14 @@ export const LOG_SOURCE_FUNCTIONS: Record<string, readonly string[]> = {
"<anonymous>",
"main"
],
"src/scripts/check-github-app-key.ts": [
"<anonymous>",
"checkGitHubAppKey"
],
"build/scripts/check-github-app-key.js": [
"<anonymous>",
"checkGitHubAppKey"
],
"src/scripts/create-performance-indexes.ts": [
"<anonymous>",
"main"
@@ -2860,4 +2906,4 @@ export const LOG_SOURCE_FUNCTIONS: Record<string, readonly string[]> = {
]
};
export const LOG_ERROR_TYPES = ["Error","TypeError","RangeError","ReferenceError","SyntaxError","URIError","EvalError","AggregateError","AnonymousError","HTTPError","RequestError","MongoServerError","MongoNetworkError","MongoServerSelectionError","ValidationError","TimeoutError","AbortError","UnknownError"];
export const LOG_ROUTES = ["/","other","/dashboard","/anonymize","/healthcheck","/api/healthcheck","/api","/api/download","/api/stat","/api/stat/history","/api/options","/api/message","/api/user","/api/user/quota","/api/user/dashboard","/api/repo","/api/anonymize-preview","/api/admin/overview","/api/admin/performance","/api/admin/errors","/api/admin/errors/stats","/api/admin/queues","/api/admin/queues/metrics","/github/app/webhook","/api/repo/:repoId","/api/repo/:repoId/file/:path","/api/repo/:repoId/files","/api/repo/:repoId/files/counts","/api/repo/:repoId/files/search","/api/repo/:repoId/zip","/api/repo/:repoId/options","/api/repo/:repoId/refresh","/api/repo/:repoId/coauthors","/api/repo/:repoId/extend","/api/repo/:owner/:repo/readme","/api/repo/:owner/:repo/branches","/w/:repoId/:path","/r/:repoId/:path","/api/pr/:id/:path","/api/gist/:id/:path","/assets/:path","/github/:action","/api/admin/:action","/api/user/:action","/api/repo/:id/file/:path","/api/repo/:id/files/counts","/api/repo/:id/files/search","/api/repo/:id/files","/api/repo/:id/zip","/api/repo/:id","/viewer/:id/:path"];
export const LOG_ROUTES = ["/","other","/dashboard","/anonymize","/healthcheck","/api/healthcheck","/api","/api/download","/api/stat","/api/stat/history","/api/options","/api/message","/api/user","/api/user/quota","/api/user/dashboard","/api/repo","/api/anonymize-preview","/api/admin/overview","/api/admin/performance","/api/admin/errors","/api/admin/errors/stats","/api/admin/queues","/api/admin/queues/metrics","/github/app/webhook","/api/repo/:repoId","/api/repo/:repoId/file/:path","/api/repo/:repoId/files","/api/repo/:repoId/files/counts","/api/repo/:repoId/files/search","/api/repo/:repoId/zip","/api/repo/:repoId/options","/api/repo/:repoId/refresh","/api/repo/:repoId/coauthors","/api/repo/:repoId/extend","/api/repo/:owner/:repo","/api/repo/:owner/:repo/tree","/api/repo/:owner/:repo/file","/api/repo/:owner/:repo/readme","/api/repo/:owner/:repo/branches","/w/:repoId/:path","/r/:repoId/:path","/api/pr/:id/:path","/api/gist/:id/:path","/assets/:path","/github/:action","/api/admin/:action","/api/user/:action","/api/repo/:id/file/:path","/api/repo/:id/files/counts","/api/repo/:id/files/search","/api/repo/:id/files","/api/repo/:id/zip","/api/repo/:id","/viewer/:id/:path"];
+19 -7
View File
@@ -23,14 +23,14 @@ const errorCodes = new Set([
"ECONNRESET", "ECONNREFUSED", "ETIMEDOUT", "ENOTFOUND", "EPIPE", "ABORT_ERR",
"upstream_error", "upstream_timeout", "invalid_signature", "webhook_processing_failed",
]);
const numbers = ["status", "httpStatus", "ms", "firstByteMs", "responseMs", "bytes", "bytesPerSecond",
export const LOG_NUMBERS = ["expectedBytes", "measuredMs", "unattributedMs", "overlapMs","status", "httpStatus", "ms", "firstByteMs", "responseMs", "bytes", "bytesPerSecond",
"attempt", "rss", "memoryLimitBytes", "memoryRatio", "uptime", "activeRequests", "waiting", "active", "delayed", "failed",
"oldestWaitingMs", "droppedLogs", "loopP95Ms", "upstreamStatus"];
const stages = ["repository", "credential_resolution", "upstream", "cache_lookup", "cache_fill", "worker_wait", "anonymize", "streamer_headers"];
export const LOG_STAGES = ["transfer", "database","repository", "credential_resolution", "upstream", "cache_lookup", "cache_fill", "worker_wait", "anonymize", "streamer_headers"];
export function logRecord(level: string, module: string, args: unknown[]) {
const context = logContext.getStore() || {};
const details: Record<string, unknown> = {};
const keys = [...numbers, "requestId", "jobId", "code", "message", "outcome", "method", "route", "queue",
const keys = [...LOG_NUMBERS, "requestId", "jobId", "hopId", "parentHopId", "cancellationOrigin", "cancellationEvidence", "endpointKind", "stageCounts", "stageOpen", "code", "message", "outcome", "method", "route", "queue",
"errorType", "errorStack", "causeType", "causeStack", "requestPath", "stages", "reason", "phase", "event", "deliveryId", "hostname", "service"];
// Do not traverse arbitrary objects or invoke error getters while logging.
for (const arg of args.slice(0, 8)) if (arg && typeof arg === "object") {
@@ -50,12 +50,12 @@ export function logRecord(level: string, module: string, args: unknown[]) {
service: context.service || (allowed(["api", "streamer", "worker"], details.service) ? details.service : undefined) || (process.env.SERVICE_NAME?.toLowerCase() === "streamer" ? "streamer" : "api"),
instance, release: /^[a-f0-9]{7,40}$/.test(process.env.APP_RELEASE || "") ? process.env.APP_RELEASE : "unknown",
event, message: event, module: LOG_MODULES.includes(module) ? module : "application" };
for (const key of ["requestId", "jobId"] as const) {
for (const key of ["requestId", "jobId", "hopId", "parentHopId"] as const) {
const value = details[key] || context[key];
if (validTraceId(value)) record[key] = value;
}
if (Number.isSafeInteger(context.attempt)) record.attempt = context.attempt;
for (const key of numbers) if (typeof details[key] === "number" && Number.isFinite(details[key])) record[key] = details[key];
for (const key of LOG_NUMBERS) if (typeof details[key] === "number" && Number.isFinite(details[key])) record[key] = details[key];
const code = [details.code, details.message].find(value => typeof value === "string" && errorCodes.has(value));
if (code) record.errorCode = code;
if (allowed(["protocol", "credentials", "port", "host"], details.reason)) record.reason = details.reason;
@@ -64,21 +64,33 @@ export function logRecord(level: string, module: string, args: unknown[]) {
if (validTraceId(details.deliveryId)) record.deliveryId = details.deliveryId;
if (allowed(["github.com", "api.github.com", "codeload.github.com", "raw.githubusercontent.com", "media.githubusercontent.com"], details.hostname)) record.upstreamHost = details.hostname;
else if (details.hostname) record.upstreamHost = "other";
for (const [key, values] of Object.entries({ cancellationOrigin: ["unknown", "api", "streamer", "timeout", "upstream"],
cancellationEvidence: ["response_close", "request_aborted", "upstream_error", "timeout", "local_abort"], endpointKind: ["public", "internal"] })) {
if (allowed(values, details[key])) record[key] = details[key];
}
if (allowed(["completed", "interrupted"], details.outcome)) record.outcome = details.outcome;
if (allowed(["GET", "HEAD", "POST", "PUT", "PATCH", "DELETE", "OPTIONS", "OTHER"], details.method)) record.method = details.method;
// Only fixed route labels survive; never raw URLs or request parameters.
const route = context.route || details.route;
if (typeof route === "string" && LOG_ROUTES.includes(route)) record.route = route;
if (typeof route === "string" && LOG_ROUTES.includes(route)) record.route = route.replace(/^\/api\/repo\/:id(?=\/|$)/, "/api/repo/:repoId");
if (context.method && allowed(["GET", "HEAD", "POST", "PUT", "PATCH", "DELETE", "OPTIONS", "OTHER"], context.method)) record.method = context.method;
Object.assign(record, safeDiagnosticFields(details));
// Anonymous request paths are diagnostic fields, never metric labels.
const requestPath = safeReproductionPath(context.requestPath || details.requestPath, record.method);
if (requestPath) Object.assign(record, requestReproduction(requestPath, record.method));
if (allowed(["download", "remove", "cache"], details.queue)) record.queue = details.queue;
if (details.stages && typeof details.stages === "object") for (const key of stages) {
if (details.stages && typeof details.stages === "object") for (const key of LOG_STAGES) {
let value: unknown;
try { value = Object.getOwnPropertyDescriptor(details.stages, key)?.value; } catch { continue; }
if (typeof value === "number" && Number.isFinite(value)) record[`stage_${key}_ms`] = value;
}
for (const [field, suffix] of [["stageCounts", "count"], ["stageOpen", "open"]]) {
if (!details[field] || typeof details[field] !== "object") continue;
for (const stage of LOG_STAGES) {
let value: unknown;
try { value = Object.getOwnPropertyDescriptor(details[field], stage)?.value; } catch { continue; }
if (typeof value === "number" && Number.isSafeInteger(value) && value >= 0) record[`stage_${stage}_${suffix}`] = value;
}
}
return record;
}
+22
View File
@@ -0,0 +1,22 @@
export type FailureOrigin = "unknown" | "api" | "streamer" | "timeout" | "upstream";
export type FailureEvidence = "response_close" | "request_aborted" | "upstream_error" | "timeout" | "local_abort";
const evidence = new WeakMap<object, { cancellationOrigin: FailureOrigin; cancellationEvidence: FailureEvidence }>();
/** Record at the causal boundary, before destroying the response. First evidence wins. */
export function recordResponseFailure(response: object, origin: FailureOrigin, reason: FailureEvidence) {
if (!evidence.has(response)) evidence.set(response, { cancellationOrigin: origin, cancellationEvidence: reason });
}
export function recordUpstreamFailure(response: object, error: unknown, origin: "streamer" | "upstream" = "upstream") {
const code = error && typeof error === "object" ? Object.getOwnPropertyDescriptor(error, "code")?.value : undefined;
recordResponseFailure(response, code === "ETIMEDOUT" ? "timeout" : origin, code === "ETIMEDOUT" ? "timeout" : "upstream_error");
}
export function responseEvidence(response: object, aborted = false) {
return evidence.get(response) || { cancellationOrigin: "unknown", cancellationEvidence: aborted ? "request_aborted" : "response_close" };
}
/** Only output representation length is comparable to response writes. */
export function expectedResponseBytes(method: string, status: number, length: unknown): number | undefined {
if (method === "HEAD" || status === 204 || status === 304) return 0;
if ((typeof length !== "string" && typeof length !== "number") || !/^\d+$/.test(String(length))) return undefined;
const value = Number(length);
return Number.isSafeInteger(value) && value >= 0 ? value : undefined;
}
+69 -13
View File
@@ -1,5 +1,6 @@
import { responseEvidence, expectedResponseBytes } from "./request-evidence";
import { requestReproduction } from "./log-reproduction";
import { logContext } from "./log-context";
import { logContext, validTraceId } from "./log-context";
import { AsyncLocalStorage } from "async_hooks";
import { randomUUID } from "crypto";
import { performance } from "perf_hooks";
@@ -8,13 +9,55 @@ import { createLogger } from "./logger";
export const LATENCY_BUCKETS = [10, 50, 100, 250, 500, 1000, 2500, 5000, 10000, 30000, 60000];
export type Service = "api" | "streamer";
export type Stage = "repository" | "credential_resolution" | "upstream" | "cache_lookup" | "cache_fill" | "worker_wait" | "anonymize" | "streamer_headers";
export type Stage = "transfer" | "database" | "repository" | "credential_resolution" | "upstream" | "cache_lookup" | "cache_fill" | "worker_wait" | "anonymize" | "streamer_headers";
export interface Metric {
metric: string; route: string; method: string; count: number; sumMs: number;
bytes: number; aborted: number; errors: number; slow: number; buckets: number[];
}
const logger = createLogger("requests");
const context = new AsyncLocalStorage<{ id: string; service: Service; traffic?: string; stages: Partial<Record<Stage, number>> }>();
interface Trace { id: string; hopId: string; service: Service; traffic?: string; timing: RequestTiming }
const context = new AsyncLocalStorage<Trace>();
/** Constant storage: integrate active spans instead of retaining a per-call timeline. */
export class RequestTiming {
stages: Partial<Record<Stage, number>> = {};
counts: Partial<Record<Stage, number>> = {};
open: Partial<Record<Stage, number>> = {};
private last: number;
private covered = 0;
private concurrent = 0;
private ended = false;
constructor(private started = performance.now(), private now = () => performance.now()) { this.last = started; }
private advance(at: number) {
const elapsed = Math.max(0, at - this.last);
let active = 0;
for (const stage of Object.keys(this.open) as Stage[]) {
const count = this.open[stage] || 0;
this.stages[stage] = (this.stages[stage] || 0) + elapsed * count;
active += count;
}
if (active) this.covered += elapsed;
this.concurrent += elapsed * Math.max(0, active - 1);
this.last = at;
}
start(stage: Stage) {
if (this.ended) return () => undefined;
this.advance(this.now());
this.counts[stage] = (this.counts[stage] || 0) + 1;
this.open[stage] = (this.open[stage] || 0) + 1;
let done = false;
return () => {
if (done || this.ended) return;
done = true; this.advance(this.now()); this.open[stage]!--;
};
}
finish(at = this.now()) {
if (!this.ended) { this.advance(at); this.ended = true; }
return { stages: { ...this.stages }, stageCounts: { ...this.counts }, stageOpen: { ...this.open },
measuredMs: this.covered, unattributedMs: Math.max(0, this.last - this.started - this.covered), overlapMs: this.concurrent };
}
}
const methods = new Set(["GET", "HEAD", "POST", "PUT", "PATCH", "DELETE", "OPTIONS"]);
const staticRoutes = new Set([
"/", "/dashboard", "/anonymize", "/healthcheck", "/api/healthcheck", "/api/stat", "/api/stat/history",
@@ -24,11 +67,16 @@ const staticRoutes = new Set([
]);
/** Fixed labels: never use repository IDs, file names or query strings as metric keys. */
export function requestRoute(url: string, service: Service): string {
export function requestRoute(url: string, service: Service, method = "GET"): string {
const path = url.split("?", 1)[0].replace(/\/+$/, "") || "/";
if (service === "streamer") return ["/api/download", "/api", "/healthcheck"].includes(path) ? path : "other";
if (staticRoutes.has(path)) return path;
if (/^\/api\/repo\//.test(path)) {
if (["GET", "HEAD"].includes(method)) {
const source = path.match(/^\/api\/repo\/[^/]+\/[^/]+\/(readme|branches|tree|file)$/);
if (source && !/^\/api\/repo\/[^/]+\/file\//.test(path)) return `/api/repo/:owner/:repo/${source[1]}`;
if (/^\/api\/repo\/[^/]+\/[^/]+$/.test(path) && !/\/(file|files|zip|options)$/.test(path)) return "/api/repo/:owner/:repo";
}
const match = path.match(/^\/api\/repo\/[^/]+\/(file|files|zip|options|refresh|coauthors|extend)(?:\/|$)/);
if (match) {
const subroute = match[1] === "files" && /\/(counts|search)$/.exec(path);
@@ -79,7 +127,7 @@ let activeRequests = 0;
export function activeRequestCount() { return activeRequests; }
export function requestHeaders(): Record<string, string> {
const id = context.getStore()?.id;
return id ? { "x-request-id": id, "x-cache-traffic": cacheTraffic() } : {};
return id ? { "x-request-id": id, "x-cache-traffic": cacheTraffic(), "x-parent-hop-id": context.getStore()!.hopId } : {};
}
export function cacheTraffic() { return context.getStore()?.traffic || "background"; }
export function recordCacheHit(cache: "source" | "transformed", hit: boolean) {
@@ -90,12 +138,13 @@ export function recordCacheHit(cache: "source" | "transformed", hit: boolean) {
export function startStage(stage: Stage) {
const started = performance.now();
const trace = context.getStore();
const stop = trace?.timing.start(stage);
let ended = false;
return (error = false) => {
if (ended) return; ended = true;
if (!trace) return;
const ms = performance.now() - started;
trace.stages[stage] = (trace.stages[stage] || 0) + ms;
stop?.();
requestMetrics.observe(stage, "all", "all", ms, { error });
};
}
@@ -109,25 +158,29 @@ export function monitorRequests(service: Service, metrics = requestMetrics,
log: Pick<typeof logger, "info" | "warn"> = logger): RequestHandler {
return (req, res, next) => {
const forwarded = service === "streamer" ? req.get("x-request-id") : undefined;
const id = forwarded && /^[a-f0-9-]{36}$/.test(forwarded) ? forwarded : randomUUID();
const id = validTraceId(forwarded) ? forwarded : randomUUID();
const forwardedTraffic = service === "streamer" ? req.get("x-cache-traffic") : undefined;
const traffic = forwardedTraffic && ["file", "website", "zip", "monitor", "background"].includes(forwardedTraffic)
? forwardedTraffic : /uptime|healthcheck|monitor/i.test(req.get("user-agent") || "") ? "monitor"
: /\/zip(?:[/?]|$)|\/download(?:[/?]|$)/.test(req.originalUrl) ? "zip"
: req.originalUrl.startsWith("/w/") ? "website" : "file";
const trace = { id, service, traffic, stages: {} };
const route = requestRoute(req.originalUrl, service);
const started = performance.now();
const hopId = randomUUID();
const parent = service === "streamer" ? req.get("x-parent-hop-id") : undefined;
const parentHopId = validTraceId(parent) ? parent : undefined;
const trace = { id, hopId, service, traffic, timing: new RequestTiming(started) };
const route = requestRoute(req.originalUrl, service, req.method);
const reproduction = service === "api" ? requestReproduction(req.originalUrl, req.method) : {};
const method = methods.has(req.method) ? req.method : "OTHER";
const started = performance.now();
let firstByteMs: number | undefined;
let transferDone: (() => void) | undefined;
let bytes = 0;
let ended = false;
activeRequests++;
res.setHeader("X-Request-ID", id);
const writeHead = res.writeHead;
res.writeHead = function (this: typeof res, ...args: unknown[]): typeof res {
firstByteMs ??= performance.now() - started;
if (firstByteMs == null) { firstByteMs = performance.now() - started; transferDone = trace.timing.start("transfer"); }
return Reflect.apply(writeHead, this, args);
};
const count = (chunk: unknown, encoding: unknown) => {
@@ -151,6 +204,7 @@ export function monitorRequests(service: Service, metrics = requestMetrics,
if (ended) return; ended = true;
activeRequests--;
res.removeListener("finish", done); res.removeListener("close", done);
transferDone?.();
const ms = performance.now() - started;
const aborted = !res.writableFinished;
const transfer = /\/(file|zip)(?:\/|$)/.test(route) || route.startsWith("/w/") || service === "streamer";
@@ -162,11 +216,13 @@ export function monitorRequests(service: Service, metrics = requestMetrics,
outcome: aborted ? "interrupted" : "completed", ms: Math.round(ms), firstByteMs: firstByteMs == null ? null : Math.round(firstByteMs),
responseMs: responseMs == null ? null : Math.round(responseMs),
bytesPerSecond: responseMs != null && responseMs >= 1 && bytes > 0 ? Math.round(bytes * 1000 / responseMs) : null,
bytes, stages: trace.stages };
bytes, expectedBytes: expectedResponseBytes(req.method, res.statusCode, res.getHeader("content-length")),
endpointKind: service === "streamer" ? "internal" : "public", hopId, parentHopId,
...(aborted ? responseEvidence(res, req.aborted) : {}), ...trace.timing.finish(started + ms) };
if (slow || aborted) log.warn(aborted ? "request interrupted" : "slow request", detail);
else if (!route.includes("healthcheck")) log.info("request", detail);
};
res.once("finish", done); res.once("close", done);
context.run(trace, () => logContext.run({ ...reproduction, requestId: id, service, route, method }, next));
context.run(trace, () => logContext.run({ ...reproduction, requestId: id, hopId, parentHopId, service, route, method }, next));
};
}
+2 -1
View File
@@ -1,3 +1,4 @@
import { recordUpstreamFailure } from "./request-evidence";
import { PassThrough, Readable, Writable } from "stream";
/** Cancel upstream on every response terminal event, including a pre-header close. */
@@ -19,7 +20,7 @@ export function streamResponse(source: Readable, response: Writable,
};
const done = () => { cleanup(); resolve(); };
const failed = (error: Error) => { cleanup(); reject(error); };
const upstreamFailed = (error: Error) => { onError(error); done(); };
const upstreamFailed = (error: Error) => { if (!response.destroyed) recordUpstreamFailure(response, error, "streamer"); onError(error); done(); };
response.once("close", done);
response.once("finish", done);
response.once("error", failed);
+3 -1
View File
@@ -1,3 +1,4 @@
import { recordUpstreamFailure, recordResponseFailure } from "./request-evidence";
import got from "got";
import config from "../config";
import { MAX_SUBMODULE_ENTRIES, SubmoduleMount, safeRepositoryPath, validateMounts } from "./source/github-submodules";
@@ -169,6 +170,7 @@ export async function streamAnonymizedZip(
};
const fail = (error: Error) => {
if (stopped) return;
recordResponseFailure(res, "unknown", "local_abort");
logger.error("zip stream failed", serializeError(error));
stop();
const destroyable = res as unknown as {
@@ -197,7 +199,7 @@ export async function streamAnonymizedZip(
input.once("close", () => upstreamDone(!input.readableEnded));
parser = Parse() as Transform;
downloadStream
.on("error", fail)
.on("error", error => { recordUpstreamFailure(res, error); fail(error); })
.pipe(parser)
.on("entry", (entry: Readable & { type: string; path: string; autodrain: () => void }) => {
if (stopped) {
+6 -3
View File
@@ -1,5 +1,5 @@
import { createHash } from "crypto";
import { logRecord } from "../core/log-record";
import { logRecord, LOG_NUMBERS, LOG_STAGES } from "../core/log-record";
import { logContext, validTraceId } from "../core/log-context";
import { LOG_EVENTS } from "../core/log-schema";
@@ -50,13 +50,16 @@ function safeRow(input: Record<string, unknown>): Row | null {
const time = typeof input._time === "string" ? input._time : input.timestamp;
if (typeof time !== "string" || !Number.isFinite(Date.parse(time))) return null;
// Reuse the application's reviewed projection. Never return raw backend data.
const numeric = ["status", "httpStatus", "ms", "firstByteMs", "responseMs", "attempt", "rss", "memoryRatio", "droppedLogs", "oldestWaitingMs", "waiting"];
const numeric = [...LOG_NUMBERS, ...LOG_STAGES.flatMap(stage => [`stage_${stage}_ms`, `stage_${stage}_count`, `stage_${stage}_open`])];
const data: Record<string, unknown> = { ...input, code: input.errorCode };
for (const key of numeric) if (typeof input[key] === "string" && /^-?\d+(?:\.\d+)?$/.test(input[key] as string)) data[key] = Number(input[key]);
for (const [field, suffix] of [["stages", "ms"], ["stageCounts", "count"], ["stageOpen", "open"]]) {
data[field] = Object.fromEntries(LOG_STAGES.map(stage => [stage, data[`stage_${stage}_${suffix}`]]));
}
// Revalidate finite route and diagnostic allowlists; never pass raw strings.
const projected = logContext.run({}, () => logRecord(["error", "warn", "info", "debug"].includes(String(input.severity)) ? String(input.severity) : "info", String(input.module || "application"), ["", data]));
const result: Row = { timestamp: /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d{1,9})?Z$/.test(time) ? time : new Date(time).toISOString(), severity: String(projected.severity), service: services.includes(String(input.service)) ? String(input.service) : "unknown", event: events.has(String(input.event)) ? String(input.event) : "application_log", module: String(projected.module), release: /^(?:[a-f0-9]{7,40}|unknown)$/.test(String(input.release)) ? String(input.release) : "unknown" };
for (const key of [...numeric, "errorCode", "requestId", "jobId", "queue", "route", "method", "errorType", "errorStack", "causeType", "causeStack", "repoId", "requestPath"]) if (typeof projected[key] === "string" || typeof projected[key] === "number") result[key] = projected[key] as string | number;
for (const key of [...numeric, "outcome", "endpointKind", "cancellationOrigin", "cancellationEvidence", "hopId", "parentHopId", "errorCode", "requestId", "jobId", "queue", "route", "method", "errorType", "errorStack", "causeType", "causeStack", "repoId", "requestPath"]) if (typeof projected[key] === "string" || typeof projected[key] === "number") result[key] = projected[key] as string | number;
if (validTraceId(input.instance)) result.instance = input.instance;
return result;
}
+2
View File
@@ -1,3 +1,4 @@
import { installDatabaseTiming } from "../core/database-timing";
import InstallationModel from "../core/model/github-installation";
import { credentialCipher } from "../core/credentials";
import CredentialModel from "../core/model/credentials/credentials.model";
@@ -24,6 +25,7 @@ export const database = mongoose.connection;
export let isConnected = false;
export async function connect(overrides: Pick<ConnectOptions, "appName" | "maxPoolSize" | "minPoolSize"> = {}) {
installDatabaseTiming();
credentialCipher(); // Refuse to serve persisted credentials without a valid keyring.
mongoose.set("strictQuery", false);
const options: ConnectOptions = {
+2
View File
@@ -1,3 +1,4 @@
import { recordUpstreamFailure } from "../../core/request-evidence";
import { jobTrace } from "../../core/log-context";
import { hasVisibleSubmodules } from "../../core/file-submodules";
import { FILE_LISTING_LIMIT } from "../../core/file-tree";
@@ -89,6 +90,7 @@ router.get(
},
})
.on("error", (err: Error & { response?: { statusCode?: number; body?: unknown } }) => {
if (!res.destroyed) recordUpstreamFailure(res, err, "streamer");
headersDone(true);
const upstreamStatus = err?.response?.statusCode;
let upstreamBody: string | undefined;
+3 -2
View File
@@ -1,3 +1,4 @@
import { recordResponseFailure } from "../core/request-evidence";
import { sendSharedContent } from "../core/cached-response";
import * as express from "express";
import GitHubStream from "../core/source/GitHubStream";
@@ -125,9 +126,9 @@ router.post("/", async (req: express.Request, res: express.Response) => {
anonymizer.destroy();
}
content
.on("error", handleStreamError)
.on("error", error => { if (!res.destroyed) recordResponseFailure(res, "unknown", "local_abort"); handleStreamError(error); })
.pipe(anonymizer)
.on("error", handleStreamError)
.on("error", error => { if (!res.destroyed) recordResponseFailure(res, "streamer", "local_abort"); handleStreamError(error); })
.pipe(res)
.on("error", handleStreamError)
.on("close", () => {
+28
View File
@@ -0,0 +1,28 @@
const {expect}=require('chai');
require('ts-node/register/transpile-only');
const mongoose=require('mongoose');
const {installDatabaseTiming}=require('../src/core/database-timing');
const {monitorRequests,RequestMetrics}=require('../src/core/request-monitoring');
const express=require('express');
const http=require('node:http');
describe('database execution timing',()=>{
it('preserves query receiver, result and failure while measuring execution once',async()=>{
const Model=mongoose.model("TimingFixture",new mongoose.Schema({}));
const logs=[];const app=express();let server;
try{
installDatabaseTiming();const wrapped=mongoose.Query.prototype.exec;installDatabaseTiming();expect(mongoose.Query.prototype.exec).equal(wrapped);
app.use(monitorRequests('api',new RequestMetrics(),{info:(_,row)=>logs.push(row),warn:(_,row)=>logs.push(row)}));
app.get('/',async(_req,res)=>{
const query=Model.find();query._find=async function(){expect(this).equal(query);await new Promise(r=>setTimeout(r,20));return ['ok'];};
expect(await query.exec()).deep.equal(['ok']);
const failed=Model.find();const error=new Error('PRIVATE query');failed._find=async()=>{throw error;};
try{await failed.exec();throw Error('expected failure');}catch(e){expect(e).equal(error);}
res.end('ok');
});
server=await new Promise(resolve=>{const s=app.listen(0,'127.0.0.1',()=>resolve(s));});
await new Promise((resolve,reject)=>http.get({host:'127.0.0.1',port:server.address().port},res=>{res.resume();res.on('end',resolve);}).on('error',reject));
expect(logs[0].stageCounts.database).equal(2);expect(logs[0].stages.database).greaterThan(15);expect(logs[0].stageOpen.database).equal(0);
expect(JSON.stringify(logs)).not.include('PRIVATE');
}finally{mongoose.deleteModel("TimingFixture");if(server)await new Promise(resolve=>server.close(resolve));}
});
});
+68
View File
@@ -0,0 +1,68 @@
const { expect } = require('chai');
require('ts-node/register/transpile-only');
const { RequestTiming, requestRoute } = require('../src/core/request-monitoring');
const { recordResponseFailure, recordUpstreamFailure, responseEvidence, expectedResponseBytes } = require('../src/core/request-evidence');
const { logRecord } = require('../src/core/log-record');
const { safeRedisLog } = require('../src/server/admin-log-investigation');
describe('request causal evidence and timing accounting', () => {
it('integrates nested, parallel, repeated and unfinished spans without invented totals', () => {
let now = 0;
const timing = new RequestTiming(0, () => now);
now = 10; const credential = timing.start('credential_resolution');
now = 20; const db = timing.start('database');
now = 30; const upstream = timing.start('upstream');
now = 40; db(); db();
now = 50; upstream(); credential();
now = 60; timing.start('upstream');
now = 70; const snapshot = timing.finish();
expect(snapshot).deep.equal({stages:{credential_resolution:40,database:20,upstream:30},
stageCounts:{credential_resolution:1,database:1,upstream:2},stageOpen:{credential_resolution:0,database:0,upstream:1},
measuredMs:50,unattributedMs:20,overlapMs:40});
now = 100; upstream(); timing.start('database')();
expect(timing.finish()).deep.equal(snapshot);
});
it('bounds storage across repeated spans', () => {
let now = 0; const timing = new RequestTiming(0, () => now++);
for(let i=0;i<100000;i++) timing.start('upstream')();
const result=timing.finish();
expect(result.stageCounts.upstream).equal(100000);
expect(JSON.stringify(timing).length).lessThan(500);
});
it('keeps disconnect origins unknown and preserves the first causal evidence', () => {
expect(responseEvidence({})).deep.equal({cancellationOrigin:'unknown',cancellationEvidence:'response_close'});
expect(responseEvidence({},true).cancellationOrigin).equal('unknown');
const res={}; recordUpstreamFailure(res,Object.assign(new Error('PRIVATE'),{code:'ETIMEDOUT'}));
recordResponseFailure(res,'api','local_abort');
expect(responseEvidence(res)).deep.equal({cancellationOrigin:'timeout',cancellationEvidence:'timeout'});
const upstream={};recordUpstreamFailure(upstream,new Error('PRIVATE'),'streamer');
expect(responseEvidence(upstream)).deep.equal({cancellationOrigin:'streamer',cancellationEvidence:'upstream_error'});
const local={};recordResponseFailure(local,'api','local_abort');expect(responseEvidence(local).cancellationOrigin).equal('api');
});
it('only compares body bytes with the output content length', () => {
expect(expectedResponseBytes('GET',200,'123')).equal(123);
expect(expectedResponseBytes('HEAD',200,'123')).equal(0);
expect(expectedResponseBytes('GET',304,'123')).equal(0);
for(const value of [undefined,-1,'1.2','9e5','9007199254740992',{},['123']]) expect(expectedResponseBytes('GET',200,value)).equal(undefined);
});
it('preserves bounded evidence and timings through the admin projection', () => {
const row=logRecord('warn','requests',['request interrupted',{outcome:'interrupted',bytes:12,expectedBytes:50,
cancellationOrigin:'unknown',cancellationEvidence:'response_close',endpointKind:'public',measuredMs:20,unattributedMs:10,overlapMs:5,
stages:{database:3,upstream:17,PRIVATE:123},stageCounts:{upstream:2},stageOpen:{upstream:1}}]);
const safe=safeRedisLog({ts:row.timestamp,level:'warn',module:'requests',raw:[row]}).raw[0];
expect(safe).include({bytes:12,expectedBytes:50,cancellationOrigin:'unknown',cancellationEvidence:'response_close',
endpointKind:'public',measuredMs:20,unattributedMs:10,overlapMs:5,stage_database_ms:3,stage_upstream_ms:17,stage_upstream_count:2,stage_upstream_open:1});
expect(JSON.stringify(safe)).not.include('PRIVATE');
const bad=logRecord('warn','requests',['request interrupted',{cancellationOrigin:'PRIVATE',cancellationEvidence:'PRIVATE',endpointKind:'PRIVATE'}]);
expect(JSON.stringify(bad)).not.include('PRIVATE');
});
it('respects source handler precedence and canonical anonymous route IDs', () => {
for(const action of ['readme','branches','tree','file']) expect(requestRoute('/api/repo/private/files/'+action,'api')).equal('/api/repo/:owner/:repo/'+action);
expect(requestRoute('/api/repo/demo/file/readme','api')).equal('/api/repo/:repoId/file/:path');
for(const action of ['coauthors','extend','refresh']) {
expect(requestRoute('/api/repo/private/'+action,'api','GET')).equal('/api/repo/:owner/:repo');
expect(requestRoute('/api/repo/demo/'+action,'api','POST')).equal('/api/repo/:repoId/'+action);
}
expect(logRecord('info','requests',['request',{route:'/api/repo/:id/files'}]).route).equal('/api/repo/:repoId/files');
});
});
+45
View File
@@ -174,6 +174,7 @@ describe("bounded request monitoring", function () {
await ready; client.destroy(); await terminal;
expect(logs).to.have.length(1); expect(logs[0][0]).to.equal("request interrupted");
expect(logs[0][1].outcome).to.equal("interrupted");
expect(logs[0][1].cancellationOrigin).to.equal("unknown");
expect(logs[0][1]).include({requestPath:"/api/repo/demo-123/zip",repoId:"demo-123"});
expect(JSON.stringify(logs)).not.include("SECRET");
const row = metrics.drain().find(row => row.data.metric === "request").data;
@@ -186,6 +187,50 @@ describe("bounded request monitoring", function () {
});
}
for (const code of ["ETIMEDOUT", "ECONNRESET"]) {
it(`records causal ${code} before a truncated 200 closes`, async function () {
const { streamResponse } = require("../src/core/response-stream");
app.get("/", (_req, res) => {
const source = new Readable({ read() {} });
res.setHeader("Content-Length", "100");
void streamResponse(source, res, () => res.destroy());
source.push("partial");
setTimeout(() => source.destroy(Object.assign(new Error("PRIVATE"), {code})), 20);
});
await listen();
await fetch(server, "/").catch(() => undefined);
await delay(20);
const detail = logs[0][1];
expect(detail).include({status:200,outcome:"interrupted",bytes:7,expectedBytes:100,
cancellationOrigin:code === "ETIMEDOUT" ? "timeout" : "streamer"});
expect(JSON.stringify(logs)).not.include("PRIVATE");
});
}
it("correlates repeated streamer hops without accepting public caller IDs", async function () {
const upstream = express(); const hops=[];
upstream.use(monitorRequests("streamer",new RequestMetrics(),{info:(_,row)=>hops.push(row),warn:(_,row)=>hops.push(row)}));
upstream.post("/api",(_req,res)=>res.end("ok"));
const streamer = await new Promise(resolve => {const s=upstream.listen(0,"127.0.0.1",()=>resolve(s));});
try {
app.get("/", async (_req,res)=>{
await fetch(streamer,"/api",{method:"POST",headers:requestHeaders()});
await fetch(streamer,"/api",{method:"POST",headers:requestHeaders()});
res.end("ok");
});
await listen();
const forged="11111111-1111-1111-1111-111111111111";
await fetch(server,"/",{headers:{"x-request-id":forged,"x-parent-hop-id":forged}});
const parent=logs[0][1];
expect(parent.requestId).not.equal(forged);
expect(parent.parentHopId).equal(undefined);
expect(hops).length(2);
expect(hops[0].hopId).not.equal(hops[1].hopId);
for(const hop of hops) expect(hop).include({requestId:parent.requestId,parentHopId:parent.hopId,endpointKind:"internal",outcome:"completed"});
expect(parent).not.have.property("cancellationOrigin");
} finally {streamer.closeAllConnections();await new Promise(resolve=>streamer.close(resolve));}
});
it("keeps overlapping request IDs and asynchronous stages isolated", async function () {
let doneA, doneB;
app.get("/a", (_req, res) => { doneA = startStage("upstream"); setTimeout(() => { doneA(); doneA(); res.json(requestHeaders()); }, 30); });
+24 -2
View File
@@ -26,7 +26,7 @@ function request(port, path, encoding = "gzip", extra = {}) {
}
describe("shared cache HTTP and telemetry integration", function () {
this.timeout(15000);
let root, old, server, port, fills, body;
let root, old, server, port, fills, body, requestLogs;
beforeEach(async () => {
old = { ...config }; closeBlobCaches(); closeCacheMetrics();
root = await fs.mkdtemp(join(tmpdir(), "shared-http-"));
@@ -34,7 +34,8 @@ describe("shared cache HTTP and telemetry integration", function () {
SHARED_CACHE_BYTES: 16 * 1024 ** 2, SHARED_CACHE_RESULT_BYTES: 8 * 1024 ** 2, SHARED_CACHE_MAX_FILE_BYTES: 1024 ** 2,
SHARED_CACHE_FREE_BYTES: 0, CACHE_MONITORING_ENABLED: true });
fills = 0; body = Buffer.from("Public documentation with no identifying names.\n".repeat(100));
const app = express(); app.use(monitorRequests("api", undefined, { info() {}, warn() {} }));
requestLogs = [];
const app = express(); app.use(monitorRequests("api", undefined, { info: (_, row) => requestLogs.push(row), warn: (_, row) => requestLogs.push(row) }));
app.get("/:project/:mode", async (req, res) => {
if (req.headers.authorization !== "test-access") return res.status(403).end();
const source = new GitHubStream({ repoId: req.params.project, organization: "org", repoName: "repo", commit: "commit", getToken: async () => "" });
@@ -56,6 +57,27 @@ describe("shared cache HTTP and telemetry integration", function () {
await new Promise(resolve => server.close(resolve)); closeBlobCaches(); closeCacheMetrics(); Object.assign(config, old);
await fs.rm(root, { recursive: true, force: true });
});
it("records a shared result cache read failure before pipeline destroys the response", async () => {
await request(port, "/a/changed", "identity", {authorization:"test-access"});
const results = resultBlobCache(), original = results.open;
results.open = async function (...args) {
const stored = await Reflect.apply(original, this, args);
if (!stored) return stored;
const broken = new Readable({read() {}});
broken.cachedBlob = stored.cachedBlob; stored.destroy();
setTimeout(() => broken.push(Buffer.from("partial")), 10);
setTimeout(() => broken.destroy(Object.assign(new Error("PRIVATE local cache"), {code:"EIO"})), 30);
return broken;
};
try {
await assert.rejects(request(port, "/a/changed", "identity", {authorization:"test-access"}));
await new Promise(resolve => setTimeout(resolve, 20));
const row = requestLogs.at(-1);
assert.equal(row.status, 200); assert.equal(row.outcome, "interrupted");
assert.equal(row.cancellationOrigin, "unknown"); assert.equal(row.cancellationEvidence, "local_abort");
assert.equal(row.bytes, 7); assert.ok(row.expectedBytes > 7);
} finally { results.open = original; }
});
for (const mode of ["fill", "hit", "bypass"]) {
it(`enforces resolved-byte billing admission on shared source ${mode}`, async () => {
const quotas = require("../src/core/billing/quotas");
+38
View File
@@ -0,0 +1,38 @@
const {expect}=require('chai');
const express=require('express');
const http=require('node:http');
const {Readable}=require('node:stream');
require('ts-node/register/transpile-only');
const {monitorRequests,RequestMetrics}=require('../src/core/request-monitoring');
const GitHubStream=require('../src/core/source/GitHubStream').default;
const {AnonymizeTransformer}=require('../src/core/anonymize-utils');
const streamer=require('../src/streamer/route').default;
const config=require('../src/config').default;
describe('streamer failure provenance',()=>{
for(const mode of ['cache','transform']) it(`records ${mode} failures before teardown without inventing upstream provenance`,async()=>{
const originalCache=GitHubStream.prototype.getFileContentCache,originalTransform=AnonymizeTransformer.prototype._transform,shared=config.SHARED_CACHE_ENABLED;
const logs=[];let server;
try{
config.SHARED_CACHE_ENABLED=false;
GitHubStream.prototype.getFileContentCache=async()=>{
const input=new Readable({read(){}});
setTimeout(()=>input.push(Buffer.from('partial')),10);
if(mode==='cache')setTimeout(()=>input.destroy(Object.assign(new Error('PRIVATE cache path'),{code:'EIO'})),30);
else setTimeout(()=>input.push(Buffer.from('fail')),30);
return input;
};
AnonymizeTransformer.prototype._transform=function(chunk,_encoding,done){
if(mode==='transform'&&chunk.toString()==='fail')done(new Error('PRIVATE output spool'));
else {this.push(chunk);done();}
};
const app=express();app.use(monitorRequests('streamer',new RequestMetrics(),{info:(_,r)=>logs.push(r),warn:(_,r)=>logs.push(r)}));app.use(express.json());app.use('/api',streamer);
server=await new Promise(resolve=>{const s=app.listen(0,'127.0.0.1',()=>resolve(s));});
await new Promise(resolve=>{
const req=http.request({host:'127.0.0.1',port:server.address().port,path:'/api',method:'POST',headers:{'content-type':'application/json'}},res=>{res.resume();res.on('close',resolve);});
req.on('error',resolve);req.end(JSON.stringify({token:'',repoFullName:'PRIVATE/repo',repoId:'demo',filePath:'file.txt',anonymizerOptions:{terms:[],filePath:'file.txt'}}));
});
await new Promise(resolve=>setTimeout(resolve,20));
expect(logs).length(1);expect(logs[0]).include({status:200,outcome:'interrupted',cancellationOrigin:mode==='transform'?'streamer':'unknown',cancellationEvidence:'local_abort',bytes:7});
}finally{GitHubStream.prototype.getFileContentCache=originalCache;AnonymizeTransformer.prototype._transform=originalTransform;config.SHARED_CACHE_ENABLED=shared;if(server){server.closeAllConnections();await new Promise(resolve=>server.close(resolve));}}
});
});