fix: explain interrupted requests and account for overlapping latency

This commit is contained in:
Thomas Durieux committed 2026-11-15 13:31:16 +00:00
1 parent 1e617d2df0
commit d2e3954d21
18 files changed
+394 -33

No files matched your search

+53
View File
@@ -359,3 +359,56 @@ Application requests to the supported anonymous repository endpoints retain thei
For the focused restricted-file/rotation regression, run
`LOGGING_ACCESS_ONLY=1 python3 scripts/test-logging.py`. This uses synthetic
`0640` logs in an isolated Docker project and preserves capability restrictions.
## Request evidence contract
`outcome=completed` means Node emitted response `finish`. It does not prove that
an end user received the body. A 200 can have `outcome=interrupted`. Count public
API outcomes once; correlated streamer hops describe the same operation and must
not be added to public failure counts. `endpointKind` distinguishes public API
from internal streamer requests. Each middleware invocation has a fresh `hopId`;
`parentHopId` links streamer calls to the API hop, including repeated cache probes.
Public caller correlation headers are ignored. Internal IDs are diagnostics, not
proof of caller identity. Existing job UUIDs and attempt numbers remain separate.
For interrupted responses, `cancellationOrigin` and `cancellationEvidence` retain
the first observed causal boundary. Explicit upstream errors identify the failing
upstream hop; ETIMEDOUT identifies a timeout. A request abort or response close
alone has origin `unknown`: the peer may be a proxy, API or browser, and a close
can also follow server teardown. No client-action claim is inferred from 200,
socket close, or lack of error logs. ZIP processing failures with no identified
upstream retain unknown origin with local-abort evidence. Historical records lack
this evidence and cannot be reclassified reliably.
`bytes` counts encoded body bytes passed to Node response writes, including
buffered writes; it is not an acknowledgement from the peer. `expectedBytes` is
only the response Content-Length, absent when unknown. Source file size is not
comparable after anonymization/compression. HEAD, 204 and 304 have zero expected
body bytes. `firstByteMs` is the existing header-write boundary, not a measured
network first byte. `responseMs` and `stage_transfer_ms` cover header write through
finish/close, including backpressure and waits for content.
Named stage durations are inclusive sums of instrumented spans, clipped at the
response terminal event. Repeated and parallel spans add to their stage totals.
`stage_<name>_count` counts instrumented calls, not hidden library retries;
`stage_<name>_open` counts spans still running at termination. Uninstrumented
library retries/backoff stay inside their enclosing span and cannot be separated.
`measuredMs` is the union of all spans, `overlapMs` is the excess of their summed
durations over that union, and `unattributedMs` is request time outside any span.
Do not add transfer, credential, database, cache, upstream or anonymization totals
to estimate request duration. Millisecond rounding of legacy request totals may
cause sub-millisecond differences. Storage is fixed per stage, not per call.
`database` measures Mongoose Query/Aggregate execution after database setup,
including pool/buffering wait and conversion. It excludes document saves, bulk
writes and direct driver operations. `repository` remains the application
repository lookup, which can enclose database time. Credential resolution can
enclose database and GitHub work. GitHub `upstream` spans cover the existing
Octokit request wrapper or explicitly instrumented stream intervals. Cache lookup,
fill, worker admission, anonymization and streamer-header spans retain their
existing call boundaries. Missing stages mean unavailable instrumentation, never
zero cost. Unattributed time is visible; it is not evidence of a stall.
The admin API and Redis projection preserve these allowlisted fields. Collector
and application versions must be updated together. No ordinary request depends
on the collector, Grafana or VictoriaLogs being available.
+7 -4
View File
@@ -46,18 +46,21 @@ if service == "mongodb" {
if includes(["github_app_authorization", "installation", "installation_repositories"], record.webhookEvent) { out.webhookEvent = record.webhookEvent }
if includes(["github.com", "api.github.com", "codeload.github.com", "raw.githubusercontent.com", "media.githubusercontent.com", "other"], record.upstreamHost) { out.upstreamHost = record.upstreamHost }
if includes(["download", "remove", "cache"], record.queue) { out.queue = record.queue }
if includes(["unknown", "api", "streamer", "timeout", "upstream"], record.cancellationOrigin) { out.cancellationOrigin = record.cancellationOrigin }
if includes(["response_close", "request_aborted", "upstream_error", "timeout", "local_abort"], record.cancellationEvidence) { out.cancellationEvidence = record.cancellationEvidence }
if includes(["public", "internal"], record.endpointKind) { out.endpointKind = record.endpointKind }
if includes(["completed", "interrupted"], record.outcome) { out.outcome = record.outcome }
}
for_each(["requestId", "jobId", "deliveryId"]) -> |_index, key| {
for_each(["requestId", "jobId", "deliveryId", "hopId", "parentHopId"]) -> |_index, key| {
value = get(record, [key]) ?? null
if is_string(value) && match(string!(value), r'^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$') { out = set!(out, [key], value) }
}
for_each(["status", "httpStatus", "ms", "firstByteMs", "responseMs", "bytes", "bytesPerSecond", "attempt", "rss", "memoryLimitBytes", "memoryRatio", "uptime", "activeRequests", "waiting", "active", "delayed", "failed", "oldestWaitingMs", "droppedLogs", "loopP95Ms", "upstreamStatus", "stage_repository_ms", "stage_credential_resolution_ms", "stage_upstream_ms", "stage_cache_lookup_ms", "stage_cache_fill_ms", "stage_worker_wait_ms", "stage_anonymize_ms", "stage_streamer_headers_ms"]) -> |_index, key| {
for_each(["stage_transfer_ms", "stage_transfer_count", "stage_transfer_open", "stage_database_ms", "stage_database_count", "stage_database_open", "expectedBytes", "measuredMs", "unattributedMs", "overlapMs", "stage_repository_count", "stage_repository_open", "stage_credential_resolution_count", "stage_credential_resolution_open", "stage_upstream_count", "stage_upstream_open", "stage_cache_lookup_count", "stage_cache_lookup_open", "stage_cache_fill_count", "stage_cache_fill_open", "stage_worker_wait_count", "stage_worker_wait_open", "stage_anonymize_count", "stage_anonymize_open", "stage_streamer_headers_count", "stage_streamer_headers_open", "status", "httpStatus", "ms", "firstByteMs", "responseMs", "bytes", "bytesPerSecond", "attempt", "rss", "memoryLimitBytes", "memoryRatio", "uptime", "activeRequests", "waiting", "active", "delayed", "failed", "oldestWaitingMs", "droppedLogs", "loopP95Ms", "upstreamStatus", "stage_repository_ms", "stage_credential_resolution_ms", "stage_upstream_ms", "stage_cache_lookup_ms", "stage_cache_fill_ms", "stage_worker_wait_ms", "stage_anonymize_ms", "stage_streamer_headers_ms"]) -> |_index, key| {
value = get(record, [key]) ?? null
if is_integer(value) || is_float(value) { out = set!(out, [key], value) }
}
if includes(["GET", "HEAD", "POST", "PUT", "PATCH", "DELETE", "OPTIONS", "OTHER"], record.method) { out.method = record.method }
if includes(["/", "other", "/dashboard", "/anonymize", "/healthcheck", "/api/healthcheck", "/api", "/api/download", "/api/stat", "/api/stat/history", "/api/options", "/api/message", "/api/user", "/api/user/quota", "/api/user/dashboard", "/api/repo", "/api/anonymize-preview", "/api/admin/overview", "/api/admin/performance", "/api/admin/errors", "/api/admin/errors/stats", "/api/admin/queues", "/api/admin/queues/metrics", "/github/app/webhook", "/api/repo/:repoId", "/api/repo/:repoId/file/:path", "/api/repo/:repoId/files", "/api/repo/:repoId/files/counts", "/api/repo/:repoId/files/search", "/api/repo/:repoId/zip", "/api/repo/:repoId/options", "/api/repo/:repoId/refresh", "/api/repo/:repoId/coauthors", "/api/repo/:repoId/extend", "/api/repo/:owner/:repo/readme", "/api/repo/:owner/:repo/branches", "/w/:repoId/:path", "/r/:repoId/:path", "/api/pr/:id/:path", "/api/gist/:id/:path", "/assets/:path", "/github/:action", "/api/admin/:action", "/api/user/:action", "/api/repo/:id/file/:path", "/api/repo/:id/files/counts", "/api/repo/:id/files/search", "/api/repo/:id/files", "/api/repo/:id/zip", "/api/repo/:id", "/viewer/:id/:path"], record.route) { out.route = record.route }
if includes(["/", "other", "/dashboard", "/anonymize", "/healthcheck", "/api/healthcheck", "/api", "/api/download", "/api/stat", "/api/stat/history", "/api/options", "/api/message", "/api/user", "/api/user/quota", "/api/user/dashboard", "/api/repo", "/api/anonymize-preview", "/api/admin/overview", "/api/admin/performance", "/api/admin/errors", "/api/admin/errors/stats", "/api/admin/queues", "/api/admin/queues/metrics", "/github/app/webhook", "/api/repo/:repoId", "/api/repo/:repoId/file/:path", "/api/repo/:repoId/files", "/api/repo/:repoId/files/counts", "/api/repo/:repoId/files/search", "/api/repo/:repoId/zip", "/api/repo/:repoId/options", "/api/repo/:repoId/refresh", "/api/repo/:repoId/coauthors", "/api/repo/:repoId/extend", "/api/repo/:owner/:repo", "/api/repo/:owner/:repo/tree", "/api/repo/:owner/:repo/file", "/api/repo/:owner/:repo/readme", "/api/repo/:owner/:repo/branches", "/w/:repoId/:path", "/r/:repoId/:path", "/api/pr/:id/:path", "/api/gist/:id/:path", "/assets/:path", "/github/:action", "/api/admin/:action", "/api/user/:action", "/api/repo/:id/file/:path", "/api/repo/:id/files/counts", "/api/repo/:id/files/search", "/api/repo/:id/files", "/api/repo/:id/zip", "/api/repo/:id", "/viewer/:id/:path"], record.route) { out.route = record.route }
# VictoriaLogs consumes its message field; keep event independently searchable.
# Anonymous application request context at every outcome, never query strings.
{
@@ -72,7 +75,7 @@ if includes(["/", "other", "/dashboard", "/anonymize", "/healthcheck", "/api/hea
}
}
# BEGIN GENERATED DIAGNOSTICS
source_functions = {"src/cli/index.ts":["<anonymous>","generateRandomFileName","main"],"build/cli/index.js":["<anonymous>","generateRandomFileName","main"],"src/config.ts":["<anonymous>"],"build/config.js":["<anonymous>"],"src/core/AnonymizedFile.ts":["<anonymous>","anonymizedContent","assertContentFile","cancel","content","defaultMaskCandidateRegex","escapeRegex","extension","forwardHeaders","getFileInfo","handleStreamError","isFileSupported","isImage","originalPath","probe","recoverTruncatedFile","send","sha","size","streamerErrorToAnonymous","visibleFile"],"build/core/AnonymizedFile.js":["<anonymous>","anonymizedContent","assertContentFile","cancel","content","defaultMaskCandidateRegex","escapeRegex","extension","forwardHeaders","getFileInfo","handleStreamError","isFileSupported","isImage","originalPath","probe","recoverTruncatedFile","send","sha","size","streamerErrorToAnonymous","visibleFile"],"src/core/AnonymousError.ts":["<anonymous>","detail","toString","url"],"build/core/AnonymousError.js":["<anonymous>","detail","toString","url"],"src/core/Conference.ts":["<anonymous>","expire","isExpired","remove","repositories","toJSON","updateStatus"],"build/core/Conference.js":["<anonymous>","expire","isExpired","remove","repositories","toJSON","updateStatus"],"src/core/Gist.ts":["<anonymous>","anonymize","check","conference","content","countView","download","downloadContent","downloadWithAccess","expire","getAccess","getToken","markExpired","remove","resetSate","toJSON","updateIfNeeded","updateStatus"],"build/core/Gist.js":["<anonymous>","anonymize","check","conference","content","countView","download","downloadContent","downloadWithAccess","expire","getAccess","getToken","markExpired","remove","resetSate","toJSON","updateIfNeeded","updateStatus"],"src/core/GitHubUtils.ts":["<anonymous>","abortableGateLookup","cancelled","checkToken","ensureRedisGateClient","getRedisGateResetAt","getToken","getTokenGateResetAt","octokit","rateLimitDetail","resolveRepositoryToken","run","setRedisGate","setRedisGateFromWorker","setTokenGate","waitForTokenGate"],"build/core/GitHubUtils.js":["<anonymous>","abortableGateLookup","cancelled","checkToken","ensureRedisGateClient","getRedisGateResetAt","getToken","getTokenGateResetAt","octokit","rateLimitDetail","resolveRepositoryToken","run","setRedisGate","setRedisGateFromWorker","setTokenGate","waitForTokenGate"],"src/core/PullRequest.ts":["<anonymous>","anonymize","check","conference","content","countView","download","downloadContent","expire","getToken","markExpired","remove","resetSate","toJSON","updateIfNeeded","updateStatus"],"build/core/PullRequest.js":["<anonymous>","anonymize","check","conference","content","countView","download","downloadContent","expire","getToken","markExpired","remove","resetSate","toJSON","updateIfNeeded","updateStatus"],"src/core/Repository.ts":["<anonymous>","anonymize","anonymizeTreeRecursive","anonymizedFiles","assertNotArchived","buildPathIndex","check","checkRefreshWrite","cleanupRetiredFileTrees","cleanupStagedFileTree","clearRows","completeRecoveredFileMetadata","computeSize","conference","countView","current","ensureFileTree","expire","fileSnapshotFilter","files","findAnonymizedPath","generateAnonymizeTransformer","getToken","hasEmptyTree","indexPaths","insertBatch","invalidateFileMetadata","isReady","markCachePresent","markExpired","pathKey","readFiles","refresh","refreshFilter","refreshIfNeeded","remove","removeCache","resetSate","searchFiles","snapshotValue","submoduleMounts","toJSON","updateIfNeeded","updateProgressMessage","updateRetiringPathIndex","updateStatus","zip"],"build/core/Repository.js":["<anonymous>","anonymize","anonymizeTreeRecursive","anonymizedFiles","assertNotArchived","buildPathIndex","check","checkRefreshWrite","cleanupRetiredFileTrees","cleanupStagedFileTree","clearRows","completeRecoveredFileMetadata","computeSize","conference","countView","current","ensureFileTree","expire","fileSnapshotFilter","files","findAnonymizedPath","generateAnonymizeTransformer","getToken","hasEmptyTree","indexPaths","insertBatch","invalidateFileMetadata","isReady","markCachePresent","markExpired","pathKey","readFiles","refresh","refreshFilter","refreshIfNeeded","remove","removeCache","resetSate","searchFiles","snapshotValue","submoduleMounts","toJSON","updateIfNeeded","updateProgressMessage","updateRetiringPathIndex","updateStatus","zip"],"src/core/User.ts":["<anonymous>","getAccessToken","getGists","getGitHubRepositories","getPullRequests","getRepositories","markListExpired","quotaRepositories","repositoryMembership","toJSON"],"build/core/User.js":["<anonymous>","getAccessToken","getGists","getGitHubRepositories","getPullRequests","getRepositories","markListExpired","quotaRepositories","repositoryMembership","toJSON"],"src/core/anonymization-pool.ts":["<anonymous>","anonymizeOnWorker","cancel","exited","failed","finish","getAnonymizationPoolStats","message","pump","release","releaseSpool","reserveSpool"],"build/core/anonymization-pool.js":["Line truncated
source_functions = {"src/cli/index.ts":["<anonymous>","generateRandomFileName","main"],"build/cli/index.js":["<anonymous>","generateRandomFileName","main"],"src/config.ts":["<anonymous>"],"build/config.js":["<anonymous>"],"src/core/AnonymizedFile.ts":["<anonymous>","anonymizedContent","assertContentFile","cancel","content","defaultMaskCandidateRegex","escapeRegex","extension","forwardHeaders","getFileInfo","handleStreamError","isFileSupported","isImage","originalPath","probe","recoverTruncatedFile","send","sha","size","streamerErrorToAnonymous","visibleFile"],"build/core/AnonymizedFile.js":["<anonymous>","anonymizedContent","assertContentFile","cancel","content","defaultMaskCandidateRegex","escapeRegex","extension","forwardHeaders","getFileInfo","handleStreamError","isFileSupported","isImage","originalPath","probe","recoverTruncatedFile","send","sha","size","streamerErrorToAnonymous","visibleFile"],"src/core/AnonymousError.ts":["<anonymous>","detail","toString","url"],"build/core/AnonymousError.js":["<anonymous>","detail","toString","url"],"src/core/Conference.ts":["<anonymous>","expire","isExpired","remove","repositories","toJSON","updateStatus"],"build/core/Conference.js":["<anonymous>","expire","isExpired","remove","repositories","toJSON","updateStatus"],"src/core/Gist.ts":["<anonymous>","anonymize","check","conference","content","countView","download","downloadContent","downloadWithAccess","expire","getAccess","getToken","markExpired","remove","resetSate","toJSON","updateIfNeeded","updateStatus"],"build/core/Gist.js":["<anonymous>","anonymize","check","conference","content","countView","download","downloadContent","downloadWithAccess","expire","getAccess","getToken","markExpired","remove","resetSate","toJSON","updateIfNeeded","updateStatus"],"src/core/GitHubUtils.ts":["<anonymous>","abortableGateLookup","cancelled","checkToken","ensureRedisGateClient","getRedisGateResetAt","getToken","getTokenGateResetAt","octokit","rateLimitDetail","resolveRepositoryToken","run","setRedisGate","setRedisGateFromWorker","setTokenGate","waitForTokenGate"],"build/core/GitHubUtils.js":["<anonymous>","abortableGateLookup","cancelled","checkToken","ensureRedisGateClient","getRedisGateResetAt","getToken","getTokenGateResetAt","octokit","rateLimitDetail","resolveRepositoryToken","run","setRedisGate","setRedisGateFromWorker","setTokenGate","waitForTokenGate"],"src/core/PullRequest.ts":["<anonymous>","anonymize","check","conference","content","countView","download","downloadContent","expire","getToken","markExpired","remove","resetSate","toJSON","updateIfNeeded","updateStatus"],"build/core/PullRequest.js":["<anonymous>","anonymize","check","conference","content","countView","download","downloadContent","expire","getToken","markExpired","remove","resetSate","toJSON","updateIfNeeded","updateStatus"],"src/core/Repository.ts":["<anonymous>","anonymize","anonymizeTreeRecursive","anonymizedFiles","assertNotArchived","buildPathIndex","check","checkRefreshWrite","cleanupRetiredFileTrees","cleanupStagedFileTree","clearRows","completeRecoveredFileMetadata","computeSize","conference","countView","current","ensureFileTree","expire","fileSnapshotFilter","files","findAnonymizedPath","generateAnonymizeTransformer","getToken","hasEmptyTree","indexPaths","insertBatch","invalidateFileMetadata","isReady","markCachePresent","markExpired","pathKey","readFiles","refresh","refreshFilter","refreshIfNeeded","remove","removeCache","resetSate","searchFiles","snapshotValue","submoduleMounts","toJSON","updateIfNeeded","updateProgressMessage","updateRetiringPathIndex","updateStatus","zip"],"build/core/Repository.js":["<anonymous>","anonymize","anonymizeTreeRecursive","anonymizedFiles","assertNotArchived","buildPathIndex","check","checkRefreshWrite","cleanupRetiredFileTrees","cleanupStagedFileTree","clearRows","completeRecoveredFileMetadata","computeSize","conference","countView","current","ensureFileTree","expire","fileSnapshotFilter","files","findAnonymizedPath","generateAnonymizeTransformer","getToken","hasEmptyTree","indexPaths","insertBatch","invalidateFileMetadata","isReady","markCachePresent","markExpired","pathKey","readFiles","refresh","refreshFilter","refreshIfNeeded","remove","removeCache","resetSate","searchFiles","snapshotValue","submoduleMounts","toJSON","updateIfNeeded","updateProgressMessage","updateRetiringPathIndex","updateStatus","zip"],"src/core/User.ts":["<anonymous>","getAccessToken","getGists","getGitHubRepositories","getPullRequests","getRepositories","markListExpired","quotaRepositories","repositoryMembership","toJSON"],"build/core/User.js":["<anonymous>","getAccessToken","getGists","getGitHubRepositories","getPullRequests","getRepositories","markListExpired","quotaRepositories","repositoryMembership","toJSON"],"src/core/anonymization-pool.ts":["<anonymous>","anonymizeOnWorker","cancel","exited","failed","finish","getAnonymizationPoolStats","message","pump","release","releaseSpool","reserveSpool"],"build/core/anonymization-pool.js":["Line truncated
for_each(["errorType", "causeType"]) -> |_index, key| {
value = get(record, [key]) ?? null
if includes(["Error","TypeError","RangeError","ReferenceError","SyntaxError","URIError","EvalError","AggregateError","AnonymousError","HTTPError","RequestError","MongoServerError","MongoNetworkError","MongoServerSelectionError","ValidationError","TimeoutError","AbortError","UnknownError"], value) { out = set!(out, [key], value) }