mirror of
https://github.com/tdurieux/anonymous_github.git
synced 2026-11-18 01:54:23 +01:00
fix: explain interrupted requests and account for overlapping latency
This commit is contained in:
1 parent
1e617d2df0
commit
d2e3954d21
18 files changed
+394
-33
No files matched your search
@@ -359,3 +359,56 @@ Application requests to the supported anonymous repository endpoints retain thei
|
||||
For the focused restricted-file/rotation regression, run
|
||||
`LOGGING_ACCESS_ONLY=1 python3 scripts/test-logging.py`. This uses synthetic
|
||||
`0640` logs in an isolated Docker project and preserves capability restrictions.
|
||||
|
||||
## Request evidence contract
|
||||
|
||||
`outcome=completed` means Node emitted response `finish`. It does not prove that
|
||||
an end user received the body. A 200 can have `outcome=interrupted`. Count public
|
||||
API outcomes once; correlated streamer hops describe the same operation and must
|
||||
not be added to public failure counts. `endpointKind` distinguishes public API
|
||||
from internal streamer requests. Each middleware invocation has a fresh `hopId`;
|
||||
`parentHopId` links streamer calls to the API hop, including repeated cache probes.
|
||||
Public caller correlation headers are ignored. Internal IDs are diagnostics, not
|
||||
proof of caller identity. Existing job UUIDs and attempt numbers remain separate.
|
||||
|
||||
For interrupted responses, `cancellationOrigin` and `cancellationEvidence` retain
|
||||
the first observed causal boundary. Explicit upstream errors identify the failing
|
||||
upstream hop; ETIMEDOUT identifies a timeout. A request abort or response close
|
||||
alone has origin `unknown`: the peer may be a proxy, API or browser, and a close
|
||||
can also follow server teardown. No client-action claim is inferred from 200,
|
||||
socket close, or lack of error logs. ZIP processing failures with no identified
|
||||
upstream retain unknown origin with local-abort evidence. Historical records lack
|
||||
this evidence and cannot be reclassified reliably.
|
||||
|
||||
`bytes` counts encoded body bytes passed to Node response writes, including
|
||||
buffered writes; it is not an acknowledgement from the peer. `expectedBytes` is
|
||||
only the response Content-Length, absent when unknown. Source file size is not
|
||||
comparable after anonymization/compression. HEAD, 204 and 304 have zero expected
|
||||
body bytes. `firstByteMs` is the existing header-write boundary, not a measured
|
||||
network first byte. `responseMs` and `stage_transfer_ms` cover header write through
|
||||
finish/close, including backpressure and waits for content.
|
||||
|
||||
Named stage durations are inclusive sums of instrumented spans, clipped at the
|
||||
response terminal event. Repeated and parallel spans add to their stage totals.
|
||||
`stage_<name>_count` counts instrumented calls, not hidden library retries;
|
||||
`stage_<name>_open` counts spans still running at termination. Uninstrumented
|
||||
library retries/backoff stay inside their enclosing span and cannot be separated.
|
||||
`measuredMs` is the union of all spans, `overlapMs` is the excess of their summed
|
||||
durations over that union, and `unattributedMs` is request time outside any span.
|
||||
Do not add transfer, credential, database, cache, upstream or anonymization totals
|
||||
to estimate request duration. Millisecond rounding of legacy request totals may
|
||||
cause sub-millisecond differences. Storage is fixed per stage, not per call.
|
||||
|
||||
`database` measures Mongoose Query/Aggregate execution after database setup,
|
||||
including pool/buffering wait and conversion. It excludes document saves, bulk
|
||||
writes and direct driver operations. `repository` remains the application
|
||||
repository lookup, which can enclose database time. Credential resolution can
|
||||
enclose database and GitHub work. GitHub `upstream` spans cover the existing
|
||||
Octokit request wrapper or explicitly instrumented stream intervals. Cache lookup,
|
||||
fill, worker admission, anonymization and streamer-header spans retain their
|
||||
existing call boundaries. Missing stages mean unavailable instrumentation, never
|
||||
zero cost. Unattributed time is visible; it is not evidence of a stall.
|
||||
|
||||
The admin API and Redis projection preserve these allowlisted fields. Collector
|
||||
and application versions must be updated together. No ordinary request depends
|
||||
on the collector, Grafana or VictoriaLogs being available.
|
||||
@@ -46,18 +46,21 @@ if service == "mongodb" {
|
||||
if includes(["github_app_authorization", "installation", "installation_repositories"], record.webhookEvent) { out.webhookEvent = record.webhookEvent }
|
||||
if includes(["github.com", "api.github.com", "codeload.github.com", "raw.githubusercontent.com", "media.githubusercontent.com", "other"], record.upstreamHost) { out.upstreamHost = record.upstreamHost }
|
||||
if includes(["download", "remove", "cache"], record.queue) { out.queue = record.queue }
|
||||
if includes(["unknown", "api", "streamer", "timeout", "upstream"], record.cancellationOrigin) { out.cancellationOrigin = record.cancellationOrigin }
|
||||
if includes(["response_close", "request_aborted", "upstream_error", "timeout", "local_abort"], record.cancellationEvidence) { out.cancellationEvidence = record.cancellationEvidence }
|
||||
if includes(["public", "internal"], record.endpointKind) { out.endpointKind = record.endpointKind }
|
||||
if includes(["completed", "interrupted"], record.outcome) { out.outcome = record.outcome }
|
||||
}
|
||||
for_each(["requestId", "jobId", "deliveryId"]) -> |_index, key| {
|
||||
for_each(["requestId", "jobId", "deliveryId", "hopId", "parentHopId"]) -> |_index, key| {
|
||||
value = get(record, [key]) ?? null
|
||||
if is_string(value) && match(string!(value), r'^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$') { out = set!(out, [key], value) }
|
||||
}
|
||||
for_each(["status", "httpStatus", "ms", "firstByteMs", "responseMs", "bytes", "bytesPerSecond", "attempt", "rss", "memoryLimitBytes", "memoryRatio", "uptime", "activeRequests", "waiting", "active", "delayed", "failed", "oldestWaitingMs", "droppedLogs", "loopP95Ms", "upstreamStatus", "stage_repository_ms", "stage_credential_resolution_ms", "stage_upstream_ms", "stage_cache_lookup_ms", "stage_cache_fill_ms", "stage_worker_wait_ms", "stage_anonymize_ms", "stage_streamer_headers_ms"]) -> |_index, key| {
|
||||
for_each(["stage_transfer_ms", "stage_transfer_count", "stage_transfer_open", "stage_database_ms", "stage_database_count", "stage_database_open", "expectedBytes", "measuredMs", "unattributedMs", "overlapMs", "stage_repository_count", "stage_repository_open", "stage_credential_resolution_count", "stage_credential_resolution_open", "stage_upstream_count", "stage_upstream_open", "stage_cache_lookup_count", "stage_cache_lookup_open", "stage_cache_fill_count", "stage_cache_fill_open", "stage_worker_wait_count", "stage_worker_wait_open", "stage_anonymize_count", "stage_anonymize_open", "stage_streamer_headers_count", "stage_streamer_headers_open", "status", "httpStatus", "ms", "firstByteMs", "responseMs", "bytes", "bytesPerSecond", "attempt", "rss", "memoryLimitBytes", "memoryRatio", "uptime", "activeRequests", "waiting", "active", "delayed", "failed", "oldestWaitingMs", "droppedLogs", "loopP95Ms", "upstreamStatus", "stage_repository_ms", "stage_credential_resolution_ms", "stage_upstream_ms", "stage_cache_lookup_ms", "stage_cache_fill_ms", "stage_worker_wait_ms", "stage_anonymize_ms", "stage_streamer_headers_ms"]) -> |_index, key| {
|
||||
value = get(record, [key]) ?? null
|
||||
if is_integer(value) || is_float(value) { out = set!(out, [key], value) }
|
||||
}
|
||||
if includes(["GET", "HEAD", "POST", "PUT", "PATCH", "DELETE", "OPTIONS", "OTHER"], record.method) { out.method = record.method }
|
||||
if includes(["/", "other", "/dashboard", "/anonymize", "/healthcheck", "/api/healthcheck", "/api", "/api/download", "/api/stat", "/api/stat/history", "/api/options", "/api/message", "/api/user", "/api/user/quota", "/api/user/dashboard", "/api/repo", "/api/anonymize-preview", "/api/admin/overview", "/api/admin/performance", "/api/admin/errors", "/api/admin/errors/stats", "/api/admin/queues", "/api/admin/queues/metrics", "/github/app/webhook", "/api/repo/:repoId", "/api/repo/:repoId/file/:path", "/api/repo/:repoId/files", "/api/repo/:repoId/files/counts", "/api/repo/:repoId/files/search", "/api/repo/:repoId/zip", "/api/repo/:repoId/options", "/api/repo/:repoId/refresh", "/api/repo/:repoId/coauthors", "/api/repo/:repoId/extend", "/api/repo/:owner/:repo/readme", "/api/repo/:owner/:repo/branches", "/w/:repoId/:path", "/r/:repoId/:path", "/api/pr/:id/:path", "/api/gist/:id/:path", "/assets/:path", "/github/:action", "/api/admin/:action", "/api/user/:action", "/api/repo/:id/file/:path", "/api/repo/:id/files/counts", "/api/repo/:id/files/search", "/api/repo/:id/files", "/api/repo/:id/zip", "/api/repo/:id", "/viewer/:id/:path"], record.route) { out.route = record.route }
|
||||
if includes(["/", "other", "/dashboard", "/anonymize", "/healthcheck", "/api/healthcheck", "/api", "/api/download", "/api/stat", "/api/stat/history", "/api/options", "/api/message", "/api/user", "/api/user/quota", "/api/user/dashboard", "/api/repo", "/api/anonymize-preview", "/api/admin/overview", "/api/admin/performance", "/api/admin/errors", "/api/admin/errors/stats", "/api/admin/queues", "/api/admin/queues/metrics", "/github/app/webhook", "/api/repo/:repoId", "/api/repo/:repoId/file/:path", "/api/repo/:repoId/files", "/api/repo/:repoId/files/counts", "/api/repo/:repoId/files/search", "/api/repo/:repoId/zip", "/api/repo/:repoId/options", "/api/repo/:repoId/refresh", "/api/repo/:repoId/coauthors", "/api/repo/:repoId/extend", "/api/repo/:owner/:repo", "/api/repo/:owner/:repo/tree", "/api/repo/:owner/:repo/file", "/api/repo/:owner/:repo/readme", "/api/repo/:owner/:repo/branches", "/w/:repoId/:path", "/r/:repoId/:path", "/api/pr/:id/:path", "/api/gist/:id/:path", "/assets/:path", "/github/:action", "/api/admin/:action", "/api/user/:action", "/api/repo/:id/file/:path", "/api/repo/:id/files/counts", "/api/repo/:id/files/search", "/api/repo/:id/files", "/api/repo/:id/zip", "/api/repo/:id", "/viewer/:id/:path"], record.route) { out.route = record.route }
|
||||
# VictoriaLogs consumes its message field; keep event independently searchable.
|
||||
# Anonymous application request context at every outcome, never query strings.
|
||||
{
|
||||
@@ -72,7 +75,7 @@ if includes(["/", "other", "/dashboard", "/anonymize", "/healthcheck", "/api/hea
|
||||
}
|
||||
}
|
||||
# BEGIN GENERATED DIAGNOSTICS
|
||||
source_functions = {"src/cli/index.ts":["<anonymous>","generateRandomFileName","main"],"build/cli/index.js":["<anonymous>","generateRandomFileName","main"],"src/config.ts":["<anonymous>"],"build/config.js":["<anonymous>"],"src/core/AnonymizedFile.ts":["<anonymous>","anonymizedContent","assertContentFile","cancel","content","defaultMaskCandidateRegex","escapeRegex","extension","forwardHeaders","getFileInfo","handleStreamError","isFileSupported","isImage","originalPath","probe","recoverTruncatedFile","send","sha","size","streamerErrorToAnonymous","visibleFile"],"build/core/AnonymizedFile.js":["<anonymous>","anonymizedContent","assertContentFile","cancel","content","defaultMaskCandidateRegex","escapeRegex","extension","forwardHeaders","getFileInfo","handleStreamError","isFileSupported","isImage","originalPath","probe","recoverTruncatedFile","send","sha","size","streamerErrorToAnonymous","visibleFile"],"src/core/AnonymousError.ts":["<anonymous>","detail","toString","url"],"build/core/AnonymousError.js":["<anonymous>","detail","toString","url"],"src/core/Conference.ts":["<anonymous>","expire","isExpired","remove","repositories","toJSON","updateStatus"],"build/core/Conference.js":["<anonymous>","expire","isExpired","remove","repositories","toJSON","updateStatus"],"src/core/Gist.ts":["<anonymous>","anonymize","check","conference","content","countView","download","downloadContent","downloadWithAccess","expire","getAccess","getToken","markExpired","remove","resetSate","toJSON","updateIfNeeded","updateStatus"],"build/core/Gist.js":["<anonymous>","anonymize","check","conference","content","countView","download","downloadContent","downloadWithAccess","expire","getAccess","getToken","markExpired","remove","resetSate","toJSON","updateIfNeeded","updateStatus"],"src/core/GitHubUtils.ts":["<anonymous>","abortableGateLookup","cancelled","checkToken","ensureRedisGateClient","getRedisGateResetAt","getToken","getTokenGateResetAt","octokit","rateLimitDetail","resolveRepositoryToken","run","setRedisGate","setRedisGateFromWorker","setTokenGate","waitForTokenGate"],"build/core/GitHubUtils.js":["<anonymous>","abortableGateLookup","cancelled","checkToken","ensureRedisGateClient","getRedisGateResetAt","getToken","getTokenGateResetAt","octokit","rateLimitDetail","resolveRepositoryToken","run","setRedisGate","setRedisGateFromWorker","setTokenGate","waitForTokenGate"],"src/core/PullRequest.ts":["<anonymous>","anonymize","check","conference","content","countView","download","downloadContent","expire","getToken","markExpired","remove","resetSate","toJSON","updateIfNeeded","updateStatus"],"build/core/PullRequest.js":["<anonymous>","anonymize","check","conference","content","countView","download","downloadContent","expire","getToken","markExpired","remove","resetSate","toJSON","updateIfNeeded","updateStatus"],"src/core/Repository.ts":["<anonymous>","anonymize","anonymizeTreeRecursive","anonymizedFiles","assertNotArchived","buildPathIndex","check","checkRefreshWrite","cleanupRetiredFileTrees","cleanupStagedFileTree","clearRows","completeRecoveredFileMetadata","computeSize","conference","countView","current","ensureFileTree","expire","fileSnapshotFilter","files","findAnonymizedPath","generateAnonymizeTransformer","getToken","hasEmptyTree","indexPaths","insertBatch","invalidateFileMetadata","isReady","markCachePresent","markExpired","pathKey","readFiles","refresh","refreshFilter","refreshIfNeeded","remove","removeCache","resetSate","searchFiles","snapshotValue","submoduleMounts","toJSON","updateIfNeeded","updateProgressMessage","updateRetiringPathIndex","updateStatus","zip"],"build/core/Repository.js":["<anonymous>","anonymize","anonymizeTreeRecursive","anonymizedFiles","assertNotArchived","buildPathIndex","check","checkRefreshWrite","cleanupRetiredFileTrees","cleanupStagedFileTree","clearRows","completeRecoveredFileMetadata","computeSize","conference","countView","current","ensureFileTree","expire","fileSnapshotFilter","files","findAnonymizedPath","generateAnonymizeTransformer","getToken","hasEmptyTree","indexPaths","insertBatch","invalidateFileMetadata","isReady","markCachePresent","markExpired","pathKey","readFiles","refresh","refreshFilter","refreshIfNeeded","remove","removeCache","resetSate","searchFiles","snapshotValue","submoduleMounts","toJSON","updateIfNeeded","updateProgressMessage","updateRetiringPathIndex","updateStatus","zip"],"src/core/User.ts":["<anonymous>","getAccessToken","getGists","getGitHubRepositories","getPullRequests","getRepositories","markListExpired","quotaRepositories","repositoryMembership","toJSON"],"build/core/User.js":["<anonymous>","getAccessToken","getGists","getGitHubRepositories","getPullRequests","getRepositories","markListExpired","quotaRepositories","repositoryMembership","toJSON"],"src/core/anonymization-pool.ts":["<anonymous>","anonymizeOnWorker","cancel","exited","failed","finish","getAnonymizationPoolStats","message","pump","release","releaseSpool","reserveSpool"],"build/core/anonymization-pool.js":["Line truncated
|
||||
source_functions = {"src/cli/index.ts":["<anonymous>","generateRandomFileName","main"],"build/cli/index.js":["<anonymous>","generateRandomFileName","main"],"src/config.ts":["<anonymous>"],"build/config.js":["<anonymous>"],"src/core/AnonymizedFile.ts":["<anonymous>","anonymizedContent","assertContentFile","cancel","content","defaultMaskCandidateRegex","escapeRegex","extension","forwardHeaders","getFileInfo","handleStreamError","isFileSupported","isImage","originalPath","probe","recoverTruncatedFile","send","sha","size","streamerErrorToAnonymous","visibleFile"],"build/core/AnonymizedFile.js":["<anonymous>","anonymizedContent","assertContentFile","cancel","content","defaultMaskCandidateRegex","escapeRegex","extension","forwardHeaders","getFileInfo","handleStreamError","isFileSupported","isImage","originalPath","probe","recoverTruncatedFile","send","sha","size","streamerErrorToAnonymous","visibleFile"],"src/core/AnonymousError.ts":["<anonymous>","detail","toString","url"],"build/core/AnonymousError.js":["<anonymous>","detail","toString","url"],"src/core/Conference.ts":["<anonymous>","expire","isExpired","remove","repositories","toJSON","updateStatus"],"build/core/Conference.js":["<anonymous>","expire","isExpired","remove","repositories","toJSON","updateStatus"],"src/core/Gist.ts":["<anonymous>","anonymize","check","conference","content","countView","download","downloadContent","downloadWithAccess","expire","getAccess","getToken","markExpired","remove","resetSate","toJSON","updateIfNeeded","updateStatus"],"build/core/Gist.js":["<anonymous>","anonymize","check","conference","content","countView","download","downloadContent","downloadWithAccess","expire","getAccess","getToken","markExpired","remove","resetSate","toJSON","updateIfNeeded","updateStatus"],"src/core/GitHubUtils.ts":["<anonymous>","abortableGateLookup","cancelled","checkToken","ensureRedisGateClient","getRedisGateResetAt","getToken","getTokenGateResetAt","octokit","rateLimitDetail","resolveRepositoryToken","run","setRedisGate","setRedisGateFromWorker","setTokenGate","waitForTokenGate"],"build/core/GitHubUtils.js":["<anonymous>","abortableGateLookup","cancelled","checkToken","ensureRedisGateClient","getRedisGateResetAt","getToken","getTokenGateResetAt","octokit","rateLimitDetail","resolveRepositoryToken","run","setRedisGate","setRedisGateFromWorker","setTokenGate","waitForTokenGate"],"src/core/PullRequest.ts":["<anonymous>","anonymize","check","conference","content","countView","download","downloadContent","expire","getToken","markExpired","remove","resetSate","toJSON","updateIfNeeded","updateStatus"],"build/core/PullRequest.js":["<anonymous>","anonymize","check","conference","content","countView","download","downloadContent","expire","getToken","markExpired","remove","resetSate","toJSON","updateIfNeeded","updateStatus"],"src/core/Repository.ts":["<anonymous>","anonymize","anonymizeTreeRecursive","anonymizedFiles","assertNotArchived","buildPathIndex","check","checkRefreshWrite","cleanupRetiredFileTrees","cleanupStagedFileTree","clearRows","completeRecoveredFileMetadata","computeSize","conference","countView","current","ensureFileTree","expire","fileSnapshotFilter","files","findAnonymizedPath","generateAnonymizeTransformer","getToken","hasEmptyTree","indexPaths","insertBatch","invalidateFileMetadata","isReady","markCachePresent","markExpired","pathKey","readFiles","refresh","refreshFilter","refreshIfNeeded","remove","removeCache","resetSate","searchFiles","snapshotValue","submoduleMounts","toJSON","updateIfNeeded","updateProgressMessage","updateRetiringPathIndex","updateStatus","zip"],"build/core/Repository.js":["<anonymous>","anonymize","anonymizeTreeRecursive","anonymizedFiles","assertNotArchived","buildPathIndex","check","checkRefreshWrite","cleanupRetiredFileTrees","cleanupStagedFileTree","clearRows","completeRecoveredFileMetadata","computeSize","conference","countView","current","ensureFileTree","expire","fileSnapshotFilter","files","findAnonymizedPath","generateAnonymizeTransformer","getToken","hasEmptyTree","indexPaths","insertBatch","invalidateFileMetadata","isReady","markCachePresent","markExpired","pathKey","readFiles","refresh","refreshFilter","refreshIfNeeded","remove","removeCache","resetSate","searchFiles","snapshotValue","submoduleMounts","toJSON","updateIfNeeded","updateProgressMessage","updateRetiringPathIndex","updateStatus","zip"],"src/core/User.ts":["<anonymous>","getAccessToken","getGists","getGitHubRepositories","getPullRequests","getRepositories","markListExpired","quotaRepositories","repositoryMembership","toJSON"],"build/core/User.js":["<anonymous>","getAccessToken","getGists","getGitHubRepositories","getPullRequests","getRepositories","markListExpired","quotaRepositories","repositoryMembership","toJSON"],"src/core/anonymization-pool.ts":["<anonymous>","anonymizeOnWorker","cancel","exited","failed","finish","getAnonymizationPoolStats","message","pump","release","releaseSpool","reserveSpool"],"build/core/anonymization-pool.js":["Line truncated
|
||||
for_each(["errorType", "causeType"]) -> |_index, key| {
|
||||
value = get(record, [key]) ?? null
|
||||
if includes(["Error","TypeError","RangeError","ReferenceError","SyntaxError","URIError","EvalError","AggregateError","AnonymousError","HTTPError","RequestError","MongoServerError","MongoNetworkError","MongoServerSelectionError","ValidationError","TimeoutError","AbortError","UnknownError"], value) { out = set!(out, [key], value) }
|
||||
|
||||
Reference in new issue
Block a user