feat: retain bounded failed-request reproduction context

This commit is contained in:
Thomas Durieux committed 2026-11-14 21:44:48 +00:00
1 parent 3a1d013b11
commit 5dbf3e7829
22 files changed
+198 -67

No files matched your search

+7 -2
View File
@@ -326,8 +326,9 @@ The fixture uses only synthetic counters and removes its container on exit.
New application errors retain the normalized route template, HTTP method/status,
a finite error type/code, and up to eight recognized application frames (1 KiB maximum per stack) for the
exception and the deepest inspected cause (at most three levels, cycles stopped). Raw messages,
URLs, query strings, repository/file parameters, external frames and absolute
filesystem roots are not stored. Unknown error types use `UnknownError`; unknown
Upstream URLs, query strings, external frames and absolute filesystem roots are
not stored. Repository/file parameters are excluded except for the explicitly
scoped failed-request reproduction context below. Unknown error types use `UnknownError`; unknown
function names become `<anonymous>`. Source locations are build-relative for
compiled code and source-relative in development; line numbers refer to that
release, not an inferred source map.
@@ -338,3 +339,7 @@ validation. Regenerate it when adding diagnostic code locations. The admin API
revalidates every field for both Redis and central logs. Deploy matching app and
collector versions to preserve new fields; older collectors deliberately drop
unknown fields. Existing discarded exception details cannot be recovered.
### Admin reproduction context
Failed application file requests now retain their public anonymous repository ID and encoded request pathname, up to 2 KiB, under the existing admin log access controls. This is an explicit exception to template-only diagnostics: repository-relative filenames are retained for reproduction. Query strings, headers, bodies and upstream owner/repository URLs are still excluded. Filenames are operator-visible data under this policy; no generic filter can guarantee that a user has not put sensitive text in a filename. Successful requests and metrics keep only the normalized route. Only `/api/repo/:repoId/file/:path` is supported; other endpoints and unsafe, oversized, malformed or traversal paths omit reproduction context. Historical records cannot be backfilled. The UI copies the exact captured pathname without a host or query.
+13 -3
View File
@@ -59,8 +59,18 @@ for_each(["status", "httpStatus", "ms", "firstByteMs", "responseMs", "bytes", "b
if includes(["GET", "HEAD", "POST", "PUT", "PATCH", "DELETE", "OPTIONS", "OTHER"], record.method) { out.method = record.method }
if includes(["/", "other", "/dashboard", "/anonymize", "/healthcheck", "/api/healthcheck", "/api", "/api/download", "/api/stat", "/api/stat/history", "/api/options", "/api/message", "/api/user", "/api/user/quota", "/api/user/dashboard", "/api/repo", "/api/anonymize-preview", "/api/admin/overview", "/api/admin/performance", "/api/admin/errors", "/api/admin/errors/stats", "/api/admin/queues", "/api/admin/queues/metrics", "/github/app/webhook", "/api/repo/:repoId", "/api/repo/:repoId/file/:path", "/api/repo/:repoId/files", "/api/repo/:repoId/files/counts", "/api/repo/:repoId/files/search", "/api/repo/:repoId/zip", "/api/repo/:repoId/options", "/api/repo/:repoId/refresh", "/api/repo/:repoId/coauthors", "/api/repo/:repoId/extend", "/api/repo/:owner/:repo/readme", "/api/repo/:owner/:repo/branches", "/w/:repoId/:path", "/r/:repoId/:path", "/api/pr/:id/:path", "/api/gist/:id/:path", "/assets/:path", "/github/:action", "/api/admin/:action", "/api/user/:action", "/api/repo/:id/file/:path", "/api/repo/:id/files/counts", "/api/repo/:id/files/search", "/api/repo/:id/files", "/api/repo/:id/zip", "/api/repo/:id", "/viewer/:id/:path"], record.route) { out.route = record.route }
# VictoriaLogs consumes its message field; keep event independently searchable.
# Explicit reproduction context for failed application file requests, never query strings.
if includes(["error", "fatal"], out.severity) || to_int(out.status) >= 400 || to_int(out.httpStatus) >= 400 {
path = to_string(record.requestPath) ?? ""
parsed, path_error = parse_regex(path, r'^/api/repo/(?P<repo>[A-Za-z0-9_-]{3,200})/file/(?P<file>[A-Za-z0-9._~!$&\'()+,;=@%/-]+)$')
decoded = decode_percent(to_string(parsed.file))
if path_error == null && length(path) <= 2048 && !match(path, r'%(?:[^a-fA-F0-9]|[a-fA-F0-9][^a-fA-F0-9]|[a-fA-F0-9]?$)') && !match(decoded, r'[\x00-\x1f\x7f-\x9f\x{2028}-\x{202e}\x{2066}-\x{2069}\x{fffd}\\:?#%]|^/|(^|/)\.\.?(/|$)') {
out.requestPath = path
out.repoId = parsed.repo
}
}
# BEGIN GENERATED DIAGNOSTICS
source_functions = {"src/cli/index.ts":["<anonymous>","generateRandomFileName","main"],"build/cli/index.js":["<anonymous>","generateRandomFileName","main"],"src/config.ts":["<anonymous>"],"build/config.js":["<anonymous>"],"src/core/AnonymizedFile.ts":["<anonymous>","anonymizedContent","assertContentFile","cancel","content","defaultMaskCandidateRegex","escapeRegex","extension","forwardHeaders","getFileInfo","handleStreamError","isFileSupported","isImage","originalPath","probe","recoverTruncatedFile","send","sha","size","streamerErrorToAnonymous","visibleFile"],"build/core/AnonymizedFile.js":["<anonymous>","anonymizedContent","assertContentFile","cancel","content","defaultMaskCandidateRegex","escapeRegex","extension","forwardHeaders","getFileInfo","handleStreamError","isFileSupported","isImage","originalPath","probe","recoverTruncatedFile","send","sha","size","streamerErrorToAnonymous","visibleFile"],"src/core/AnonymousError.ts":["<anonymous>","detail","toString","url"],"build/core/AnonymousError.js":["<anonymous>","detail","toString","url"],"src/core/Conference.ts":["<anonymous>","expire","isExpired","remove","repositories","toJSON","updateStatus"],"build/core/Conference.js":["<anonymous>","expire","isExpired","remove","repositories","toJSON","updateStatus"],"src/core/Gist.ts":["<anonymous>","anonymize","check","conference","content","countView","download","downloadContent","downloadWithAccess","expire","getAccess","getToken","markExpired","remove","resetSate","toJSON","updateIfNeeded","updateStatus"],"build/core/Gist.js":["<anonymous>","anonymize","check","conference","content","countView","download","downloadContent","downloadWithAccess","expire","getAccess","getToken","markExpired","remove","resetSate","toJSON","updateIfNeeded","updateStatus"],"src/core/GitHubUtils.ts":["<anonymous>","abortableGateLookup","cancelled","checkToken","ensureRedisGateClient","getRedisGateResetAt","getToken","getTokenGateResetAt","octokit","rateLimitDetail","resolveRepositoryToken","run","setRedisGate","setRedisGateFromWorker","setTokenGate","waitForTokenGate"],"build/core/GitHubUtils.js":["<anonymous>","abortableGateLookup","cancelled","checkToken","ensureRedisGateClient","getRedisGateResetAt","getToken","getTokenGateResetAt","octokit","rateLimitDetail","resolveRepositoryToken","run","setRedisGate","setRedisGateFromWorker","setTokenGate","waitForTokenGate"],"src/core/PullRequest.ts":["<anonymous>","anonymize","check","conference","content","countView","download","downloadContent","expire","getToken","markExpired","remove","resetSate","toJSON","updateIfNeeded","updateStatus"],"build/core/PullRequest.js":["<anonymous>","anonymize","check","conference","content","countView","download","downloadContent","expire","getToken","markExpired","remove","resetSate","toJSON","updateIfNeeded","updateStatus"],"src/core/Repository.ts":["<anonymous>","anonymize","anonymizeTreeRecursive","anonymizedFiles","assertNotArchived","buildPathIndex","check","checkRefreshWrite","cleanupRetiredFileTrees","cleanupStagedFileTree","clearRows","completeRecoveredFileMetadata","computeSize","conference","countView","current","ensureFileTree","expire","fileSnapshotFilter","files","findAnonymizedPath","generateAnonymizeTransformer","getToken","hasEmptyTree","indexPaths","insertBatch","invalidateFileMetadata","isReady","markCachePresent","markExpired","pathKey","readFiles","refresh","refreshFilter","refreshIfNeeded","remove","removeCache","resetSate","searchFiles","snapshotValue","submoduleMounts","toJSON","updateIfNeeded","updateProgressMessage","updateRetiringPathIndex","updateStatus","zip"],"build/core/Repository.js":["<anonymous>","anonymize","anonymizeTreeRecursive","anonymizedFiles","assertNotArchived","buildPathIndex","check","checkRefreshWrite","cleanupRetiredFileTrees","cleanupStagedFileTree","clearRows","completeRecoveredFileMetadata","computeSize","conference","countView","current","ensureFileTree","expire","fileSnapshotFilter","files","findAnonymizedPath","generateAnonymizeTransformer","getToken","hasEmptyTree","indexPaths","insertBatch","invalidateFileMetadata","isReady","markCachePresent","markExpired","pathKey","readFiles","refresh","refreshFilter","refreshIfNeeded","remove","removeCache","resetSate","searchFiles","snapshotValue","submoduleMounts","toJSON","updateIfNeeded","updateProgressMessage","updateRetiringPathIndex","updateStatus","zip"],"src/core/User.ts":["<anonymous>","getAccessToken","getGists","getGitHubRepositories","getPullRequests","getRepositories","markListExpired","quotaRepositories","repositoryMembership","toJSON"],"build/core/User.js":["<anonymous>","getAccessToken","getGists","getGitHubRepositories","getPullRequests","getRepositories","markListExpired","quotaRepositories","repositoryMembership","toJSON"],"src/core/anonymization-pool.ts":["<anonymous>","anonymizeOnWorker","cancel","exited","failed","finish","getAnonymizationPoolStats","message","pump","release","releaseSpool","reserveSpool"],"build/core/anonymization-pool.js":["Line truncated
source_functions = {"src/cli/index.ts":["<anonymous>","generateRandomFileName","main"],"build/cli/index.js":["<anonymous>","generateRandomFileName","main"],"src/config.ts":["<anonymous>"],"build/config.js":["<anonymous>"],"src/core/AnonymizedFile.ts":["<anonymous>","anonymizedContent","assertContentFile","cancel","content","defaultMaskCandidateRegex","escapeRegex","extension","forwardHeaders","getFileInfo","handleStreamError","isFileSupported","isImage","originalPath","probe","recoverTruncatedFile","send","sha","size","streamerErrorToAnonymous","visibleFile"],"build/core/AnonymizedFile.js":["<anonymous>","anonymizedContent","assertContentFile","cancel","content","defaultMaskCandidateRegex","escapeRegex","extension","forwardHeaders","getFileInfo","handleStreamError","isFileSupported","isImage","originalPath","probe","recoverTruncatedFile","send","sha","size","streamerErrorToAnonymous","visibleFile"],"src/core/AnonymousError.ts":["<anonymous>","detail","toString","url"],"build/core/AnonymousError.js":["<anonymous>","detail","toString","url"],"src/core/Conference.ts":["<anonymous>","expire","isExpired","remove","repositories","toJSON","updateStatus"],"build/core/Conference.js":["<anonymous>","expire","isExpired","remove","repositories","toJSON","updateStatus"],"src/core/Gist.ts":["<anonymous>","anonymize","check","conference","content","countView","download","downloadContent","downloadWithAccess","expire","getAccess","getToken","markExpired","remove","resetSate","toJSON","updateIfNeeded","updateStatus"],"build/core/Gist.js":["<anonymous>","anonymize","check","conference","content","countView","download","downloadContent","downloadWithAccess","expire","getAccess","getToken","markExpired","remove","resetSate","toJSON","updateIfNeeded","updateStatus"],"src/core/GitHubUtils.ts":["<anonymous>","abortableGateLookup","cancelled","checkToken","ensureRedisGateClient","getRedisGateResetAt","getToken","getTokenGateResetAt","octokit","rateLimitDetail","resolveRepositoryToken","run","setRedisGate","setRedisGateFromWorker","setTokenGate","waitForTokenGate"],"build/core/GitHubUtils.js":["<anonymous>","abortableGateLookup","cancelled","checkToken","ensureRedisGateClient","getRedisGateResetAt","getToken","getTokenGateResetAt","octokit","rateLimitDetail","resolveRepositoryToken","run","setRedisGate","setRedisGateFromWorker","setTokenGate","waitForTokenGate"],"src/core/PullRequest.ts":["<anonymous>","anonymize","check","conference","content","countView","download","downloadContent","expire","getToken","markExpired","remove","resetSate","toJSON","updateIfNeeded","updateStatus"],"build/core/PullRequest.js":["<anonymous>","anonymize","check","conference","content","countView","download","downloadContent","expire","getToken","markExpired","remove","resetSate","toJSON","updateIfNeeded","updateStatus"],"src/core/Repository.ts":["<anonymous>","anonymize","anonymizeTreeRecursive","anonymizedFiles","assertNotArchived","buildPathIndex","check","checkRefreshWrite","cleanupRetiredFileTrees","cleanupStagedFileTree","clearRows","completeRecoveredFileMetadata","computeSize","conference","countView","current","ensureFileTree","expire","fileSnapshotFilter","files","findAnonymizedPath","generateAnonymizeTransformer","getToken","hasEmptyTree","indexPaths","insertBatch","invalidateFileMetadata","isReady","markCachePresent","markExpired","pathKey","readFiles","refresh","refreshFilter","refreshIfNeeded","remove","removeCache","resetSate","searchFiles","snapshotValue","submoduleMounts","toJSON","updateIfNeeded","updateProgressMessage","updateRetiringPathIndex","updateStatus","zip"],"build/core/Repository.js":["<anonymous>","anonymize","anonymizeTreeRecursive","anonymizedFiles","assertNotArchived","buildPathIndex","check","checkRefreshWrite","cleanupRetiredFileTrees","cleanupStagedFileTree","clearRows","completeRecoveredFileMetadata","computeSize","conference","countView","current","ensureFileTree","expire","fileSnapshotFilter","files","findAnonymizedPath","generateAnonymizeTransformer","getToken","hasEmptyTree","indexPaths","insertBatch","invalidateFileMetadata","isReady","markCachePresent","markExpired","pathKey","readFiles","refresh","refreshFilter","refreshIfNeeded","remove","removeCache","resetSate","searchFiles","snapshotValue","submoduleMounts","toJSON","updateIfNeeded","updateProgressMessage","updateRetiringPathIndex","updateStatus","zip"],"src/core/User.ts":["<anonymous>","getAccessToken","getGists","getGitHubRepositories","getPullRequests","getRepositories","markListExpired","quotaRepositories","repositoryMembership","toJSON"],"build/core/User.js":["<anonymous>","getAccessToken","getGists","getGitHubRepositories","getPullRequests","getRepositories","markListExpired","quotaRepositories","repositoryMembership","toJSON"],"src/core/anonymization-pool.ts":["<anonymous>","anonymizeOnWorker","cancel","exited","failed","finish","getAnonymizationPoolStats","message","pump","release","releaseSpool","reserveSpool"],"build/core/anonymization-pool.js":["Line truncated
for_each(["errorType", "causeType"]) -> |_index, key| {
value = get(record, [key]) ?? null
if includes(["Error","TypeError","RangeError","ReferenceError","SyntaxError","URIError","EvalError","AggregateError","AnonymousError","HTTPError","RequestError","MongoServerError","MongoNetworkError","MongoServerSelectionError","ValidationError","TimeoutError","AbortError","UnknownError"], value) { out = set!(out, [key], value) }
@@ -70,8 +80,8 @@ for_each(["errorStack", "causeStack"]) -> |_index, key| {
if is_string(value) && length(string!(value)) <= 1024 {
frames = []
for_each(split(string!(value), "\n", limit: 9)) -> |index, frame| {
parts, err = parse_regex(frame, r'^at (?P<function>[A-Za-z_$][A-Za-z0-9_$]*|<anonymous>) \((?P<file>(?:src|build)/[A-Za-z0-9_/-]+\.(?:ts|js)):(?P<line>[1-9][0-9]{0,5}):(?P<column>[1-9][0-9]{0,4})\)$')
allowed = get(source_functions, [to_string(parts.file)]) ?? []
parts, err = parse_regex(frame, r'^at (?P<function>[A-Za-z_$][A-Za-z0-9_$]*|<anonymous>) \((?P<file>(?:src|build)/[A-Za-z0-9_./-]+\.(?:ts|js)):(?P<line>[1-9][0-9]{0,5}):(?P<column>[1-9][0-9]{0,4})\)$')
allowed = array(get(source_functions, [to_string(parts.file)]) ?? null) ?? []
if index < 8 && err == null && includes(array!(allowed), parts.function) { frames = push(frames, frame) }
}
if length(frames) > 0 { out = set!(out, [key], join!(frames, "\n")) }