mirror of
https://github.com/tdurieux/anonymous_github.git
synced 2026-11-17 17:44:09 +01:00
Merge pull request #970 from tdurieux/feat/operational-logging-evidence
fix: explain interrupted requests and account for overlapping latency
This commit is contained in:
21 files changed
+467
-38
No files matched your search
@@ -359,3 +359,56 @@ Application requests to the supported anonymous repository endpoints retain thei
|
||||
For the focused restricted-file/rotation regression, run
|
||||
`LOGGING_ACCESS_ONLY=1 python3 scripts/test-logging.py`. This uses synthetic
|
||||
`0640` logs in an isolated Docker project and preserves capability restrictions.
|
||||
|
||||
## Request evidence contract
|
||||
|
||||
`outcome=completed` means Node emitted response `finish`. It does not prove that
|
||||
an end user received the body. A 200 can have `outcome=interrupted`. Count public
|
||||
API outcomes once; correlated streamer hops describe the same operation and must
|
||||
not be added to public failure counts. `endpointKind` distinguishes public API
|
||||
from internal streamer requests. Each middleware invocation has a fresh `hopId`;
|
||||
`parentHopId` links streamer calls to the API hop, including repeated cache probes.
|
||||
Public caller correlation headers are ignored. Internal IDs are diagnostics, not
|
||||
proof of caller identity. Existing job UUIDs and attempt numbers remain separate.
|
||||
|
||||
For interrupted responses, `cancellationOrigin` and `cancellationEvidence` retain
|
||||
the first observed causal boundary. Explicit upstream errors identify the failing
|
||||
upstream hop; ETIMEDOUT identifies a timeout. A request abort or response close
|
||||
alone has origin `unknown`: the peer may be a proxy, API or browser, and a close
|
||||
can also follow server teardown. No client-action claim is inferred from 200,
|
||||
socket close, or lack of error logs. ZIP processing failures with no identified
|
||||
upstream retain unknown origin with local-abort evidence. Historical records lack
|
||||
this evidence and cannot be reclassified reliably.
|
||||
|
||||
`bytes` counts encoded body bytes passed to Node response writes, including
|
||||
buffered writes; it is not an acknowledgement from the peer. `expectedBytes` is
|
||||
only the response Content-Length, absent when unknown. Source file size is not
|
||||
comparable after anonymization/compression. HEAD, 204 and 304 have zero expected
|
||||
body bytes. `firstByteMs` is the existing header-write boundary, not a measured
|
||||
network first byte. `responseMs` and `stage_transfer_ms` cover header write through
|
||||
finish/close, including backpressure and waits for content.
|
||||
|
||||
Named stage durations are inclusive sums of instrumented spans, clipped at the
|
||||
response terminal event. Repeated and parallel spans add to their stage totals.
|
||||
`stage_<name>_count` counts instrumented calls, not hidden library retries;
|
||||
`stage_<name>_open` counts spans still running at termination. Uninstrumented
|
||||
library retries/backoff stay inside their enclosing span and cannot be separated.
|
||||
`measuredMs` is the union of all spans, `overlapMs` is the excess of their summed
|
||||
durations over that union, and `unattributedMs` is request time outside any span.
|
||||
Do not add transfer, credential, database, cache, upstream or anonymization totals
|
||||
to estimate request duration. Millisecond rounding of legacy request totals may
|
||||
cause sub-millisecond differences. Storage is fixed per stage, not per call.
|
||||
|
||||
`database` measures Mongoose Query/Aggregate execution after database setup,
|
||||
including pool/buffering wait and conversion. It excludes document saves, bulk
|
||||
writes and direct driver operations. `repository` remains the application
|
||||
repository lookup, which can enclose database time. Credential resolution can
|
||||
enclose database and GitHub work. GitHub `upstream` spans cover the existing
|
||||
Octokit request wrapper or explicitly instrumented stream intervals. Cache lookup,
|
||||
fill, worker admission, anonymization and streamer-header spans retain their
|
||||
existing call boundaries. Missing stages mean unavailable instrumentation, never
|
||||
zero cost. Unattributed time is visible; it is not evidence of a stall.
|
||||
|
||||
The admin API and Redis projection preserve these allowlisted fields. Collector
|
||||
and application versions must be updated together. No ordinary request depends
|
||||
on the collector, Grafana or VictoriaLogs being available.
|
||||
@@ -46,18 +46,21 @@ if service == "mongodb" {
|
||||
if includes(["github_app_authorization", "installation", "installation_repositories"], record.webhookEvent) { out.webhookEvent = record.webhookEvent }
|
||||
if includes(["github.com", "api.github.com", "codeload.github.com", "raw.githubusercontent.com", "media.githubusercontent.com", "other"], record.upstreamHost) { out.upstreamHost = record.upstreamHost }
|
||||
if includes(["download", "remove", "cache"], record.queue) { out.queue = record.queue }
|
||||
if includes(["unknown", "api", "streamer", "timeout", "upstream"], record.cancellationOrigin) { out.cancellationOrigin = record.cancellationOrigin }
|
||||
if includes(["response_close", "request_aborted", "upstream_error", "timeout", "local_abort"], record.cancellationEvidence) { out.cancellationEvidence = record.cancellationEvidence }
|
||||
if includes(["public", "internal"], record.endpointKind) { out.endpointKind = record.endpointKind }
|
||||
if includes(["completed", "interrupted"], record.outcome) { out.outcome = record.outcome }
|
||||
}
|
||||
for_each(["requestId", "jobId", "deliveryId"]) -> |_index, key| {
|
||||
for_each(["requestId", "jobId", "deliveryId", "hopId", "parentHopId"]) -> |_index, key| {
|
||||
value = get(record, [key]) ?? null
|
||||
if is_string(value) && match(string!(value), r'^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$') { out = set!(out, [key], value) }
|
||||
}
|
||||
for_each(["status", "httpStatus", "ms", "firstByteMs", "responseMs", "bytes", "bytesPerSecond", "attempt", "rss", "memoryLimitBytes", "memoryRatio", "uptime", "activeRequests", "waiting", "active", "delayed", "failed", "oldestWaitingMs", "droppedLogs", "loopP95Ms", "upstreamStatus", "stage_repository_ms", "stage_credential_resolution_ms", "stage_upstream_ms", "stage_cache_lookup_ms", "stage_cache_fill_ms", "stage_worker_wait_ms", "stage_anonymize_ms", "stage_streamer_headers_ms"]) -> |_index, key| {
|
||||
for_each(["stage_transfer_ms", "stage_transfer_count", "stage_transfer_open", "stage_database_ms", "stage_database_count", "stage_database_open", "expectedBytes", "measuredMs", "unattributedMs", "overlapMs", "stage_repository_count", "stage_repository_open", "stage_credential_resolution_count", "stage_credential_resolution_open", "stage_upstream_count", "stage_upstream_open", "stage_cache_lookup_count", "stage_cache_lookup_open", "stage_cache_fill_count", "stage_cache_fill_open", "stage_worker_wait_count", "stage_worker_wait_open", "stage_anonymize_count", "stage_anonymize_open", "stage_streamer_headers_count", "stage_streamer_headers_open", "status", "httpStatus", "ms", "firstByteMs", "responseMs", "bytes", "bytesPerSecond", "attempt", "rss", "memoryLimitBytes", "memoryRatio", "uptime", "activeRequests", "waiting", "active", "delayed", "failed", "oldestWaitingMs", "droppedLogs", "loopP95Ms", "upstreamStatus", "stage_repository_ms", "stage_credential_resolution_ms", "stage_upstream_ms", "stage_cache_lookup_ms", "stage_cache_fill_ms", "stage_worker_wait_ms", "stage_anonymize_ms", "stage_streamer_headers_ms"]) -> |_index, key| {
|
||||
value = get(record, [key]) ?? null
|
||||
if is_integer(value) || is_float(value) { out = set!(out, [key], value) }
|
||||
}
|
||||
if includes(["GET", "HEAD", "POST", "PUT", "PATCH", "DELETE", "OPTIONS", "OTHER"], record.method) { out.method = record.method }
|
||||
if includes(["/", "other", "/dashboard", "/anonymize", "/healthcheck", "/api/healthcheck", "/api", "/api/download", "/api/stat", "/api/stat/history", "/api/options", "/api/message", "/api/user", "/api/user/quota", "/api/user/dashboard", "/api/repo", "/api/anonymize-preview", "/api/admin/overview", "/api/admin/performance", "/api/admin/errors", "/api/admin/errors/stats", "/api/admin/queues", "/api/admin/queues/metrics", "/github/app/webhook", "/api/repo/:repoId", "/api/repo/:repoId/file/:path", "/api/repo/:repoId/files", "/api/repo/:repoId/files/counts", "/api/repo/:repoId/files/search", "/api/repo/:repoId/zip", "/api/repo/:repoId/options", "/api/repo/:repoId/refresh", "/api/repo/:repoId/coauthors", "/api/repo/:repoId/extend", "/api/repo/:owner/:repo/readme", "/api/repo/:owner/:repo/branches", "/w/:repoId/:path", "/r/:repoId/:path", "/api/pr/:id/:path", "/api/gist/:id/:path", "/assets/:path", "/github/:action", "/api/admin/:action", "/api/user/:action", "/api/repo/:id/file/:path", "/api/repo/:id/files/counts", "/api/repo/:id/files/search", "/api/repo/:id/files", "/api/repo/:id/zip", "/api/repo/:id", "/viewer/:id/:path"], record.route) { out.route = record.route }
|
||||
if includes(["/", "other", "/dashboard", "/anonymize", "/healthcheck", "/api/healthcheck", "/api", "/api/download", "/api/stat", "/api/stat/history", "/api/options", "/api/message", "/api/user", "/api/user/quota", "/api/user/dashboard", "/api/repo", "/api/anonymize-preview", "/api/admin/overview", "/api/admin/performance", "/api/admin/errors", "/api/admin/errors/stats", "/api/admin/queues", "/api/admin/queues/metrics", "/github/app/webhook", "/api/repo/:repoId", "/api/repo/:repoId/file/:path", "/api/repo/:repoId/files", "/api/repo/:repoId/files/counts", "/api/repo/:repoId/files/search", "/api/repo/:repoId/zip", "/api/repo/:repoId/options", "/api/repo/:repoId/refresh", "/api/repo/:repoId/coauthors", "/api/repo/:repoId/extend", "/api/repo/:owner/:repo", "/api/repo/:owner/:repo/tree", "/api/repo/:owner/:repo/file", "/api/repo/:owner/:repo/readme", "/api/repo/:owner/:repo/branches", "/w/:repoId/:path", "/r/:repoId/:path", "/api/pr/:id/:path", "/api/gist/:id/:path", "/assets/:path", "/github/:action", "/api/admin/:action", "/api/user/:action", "/api/repo/:id/file/:path", "/api/repo/:id/files/counts", "/api/repo/:id/files/search", "/api/repo/:id/files", "/api/repo/:id/zip", "/api/repo/:id", "/viewer/:id/:path"], record.route) { out.route = record.route }
|
||||
# VictoriaLogs consumes its message field; keep event independently searchable.
|
||||
# Anonymous application request context at every outcome, never query strings.
|
||||
{
|
||||
@@ -72,7 +75,7 @@ if includes(["/", "other", "/dashboard", "/anonymize", "/healthcheck", "/api/hea
|
||||
}
|
||||
}
|
||||
# BEGIN GENERATED DIAGNOSTICS
|
||||
source_functions = {"src/cli/index.ts":["<anonymous>","generateRandomFileName","main"],"build/cli/index.js":["<anonymous>","generateRandomFileName","main"],"src/config.ts":["<anonymous>"],"build/config.js":["<anonymous>"],"src/core/AnonymizedFile.ts":["<anonymous>","anonymizedContent","assertContentFile","cancel","content","defaultMaskCandidateRegex","escapeRegex","extension","forwardHeaders","getFileInfo","handleStreamError","isFileSupported","isImage","originalPath","probe","recoverTruncatedFile","send","sha","size","streamerErrorToAnonymous","visibleFile"],"build/core/AnonymizedFile.js":["<anonymous>","anonymizedContent","assertContentFile","cancel","content","defaultMaskCandidateRegex","escapeRegex","extension","forwardHeaders","getFileInfo","handleStreamError","isFileSupported","isImage","originalPath","probe","recoverTruncatedFile","send","sha","size","streamerErrorToAnonymous","visibleFile"],"src/core/AnonymousError.ts":["<anonymous>","detail","toString","url"],"build/core/AnonymousError.js":["<anonymous>","detail","toString","url"],"src/core/Conference.ts":["<anonymous>","expire","isExpired","remove","repositories","toJSON","updateStatus"],"build/core/Conference.js":["<anonymous>","expire","isExpired","remove","repositories","toJSON","updateStatus"],"src/core/Gist.ts":["<anonymous>","anonymize","check","conference","content","countView","download","downloadContent","downloadWithAccess","expire","getAccess","getToken","markExpired","remove","resetSate","toJSON","updateIfNeeded","updateStatus"],"build/core/Gist.js":["<anonymous>","anonymize","check","conference","content","countView","download","downloadContent","downloadWithAccess","expire","getAccess","getToken","markExpired","remove","resetSate","toJSON","updateIfNeeded","updateStatus"],"src/core/GitHubUtils.ts":["<anonymous>","abortableGateLookup","cancelled","checkToken","ensureRedisGateClient","getRedisGateResetAt","getToken","getTokenGateResetAt","octokit","rateLimitDetail","resolveRepositoryToken","run","setRedisGate","setRedisGateFromWorker","setTokenGate","waitForTokenGate"],"build/core/GitHubUtils.js":["<anonymous>","abortableGateLookup","cancelled","checkToken","ensureRedisGateClient","getRedisGateResetAt","getToken","getTokenGateResetAt","octokit","rateLimitDetail","resolveRepositoryToken","run","setRedisGate","setRedisGateFromWorker","setTokenGate","waitForTokenGate"],"src/core/PullRequest.ts":["<anonymous>","anonymize","check","conference","content","countView","download","downloadContent","expire","getToken","markExpired","remove","resetSate","toJSON","updateIfNeeded","updateStatus"],"build/core/PullRequest.js":["<anonymous>","anonymize","check","conference","content","countView","download","downloadContent","expire","getToken","markExpired","remove","resetSate","toJSON","updateIfNeeded","updateStatus"],"src/core/Repository.ts":["<anonymous>","anonymize","anonymizeTreeRecursive","anonymizedFiles","assertNotArchived","buildPathIndex","check","checkRefreshWrite","cleanupRetiredFileTrees","cleanupStagedFileTree","clearRows","completeRecoveredFileMetadata","computeSize","conference","countView","current","ensureFileTree","expire","fileSnapshotFilter","files","findAnonymizedPath","generateAnonymizeTransformer","getToken","hasEmptyTree","indexPaths","insertBatch","invalidateFileMetadata","isReady","markCachePresent","markExpired","pathKey","readFiles","refresh","refreshFilter","refreshIfNeeded","remove","removeCache","resetSate","searchFiles","snapshotValue","submoduleMounts","toJSON","updateIfNeeded","updateProgressMessage","updateRetiringPathIndex","updateStatus","zip"],"build/core/Repository.js":["<anonymous>","anonymize","anonymizeTreeRecursive","anonymizedFiles","assertNotArchived","buildPathIndex","check","checkRefreshWrite","cleanupRetiredFileTrees","cleanupStagedFileTree","clearRows","completeRecoveredFileMetadata","computeSize","conference","countView","current","ensureFileTree","expire","fileSnapshotFilter","files","findAnonymizedPath","generateAnonymizeTransformer","getToken","hasEmptyTree","indexPaths","insertBatch","invalidateFileMetadata","isReady","markCachePresent","markExpired","pathKey","readFiles","refresh","refreshFilter","refreshIfNeeded","remove","removeCache","resetSate","searchFiles","snapshotValue","submoduleMounts","toJSON","updateIfNeeded","updateProgressMessage","updateRetiringPathIndex","updateStatus","zip"],"src/core/User.ts":["<anonymous>","getAccessToken","getGists","getGitHubRepositories","getPullRequests","getRepositories","markListExpired","quotaRepositories","repositoryMembership","toJSON"],"build/core/User.js":["<anonymous>","getAccessToken","getGists","getGitHubRepositories","getPullRequests","getRepositories","markListExpired","quotaRepositories","repositoryMembership","toJSON"],"src/core/anonymization-pool.ts":["<anonymous>","anonymizeOnWorker","cancel","exited","failed","finish","getAnonymizationPoolStats","message","pump","release","releaseSpool","reserveSpool"],"build/core/anonymization-pool.js":["Line truncated
|
||||
source_functions = {"src/cli/index.ts":["<anonymous>","generateRandomFileName","main"],"build/cli/index.js":["<anonymous>","generateRandomFileName","main"],"src/config.ts":["<anonymous>"],"build/config.js":["<anonymous>"],"src/core/AnonymizedFile.ts":["<anonymous>","anonymizedContent","assertContentFile","cancel","content","defaultMaskCandidateRegex","escapeRegex","extension","forwardHeaders","getFileInfo","handleStreamError","isFileSupported","isImage","originalPath","probe","recoverTruncatedFile","send","sha","size","streamerErrorToAnonymous","visibleFile"],"build/core/AnonymizedFile.js":["<anonymous>","anonymizedContent","assertContentFile","cancel","content","defaultMaskCandidateRegex","escapeRegex","extension","forwardHeaders","getFileInfo","handleStreamError","isFileSupported","isImage","originalPath","probe","recoverTruncatedFile","send","sha","size","streamerErrorToAnonymous","visibleFile"],"src/core/AnonymousError.ts":["<anonymous>","detail","toString","url"],"build/core/AnonymousError.js":["<anonymous>","detail","toString","url"],"src/core/Conference.ts":["<anonymous>","expire","isExpired","remove","repositories","toJSON","updateStatus"],"build/core/Conference.js":["<anonymous>","expire","isExpired","remove","repositories","toJSON","updateStatus"],"src/core/Gist.ts":["<anonymous>","anonymize","check","conference","content","countView","download","downloadContent","downloadWithAccess","expire","getAccess","getToken","markExpired","remove","resetSate","toJSON","updateIfNeeded","updateStatus"],"build/core/Gist.js":["<anonymous>","anonymize","check","conference","content","countView","download","downloadContent","downloadWithAccess","expire","getAccess","getToken","markExpired","remove","resetSate","toJSON","updateIfNeeded","updateStatus"],"src/core/GitHubUtils.ts":["<anonymous>","abortableGateLookup","cancelled","checkToken","ensureRedisGateClient","getRedisGateResetAt","getToken","getTokenGateResetAt","octokit","rateLimitDetail","resolveRepositoryToken","run","setRedisGate","setRedisGateFromWorker","setTokenGate","waitForTokenGate"],"build/core/GitHubUtils.js":["<anonymous>","abortableGateLookup","cancelled","checkToken","ensureRedisGateClient","getRedisGateResetAt","getToken","getTokenGateResetAt","octokit","rateLimitDetail","resolveRepositoryToken","run","setRedisGate","setRedisGateFromWorker","setTokenGate","waitForTokenGate"],"src/core/PullRequest.ts":["<anonymous>","anonymize","check","conference","content","countView","download","downloadContent","expire","getToken","markExpired","remove","resetSate","toJSON","updateIfNeeded","updateStatus"],"build/core/PullRequest.js":["<anonymous>","anonymize","check","conference","content","countView","download","downloadContent","expire","getToken","markExpired","remove","resetSate","toJSON","updateIfNeeded","updateStatus"],"src/core/Repository.ts":["<anonymous>","anonymize","anonymizeTreeRecursive","anonymizedFiles","assertNotArchived","buildPathIndex","check","checkRefreshWrite","cleanupRetiredFileTrees","cleanupStagedFileTree","clearRows","completeRecoveredFileMetadata","computeSize","conference","countView","current","ensureFileTree","expire","fileSnapshotFilter","files","findAnonymizedPath","generateAnonymizeTransformer","getToken","hasEmptyTree","indexPaths","insertBatch","invalidateFileMetadata","isReady","markCachePresent","markExpired","pathKey","readFiles","refresh","refreshFilter","refreshIfNeeded","remove","removeCache","resetSate","searchFiles","snapshotValue","submoduleMounts","toJSON","updateIfNeeded","updateProgressMessage","updateRetiringPathIndex","updateStatus","zip"],"build/core/Repository.js":["<anonymous>","anonymize","anonymizeTreeRecursive","anonymizedFiles","assertNotArchived","buildPathIndex","check","checkRefreshWrite","cleanupRetiredFileTrees","cleanupStagedFileTree","clearRows","completeRecoveredFileMetadata","computeSize","conference","countView","current","ensureFileTree","expire","fileSnapshotFilter","files","findAnonymizedPath","generateAnonymizeTransformer","getToken","hasEmptyTree","indexPaths","insertBatch","invalidateFileMetadata","isReady","markCachePresent","markExpired","pathKey","readFiles","refresh","refreshFilter","refreshIfNeeded","remove","removeCache","resetSate","searchFiles","snapshotValue","submoduleMounts","toJSON","updateIfNeeded","updateProgressMessage","updateRetiringPathIndex","updateStatus","zip"],"src/core/User.ts":["<anonymous>","getAccessToken","getGists","getGitHubRepositories","getPullRequests","getRepositories","markListExpired","quotaRepositories","repositoryMembership","toJSON"],"build/core/User.js":["<anonymous>","getAccessToken","getGists","getGitHubRepositories","getPullRequests","getRepositories","markListExpired","quotaRepositories","repositoryMembership","toJSON"],"src/core/anonymization-pool.ts":["<anonymous>","anonymizeOnWorker","cancel","exited","failed","finish","getAnonymizationPoolStats","message","pump","release","releaseSpool","reserveSpool"],"build/core/anonymization-pool.js":["Line truncated
|
||||
for_each(["errorType", "causeType"]) -> |_index, key| {
|
||||
value = get(record, [key]) ?? null
|
||||
if includes(["Error","TypeError","RangeError","ReferenceError","SyntaxError","URIError","EvalError","AggregateError","AnonymousError","HTTPError","RequestError","MongoServerError","MongoNetworkError","MongoServerSelectionError","ValidationError","TimeoutError","AbortError","UnknownError"], value) { out = set!(out, [key], value) }
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { recordResponseFailure } from "./request-evidence";
|
||||
import { rejectStreamerRedirects } from "./streamer-response";
|
||||
import { isResponseCancellation } from "./response-cancellation";
|
||||
import { canReadCachedRepository } from "./cached-repository-access";
|
||||
@@ -585,9 +586,9 @@ export default class AnonymizedFile {
|
||||
reject(error);
|
||||
}
|
||||
content
|
||||
.on("error", handleStreamError)
|
||||
.on("error", error => { if (!res.destroyed) recordResponseFailure(res, "unknown", "local_abort"); handleStreamError(error); })
|
||||
.pipe(anonymizer)
|
||||
.on("error", handleStreamError)
|
||||
.on("error", error => { if (!res.destroyed) recordResponseFailure(res, "api", "local_abort"); handleStreamError(error); })
|
||||
.pipe(res)
|
||||
.on("error", handleStreamError)
|
||||
.on("finish", () => {
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { recordResponseFailure } from "./request-evidence";
|
||||
import { createGzip } from "zlib";
|
||||
import { createHash } from "crypto";
|
||||
import { Readable, PassThrough } from "stream";
|
||||
@@ -49,9 +50,14 @@ async function sendCached(content: Readable, res: Response,
|
||||
if (isText && !res.getHeader("Content-Type")) res.type("text/plain");
|
||||
if (isText) res.setHeader("Accept-Ranges", "none");
|
||||
else res.removeHeader("Accept-Ranges");
|
||||
// Install evidence before pipeline's listeners destroy the response.
|
||||
const localFailure = () => { if (!res.destroyed) recordResponseFailure(res, "unknown", "local_abort"); };
|
||||
stream.once("error", localFailure);
|
||||
if (record.encoding === "identity" && !res.req.acceptsEncodings("identity")) {
|
||||
res.setHeader("Content-Encoding", "gzip"); res.removeHeader("Content-Length");
|
||||
await pipeline(stream, createGzip(), res);
|
||||
const gzip = createGzip();
|
||||
gzip.once("error", localFailure);
|
||||
await pipeline(stream, gzip, res);
|
||||
} else await pipeline(stream, res);
|
||||
return true;
|
||||
};
|
||||
|
||||
@@ -0,0 +1,18 @@
|
||||
import mongoose from "mongoose";
|
||||
import { measureStage } from "./request-monitoring";
|
||||
|
||||
let installed = false;
|
||||
/** Measure Mongoose query/aggregate execution, including pool/buffering wait.
|
||||
* No query text, model names, results or parameters enter the logging context.
|
||||
* Direct driver operations, document saves and bulk writes are not covered.
|
||||
*/
|
||||
export function installDatabaseTiming() {
|
||||
if (installed) return;
|
||||
installed = true;
|
||||
for (const prototype of [mongoose.Query.prototype, mongoose.Aggregate.prototype]) {
|
||||
const original = prototype.exec;
|
||||
prototype.exec = function (this: unknown, ...args: unknown[]) {
|
||||
return measureStage("database", () => Reflect.apply(original, this, args));
|
||||
} as typeof original;
|
||||
}
|
||||
}
|
||||
@@ -1,7 +1,7 @@
|
||||
import { AsyncLocalStorage } from "async_hooks";
|
||||
import { randomUUID } from "crypto";
|
||||
|
||||
export interface LogContext { requestId?: string; jobId?: string; attempt?: number; service?: string; route?: string; method?: string; requestPath?: string }
|
||||
export interface LogContext { requestId?: string; hopId?: string; parentHopId?: string; jobId?: string; attempt?: number; service?: string; route?: string; method?: string; requestPath?: string }
|
||||
export const logContext = new AsyncLocalStorage<LogContext>();
|
||||
export const validTraceId = (value: unknown): value is string => typeof value === "string" &&
|
||||
/^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$/.test(value);
|
||||
|
||||
@@ -785,6 +785,7 @@ export const LOG_SOURCE_FUNCTIONS: Record<string, readonly string[]> = {
|
||||
"abort",
|
||||
"assertActive",
|
||||
"decisionKey",
|
||||
"localFailure",
|
||||
"send",
|
||||
"sendCached",
|
||||
"sendSharedContent"
|
||||
@@ -794,6 +795,7 @@ export const LOG_SOURCE_FUNCTIONS: Record<string, readonly string[]> = {
|
||||
"abort",
|
||||
"assertActive",
|
||||
"decisionKey",
|
||||
"localFailure",
|
||||
"send",
|
||||
"sendCached",
|
||||
"sendSharedContent"
|
||||
@@ -858,6 +860,14 @@ export const LOG_SOURCE_FUNCTIONS: Record<string, readonly string[]> = {
|
||||
"replaceCredential",
|
||||
"setCredential"
|
||||
],
|
||||
"src/core/database-timing.ts": [
|
||||
"<anonymous>",
|
||||
"installDatabaseTiming"
|
||||
],
|
||||
"build/core/database-timing.js": [
|
||||
"<anonymous>",
|
||||
"installDatabaseTiming"
|
||||
],
|
||||
"src/core/email-history.ts": [
|
||||
"<anonymous>",
|
||||
"complete",
|
||||
@@ -960,6 +970,14 @@ export const LOG_SOURCE_FUNCTIONS: Record<string, readonly string[]> = {
|
||||
"<anonymous>",
|
||||
"isGitHubRateLimitError"
|
||||
],
|
||||
"src/core/github-repository-picker.ts": [
|
||||
"<anonymous>",
|
||||
"repositoryPickerPage"
|
||||
],
|
||||
"build/core/github-repository-picker.js": [
|
||||
"<anonymous>",
|
||||
"repositoryPickerPage"
|
||||
],
|
||||
"src/core/github-token-context.ts": [
|
||||
"<anonymous>",
|
||||
"githubQuotaKey",
|
||||
@@ -1482,34 +1500,54 @@ export const LOG_SOURCE_FUNCTIONS: Record<string, readonly string[]> = {
|
||||
"build/core/repository-id.js": [
|
||||
"<anonymous>"
|
||||
],
|
||||
"src/core/request-evidence.ts": [
|
||||
"<anonymous>",
|
||||
"expectedResponseBytes",
|
||||
"recordResponseFailure",
|
||||
"recordUpstreamFailure",
|
||||
"responseEvidence"
|
||||
],
|
||||
"build/core/request-evidence.js": [
|
||||
"<anonymous>",
|
||||
"expectedResponseBytes",
|
||||
"recordResponseFailure",
|
||||
"recordUpstreamFailure",
|
||||
"responseEvidence"
|
||||
],
|
||||
"src/core/request-monitoring.ts": [
|
||||
"<anonymous>",
|
||||
"activeRequestCount",
|
||||
"advance",
|
||||
"cacheTraffic",
|
||||
"count",
|
||||
"done",
|
||||
"drain",
|
||||
"finish",
|
||||
"measureStage",
|
||||
"monitorRequests",
|
||||
"observe",
|
||||
"recordCacheHit",
|
||||
"requestHeaders",
|
||||
"requestRoute",
|
||||
"start",
|
||||
"startStage"
|
||||
],
|
||||
"build/core/request-monitoring.js": [
|
||||
"<anonymous>",
|
||||
"activeRequestCount",
|
||||
"advance",
|
||||
"cacheTraffic",
|
||||
"count",
|
||||
"done",
|
||||
"drain",
|
||||
"finish",
|
||||
"measureStage",
|
||||
"monitorRequests",
|
||||
"observe",
|
||||
"recordCacheHit",
|
||||
"requestHeaders",
|
||||
"requestRoute",
|
||||
"start",
|
||||
"startStage"
|
||||
],
|
||||
"src/core/response-cancellation.ts": [
|
||||
@@ -2104,6 +2142,14 @@ export const LOG_SOURCE_FUNCTIONS: Record<string, readonly string[]> = {
|
||||
"<anonymous>",
|
||||
"main"
|
||||
],
|
||||
"src/scripts/check-github-app-key.ts": [
|
||||
"<anonymous>",
|
||||
"checkGitHubAppKey"
|
||||
],
|
||||
"build/scripts/check-github-app-key.js": [
|
||||
"<anonymous>",
|
||||
"checkGitHubAppKey"
|
||||
],
|
||||
"src/scripts/create-performance-indexes.ts": [
|
||||
"<anonymous>",
|
||||
"main"
|
||||
@@ -2860,4 +2906,4 @@ export const LOG_SOURCE_FUNCTIONS: Record<string, readonly string[]> = {
|
||||
]
|
||||
};
|
||||
export const LOG_ERROR_TYPES = ["Error","TypeError","RangeError","ReferenceError","SyntaxError","URIError","EvalError","AggregateError","AnonymousError","HTTPError","RequestError","MongoServerError","MongoNetworkError","MongoServerSelectionError","ValidationError","TimeoutError","AbortError","UnknownError"];
|
||||
export const LOG_ROUTES = ["/","other","/dashboard","/anonymize","/healthcheck","/api/healthcheck","/api","/api/download","/api/stat","/api/stat/history","/api/options","/api/message","/api/user","/api/user/quota","/api/user/dashboard","/api/repo","/api/anonymize-preview","/api/admin/overview","/api/admin/performance","/api/admin/errors","/api/admin/errors/stats","/api/admin/queues","/api/admin/queues/metrics","/github/app/webhook","/api/repo/:repoId","/api/repo/:repoId/file/:path","/api/repo/:repoId/files","/api/repo/:repoId/files/counts","/api/repo/:repoId/files/search","/api/repo/:repoId/zip","/api/repo/:repoId/options","/api/repo/:repoId/refresh","/api/repo/:repoId/coauthors","/api/repo/:repoId/extend","/api/repo/:owner/:repo/readme","/api/repo/:owner/:repo/branches","/w/:repoId/:path","/r/:repoId/:path","/api/pr/:id/:path","/api/gist/:id/:path","/assets/:path","/github/:action","/api/admin/:action","/api/user/:action","/api/repo/:id/file/:path","/api/repo/:id/files/counts","/api/repo/:id/files/search","/api/repo/:id/files","/api/repo/:id/zip","/api/repo/:id","/viewer/:id/:path"];
|
||||
export const LOG_ROUTES = ["/","other","/dashboard","/anonymize","/healthcheck","/api/healthcheck","/api","/api/download","/api/stat","/api/stat/history","/api/options","/api/message","/api/user","/api/user/quota","/api/user/dashboard","/api/repo","/api/anonymize-preview","/api/admin/overview","/api/admin/performance","/api/admin/errors","/api/admin/errors/stats","/api/admin/queues","/api/admin/queues/metrics","/github/app/webhook","/api/repo/:repoId","/api/repo/:repoId/file/:path","/api/repo/:repoId/files","/api/repo/:repoId/files/counts","/api/repo/:repoId/files/search","/api/repo/:repoId/zip","/api/repo/:repoId/options","/api/repo/:repoId/refresh","/api/repo/:repoId/coauthors","/api/repo/:repoId/extend","/api/repo/:owner/:repo","/api/repo/:owner/:repo/tree","/api/repo/:owner/:repo/file","/api/repo/:owner/:repo/readme","/api/repo/:owner/:repo/branches","/w/:repoId/:path","/r/:repoId/:path","/api/pr/:id/:path","/api/gist/:id/:path","/assets/:path","/github/:action","/api/admin/:action","/api/user/:action","/api/repo/:id/file/:path","/api/repo/:id/files/counts","/api/repo/:id/files/search","/api/repo/:id/files","/api/repo/:id/zip","/api/repo/:id","/viewer/:id/:path"];
|
||||
+19
-7
@@ -23,14 +23,14 @@ const errorCodes = new Set([
|
||||
"ECONNRESET", "ECONNREFUSED", "ETIMEDOUT", "ENOTFOUND", "EPIPE", "ABORT_ERR",
|
||||
"upstream_error", "upstream_timeout", "invalid_signature", "webhook_processing_failed",
|
||||
]);
|
||||
const numbers = ["status", "httpStatus", "ms", "firstByteMs", "responseMs", "bytes", "bytesPerSecond",
|
||||
export const LOG_NUMBERS = ["expectedBytes", "measuredMs", "unattributedMs", "overlapMs","status", "httpStatus", "ms", "firstByteMs", "responseMs", "bytes", "bytesPerSecond",
|
||||
"attempt", "rss", "memoryLimitBytes", "memoryRatio", "uptime", "activeRequests", "waiting", "active", "delayed", "failed",
|
||||
"oldestWaitingMs", "droppedLogs", "loopP95Ms", "upstreamStatus"];
|
||||
const stages = ["repository", "credential_resolution", "upstream", "cache_lookup", "cache_fill", "worker_wait", "anonymize", "streamer_headers"];
|
||||
export const LOG_STAGES = ["transfer", "database","repository", "credential_resolution", "upstream", "cache_lookup", "cache_fill", "worker_wait", "anonymize", "streamer_headers"];
|
||||
export function logRecord(level: string, module: string, args: unknown[]) {
|
||||
const context = logContext.getStore() || {};
|
||||
const details: Record<string, unknown> = {};
|
||||
const keys = [...numbers, "requestId", "jobId", "code", "message", "outcome", "method", "route", "queue",
|
||||
const keys = [...LOG_NUMBERS, "requestId", "jobId", "hopId", "parentHopId", "cancellationOrigin", "cancellationEvidence", "endpointKind", "stageCounts", "stageOpen", "code", "message", "outcome", "method", "route", "queue",
|
||||
"errorType", "errorStack", "causeType", "causeStack", "requestPath", "stages", "reason", "phase", "event", "deliveryId", "hostname", "service"];
|
||||
// Do not traverse arbitrary objects or invoke error getters while logging.
|
||||
for (const arg of args.slice(0, 8)) if (arg && typeof arg === "object") {
|
||||
@@ -50,12 +50,12 @@ export function logRecord(level: string, module: string, args: unknown[]) {
|
||||
service: context.service || (allowed(["api", "streamer", "worker"], details.service) ? details.service : undefined) || (process.env.SERVICE_NAME?.toLowerCase() === "streamer" ? "streamer" : "api"),
|
||||
instance, release: /^[a-f0-9]{7,40}$/.test(process.env.APP_RELEASE || "") ? process.env.APP_RELEASE : "unknown",
|
||||
event, message: event, module: LOG_MODULES.includes(module) ? module : "application" };
|
||||
for (const key of ["requestId", "jobId"] as const) {
|
||||
for (const key of ["requestId", "jobId", "hopId", "parentHopId"] as const) {
|
||||
const value = details[key] || context[key];
|
||||
if (validTraceId(value)) record[key] = value;
|
||||
}
|
||||
if (Number.isSafeInteger(context.attempt)) record.attempt = context.attempt;
|
||||
for (const key of numbers) if (typeof details[key] === "number" && Number.isFinite(details[key])) record[key] = details[key];
|
||||
for (const key of LOG_NUMBERS) if (typeof details[key] === "number" && Number.isFinite(details[key])) record[key] = details[key];
|
||||
const code = [details.code, details.message].find(value => typeof value === "string" && errorCodes.has(value));
|
||||
if (code) record.errorCode = code;
|
||||
if (allowed(["protocol", "credentials", "port", "host"], details.reason)) record.reason = details.reason;
|
||||
@@ -64,21 +64,33 @@ export function logRecord(level: string, module: string, args: unknown[]) {
|
||||
if (validTraceId(details.deliveryId)) record.deliveryId = details.deliveryId;
|
||||
if (allowed(["github.com", "api.github.com", "codeload.github.com", "raw.githubusercontent.com", "media.githubusercontent.com"], details.hostname)) record.upstreamHost = details.hostname;
|
||||
else if (details.hostname) record.upstreamHost = "other";
|
||||
for (const [key, values] of Object.entries({ cancellationOrigin: ["unknown", "api", "streamer", "timeout", "upstream"],
|
||||
cancellationEvidence: ["response_close", "request_aborted", "upstream_error", "timeout", "local_abort"], endpointKind: ["public", "internal"] })) {
|
||||
if (allowed(values, details[key])) record[key] = details[key];
|
||||
}
|
||||
if (allowed(["completed", "interrupted"], details.outcome)) record.outcome = details.outcome;
|
||||
if (allowed(["GET", "HEAD", "POST", "PUT", "PATCH", "DELETE", "OPTIONS", "OTHER"], details.method)) record.method = details.method;
|
||||
// Only fixed route labels survive; never raw URLs or request parameters.
|
||||
const route = context.route || details.route;
|
||||
if (typeof route === "string" && LOG_ROUTES.includes(route)) record.route = route;
|
||||
if (typeof route === "string" && LOG_ROUTES.includes(route)) record.route = route.replace(/^\/api\/repo\/:id(?=\/|$)/, "/api/repo/:repoId");
|
||||
if (context.method && allowed(["GET", "HEAD", "POST", "PUT", "PATCH", "DELETE", "OPTIONS", "OTHER"], context.method)) record.method = context.method;
|
||||
Object.assign(record, safeDiagnosticFields(details));
|
||||
// Anonymous request paths are diagnostic fields, never metric labels.
|
||||
const requestPath = safeReproductionPath(context.requestPath || details.requestPath, record.method);
|
||||
if (requestPath) Object.assign(record, requestReproduction(requestPath, record.method));
|
||||
if (allowed(["download", "remove", "cache"], details.queue)) record.queue = details.queue;
|
||||
if (details.stages && typeof details.stages === "object") for (const key of stages) {
|
||||
if (details.stages && typeof details.stages === "object") for (const key of LOG_STAGES) {
|
||||
let value: unknown;
|
||||
try { value = Object.getOwnPropertyDescriptor(details.stages, key)?.value; } catch { continue; }
|
||||
if (typeof value === "number" && Number.isFinite(value)) record[`stage_${key}_ms`] = value;
|
||||
}
|
||||
for (const [field, suffix] of [["stageCounts", "count"], ["stageOpen", "open"]]) {
|
||||
if (!details[field] || typeof details[field] !== "object") continue;
|
||||
for (const stage of LOG_STAGES) {
|
||||
let value: unknown;
|
||||
try { value = Object.getOwnPropertyDescriptor(details[field], stage)?.value; } catch { continue; }
|
||||
if (typeof value === "number" && Number.isSafeInteger(value) && value >= 0) record[`stage_${stage}_${suffix}`] = value;
|
||||
}
|
||||
}
|
||||
return record;
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
export type FailureOrigin = "unknown" | "api" | "streamer" | "timeout" | "upstream";
|
||||
export type FailureEvidence = "response_close" | "request_aborted" | "upstream_error" | "timeout" | "local_abort";
|
||||
const evidence = new WeakMap<object, { cancellationOrigin: FailureOrigin; cancellationEvidence: FailureEvidence }>();
|
||||
|
||||
/** Record at the causal boundary, before destroying the response. First evidence wins. */
|
||||
export function recordResponseFailure(response: object, origin: FailureOrigin, reason: FailureEvidence) {
|
||||
if (!evidence.has(response)) evidence.set(response, { cancellationOrigin: origin, cancellationEvidence: reason });
|
||||
}
|
||||
export function recordUpstreamFailure(response: object, error: unknown, origin: "streamer" | "upstream" = "upstream") {
|
||||
const code = error && typeof error === "object" ? Object.getOwnPropertyDescriptor(error, "code")?.value : undefined;
|
||||
recordResponseFailure(response, code === "ETIMEDOUT" ? "timeout" : origin, code === "ETIMEDOUT" ? "timeout" : "upstream_error");
|
||||
}
|
||||
export function responseEvidence(response: object, aborted = false) {
|
||||
return evidence.get(response) || { cancellationOrigin: "unknown", cancellationEvidence: aborted ? "request_aborted" : "response_close" };
|
||||
}
|
||||
/** Only output representation length is comparable to response writes. */
|
||||
export function expectedResponseBytes(method: string, status: number, length: unknown): number | undefined {
|
||||
if (method === "HEAD" || status === 204 || status === 304) return 0;
|
||||
if ((typeof length !== "string" && typeof length !== "number") || !/^\d+$/.test(String(length))) return undefined;
|
||||
const value = Number(length);
|
||||
return Number.isSafeInteger(value) && value >= 0 ? value : undefined;
|
||||
}
|
||||
@@ -1,5 +1,6 @@
|
||||
import { responseEvidence, expectedResponseBytes } from "./request-evidence";
|
||||
import { requestReproduction } from "./log-reproduction";
|
||||
import { logContext } from "./log-context";
|
||||
import { logContext, validTraceId } from "./log-context";
|
||||
import { AsyncLocalStorage } from "async_hooks";
|
||||
import { randomUUID } from "crypto";
|
||||
import { performance } from "perf_hooks";
|
||||
@@ -8,13 +9,55 @@ import { createLogger } from "./logger";
|
||||
|
||||
export const LATENCY_BUCKETS = [10, 50, 100, 250, 500, 1000, 2500, 5000, 10000, 30000, 60000];
|
||||
export type Service = "api" | "streamer";
|
||||
export type Stage = "repository" | "credential_resolution" | "upstream" | "cache_lookup" | "cache_fill" | "worker_wait" | "anonymize" | "streamer_headers";
|
||||
export type Stage = "transfer" | "database" | "repository" | "credential_resolution" | "upstream" | "cache_lookup" | "cache_fill" | "worker_wait" | "anonymize" | "streamer_headers";
|
||||
export interface Metric {
|
||||
metric: string; route: string; method: string; count: number; sumMs: number;
|
||||
bytes: number; aborted: number; errors: number; slow: number; buckets: number[];
|
||||
}
|
||||
const logger = createLogger("requests");
|
||||
const context = new AsyncLocalStorage<{ id: string; service: Service; traffic?: string; stages: Partial<Record<Stage, number>> }>();
|
||||
interface Trace { id: string; hopId: string; service: Service; traffic?: string; timing: RequestTiming }
|
||||
const context = new AsyncLocalStorage<Trace>();
|
||||
|
||||
/** Constant storage: integrate active spans instead of retaining a per-call timeline. */
|
||||
export class RequestTiming {
|
||||
stages: Partial<Record<Stage, number>> = {};
|
||||
counts: Partial<Record<Stage, number>> = {};
|
||||
open: Partial<Record<Stage, number>> = {};
|
||||
private last: number;
|
||||
private covered = 0;
|
||||
private concurrent = 0;
|
||||
private ended = false;
|
||||
constructor(private started = performance.now(), private now = () => performance.now()) { this.last = started; }
|
||||
private advance(at: number) {
|
||||
const elapsed = Math.max(0, at - this.last);
|
||||
let active = 0;
|
||||
for (const stage of Object.keys(this.open) as Stage[]) {
|
||||
const count = this.open[stage] || 0;
|
||||
this.stages[stage] = (this.stages[stage] || 0) + elapsed * count;
|
||||
active += count;
|
||||
}
|
||||
if (active) this.covered += elapsed;
|
||||
this.concurrent += elapsed * Math.max(0, active - 1);
|
||||
this.last = at;
|
||||
}
|
||||
start(stage: Stage) {
|
||||
if (this.ended) return () => undefined;
|
||||
this.advance(this.now());
|
||||
this.counts[stage] = (this.counts[stage] || 0) + 1;
|
||||
this.open[stage] = (this.open[stage] || 0) + 1;
|
||||
let done = false;
|
||||
return () => {
|
||||
if (done || this.ended) return;
|
||||
done = true; this.advance(this.now()); this.open[stage]!--;
|
||||
};
|
||||
}
|
||||
finish(at = this.now()) {
|
||||
if (!this.ended) { this.advance(at); this.ended = true; }
|
||||
return { stages: { ...this.stages }, stageCounts: { ...this.counts }, stageOpen: { ...this.open },
|
||||
measuredMs: this.covered, unattributedMs: Math.max(0, this.last - this.started - this.covered), overlapMs: this.concurrent };
|
||||
}
|
||||
}
|
||||
|
||||
const methods = new Set(["GET", "HEAD", "POST", "PUT", "PATCH", "DELETE", "OPTIONS"]);
|
||||
const staticRoutes = new Set([
|
||||
"/", "/dashboard", "/anonymize", "/healthcheck", "/api/healthcheck", "/api/stat", "/api/stat/history",
|
||||
@@ -24,11 +67,16 @@ const staticRoutes = new Set([
|
||||
]);
|
||||
|
||||
/** Fixed labels: never use repository IDs, file names or query strings as metric keys. */
|
||||
export function requestRoute(url: string, service: Service): string {
|
||||
export function requestRoute(url: string, service: Service, method = "GET"): string {
|
||||
const path = url.split("?", 1)[0].replace(/\/+$/, "") || "/";
|
||||
if (service === "streamer") return ["/api/download", "/api", "/healthcheck"].includes(path) ? path : "other";
|
||||
if (staticRoutes.has(path)) return path;
|
||||
if (/^\/api\/repo\//.test(path)) {
|
||||
if (["GET", "HEAD"].includes(method)) {
|
||||
const source = path.match(/^\/api\/repo\/[^/]+\/[^/]+\/(readme|branches|tree|file)$/);
|
||||
if (source && !/^\/api\/repo\/[^/]+\/file\//.test(path)) return `/api/repo/:owner/:repo/${source[1]}`;
|
||||
if (/^\/api\/repo\/[^/]+\/[^/]+$/.test(path) && !/\/(file|files|zip|options)$/.test(path)) return "/api/repo/:owner/:repo";
|
||||
}
|
||||
const match = path.match(/^\/api\/repo\/[^/]+\/(file|files|zip|options|refresh|coauthors|extend)(?:\/|$)/);
|
||||
if (match) {
|
||||
const subroute = match[1] === "files" && /\/(counts|search)$/.exec(path);
|
||||
@@ -79,7 +127,7 @@ let activeRequests = 0;
|
||||
export function activeRequestCount() { return activeRequests; }
|
||||
export function requestHeaders(): Record<string, string> {
|
||||
const id = context.getStore()?.id;
|
||||
return id ? { "x-request-id": id, "x-cache-traffic": cacheTraffic() } : {};
|
||||
return id ? { "x-request-id": id, "x-cache-traffic": cacheTraffic(), "x-parent-hop-id": context.getStore()!.hopId } : {};
|
||||
}
|
||||
export function cacheTraffic() { return context.getStore()?.traffic || "background"; }
|
||||
export function recordCacheHit(cache: "source" | "transformed", hit: boolean) {
|
||||
@@ -90,12 +138,13 @@ export function recordCacheHit(cache: "source" | "transformed", hit: boolean) {
|
||||
export function startStage(stage: Stage) {
|
||||
const started = performance.now();
|
||||
const trace = context.getStore();
|
||||
const stop = trace?.timing.start(stage);
|
||||
let ended = false;
|
||||
return (error = false) => {
|
||||
if (ended) return; ended = true;
|
||||
if (!trace) return;
|
||||
const ms = performance.now() - started;
|
||||
trace.stages[stage] = (trace.stages[stage] || 0) + ms;
|
||||
stop?.();
|
||||
requestMetrics.observe(stage, "all", "all", ms, { error });
|
||||
};
|
||||
}
|
||||
@@ -109,25 +158,29 @@ export function monitorRequests(service: Service, metrics = requestMetrics,
|
||||
log: Pick<typeof logger, "info" | "warn"> = logger): RequestHandler {
|
||||
return (req, res, next) => {
|
||||
const forwarded = service === "streamer" ? req.get("x-request-id") : undefined;
|
||||
const id = forwarded && /^[a-f0-9-]{36}$/.test(forwarded) ? forwarded : randomUUID();
|
||||
const id = validTraceId(forwarded) ? forwarded : randomUUID();
|
||||
const forwardedTraffic = service === "streamer" ? req.get("x-cache-traffic") : undefined;
|
||||
const traffic = forwardedTraffic && ["file", "website", "zip", "monitor", "background"].includes(forwardedTraffic)
|
||||
? forwardedTraffic : /uptime|healthcheck|monitor/i.test(req.get("user-agent") || "") ? "monitor"
|
||||
: /\/zip(?:[/?]|$)|\/download(?:[/?]|$)/.test(req.originalUrl) ? "zip"
|
||||
: req.originalUrl.startsWith("/w/") ? "website" : "file";
|
||||
const trace = { id, service, traffic, stages: {} };
|
||||
const route = requestRoute(req.originalUrl, service);
|
||||
const started = performance.now();
|
||||
const hopId = randomUUID();
|
||||
const parent = service === "streamer" ? req.get("x-parent-hop-id") : undefined;
|
||||
const parentHopId = validTraceId(parent) ? parent : undefined;
|
||||
const trace = { id, hopId, service, traffic, timing: new RequestTiming(started) };
|
||||
const route = requestRoute(req.originalUrl, service, req.method);
|
||||
const reproduction = service === "api" ? requestReproduction(req.originalUrl, req.method) : {};
|
||||
const method = methods.has(req.method) ? req.method : "OTHER";
|
||||
const started = performance.now();
|
||||
let firstByteMs: number | undefined;
|
||||
let transferDone: (() => void) | undefined;
|
||||
let bytes = 0;
|
||||
let ended = false;
|
||||
activeRequests++;
|
||||
res.setHeader("X-Request-ID", id);
|
||||
const writeHead = res.writeHead;
|
||||
res.writeHead = function (this: typeof res, ...args: unknown[]): typeof res {
|
||||
firstByteMs ??= performance.now() - started;
|
||||
if (firstByteMs == null) { firstByteMs = performance.now() - started; transferDone = trace.timing.start("transfer"); }
|
||||
return Reflect.apply(writeHead, this, args);
|
||||
};
|
||||
const count = (chunk: unknown, encoding: unknown) => {
|
||||
@@ -151,6 +204,7 @@ export function monitorRequests(service: Service, metrics = requestMetrics,
|
||||
if (ended) return; ended = true;
|
||||
activeRequests--;
|
||||
res.removeListener("finish", done); res.removeListener("close", done);
|
||||
transferDone?.();
|
||||
const ms = performance.now() - started;
|
||||
const aborted = !res.writableFinished;
|
||||
const transfer = /\/(file|zip)(?:\/|$)/.test(route) || route.startsWith("/w/") || service === "streamer";
|
||||
@@ -162,11 +216,13 @@ export function monitorRequests(service: Service, metrics = requestMetrics,
|
||||
outcome: aborted ? "interrupted" : "completed", ms: Math.round(ms), firstByteMs: firstByteMs == null ? null : Math.round(firstByteMs),
|
||||
responseMs: responseMs == null ? null : Math.round(responseMs),
|
||||
bytesPerSecond: responseMs != null && responseMs >= 1 && bytes > 0 ? Math.round(bytes * 1000 / responseMs) : null,
|
||||
bytes, stages: trace.stages };
|
||||
bytes, expectedBytes: expectedResponseBytes(req.method, res.statusCode, res.getHeader("content-length")),
|
||||
endpointKind: service === "streamer" ? "internal" : "public", hopId, parentHopId,
|
||||
...(aborted ? responseEvidence(res, req.aborted) : {}), ...trace.timing.finish(started + ms) };
|
||||
if (slow || aborted) log.warn(aborted ? "request interrupted" : "slow request", detail);
|
||||
else if (!route.includes("healthcheck")) log.info("request", detail);
|
||||
};
|
||||
res.once("finish", done); res.once("close", done);
|
||||
context.run(trace, () => logContext.run({ ...reproduction, requestId: id, service, route, method }, next));
|
||||
context.run(trace, () => logContext.run({ ...reproduction, requestId: id, hopId, parentHopId, service, route, method }, next));
|
||||
};
|
||||
}
|
||||
@@ -1,3 +1,4 @@
|
||||
import { recordUpstreamFailure } from "./request-evidence";
|
||||
import { PassThrough, Readable, Writable } from "stream";
|
||||
|
||||
/** Cancel upstream on every response terminal event, including a pre-header close. */
|
||||
@@ -19,7 +20,7 @@ export function streamResponse(source: Readable, response: Writable,
|
||||
};
|
||||
const done = () => { cleanup(); resolve(); };
|
||||
const failed = (error: Error) => { cleanup(); reject(error); };
|
||||
const upstreamFailed = (error: Error) => { onError(error); done(); };
|
||||
const upstreamFailed = (error: Error) => { if (!response.destroyed) recordUpstreamFailure(response, error, "streamer"); onError(error); done(); };
|
||||
response.once("close", done);
|
||||
response.once("finish", done);
|
||||
response.once("error", failed);
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { recordUpstreamFailure, recordResponseFailure } from "./request-evidence";
|
||||
import got from "got";
|
||||
import config from "../config";
|
||||
import { MAX_SUBMODULE_ENTRIES, SubmoduleMount, safeRepositoryPath, validateMounts } from "./source/github-submodules";
|
||||
@@ -169,6 +170,7 @@ export async function streamAnonymizedZip(
|
||||
};
|
||||
const fail = (error: Error) => {
|
||||
if (stopped) return;
|
||||
recordResponseFailure(res, "unknown", "local_abort");
|
||||
logger.error("zip stream failed", serializeError(error));
|
||||
stop();
|
||||
const destroyable = res as unknown as {
|
||||
@@ -197,7 +199,7 @@ export async function streamAnonymizedZip(
|
||||
input.once("close", () => upstreamDone(!input.readableEnded));
|
||||
parser = Parse() as Transform;
|
||||
downloadStream
|
||||
.on("error", fail)
|
||||
.on("error", error => { recordUpstreamFailure(res, error); fail(error); })
|
||||
.pipe(parser)
|
||||
.on("entry", (entry: Readable & { type: string; path: string; autodrain: () => void }) => {
|
||||
if (stopped) {
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { createHash } from "crypto";
|
||||
import { logRecord } from "../core/log-record";
|
||||
import { logRecord, LOG_NUMBERS, LOG_STAGES } from "../core/log-record";
|
||||
import { logContext, validTraceId } from "../core/log-context";
|
||||
import { LOG_EVENTS } from "../core/log-schema";
|
||||
|
||||
@@ -50,13 +50,16 @@ function safeRow(input: Record<string, unknown>): Row | null {
|
||||
const time = typeof input._time === "string" ? input._time : input.timestamp;
|
||||
if (typeof time !== "string" || !Number.isFinite(Date.parse(time))) return null;
|
||||
// Reuse the application's reviewed projection. Never return raw backend data.
|
||||
const numeric = ["status", "httpStatus", "ms", "firstByteMs", "responseMs", "attempt", "rss", "memoryRatio", "droppedLogs", "oldestWaitingMs", "waiting"];
|
||||
const numeric = [...LOG_NUMBERS, ...LOG_STAGES.flatMap(stage => [`stage_${stage}_ms`, `stage_${stage}_count`, `stage_${stage}_open`])];
|
||||
const data: Record<string, unknown> = { ...input, code: input.errorCode };
|
||||
for (const key of numeric) if (typeof input[key] === "string" && /^-?\d+(?:\.\d+)?$/.test(input[key] as string)) data[key] = Number(input[key]);
|
||||
for (const [field, suffix] of [["stages", "ms"], ["stageCounts", "count"], ["stageOpen", "open"]]) {
|
||||
data[field] = Object.fromEntries(LOG_STAGES.map(stage => [stage, data[`stage_${stage}_${suffix}`]]));
|
||||
}
|
||||
// Revalidate finite route and diagnostic allowlists; never pass raw strings.
|
||||
const projected = logContext.run({}, () => logRecord(["error", "warn", "info", "debug"].includes(String(input.severity)) ? String(input.severity) : "info", String(input.module || "application"), ["", data]));
|
||||
const result: Row = { timestamp: /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d{1,9})?Z$/.test(time) ? time : new Date(time).toISOString(), severity: String(projected.severity), service: services.includes(String(input.service)) ? String(input.service) : "unknown", event: events.has(String(input.event)) ? String(input.event) : "application_log", module: String(projected.module), release: /^(?:[a-f0-9]{7,40}|unknown)$/.test(String(input.release)) ? String(input.release) : "unknown" };
|
||||
for (const key of [...numeric, "errorCode", "requestId", "jobId", "queue", "route", "method", "errorType", "errorStack", "causeType", "causeStack", "repoId", "requestPath"]) if (typeof projected[key] === "string" || typeof projected[key] === "number") result[key] = projected[key] as string | number;
|
||||
for (const key of [...numeric, "outcome", "endpointKind", "cancellationOrigin", "cancellationEvidence", "hopId", "parentHopId", "errorCode", "requestId", "jobId", "queue", "route", "method", "errorType", "errorStack", "causeType", "causeStack", "repoId", "requestPath"]) if (typeof projected[key] === "string" || typeof projected[key] === "number") result[key] = projected[key] as string | number;
|
||||
if (validTraceId(input.instance)) result.instance = input.instance;
|
||||
return result;
|
||||
}
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { installDatabaseTiming } from "../core/database-timing";
|
||||
import InstallationModel from "../core/model/github-installation";
|
||||
import { credentialCipher } from "../core/credentials";
|
||||
import CredentialModel from "../core/model/credentials/credentials.model";
|
||||
@@ -24,6 +25,7 @@ export const database = mongoose.connection;
|
||||
export let isConnected = false;
|
||||
|
||||
export async function connect(overrides: Pick<ConnectOptions, "appName" | "maxPoolSize" | "minPoolSize"> = {}) {
|
||||
installDatabaseTiming();
|
||||
credentialCipher(); // Refuse to serve persisted credentials without a valid keyring.
|
||||
mongoose.set("strictQuery", false);
|
||||
const options: ConnectOptions = {
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { recordUpstreamFailure } from "../../core/request-evidence";
|
||||
import { jobTrace } from "../../core/log-context";
|
||||
import { hasVisibleSubmodules } from "../../core/file-submodules";
|
||||
import { FILE_LISTING_LIMIT } from "../../core/file-tree";
|
||||
@@ -89,6 +90,7 @@ router.get(
|
||||
},
|
||||
})
|
||||
.on("error", (err: Error & { response?: { statusCode?: number; body?: unknown } }) => {
|
||||
if (!res.destroyed) recordUpstreamFailure(res, err, "streamer");
|
||||
headersDone(true);
|
||||
const upstreamStatus = err?.response?.statusCode;
|
||||
let upstreamBody: string | undefined;
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { recordResponseFailure } from "../core/request-evidence";
|
||||
import { sendSharedContent } from "../core/cached-response";
|
||||
import * as express from "express";
|
||||
import GitHubStream from "../core/source/GitHubStream";
|
||||
@@ -125,9 +126,9 @@ router.post("/", async (req: express.Request, res: express.Response) => {
|
||||
anonymizer.destroy();
|
||||
}
|
||||
content
|
||||
.on("error", handleStreamError)
|
||||
.on("error", error => { if (!res.destroyed) recordResponseFailure(res, "unknown", "local_abort"); handleStreamError(error); })
|
||||
.pipe(anonymizer)
|
||||
.on("error", handleStreamError)
|
||||
.on("error", error => { if (!res.destroyed) recordResponseFailure(res, "streamer", "local_abort"); handleStreamError(error); })
|
||||
.pipe(res)
|
||||
.on("error", handleStreamError)
|
||||
.on("close", () => {
|
||||
|
||||
@@ -0,0 +1,28 @@
|
||||
const {expect}=require('chai');
|
||||
require('ts-node/register/transpile-only');
|
||||
const mongoose=require('mongoose');
|
||||
const {installDatabaseTiming}=require('../src/core/database-timing');
|
||||
const {monitorRequests,RequestMetrics}=require('../src/core/request-monitoring');
|
||||
const express=require('express');
|
||||
const http=require('node:http');
|
||||
describe('database execution timing',()=>{
|
||||
it('preserves query receiver, result and failure while measuring execution once',async()=>{
|
||||
const Model=mongoose.model("TimingFixture",new mongoose.Schema({}));
|
||||
const logs=[];const app=express();let server;
|
||||
try{
|
||||
installDatabaseTiming();const wrapped=mongoose.Query.prototype.exec;installDatabaseTiming();expect(mongoose.Query.prototype.exec).equal(wrapped);
|
||||
app.use(monitorRequests('api',new RequestMetrics(),{info:(_,row)=>logs.push(row),warn:(_,row)=>logs.push(row)}));
|
||||
app.get('/',async(_req,res)=>{
|
||||
const query=Model.find();query._find=async function(){expect(this).equal(query);await new Promise(r=>setTimeout(r,20));return ['ok'];};
|
||||
expect(await query.exec()).deep.equal(['ok']);
|
||||
const failed=Model.find();const error=new Error('PRIVATE query');failed._find=async()=>{throw error;};
|
||||
try{await failed.exec();throw Error('expected failure');}catch(e){expect(e).equal(error);}
|
||||
res.end('ok');
|
||||
});
|
||||
server=await new Promise(resolve=>{const s=app.listen(0,'127.0.0.1',()=>resolve(s));});
|
||||
await new Promise((resolve,reject)=>http.get({host:'127.0.0.1',port:server.address().port},res=>{res.resume();res.on('end',resolve);}).on('error',reject));
|
||||
expect(logs[0].stageCounts.database).equal(2);expect(logs[0].stages.database).greaterThan(15);expect(logs[0].stageOpen.database).equal(0);
|
||||
expect(JSON.stringify(logs)).not.include('PRIVATE');
|
||||
}finally{mongoose.deleteModel("TimingFixture");if(server)await new Promise(resolve=>server.close(resolve));}
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,68 @@
|
||||
const { expect } = require('chai');
|
||||
require('ts-node/register/transpile-only');
|
||||
const { RequestTiming, requestRoute } = require('../src/core/request-monitoring');
|
||||
const { recordResponseFailure, recordUpstreamFailure, responseEvidence, expectedResponseBytes } = require('../src/core/request-evidence');
|
||||
const { logRecord } = require('../src/core/log-record');
|
||||
const { safeRedisLog } = require('../src/server/admin-log-investigation');
|
||||
|
||||
describe('request causal evidence and timing accounting', () => {
|
||||
it('integrates nested, parallel, repeated and unfinished spans without invented totals', () => {
|
||||
let now = 0;
|
||||
const timing = new RequestTiming(0, () => now);
|
||||
now = 10; const credential = timing.start('credential_resolution');
|
||||
now = 20; const db = timing.start('database');
|
||||
now = 30; const upstream = timing.start('upstream');
|
||||
now = 40; db(); db();
|
||||
now = 50; upstream(); credential();
|
||||
now = 60; timing.start('upstream');
|
||||
now = 70; const snapshot = timing.finish();
|
||||
expect(snapshot).deep.equal({stages:{credential_resolution:40,database:20,upstream:30},
|
||||
stageCounts:{credential_resolution:1,database:1,upstream:2},stageOpen:{credential_resolution:0,database:0,upstream:1},
|
||||
measuredMs:50,unattributedMs:20,overlapMs:40});
|
||||
now = 100; upstream(); timing.start('database')();
|
||||
expect(timing.finish()).deep.equal(snapshot);
|
||||
});
|
||||
it('bounds storage across repeated spans', () => {
|
||||
let now = 0; const timing = new RequestTiming(0, () => now++);
|
||||
for(let i=0;i<100000;i++) timing.start('upstream')();
|
||||
const result=timing.finish();
|
||||
expect(result.stageCounts.upstream).equal(100000);
|
||||
expect(JSON.stringify(timing).length).lessThan(500);
|
||||
});
|
||||
it('keeps disconnect origins unknown and preserves the first causal evidence', () => {
|
||||
expect(responseEvidence({})).deep.equal({cancellationOrigin:'unknown',cancellationEvidence:'response_close'});
|
||||
expect(responseEvidence({},true).cancellationOrigin).equal('unknown');
|
||||
const res={}; recordUpstreamFailure(res,Object.assign(new Error('PRIVATE'),{code:'ETIMEDOUT'}));
|
||||
recordResponseFailure(res,'api','local_abort');
|
||||
expect(responseEvidence(res)).deep.equal({cancellationOrigin:'timeout',cancellationEvidence:'timeout'});
|
||||
const upstream={};recordUpstreamFailure(upstream,new Error('PRIVATE'),'streamer');
|
||||
expect(responseEvidence(upstream)).deep.equal({cancellationOrigin:'streamer',cancellationEvidence:'upstream_error'});
|
||||
const local={};recordResponseFailure(local,'api','local_abort');expect(responseEvidence(local).cancellationOrigin).equal('api');
|
||||
});
|
||||
it('only compares body bytes with the output content length', () => {
|
||||
expect(expectedResponseBytes('GET',200,'123')).equal(123);
|
||||
expect(expectedResponseBytes('HEAD',200,'123')).equal(0);
|
||||
expect(expectedResponseBytes('GET',304,'123')).equal(0);
|
||||
for(const value of [undefined,-1,'1.2','9e5','9007199254740992',{},['123']]) expect(expectedResponseBytes('GET',200,value)).equal(undefined);
|
||||
});
|
||||
it('preserves bounded evidence and timings through the admin projection', () => {
|
||||
const row=logRecord('warn','requests',['request interrupted',{outcome:'interrupted',bytes:12,expectedBytes:50,
|
||||
cancellationOrigin:'unknown',cancellationEvidence:'response_close',endpointKind:'public',measuredMs:20,unattributedMs:10,overlapMs:5,
|
||||
stages:{database:3,upstream:17,PRIVATE:123},stageCounts:{upstream:2},stageOpen:{upstream:1}}]);
|
||||
const safe=safeRedisLog({ts:row.timestamp,level:'warn',module:'requests',raw:[row]}).raw[0];
|
||||
expect(safe).include({bytes:12,expectedBytes:50,cancellationOrigin:'unknown',cancellationEvidence:'response_close',
|
||||
endpointKind:'public',measuredMs:20,unattributedMs:10,overlapMs:5,stage_database_ms:3,stage_upstream_ms:17,stage_upstream_count:2,stage_upstream_open:1});
|
||||
expect(JSON.stringify(safe)).not.include('PRIVATE');
|
||||
const bad=logRecord('warn','requests',['request interrupted',{cancellationOrigin:'PRIVATE',cancellationEvidence:'PRIVATE',endpointKind:'PRIVATE'}]);
|
||||
expect(JSON.stringify(bad)).not.include('PRIVATE');
|
||||
});
|
||||
it('respects source handler precedence and canonical anonymous route IDs', () => {
|
||||
for(const action of ['readme','branches','tree','file']) expect(requestRoute('/api/repo/private/files/'+action,'api')).equal('/api/repo/:owner/:repo/'+action);
|
||||
expect(requestRoute('/api/repo/demo/file/readme','api')).equal('/api/repo/:repoId/file/:path');
|
||||
for(const action of ['coauthors','extend','refresh']) {
|
||||
expect(requestRoute('/api/repo/private/'+action,'api','GET')).equal('/api/repo/:owner/:repo');
|
||||
expect(requestRoute('/api/repo/demo/'+action,'api','POST')).equal('/api/repo/:repoId/'+action);
|
||||
}
|
||||
expect(logRecord('info','requests',['request',{route:'/api/repo/:id/files'}]).route).equal('/api/repo/:repoId/files');
|
||||
});
|
||||
});
|
||||
@@ -174,6 +174,7 @@ describe("bounded request monitoring", function () {
|
||||
await ready; client.destroy(); await terminal;
|
||||
expect(logs).to.have.length(1); expect(logs[0][0]).to.equal("request interrupted");
|
||||
expect(logs[0][1].outcome).to.equal("interrupted");
|
||||
expect(logs[0][1].cancellationOrigin).to.equal("unknown");
|
||||
expect(logs[0][1]).include({requestPath:"/api/repo/demo-123/zip",repoId:"demo-123"});
|
||||
expect(JSON.stringify(logs)).not.include("SECRET");
|
||||
const row = metrics.drain().find(row => row.data.metric === "request").data;
|
||||
@@ -186,6 +187,50 @@ describe("bounded request monitoring", function () {
|
||||
});
|
||||
}
|
||||
|
||||
for (const code of ["ETIMEDOUT", "ECONNRESET"]) {
|
||||
it(`records causal ${code} before a truncated 200 closes`, async function () {
|
||||
const { streamResponse } = require("../src/core/response-stream");
|
||||
app.get("/", (_req, res) => {
|
||||
const source = new Readable({ read() {} });
|
||||
res.setHeader("Content-Length", "100");
|
||||
void streamResponse(source, res, () => res.destroy());
|
||||
source.push("partial");
|
||||
setTimeout(() => source.destroy(Object.assign(new Error("PRIVATE"), {code})), 20);
|
||||
});
|
||||
await listen();
|
||||
await fetch(server, "/").catch(() => undefined);
|
||||
await delay(20);
|
||||
const detail = logs[0][1];
|
||||
expect(detail).include({status:200,outcome:"interrupted",bytes:7,expectedBytes:100,
|
||||
cancellationOrigin:code === "ETIMEDOUT" ? "timeout" : "streamer"});
|
||||
expect(JSON.stringify(logs)).not.include("PRIVATE");
|
||||
});
|
||||
}
|
||||
|
||||
it("correlates repeated streamer hops without accepting public caller IDs", async function () {
|
||||
const upstream = express(); const hops=[];
|
||||
upstream.use(monitorRequests("streamer",new RequestMetrics(),{info:(_,row)=>hops.push(row),warn:(_,row)=>hops.push(row)}));
|
||||
upstream.post("/api",(_req,res)=>res.end("ok"));
|
||||
const streamer = await new Promise(resolve => {const s=upstream.listen(0,"127.0.0.1",()=>resolve(s));});
|
||||
try {
|
||||
app.get("/", async (_req,res)=>{
|
||||
await fetch(streamer,"/api",{method:"POST",headers:requestHeaders()});
|
||||
await fetch(streamer,"/api",{method:"POST",headers:requestHeaders()});
|
||||
res.end("ok");
|
||||
});
|
||||
await listen();
|
||||
const forged="11111111-1111-1111-1111-111111111111";
|
||||
await fetch(server,"/",{headers:{"x-request-id":forged,"x-parent-hop-id":forged}});
|
||||
const parent=logs[0][1];
|
||||
expect(parent.requestId).not.equal(forged);
|
||||
expect(parent.parentHopId).equal(undefined);
|
||||
expect(hops).length(2);
|
||||
expect(hops[0].hopId).not.equal(hops[1].hopId);
|
||||
for(const hop of hops) expect(hop).include({requestId:parent.requestId,parentHopId:parent.hopId,endpointKind:"internal",outcome:"completed"});
|
||||
expect(parent).not.have.property("cancellationOrigin");
|
||||
} finally {streamer.closeAllConnections();await new Promise(resolve=>streamer.close(resolve));}
|
||||
});
|
||||
|
||||
it("keeps overlapping request IDs and asynchronous stages isolated", async function () {
|
||||
let doneA, doneB;
|
||||
app.get("/a", (_req, res) => { doneA = startStage("upstream"); setTimeout(() => { doneA(); doneA(); res.json(requestHeaders()); }, 30); });
|
||||
|
||||
@@ -26,7 +26,7 @@ function request(port, path, encoding = "gzip", extra = {}) {
|
||||
}
|
||||
describe("shared cache HTTP and telemetry integration", function () {
|
||||
this.timeout(15000);
|
||||
let root, old, server, port, fills, body;
|
||||
let root, old, server, port, fills, body, requestLogs;
|
||||
beforeEach(async () => {
|
||||
old = { ...config }; closeBlobCaches(); closeCacheMetrics();
|
||||
root = await fs.mkdtemp(join(tmpdir(), "shared-http-"));
|
||||
@@ -34,7 +34,8 @@ describe("shared cache HTTP and telemetry integration", function () {
|
||||
SHARED_CACHE_BYTES: 16 * 1024 ** 2, SHARED_CACHE_RESULT_BYTES: 8 * 1024 ** 2, SHARED_CACHE_MAX_FILE_BYTES: 1024 ** 2,
|
||||
SHARED_CACHE_FREE_BYTES: 0, CACHE_MONITORING_ENABLED: true });
|
||||
fills = 0; body = Buffer.from("Public documentation with no identifying names.\n".repeat(100));
|
||||
const app = express(); app.use(monitorRequests("api", undefined, { info() {}, warn() {} }));
|
||||
requestLogs = [];
|
||||
const app = express(); app.use(monitorRequests("api", undefined, { info: (_, row) => requestLogs.push(row), warn: (_, row) => requestLogs.push(row) }));
|
||||
app.get("/:project/:mode", async (req, res) => {
|
||||
if (req.headers.authorization !== "test-access") return res.status(403).end();
|
||||
const source = new GitHubStream({ repoId: req.params.project, organization: "org", repoName: "repo", commit: "commit", getToken: async () => "" });
|
||||
@@ -56,6 +57,27 @@ describe("shared cache HTTP and telemetry integration", function () {
|
||||
await new Promise(resolve => server.close(resolve)); closeBlobCaches(); closeCacheMetrics(); Object.assign(config, old);
|
||||
await fs.rm(root, { recursive: true, force: true });
|
||||
});
|
||||
it("records a shared result cache read failure before pipeline destroys the response", async () => {
|
||||
await request(port, "/a/changed", "identity", {authorization:"test-access"});
|
||||
const results = resultBlobCache(), original = results.open;
|
||||
results.open = async function (...args) {
|
||||
const stored = await Reflect.apply(original, this, args);
|
||||
if (!stored) return stored;
|
||||
const broken = new Readable({read() {}});
|
||||
broken.cachedBlob = stored.cachedBlob; stored.destroy();
|
||||
setTimeout(() => broken.push(Buffer.from("partial")), 10);
|
||||
setTimeout(() => broken.destroy(Object.assign(new Error("PRIVATE local cache"), {code:"EIO"})), 30);
|
||||
return broken;
|
||||
};
|
||||
try {
|
||||
await assert.rejects(request(port, "/a/changed", "identity", {authorization:"test-access"}));
|
||||
await new Promise(resolve => setTimeout(resolve, 20));
|
||||
const row = requestLogs.at(-1);
|
||||
assert.equal(row.status, 200); assert.equal(row.outcome, "interrupted");
|
||||
assert.equal(row.cancellationOrigin, "unknown"); assert.equal(row.cancellationEvidence, "local_abort");
|
||||
assert.equal(row.bytes, 7); assert.ok(row.expectedBytes > 7);
|
||||
} finally { results.open = original; }
|
||||
});
|
||||
for (const mode of ["fill", "hit", "bypass"]) {
|
||||
it(`enforces resolved-byte billing admission on shared source ${mode}`, async () => {
|
||||
const quotas = require("../src/core/billing/quotas");
|
||||
|
||||
@@ -0,0 +1,38 @@
|
||||
const {expect}=require('chai');
|
||||
const express=require('express');
|
||||
const http=require('node:http');
|
||||
const {Readable}=require('node:stream');
|
||||
require('ts-node/register/transpile-only');
|
||||
const {monitorRequests,RequestMetrics}=require('../src/core/request-monitoring');
|
||||
const GitHubStream=require('../src/core/source/GitHubStream').default;
|
||||
const {AnonymizeTransformer}=require('../src/core/anonymize-utils');
|
||||
const streamer=require('../src/streamer/route').default;
|
||||
const config=require('../src/config').default;
|
||||
describe('streamer failure provenance',()=>{
|
||||
for(const mode of ['cache','transform']) it(`records ${mode} failures before teardown without inventing upstream provenance`,async()=>{
|
||||
const originalCache=GitHubStream.prototype.getFileContentCache,originalTransform=AnonymizeTransformer.prototype._transform,shared=config.SHARED_CACHE_ENABLED;
|
||||
const logs=[];let server;
|
||||
try{
|
||||
config.SHARED_CACHE_ENABLED=false;
|
||||
GitHubStream.prototype.getFileContentCache=async()=>{
|
||||
const input=new Readable({read(){}});
|
||||
setTimeout(()=>input.push(Buffer.from('partial')),10);
|
||||
if(mode==='cache')setTimeout(()=>input.destroy(Object.assign(new Error('PRIVATE cache path'),{code:'EIO'})),30);
|
||||
else setTimeout(()=>input.push(Buffer.from('fail')),30);
|
||||
return input;
|
||||
};
|
||||
AnonymizeTransformer.prototype._transform=function(chunk,_encoding,done){
|
||||
if(mode==='transform'&&chunk.toString()==='fail')done(new Error('PRIVATE output spool'));
|
||||
else {this.push(chunk);done();}
|
||||
};
|
||||
const app=express();app.use(monitorRequests('streamer',new RequestMetrics(),{info:(_,r)=>logs.push(r),warn:(_,r)=>logs.push(r)}));app.use(express.json());app.use('/api',streamer);
|
||||
server=await new Promise(resolve=>{const s=app.listen(0,'127.0.0.1',()=>resolve(s));});
|
||||
await new Promise(resolve=>{
|
||||
const req=http.request({host:'127.0.0.1',port:server.address().port,path:'/api',method:'POST',headers:{'content-type':'application/json'}},res=>{res.resume();res.on('close',resolve);});
|
||||
req.on('error',resolve);req.end(JSON.stringify({token:'',repoFullName:'PRIVATE/repo',repoId:'demo',filePath:'file.txt',anonymizerOptions:{terms:[],filePath:'file.txt'}}));
|
||||
});
|
||||
await new Promise(resolve=>setTimeout(resolve,20));
|
||||
expect(logs).length(1);expect(logs[0]).include({status:200,outcome:'interrupted',cancellationOrigin:mode==='transform'?'streamer':'unknown',cancellationEvidence:'local_abort',bytes:7});
|
||||
}finally{GitHubStream.prototype.getFileContentCache=originalCache;AnonymizeTransformer.prototype._transform=originalTransform;config.SHARED_CACHE_ENABLED=shared;if(server){server.closeAllConnections();await new Promise(resolve=>server.close(resolve));}}
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user