From 953767b2cd609b36bfe8d9516451bdb952d946ed Mon Sep 17 00:00:00 2001 From: Thomas Durieux <5577568+tdurieux@users.noreply.github.com> Date: Sun, 15 Nov 2026 00:01:21 +0000 Subject: [PATCH 1/3] fix: retain actual anonymous request paths across outcomes --- deploy/logging/README.md | 4 ++-- deploy/logging/sanitize.vrl | 8 ++++---- scripts/test-admin-logs.js | 13 +++++++++++++ src/core/log-record.ts | 8 +++----- src/core/log-reproduction.ts | 10 +++++----- src/core/request-monitoring.ts | 2 +- test/log-diagnostics.test.js | 17 +++++++++++++++-- test/request-monitoring.test.js | 17 ++++++++++++++--- 8 files changed, 57 insertions(+), 22 deletions(-) diff --git a/deploy/logging/README.md b/deploy/logging/README.md index 6a4bc0c..18c097d 100644 --- a/deploy/logging/README.md +++ b/deploy/logging/README.md @@ -341,7 +341,7 @@ a finite error type/code, and up to eight recognized application frames (1 KiB m exception and the deepest inspected cause (at most three levels, cycles stopped). Raw messages, Upstream URLs, query strings, external frames and absolute filesystem roots are not stored. Repository/file parameters are excluded except for the explicitly -scoped failed-request reproduction context below. Unknown error types use `UnknownError`; unknown +scoped anonymous-request reproduction context below. Unknown error types use `UnknownError`; unknown function names become ``. Source locations are build-relative for compiled code and source-relative in development; line numbers refer to that release, not an inferred source map. @@ -355,7 +355,7 @@ unknown fields. Existing discarded exception details cannot be recovered. ### Admin reproduction context -Failed application file requests now retain their public anonymous repository ID and encoded request pathname, up to 2 KiB, under the existing admin log access controls. This is an explicit exception to template-only diagnostics: repository-relative filenames are retained for reproduction. Query strings, headers, bodies and upstream owner/repository URLs are still excluded. Filenames are operator-visible data under this policy; no generic filter can guarantee that a user has not put sensitive text in a filename. Successful requests and metrics keep only the normalized route. Only `/api/repo/:repoId/file/:path` is supported; other endpoints and unsafe, oversized, malformed or traversal paths omit reproduction context. Historical records cannot be backfilled. The UI copies the exact captured pathname without a host or query. +Application requests to the supported anonymous repository endpoints retain their public anonymous repository ID and encoded request pathname, up to 2 KiB, under the existing admin log access controls. This is an explicit exception to template-only diagnostics: repository-relative filenames are retained for reproduction. Query strings, headers, bodies and upstream owner/repository URLs are still excluded. Filenames are operator-visible data under this policy; no generic filter can guarantee that a user has not put sensitive text in a filename. Capture includes successful, failed and interrupted requests; metrics still use only normalized routes. Supported paths are repository metadata, file contents, listings/counts/search, ZIP, options, refresh, coauthors and extend under `/api/repo/:repoId`; other endpoints and unsafe, oversized, malformed or traversal paths omit reproduction context. Historical records cannot be backfilled. The UI copies the exact captured pathname without a host or query. For the focused restricted-file/rotation regression, run `LOGGING_ACCESS_ONLY=1 python3 scripts/test-logging.py`. This uses synthetic `0640` logs in an isolated Docker project and preserves capability restrictions. diff --git a/deploy/logging/sanitize.vrl b/deploy/logging/sanitize.vrl index 53c31a7..2afb127 100644 --- a/deploy/logging/sanitize.vrl +++ b/deploy/logging/sanitize.vrl @@ -59,12 +59,12 @@ for_each(["status", "httpStatus", "ms", "firstByteMs", "responseMs", "bytes", "b if includes(["GET", "HEAD", "POST", "PUT", "PATCH", "DELETE", "OPTIONS", "OTHER"], record.method) { out.method = record.method } if includes(["/", "other", "/dashboard", "/anonymize", "/healthcheck", "/api/healthcheck", "/api", "/api/download", "/api/stat", "/api/stat/history", "/api/options", "/api/message", "/api/user", "/api/user/quota", "/api/user/dashboard", "/api/repo", "/api/anonymize-preview", "/api/admin/overview", "/api/admin/performance", "/api/admin/errors", "/api/admin/errors/stats", "/api/admin/queues", "/api/admin/queues/metrics", "/github/app/webhook", "/api/repo/:repoId", "/api/repo/:repoId/file/:path", "/api/repo/:repoId/files", "/api/repo/:repoId/files/counts", "/api/repo/:repoId/files/search", "/api/repo/:repoId/zip", "/api/repo/:repoId/options", "/api/repo/:repoId/refresh", "/api/repo/:repoId/coauthors", "/api/repo/:repoId/extend", "/api/repo/:owner/:repo/readme", "/api/repo/:owner/:repo/branches", "/w/:repoId/:path", "/r/:repoId/:path", "/api/pr/:id/:path", "/api/gist/:id/:path", "/assets/:path", "/github/:action", "/api/admin/:action", "/api/user/:action", "/api/repo/:id/file/:path", "/api/repo/:id/files/counts", "/api/repo/:id/files/search", "/api/repo/:id/files", "/api/repo/:id/zip", "/api/repo/:id", "/viewer/:id/:path"], record.route) { out.route = record.route } # VictoriaLogs consumes its message field; keep event independently searchable. -# Explicit reproduction context for failed application file requests, never query strings. -if includes(["error", "fatal"], out.severity) || to_int(out.status) >= 400 || to_int(out.httpStatus) >= 400 { +# Anonymous application request context at every outcome, never query strings. +{ path = to_string(record.requestPath) ?? "" - parsed, path_error = parse_regex(path, r'^/api/repo/(?P[A-Za-z0-9_-]{3,200})/file/(?P[A-Za-z0-9._~!$&\'()+,;=@%/-]+)$') + parsed, path_error = parse_regex(path, r'^/api/repo/(?P[A-Za-z0-9_-]{3,200})(?:/(?Pfile/[A-Za-z0-9._~!$&\'()+,;=@%/-]+|files(?:/[A-Za-z0-9._~!$&\'()+,;=@%/-]*)?|(?:zip|options|refresh|coauthors|extend)/?))?/?$') decoded = decode_percent(to_string(parsed.file)) - if path_error == null && length(path) <= 2048 && !match(path, r'%(?:[^a-fA-F0-9]|[a-fA-F0-9][^a-fA-F0-9]|[a-fA-F0-9]?$)') && !match(decoded, r'[\x00-\x1f\x7f-\x9f\x{2028}-\x{202e}\x{2066}-\x{2069}\x{fffd}\\:?#%]|^/|(^|/)\.\.?(/|$)') { + if path_error == null && length(path) <= 2048 && !match(path, r'%(?:[^a-fA-F0-9]|[a-fA-F0-9][^a-fA-F0-9]|[a-fA-F0-9]?$)') && !match(decoded, r'[\x00-\x1f\x7f-\x9f\x{2028}-\x{202e}\x{2066}-\x{2069}\x{fffd}\\:?#%]|//|(^|/)\.\.?(/|$)') { out.requestPath = path out.repoId = parsed.repo } diff --git a/scripts/test-admin-logs.js b/scripts/test-admin-logs.js index bd5fca5..beaddf5 100644 --- a/scripts/test-admin-logs.js +++ b/scripts/test-admin-logs.js @@ -90,6 +90,19 @@ const sleep = ms => new Promise(resolve=>setTimeout(resolve,ms)); const traced=await investigateLogs(logOptions({kind:'requestId',id:diagnosticId},true),true); const redis=safeRedisLog({ts:diagnostic.timestamp,level:'error',module:'route',raw:[diagnostic]}); for(const row of [traced.events[0],redis.raw[0]]) {assert.equal(row.requestPath,diagnostic.requestPath);assert.equal(row.repoId,'demo-123');assert.equal(row.route,diagnostic.route);assert.equal(row.method,'GET');assert.equal(row.errorStack,projected.errorStack);assert.equal(row.causeType,'RangeError');assert(!JSON.stringify(row).includes(secret));} + const paths=['/api/repo/demo-123','/api/repo/demo-123/zip','/api/repo/demo-123/files/docs/nested%20folder/','/api/repo/demo-123/files/counts','/api/repo/demo-123/options','/api/repo/demo-123/file/README.md']; + const pathRecords=paths.map((requestPath,i)=>logContext.run({requestId:randomUUID(),requestPath},()=>logRecord(i%2?'warn':'info','requests',['request',{status:200,outcome:i%2?'interrupted':'completed'}]))); + const pathSources=pathRecords.map(record=>({...rawSource,message:JSON.stringify({attrs:{tag:'api'},log:JSON.stringify(record)})})); + const pathOutput=execFileSync('docker',['run','--rm','-i','--network','none','--memory','256m',vectorImage,'vrl','-i','/dev/stdin',fs.readFileSync('deploy/logging/sanitize.vrl','utf8')+'\nencode_json(.)'],{encoding:'utf8',input:pathSources.map(JSON.stringify).join('\n')+'\n'}).trim().split('\n').map(line=>JSON.parse(JSON.parse(line))); + assert.deepEqual(pathOutput.map(x=>x.requestPath),paths); + const pathInsert=await fetch(process.env.ADMIN_LOG_URL+'/insert/jsonline?_time_field=timestamp',{method:'POST',body:pathOutput.map(JSON.stringify).join('\n'),headers:{'Content-Type':'application/stream+json'}}); + assert(pathInsert.ok);await sleep(2000); + for(let i=0;i= 400) { - const requestPath = safeReproductionPath(context.requestPath || details.requestPath); - if (requestPath) Object.assign(record, requestReproduction(requestPath)); - } + // Anonymous request paths are diagnostic fields, never metric labels. + const requestPath = safeReproductionPath(context.requestPath || details.requestPath); + if (requestPath) Object.assign(record, requestReproduction(requestPath)); if (allowed(["download", "remove", "cache"], details.queue)) record.queue = details.queue; if (details.stages && typeof details.stages === "object") for (const key of stages) { let value: unknown; diff --git a/src/core/log-reproduction.ts b/src/core/log-reproduction.ts index f854dad..649057d 100644 --- a/src/core/log-reproduction.ts +++ b/src/core/log-reproduction.ts @@ -1,19 +1,19 @@ // Explicit admin diagnostics: public anonymous repository ID and request pathname only. // Never collect upstream owner/repository URLs, headers, query strings or bodies. -const fileRequest = /^\/api\/repo\/([A-Za-z0-9_-]{3,200})\/file\/([A-Za-z0-9._~!$&'()+,;=@%/-]+)$/; +const anonymousRequest = /^\/api\/repo\/([A-Za-z0-9_-]{3,200})(?:\/(file\/[A-Za-z0-9._~!$&'()+,;=@%/-]+|files(?:\/[A-Za-z0-9._~!$&'()+,;=@%/-]*)?|(?:zip|options|refresh|coauthors|extend)\/?))?\/?$/; export function safeReproductionPath(value: unknown): string | undefined { if (typeof value !== "string" || value.length > 2048) return undefined; - const match = fileRequest.exec(value); + const match = anonymousRequest.exec(value); if (!match || /%(?![a-fA-F0-9]{2})/.test(value)) return undefined; try { - const path = decodeURIComponent(match[2]); + const path = decodeURIComponent(match[2] || ""); // Reject controls, traversal, URL syntax, encoded delimiters and double encoding. // eslint-disable-next-line no-control-regex -- reject control characters from stored paths - if (/[\x00-\x1f\x7f-\x9f\u2028-\u202e\u2066-\u2069\ufffd\\:?#%]/.test(path) || path.startsWith("/") || path.split("/").some(part => part === "." || part === "..")) return undefined; + if (/[\x00-\x1f\x7f-\x9f\u2028-\u202e\u2066-\u2069\ufffd\\:?#%]/.test(path) || path.includes("//") || path.split("/").some(part => part === "." || part === "..")) return undefined; } catch { return undefined; } return value; } export function requestReproduction(url: string) { const requestPath = safeReproductionPath(url.split("?", 1)[0]); - return requestPath ? { requestPath, repoId: fileRequest.exec(requestPath)![1] } : {}; + return requestPath ? { requestPath, repoId: anonymousRequest.exec(requestPath)![1] } : {}; } diff --git a/src/core/request-monitoring.ts b/src/core/request-monitoring.ts index 1ebb11c..f1c0c78 100644 --- a/src/core/request-monitoring.ts +++ b/src/core/request-monitoring.ts @@ -158,7 +158,7 @@ export function monitorRequests(service: Service, metrics = requestMetrics, metrics.observe("request", route, method, ms, { bytes, aborted, error: res.statusCode >= 500, slow }); if (firstByteMs != null) metrics.observe("first_byte", route, method, firstByteMs); const responseMs = firstByteMs == null ? null : Math.max(0, ms - firstByteMs); - const detail = { ...(res.statusCode >= 400 ? reproduction : {}), requestId: id, service, method, route, status: res.statusCode, + const detail = { ...reproduction, requestId: id, service, method, route, status: res.statusCode, outcome: aborted ? "interrupted" : "completed", ms: Math.round(ms), firstByteMs: firstByteMs == null ? null : Math.round(firstByteMs), responseMs: responseMs == null ? null : Math.round(responseMs), bytesPerSecond: responseMs != null && responseMs >= 1 && bytes > 0 ? Math.round(bytes * 1000 / responseMs) : null, diff --git a/test/log-diagnostics.test.js b/test/log-diagnostics.test.js index 470fd87..918efc5 100644 --- a/test/log-diagnostics.test.js +++ b/test/log-diagnostics.test.js @@ -13,15 +13,28 @@ const secret='PRIVATE_TOKEN_owner_filename'; const frame='at printError (src/server/routes/route-utils.ts:200:7)'; function failure(){ const error=new TypeError('https://user:'+secret+'@github.com/private/repo?token='+secret); error.stack=error.message+'\n at printError ('+path.resolve('src/server/routes/route-utils.ts')+':200:7)\n at '+secret+' ('+path.resolve('src/server/routes/route-utils.ts')+':201:9)\n at secret (/home/private/'+secret+':1:2)\n at fn (https://private/'+secret+':1:2)';return error; } describe('safe actionable exception diagnostics',()=>{ - it('keeps bounded anonymous file request context for failures without queries or unsafe paths',()=>{ + it('keeps bounded anonymous request context without queries or unsafe paths',()=>{ const requestPath='/api/repo/demo-123/file/docs/reproduce%20issue.md'; expect(requestReproduction(requestPath+'?token=SECRET')).deep.equal({requestPath,repoId:'demo-123'}); for(const value of ['/api/repo/owner/name/readme','https://example.test'+requestPath,requestPath+'?token=SECRET','/api/repo/demo-123/file/%0aSECRET','/api/repo/demo-123/file/../secret','/api/repo/demo-123/file/%252e%252e/secret','/api/repo/demo-123/file/https%3a%2f%2fuser:token@example.test','/api/repo/demo-123/file/'+ 'x'.repeat(2048)]) expect(safeReproductionPath(value),value).equal(undefined); - expect(logContext.run({requestPath},()=>logRecord('info','route',['request',{status:200}]))).not.have.property('requestPath'); + expect(logContext.run({requestPath},()=>logRecord('info','route',['request',{status:200}]))).have.property('requestPath',requestPath); const record=logContext.run({requestPath},()=>logRecord('error','route',['failure',{httpStatus:500}])); expect(record).include({requestPath,repoId:'demo-123'}); expect(safeRedisLog({ts:record.timestamp,level:'error',module:'route',raw:[record]}).raw[0]).include({requestPath,repoId:'demo-123'}); }); + it('preserves anonymous endpoints at all outcomes through both API projections',async()=>{ + for(const suffix of ['', '/', '/zip', '/files', '/files/docs/nested%20folder/', '/files/counts', '/files/search', '/options', '/refresh', '/coauthors', '/extend', '/file/README.md']){ + const requestPath='/api/repo/demo-123'+suffix; + expect(requestReproduction(requestPath+'?token=SECRET')).deep.equal({requestPath,repoId:'demo-123'}); + for(const status of [200,304,404,500]){ + const row=logContext.run({requestPath},()=>logRecord('info','requests',['request',{status,outcome:'interrupted'}])); + expect(row).include({requestPath,repoId:'demo-123'}); + expect(safeRedisLog({ts:row.timestamp,level:'info',module:'requests',raw:[row]}).raw[0]).include({requestPath,repoId:'demo-123'}); + } + } + for(const suffix of ['/files/../secret','/files/%2fsecret','/files/%0aSECRET','/files/%252e%252e/secret','/zip?token=SECRET']) + expect(safeReproductionPath('/api/repo/demo-123'+suffix)).equal(undefined); + }); it('preserves dotted allowlisted source files and anonymous/named async frames',()=>{ const error=new Error(secret); error.stack='Error: '+secret+'\n at async '+path.resolve('src/server/routes/repository-public.ts')+':224:23\n at async printError ('+path.resolve('src/server/routes/route-utils.ts')+':200:7)\n at '+path.resolve('src/core/model/owner-alert-lifecycle.schema.ts')+':12:3\n at async https://'+secret+':1:2'; diff --git a/test/request-monitoring.test.js b/test/request-monitoring.test.js index bdeeaa5..46f6845 100644 --- a/test/request-monitoring.test.js +++ b/test/request-monitoring.test.js @@ -47,6 +47,14 @@ describe("bounded request monitoring", function () { expect(metrics.drain().every(row=>!JSON.stringify(row).includes('demo-123'))).equal(true); }); + it("records the actual listing path on successful requests while keeping metrics templated", async function () { + app.get('/api/repo/:repoId/files/*path',(_req,res)=>res.json([])); + await listen(); await fetch(server,'/api/repo/demo-123/files/docs/nested%20folder/?token=SECRET'); + expect(logs[0][1]).include({status:200,requestPath:'/api/repo/demo-123/files/docs/nested%20folder/',repoId:'demo-123'}); + expect(JSON.stringify(logs)).not.include('SECRET'); + expect(JSON.stringify(metrics.drain())).not.include('demo-123'); + }); + it("keeps labels bounded and removes source identities and query values", function () { expect(requestRoute("/api/repo/private-name/file/secret.txt?token=abc", "api")).to.equal("/api/repo/:repoId/file/:path"); expect(requestRoute("/api/repo/secret-owner/secret-repo/readme?branch=secret", "api")).to.equal("/api/repo/:owner/:repo/readme"); @@ -94,7 +102,8 @@ describe("bounded request monitoring", function () { expect(callbacks).to.equal(gzip ? 0 : body.length / 16384); const row = metrics.drain().find(row => row.data.metric === "request").data; expect(row.bytes).to.equal(result.body.length); expect(row.aborted).to.equal(0); expect(row.count).to.equal(1); - expect(JSON.stringify(logs)).not.to.include("private.txt"); expect(JSON.stringify(logs)).not.to.include("secret"); + expect(logs[0][1].requestPath).to.equal("/api/repo/private/file/private.txt"); + expect(JSON.stringify(row)).not.to.include("private.txt"); expect(JSON.stringify(logs)).not.to.include("secret"); expect(logs[0][1].requestId).to.equal(result.headers["x-request-id"]); expect(activeRequestCount()).to.equal(0); }); @@ -124,12 +133,14 @@ describe("bounded request monitoring", function () { it(`records a disconnect ${headersSent ? "after" : "before"} headers exactly once`, async function () { let arrived; const ready = new Promise(resolve => { arrived = resolve; }); let finished; const terminal = new Promise(resolve => { finished = resolve; }); - app.get("/", (_req, res) => { if (headersSent) { res.type("text/plain"); res.write("partial"); } arrived(); res.on("close", finished); }); + app.get("/api/repo/:repoId/zip", (_req, res) => { if (headersSent) { res.type("text/plain"); res.write("partial"); } arrived(); res.on("close", finished); }); await listen(); - const client = http.get({ host: "127.0.0.1", port: server.address().port, path: "/" }); client.on("error", () => {}); + const client = http.get({ host: "127.0.0.1", port: server.address().port, path: "/api/repo/demo-123/zip?token=SECRET" }); client.on("error", () => {}); await ready; client.destroy(); await terminal; expect(logs).to.have.length(1); expect(logs[0][0]).to.equal("request interrupted"); expect(logs[0][1].outcome).to.equal("interrupted"); + expect(logs[0][1]).include({requestPath:"/api/repo/demo-123/zip",repoId:"demo-123"}); + expect(JSON.stringify(logs)).not.include("SECRET"); const row = metrics.drain().find(row => row.data.metric === "request").data; expect(row.count).to.equal(1); expect(row.aborted).to.equal(1); expect(activeRequestCount()).to.equal(0); if (!headersSent) { From 7d044525ce6e0ddf893b526bc5421b882f73f283 Mon Sep 17 00:00:00 2001 From: Thomas Durieux <5577568+tdurieux@users.noreply.github.com> Date: Sun, 15 Nov 2026 00:08:12 +0000 Subject: [PATCH 2/3] fix: exclude source repository lookups from captured paths --- deploy/logging/README.md | 2 +- deploy/logging/sanitize.vrl | 2 +- scripts/test-admin-logs.js | 6 +++--- src/core/log-reproduction.ts | 4 +++- test/log-diagnostics.test.js | 16 +++++++++++++++- test/request-monitoring.test.js | 19 ++++++++++++++++--- 6 files changed, 39 insertions(+), 10 deletions(-) diff --git a/deploy/logging/README.md b/deploy/logging/README.md index 18c097d..8be8fb0 100644 --- a/deploy/logging/README.md +++ b/deploy/logging/README.md @@ -355,7 +355,7 @@ unknown fields. Existing discarded exception details cannot be recovered. ### Admin reproduction context -Application requests to the supported anonymous repository endpoints retain their public anonymous repository ID and encoded request pathname, up to 2 KiB, under the existing admin log access controls. This is an explicit exception to template-only diagnostics: repository-relative filenames are retained for reproduction. Query strings, headers, bodies and upstream owner/repository URLs are still excluded. Filenames are operator-visible data under this policy; no generic filter can guarantee that a user has not put sensitive text in a filename. Capture includes successful, failed and interrupted requests; metrics still use only normalized routes. Supported paths are repository metadata, file contents, listings/counts/search, ZIP, options, refresh, coauthors and extend under `/api/repo/:repoId`; other endpoints and unsafe, oversized, malformed or traversal paths omit reproduction context. Historical records cannot be backfilled. The UI copies the exact captured pathname without a host or query. +Application requests to the supported anonymous repository endpoints retain their public anonymous repository ID and encoded request pathname, up to 2 KiB, under the existing admin log access controls. This is an explicit exception to template-only diagnostics: repository-relative filenames are retained for reproduction. Query strings, headers, bodies and upstream owner/repository URLs are still excluded. Filenames are operator-visible data under this policy; no generic filter can guarantee that a user has not put sensitive text in a filename. Capture includes successful, failed and interrupted requests; metrics still use only normalized routes. Supported paths are repository metadata, file contents, listings/counts/search, ZIP, options, refresh, coauthors and extend under `/api/repo/:repoId`; other endpoints and unsafe, oversized, malformed or traversal paths omit reproduction context. Listing capture is limited to the actual `/files`, `/files/counts` and `/files/search` endpoints; folder/search query values are not retained. Source lookups such as `/api/repo/:owner/files/readme` are excluded. Historical records cannot be backfilled. The UI copies the exact captured pathname without a host or query. For the focused restricted-file/rotation regression, run `LOGGING_ACCESS_ONLY=1 python3 scripts/test-logging.py`. This uses synthetic `0640` logs in an isolated Docker project and preserves capability restrictions. diff --git a/deploy/logging/sanitize.vrl b/deploy/logging/sanitize.vrl index 2afb127..3046075 100644 --- a/deploy/logging/sanitize.vrl +++ b/deploy/logging/sanitize.vrl @@ -62,7 +62,7 @@ if includes(["/", "other", "/dashboard", "/anonymize", "/healthcheck", "/api/hea # Anonymous application request context at every outcome, never query strings. { path = to_string(record.requestPath) ?? "" - parsed, path_error = parse_regex(path, r'^/api/repo/(?P[A-Za-z0-9_-]{3,200})(?:/(?Pfile/[A-Za-z0-9._~!$&\'()+,;=@%/-]+|files(?:/[A-Za-z0-9._~!$&\'()+,;=@%/-]*)?|(?:zip|options|refresh|coauthors|extend)/?))?/?$') + parsed, path_error = parse_regex(path, r'^/api/repo/(?P[A-Za-z0-9_-]{3,200})(?:/(?Pfile/[A-Za-z0-9._~!$&\'()+,;=@%/-]+|files(?:/(?:counts|search))?/?|(?:zip|options|refresh|coauthors|extend)/?))?/?$') decoded = decode_percent(to_string(parsed.file)) if path_error == null && length(path) <= 2048 && !match(path, r'%(?:[^a-fA-F0-9]|[a-fA-F0-9][^a-fA-F0-9]|[a-fA-F0-9]?$)') && !match(decoded, r'[\x00-\x1f\x7f-\x9f\x{2028}-\x{202e}\x{2066}-\x{2069}\x{fffd}\\:?#%]|//|(^|/)\.\.?(/|$)') { out.requestPath = path diff --git a/scripts/test-admin-logs.js b/scripts/test-admin-logs.js index beaddf5..4439ce1 100644 --- a/scripts/test-admin-logs.js +++ b/scripts/test-admin-logs.js @@ -81,8 +81,8 @@ const sleep = ms => new Promise(resolve=>setTimeout(resolve,ms)); assert(projected.errorStack.includes('at (src/server/routes/repository-public.ts:224:23)')); assert.match(projected.errorStack,/at requestRoute \(src\/core\/request-monitoring.ts:\d+:\d+\)/); assert(!JSON.stringify(projected).includes(secret)); - const invalidPaths=['/api/repo/demo-123/file/a?token='+secret,'/api/repo/demo-123/file/%0a'+secret,'/api/repo/demo-123/file/../'+secret,'/api/repo/demo-123/file/%252e%252e/'+secret,'https://user:'+secret+'@example.test/file']; - const invalidSources=invalidPaths.map(requestPath=>({...rawSource,message:JSON.stringify({attrs:{tag:'api'},log:JSON.stringify({...diagnostic,requestPath,errorStack:'at (src/core/model/private.schema.ts:1:2)'})})})); + const invalidPaths=[...['readme','branches','tree','file'].flatMap(action=>['','/'].map(tail=>'/api/repo/private-owner/files/'+action+tail)),'/api/repo/demo-123/file/a?token='+secret,'/api/repo/demo-123/file/%0a'+secret,'/api/repo/demo-123/file/../'+secret,'/api/repo/demo-123/file/%252e%252e/'+secret,'https://user:'+secret+'@example.test/file']; + const invalidSources=invalidPaths.flatMap(requestPath=>[200,403,500].map(status=>({...rawSource,message:JSON.stringify({attrs:{tag:'api'},log:JSON.stringify({...diagnostic,status,httpStatus:status,severity:status===200?'info':'error',requestPath,errorStack:'at (src/core/model/private.schema.ts:1:2)'})})}))); const rejected=execFileSync('docker',['run','--rm','-i','--network','none','--memory','256m',vectorImage,'vrl','-i','/dev/stdin',fs.readFileSync('deploy/logging/sanitize.vrl','utf8')+'\nencode_json(.)'],{encoding:'utf8',input:invalidSources.map(JSON.stringify).join('\n')+'\n'}).trim().split('\n').map(line=>JSON.parse(JSON.parse(line))); assert(rejected.every(row=>!row.requestPath && !row.repoId && !row.errorStack)); assert(!JSON.stringify(rejected).includes(secret)); @@ -90,7 +90,7 @@ const sleep = ms => new Promise(resolve=>setTimeout(resolve,ms)); const traced=await investigateLogs(logOptions({kind:'requestId',id:diagnosticId},true),true); const redis=safeRedisLog({ts:diagnostic.timestamp,level:'error',module:'route',raw:[diagnostic]}); for(const row of [traced.events[0],redis.raw[0]]) {assert.equal(row.requestPath,diagnostic.requestPath);assert.equal(row.repoId,'demo-123');assert.equal(row.route,diagnostic.route);assert.equal(row.method,'GET');assert.equal(row.errorStack,projected.errorStack);assert.equal(row.causeType,'RangeError');assert(!JSON.stringify(row).includes(secret));} - const paths=['/api/repo/demo-123','/api/repo/demo-123/zip','/api/repo/demo-123/files/docs/nested%20folder/','/api/repo/demo-123/files/counts','/api/repo/demo-123/options','/api/repo/demo-123/file/README.md']; + const paths=['/api/repo/demo-123','/api/repo/demo-123/zip','/api/repo/demo-123/files','/api/repo/demo-123/files/counts','/api/repo/demo-123/options','/api/repo/demo-123/file/README.md']; const pathRecords=paths.map((requestPath,i)=>logContext.run({requestId:randomUUID(),requestPath},()=>logRecord(i%2?'warn':'info','requests',['request',{status:200,outcome:i%2?'interrupted':'completed'}]))); const pathSources=pathRecords.map(record=>({...rawSource,message:JSON.stringify({attrs:{tag:'api'},log:JSON.stringify(record)})})); const pathOutput=execFileSync('docker',['run','--rm','-i','--network','none','--memory','256m',vectorImage,'vrl','-i','/dev/stdin',fs.readFileSync('deploy/logging/sanitize.vrl','utf8')+'\nencode_json(.)'],{encoding:'utf8',input:pathSources.map(JSON.stringify).join('\n')+'\n'}).trim().split('\n').map(line=>JSON.parse(JSON.parse(line))); diff --git a/src/core/log-reproduction.ts b/src/core/log-reproduction.ts index 649057d..9e18d4b 100644 --- a/src/core/log-reproduction.ts +++ b/src/core/log-reproduction.ts @@ -1,6 +1,8 @@ // Explicit admin diagnostics: public anonymous repository ID and request pathname only. // Never collect upstream owner/repository URLs, headers, query strings or bodies. -const anonymousRequest = /^\/api\/repo\/([A-Za-z0-9_-]{3,200})(?:\/(file\/[A-Za-z0-9._~!$&'()+,;=@%/-]+|files(?:\/[A-Za-z0-9._~!$&'()+,;=@%/-]*)?|(?:zip|options|refresh|coauthors|extend)\/?))?\/?$/; +// Listing folders use query parameters; a wildcard here would capture upstream +// /:owner/files/{readme,branches,tree,file} source lookups. +const anonymousRequest = /^\/api\/repo\/([A-Za-z0-9_-]{3,200})(?:\/(file\/[A-Za-z0-9._~!$&'()+,;=@%/-]+|files(?:\/(?:counts|search))?\/?|(?:zip|options|refresh|coauthors|extend)\/?))?\/?$/; export function safeReproductionPath(value: unknown): string | undefined { if (typeof value !== "string" || value.length > 2048) return undefined; const match = anonymousRequest.exec(value); diff --git a/test/log-diagnostics.test.js b/test/log-diagnostics.test.js index 918efc5..866e63a 100644 --- a/test/log-diagnostics.test.js +++ b/test/log-diagnostics.test.js @@ -23,7 +23,7 @@ describe('safe actionable exception diagnostics',()=>{ expect(safeRedisLog({ts:record.timestamp,level:'error',module:'route',raw:[record]}).raw[0]).include({requestPath,repoId:'demo-123'}); }); it('preserves anonymous endpoints at all outcomes through both API projections',async()=>{ - for(const suffix of ['', '/', '/zip', '/files', '/files/docs/nested%20folder/', '/files/counts', '/files/search', '/options', '/refresh', '/coauthors', '/extend', '/file/README.md']){ + for(const suffix of ['', '/', '/zip', '/files', '/files/counts', '/files/search', '/options', '/refresh', '/coauthors', '/extend', '/file/README.md']){ const requestPath='/api/repo/demo-123'+suffix; expect(requestReproduction(requestPath+'?token=SECRET')).deep.equal({requestPath,repoId:'demo-123'}); for(const status of [200,304,404,500]){ @@ -35,6 +35,20 @@ describe('safe actionable exception diagnostics',()=>{ for(const suffix of ['/files/../secret','/files/%2fsecret','/files/%0aSECRET','/files/%252e%252e/secret','/zip?token=SECRET']) expect(safeReproductionPath('/api/repo/demo-123'+suffix)).equal(undefined); }); + it('rejects upstream lookups for repositories named files at every logging boundary',()=>{ + for(const action of ['readme','branches','tree','file']){ + const requestPath='/api/repo/private-owner/files/'+action; + for(const suffix of ['', '/', '?token=SECRET']){ + expect(requestReproduction(requestPath+suffix)).deep.equal({}); + for(const status of [200,403,500]){ + const row=logContext.run({requestPath:requestPath+suffix},()=>logRecord('error','route',['failure',{httpStatus:status}])); + expect(row).not.have.property('requestPath');expect(row).not.have.property('repoId'); + const redis=safeRedisLog({ts:row.timestamp,level:'error',module:'route',raw:[{...row,requestPath:requestPath+suffix,repoId:'private-owner'}]}); + expect(redis.raw[0]).not.have.property('requestPath');expect(redis.raw[0]).not.have.property('repoId'); + } + } + } + }); it('preserves dotted allowlisted source files and anonymous/named async frames',()=>{ const error=new Error(secret); error.stack='Error: '+secret+'\n at async '+path.resolve('src/server/routes/repository-public.ts')+':224:23\n at async printError ('+path.resolve('src/server/routes/route-utils.ts')+':200:7)\n at '+path.resolve('src/core/model/owner-alert-lifecycle.schema.ts')+':12:3\n at async https://'+secret+':1:2'; diff --git a/test/request-monitoring.test.js b/test/request-monitoring.test.js index 46f6845..a24fb6d 100644 --- a/test/request-monitoring.test.js +++ b/test/request-monitoring.test.js @@ -48,13 +48,26 @@ describe("bounded request monitoring", function () { }); it("records the actual listing path on successful requests while keeping metrics templated", async function () { - app.get('/api/repo/:repoId/files/*path',(_req,res)=>res.json([])); - await listen(); await fetch(server,'/api/repo/demo-123/files/docs/nested%20folder/?token=SECRET'); - expect(logs[0][1]).include({status:200,requestPath:'/api/repo/demo-123/files/docs/nested%20folder/',repoId:'demo-123'}); + app.get('/api/repo/:repoId/files',(_req,res)=>res.json([])); + await listen(); await fetch(server,'/api/repo/demo-123/files?path=docs%2Fnested&token=SECRET'); + expect(logs[0][1]).include({status:200,requestPath:'/api/repo/demo-123/files',repoId:'demo-123'}); expect(JSON.stringify(logs)).not.include('SECRET'); expect(JSON.stringify(metrics.drain())).not.include('demo-123'); }); + it("never captures upstream identities from files-named source repository handlers", async function () { + const {logRecord}=require('../src/core/log-record'); + const records=[]; + for(const action of ['readme','branches','tree','file']) + app.get('/api/repo/:owner/:repo/'+action,(_req,res)=>{ + records.push(logRecord('error','route',['failure',{httpStatus:403}]));res.status(403).end(); + }); + await listen(); + for(const action of ['readme','branches','tree','file']) + await fetch(server,'/api/repo/private-owner/files/'+action); + expect(JSON.stringify([...records,...logs])).not.include('private-owner'); + }); + it("keeps labels bounded and removes source identities and query values", function () { expect(requestRoute("/api/repo/private-name/file/secret.txt?token=abc", "api")).to.equal("/api/repo/:repoId/file/:path"); expect(requestRoute("/api/repo/secret-owner/secret-repo/readme?branch=secret", "api")).to.equal("/api/repo/:owner/:repo/readme"); From 1723f5e927725ea8aad20223894f83569f7f017d Mon Sep 17 00:00:00 2001 From: Thomas Durieux <5577568+tdurieux@users.noreply.github.com> Date: Sun, 15 Nov 2026 00:14:01 +0000 Subject: [PATCH 3/3] fix: require anonymous endpoint methods for logged paths --- deploy/logging/README.md | 2 +- deploy/logging/sanitize.vrl | 4 +++- scripts/test-admin-logs.js | 6 +++--- src/core/log-record.ts | 4 ++-- src/core/log-reproduction.ts | 11 +++++++--- src/core/request-monitoring.ts | 2 +- test/log-diagnostics.test.js | 36 ++++++++++++++++++++++++--------- test/request-monitoring.test.js | 22 ++++++++++++++++++++ test/structured-logging.test.js | 2 +- 9 files changed, 67 insertions(+), 22 deletions(-) diff --git a/deploy/logging/README.md b/deploy/logging/README.md index 8be8fb0..b427208 100644 --- a/deploy/logging/README.md +++ b/deploy/logging/README.md @@ -355,7 +355,7 @@ unknown fields. Existing discarded exception details cannot be recovered. ### Admin reproduction context -Application requests to the supported anonymous repository endpoints retain their public anonymous repository ID and encoded request pathname, up to 2 KiB, under the existing admin log access controls. This is an explicit exception to template-only diagnostics: repository-relative filenames are retained for reproduction. Query strings, headers, bodies and upstream owner/repository URLs are still excluded. Filenames are operator-visible data under this policy; no generic filter can guarantee that a user has not put sensitive text in a filename. Capture includes successful, failed and interrupted requests; metrics still use only normalized routes. Supported paths are repository metadata, file contents, listings/counts/search, ZIP, options, refresh, coauthors and extend under `/api/repo/:repoId`; other endpoints and unsafe, oversized, malformed or traversal paths omit reproduction context. Listing capture is limited to the actual `/files`, `/files/counts` and `/files/search` endpoints; folder/search query values are not retained. Source lookups such as `/api/repo/:owner/files/readme` are excluded. Historical records cannot be backfilled. The UI copies the exact captured pathname without a host or query. +Application requests to the supported anonymous repository endpoints retain their public anonymous repository ID and encoded request pathname, up to 2 KiB, under the existing admin log access controls. This is an explicit exception to template-only diagnostics: repository-relative filenames are retained for reproduction. Query strings, headers, bodies and upstream owner/repository URLs are still excluded. Filenames are operator-visible data under this policy; no generic filter can guarantee that a user has not put sensitive text in a filename. Capture includes successful, failed and interrupted requests; metrics still use only normalized routes. Supported paths are repository metadata, file contents, listings/counts/search, ZIP, options, refresh, coauthors and extend under `/api/repo/:repoId`; other endpoints and unsafe, oversized, malformed or traversal paths omit reproduction context. Listing capture is limited to the actual `/files`, `/files/counts` and `/files/search` endpoints; folder/search query values are not retained. Action paths refresh, extend and coauthors require POST; GET/HEAD source metadata lookups using those repository names are excluded. Missing or unsupported methods omit reproduction context. Source lookups such as `/api/repo/:owner/files/readme` are excluded. Historical records cannot be backfilled. The UI copies the exact captured pathname without a host or query. For the focused restricted-file/rotation regression, run `LOGGING_ACCESS_ONLY=1 python3 scripts/test-logging.py`. This uses synthetic `0640` logs in an isolated Docker project and preserves capability restrictions. diff --git a/deploy/logging/sanitize.vrl b/deploy/logging/sanitize.vrl index 3046075..a19442a 100644 --- a/deploy/logging/sanitize.vrl +++ b/deploy/logging/sanitize.vrl @@ -63,8 +63,10 @@ if includes(["/", "other", "/dashboard", "/anonymize", "/healthcheck", "/api/hea { path = to_string(record.requestPath) ?? "" parsed, path_error = parse_regex(path, r'^/api/repo/(?P[A-Za-z0-9_-]{3,200})(?:/(?Pfile/[A-Za-z0-9._~!$&\'()+,;=@%/-]+|files(?:/(?:counts|search))?/?|(?:zip|options|refresh|coauthors|extend)/?))?/?$') + action = replace(to_string(parsed.file), r'/+$', "") + allowed_methods = if action == "" { ["GET", "HEAD", "POST", "DELETE"] } else if includes(["refresh", "extend", "coauthors"], action) { ["POST"] } else { ["GET", "HEAD"] } decoded = decode_percent(to_string(parsed.file)) - if path_error == null && length(path) <= 2048 && !match(path, r'%(?:[^a-fA-F0-9]|[a-fA-F0-9][^a-fA-F0-9]|[a-fA-F0-9]?$)') && !match(decoded, r'[\x00-\x1f\x7f-\x9f\x{2028}-\x{202e}\x{2066}-\x{2069}\x{fffd}\\:?#%]|//|(^|/)\.\.?(/|$)') { + if path_error == null && includes(allowed_methods, out.method) && length(path) <= 2048 && !match(path, r'%(?:[^a-fA-F0-9]|[a-fA-F0-9][^a-fA-F0-9]|[a-fA-F0-9]?$)') && !match(decoded, r'[\x00-\x1f\x7f-\x9f\x{2028}-\x{202e}\x{2066}-\x{2069}\x{fffd}\\:?#%]|//|(^|/)\.\.?(/|$)') { out.requestPath = path out.repoId = parsed.repo } diff --git a/scripts/test-admin-logs.js b/scripts/test-admin-logs.js index 4439ce1..4b0871d 100644 --- a/scripts/test-admin-logs.js +++ b/scripts/test-admin-logs.js @@ -81,7 +81,7 @@ const sleep = ms => new Promise(resolve=>setTimeout(resolve,ms)); assert(projected.errorStack.includes('at (src/server/routes/repository-public.ts:224:23)')); assert.match(projected.errorStack,/at requestRoute \(src\/core\/request-monitoring.ts:\d+:\d+\)/); assert(!JSON.stringify(projected).includes(secret)); - const invalidPaths=[...['readme','branches','tree','file'].flatMap(action=>['','/'].map(tail=>'/api/repo/private-owner/files/'+action+tail)),'/api/repo/demo-123/file/a?token='+secret,'/api/repo/demo-123/file/%0a'+secret,'/api/repo/demo-123/file/../'+secret,'/api/repo/demo-123/file/%252e%252e/'+secret,'https://user:'+secret+'@example.test/file']; + const invalidPaths=[...['refresh','extend','coauthors'].flatMap(action=>['','/'].map(tail=>'/api/repo/private-owner/'+action+tail)),...['readme','branches','tree','file'].flatMap(action=>['','/'].map(tail=>'/api/repo/private-owner/files/'+action+tail)),'/api/repo/demo-123/file/a?token='+secret,'/api/repo/demo-123/file/%0a'+secret,'/api/repo/demo-123/file/../'+secret,'/api/repo/demo-123/file/%252e%252e/'+secret,'https://user:'+secret+'@example.test/file']; const invalidSources=invalidPaths.flatMap(requestPath=>[200,403,500].map(status=>({...rawSource,message:JSON.stringify({attrs:{tag:'api'},log:JSON.stringify({...diagnostic,status,httpStatus:status,severity:status===200?'info':'error',requestPath,errorStack:'at (src/core/model/private.schema.ts:1:2)'})})}))); const rejected=execFileSync('docker',['run','--rm','-i','--network','none','--memory','256m',vectorImage,'vrl','-i','/dev/stdin',fs.readFileSync('deploy/logging/sanitize.vrl','utf8')+'\nencode_json(.)'],{encoding:'utf8',input:invalidSources.map(JSON.stringify).join('\n')+'\n'}).trim().split('\n').map(line=>JSON.parse(JSON.parse(line))); assert(rejected.every(row=>!row.requestPath && !row.repoId && !row.errorStack)); @@ -90,8 +90,8 @@ const sleep = ms => new Promise(resolve=>setTimeout(resolve,ms)); const traced=await investigateLogs(logOptions({kind:'requestId',id:diagnosticId},true),true); const redis=safeRedisLog({ts:diagnostic.timestamp,level:'error',module:'route',raw:[diagnostic]}); for(const row of [traced.events[0],redis.raw[0]]) {assert.equal(row.requestPath,diagnostic.requestPath);assert.equal(row.repoId,'demo-123');assert.equal(row.route,diagnostic.route);assert.equal(row.method,'GET');assert.equal(row.errorStack,projected.errorStack);assert.equal(row.causeType,'RangeError');assert(!JSON.stringify(row).includes(secret));} - const paths=['/api/repo/demo-123','/api/repo/demo-123/zip','/api/repo/demo-123/files','/api/repo/demo-123/files/counts','/api/repo/demo-123/options','/api/repo/demo-123/file/README.md']; - const pathRecords=paths.map((requestPath,i)=>logContext.run({requestId:randomUUID(),requestPath},()=>logRecord(i%2?'warn':'info','requests',['request',{status:200,outcome:i%2?'interrupted':'completed'}]))); + const paths=['/api/repo/demo-123/refresh','/api/repo/demo-123/extend','/api/repo/demo-123/coauthors','/api/repo/demo-123','/api/repo/demo-123/zip','/api/repo/demo-123/files','/api/repo/demo-123/files/counts','/api/repo/demo-123/options','/api/repo/demo-123/file/README.md']; + const pathRecords=paths.map((requestPath,i)=>logContext.run({requestId:randomUUID(),requestPath,method:/\/(refresh|extend|coauthors)$/.test(requestPath)?'POST':'GET'},()=>logRecord(i%2?'warn':'info','requests',['request',{status:200,outcome:i%2?'interrupted':'completed'}]))); const pathSources=pathRecords.map(record=>({...rawSource,message:JSON.stringify({attrs:{tag:'api'},log:JSON.stringify(record)})})); const pathOutput=execFileSync('docker',['run','--rm','-i','--network','none','--memory','256m',vectorImage,'vrl','-i','/dev/stdin',fs.readFileSync('deploy/logging/sanitize.vrl','utf8')+'\nencode_json(.)'],{encoding:'utf8',input:pathSources.map(JSON.stringify).join('\n')+'\n'}).trim().split('\n').map(line=>JSON.parse(JSON.parse(line))); assert.deepEqual(pathOutput.map(x=>x.requestPath),paths); diff --git a/src/core/log-record.ts b/src/core/log-record.ts index d537ca6..60facdc 100644 --- a/src/core/log-record.ts +++ b/src/core/log-record.ts @@ -72,8 +72,8 @@ export function logRecord(level: string, module: string, args: unknown[]) { if (context.method && allowed(["GET", "HEAD", "POST", "PUT", "PATCH", "DELETE", "OPTIONS", "OTHER"], context.method)) record.method = context.method; Object.assign(record, safeDiagnosticFields(details)); // Anonymous request paths are diagnostic fields, never metric labels. - const requestPath = safeReproductionPath(context.requestPath || details.requestPath); - if (requestPath) Object.assign(record, requestReproduction(requestPath)); + const requestPath = safeReproductionPath(context.requestPath || details.requestPath, record.method); + if (requestPath) Object.assign(record, requestReproduction(requestPath, record.method)); if (allowed(["download", "remove", "cache"], details.queue)) record.queue = details.queue; if (details.stages && typeof details.stages === "object") for (const key of stages) { let value: unknown; diff --git a/src/core/log-reproduction.ts b/src/core/log-reproduction.ts index 9e18d4b..7813ef6 100644 --- a/src/core/log-reproduction.ts +++ b/src/core/log-reproduction.ts @@ -3,10 +3,15 @@ // Listing folders use query parameters; a wildcard here would capture upstream // /:owner/files/{readme,branches,tree,file} source lookups. const anonymousRequest = /^\/api\/repo\/([A-Za-z0-9_-]{3,200})(?:\/(file\/[A-Za-z0-9._~!$&'()+,;=@%/-]+|files(?:\/(?:counts|search))?\/?|(?:zip|options|refresh|coauthors|extend)\/?))?\/?$/; -export function safeReproductionPath(value: unknown): string | undefined { +export function safeReproductionPath(value: unknown, method: unknown): string | undefined { if (typeof value !== "string" || value.length > 2048) return undefined; const match = anonymousRequest.exec(value); if (!match || /%(?![a-fA-F0-9]{2})/.test(value)) return undefined; + const action = (match[2] || "").replace(/\/+$/, ""); + const allowedMethods = !action ? ["GET", "HEAD", "POST", "DELETE"] + : ["refresh", "extend", "coauthors"].includes(action) ? ["POST"] : ["GET", "HEAD"]; + // GET coauthors is shadowed by the earlier source metadata handler. + if (typeof method !== "string" || !allowedMethods.includes(method)) return undefined; try { const path = decodeURIComponent(match[2] || ""); // Reject controls, traversal, URL syntax, encoded delimiters and double encoding. @@ -15,7 +20,7 @@ export function safeReproductionPath(value: unknown): string | undefined { } catch { return undefined; } return value; } -export function requestReproduction(url: string) { - const requestPath = safeReproductionPath(url.split("?", 1)[0]); +export function requestReproduction(url: string, method: unknown) { + const requestPath = safeReproductionPath(url.split("?", 1)[0], method); return requestPath ? { requestPath, repoId: anonymousRequest.exec(requestPath)![1] } : {}; } diff --git a/src/core/request-monitoring.ts b/src/core/request-monitoring.ts index f1c0c78..5d4510e 100644 --- a/src/core/request-monitoring.ts +++ b/src/core/request-monitoring.ts @@ -117,7 +117,7 @@ export function monitorRequests(service: Service, metrics = requestMetrics, : req.originalUrl.startsWith("/w/") ? "website" : "file"; const trace = { id, service, traffic, stages: {} }; const route = requestRoute(req.originalUrl, service); - const reproduction = service === "api" ? requestReproduction(req.originalUrl) : {}; + const reproduction = service === "api" ? requestReproduction(req.originalUrl, req.method) : {}; const method = methods.has(req.method) ? req.method : "OTHER"; const started = performance.now(); let firstByteMs: number | undefined; diff --git a/test/log-diagnostics.test.js b/test/log-diagnostics.test.js index 866e63a..0e36b73 100644 --- a/test/log-diagnostics.test.js +++ b/test/log-diagnostics.test.js @@ -14,34 +14,35 @@ const frame='at printError (src/server/routes/route-utils.ts:200:7)'; function failure(){ const error=new TypeError('https://user:'+secret+'@github.com/private/repo?token='+secret); error.stack=error.message+'\n at printError ('+path.resolve('src/server/routes/route-utils.ts')+':200:7)\n at '+secret+' ('+path.resolve('src/server/routes/route-utils.ts')+':201:9)\n at secret (/home/private/'+secret+':1:2)\n at fn (https://private/'+secret+':1:2)';return error; } describe('safe actionable exception diagnostics',()=>{ it('keeps bounded anonymous request context without queries or unsafe paths',()=>{ - const requestPath='/api/repo/demo-123/file/docs/reproduce%20issue.md'; - expect(requestReproduction(requestPath+'?token=SECRET')).deep.equal({requestPath,repoId:'demo-123'}); - for(const value of ['/api/repo/owner/name/readme','https://example.test'+requestPath,requestPath+'?token=SECRET','/api/repo/demo-123/file/%0aSECRET','/api/repo/demo-123/file/../secret','/api/repo/demo-123/file/%252e%252e/secret','/api/repo/demo-123/file/https%3a%2f%2fuser:token@example.test','/api/repo/demo-123/file/'+ 'x'.repeat(2048)]) expect(safeReproductionPath(value),value).equal(undefined); - expect(logContext.run({requestPath},()=>logRecord('info','route',['request',{status:200}]))).have.property('requestPath',requestPath); - const record=logContext.run({requestPath},()=>logRecord('error','route',['failure',{httpStatus:500}])); + const method='GET', requestPath='/api/repo/demo-123/file/docs/reproduce%20issue.md'; + expect(requestReproduction(requestPath+'?token=SECRET',method)).deep.equal({requestPath,repoId:'demo-123'}); + for(const value of ['/api/repo/owner/name/readme','https://example.test'+requestPath,requestPath+'?token=SECRET','/api/repo/demo-123/file/%0aSECRET','/api/repo/demo-123/file/../secret','/api/repo/demo-123/file/%252e%252e/secret','/api/repo/demo-123/file/https%3a%2f%2fuser:token@example.test','/api/repo/demo-123/file/'+ 'x'.repeat(2048)]) expect(safeReproductionPath(value,"GET"),value).equal(undefined); + expect(logContext.run({requestPath,method},()=>logRecord('info','route',['request',{status:200}]))).have.property('requestPath',requestPath); + const record=logContext.run({requestPath,method},()=>logRecord('error','route',['failure',{httpStatus:500}])); expect(record).include({requestPath,repoId:'demo-123'}); expect(safeRedisLog({ts:record.timestamp,level:'error',module:'route',raw:[record]}).raw[0]).include({requestPath,repoId:'demo-123'}); }); it('preserves anonymous endpoints at all outcomes through both API projections',async()=>{ for(const suffix of ['', '/', '/zip', '/files', '/files/counts', '/files/search', '/options', '/refresh', '/coauthors', '/extend', '/file/README.md']){ + const method=['/refresh','/extend','/coauthors'].includes(suffix)?'POST':'GET'; const requestPath='/api/repo/demo-123'+suffix; - expect(requestReproduction(requestPath+'?token=SECRET')).deep.equal({requestPath,repoId:'demo-123'}); + expect(requestReproduction(requestPath+'?token=SECRET',method)).deep.equal({requestPath,repoId:'demo-123'}); for(const status of [200,304,404,500]){ - const row=logContext.run({requestPath},()=>logRecord('info','requests',['request',{status,outcome:'interrupted'}])); + const row=logContext.run({requestPath,method},()=>logRecord('info','requests',['request',{status,outcome:'interrupted'}])); expect(row).include({requestPath,repoId:'demo-123'}); expect(safeRedisLog({ts:row.timestamp,level:'info',module:'requests',raw:[row]}).raw[0]).include({requestPath,repoId:'demo-123'}); } } for(const suffix of ['/files/../secret','/files/%2fsecret','/files/%0aSECRET','/files/%252e%252e/secret','/zip?token=SECRET']) - expect(safeReproductionPath('/api/repo/demo-123'+suffix)).equal(undefined); + expect(safeReproductionPath('/api/repo/demo-123'+suffix,'GET')).equal(undefined); }); it('rejects upstream lookups for repositories named files at every logging boundary',()=>{ for(const action of ['readme','branches','tree','file']){ const requestPath='/api/repo/private-owner/files/'+action; for(const suffix of ['', '/', '?token=SECRET']){ - expect(requestReproduction(requestPath+suffix)).deep.equal({}); + expect(requestReproduction(requestPath+suffix,"GET")).deep.equal({}); for(const status of [200,403,500]){ - const row=logContext.run({requestPath:requestPath+suffix},()=>logRecord('error','route',['failure',{httpStatus:status}])); + const row=logContext.run({requestPath:requestPath+suffix,method:"GET"},()=>logRecord('error','route',['failure',{httpStatus:status}])); expect(row).not.have.property('requestPath');expect(row).not.have.property('repoId'); const redis=safeRedisLog({ts:row.timestamp,level:'error',module:'route',raw:[{...row,requestPath:requestPath+suffix,repoId:'private-owner'}]}); expect(redis.raw[0]).not.have.property('requestPath');expect(redis.raw[0]).not.have.property('repoId'); @@ -49,6 +50,21 @@ describe('safe actionable exception diagnostics',()=>{ } } }); + it('requires the anonymous action method and omits ambiguous GET source metadata',()=>{ + for(const action of ['refresh','extend','coauthors']){ + for(const suffix of ['', '/']){ + const requestPath='/api/repo/private-owner/'+action+suffix; + for(const method of ['GET','HEAD',undefined,'OPTIONS']){ + expect(safeReproductionPath(requestPath,method)).equal(undefined); + const row=logContext.run({requestPath,method},()=>logRecord('error','route',['failure',{httpStatus:403}])); + expect(row).not.have.property('requestPath'); + const redis=safeRedisLog({ts:row.timestamp,level:'error',module:'route',raw:[{...row,requestPath,repoId:'private-owner'}]}); + expect(redis.raw[0]).not.have.property('requestPath'); + } + expect(requestReproduction(requestPath,'POST')).deep.equal({requestPath,repoId:'private-owner'}); + } + } + }); it('preserves dotted allowlisted source files and anonymous/named async frames',()=>{ const error=new Error(secret); error.stack='Error: '+secret+'\n at async '+path.resolve('src/server/routes/repository-public.ts')+':224:23\n at async printError ('+path.resolve('src/server/routes/route-utils.ts')+':200:7)\n at '+path.resolve('src/core/model/owner-alert-lifecycle.schema.ts')+':12:3\n at async https://'+secret+':1:2'; diff --git a/test/request-monitoring.test.js b/test/request-monitoring.test.js index a24fb6d..4228587 100644 --- a/test/request-monitoring.test.js +++ b/test/request-monitoring.test.js @@ -68,6 +68,28 @@ describe("bounded request monitoring", function () { expect(JSON.stringify([...records,...logs])).not.include('private-owner'); }); + it("distinguishes anonymous POST actions from GET source metadata at the same URL", async function () { + const {logRecord}=require('../src/core/log-record'); + const records=[]; + for(const action of ['refresh','extend','coauthors']) + app.post('/api/repo/:repoId/'+action,(_req,res)=>{ + records.push(logRecord('info','route',['request',{status:200}]));res.end(); + }); + app.get('/api/repo/:owner/:repo/',(_req,res)=>{ + records.push(logRecord('error','route',['failure',{httpStatus:403}]));res.status(403).end(); + }); + await listen(); + for(const action of ['refresh','extend','coauthors']){ + for(const method of ['GET','HEAD','POST']){ + await fetch(server,'/api/repo/private-owner/'+action+'/?token=SECRET',{method}); + const detail=logs.at(-1)[1]; + if(method==='POST')expect(detail.requestPath).equal('/api/repo/private-owner/'+action+'/'); + else {expect(detail).not.have.property('requestPath');expect(records.at(-1)).not.have.property('requestPath');} + } + } + expect(JSON.stringify(logs)).not.include('SECRET'); + }); + it("keeps labels bounded and removes source identities and query values", function () { expect(requestRoute("/api/repo/private-name/file/secret.txt?token=abc", "api")).to.equal("/api/repo/:repoId/file/:path"); expect(requestRoute("/api/repo/secret-owner/secret-repo/readme?branch=secret", "api")).to.equal("/api/repo/:owner/:repo/readme"); diff --git a/test/structured-logging.test.js b/test/structured-logging.test.js index 5004a9f..b521d79 100644 --- a/test/structured-logging.test.js +++ b/test/structured-logging.test.js @@ -106,7 +106,7 @@ describe('structured logging privacy and context', function () { const requestPath='/api/repo/demo-123/file/'+ 'a'.repeat(1950); const frame='at (src/core/model/owner-alert-lifecycle.schema.ts:123456:12345)'; const stack=Array(8).fill(frame).join('\\n'); - require('./src/core/logger').createLogger('route').error('failure',{...Object.fromEntries(['ms','firstByteMs','responseMs','bytes','bytesPerSecond','attempt','rss','memoryLimitBytes','memoryRatio','uptime','activeRequests','waiting','active','delayed','failed','oldestWaitingMs','droppedLogs','loopP95Ms','upstreamStatus'].map(key=>[key,Number.MAX_VALUE])),requestPath,errorType:'TypeError',errorStack:stack,causeType:'Error',causeStack:stack,httpStatus:500}); + require('./src/core/logger').createLogger('route').error('failure',{...Object.fromEntries(['ms','firstByteMs','responseMs','bytes','bytesPerSecond','attempt','rss','memoryLimitBytes','memoryRatio','uptime','activeRequests','waiting','active','delayed','failed','oldestWaitingMs','droppedLogs','loopP95Ms','upstreamStatus'].map(key=>[key,Number.MAX_VALUE])),requestPath,method:'GET',errorType:'TypeError',errorStack:stack,causeType:'Error',causeStack:stack,httpStatus:500}); assert(payload.length<=4096);const row=JSON.parse(payload).raw[0];assert.equal(row.requestPath,requestPath);assert.equal(row.repoId,'demo-123');assert.equal(row.errorStack,frame); `],{env:{...process.env,LOG_REDIS_MIRROR:'on'},encoding:'utf8',timeout:10000}); expect(child.status,child.stderr).equal(0);