fix: retain actual anonymous request paths across outcomes

This commit is contained in:
Thomas Durieux committed 2026-11-15 00:01:21 +00:00
1 parent b54cf72bf8
commit 953767b2cd
8 files changed
+57 -22

No files matched your search

+2 -2
View File
@@ -341,7 +341,7 @@ a finite error type/code, and up to eight recognized application frames (1 KiB m
exception and the deepest inspected cause (at most three levels, cycles stopped). Raw messages,
Upstream URLs, query strings, external frames and absolute filesystem roots are
not stored. Repository/file parameters are excluded except for the explicitly
scoped failed-request reproduction context below. Unknown error types use `UnknownError`; unknown
scoped anonymous-request reproduction context below. Unknown error types use `UnknownError`; unknown
function names become `<anonymous>`. Source locations are build-relative for
compiled code and source-relative in development; line numbers refer to that
release, not an inferred source map.
@@ -355,7 +355,7 @@ unknown fields. Existing discarded exception details cannot be recovered.
### Admin reproduction context
Failed application file requests now retain their public anonymous repository ID and encoded request pathname, up to 2 KiB, under the existing admin log access controls. This is an explicit exception to template-only diagnostics: repository-relative filenames are retained for reproduction. Query strings, headers, bodies and upstream owner/repository URLs are still excluded. Filenames are operator-visible data under this policy; no generic filter can guarantee that a user has not put sensitive text in a filename. Successful requests and metrics keep only the normalized route. Only `/api/repo/:repoId/file/:path` is supported; other endpoints and unsafe, oversized, malformed or traversal paths omit reproduction context. Historical records cannot be backfilled. The UI copies the exact captured pathname without a host or query.
Application requests to the supported anonymous repository endpoints retain their public anonymous repository ID and encoded request pathname, up to 2 KiB, under the existing admin log access controls. This is an explicit exception to template-only diagnostics: repository-relative filenames are retained for reproduction. Query strings, headers, bodies and upstream owner/repository URLs are still excluded. Filenames are operator-visible data under this policy; no generic filter can guarantee that a user has not put sensitive text in a filename. Capture includes successful, failed and interrupted requests; metrics still use only normalized routes. Supported paths are repository metadata, file contents, listings/counts/search, ZIP, options, refresh, coauthors and extend under `/api/repo/:repoId`; other endpoints and unsafe, oversized, malformed or traversal paths omit reproduction context. Historical records cannot be backfilled. The UI copies the exact captured pathname without a host or query.
For the focused restricted-file/rotation regression, run
`LOGGING_ACCESS_ONLY=1 python3 scripts/test-logging.py`. This uses synthetic
`0640` logs in an isolated Docker project and preserves capability restrictions.
+4 -4
View File
@@ -59,12 +59,12 @@ for_each(["status", "httpStatus", "ms", "firstByteMs", "responseMs", "bytes", "b
if includes(["GET", "HEAD", "POST", "PUT", "PATCH", "DELETE", "OPTIONS", "OTHER"], record.method) { out.method = record.method }
if includes(["/", "other", "/dashboard", "/anonymize", "/healthcheck", "/api/healthcheck", "/api", "/api/download", "/api/stat", "/api/stat/history", "/api/options", "/api/message", "/api/user", "/api/user/quota", "/api/user/dashboard", "/api/repo", "/api/anonymize-preview", "/api/admin/overview", "/api/admin/performance", "/api/admin/errors", "/api/admin/errors/stats", "/api/admin/queues", "/api/admin/queues/metrics", "/github/app/webhook", "/api/repo/:repoId", "/api/repo/:repoId/file/:path", "/api/repo/:repoId/files", "/api/repo/:repoId/files/counts", "/api/repo/:repoId/files/search", "/api/repo/:repoId/zip", "/api/repo/:repoId/options", "/api/repo/:repoId/refresh", "/api/repo/:repoId/coauthors", "/api/repo/:repoId/extend", "/api/repo/:owner/:repo/readme", "/api/repo/:owner/:repo/branches", "/w/:repoId/:path", "/r/:repoId/:path", "/api/pr/:id/:path", "/api/gist/:id/:path", "/assets/:path", "/github/:action", "/api/admin/:action", "/api/user/:action", "/api/repo/:id/file/:path", "/api/repo/:id/files/counts", "/api/repo/:id/files/search", "/api/repo/:id/files", "/api/repo/:id/zip", "/api/repo/:id", "/viewer/:id/:path"], record.route) { out.route = record.route }
# VictoriaLogs consumes its message field; keep event independently searchable.
# Explicit reproduction context for failed application file requests, never query strings.
if includes(["error", "fatal"], out.severity) || to_int(out.status) >= 400 || to_int(out.httpStatus) >= 400 {
# Anonymous application request context at every outcome, never query strings.
{
path = to_string(record.requestPath) ?? ""
parsed, path_error = parse_regex(path, r'^/api/repo/(?P<repo>[A-Za-z0-9_-]{3,200})/file/(?P<file>[A-Za-z0-9._~!$&\'()+,;=@%/-]+)$')
parsed, path_error = parse_regex(path, r'^/api/repo/(?P<repo>[A-Za-z0-9_-]{3,200})(?:/(?P<file>file/[A-Za-z0-9._~!$&\'()+,;=@%/-]+|files(?:/[A-Za-z0-9._~!$&\'()+,;=@%/-]*)?|(?:zip|options|refresh|coauthors|extend)/?))?/?$')
decoded = decode_percent(to_string(parsed.file))
if path_error == null && length(path) <= 2048 && !match(path, r'%(?:[^a-fA-F0-9]|[a-fA-F0-9][^a-fA-F0-9]|[a-fA-F0-9]?$)') && !match(decoded, r'[\x00-\x1f\x7f-\x9f\x{2028}-\x{202e}\x{2066}-\x{2069}\x{fffd}\\:?#%]|^/|(^|/)\.\.?(/|$)') {
if path_error == null && length(path) <= 2048 && !match(path, r'%(?:[^a-fA-F0-9]|[a-fA-F0-9][^a-fA-F0-9]|[a-fA-F0-9]?$)') && !match(decoded, r'[\x00-\x1f\x7f-\x9f\x{2028}-\x{202e}\x{2066}-\x{2069}\x{fffd}\\:?#%]|//|(^|/)\.\.?(/|$)') {
out.requestPath = path
out.repoId = parsed.repo
}
+13
View File
@@ -90,6 +90,19 @@ const sleep = ms => new Promise(resolve=>setTimeout(resolve,ms));
const traced=await investigateLogs(logOptions({kind:'requestId',id:diagnosticId},true),true);
const redis=safeRedisLog({ts:diagnostic.timestamp,level:'error',module:'route',raw:[diagnostic]});
for(const row of [traced.events[0],redis.raw[0]]) {assert.equal(row.requestPath,diagnostic.requestPath);assert.equal(row.repoId,'demo-123');assert.equal(row.route,diagnostic.route);assert.equal(row.method,'GET');assert.equal(row.errorStack,projected.errorStack);assert.equal(row.causeType,'RangeError');assert(!JSON.stringify(row).includes(secret));}
const paths=['/api/repo/demo-123','/api/repo/demo-123/zip','/api/repo/demo-123/files/docs/nested%20folder/','/api/repo/demo-123/files/counts','/api/repo/demo-123/options','/api/repo/demo-123/file/README.md'];
const pathRecords=paths.map((requestPath,i)=>logContext.run({requestId:randomUUID(),requestPath},()=>logRecord(i%2?'warn':'info','requests',['request',{status:200,outcome:i%2?'interrupted':'completed'}])));
const pathSources=pathRecords.map(record=>({...rawSource,message:JSON.stringify({attrs:{tag:'api'},log:JSON.stringify(record)})}));
const pathOutput=execFileSync('docker',['run','--rm','-i','--network','none','--memory','256m',vectorImage,'vrl','-i','/dev/stdin',fs.readFileSync('deploy/logging/sanitize.vrl','utf8')+'\nencode_json(.)'],{encoding:'utf8',input:pathSources.map(JSON.stringify).join('\n')+'\n'}).trim().split('\n').map(line=>JSON.parse(JSON.parse(line)));
assert.deepEqual(pathOutput.map(x=>x.requestPath),paths);
const pathInsert=await fetch(process.env.ADMIN_LOG_URL+'/insert/jsonline?_time_field=timestamp',{method:'POST',body:pathOutput.map(JSON.stringify).join('\n'),headers:{'Content-Type':'application/stream+json'}});
assert(pathInsert.ok);await sleep(2000);
for(let i=0;i<paths.length;i++){
const result=await investigateLogs(logOptions({kind:'requestId',id:pathRecords[i].requestId},true),true);
assert.equal(result.events[0].requestPath,paths[i]);
assert.equal(result.events[0].repoId,'demo-123');
}
console.log('PASS successful and interrupted anonymous endpoint paths through collector and central API');
console.log('PASS synthetic exception logger → pinned collector → VictoriaLogs/Redis safe API with normalized route and bounded frames');
docker('stop',name);
const unavailable=await investigateLogs(logOptions({})); assert.equal(unavailable.state,'unreachable');
+3 -5
View File
@@ -71,11 +71,9 @@ export function logRecord(level: string, module: string, args: unknown[]) {
if (typeof route === "string" && LOG_ROUTES.includes(route)) record.route = route;
if (context.method && allowed(["GET", "HEAD", "POST", "PUT", "PATCH", "DELETE", "OPTIONS", "OTHER"], context.method)) record.method = context.method;
Object.assign(record, safeDiagnosticFields(details));
// Reproduction data is restricted to failures, not successful traffic/metrics.
if (["error", "fatal"].includes(level) || Number(details.status || details.httpStatus) >= 400) {
const requestPath = safeReproductionPath(context.requestPath || details.requestPath);
if (requestPath) Object.assign(record, requestReproduction(requestPath));
}
// Anonymous request paths are diagnostic fields, never metric labels.
const requestPath = safeReproductionPath(context.requestPath || details.requestPath);
if (requestPath) Object.assign(record, requestReproduction(requestPath));
if (allowed(["download", "remove", "cache"], details.queue)) record.queue = details.queue;
if (details.stages && typeof details.stages === "object") for (const key of stages) {
let value: unknown;
+5 -5
View File
@@ -1,19 +1,19 @@
// Explicit admin diagnostics: public anonymous repository ID and request pathname only.
// Never collect upstream owner/repository URLs, headers, query strings or bodies.
const fileRequest = /^\/api\/repo\/([A-Za-z0-9_-]{3,200})\/file\/([A-Za-z0-9._~!$&'()+,;=@%/-]+)$/;
const anonymousRequest = /^\/api\/repo\/([A-Za-z0-9_-]{3,200})(?:\/(file\/[A-Za-z0-9._~!$&'()+,;=@%/-]+|files(?:\/[A-Za-z0-9._~!$&'()+,;=@%/-]*)?|(?:zip|options|refresh|coauthors|extend)\/?))?\/?$/;
export function safeReproductionPath(value: unknown): string | undefined {
if (typeof value !== "string" || value.length > 2048) return undefined;
const match = fileRequest.exec(value);
const match = anonymousRequest.exec(value);
if (!match || /%(?![a-fA-F0-9]{2})/.test(value)) return undefined;
try {
const path = decodeURIComponent(match[2]);
const path = decodeURIComponent(match[2] || "");
// Reject controls, traversal, URL syntax, encoded delimiters and double encoding.
// eslint-disable-next-line no-control-regex -- reject control characters from stored paths
if (/[\x00-\x1f\x7f-\x9f\u2028-\u202e\u2066-\u2069\ufffd\\:?#%]/.test(path) || path.startsWith("/") || path.split("/").some(part => part === "." || part === "..")) return undefined;
if (/[\x00-\x1f\x7f-\x9f\u2028-\u202e\u2066-\u2069\ufffd\\:?#%]/.test(path) || path.includes("//") || path.split("/").some(part => part === "." || part === "..")) return undefined;
} catch { return undefined; }
return value;
}
export function requestReproduction(url: string) {
const requestPath = safeReproductionPath(url.split("?", 1)[0]);
return requestPath ? { requestPath, repoId: fileRequest.exec(requestPath)![1] } : {};
return requestPath ? { requestPath, repoId: anonymousRequest.exec(requestPath)![1] } : {};
}
+1 -1
View File
@@ -158,7 +158,7 @@ export function monitorRequests(service: Service, metrics = requestMetrics,
metrics.observe("request", route, method, ms, { bytes, aborted, error: res.statusCode >= 500, slow });
if (firstByteMs != null) metrics.observe("first_byte", route, method, firstByteMs);
const responseMs = firstByteMs == null ? null : Math.max(0, ms - firstByteMs);
const detail = { ...(res.statusCode >= 400 ? reproduction : {}), requestId: id, service, method, route, status: res.statusCode,
const detail = { ...reproduction, requestId: id, service, method, route, status: res.statusCode,
outcome: aborted ? "interrupted" : "completed", ms: Math.round(ms), firstByteMs: firstByteMs == null ? null : Math.round(firstByteMs),
responseMs: responseMs == null ? null : Math.round(responseMs),
bytesPerSecond: responseMs != null && responseMs >= 1 && bytes > 0 ? Math.round(bytes * 1000 / responseMs) : null,
+15 -2
View File
@@ -13,15 +13,28 @@ const secret='PRIVATE_TOKEN_owner_filename';
const frame='at printError (src/server/routes/route-utils.ts:200:7)';
function failure(){ const error=new TypeError('https://user:'+secret+'@github.com/private/repo?token='+secret); error.stack=error.message+'\n at printError ('+path.resolve('src/server/routes/route-utils.ts')+':200:7)\n at '+secret+' ('+path.resolve('src/server/routes/route-utils.ts')+':201:9)\n at secret (/home/private/'+secret+':1:2)\n at fn (https://private/'+secret+':1:2)';return error; }
describe('safe actionable exception diagnostics',()=>{
it('keeps bounded anonymous file request context for failures without queries or unsafe paths',()=>{
it('keeps bounded anonymous request context without queries or unsafe paths',()=>{
const requestPath='/api/repo/demo-123/file/docs/reproduce%20issue.md';
expect(requestReproduction(requestPath+'?token=SECRET')).deep.equal({requestPath,repoId:'demo-123'});
for(const value of ['/api/repo/owner/name/readme','https://example.test'+requestPath,requestPath+'?token=SECRET','/api/repo/demo-123/file/%0aSECRET','/api/repo/demo-123/file/../secret','/api/repo/demo-123/file/%252e%252e/secret','/api/repo/demo-123/file/https%3a%2f%2fuser:token@example.test','/api/repo/demo-123/file/'+ 'x'.repeat(2048)]) expect(safeReproductionPath(value),value).equal(undefined);
expect(logContext.run({requestPath},()=>logRecord('info','route',['request',{status:200}]))).not.have.property('requestPath');
expect(logContext.run({requestPath},()=>logRecord('info','route',['request',{status:200}]))).have.property('requestPath',requestPath);
const record=logContext.run({requestPath},()=>logRecord('error','route',['failure',{httpStatus:500}]));
expect(record).include({requestPath,repoId:'demo-123'});
expect(safeRedisLog({ts:record.timestamp,level:'error',module:'route',raw:[record]}).raw[0]).include({requestPath,repoId:'demo-123'});
});
it('preserves anonymous endpoints at all outcomes through both API projections',async()=>{
for(const suffix of ['', '/', '/zip', '/files', '/files/docs/nested%20folder/', '/files/counts', '/files/search', '/options', '/refresh', '/coauthors', '/extend', '/file/README.md']){
const requestPath='/api/repo/demo-123'+suffix;
expect(requestReproduction(requestPath+'?token=SECRET')).deep.equal({requestPath,repoId:'demo-123'});
for(const status of [200,304,404,500]){
const row=logContext.run({requestPath},()=>logRecord('info','requests',['request',{status,outcome:'interrupted'}]));
expect(row).include({requestPath,repoId:'demo-123'});
expect(safeRedisLog({ts:row.timestamp,level:'info',module:'requests',raw:[row]}).raw[0]).include({requestPath,repoId:'demo-123'});
}
}
for(const suffix of ['/files/../secret','/files/%2fsecret','/files/%0aSECRET','/files/%252e%252e/secret','/zip?token=SECRET'])
expect(safeReproductionPath('/api/repo/demo-123'+suffix)).equal(undefined);
});
it('preserves dotted allowlisted source files and anonymous/named async frames',()=>{
const error=new Error(secret);
error.stack='Error: '+secret+'\n at async '+path.resolve('src/server/routes/repository-public.ts')+':224:23\n at async printError ('+path.resolve('src/server/routes/route-utils.ts')+':200:7)\n at '+path.resolve('src/core/model/owner-alert-lifecycle.schema.ts')+':12:3\n at async https://'+secret+':1:2';
+14 -3
View File
@@ -47,6 +47,14 @@ describe("bounded request monitoring", function () {
expect(metrics.drain().every(row=>!JSON.stringify(row).includes('demo-123'))).equal(true);
});
it("records the actual listing path on successful requests while keeping metrics templated", async function () {
app.get('/api/repo/:repoId/files/*path',(_req,res)=>res.json([]));
await listen(); await fetch(server,'/api/repo/demo-123/files/docs/nested%20folder/?token=SECRET');
expect(logs[0][1]).include({status:200,requestPath:'/api/repo/demo-123/files/docs/nested%20folder/',repoId:'demo-123'});
expect(JSON.stringify(logs)).not.include('SECRET');
expect(JSON.stringify(metrics.drain())).not.include('demo-123');
});
it("keeps labels bounded and removes source identities and query values", function () {
expect(requestRoute("/api/repo/private-name/file/secret.txt?token=abc", "api")).to.equal("/api/repo/:repoId/file/:path");
expect(requestRoute("/api/repo/secret-owner/secret-repo/readme?branch=secret", "api")).to.equal("/api/repo/:owner/:repo/readme");
@@ -94,7 +102,8 @@ describe("bounded request monitoring", function () {
expect(callbacks).to.equal(gzip ? 0 : body.length / 16384);
const row = metrics.drain().find(row => row.data.metric === "request").data;
expect(row.bytes).to.equal(result.body.length); expect(row.aborted).to.equal(0); expect(row.count).to.equal(1);
expect(JSON.stringify(logs)).not.to.include("private.txt"); expect(JSON.stringify(logs)).not.to.include("secret");
expect(logs[0][1].requestPath).to.equal("/api/repo/private/file/private.txt");
expect(JSON.stringify(row)).not.to.include("private.txt"); expect(JSON.stringify(logs)).not.to.include("secret");
expect(logs[0][1].requestId).to.equal(result.headers["x-request-id"]);
expect(activeRequestCount()).to.equal(0);
});
@@ -124,12 +133,14 @@ describe("bounded request monitoring", function () {
it(`records a disconnect ${headersSent ? "after" : "before"} headers exactly once`, async function () {
let arrived; const ready = new Promise(resolve => { arrived = resolve; });
let finished; const terminal = new Promise(resolve => { finished = resolve; });
app.get("/", (_req, res) => { if (headersSent) { res.type("text/plain"); res.write("partial"); } arrived(); res.on("close", finished); });
app.get("/api/repo/:repoId/zip", (_req, res) => { if (headersSent) { res.type("text/plain"); res.write("partial"); } arrived(); res.on("close", finished); });
await listen();
const client = http.get({ host: "127.0.0.1", port: server.address().port, path: "/" }); client.on("error", () => {});
const client = http.get({ host: "127.0.0.1", port: server.address().port, path: "/api/repo/demo-123/zip?token=SECRET" }); client.on("error", () => {});
await ready; client.destroy(); await terminal;
expect(logs).to.have.length(1); expect(logs[0][0]).to.equal("request interrupted");
expect(logs[0][1].outcome).to.equal("interrupted");
expect(logs[0][1]).include({requestPath:"/api/repo/demo-123/zip",repoId:"demo-123"});
expect(JSON.stringify(logs)).not.include("SECRET");
const row = metrics.drain().find(row => row.data.metric === "request").data;
expect(row.count).to.equal(1); expect(row.aborted).to.equal(1); expect(activeRequestCount()).to.equal(0);
if (!headersSent) {