fix: exclude source repository lookups from captured paths

This commit is contained in:
Thomas Durieux committed 2026-11-15 00:08:12 +00:00
1 parent 953767b2cd
commit 7d044525ce
6 files changed
+39 -10

No files matched your search

+1 -1
View File
@@ -355,7 +355,7 @@ unknown fields. Existing discarded exception details cannot be recovered.
### Admin reproduction context
Application requests to the supported anonymous repository endpoints retain their public anonymous repository ID and encoded request pathname, up to 2 KiB, under the existing admin log access controls. This is an explicit exception to template-only diagnostics: repository-relative filenames are retained for reproduction. Query strings, headers, bodies and upstream owner/repository URLs are still excluded. Filenames are operator-visible data under this policy; no generic filter can guarantee that a user has not put sensitive text in a filename. Capture includes successful, failed and interrupted requests; metrics still use only normalized routes. Supported paths are repository metadata, file contents, listings/counts/search, ZIP, options, refresh, coauthors and extend under `/api/repo/:repoId`; other endpoints and unsafe, oversized, malformed or traversal paths omit reproduction context. Historical records cannot be backfilled. The UI copies the exact captured pathname without a host or query.
Application requests to the supported anonymous repository endpoints retain their public anonymous repository ID and encoded request pathname, up to 2 KiB, under the existing admin log access controls. This is an explicit exception to template-only diagnostics: repository-relative filenames are retained for reproduction. Query strings, headers, bodies and upstream owner/repository URLs are still excluded. Filenames are operator-visible data under this policy; no generic filter can guarantee that a user has not put sensitive text in a filename. Capture includes successful, failed and interrupted requests; metrics still use only normalized routes. Supported paths are repository metadata, file contents, listings/counts/search, ZIP, options, refresh, coauthors and extend under `/api/repo/:repoId`; other endpoints and unsafe, oversized, malformed or traversal paths omit reproduction context. Listing capture is limited to the actual `/files`, `/files/counts` and `/files/search` endpoints; folder/search query values are not retained. Source lookups such as `/api/repo/:owner/files/readme` are excluded. Historical records cannot be backfilled. The UI copies the exact captured pathname without a host or query.
For the focused restricted-file/rotation regression, run
`LOGGING_ACCESS_ONLY=1 python3 scripts/test-logging.py`. This uses synthetic
`0640` logs in an isolated Docker project and preserves capability restrictions.
+1 -1
View File
@@ -62,7 +62,7 @@ if includes(["/", "other", "/dashboard", "/anonymize", "/healthcheck", "/api/hea
# Anonymous application request context at every outcome, never query strings.
{
path = to_string(record.requestPath) ?? ""
parsed, path_error = parse_regex(path, r'^/api/repo/(?P<repo>[A-Za-z0-9_-]{3,200})(?:/(?P<file>file/[A-Za-z0-9._~!$&\'()+,;=@%/-]+|files(?:/[A-Za-z0-9._~!$&\'()+,;=@%/-]*)?|(?:zip|options|refresh|coauthors|extend)/?))?/?$')
parsed, path_error = parse_regex(path, r'^/api/repo/(?P<repo>[A-Za-z0-9_-]{3,200})(?:/(?P<file>file/[A-Za-z0-9._~!$&\'()+,;=@%/-]+|files(?:/(?:counts|search))?/?|(?:zip|options|refresh|coauthors|extend)/?))?/?$')
decoded = decode_percent(to_string(parsed.file))
if path_error == null && length(path) <= 2048 && !match(path, r'%(?:[^a-fA-F0-9]|[a-fA-F0-9][^a-fA-F0-9]|[a-fA-F0-9]?$)') && !match(decoded, r'[\x00-\x1f\x7f-\x9f\x{2028}-\x{202e}\x{2066}-\x{2069}\x{fffd}\\:?#%]|//|(^|/)\.\.?(/|$)') {
out.requestPath = path
+3 -3
View File
@@ -81,8 +81,8 @@ const sleep = ms => new Promise(resolve=>setTimeout(resolve,ms));
assert(projected.errorStack.includes('at <anonymous> (src/server/routes/repository-public.ts:224:23)'));
assert.match(projected.errorStack,/at requestRoute \(src\/core\/request-monitoring.ts:\d+:\d+\)/);
assert(!JSON.stringify(projected).includes(secret));
const invalidPaths=['/api/repo/demo-123/file/a?token='+secret,'/api/repo/demo-123/file/%0a'+secret,'/api/repo/demo-123/file/../'+secret,'/api/repo/demo-123/file/%252e%252e/'+secret,'https://user:'+secret+'@example.test/file'];
const invalidSources=invalidPaths.map(requestPath=>({...rawSource,message:JSON.stringify({attrs:{tag:'api'},log:JSON.stringify({...diagnostic,requestPath,errorStack:'at <anonymous> (src/core/model/private.schema.ts:1:2)'})})}));
const invalidPaths=[...['readme','branches','tree','file'].flatMap(action=>['','/'].map(tail=>'/api/repo/private-owner/files/'+action+tail)),'/api/repo/demo-123/file/a?token='+secret,'/api/repo/demo-123/file/%0a'+secret,'/api/repo/demo-123/file/../'+secret,'/api/repo/demo-123/file/%252e%252e/'+secret,'https://user:'+secret+'@example.test/file'];
const invalidSources=invalidPaths.flatMap(requestPath=>[200,403,500].map(status=>({...rawSource,message:JSON.stringify({attrs:{tag:'api'},log:JSON.stringify({...diagnostic,status,httpStatus:status,severity:status===200?'info':'error',requestPath,errorStack:'at <anonymous> (src/core/model/private.schema.ts:1:2)'})})})));
const rejected=execFileSync('docker',['run','--rm','-i','--network','none','--memory','256m',vectorImage,'vrl','-i','/dev/stdin',fs.readFileSync('deploy/logging/sanitize.vrl','utf8')+'\nencode_json(.)'],{encoding:'utf8',input:invalidSources.map(JSON.stringify).join('\n')+'\n'}).trim().split('\n').map(line=>JSON.parse(JSON.parse(line)));
assert(rejected.every(row=>!row.requestPath && !row.repoId && !row.errorStack));
assert(!JSON.stringify(rejected).includes(secret));
@@ -90,7 +90,7 @@ const sleep = ms => new Promise(resolve=>setTimeout(resolve,ms));
const traced=await investigateLogs(logOptions({kind:'requestId',id:diagnosticId},true),true);
const redis=safeRedisLog({ts:diagnostic.timestamp,level:'error',module:'route',raw:[diagnostic]});
for(const row of [traced.events[0],redis.raw[0]]) {assert.equal(row.requestPath,diagnostic.requestPath);assert.equal(row.repoId,'demo-123');assert.equal(row.route,diagnostic.route);assert.equal(row.method,'GET');assert.equal(row.errorStack,projected.errorStack);assert.equal(row.causeType,'RangeError');assert(!JSON.stringify(row).includes(secret));}
const paths=['/api/repo/demo-123','/api/repo/demo-123/zip','/api/repo/demo-123/files/docs/nested%20folder/','/api/repo/demo-123/files/counts','/api/repo/demo-123/options','/api/repo/demo-123/file/README.md'];
const paths=['/api/repo/demo-123','/api/repo/demo-123/zip','/api/repo/demo-123/files','/api/repo/demo-123/files/counts','/api/repo/demo-123/options','/api/repo/demo-123/file/README.md'];
const pathRecords=paths.map((requestPath,i)=>logContext.run({requestId:randomUUID(),requestPath},()=>logRecord(i%2?'warn':'info','requests',['request',{status:200,outcome:i%2?'interrupted':'completed'}])));
const pathSources=pathRecords.map(record=>({...rawSource,message:JSON.stringify({attrs:{tag:'api'},log:JSON.stringify(record)})}));
const pathOutput=execFileSync('docker',['run','--rm','-i','--network','none','--memory','256m',vectorImage,'vrl','-i','/dev/stdin',fs.readFileSync('deploy/logging/sanitize.vrl','utf8')+'\nencode_json(.)'],{encoding:'utf8',input:pathSources.map(JSON.stringify).join('\n')+'\n'}).trim().split('\n').map(line=>JSON.parse(JSON.parse(line)));
+3 -1
View File
@@ -1,6 +1,8 @@
// Explicit admin diagnostics: public anonymous repository ID and request pathname only.
// Never collect upstream owner/repository URLs, headers, query strings or bodies.
const anonymousRequest = /^\/api\/repo\/([A-Za-z0-9_-]{3,200})(?:\/(file\/[A-Za-z0-9._~!$&'()+,;=@%/-]+|files(?:\/[A-Za-z0-9._~!$&'()+,;=@%/-]*)?|(?:zip|options|refresh|coauthors|extend)\/?))?\/?$/;
// Listing folders use query parameters; a wildcard here would capture upstream
// /:owner/files/{readme,branches,tree,file} source lookups.
const anonymousRequest = /^\/api\/repo\/([A-Za-z0-9_-]{3,200})(?:\/(file\/[A-Za-z0-9._~!$&'()+,;=@%/-]+|files(?:\/(?:counts|search))?\/?|(?:zip|options|refresh|coauthors|extend)\/?))?\/?$/;
export function safeReproductionPath(value: unknown): string | undefined {
if (typeof value !== "string" || value.length > 2048) return undefined;
const match = anonymousRequest.exec(value);
+15 -1
View File
@@ -23,7 +23,7 @@ describe('safe actionable exception diagnostics',()=>{
expect(safeRedisLog({ts:record.timestamp,level:'error',module:'route',raw:[record]}).raw[0]).include({requestPath,repoId:'demo-123'});
});
it('preserves anonymous endpoints at all outcomes through both API projections',async()=>{
for(const suffix of ['', '/', '/zip', '/files', '/files/docs/nested%20folder/', '/files/counts', '/files/search', '/options', '/refresh', '/coauthors', '/extend', '/file/README.md']){
for(const suffix of ['', '/', '/zip', '/files', '/files/counts', '/files/search', '/options', '/refresh', '/coauthors', '/extend', '/file/README.md']){
const requestPath='/api/repo/demo-123'+suffix;
expect(requestReproduction(requestPath+'?token=SECRET')).deep.equal({requestPath,repoId:'demo-123'});
for(const status of [200,304,404,500]){
@@ -35,6 +35,20 @@ describe('safe actionable exception diagnostics',()=>{
for(const suffix of ['/files/../secret','/files/%2fsecret','/files/%0aSECRET','/files/%252e%252e/secret','/zip?token=SECRET'])
expect(safeReproductionPath('/api/repo/demo-123'+suffix)).equal(undefined);
});
it('rejects upstream lookups for repositories named files at every logging boundary',()=>{
for(const action of ['readme','branches','tree','file']){
const requestPath='/api/repo/private-owner/files/'+action;
for(const suffix of ['', '/', '?token=SECRET']){
expect(requestReproduction(requestPath+suffix)).deep.equal({});
for(const status of [200,403,500]){
const row=logContext.run({requestPath:requestPath+suffix},()=>logRecord('error','route',['failure',{httpStatus:status}]));
expect(row).not.have.property('requestPath');expect(row).not.have.property('repoId');
const redis=safeRedisLog({ts:row.timestamp,level:'error',module:'route',raw:[{...row,requestPath:requestPath+suffix,repoId:'private-owner'}]});
expect(redis.raw[0]).not.have.property('requestPath');expect(redis.raw[0]).not.have.property('repoId');
}
}
}
});
it('preserves dotted allowlisted source files and anonymous/named async frames',()=>{
const error=new Error(secret);
error.stack='Error: '+secret+'\n at async '+path.resolve('src/server/routes/repository-public.ts')+':224:23\n at async printError ('+path.resolve('src/server/routes/route-utils.ts')+':200:7)\n at '+path.resolve('src/core/model/owner-alert-lifecycle.schema.ts')+':12:3\n at async https://'+secret+':1:2';
+16 -3
View File
@@ -48,13 +48,26 @@ describe("bounded request monitoring", function () {
});
it("records the actual listing path on successful requests while keeping metrics templated", async function () {
app.get('/api/repo/:repoId/files/*path',(_req,res)=>res.json([]));
await listen(); await fetch(server,'/api/repo/demo-123/files/docs/nested%20folder/?token=SECRET');
expect(logs[0][1]).include({status:200,requestPath:'/api/repo/demo-123/files/docs/nested%20folder/',repoId:'demo-123'});
app.get('/api/repo/:repoId/files',(_req,res)=>res.json([]));
await listen(); await fetch(server,'/api/repo/demo-123/files?path=docs%2Fnested&token=SECRET');
expect(logs[0][1]).include({status:200,requestPath:'/api/repo/demo-123/files',repoId:'demo-123'});
expect(JSON.stringify(logs)).not.include('SECRET');
expect(JSON.stringify(metrics.drain())).not.include('demo-123');
});
it("never captures upstream identities from files-named source repository handlers", async function () {
const {logRecord}=require('../src/core/log-record');
const records=[];
for(const action of ['readme','branches','tree','file'])
app.get('/api/repo/:owner/:repo/'+action,(_req,res)=>{
records.push(logRecord('error','route',['failure',{httpStatus:403}]));res.status(403).end();
});
await listen();
for(const action of ['readme','branches','tree','file'])
await fetch(server,'/api/repo/private-owner/files/'+action);
expect(JSON.stringify([...records,...logs])).not.include('private-owner');
});
it("keeps labels bounded and removes source identities and query values", function () {
expect(requestRoute("/api/repo/private-name/file/secret.txt?token=abc", "api")).to.equal("/api/repo/:repoId/file/:path");
expect(requestRoute("/api/repo/secret-owner/secret-repo/readme?branch=secret", "api")).to.equal("/api/repo/:owner/:repo/readme");