mirror of
https://github.com/tdurieux/anonymous_github.git
synced 2026-11-17 17:44:09 +01:00
fix: require anonymous endpoint methods for logged paths
This commit is contained in:
1 parent
7d044525ce
commit
1723f5e927
9 files changed
+67
-22
No files matched your search
@@ -355,7 +355,7 @@ unknown fields. Existing discarded exception details cannot be recovered.
|
||||
|
||||
### Admin reproduction context
|
||||
|
||||
Application requests to the supported anonymous repository endpoints retain their public anonymous repository ID and encoded request pathname, up to 2 KiB, under the existing admin log access controls. This is an explicit exception to template-only diagnostics: repository-relative filenames are retained for reproduction. Query strings, headers, bodies and upstream owner/repository URLs are still excluded. Filenames are operator-visible data under this policy; no generic filter can guarantee that a user has not put sensitive text in a filename. Capture includes successful, failed and interrupted requests; metrics still use only normalized routes. Supported paths are repository metadata, file contents, listings/counts/search, ZIP, options, refresh, coauthors and extend under `/api/repo/:repoId`; other endpoints and unsafe, oversized, malformed or traversal paths omit reproduction context. Listing capture is limited to the actual `/files`, `/files/counts` and `/files/search` endpoints; folder/search query values are not retained. Source lookups such as `/api/repo/:owner/files/readme` are excluded. Historical records cannot be backfilled. The UI copies the exact captured pathname without a host or query.
|
||||
Application requests to the supported anonymous repository endpoints retain their public anonymous repository ID and encoded request pathname, up to 2 KiB, under the existing admin log access controls. This is an explicit exception to template-only diagnostics: repository-relative filenames are retained for reproduction. Query strings, headers, bodies and upstream owner/repository URLs are still excluded. Filenames are operator-visible data under this policy; no generic filter can guarantee that a user has not put sensitive text in a filename. Capture includes successful, failed and interrupted requests; metrics still use only normalized routes. Supported paths are repository metadata, file contents, listings/counts/search, ZIP, options, refresh, coauthors and extend under `/api/repo/:repoId`; other endpoints and unsafe, oversized, malformed or traversal paths omit reproduction context. Listing capture is limited to the actual `/files`, `/files/counts` and `/files/search` endpoints; folder/search query values are not retained. Action paths refresh, extend and coauthors require POST; GET/HEAD source metadata lookups using those repository names are excluded. Missing or unsupported methods omit reproduction context. Source lookups such as `/api/repo/:owner/files/readme` are excluded. Historical records cannot be backfilled. The UI copies the exact captured pathname without a host or query.
|
||||
For the focused restricted-file/rotation regression, run
|
||||
`LOGGING_ACCESS_ONLY=1 python3 scripts/test-logging.py`. This uses synthetic
|
||||
`0640` logs in an isolated Docker project and preserves capability restrictions.
|
||||
@@ -63,8 +63,10 @@ if includes(["/", "other", "/dashboard", "/anonymize", "/healthcheck", "/api/hea
|
||||
{
|
||||
path = to_string(record.requestPath) ?? ""
|
||||
parsed, path_error = parse_regex(path, r'^/api/repo/(?P<repo>[A-Za-z0-9_-]{3,200})(?:/(?P<file>file/[A-Za-z0-9._~!$&\'()+,;=@%/-]+|files(?:/(?:counts|search))?/?|(?:zip|options|refresh|coauthors|extend)/?))?/?$')
|
||||
action = replace(to_string(parsed.file), r'/+$', "")
|
||||
allowed_methods = if action == "" { ["GET", "HEAD", "POST", "DELETE"] } else if includes(["refresh", "extend", "coauthors"], action) { ["POST"] } else { ["GET", "HEAD"] }
|
||||
decoded = decode_percent(to_string(parsed.file))
|
||||
if path_error == null && length(path) <= 2048 && !match(path, r'%(?:[^a-fA-F0-9]|[a-fA-F0-9][^a-fA-F0-9]|[a-fA-F0-9]?$)') && !match(decoded, r'[\x00-\x1f\x7f-\x9f\x{2028}-\x{202e}\x{2066}-\x{2069}\x{fffd}\\:?#%]|//|(^|/)\.\.?(/|$)') {
|
||||
if path_error == null && includes(allowed_methods, out.method) && length(path) <= 2048 && !match(path, r'%(?:[^a-fA-F0-9]|[a-fA-F0-9][^a-fA-F0-9]|[a-fA-F0-9]?$)') && !match(decoded, r'[\x00-\x1f\x7f-\x9f\x{2028}-\x{202e}\x{2066}-\x{2069}\x{fffd}\\:?#%]|//|(^|/)\.\.?(/|$)') {
|
||||
out.requestPath = path
|
||||
out.repoId = parsed.repo
|
||||
}
|
||||
|
||||
@@ -81,7 +81,7 @@ const sleep = ms => new Promise(resolve=>setTimeout(resolve,ms));
|
||||
assert(projected.errorStack.includes('at <anonymous> (src/server/routes/repository-public.ts:224:23)'));
|
||||
assert.match(projected.errorStack,/at requestRoute \(src\/core\/request-monitoring.ts:\d+:\d+\)/);
|
||||
assert(!JSON.stringify(projected).includes(secret));
|
||||
const invalidPaths=[...['readme','branches','tree','file'].flatMap(action=>['','/'].map(tail=>'/api/repo/private-owner/files/'+action+tail)),'/api/repo/demo-123/file/a?token='+secret,'/api/repo/demo-123/file/%0a'+secret,'/api/repo/demo-123/file/../'+secret,'/api/repo/demo-123/file/%252e%252e/'+secret,'https://user:'+secret+'@example.test/file'];
|
||||
const invalidPaths=[...['refresh','extend','coauthors'].flatMap(action=>['','/'].map(tail=>'/api/repo/private-owner/'+action+tail)),...['readme','branches','tree','file'].flatMap(action=>['','/'].map(tail=>'/api/repo/private-owner/files/'+action+tail)),'/api/repo/demo-123/file/a?token='+secret,'/api/repo/demo-123/file/%0a'+secret,'/api/repo/demo-123/file/../'+secret,'/api/repo/demo-123/file/%252e%252e/'+secret,'https://user:'+secret+'@example.test/file'];
|
||||
const invalidSources=invalidPaths.flatMap(requestPath=>[200,403,500].map(status=>({...rawSource,message:JSON.stringify({attrs:{tag:'api'},log:JSON.stringify({...diagnostic,status,httpStatus:status,severity:status===200?'info':'error',requestPath,errorStack:'at <anonymous> (src/core/model/private.schema.ts:1:2)'})})})));
|
||||
const rejected=execFileSync('docker',['run','--rm','-i','--network','none','--memory','256m',vectorImage,'vrl','-i','/dev/stdin',fs.readFileSync('deploy/logging/sanitize.vrl','utf8')+'\nencode_json(.)'],{encoding:'utf8',input:invalidSources.map(JSON.stringify).join('\n')+'\n'}).trim().split('\n').map(line=>JSON.parse(JSON.parse(line)));
|
||||
assert(rejected.every(row=>!row.requestPath && !row.repoId && !row.errorStack));
|
||||
@@ -90,8 +90,8 @@ const sleep = ms => new Promise(resolve=>setTimeout(resolve,ms));
|
||||
const traced=await investigateLogs(logOptions({kind:'requestId',id:diagnosticId},true),true);
|
||||
const redis=safeRedisLog({ts:diagnostic.timestamp,level:'error',module:'route',raw:[diagnostic]});
|
||||
for(const row of [traced.events[0],redis.raw[0]]) {assert.equal(row.requestPath,diagnostic.requestPath);assert.equal(row.repoId,'demo-123');assert.equal(row.route,diagnostic.route);assert.equal(row.method,'GET');assert.equal(row.errorStack,projected.errorStack);assert.equal(row.causeType,'RangeError');assert(!JSON.stringify(row).includes(secret));}
|
||||
const paths=['/api/repo/demo-123','/api/repo/demo-123/zip','/api/repo/demo-123/files','/api/repo/demo-123/files/counts','/api/repo/demo-123/options','/api/repo/demo-123/file/README.md'];
|
||||
const pathRecords=paths.map((requestPath,i)=>logContext.run({requestId:randomUUID(),requestPath},()=>logRecord(i%2?'warn':'info','requests',['request',{status:200,outcome:i%2?'interrupted':'completed'}])));
|
||||
const paths=['/api/repo/demo-123/refresh','/api/repo/demo-123/extend','/api/repo/demo-123/coauthors','/api/repo/demo-123','/api/repo/demo-123/zip','/api/repo/demo-123/files','/api/repo/demo-123/files/counts','/api/repo/demo-123/options','/api/repo/demo-123/file/README.md'];
|
||||
const pathRecords=paths.map((requestPath,i)=>logContext.run({requestId:randomUUID(),requestPath,method:/\/(refresh|extend|coauthors)$/.test(requestPath)?'POST':'GET'},()=>logRecord(i%2?'warn':'info','requests',['request',{status:200,outcome:i%2?'interrupted':'completed'}])));
|
||||
const pathSources=pathRecords.map(record=>({...rawSource,message:JSON.stringify({attrs:{tag:'api'},log:JSON.stringify(record)})}));
|
||||
const pathOutput=execFileSync('docker',['run','--rm','-i','--network','none','--memory','256m',vectorImage,'vrl','-i','/dev/stdin',fs.readFileSync('deploy/logging/sanitize.vrl','utf8')+'\nencode_json(.)'],{encoding:'utf8',input:pathSources.map(JSON.stringify).join('\n')+'\n'}).trim().split('\n').map(line=>JSON.parse(JSON.parse(line)));
|
||||
assert.deepEqual(pathOutput.map(x=>x.requestPath),paths);
|
||||
|
||||
@@ -72,8 +72,8 @@ export function logRecord(level: string, module: string, args: unknown[]) {
|
||||
if (context.method && allowed(["GET", "HEAD", "POST", "PUT", "PATCH", "DELETE", "OPTIONS", "OTHER"], context.method)) record.method = context.method;
|
||||
Object.assign(record, safeDiagnosticFields(details));
|
||||
// Anonymous request paths are diagnostic fields, never metric labels.
|
||||
const requestPath = safeReproductionPath(context.requestPath || details.requestPath);
|
||||
if (requestPath) Object.assign(record, requestReproduction(requestPath));
|
||||
const requestPath = safeReproductionPath(context.requestPath || details.requestPath, record.method);
|
||||
if (requestPath) Object.assign(record, requestReproduction(requestPath, record.method));
|
||||
if (allowed(["download", "remove", "cache"], details.queue)) record.queue = details.queue;
|
||||
if (details.stages && typeof details.stages === "object") for (const key of stages) {
|
||||
let value: unknown;
|
||||
|
||||
@@ -3,10 +3,15 @@
|
||||
// Listing folders use query parameters; a wildcard here would capture upstream
|
||||
// /:owner/files/{readme,branches,tree,file} source lookups.
|
||||
const anonymousRequest = /^\/api\/repo\/([A-Za-z0-9_-]{3,200})(?:\/(file\/[A-Za-z0-9._~!$&'()+,;=@%/-]+|files(?:\/(?:counts|search))?\/?|(?:zip|options|refresh|coauthors|extend)\/?))?\/?$/;
|
||||
export function safeReproductionPath(value: unknown): string | undefined {
|
||||
export function safeReproductionPath(value: unknown, method: unknown): string | undefined {
|
||||
if (typeof value !== "string" || value.length > 2048) return undefined;
|
||||
const match = anonymousRequest.exec(value);
|
||||
if (!match || /%(?![a-fA-F0-9]{2})/.test(value)) return undefined;
|
||||
const action = (match[2] || "").replace(/\/+$/, "");
|
||||
const allowedMethods = !action ? ["GET", "HEAD", "POST", "DELETE"]
|
||||
: ["refresh", "extend", "coauthors"].includes(action) ? ["POST"] : ["GET", "HEAD"];
|
||||
// GET coauthors is shadowed by the earlier source metadata handler.
|
||||
if (typeof method !== "string" || !allowedMethods.includes(method)) return undefined;
|
||||
try {
|
||||
const path = decodeURIComponent(match[2] || "");
|
||||
// Reject controls, traversal, URL syntax, encoded delimiters and double encoding.
|
||||
@@ -15,7 +20,7 @@ export function safeReproductionPath(value: unknown): string | undefined {
|
||||
} catch { return undefined; }
|
||||
return value;
|
||||
}
|
||||
export function requestReproduction(url: string) {
|
||||
const requestPath = safeReproductionPath(url.split("?", 1)[0]);
|
||||
export function requestReproduction(url: string, method: unknown) {
|
||||
const requestPath = safeReproductionPath(url.split("?", 1)[0], method);
|
||||
return requestPath ? { requestPath, repoId: anonymousRequest.exec(requestPath)![1] } : {};
|
||||
}
|
||||
@@ -117,7 +117,7 @@ export function monitorRequests(service: Service, metrics = requestMetrics,
|
||||
: req.originalUrl.startsWith("/w/") ? "website" : "file";
|
||||
const trace = { id, service, traffic, stages: {} };
|
||||
const route = requestRoute(req.originalUrl, service);
|
||||
const reproduction = service === "api" ? requestReproduction(req.originalUrl) : {};
|
||||
const reproduction = service === "api" ? requestReproduction(req.originalUrl, req.method) : {};
|
||||
const method = methods.has(req.method) ? req.method : "OTHER";
|
||||
const started = performance.now();
|
||||
let firstByteMs: number | undefined;
|
||||
|
||||
@@ -14,34 +14,35 @@ const frame='at printError (src/server/routes/route-utils.ts:200:7)';
|
||||
function failure(){ const error=new TypeError('https://user:'+secret+'@github.com/private/repo?token='+secret); error.stack=error.message+'\n at printError ('+path.resolve('src/server/routes/route-utils.ts')+':200:7)\n at '+secret+' ('+path.resolve('src/server/routes/route-utils.ts')+':201:9)\n at secret (/home/private/'+secret+':1:2)\n at fn (https://private/'+secret+':1:2)';return error; }
|
||||
describe('safe actionable exception diagnostics',()=>{
|
||||
it('keeps bounded anonymous request context without queries or unsafe paths',()=>{
|
||||
const requestPath='/api/repo/demo-123/file/docs/reproduce%20issue.md';
|
||||
expect(requestReproduction(requestPath+'?token=SECRET')).deep.equal({requestPath,repoId:'demo-123'});
|
||||
for(const value of ['/api/repo/owner/name/readme','https://example.test'+requestPath,requestPath+'?token=SECRET','/api/repo/demo-123/file/%0aSECRET','/api/repo/demo-123/file/../secret','/api/repo/demo-123/file/%252e%252e/secret','/api/repo/demo-123/file/https%3a%2f%2fuser:token@example.test','/api/repo/demo-123/file/'+ 'x'.repeat(2048)]) expect(safeReproductionPath(value),value).equal(undefined);
|
||||
expect(logContext.run({requestPath},()=>logRecord('info','route',['request',{status:200}]))).have.property('requestPath',requestPath);
|
||||
const record=logContext.run({requestPath},()=>logRecord('error','route',['failure',{httpStatus:500}]));
|
||||
const method='GET', requestPath='/api/repo/demo-123/file/docs/reproduce%20issue.md';
|
||||
expect(requestReproduction(requestPath+'?token=SECRET',method)).deep.equal({requestPath,repoId:'demo-123'});
|
||||
for(const value of ['/api/repo/owner/name/readme','https://example.test'+requestPath,requestPath+'?token=SECRET','/api/repo/demo-123/file/%0aSECRET','/api/repo/demo-123/file/../secret','/api/repo/demo-123/file/%252e%252e/secret','/api/repo/demo-123/file/https%3a%2f%2fuser:token@example.test','/api/repo/demo-123/file/'+ 'x'.repeat(2048)]) expect(safeReproductionPath(value,"GET"),value).equal(undefined);
|
||||
expect(logContext.run({requestPath,method},()=>logRecord('info','route',['request',{status:200}]))).have.property('requestPath',requestPath);
|
||||
const record=logContext.run({requestPath,method},()=>logRecord('error','route',['failure',{httpStatus:500}]));
|
||||
expect(record).include({requestPath,repoId:'demo-123'});
|
||||
expect(safeRedisLog({ts:record.timestamp,level:'error',module:'route',raw:[record]}).raw[0]).include({requestPath,repoId:'demo-123'});
|
||||
});
|
||||
it('preserves anonymous endpoints at all outcomes through both API projections',async()=>{
|
||||
for(const suffix of ['', '/', '/zip', '/files', '/files/counts', '/files/search', '/options', '/refresh', '/coauthors', '/extend', '/file/README.md']){
|
||||
const method=['/refresh','/extend','/coauthors'].includes(suffix)?'POST':'GET';
|
||||
const requestPath='/api/repo/demo-123'+suffix;
|
||||
expect(requestReproduction(requestPath+'?token=SECRET')).deep.equal({requestPath,repoId:'demo-123'});
|
||||
expect(requestReproduction(requestPath+'?token=SECRET',method)).deep.equal({requestPath,repoId:'demo-123'});
|
||||
for(const status of [200,304,404,500]){
|
||||
const row=logContext.run({requestPath},()=>logRecord('info','requests',['request',{status,outcome:'interrupted'}]));
|
||||
const row=logContext.run({requestPath,method},()=>logRecord('info','requests',['request',{status,outcome:'interrupted'}]));
|
||||
expect(row).include({requestPath,repoId:'demo-123'});
|
||||
expect(safeRedisLog({ts:row.timestamp,level:'info',module:'requests',raw:[row]}).raw[0]).include({requestPath,repoId:'demo-123'});
|
||||
}
|
||||
}
|
||||
for(const suffix of ['/files/../secret','/files/%2fsecret','/files/%0aSECRET','/files/%252e%252e/secret','/zip?token=SECRET'])
|
||||
expect(safeReproductionPath('/api/repo/demo-123'+suffix)).equal(undefined);
|
||||
expect(safeReproductionPath('/api/repo/demo-123'+suffix,'GET')).equal(undefined);
|
||||
});
|
||||
it('rejects upstream lookups for repositories named files at every logging boundary',()=>{
|
||||
for(const action of ['readme','branches','tree','file']){
|
||||
const requestPath='/api/repo/private-owner/files/'+action;
|
||||
for(const suffix of ['', '/', '?token=SECRET']){
|
||||
expect(requestReproduction(requestPath+suffix)).deep.equal({});
|
||||
expect(requestReproduction(requestPath+suffix,"GET")).deep.equal({});
|
||||
for(const status of [200,403,500]){
|
||||
const row=logContext.run({requestPath:requestPath+suffix},()=>logRecord('error','route',['failure',{httpStatus:status}]));
|
||||
const row=logContext.run({requestPath:requestPath+suffix,method:"GET"},()=>logRecord('error','route',['failure',{httpStatus:status}]));
|
||||
expect(row).not.have.property('requestPath');expect(row).not.have.property('repoId');
|
||||
const redis=safeRedisLog({ts:row.timestamp,level:'error',module:'route',raw:[{...row,requestPath:requestPath+suffix,repoId:'private-owner'}]});
|
||||
expect(redis.raw[0]).not.have.property('requestPath');expect(redis.raw[0]).not.have.property('repoId');
|
||||
@@ -49,6 +50,21 @@ describe('safe actionable exception diagnostics',()=>{
|
||||
}
|
||||
}
|
||||
});
|
||||
it('requires the anonymous action method and omits ambiguous GET source metadata',()=>{
|
||||
for(const action of ['refresh','extend','coauthors']){
|
||||
for(const suffix of ['', '/']){
|
||||
const requestPath='/api/repo/private-owner/'+action+suffix;
|
||||
for(const method of ['GET','HEAD',undefined,'OPTIONS']){
|
||||
expect(safeReproductionPath(requestPath,method)).equal(undefined);
|
||||
const row=logContext.run({requestPath,method},()=>logRecord('error','route',['failure',{httpStatus:403}]));
|
||||
expect(row).not.have.property('requestPath');
|
||||
const redis=safeRedisLog({ts:row.timestamp,level:'error',module:'route',raw:[{...row,requestPath,repoId:'private-owner'}]});
|
||||
expect(redis.raw[0]).not.have.property('requestPath');
|
||||
}
|
||||
expect(requestReproduction(requestPath,'POST')).deep.equal({requestPath,repoId:'private-owner'});
|
||||
}
|
||||
}
|
||||
});
|
||||
it('preserves dotted allowlisted source files and anonymous/named async frames',()=>{
|
||||
const error=new Error(secret);
|
||||
error.stack='Error: '+secret+'\n at async '+path.resolve('src/server/routes/repository-public.ts')+':224:23\n at async printError ('+path.resolve('src/server/routes/route-utils.ts')+':200:7)\n at '+path.resolve('src/core/model/owner-alert-lifecycle.schema.ts')+':12:3\n at async https://'+secret+':1:2';
|
||||
|
||||
@@ -68,6 +68,28 @@ describe("bounded request monitoring", function () {
|
||||
expect(JSON.stringify([...records,...logs])).not.include('private-owner');
|
||||
});
|
||||
|
||||
it("distinguishes anonymous POST actions from GET source metadata at the same URL", async function () {
|
||||
const {logRecord}=require('../src/core/log-record');
|
||||
const records=[];
|
||||
for(const action of ['refresh','extend','coauthors'])
|
||||
app.post('/api/repo/:repoId/'+action,(_req,res)=>{
|
||||
records.push(logRecord('info','route',['request',{status:200}]));res.end();
|
||||
});
|
||||
app.get('/api/repo/:owner/:repo/',(_req,res)=>{
|
||||
records.push(logRecord('error','route',['failure',{httpStatus:403}]));res.status(403).end();
|
||||
});
|
||||
await listen();
|
||||
for(const action of ['refresh','extend','coauthors']){
|
||||
for(const method of ['GET','HEAD','POST']){
|
||||
await fetch(server,'/api/repo/private-owner/'+action+'/?token=SECRET',{method});
|
||||
const detail=logs.at(-1)[1];
|
||||
if(method==='POST')expect(detail.requestPath).equal('/api/repo/private-owner/'+action+'/');
|
||||
else {expect(detail).not.have.property('requestPath');expect(records.at(-1)).not.have.property('requestPath');}
|
||||
}
|
||||
}
|
||||
expect(JSON.stringify(logs)).not.include('SECRET');
|
||||
});
|
||||
|
||||
it("keeps labels bounded and removes source identities and query values", function () {
|
||||
expect(requestRoute("/api/repo/private-name/file/secret.txt?token=abc", "api")).to.equal("/api/repo/:repoId/file/:path");
|
||||
expect(requestRoute("/api/repo/secret-owner/secret-repo/readme?branch=secret", "api")).to.equal("/api/repo/:owner/:repo/readme");
|
||||
|
||||
@@ -106,7 +106,7 @@ describe('structured logging privacy and context', function () {
|
||||
const requestPath='/api/repo/demo-123/file/'+ 'a'.repeat(1950);
|
||||
const frame='at <anonymous> (src/core/model/owner-alert-lifecycle.schema.ts:123456:12345)';
|
||||
const stack=Array(8).fill(frame).join('\\n');
|
||||
require('./src/core/logger').createLogger('route').error('failure',{...Object.fromEntries(['ms','firstByteMs','responseMs','bytes','bytesPerSecond','attempt','rss','memoryLimitBytes','memoryRatio','uptime','activeRequests','waiting','active','delayed','failed','oldestWaitingMs','droppedLogs','loopP95Ms','upstreamStatus'].map(key=>[key,Number.MAX_VALUE])),requestPath,errorType:'TypeError',errorStack:stack,causeType:'Error',causeStack:stack,httpStatus:500});
|
||||
require('./src/core/logger').createLogger('route').error('failure',{...Object.fromEntries(['ms','firstByteMs','responseMs','bytes','bytesPerSecond','attempt','rss','memoryLimitBytes','memoryRatio','uptime','activeRequests','waiting','active','delayed','failed','oldestWaitingMs','droppedLogs','loopP95Ms','upstreamStatus'].map(key=>[key,Number.MAX_VALUE])),requestPath,method:'GET',errorType:'TypeError',errorStack:stack,causeType:'Error',causeStack:stack,httpStatus:500});
|
||||
assert(payload.length<=4096);const row=JSON.parse(payload).raw[0];assert.equal(row.requestPath,requestPath);assert.equal(row.repoId,'demo-123');assert.equal(row.errorStack,frame);
|
||||
`],{env:{...process.env,LOG_REDIS_MIRROR:'on'},encoding:'utf8',timeout:10000});
|
||||
expect(child.status,child.stderr).equal(0);
|
||||
|
||||
Reference in new issue
Block a user