Merge remote-tracking branch 'origin/main' into integrate/source-errors

# Conflicts:
#	public/asset-manifest.json
#	public/css/all.min.css
#	public/script/vendor.min.js
This commit is contained in:
Thomas Durieux committed 2026-11-14 23:23:19 +00:00
commit bc44879b43
24 files changed
+3370 -73

No files matched your search

+47
View File
@@ -0,0 +1,47 @@
// Generate finite diagnostic allowlists from repository code, never runtime input.
const fs = require('fs'), path = require('path'), ts = require('typescript');
const files = {};
function scan(dir) {
for (const entry of fs.readdirSync(dir, {withFileTypes:true})) {
const file = path.join(dir,entry.name);
if(entry.isDirectory()) scan(file);
else if(file.endsWith('.ts') && !file.endsWith('.d.ts') && !file.endsWith('log-diagnostic-schema.ts')) {
const source=ts.createSourceFile(file,fs.readFileSync(file,'utf8'),ts.ScriptTarget.Latest,true);
const names=new Set(['<anonymous>']);
function visit(node) {
if ((ts.isFunctionDeclaration(node)||ts.isMethodDeclaration(node)||ts.isFunctionExpression(node)) && node.name && ts.isIdentifier(node.name)) names.add(node.name.text);
if(ts.isVariableDeclaration(node)&&ts.isIdentifier(node.name)&&node.initializer&&(ts.isArrowFunction(node.initializer)||ts.isFunctionExpression(node.initializer))) names.add(node.name.text);
ts.forEachChild(node,visit);
}
visit(source);
files[file]=[...names].sort();
files[file.replace(/^src\//,'build/').replace(/\.ts$/,'.js')]=files[file];
}
}
}
scan('src');
const types=['Error','TypeError','RangeError','ReferenceError','SyntaxError','URIError','EvalError','AggregateError','AnonymousError','HTTPError','RequestError','MongoServerError','MongoNetworkError','MongoServerSelectionError','ValidationError','TimeoutError','AbortError','UnknownError'];
const vrl=fs.readFileSync('deploy/logging/sanitize.vrl','utf8');
const routes=JSON.parse(vrl.match(/if includes\((\[.*?\]), record.route\)/)[1]);
fs.writeFileSync('src/core/log-diagnostic-schema.ts','// Generated by scripts/generate-log-diagnostics.js; do not edit.\nexport const LOG_SOURCE_FUNCTIONS: Record<string, readonly string[]> = '+JSON.stringify(files,null,2)+';\nexport const LOG_ERROR_TYPES = '+JSON.stringify(types)+';\nexport const LOG_ROUTES = '+JSON.stringify(routes)+';\n');
const block=`# BEGIN GENERATED DIAGNOSTICS
source_functions = ${JSON.stringify(files)}
for_each(["errorType", "causeType"]) -> |_index, key| {
value = get(record, [key]) ?? null
if includes(${JSON.stringify(types)}, value) { out = set!(out, [key], value) }
}
for_each(["errorStack", "causeStack"]) -> |_index, key| {
value = get(record, [key]) ?? null
if is_string(value) && length(string!(value)) <= 1024 {
frames = []
for_each(split(string!(value), "\\n", limit: 9)) -> |index, frame| {
parts, err = parse_regex(frame, r'^at (?P<function>[A-Za-z_$][A-Za-z0-9_$]*|<anonymous>) \\((?P<file>(?:src|build)/[A-Za-z0-9_./-]+\\.(?:ts|js)):(?P<line>[1-9][0-9]{0,5}):(?P<column>[1-9][0-9]{0,4})\\)$')
allowed = array(get(source_functions, [to_string(parts.file)]) ?? null) ?? []
if index < 8 && err == null && includes(array!(allowed), parts.function) { frames = push(frames, frame) }
}
if length(frames) > 0 { out = set!(out, [key], join!(frames, "\\n")) }
}
}
# END GENERATED DIAGNOSTICS`;
const clean=vrl.replace(/# BEGIN GENERATED DIAGNOSTICS[\s\S]*?# END GENERATED DIAGNOSTICS\n?/,'');
fs.writeFileSync('deploy/logging/sanitize.vrl',clean.replace('out._msg = out.event',()=>block+'\nout._msg = out.event'));
+3 -1
View File
@@ -25,12 +25,14 @@ app.use('/api', async (req, res) => {
{service:'streamer',module:'download',event:'request_aborted',severity:'warn',requestId:id},
{service:'api',module:'route',event:'request_failed',errorCode:'repository_unavailable',severity:'error',status:503,requestId:id},
];
if (scenario === 'diagnostics') samples = [{service:'api',module:'route',event:'application_log',severity:'error',httpStatus:500,requestId:id,method:'GET',route:'/api/repo/:repoId/file/:path',repoId:'demo-123',requestPath:'/api/repo/demo-123/file/docs/reproduce%20issue.md',errorType:'TypeError',errorStack:'at printError (build/server/routes/route-utils.js:200:7)\nat <anonymous> (build/server/routes/repository-public.js:312:11)',causeType:'RangeError',causeStack:'at <anonymous> (build/core/repository.js:410:9)'},{service:'api',module:'requests',event:'request_completed',severity:'info',status:500,requestId:id,method:'GET',route:'/api/repo/:repoId/file/:path',ms:283}];
if (scenario === 'occurrences') samples = Array.from({length:50}, (_,i) => ({service:'api',module:'route',event:'application_log',severity:'error',httpStatus:500,requestId:id,...(i%2?{method:'GET',route:'/api/repo/:repoId/file/:path'}:{})}));
if (scenario === 'many') samples = Array.from({length:50}, (_,i) => ({...samples[i%4],errorCode:'synthetic_event_'+(i+1)}));
if (req.path === '/admin/errors/investigation') return res.json({state:scenario === 'unreachable' ? 'unreachable' : scenario === 'disabled' ? 'disabled' : scenario === 'empty' ? 'empty':'ready',window:req.query.window||'1h',start,end:scenario==='stale'?start:now,captured:['disabled','unreachable','empty'].includes(scenario)?0:samples.reduce((sum,_,i)=>sum+[18,9,5,2][i%4],0),groups:['disabled','unreachable','empty'].includes(scenario)?[]:samples.map((sample,i)=>({fingerprint:String(i),sample:{...sample,release:'abcdef1234567'},count:[18,9,5,2][i%4],first:start,last:now})),telemetry:{missing:[],instances:[{service:'api',instance:'synthetic-api',last:now,dropped:0}],collector:{state:'reachable',dropped:0}}});
if (req.path === '/admin/errors/correlation') return res.json({state:'ready',start,end:now,events:samples.map(sample=>({...sample,timestamp:now}))});
if (req.path === '/admin/errors/stats') return res.json({available:true,last24h:34,prev24h:20,severity:{error:29,warn:5,info:0},unique:{error:3,warn:1,info:0},buckets:[],dropped:0});
if (req.path === '/admin/errors' && req.method === 'DELETE') historyCleared = true;
if (req.path === '/admin/errors') return res.json({available:true,redisCollecting:true,total:historyCleared?0:samples.length,max:1000,entries:historyCleared?[]:samples.map((sample,i)=>({ts:now,level:sample.severity,module:sample.module,message:sample.errorCode||sample.event,count:i+1,raw:[sample]}))});
if (req.path === '/admin/errors') return res.json({available:true,redisCollecting:true,total:historyCleared?0:samples.length,max:1000,entries:historyCleared?[]:samples.map((sample,i)=>({ts:scenario==='occurrences'?new Date(Date.now()-i*11000).toISOString():now,level:sample.severity,module:sample.module,message:sample.errorCode||sample.errorType||sample.event,count:i+1,raw:[sample]}))});
return res.json([]);
});
app.use((req,res,next)=>{ req.url=req.url.replace(/\.[a-f0-9]{10}\.min\./,'.min.'); next(); });
+30
View File
@@ -61,6 +61,36 @@ const sleep = ms => new Promise(resolve=>setTimeout(resolve,ms));
assert.equal(trace.events.length,1); assert.equal(trace.events[0].status,[200,429,502,504][i]);
}
console.log('PASS actual collector output: missing/unknown release compatibility, edge-only 502/504, excluded 200/429 with retained timelines');
const {logRecord}=require('../src/core/log-record');
const {serializeError}=require('../src/core/logger');
const {safeRedisLog}=require('../src/server/admin-log-investigation');
const {logContext}=require('../src/core/log-context');
const {requestRoute}=require('../src/core/request-monitoring');
const diagnosticId=randomUUID(), secret='SECRET_private_owner_token';
let error;
try { requestRoute(null,'api'); } catch (failure) { error=failure; } // real synthetic TypeError in application code
assert(error instanceof TypeError);
error.message='https://user:'+secret+'@example.test/private?token='+secret;
error.stack+='\n at async '+require('node:path').resolve('src/server/routes/repository-public.ts')+':224:23';
error.cause=new RangeError(secret);error.cause.stack='RangeError: '+secret+'\n at '+require('node:path').resolve('src/core/model/owner-alert-lifecycle.schema.ts')+':12:3';
const diagnostic=logContext.run({requestId:diagnosticId,requestPath:'/api/repo/demo-123/file/docs/reproduce%20issue.md',route:'/api/repo/:repoId/file/:path',method:'GET'},()=>logRecord('error','route',['', {...serializeError(error),httpStatus:500}]));
const rawSource={file:'/host/docker/fixture/log.json',message:JSON.stringify({attrs:{tag:'api'},log:JSON.stringify({...diagnostic,url:secret,stack:secret,errorStack:diagnostic.errorStack+'\n at '+secret+' (/private/'+secret+':1:2)'})})};
const projected=execFileSync('docker',['run','--rm','-i','--network','none','--memory','256m',vectorImage,'vrl','-i','/dev/stdin',fs.readFileSync('deploy/logging/sanitize.vrl','utf8')+'\nencode_json(.)'],{encoding:'utf8',input:JSON.stringify(rawSource)+'\n'}).trim().split('\n').map(line=>JSON.parse(JSON.parse(line)))[0];
assert.equal(projected.requestPath,diagnostic.requestPath);assert.equal(projected.repoId,'demo-123');assert.equal(projected.errorType,'TypeError');assert.equal(projected.causeType,'RangeError');
assert.equal(projected.causeStack,'at <anonymous> (src/core/model/owner-alert-lifecycle.schema.ts:12:3)');
assert(projected.errorStack.includes('at <anonymous> (src/server/routes/repository-public.ts:224:23)'));
assert.match(projected.errorStack,/at requestRoute \(src\/core\/request-monitoring.ts:\d+:\d+\)/);
assert(!JSON.stringify(projected).includes(secret));
const invalidPaths=['/api/repo/demo-123/file/a?token='+secret,'/api/repo/demo-123/file/%0a'+secret,'/api/repo/demo-123/file/../'+secret,'/api/repo/demo-123/file/%252e%252e/'+secret,'https://user:'+secret+'@example.test/file'];
const invalidSources=invalidPaths.map(requestPath=>({...rawSource,message:JSON.stringify({attrs:{tag:'api'},log:JSON.stringify({...diagnostic,requestPath,errorStack:'at <anonymous> (src/core/model/private.schema.ts:1:2)'})})}));
const rejected=execFileSync('docker',['run','--rm','-i','--network','none','--memory','256m',vectorImage,'vrl','-i','/dev/stdin',fs.readFileSync('deploy/logging/sanitize.vrl','utf8')+'\nencode_json(.)'],{encoding:'utf8',input:invalidSources.map(JSON.stringify).join('\n')+'\n'}).trim().split('\n').map(line=>JSON.parse(JSON.parse(line)));
assert(rejected.every(row=>!row.requestPath && !row.repoId && !row.errorStack));
assert(!JSON.stringify(rejected).includes(secret));
const ingested=await fetch(process.env.ADMIN_LOG_URL+'/insert/jsonline?_time_field=timestamp',{method:'POST',body:JSON.stringify(projected),headers:{'Content-Type':'application/stream+json'}});assert(ingested.ok);await sleep(2000);
const traced=await investigateLogs(logOptions({kind:'requestId',id:diagnosticId},true),true);
const redis=safeRedisLog({ts:diagnostic.timestamp,level:'error',module:'route',raw:[diagnostic]});
for(const row of [traced.events[0],redis.raw[0]]) {assert.equal(row.requestPath,diagnostic.requestPath);assert.equal(row.repoId,'demo-123');assert.equal(row.route,diagnostic.route);assert.equal(row.method,'GET');assert.equal(row.errorStack,projected.errorStack);assert.equal(row.causeType,'RangeError');assert(!JSON.stringify(row).includes(secret));}
console.log('PASS synthetic exception logger → pinned collector → VictoriaLogs/Redis safe API with normalized route and bounded frames');
docker('stop',name);
const unavailable=await investigateLogs(logOptions({})); assert.equal(unavailable.state,'unreachable');
delete process.env.ADMIN_LOG_URL;