mirror of
https://github.com/tdurieux/anonymous_github.git
synced 2026-11-18 01:54:23 +01:00
feat(admin): investigate correlated logs from the Errors page
This commit is contained in:
1 parent
f084f5cbfc
commit
8b928a4c82
14 files changed
+756
-67
No files matched your search
@@ -0,0 +1,133 @@
|
||||
const { expect } = require('chai');
|
||||
const { Response } = globalThis;
|
||||
const { randomUUID } = require('crypto');
|
||||
require('ts-node/register/transpile-only');
|
||||
const { logOptions, investigateLogs, safeRedisLog } = require('../src/server/admin-log-investigation');
|
||||
const { logContext } = require('../src/core/log-context');
|
||||
const envKeys = ['ADMIN_LOG_URL', 'ADMIN_LOG_VECTOR_URL', 'ADMIN_LOG_GRAFANA_URL', 'LOG_REDIS_MIRROR'];
|
||||
|
||||
describe('bounded admin log investigation', function () {
|
||||
this.timeout(10000);
|
||||
let saved, originalFetch, calls;
|
||||
const id = randomUUID(), job = randomUUID();
|
||||
const event = extra => ({ _time: new Date().toISOString(), event: 'job_failed', severity: 'error', module: 'queue', service: 'worker', release: 'abcdef1', requestId: id, jobId: job, errorCode: 'upstream_error', ...extra });
|
||||
beforeEach(() => {
|
||||
saved = Object.fromEntries(envKeys.map(key => [key, process.env[key]])); originalFetch = global.fetch; calls = [];
|
||||
envKeys.forEach(key => delete process.env[key]);
|
||||
global.fetch = async (url, options) => { calls.push([url, options]); return new Response(''); };
|
||||
});
|
||||
afterEach(() => { global.fetch = originalFetch; for (const key of envKeys) saved[key] === undefined ? delete process.env[key] : process.env[key] = saved[key]; });
|
||||
it('is disabled without any backend calls and reports mirror off independently', async () => {
|
||||
process.env.LOG_REDIS_MIRROR = 'off';
|
||||
const result = await investigateLogs(logOptions({}));
|
||||
expect(result).to.include({state:'disabled', redisCollecting:false}); expect(calls).to.have.length(0);
|
||||
});
|
||||
it('rejects arbitrary queries, URLs, arrays, excessive windows and injected opaque IDs', () => {
|
||||
for (const query of [{q:'*'}, {url:'http://127.0.0.1'}, {window:'7d'}, {window:['1h']}, {release:'abc" OR *'}, {service:'api OR *'}]) expect(() => logOptions(query)).to.throw();
|
||||
for (const query of [{id:'a" OR *',kind:'jobId'}, {id,kind:'url'}, {id:[id],kind:'requestId'}, {id,kind:'jobId',release:'abcdef1'}]) expect(() => logOptions(query,true)).to.throw();
|
||||
});
|
||||
it('bounds results, projects private fields, groups stable fingerprints and keeps admin context out', async () => {
|
||||
process.env.ADMIN_LOG_URL = 'http://logging.test:9428';
|
||||
global.fetch = async (url, options) => {
|
||||
calls.push([url,options]); const query = options.body.get('query');
|
||||
return new Response(query.includes('severity:error') ? [event({url:'private-owner/private-repo?token=secret',message:'<img onerror=evil>',body:'secret'}), event({requestId:randomUUID(),jobId:randomUUID()})].map(JSON.stringify).join('\n') : '');
|
||||
};
|
||||
const result = await logContext.run({requestId:randomUUID()}, () => investigateLogs(logOptions({service:'worker',release:'abcdef1'})));
|
||||
expect(result.state).to.equal('stale'); expect(result.groups).to.have.length(1); expect(result.groups[0].count).to.equal(2);
|
||||
expect(result.groups[0].sample.requestId).to.equal(id);
|
||||
expect(JSON.stringify(result)).not.to.match(/private-owner|secret|onerror/);
|
||||
expect(calls.every(([url,opts]) => url === 'http://logging.test:9428/select/logsql/query' && opts.redirect === 'error')).to.equal(true);
|
||||
expect(calls[0][1].body.get('limit')).to.equal('1001');
|
||||
expect(calls[0][1].body.get('query')).to.include('service:worker release:abcdef1');
|
||||
});
|
||||
it('sanitizes legacy Redis history and does not invent correlation IDs', () => {
|
||||
const row = logContext.run({requestId:id}, () => safeRedisLog({ts:new Date().toISOString(),level:'error',module:'route',message:'secret',raw:[{message:'invalid_request',url:'private',stack:'secret',cookies:'secret',status:400}]}));
|
||||
expect(row.message).to.equal('invalid_request'); expect(JSON.stringify(row)).not.to.match(/private|secret|requestId/);
|
||||
});
|
||||
it('distinguishes empty healthy coverage, stale telemetry and partial records', async () => {
|
||||
process.env.ADMIN_LOG_URL = 'http://logging.test';
|
||||
const heartbeats = [event({event:'process_heartbeat',service:'api',instance:randomUUID()}), ...Array.from({length:4},()=>event({event:'process_heartbeat',service:'streamer',instance:randomUUID()})), ...['download','cache','remove'].map(queue=>event({event:'queue_heartbeat',service:'api',queue}))];
|
||||
let malformed = false;
|
||||
global.fetch = async (_,opts) => new Response(opts.body.get('query').includes('severity:error') ? (malformed ? '{invalid}\n' : '') : heartbeats.map(JSON.stringify).join('\n'));
|
||||
expect((await investigateLogs(logOptions({}))).state).to.equal('empty');
|
||||
malformed = true; expect((await investigateLogs(logOptions({}))).state).to.equal('partial');
|
||||
});
|
||||
it('expands a job to its request and returns a bounded cross-service timeline', async () => {
|
||||
process.env.ADMIN_LOG_URL = 'http://logging.test'; process.env.ADMIN_LOG_GRAFANA_URL = 'https://grafana.test/d/anonymous-operations?token=do-not-expose';
|
||||
global.fetch = async (url, opts) => { calls.push([url,opts]); return new Response(JSON.stringify(event({service:opts.body.get('query').includes('NOT jobId:') ? 'api':'worker'}))); };
|
||||
const result = await investigateLogs(logOptions({kind:'jobId',id:job},true),true);
|
||||
expect(result.events.map(row=>row.service)).to.have.members(['worker','api']);
|
||||
expect(calls).to.have.length(2); expect(calls[1][1].body.get('query')).to.include(`requestId:"${id}"`);
|
||||
expect(result.grafanaUrl).to.include('var-job_id='); expect(result.grafanaUrl).not.to.include('token');
|
||||
});
|
||||
it('counts buffer loss but excludes intentional transform filtering', async () => {
|
||||
process.env.ADMIN_LOG_URL = 'http://logging.test'; process.env.ADMIN_LOG_VECTOR_URL = 'http://collector.test';
|
||||
global.fetch = async url => new Response(url.includes('collector') ? 'vector_buffer_max_size_bytes{stage="0"} 268435488\nvector_buffer_discarded_events_total{intentional="true"} 7\nvector_component_discarded_events_total{intentional="true"} 9\nvector_component_discarded_events_total{intentional="false"} 2\n' : '');
|
||||
expect((await investigateLogs(logOptions({}))).telemetry.collector).to.deep.equal({state:'reachable',dropped:9});
|
||||
});
|
||||
it('reports unreachable, malformed configuration and response overflow without backend details', async () => {
|
||||
process.env.ADMIN_LOG_URL = 'http://logging.test';
|
||||
global.fetch = async () => {throw Error('private token');};
|
||||
expect((await investigateLogs(logOptions({}))).state).to.equal('unreachable');
|
||||
global.fetch = async () => new Response('x'.repeat(2*1024*1024+1));
|
||||
expect((await investigateLogs(logOptions({}))).state).to.equal('unreachable');
|
||||
process.env.ADMIN_LOG_URL = 'http://secret:password@logging.test';
|
||||
expect((await investigateLogs(logOptions({}))).state).to.equal('misconfigured');
|
||||
});
|
||||
it('limits concurrent investigations and cancels hung backends within four seconds', async () => {
|
||||
process.env.ADMIN_LOG_URL = 'http://logging.test';
|
||||
global.fetch = async (_,opts) => new Promise((resolve,reject)=>opts.signal.addEventListener('abort',()=>reject(Error('aborted'))));
|
||||
const first = investigateLogs(logOptions({})), second = investigateLogs(logOptions({}));
|
||||
expect((await investigateLogs(logOptions({}))).state).to.equal('busy');
|
||||
expect((await Promise.all([first,second])).map(value=>value.state)).to.deep.equal(['unreachable','unreachable']);
|
||||
});
|
||||
it('aborts sibling requests immediately when another backend request fails', async () => {
|
||||
process.env.ADMIN_LOG_URL = 'http://logging.test';
|
||||
const signals = [];
|
||||
global.fetch = async (_,opts) => {
|
||||
if (opts.body.get('query').includes('severity:error')) throw Error('failed');
|
||||
signals.push(opts.signal);
|
||||
return new Promise((resolve,reject)=>opts.signal.addEventListener('abort',()=>reject(Error('aborted'))));
|
||||
};
|
||||
for (let i=0;i<4;i++) expect((await investigateLogs(logOptions({}))).state).to.equal('unreachable');
|
||||
expect(signals).to.have.length(4); expect(signals.every(signal=>signal.aborted)).to.equal(true);
|
||||
});
|
||||
it('preserves repeated timeline records and submillisecond timestamps', async () => {
|
||||
process.env.ADMIN_LOG_URL = 'http://logging.test';
|
||||
const timestamp = new Date().toISOString().replace(/\.\d+Z$/, '.123456001Z');
|
||||
const records = [event({_time:timestamp}),event({_time:timestamp}),event({_time:timestamp.replace('001Z','002Z')})];
|
||||
global.fetch = async () => new Response(records.map(JSON.stringify).join('\n'));
|
||||
const result = await investigateLogs(logOptions({kind:'requestId',id},true),true);
|
||||
expect(result.state).to.equal('ready'); expect(result.events).to.have.length(3);
|
||||
expect(result.events[0].timestamp).to.equal(timestamp);
|
||||
expect(result.events[2].timestamp).to.equal(timestamp.replace('001Z','002Z'));
|
||||
});
|
||||
it('orders mixed timestamp precision and computes correct first/last occurrences', async () => {
|
||||
process.env.ADMIN_LOG_URL = 'http://logging.test';
|
||||
const base = new Date().toISOString().replace(/\.\d+Z$/, '');
|
||||
const times = [base+'.123456001Z', base+'Z', base+'.123Z'];
|
||||
global.fetch = async (_,opts) => new Response(opts.body.get('query').includes('_time:5m') ? '' : times.map(time=>JSON.stringify(event({_time:time}))).join('\n'));
|
||||
const timeline=await investigateLogs(logOptions({kind:'requestId',id},true),true);
|
||||
expect(timeline.events.map(row=>row.timestamp)).to.deep.equal([times[1],times[2],times[0]]);
|
||||
const result=await investigateLogs(logOptions({}));
|
||||
expect(result.groups[0]).to.include({first:times[1],last:times[0],count:3});
|
||||
});
|
||||
it('keeps real admin authentication and authorization ahead of log queries', async () => {
|
||||
const express = require('express');
|
||||
const utils = require('../src/server/routes/route-utils');
|
||||
const previous = utils.getUser;
|
||||
utils.getUser = async req => ({id:'fixture',model:{id:'fixture'},isAdmin:req.headers['x-admin']==='yes'});
|
||||
const app = express(); app.use((req,res,next)=>{req.isAuthenticated=()=>req.headers['x-login']==='yes';next();});
|
||||
app.use('/api/admin', require('../src/server/routes/admin').default);
|
||||
const server = app.listen(0,'127.0.0.1'); await new Promise(resolve=>server.once('listening',resolve));
|
||||
const address = `http://127.0.0.1:${server.address().port}/api/admin/errors/investigation`;
|
||||
try {
|
||||
expect((await originalFetch(address)).status).to.equal(401);
|
||||
expect((await originalFetch(address,{headers:{'x-login':'yes'}})).status).to.equal(401);
|
||||
const response = await originalFetch(address,{headers:{'x-login':'yes','x-admin':'yes'}});
|
||||
expect(response.status).to.equal(200); expect((await response.json()).state).to.equal('disabled');
|
||||
expect((await originalFetch(address+'?q=*',{headers:{'x-login':'yes','x-admin':'yes'}})).status).to.equal(400);
|
||||
expect(calls).to.have.length(0);
|
||||
} finally { utils.getUser=previous; await new Promise(resolve=>server.close(resolve)); }
|
||||
});
|
||||
});
|
||||
@@ -81,6 +81,73 @@ describe("Vue 3 UI", function () {
|
||||
let ui;
|
||||
afterEach(() => ui?.close());
|
||||
|
||||
it("investigates grouped admin logs with bounded filters and a safe correlated timeline", async () => {
|
||||
const requestId = "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa";
|
||||
const now = new Date().toISOString();
|
||||
ui = await browser("/", {
|
||||
"/api/user": {username:"admin",isAdmin:true,status:"ready"},
|
||||
"/api/admin/errors": {entries:[],available:true,redisCollecting:false,total:0},
|
||||
"/api/admin/errors/investigation": {state:"partial",window:"1h",start:now,end:now,captured:3,groups:[{fingerprint:"fixture",count:3,first:now,last:now,sample:{service:"worker",release:"abcdef1",event:"job_failed",severity:"error",module:"queue",errorCode:"<img src=x onerror=evil>",requestId}}],telemetry:{missing:["streamers"],partial:true,instances:[],collector:{state:"unreachable",dropped:null}}},
|
||||
"/api/admin/errors/correlation": {state:"ready",start:now,end:now,events:[{timestamp:now,service:"api",event:"request_completed",requestId},{timestamp:now,service:"worker",event:"job_failed",requestId,ms:25}]},
|
||||
});
|
||||
await ui.go("/admin/errors");
|
||||
const doc = ui.window.document;
|
||||
expect(doc.body.textContent).to.include("Partial results").and.include("Collection is OFF").and.include("dropped events since collector start: unknown");
|
||||
expect(doc.querySelector(".log-table img")).to.equal(null);
|
||||
doc.querySelector(".log-trace-button").click(); await delay(50);
|
||||
expect(doc.querySelectorAll(".log-events li")).to.have.length(2);
|
||||
expect(doc.activeElement.id).to.equal("log-timeline-title");
|
||||
const request = ui.requests.find(r=>r.url.pathname==="/api/admin/errors/correlation");
|
||||
expect(request.url.searchParams.get("id")).to.equal(requestId);
|
||||
expect(request.url.searchParams.get("kind")).to.equal("requestId");
|
||||
await ui.input("#log-service","worker","change");
|
||||
await ui.input("#log-window","6h","change");
|
||||
doc.querySelector(".log-filters").dispatchEvent(new ui.window.Event("submit",{bubbles:true,cancelable:true})); await delay(40);
|
||||
const last = ui.requests.filter(r=>r.url.pathname==="/api/admin/errors/investigation").at(-1);
|
||||
expect(last.url.searchParams.get("service")).to.equal("worker"); expect(last.url.searchParams.get("window")).to.equal("6h");
|
||||
expect(ui.errors).to.deep.equal([]);
|
||||
});
|
||||
|
||||
it("keeps Redis history usable when admin central lookup is disabled or unreachable", async () => {
|
||||
let state = "disabled";
|
||||
ui = await browser("/", {
|
||||
"/api/user": {username:"admin",isAdmin:true,status:"ready"},
|
||||
"/api/admin/errors": {entries:[{ts:new Date().toISOString(),level:"error",module:"queue",message:"upstream_error",raw:[{code:"upstream_error",status:502}]}],available:true,redisCollecting:true,total:1},
|
||||
"/api/admin/errors/investigation": () => ({state,groups:[]}),
|
||||
});
|
||||
await ui.go("/admin/errors");
|
||||
expect(ui.window.document.body.textContent).to.include("Central lookup is not configured").and.include("upstream_error");
|
||||
state = "unreachable";
|
||||
ui.window.document.querySelector(".log-filters").dispatchEvent(new ui.window.Event("submit",{bubbles:true,cancelable:true})); await delay(40);
|
||||
expect(ui.window.document.body.textContent).to.include("Central logging is unreachable").and.include("Redis history is reachable");
|
||||
expect(ui.errors).to.deep.equal([]);
|
||||
});
|
||||
|
||||
it("loads older safe Redis records when a whole page was rejected", async () => {
|
||||
ui = await browser("/", {
|
||||
"/api/user": {username:"admin",isAdmin:true,status:"ready"},
|
||||
"/api/admin/errors/investigation": {state:"disabled",groups:[]},
|
||||
"/api/admin/errors": request => request.url.searchParams.get("offset") === "250"
|
||||
? {entries:[{ts:new Date().toISOString(),level:"error",module:"queue",message:"upstream_error",raw:[{code:"upstream_error",status:502}]}],available:true,total:251,nextOffset:251}
|
||||
: {entries:[],available:true,total:251,nextOffset:250,redactedEntries:250},
|
||||
});
|
||||
await ui.go("/admin/errors");
|
||||
const button=ui.window.document.querySelector(".errors-pager button");
|
||||
expect(button).not.to.equal(null); button.click(); await delay(40);
|
||||
expect(ui.window.document.body.textContent).to.include("upstream_error").and.include("invalid legacy records were omitted");
|
||||
expect(ui.errors).to.deep.equal([]);
|
||||
});
|
||||
|
||||
it("marks an old central health snapshot stale without claiming current health", async () => {
|
||||
ui = await browser("/", {
|
||||
"/api/user": {username:"admin",isAdmin:true,status:"ready"},
|
||||
"/api/admin/errors/investigation": {state:"ready",end:new Date(Date.now()-120000).toISOString(),groups:[]},
|
||||
});
|
||||
await ui.go("/admin/errors");
|
||||
expect(ui.window.document.querySelector(".log-state").textContent).to.include("snapshot is stale");
|
||||
expect(ui.errors).to.deep.equal([]);
|
||||
});
|
||||
|
||||
it("starts a deep-linked file and editor before directory listings finish", async () => {
|
||||
const pendingDirectories = [];
|
||||
let releaseFile;
|
||||
|
||||
Reference in new issue
Block a user