feat(admin): investigate correlated logs from the Errors page

This commit is contained in:
Codex committed 2026-11-14 16:33:37 +00:00
1 parent f084f5cbfc
commit 8b928a4c82
14 files changed
+756 -67

No files matched your search

+133
View File
@@ -0,0 +1,133 @@
const { expect } = require('chai');
const { Response } = globalThis;
const { randomUUID } = require('crypto');
require('ts-node/register/transpile-only');
const { logOptions, investigateLogs, safeRedisLog } = require('../src/server/admin-log-investigation');
const { logContext } = require('../src/core/log-context');
const envKeys = ['ADMIN_LOG_URL', 'ADMIN_LOG_VECTOR_URL', 'ADMIN_LOG_GRAFANA_URL', 'LOG_REDIS_MIRROR'];
describe('bounded admin log investigation', function () {
this.timeout(10000);
let saved, originalFetch, calls;
const id = randomUUID(), job = randomUUID();
const event = extra => ({ _time: new Date().toISOString(), event: 'job_failed', severity: 'error', module: 'queue', service: 'worker', release: 'abcdef1', requestId: id, jobId: job, errorCode: 'upstream_error', ...extra });
beforeEach(() => {
saved = Object.fromEntries(envKeys.map(key => [key, process.env[key]])); originalFetch = global.fetch; calls = [];
envKeys.forEach(key => delete process.env[key]);
global.fetch = async (url, options) => { calls.push([url, options]); return new Response(''); };
});
afterEach(() => { global.fetch = originalFetch; for (const key of envKeys) saved[key] === undefined ? delete process.env[key] : process.env[key] = saved[key]; });
it('is disabled without any backend calls and reports mirror off independently', async () => {
process.env.LOG_REDIS_MIRROR = 'off';
const result = await investigateLogs(logOptions({}));
expect(result).to.include({state:'disabled', redisCollecting:false}); expect(calls).to.have.length(0);
});
it('rejects arbitrary queries, URLs, arrays, excessive windows and injected opaque IDs', () => {
for (const query of [{q:'*'}, {url:'http://127.0.0.1'}, {window:'7d'}, {window:['1h']}, {release:'abc" OR *'}, {service:'api OR *'}]) expect(() => logOptions(query)).to.throw();
for (const query of [{id:'a" OR *',kind:'jobId'}, {id,kind:'url'}, {id:[id],kind:'requestId'}, {id,kind:'jobId',release:'abcdef1'}]) expect(() => logOptions(query,true)).to.throw();
});
it('bounds results, projects private fields, groups stable fingerprints and keeps admin context out', async () => {
process.env.ADMIN_LOG_URL = 'http://logging.test:9428';
global.fetch = async (url, options) => {
calls.push([url,options]); const query = options.body.get('query');
return new Response(query.includes('severity:error') ? [event({url:'private-owner/private-repo?token=secret',message:'<img onerror=evil>',body:'secret'}), event({requestId:randomUUID(),jobId:randomUUID()})].map(JSON.stringify).join('\n') : '');
};
const result = await logContext.run({requestId:randomUUID()}, () => investigateLogs(logOptions({service:'worker',release:'abcdef1'})));
expect(result.state).to.equal('stale'); expect(result.groups).to.have.length(1); expect(result.groups[0].count).to.equal(2);
expect(result.groups[0].sample.requestId).to.equal(id);
expect(JSON.stringify(result)).not.to.match(/private-owner|secret|onerror/);
expect(calls.every(([url,opts]) => url === 'http://logging.test:9428/select/logsql/query' && opts.redirect === 'error')).to.equal(true);
expect(calls[0][1].body.get('limit')).to.equal('1001');
expect(calls[0][1].body.get('query')).to.include('service:worker release:abcdef1');
});
it('sanitizes legacy Redis history and does not invent correlation IDs', () => {
const row = logContext.run({requestId:id}, () => safeRedisLog({ts:new Date().toISOString(),level:'error',module:'route',message:'secret',raw:[{message:'invalid_request',url:'private',stack:'secret',cookies:'secret',status:400}]}));
expect(row.message).to.equal('invalid_request'); expect(JSON.stringify(row)).not.to.match(/private|secret|requestId/);
});
it('distinguishes empty healthy coverage, stale telemetry and partial records', async () => {
process.env.ADMIN_LOG_URL = 'http://logging.test';
const heartbeats = [event({event:'process_heartbeat',service:'api',instance:randomUUID()}), ...Array.from({length:4},()=>event({event:'process_heartbeat',service:'streamer',instance:randomUUID()})), ...['download','cache','remove'].map(queue=>event({event:'queue_heartbeat',service:'api',queue}))];
let malformed = false;
global.fetch = async (_,opts) => new Response(opts.body.get('query').includes('severity:error') ? (malformed ? '{invalid}\n' : '') : heartbeats.map(JSON.stringify).join('\n'));
expect((await investigateLogs(logOptions({}))).state).to.equal('empty');
malformed = true; expect((await investigateLogs(logOptions({}))).state).to.equal('partial');
});
it('expands a job to its request and returns a bounded cross-service timeline', async () => {
process.env.ADMIN_LOG_URL = 'http://logging.test'; process.env.ADMIN_LOG_GRAFANA_URL = 'https://grafana.test/d/anonymous-operations?token=do-not-expose';
global.fetch = async (url, opts) => { calls.push([url,opts]); return new Response(JSON.stringify(event({service:opts.body.get('query').includes('NOT jobId:') ? 'api':'worker'}))); };
const result = await investigateLogs(logOptions({kind:'jobId',id:job},true),true);
expect(result.events.map(row=>row.service)).to.have.members(['worker','api']);
expect(calls).to.have.length(2); expect(calls[1][1].body.get('query')).to.include(`requestId:"${id}"`);
expect(result.grafanaUrl).to.include('var-job_id='); expect(result.grafanaUrl).not.to.include('token');
});
it('counts buffer loss but excludes intentional transform filtering', async () => {
process.env.ADMIN_LOG_URL = 'http://logging.test'; process.env.ADMIN_LOG_VECTOR_URL = 'http://collector.test';
global.fetch = async url => new Response(url.includes('collector') ? 'vector_buffer_max_size_bytes{stage="0"} 268435488\nvector_buffer_discarded_events_total{intentional="true"} 7\nvector_component_discarded_events_total{intentional="true"} 9\nvector_component_discarded_events_total{intentional="false"} 2\n' : '');
expect((await investigateLogs(logOptions({}))).telemetry.collector).to.deep.equal({state:'reachable',dropped:9});
});
it('reports unreachable, malformed configuration and response overflow without backend details', async () => {
process.env.ADMIN_LOG_URL = 'http://logging.test';
global.fetch = async () => {throw Error('private token');};
expect((await investigateLogs(logOptions({}))).state).to.equal('unreachable');
global.fetch = async () => new Response('x'.repeat(2*1024*1024+1));
expect((await investigateLogs(logOptions({}))).state).to.equal('unreachable');
process.env.ADMIN_LOG_URL = 'http://secret:password@logging.test';
expect((await investigateLogs(logOptions({}))).state).to.equal('misconfigured');
});
it('limits concurrent investigations and cancels hung backends within four seconds', async () => {
process.env.ADMIN_LOG_URL = 'http://logging.test';
global.fetch = async (_,opts) => new Promise((resolve,reject)=>opts.signal.addEventListener('abort',()=>reject(Error('aborted'))));
const first = investigateLogs(logOptions({})), second = investigateLogs(logOptions({}));
expect((await investigateLogs(logOptions({}))).state).to.equal('busy');
expect((await Promise.all([first,second])).map(value=>value.state)).to.deep.equal(['unreachable','unreachable']);
});
it('aborts sibling requests immediately when another backend request fails', async () => {
process.env.ADMIN_LOG_URL = 'http://logging.test';
const signals = [];
global.fetch = async (_,opts) => {
if (opts.body.get('query').includes('severity:error')) throw Error('failed');
signals.push(opts.signal);
return new Promise((resolve,reject)=>opts.signal.addEventListener('abort',()=>reject(Error('aborted'))));
};
for (let i=0;i<4;i++) expect((await investigateLogs(logOptions({}))).state).to.equal('unreachable');
expect(signals).to.have.length(4); expect(signals.every(signal=>signal.aborted)).to.equal(true);
});
it('preserves repeated timeline records and submillisecond timestamps', async () => {
process.env.ADMIN_LOG_URL = 'http://logging.test';
const timestamp = new Date().toISOString().replace(/\.\d+Z$/, '.123456001Z');
const records = [event({_time:timestamp}),event({_time:timestamp}),event({_time:timestamp.replace('001Z','002Z')})];
global.fetch = async () => new Response(records.map(JSON.stringify).join('\n'));
const result = await investigateLogs(logOptions({kind:'requestId',id},true),true);
expect(result.state).to.equal('ready'); expect(result.events).to.have.length(3);
expect(result.events[0].timestamp).to.equal(timestamp);
expect(result.events[2].timestamp).to.equal(timestamp.replace('001Z','002Z'));
});
it('orders mixed timestamp precision and computes correct first/last occurrences', async () => {
process.env.ADMIN_LOG_URL = 'http://logging.test';
const base = new Date().toISOString().replace(/\.\d+Z$/, '');
const times = [base+'.123456001Z', base+'Z', base+'.123Z'];
global.fetch = async (_,opts) => new Response(opts.body.get('query').includes('_time:5m') ? '' : times.map(time=>JSON.stringify(event({_time:time}))).join('\n'));
const timeline=await investigateLogs(logOptions({kind:'requestId',id},true),true);
expect(timeline.events.map(row=>row.timestamp)).to.deep.equal([times[1],times[2],times[0]]);
const result=await investigateLogs(logOptions({}));
expect(result.groups[0]).to.include({first:times[1],last:times[0],count:3});
});
it('keeps real admin authentication and authorization ahead of log queries', async () => {
const express = require('express');
const utils = require('../src/server/routes/route-utils');
const previous = utils.getUser;
utils.getUser = async req => ({id:'fixture',model:{id:'fixture'},isAdmin:req.headers['x-admin']==='yes'});
const app = express(); app.use((req,res,next)=>{req.isAuthenticated=()=>req.headers['x-login']==='yes';next();});
app.use('/api/admin', require('../src/server/routes/admin').default);
const server = app.listen(0,'127.0.0.1'); await new Promise(resolve=>server.once('listening',resolve));
const address = `http://127.0.0.1:${server.address().port}/api/admin/errors/investigation`;
try {
expect((await originalFetch(address)).status).to.equal(401);
expect((await originalFetch(address,{headers:{'x-login':'yes'}})).status).to.equal(401);
const response = await originalFetch(address,{headers:{'x-login':'yes','x-admin':'yes'}});
expect(response.status).to.equal(200); expect((await response.json()).state).to.equal('disabled');
expect((await originalFetch(address+'?q=*',{headers:{'x-login':'yes','x-admin':'yes'}})).status).to.equal(400);
expect(calls).to.have.length(0);
} finally { utils.getUser=previous; await new Promise(resolve=>server.close(resolve)); }
});
});
+67
View File
@@ -81,6 +81,73 @@ describe("Vue 3 UI", function () {
let ui;
afterEach(() => ui?.close());
it("investigates grouped admin logs with bounded filters and a safe correlated timeline", async () => {
const requestId = "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa";
const now = new Date().toISOString();
ui = await browser("/", {
"/api/user": {username:"admin",isAdmin:true,status:"ready"},
"/api/admin/errors": {entries:[],available:true,redisCollecting:false,total:0},
"/api/admin/errors/investigation": {state:"partial",window:"1h",start:now,end:now,captured:3,groups:[{fingerprint:"fixture",count:3,first:now,last:now,sample:{service:"worker",release:"abcdef1",event:"job_failed",severity:"error",module:"queue",errorCode:"<img src=x onerror=evil>",requestId}}],telemetry:{missing:["streamers"],partial:true,instances:[],collector:{state:"unreachable",dropped:null}}},
"/api/admin/errors/correlation": {state:"ready",start:now,end:now,events:[{timestamp:now,service:"api",event:"request_completed",requestId},{timestamp:now,service:"worker",event:"job_failed",requestId,ms:25}]},
});
await ui.go("/admin/errors");
const doc = ui.window.document;
expect(doc.body.textContent).to.include("Partial results").and.include("Collection is OFF").and.include("dropped events since collector start: unknown");
expect(doc.querySelector(".log-table img")).to.equal(null);
doc.querySelector(".log-trace-button").click(); await delay(50);
expect(doc.querySelectorAll(".log-events li")).to.have.length(2);
expect(doc.activeElement.id).to.equal("log-timeline-title");
const request = ui.requests.find(r=>r.url.pathname==="/api/admin/errors/correlation");
expect(request.url.searchParams.get("id")).to.equal(requestId);
expect(request.url.searchParams.get("kind")).to.equal("requestId");
await ui.input("#log-service","worker","change");
await ui.input("#log-window","6h","change");
doc.querySelector(".log-filters").dispatchEvent(new ui.window.Event("submit",{bubbles:true,cancelable:true})); await delay(40);
const last = ui.requests.filter(r=>r.url.pathname==="/api/admin/errors/investigation").at(-1);
expect(last.url.searchParams.get("service")).to.equal("worker"); expect(last.url.searchParams.get("window")).to.equal("6h");
expect(ui.errors).to.deep.equal([]);
});
it("keeps Redis history usable when admin central lookup is disabled or unreachable", async () => {
let state = "disabled";
ui = await browser("/", {
"/api/user": {username:"admin",isAdmin:true,status:"ready"},
"/api/admin/errors": {entries:[{ts:new Date().toISOString(),level:"error",module:"queue",message:"upstream_error",raw:[{code:"upstream_error",status:502}]}],available:true,redisCollecting:true,total:1},
"/api/admin/errors/investigation": () => ({state,groups:[]}),
});
await ui.go("/admin/errors");
expect(ui.window.document.body.textContent).to.include("Central lookup is not configured").and.include("upstream_error");
state = "unreachable";
ui.window.document.querySelector(".log-filters").dispatchEvent(new ui.window.Event("submit",{bubbles:true,cancelable:true})); await delay(40);
expect(ui.window.document.body.textContent).to.include("Central logging is unreachable").and.include("Redis history is reachable");
expect(ui.errors).to.deep.equal([]);
});
it("loads older safe Redis records when a whole page was rejected", async () => {
ui = await browser("/", {
"/api/user": {username:"admin",isAdmin:true,status:"ready"},
"/api/admin/errors/investigation": {state:"disabled",groups:[]},
"/api/admin/errors": request => request.url.searchParams.get("offset") === "250"
? {entries:[{ts:new Date().toISOString(),level:"error",module:"queue",message:"upstream_error",raw:[{code:"upstream_error",status:502}]}],available:true,total:251,nextOffset:251}
: {entries:[],available:true,total:251,nextOffset:250,redactedEntries:250},
});
await ui.go("/admin/errors");
const button=ui.window.document.querySelector(".errors-pager button");
expect(button).not.to.equal(null); button.click(); await delay(40);
expect(ui.window.document.body.textContent).to.include("upstream_error").and.include("invalid legacy records were omitted");
expect(ui.errors).to.deep.equal([]);
});
it("marks an old central health snapshot stale without claiming current health", async () => {
ui = await browser("/", {
"/api/user": {username:"admin",isAdmin:true,status:"ready"},
"/api/admin/errors/investigation": {state:"ready",end:new Date(Date.now()-120000).toISOString(),groups:[]},
});
await ui.go("/admin/errors");
expect(ui.window.document.querySelector(".log-state").textContent).to.include("snapshot is stale");
expect(ui.errors).to.deep.equal([]);
});
it("starts a deep-linked file and editor before directory listings finish", async () => {
const pendingDirectories = [];
let releaseFile;