From 88d696ca7cfbd125a543828bc014fe586fa1b8b3 Mon Sep 17 00:00:00 2001 From: Codex Date: Sat, 14 Nov 2026 16:44:31 +0000 Subject: [PATCH] fix(admin): reserve Redis capacity for complete stats transactions --- deploy/logging/README.md | 13 ++++ scripts/test-admin-redis.js | 85 ++++++++++++++++++++++++++ src/server/routes/admin.ts | 117 +++++++++++++++++++++++------------- 3 files changed, 174 insertions(+), 41 deletions(-) create mode 100644 scripts/test-admin-redis.js diff --git a/deploy/logging/README.md b/deploy/logging/README.md index bb7256c..3d25087 100644 --- a/deploy/logging/README.md +++ b/deploy/logging/README.md @@ -302,3 +302,16 @@ pinned image, inserts synthetic records, verifies grouping/privacy/correlation, stops the backend and verifies unavailable/disabled responses, then removes its container. It does not read `.env`, production data or send notifications. UI regressions are in `test/vue-ui.test.js`; rebuild the UI before running them. + +The Redis admin reader admits at most four concurrent history/statistics operations, +including the overview's error counters. Excess history requests receive a busy +response; overview error counters retain their existing fallback. Its 256-command queue fits four complete 51-command statistics +transactions with headroom. Concurrent first requests share a connection attempt; +no offline command backlog or automatic reconnect loop is enabled. Admission +slots remain held until Redis work finishes, even if the HTTP client disconnects. + +Run `node scripts/test-admin-redis.js` to exercise the actual admin routes and +node-redis client against a disposable loopback-only Redis container. It checks +cold concurrent statistics, recent/grouped-history reads, overload rejection and +subsequent transaction integrity, unavailable fallback, and restart recovery. +The fixture uses only synthetic counters and removes its container on exit. diff --git a/scripts/test-admin-redis.js b/scripts/test-admin-redis.js new file mode 100644 index 0000000..9d13fac --- /dev/null +++ b/scripts/test-admin-redis.js @@ -0,0 +1,85 @@ +#!/usr/bin/env node +// Isolated real Redis + production admin routes/client. No .env or production access. +require('ts-node/register/transpile-only'); +const { fetch, AbortSignal } = globalThis; +const { setTimeout: sleep } = require('node:timers/promises'); +const { execFileSync } = require('node:child_process'); +const { randomUUID } = require('node:crypto'); +const assert = require('node:assert/strict'); +const express = require('express'); +const { createClient } = require('redis'); +const name = 'admin-redis-test-' + randomUUID().slice(0,8); +const docker = (...args) => execFileSync('docker',args,{encoding:'utf8',stdio:['ignore','pipe','pipe']}).trim(); +process.env.LOG_REDIS_MIRROR = 'off'; +(async () => { + let seed, server; + docker('run','-d','--name',name,'--memory','128m','--cpus','0.5','-p','127.0.0.1::6379','redis:7-alpine','redis-server','--save','','--appendonly','no'); + try { + const port = Number(docker('port',name,'6379').split(':').at(-1)); + const config = require('../src/config').default; + config.REDIS_HOSTNAME = '127.0.0.1'; config.REDIS_PORT = port; + seed = createClient({url:`redis://127.0.0.1:${port}`,socket:{reconnectStrategy:false}}); + seed.on('error',()=>{}); await seed.connect(); + const utils = require('../src/server/routes/route-utils'); + utils.getUser = async () => ({id:'fixture',model:{id:'fixture'},isAdmin:true}); + const app = express(); app.use((req,res,next)=>{req.isAuthenticated=()=>true;next();}); + app.use('/api/admin',require('../src/server/routes/admin').default); + server = app.listen(0,'127.0.0.1'); await new Promise(resolve=>server.once('listening',resolve)); + const url = `http://127.0.0.1:${server.address().port}/api/admin/errors`; + const request = async (suffix='',method='GET') => { + const response=await fetch(url+suffix,{method,signal:AbortSignal.timeout(5000)}); + return {status:response.status,data:await response.json()}; + }; + const seedHistory = async () => { + const key = hours => 'admin:errors:hourly:' + new Date(Date.now()-hours*3600000).toISOString().slice(0,13).replace(/[-T:]/g,''); + await seed.hSet(key(0),{total:'3','bucket:error':'2','bucket:warn':'1','cb:error:upstream_error':'2'}); + await seed.hSet(key(24),{total:'5'}); await seed.set('admin:errors:dropped','7'); + await seed.del('admin:errors'); + await seed.rPush('admin:errors',JSON.stringify({ts:new Date().toISOString(),level:'error',module:'queue',raw:[{code:'upstream_error',status:502}]})); + }; + const checkStats = result => { + assert.equal(result.status,200); assert.equal(result.data.available,true); + assert.equal(result.data.last24h,3); assert.equal(result.data.prev24h,5); + assert.deepEqual(result.data.severity,{error:2,warn:1,info:0}); + assert.equal(result.data.unique.error,1); assert.equal(result.data.buckets.length,24); assert.equal(result.data.dropped,7); + }; + await seedHistory(); + // Four uncached stats transactions share the first connection: 204 queue slots. + (await Promise.all(Array.from({length:4},()=>request('/stats')))).forEach(checkStats); + for(let round=0;round<3;round++) { + assert.equal((await request('','DELETE')).status,200); await seedHistory(); + const results=await Promise.all([request('/stats'),request('?limit=250'),request('?limit=1000'),request('/stats')]); + checkStats(results[0]); checkStats(results[3]); + for(const result of results.slice(1,3)) {assert.equal(result.status,200);assert.equal(result.data.total,1);assert.equal(result.data.entries.length,1);} + } + console.log('PASS cold/concurrent stats, recent and grouped-history reads with exact counters and intact transactions'); + await request('','DELETE'); await seedHistory(); + await seed.sendCommand(['CLIENT','PAUSE','1000','ALL']); + const load=await Promise.all(Array.from({length:12},()=>request('/stats'))); + assert.equal(load.filter(r=>r.status===200).length,4); + assert.equal(load.filter(r=>r.status===503 && r.data.available===false && r.data.error==='logging_busy').length,8); + load.filter(r=>r.status===200).forEach(checkStats); + checkStats(await request('/stats')); + assert.equal((await request()).data.entries.length,1); + console.log('PASS overload rejected before enqueue; same client remains usable without partial transaction corruption'); + await request('','DELETE'); // invalidate stats cache before outage + seed.destroy(); docker('stop',name); await sleep(50); + const start=Date.now(); + for(const result of await Promise.all([request('/stats'),request(),request('?limit=1000')])) { + assert.equal(result.status,200); assert.equal(result.data.available,false); + } + assert(Date.now()-start<4000); + console.log('PASS unavailable Redis returns bounded unavailable fallback'); + docker('start',name); + for(let i=0;i<30;i++) { try { if(docker('exec',name,'redis-cli','ping')==='PONG') break; } catch { /* startup */ } await sleep(100); } + config.REDIS_PORT = Number(docker('port',name,'6379').split(':').at(-1)); + seed = createClient({url:`redis://127.0.0.1:${config.REDIS_PORT}`,socket:{reconnectStrategy:false}}); seed.on('error',()=>{}); await seed.connect(); + await seedHistory(); checkStats(await request('/stats')); + assert.equal((await request()).data.entries.length,1); + console.log('PASS Redis restart recovers real route/client'); + } finally { + if(seed?.isOpen) seed.destroy(); + if(server) await new Promise(resolve=>server.close(resolve)); + docker('rm','-f',name); + } +})().then(()=>process.exit(0),error=>{console.error(error);process.exit(1);}); diff --git a/src/server/routes/admin.ts b/src/server/routes/admin.ts index 01e65df..d9111ea 100644 --- a/src/server/routes/admin.ts +++ b/src/server/routes/admin.ts @@ -43,13 +43,19 @@ const execFileAsync = promisify(execFile); const diskUsageCache = new AsyncCache(30000, 1); let errorLogClient: RedisClientType | null = null; +let errorLogConnecting: Promise | null = null; +// Each admitted operation queues at most 51 commands (49 stats reads + MULTI/EXEC). +// Keep headroom above 4 * 51, and reject excess work before touching the client. +const ERROR_LOG_CONCURRENCY = 4; +let errorLogActive = 0; let errorStatsCache: { data: unknown; ts: number } | null = null; const ERROR_STATS_CACHE_TTL = 30_000; async function getErrorLogClient(): Promise { - if (errorLogClient && errorLogClient.isOpen) return errorLogClient; - try { - errorLogClient = createClient({ + if (errorLogClient?.isReady) return errorLogClient; + if (errorLogConnecting) return errorLogConnecting; + errorLogConnecting = (async () => { + const client = createClient({ socket: { host: config.REDIS_HOSTNAME, port: config.REDIS_PORT, @@ -57,15 +63,21 @@ async function getErrorLogClient(): Promise { reconnectStrategy: false, }, disableOfflineQueue: true, - commandsQueueMaxLength: 32, + commandsQueueMaxLength: 256, }) as RedisClientType; - errorLogClient.on("error", () => undefined); - await errorLogClient.connect(); - return errorLogClient; - } catch (err) { - logger.error("error log redis connect failed", serializeError(err)); - return null; - } + client.on("error", () => undefined); + try { + await client.connect(); + errorLogClient = client; + return client; + } catch (err) { + if (client.isOpen) client.destroy(); + logger.error("error log redis connect failed", serializeError(err)); + return null; + } + })(); + try { return await errorLogConnecting; } + finally { errorLogConnecting = null; } } const router = express.Router(); @@ -456,6 +468,10 @@ router.get("/queues", dashboardCache, async (req, res) => { // the full ERROR_LOG_MAX entries on every poll — payloads can be a few KB // each once detail() enrichment is included. router.get("/errors", async (req, res) => { + if (errorLogActive >= ERROR_LOG_CONCURRENCY) { + return res.status(503).json({ available: false, error: "logging_busy" }); + } + errorLogActive++; try { const client = await getErrorLogClient(); if (!client) { @@ -499,12 +515,18 @@ router.get("/errors", async (req, res) => { }); } catch (error) { handleError(error, res, req); + } finally { + errorLogActive--; } }); // Aggregated stats from the precomputed hourly counters (HINCRBY on each // persistError). No JSON parsing of stored entries — O(48 small HGETALLs). router.get("/errors/stats", dashboardCache, async (req, res) => { + if (errorLogActive >= ERROR_LOG_CONCURRENCY) { + return res.status(503).json({ available: false, error: "logging_busy" }); + } + errorLogActive++; try { if ( errorStatsCache && @@ -632,10 +654,16 @@ router.get("/errors/stats", dashboardCache, async (req, res) => { res.json(data); } catch (error) { handleError(error, res, req); + } finally { + errorLogActive--; } }); router.delete("/errors", async (req, res) => { + if (errorLogActive >= ERROR_LOG_CONCURRENCY) { + return res.status(503).json({ available: false, error: "logging_busy" }); + } + errorLogActive++; try { const client = await getErrorLogClient(); if (!client) return res.json({ ok: true, cleared: 0 }); @@ -661,6 +689,8 @@ router.delete("/errors", async (req, res) => { res.json({ ok: true, cleared: len, hourlyCleared: hourlyKeys.length }); } catch (error) { handleError(error, res, req); + } finally { + errorLogActive--; } }); @@ -738,38 +768,43 @@ router.get("/overview", async (req, res) => { // Error stats (from Redis hourly counters) let errorStats = { last24h: 0, severity: { error: 0, warn: 0, info: 0 } }; - try { - const client = await getErrorLogClient(); - if (client) { - const nowDate = new Date(); - const keys: string[] = []; - for (let i = 23; i >= 0; i--) { - const d = new Date(nowDate.getTime() - i * 3600 * 1000); - const anchor = new Date( - Date.UTC(d.getUTCFullYear(), d.getUTCMonth(), d.getUTCDate(), d.getUTCHours()) - ); - const y = anchor.getUTCFullYear(); - const m = String(anchor.getUTCMonth() + 1).padStart(2, "0"); - const day = String(anchor.getUTCDate()).padStart(2, "0"); - const h = String(anchor.getUTCHours()).padStart(2, "0"); - keys.push(`${ERROR_LOG_HOURLY_PREFIX}${y}${m}${day}${h}`); + if (errorLogActive < ERROR_LOG_CONCURRENCY) { + errorLogActive++; + try { + const client = await getErrorLogClient(); + if (client) { + const nowDate = new Date(); + const keys: string[] = []; + for (let i = 23; i >= 0; i--) { + const d = new Date(nowDate.getTime() - i * 3600 * 1000); + const anchor = new Date( + Date.UTC(d.getUTCFullYear(), d.getUTCMonth(), d.getUTCDate(), d.getUTCHours()) + ); + const y = anchor.getUTCFullYear(); + const m = String(anchor.getUTCMonth() + 1).padStart(2, "0"); + const day = String(anchor.getUTCDate()).padStart(2, "0"); + const h = String(anchor.getUTCHours()).padStart(2, "0"); + keys.push(`${ERROR_LOG_HOURLY_PREFIX}${y}${m}${day}${h}`); + } + const pipe = client.multi(); + for (const k of keys) pipe.hGetAll(k); + const results = (await pipe.exec()) as unknown as Record[]; + let total = 0; + const sev = { error: 0, warn: 0, info: 0 }; + for (const h of results) { + const flat = h || {}; + total += parseInt(flat.total || "0", 10) || 0; + sev.error += parseInt(flat["bucket:error"] || "0", 10) || 0; + sev.warn += parseInt(flat["bucket:warn"] || "0", 10) || 0; + sev.info += parseInt(flat["bucket:info"] || "0", 10) || 0; + } + errorStats = { last24h: total, severity: sev }; } - const pipe = client.multi(); - for (const k of keys) pipe.hGetAll(k); - const results = (await pipe.exec()) as unknown as Record[]; - let total = 0; - const sev = { error: 0, warn: 0, info: 0 }; - for (const h of results) { - const flat = h || {}; - total += parseInt(flat.total || "0", 10) || 0; - sev.error += parseInt(flat["bucket:error"] || "0", 10) || 0; - sev.warn += parseInt(flat["bucket:warn"] || "0", 10) || 0; - sev.info += parseInt(flat["bucket:info"] || "0", 10) || 0; - } - errorStats = { last24h: total, severity: sev }; + } catch { + // Redis unavailable — keep defaults + } finally { + errorLogActive--; } - } catch { - // Redis unavailable — keep defaults } // Daily history (last 30 days) from DailyStatsModel