fix(admin): reserve Redis capacity for complete stats transactions

This commit is contained in:
Codex committed 2026-11-14 16:44:31 +00:00
1 parent 21f2c4eb4d
commit 88d696ca7c
3 files changed
+174 -41

No files matched your search

+13
View File
@@ -302,3 +302,16 @@ pinned image, inserts synthetic records, verifies grouping/privacy/correlation,
stops the backend and verifies unavailable/disabled responses, then removes its
container. It does not read `.env`, production data or send notifications. UI
regressions are in `test/vue-ui.test.js`; rebuild the UI before running them.
The Redis admin reader admits at most four concurrent history/statistics operations,
including the overview's error counters. Excess history requests receive a busy
response; overview error counters retain their existing fallback. Its 256-command queue fits four complete 51-command statistics
transactions with headroom. Concurrent first requests share a connection attempt;
no offline command backlog or automatic reconnect loop is enabled. Admission
slots remain held until Redis work finishes, even if the HTTP client disconnects.
Run `node scripts/test-admin-redis.js` to exercise the actual admin routes and
node-redis client against a disposable loopback-only Redis container. It checks
cold concurrent statistics, recent/grouped-history reads, overload rejection and
subsequent transaction integrity, unavailable fallback, and restart recovery.
The fixture uses only synthetic counters and removes its container on exit.
+85
View File
@@ -0,0 +1,85 @@
#!/usr/bin/env node
// Isolated real Redis + production admin routes/client. No .env or production access.
require('ts-node/register/transpile-only');
const { fetch, AbortSignal } = globalThis;
const { setTimeout: sleep } = require('node:timers/promises');
const { execFileSync } = require('node:child_process');
const { randomUUID } = require('node:crypto');
const assert = require('node:assert/strict');
const express = require('express');
const { createClient } = require('redis');
const name = 'admin-redis-test-' + randomUUID().slice(0,8);
const docker = (...args) => execFileSync('docker',args,{encoding:'utf8',stdio:['ignore','pipe','pipe']}).trim();
process.env.LOG_REDIS_MIRROR = 'off';
(async () => {
let seed, server;
docker('run','-d','--name',name,'--memory','128m','--cpus','0.5','-p','127.0.0.1::6379','redis:7-alpine','redis-server','--save','','--appendonly','no');
try {
const port = Number(docker('port',name,'6379').split(':').at(-1));
const config = require('../src/config').default;
config.REDIS_HOSTNAME = '127.0.0.1'; config.REDIS_PORT = port;
seed = createClient({url:`redis://127.0.0.1:${port}`,socket:{reconnectStrategy:false}});
seed.on('error',()=>{}); await seed.connect();
const utils = require('../src/server/routes/route-utils');
utils.getUser = async () => ({id:'fixture',model:{id:'fixture'},isAdmin:true});
const app = express(); app.use((req,res,next)=>{req.isAuthenticated=()=>true;next();});
app.use('/api/admin',require('../src/server/routes/admin').default);
server = app.listen(0,'127.0.0.1'); await new Promise(resolve=>server.once('listening',resolve));
const url = `http://127.0.0.1:${server.address().port}/api/admin/errors`;
const request = async (suffix='',method='GET') => {
const response=await fetch(url+suffix,{method,signal:AbortSignal.timeout(5000)});
return {status:response.status,data:await response.json()};
};
const seedHistory = async () => {
const key = hours => 'admin:errors:hourly:' + new Date(Date.now()-hours*3600000).toISOString().slice(0,13).replace(/[-T:]/g,'');
await seed.hSet(key(0),{total:'3','bucket:error':'2','bucket:warn':'1','cb:error:upstream_error':'2'});
await seed.hSet(key(24),{total:'5'}); await seed.set('admin:errors:dropped','7');
await seed.del('admin:errors');
await seed.rPush('admin:errors',JSON.stringify({ts:new Date().toISOString(),level:'error',module:'queue',raw:[{code:'upstream_error',status:502}]}));
};
const checkStats = result => {
assert.equal(result.status,200); assert.equal(result.data.available,true);
assert.equal(result.data.last24h,3); assert.equal(result.data.prev24h,5);
assert.deepEqual(result.data.severity,{error:2,warn:1,info:0});
assert.equal(result.data.unique.error,1); assert.equal(result.data.buckets.length,24); assert.equal(result.data.dropped,7);
};
await seedHistory();
// Four uncached stats transactions share the first connection: 204 queue slots.
(await Promise.all(Array.from({length:4},()=>request('/stats')))).forEach(checkStats);
for(let round=0;round<3;round++) {
assert.equal((await request('','DELETE')).status,200); await seedHistory();
const results=await Promise.all([request('/stats'),request('?limit=250'),request('?limit=1000'),request('/stats')]);
checkStats(results[0]); checkStats(results[3]);
for(const result of results.slice(1,3)) {assert.equal(result.status,200);assert.equal(result.data.total,1);assert.equal(result.data.entries.length,1);}
}
console.log('PASS cold/concurrent stats, recent and grouped-history reads with exact counters and intact transactions');
await request('','DELETE'); await seedHistory();
await seed.sendCommand(['CLIENT','PAUSE','1000','ALL']);
const load=await Promise.all(Array.from({length:12},()=>request('/stats')));
assert.equal(load.filter(r=>r.status===200).length,4);
assert.equal(load.filter(r=>r.status===503 && r.data.available===false && r.data.error==='logging_busy').length,8);
load.filter(r=>r.status===200).forEach(checkStats);
checkStats(await request('/stats'));
assert.equal((await request()).data.entries.length,1);
console.log('PASS overload rejected before enqueue; same client remains usable without partial transaction corruption');
await request('','DELETE'); // invalidate stats cache before outage
seed.destroy(); docker('stop',name); await sleep(50);
const start=Date.now();
for(const result of await Promise.all([request('/stats'),request(),request('?limit=1000')])) {
assert.equal(result.status,200); assert.equal(result.data.available,false);
}
assert(Date.now()-start<4000);
console.log('PASS unavailable Redis returns bounded unavailable fallback');
docker('start',name);
for(let i=0;i<30;i++) { try { if(docker('exec',name,'redis-cli','ping')==='PONG') break; } catch { /* startup */ } await sleep(100); }
config.REDIS_PORT = Number(docker('port',name,'6379').split(':').at(-1));
seed = createClient({url:`redis://127.0.0.1:${config.REDIS_PORT}`,socket:{reconnectStrategy:false}}); seed.on('error',()=>{}); await seed.connect();
await seedHistory(); checkStats(await request('/stats'));
assert.equal((await request()).data.entries.length,1);
console.log('PASS Redis restart recovers real route/client');
} finally {
if(seed?.isOpen) seed.destroy();
if(server) await new Promise(resolve=>server.close(resolve));
docker('rm','-f',name);
}
})().then(()=>process.exit(0),error=>{console.error(error);process.exit(1);});
+76 -41
View File
@@ -43,13 +43,19 @@ const execFileAsync = promisify(execFile);
const diskUsageCache = new AsyncCache<string>(30000, 1);
let errorLogClient: RedisClientType | null = null;
let errorLogConnecting: Promise<RedisClientType | null> | null = null;
// Each admitted operation queues at most 51 commands (49 stats reads + MULTI/EXEC).
// Keep headroom above 4 * 51, and reject excess work before touching the client.
const ERROR_LOG_CONCURRENCY = 4;
let errorLogActive = 0;
let errorStatsCache: { data: unknown; ts: number } | null = null;
const ERROR_STATS_CACHE_TTL = 30_000;
async function getErrorLogClient(): Promise<RedisClientType | null> {
if (errorLogClient && errorLogClient.isOpen) return errorLogClient;
try {
errorLogClient = createClient({
if (errorLogClient?.isReady) return errorLogClient;
if (errorLogConnecting) return errorLogConnecting;
errorLogConnecting = (async () => {
const client = createClient({
socket: {
host: config.REDIS_HOSTNAME,
port: config.REDIS_PORT,
@@ -57,15 +63,21 @@ async function getErrorLogClient(): Promise<RedisClientType | null> {
reconnectStrategy: false,
},
disableOfflineQueue: true,
commandsQueueMaxLength: 32,
commandsQueueMaxLength: 256,
}) as RedisClientType;
errorLogClient.on("error", () => undefined);
await errorLogClient.connect();
return errorLogClient;
} catch (err) {
logger.error("error log redis connect failed", serializeError(err));
return null;
}
client.on("error", () => undefined);
try {
await client.connect();
errorLogClient = client;
return client;
} catch (err) {
if (client.isOpen) client.destroy();
logger.error("error log redis connect failed", serializeError(err));
return null;
}
})();
try { return await errorLogConnecting; }
finally { errorLogConnecting = null; }
}
const router = express.Router();
@@ -456,6 +468,10 @@ router.get("/queues", dashboardCache, async (req, res) => {
// the full ERROR_LOG_MAX entries on every poll — payloads can be a few KB
// each once detail() enrichment is included.
router.get("/errors", async (req, res) => {
if (errorLogActive >= ERROR_LOG_CONCURRENCY) {
return res.status(503).json({ available: false, error: "logging_busy" });
}
errorLogActive++;
try {
const client = await getErrorLogClient();
if (!client) {
@@ -499,12 +515,18 @@ router.get("/errors", async (req, res) => {
});
} catch (error) {
handleError(error, res, req);
} finally {
errorLogActive--;
}
});
// Aggregated stats from the precomputed hourly counters (HINCRBY on each
// persistError). No JSON parsing of stored entries — O(48 small HGETALLs).
router.get("/errors/stats", dashboardCache, async (req, res) => {
if (errorLogActive >= ERROR_LOG_CONCURRENCY) {
return res.status(503).json({ available: false, error: "logging_busy" });
}
errorLogActive++;
try {
if (
errorStatsCache &&
@@ -632,10 +654,16 @@ router.get("/errors/stats", dashboardCache, async (req, res) => {
res.json(data);
} catch (error) {
handleError(error, res, req);
} finally {
errorLogActive--;
}
});
router.delete("/errors", async (req, res) => {
if (errorLogActive >= ERROR_LOG_CONCURRENCY) {
return res.status(503).json({ available: false, error: "logging_busy" });
}
errorLogActive++;
try {
const client = await getErrorLogClient();
if (!client) return res.json({ ok: true, cleared: 0 });
@@ -661,6 +689,8 @@ router.delete("/errors", async (req, res) => {
res.json({ ok: true, cleared: len, hourlyCleared: hourlyKeys.length });
} catch (error) {
handleError(error, res, req);
} finally {
errorLogActive--;
}
});
@@ -738,38 +768,43 @@ router.get("/overview", async (req, res) => {
// Error stats (from Redis hourly counters)
let errorStats = { last24h: 0, severity: { error: 0, warn: 0, info: 0 } };
try {
const client = await getErrorLogClient();
if (client) {
const nowDate = new Date();
const keys: string[] = [];
for (let i = 23; i >= 0; i--) {
const d = new Date(nowDate.getTime() - i * 3600 * 1000);
const anchor = new Date(
Date.UTC(d.getUTCFullYear(), d.getUTCMonth(), d.getUTCDate(), d.getUTCHours())
);
const y = anchor.getUTCFullYear();
const m = String(anchor.getUTCMonth() + 1).padStart(2, "0");
const day = String(anchor.getUTCDate()).padStart(2, "0");
const h = String(anchor.getUTCHours()).padStart(2, "0");
keys.push(`${ERROR_LOG_HOURLY_PREFIX}${y}${m}${day}${h}`);
if (errorLogActive < ERROR_LOG_CONCURRENCY) {
errorLogActive++;
try {
const client = await getErrorLogClient();
if (client) {
const nowDate = new Date();
const keys: string[] = [];
for (let i = 23; i >= 0; i--) {
const d = new Date(nowDate.getTime() - i * 3600 * 1000);
const anchor = new Date(
Date.UTC(d.getUTCFullYear(), d.getUTCMonth(), d.getUTCDate(), d.getUTCHours())
);
const y = anchor.getUTCFullYear();
const m = String(anchor.getUTCMonth() + 1).padStart(2, "0");
const day = String(anchor.getUTCDate()).padStart(2, "0");
const h = String(anchor.getUTCHours()).padStart(2, "0");
keys.push(`${ERROR_LOG_HOURLY_PREFIX}${y}${m}${day}${h}`);
}
const pipe = client.multi();
for (const k of keys) pipe.hGetAll(k);
const results = (await pipe.exec()) as unknown as Record<string, string>[];
let total = 0;
const sev = { error: 0, warn: 0, info: 0 };
for (const h of results) {
const flat = h || {};
total += parseInt(flat.total || "0", 10) || 0;
sev.error += parseInt(flat["bucket:error"] || "0", 10) || 0;
sev.warn += parseInt(flat["bucket:warn"] || "0", 10) || 0;
sev.info += parseInt(flat["bucket:info"] || "0", 10) || 0;
}
errorStats = { last24h: total, severity: sev };
}
const pipe = client.multi();
for (const k of keys) pipe.hGetAll(k);
const results = (await pipe.exec()) as unknown as Record<string, string>[];
let total = 0;
const sev = { error: 0, warn: 0, info: 0 };
for (const h of results) {
const flat = h || {};
total += parseInt(flat.total || "0", 10) || 0;
sev.error += parseInt(flat["bucket:error"] || "0", 10) || 0;
sev.warn += parseInt(flat["bucket:warn"] || "0", 10) || 0;
sev.info += parseInt(flat["bucket:info"] || "0", 10) || 0;
}
errorStats = { last24h: total, severity: sev };
} catch {
// Redis unavailable — keep defaults
} finally {
errorLogActive--;
}
} catch {
// Redis unavailable — keep defaults
}
// Daily history (last 30 days) from DailyStatsModel