mirror of
https://github.com/tdurieux/anonymous_github.git
synced 2026-11-17 17:44:09 +01:00
fix(admin): reserve Redis capacity for complete stats transactions
This commit is contained in:
1 parent
21f2c4eb4d
commit
88d696ca7c
3 files changed
+174
-41
No files matched your search
@@ -302,3 +302,16 @@ pinned image, inserts synthetic records, verifies grouping/privacy/correlation,
|
||||
stops the backend and verifies unavailable/disabled responses, then removes its
|
||||
container. It does not read `.env`, production data or send notifications. UI
|
||||
regressions are in `test/vue-ui.test.js`; rebuild the UI before running them.
|
||||
|
||||
The Redis admin reader admits at most four concurrent history/statistics operations,
|
||||
including the overview's error counters. Excess history requests receive a busy
|
||||
response; overview error counters retain their existing fallback. Its 256-command queue fits four complete 51-command statistics
|
||||
transactions with headroom. Concurrent first requests share a connection attempt;
|
||||
no offline command backlog or automatic reconnect loop is enabled. Admission
|
||||
slots remain held until Redis work finishes, even if the HTTP client disconnects.
|
||||
|
||||
Run `node scripts/test-admin-redis.js` to exercise the actual admin routes and
|
||||
node-redis client against a disposable loopback-only Redis container. It checks
|
||||
cold concurrent statistics, recent/grouped-history reads, overload rejection and
|
||||
subsequent transaction integrity, unavailable fallback, and restart recovery.
|
||||
The fixture uses only synthetic counters and removes its container on exit.
|
||||
@@ -0,0 +1,85 @@
|
||||
#!/usr/bin/env node
|
||||
// Isolated real Redis + production admin routes/client. No .env or production access.
|
||||
require('ts-node/register/transpile-only');
|
||||
const { fetch, AbortSignal } = globalThis;
|
||||
const { setTimeout: sleep } = require('node:timers/promises');
|
||||
const { execFileSync } = require('node:child_process');
|
||||
const { randomUUID } = require('node:crypto');
|
||||
const assert = require('node:assert/strict');
|
||||
const express = require('express');
|
||||
const { createClient } = require('redis');
|
||||
const name = 'admin-redis-test-' + randomUUID().slice(0,8);
|
||||
const docker = (...args) => execFileSync('docker',args,{encoding:'utf8',stdio:['ignore','pipe','pipe']}).trim();
|
||||
process.env.LOG_REDIS_MIRROR = 'off';
|
||||
(async () => {
|
||||
let seed, server;
|
||||
docker('run','-d','--name',name,'--memory','128m','--cpus','0.5','-p','127.0.0.1::6379','redis:7-alpine','redis-server','--save','','--appendonly','no');
|
||||
try {
|
||||
const port = Number(docker('port',name,'6379').split(':').at(-1));
|
||||
const config = require('../src/config').default;
|
||||
config.REDIS_HOSTNAME = '127.0.0.1'; config.REDIS_PORT = port;
|
||||
seed = createClient({url:`redis://127.0.0.1:${port}`,socket:{reconnectStrategy:false}});
|
||||
seed.on('error',()=>{}); await seed.connect();
|
||||
const utils = require('../src/server/routes/route-utils');
|
||||
utils.getUser = async () => ({id:'fixture',model:{id:'fixture'},isAdmin:true});
|
||||
const app = express(); app.use((req,res,next)=>{req.isAuthenticated=()=>true;next();});
|
||||
app.use('/api/admin',require('../src/server/routes/admin').default);
|
||||
server = app.listen(0,'127.0.0.1'); await new Promise(resolve=>server.once('listening',resolve));
|
||||
const url = `http://127.0.0.1:${server.address().port}/api/admin/errors`;
|
||||
const request = async (suffix='',method='GET') => {
|
||||
const response=await fetch(url+suffix,{method,signal:AbortSignal.timeout(5000)});
|
||||
return {status:response.status,data:await response.json()};
|
||||
};
|
||||
const seedHistory = async () => {
|
||||
const key = hours => 'admin:errors:hourly:' + new Date(Date.now()-hours*3600000).toISOString().slice(0,13).replace(/[-T:]/g,'');
|
||||
await seed.hSet(key(0),{total:'3','bucket:error':'2','bucket:warn':'1','cb:error:upstream_error':'2'});
|
||||
await seed.hSet(key(24),{total:'5'}); await seed.set('admin:errors:dropped','7');
|
||||
await seed.del('admin:errors');
|
||||
await seed.rPush('admin:errors',JSON.stringify({ts:new Date().toISOString(),level:'error',module:'queue',raw:[{code:'upstream_error',status:502}]}));
|
||||
};
|
||||
const checkStats = result => {
|
||||
assert.equal(result.status,200); assert.equal(result.data.available,true);
|
||||
assert.equal(result.data.last24h,3); assert.equal(result.data.prev24h,5);
|
||||
assert.deepEqual(result.data.severity,{error:2,warn:1,info:0});
|
||||
assert.equal(result.data.unique.error,1); assert.equal(result.data.buckets.length,24); assert.equal(result.data.dropped,7);
|
||||
};
|
||||
await seedHistory();
|
||||
// Four uncached stats transactions share the first connection: 204 queue slots.
|
||||
(await Promise.all(Array.from({length:4},()=>request('/stats')))).forEach(checkStats);
|
||||
for(let round=0;round<3;round++) {
|
||||
assert.equal((await request('','DELETE')).status,200); await seedHistory();
|
||||
const results=await Promise.all([request('/stats'),request('?limit=250'),request('?limit=1000'),request('/stats')]);
|
||||
checkStats(results[0]); checkStats(results[3]);
|
||||
for(const result of results.slice(1,3)) {assert.equal(result.status,200);assert.equal(result.data.total,1);assert.equal(result.data.entries.length,1);}
|
||||
}
|
||||
console.log('PASS cold/concurrent stats, recent and grouped-history reads with exact counters and intact transactions');
|
||||
await request('','DELETE'); await seedHistory();
|
||||
await seed.sendCommand(['CLIENT','PAUSE','1000','ALL']);
|
||||
const load=await Promise.all(Array.from({length:12},()=>request('/stats')));
|
||||
assert.equal(load.filter(r=>r.status===200).length,4);
|
||||
assert.equal(load.filter(r=>r.status===503 && r.data.available===false && r.data.error==='logging_busy').length,8);
|
||||
load.filter(r=>r.status===200).forEach(checkStats);
|
||||
checkStats(await request('/stats'));
|
||||
assert.equal((await request()).data.entries.length,1);
|
||||
console.log('PASS overload rejected before enqueue; same client remains usable without partial transaction corruption');
|
||||
await request('','DELETE'); // invalidate stats cache before outage
|
||||
seed.destroy(); docker('stop',name); await sleep(50);
|
||||
const start=Date.now();
|
||||
for(const result of await Promise.all([request('/stats'),request(),request('?limit=1000')])) {
|
||||
assert.equal(result.status,200); assert.equal(result.data.available,false);
|
||||
}
|
||||
assert(Date.now()-start<4000);
|
||||
console.log('PASS unavailable Redis returns bounded unavailable fallback');
|
||||
docker('start',name);
|
||||
for(let i=0;i<30;i++) { try { if(docker('exec',name,'redis-cli','ping')==='PONG') break; } catch { /* startup */ } await sleep(100); }
|
||||
config.REDIS_PORT = Number(docker('port',name,'6379').split(':').at(-1));
|
||||
seed = createClient({url:`redis://127.0.0.1:${config.REDIS_PORT}`,socket:{reconnectStrategy:false}}); seed.on('error',()=>{}); await seed.connect();
|
||||
await seedHistory(); checkStats(await request('/stats'));
|
||||
assert.equal((await request()).data.entries.length,1);
|
||||
console.log('PASS Redis restart recovers real route/client');
|
||||
} finally {
|
||||
if(seed?.isOpen) seed.destroy();
|
||||
if(server) await new Promise(resolve=>server.close(resolve));
|
||||
docker('rm','-f',name);
|
||||
}
|
||||
})().then(()=>process.exit(0),error=>{console.error(error);process.exit(1);});
|
||||
+76
-41
@@ -43,13 +43,19 @@ const execFileAsync = promisify(execFile);
|
||||
const diskUsageCache = new AsyncCache<string>(30000, 1);
|
||||
|
||||
let errorLogClient: RedisClientType | null = null;
|
||||
let errorLogConnecting: Promise<RedisClientType | null> | null = null;
|
||||
// Each admitted operation queues at most 51 commands (49 stats reads + MULTI/EXEC).
|
||||
// Keep headroom above 4 * 51, and reject excess work before touching the client.
|
||||
const ERROR_LOG_CONCURRENCY = 4;
|
||||
let errorLogActive = 0;
|
||||
|
||||
let errorStatsCache: { data: unknown; ts: number } | null = null;
|
||||
const ERROR_STATS_CACHE_TTL = 30_000;
|
||||
async function getErrorLogClient(): Promise<RedisClientType | null> {
|
||||
if (errorLogClient && errorLogClient.isOpen) return errorLogClient;
|
||||
try {
|
||||
errorLogClient = createClient({
|
||||
if (errorLogClient?.isReady) return errorLogClient;
|
||||
if (errorLogConnecting) return errorLogConnecting;
|
||||
errorLogConnecting = (async () => {
|
||||
const client = createClient({
|
||||
socket: {
|
||||
host: config.REDIS_HOSTNAME,
|
||||
port: config.REDIS_PORT,
|
||||
@@ -57,15 +63,21 @@ async function getErrorLogClient(): Promise<RedisClientType | null> {
|
||||
reconnectStrategy: false,
|
||||
},
|
||||
disableOfflineQueue: true,
|
||||
commandsQueueMaxLength: 32,
|
||||
commandsQueueMaxLength: 256,
|
||||
}) as RedisClientType;
|
||||
errorLogClient.on("error", () => undefined);
|
||||
await errorLogClient.connect();
|
||||
return errorLogClient;
|
||||
} catch (err) {
|
||||
logger.error("error log redis connect failed", serializeError(err));
|
||||
return null;
|
||||
}
|
||||
client.on("error", () => undefined);
|
||||
try {
|
||||
await client.connect();
|
||||
errorLogClient = client;
|
||||
return client;
|
||||
} catch (err) {
|
||||
if (client.isOpen) client.destroy();
|
||||
logger.error("error log redis connect failed", serializeError(err));
|
||||
return null;
|
||||
}
|
||||
})();
|
||||
try { return await errorLogConnecting; }
|
||||
finally { errorLogConnecting = null; }
|
||||
}
|
||||
|
||||
const router = express.Router();
|
||||
@@ -456,6 +468,10 @@ router.get("/queues", dashboardCache, async (req, res) => {
|
||||
// the full ERROR_LOG_MAX entries on every poll — payloads can be a few KB
|
||||
// each once detail() enrichment is included.
|
||||
router.get("/errors", async (req, res) => {
|
||||
if (errorLogActive >= ERROR_LOG_CONCURRENCY) {
|
||||
return res.status(503).json({ available: false, error: "logging_busy" });
|
||||
}
|
||||
errorLogActive++;
|
||||
try {
|
||||
const client = await getErrorLogClient();
|
||||
if (!client) {
|
||||
@@ -499,12 +515,18 @@ router.get("/errors", async (req, res) => {
|
||||
});
|
||||
} catch (error) {
|
||||
handleError(error, res, req);
|
||||
} finally {
|
||||
errorLogActive--;
|
||||
}
|
||||
});
|
||||
|
||||
// Aggregated stats from the precomputed hourly counters (HINCRBY on each
|
||||
// persistError). No JSON parsing of stored entries — O(48 small HGETALLs).
|
||||
router.get("/errors/stats", dashboardCache, async (req, res) => {
|
||||
if (errorLogActive >= ERROR_LOG_CONCURRENCY) {
|
||||
return res.status(503).json({ available: false, error: "logging_busy" });
|
||||
}
|
||||
errorLogActive++;
|
||||
try {
|
||||
if (
|
||||
errorStatsCache &&
|
||||
@@ -632,10 +654,16 @@ router.get("/errors/stats", dashboardCache, async (req, res) => {
|
||||
res.json(data);
|
||||
} catch (error) {
|
||||
handleError(error, res, req);
|
||||
} finally {
|
||||
errorLogActive--;
|
||||
}
|
||||
});
|
||||
|
||||
router.delete("/errors", async (req, res) => {
|
||||
if (errorLogActive >= ERROR_LOG_CONCURRENCY) {
|
||||
return res.status(503).json({ available: false, error: "logging_busy" });
|
||||
}
|
||||
errorLogActive++;
|
||||
try {
|
||||
const client = await getErrorLogClient();
|
||||
if (!client) return res.json({ ok: true, cleared: 0 });
|
||||
@@ -661,6 +689,8 @@ router.delete("/errors", async (req, res) => {
|
||||
res.json({ ok: true, cleared: len, hourlyCleared: hourlyKeys.length });
|
||||
} catch (error) {
|
||||
handleError(error, res, req);
|
||||
} finally {
|
||||
errorLogActive--;
|
||||
}
|
||||
});
|
||||
|
||||
@@ -738,38 +768,43 @@ router.get("/overview", async (req, res) => {
|
||||
|
||||
// Error stats (from Redis hourly counters)
|
||||
let errorStats = { last24h: 0, severity: { error: 0, warn: 0, info: 0 } };
|
||||
try {
|
||||
const client = await getErrorLogClient();
|
||||
if (client) {
|
||||
const nowDate = new Date();
|
||||
const keys: string[] = [];
|
||||
for (let i = 23; i >= 0; i--) {
|
||||
const d = new Date(nowDate.getTime() - i * 3600 * 1000);
|
||||
const anchor = new Date(
|
||||
Date.UTC(d.getUTCFullYear(), d.getUTCMonth(), d.getUTCDate(), d.getUTCHours())
|
||||
);
|
||||
const y = anchor.getUTCFullYear();
|
||||
const m = String(anchor.getUTCMonth() + 1).padStart(2, "0");
|
||||
const day = String(anchor.getUTCDate()).padStart(2, "0");
|
||||
const h = String(anchor.getUTCHours()).padStart(2, "0");
|
||||
keys.push(`${ERROR_LOG_HOURLY_PREFIX}${y}${m}${day}${h}`);
|
||||
if (errorLogActive < ERROR_LOG_CONCURRENCY) {
|
||||
errorLogActive++;
|
||||
try {
|
||||
const client = await getErrorLogClient();
|
||||
if (client) {
|
||||
const nowDate = new Date();
|
||||
const keys: string[] = [];
|
||||
for (let i = 23; i >= 0; i--) {
|
||||
const d = new Date(nowDate.getTime() - i * 3600 * 1000);
|
||||
const anchor = new Date(
|
||||
Date.UTC(d.getUTCFullYear(), d.getUTCMonth(), d.getUTCDate(), d.getUTCHours())
|
||||
);
|
||||
const y = anchor.getUTCFullYear();
|
||||
const m = String(anchor.getUTCMonth() + 1).padStart(2, "0");
|
||||
const day = String(anchor.getUTCDate()).padStart(2, "0");
|
||||
const h = String(anchor.getUTCHours()).padStart(2, "0");
|
||||
keys.push(`${ERROR_LOG_HOURLY_PREFIX}${y}${m}${day}${h}`);
|
||||
}
|
||||
const pipe = client.multi();
|
||||
for (const k of keys) pipe.hGetAll(k);
|
||||
const results = (await pipe.exec()) as unknown as Record<string, string>[];
|
||||
let total = 0;
|
||||
const sev = { error: 0, warn: 0, info: 0 };
|
||||
for (const h of results) {
|
||||
const flat = h || {};
|
||||
total += parseInt(flat.total || "0", 10) || 0;
|
||||
sev.error += parseInt(flat["bucket:error"] || "0", 10) || 0;
|
||||
sev.warn += parseInt(flat["bucket:warn"] || "0", 10) || 0;
|
||||
sev.info += parseInt(flat["bucket:info"] || "0", 10) || 0;
|
||||
}
|
||||
errorStats = { last24h: total, severity: sev };
|
||||
}
|
||||
const pipe = client.multi();
|
||||
for (const k of keys) pipe.hGetAll(k);
|
||||
const results = (await pipe.exec()) as unknown as Record<string, string>[];
|
||||
let total = 0;
|
||||
const sev = { error: 0, warn: 0, info: 0 };
|
||||
for (const h of results) {
|
||||
const flat = h || {};
|
||||
total += parseInt(flat.total || "0", 10) || 0;
|
||||
sev.error += parseInt(flat["bucket:error"] || "0", 10) || 0;
|
||||
sev.warn += parseInt(flat["bucket:warn"] || "0", 10) || 0;
|
||||
sev.info += parseInt(flat["bucket:info"] || "0", 10) || 0;
|
||||
}
|
||||
errorStats = { last24h: total, severity: sev };
|
||||
} catch {
|
||||
// Redis unavailable — keep defaults
|
||||
} finally {
|
||||
errorLogActive--;
|
||||
}
|
||||
} catch {
|
||||
// Redis unavailable — keep defaults
|
||||
}
|
||||
|
||||
// Daily history (last 30 days) from DailyStatsModel
|
||||
|
||||
Reference in new issue
Block a user