mirror of
https://github.com/tdurieux/anonymous_github.git
synced 2026-11-17 17:44:09 +01:00
fix(admin): reserve Redis capacity for complete stats transactions
This commit is contained in:
1 parent
21f2c4eb4d
commit
88d696ca7c
3 files changed
+174
-41
No files matched your search
+76
-41
@@ -43,13 +43,19 @@ const execFileAsync = promisify(execFile);
|
||||
const diskUsageCache = new AsyncCache<string>(30000, 1);
|
||||
|
||||
let errorLogClient: RedisClientType | null = null;
|
||||
let errorLogConnecting: Promise<RedisClientType | null> | null = null;
|
||||
// Each admitted operation queues at most 51 commands (49 stats reads + MULTI/EXEC).
|
||||
// Keep headroom above 4 * 51, and reject excess work before touching the client.
|
||||
const ERROR_LOG_CONCURRENCY = 4;
|
||||
let errorLogActive = 0;
|
||||
|
||||
let errorStatsCache: { data: unknown; ts: number } | null = null;
|
||||
const ERROR_STATS_CACHE_TTL = 30_000;
|
||||
async function getErrorLogClient(): Promise<RedisClientType | null> {
|
||||
if (errorLogClient && errorLogClient.isOpen) return errorLogClient;
|
||||
try {
|
||||
errorLogClient = createClient({
|
||||
if (errorLogClient?.isReady) return errorLogClient;
|
||||
if (errorLogConnecting) return errorLogConnecting;
|
||||
errorLogConnecting = (async () => {
|
||||
const client = createClient({
|
||||
socket: {
|
||||
host: config.REDIS_HOSTNAME,
|
||||
port: config.REDIS_PORT,
|
||||
@@ -57,15 +63,21 @@ async function getErrorLogClient(): Promise<RedisClientType | null> {
|
||||
reconnectStrategy: false,
|
||||
},
|
||||
disableOfflineQueue: true,
|
||||
commandsQueueMaxLength: 32,
|
||||
commandsQueueMaxLength: 256,
|
||||
}) as RedisClientType;
|
||||
errorLogClient.on("error", () => undefined);
|
||||
await errorLogClient.connect();
|
||||
return errorLogClient;
|
||||
} catch (err) {
|
||||
logger.error("error log redis connect failed", serializeError(err));
|
||||
return null;
|
||||
}
|
||||
client.on("error", () => undefined);
|
||||
try {
|
||||
await client.connect();
|
||||
errorLogClient = client;
|
||||
return client;
|
||||
} catch (err) {
|
||||
if (client.isOpen) client.destroy();
|
||||
logger.error("error log redis connect failed", serializeError(err));
|
||||
return null;
|
||||
}
|
||||
})();
|
||||
try { return await errorLogConnecting; }
|
||||
finally { errorLogConnecting = null; }
|
||||
}
|
||||
|
||||
const router = express.Router();
|
||||
@@ -456,6 +468,10 @@ router.get("/queues", dashboardCache, async (req, res) => {
|
||||
// the full ERROR_LOG_MAX entries on every poll — payloads can be a few KB
|
||||
// each once detail() enrichment is included.
|
||||
router.get("/errors", async (req, res) => {
|
||||
if (errorLogActive >= ERROR_LOG_CONCURRENCY) {
|
||||
return res.status(503).json({ available: false, error: "logging_busy" });
|
||||
}
|
||||
errorLogActive++;
|
||||
try {
|
||||
const client = await getErrorLogClient();
|
||||
if (!client) {
|
||||
@@ -499,12 +515,18 @@ router.get("/errors", async (req, res) => {
|
||||
});
|
||||
} catch (error) {
|
||||
handleError(error, res, req);
|
||||
} finally {
|
||||
errorLogActive--;
|
||||
}
|
||||
});
|
||||
|
||||
// Aggregated stats from the precomputed hourly counters (HINCRBY on each
|
||||
// persistError). No JSON parsing of stored entries — O(48 small HGETALLs).
|
||||
router.get("/errors/stats", dashboardCache, async (req, res) => {
|
||||
if (errorLogActive >= ERROR_LOG_CONCURRENCY) {
|
||||
return res.status(503).json({ available: false, error: "logging_busy" });
|
||||
}
|
||||
errorLogActive++;
|
||||
try {
|
||||
if (
|
||||
errorStatsCache &&
|
||||
@@ -632,10 +654,16 @@ router.get("/errors/stats", dashboardCache, async (req, res) => {
|
||||
res.json(data);
|
||||
} catch (error) {
|
||||
handleError(error, res, req);
|
||||
} finally {
|
||||
errorLogActive--;
|
||||
}
|
||||
});
|
||||
|
||||
router.delete("/errors", async (req, res) => {
|
||||
if (errorLogActive >= ERROR_LOG_CONCURRENCY) {
|
||||
return res.status(503).json({ available: false, error: "logging_busy" });
|
||||
}
|
||||
errorLogActive++;
|
||||
try {
|
||||
const client = await getErrorLogClient();
|
||||
if (!client) return res.json({ ok: true, cleared: 0 });
|
||||
@@ -661,6 +689,8 @@ router.delete("/errors", async (req, res) => {
|
||||
res.json({ ok: true, cleared: len, hourlyCleared: hourlyKeys.length });
|
||||
} catch (error) {
|
||||
handleError(error, res, req);
|
||||
} finally {
|
||||
errorLogActive--;
|
||||
}
|
||||
});
|
||||
|
||||
@@ -738,38 +768,43 @@ router.get("/overview", async (req, res) => {
|
||||
|
||||
// Error stats (from Redis hourly counters)
|
||||
let errorStats = { last24h: 0, severity: { error: 0, warn: 0, info: 0 } };
|
||||
try {
|
||||
const client = await getErrorLogClient();
|
||||
if (client) {
|
||||
const nowDate = new Date();
|
||||
const keys: string[] = [];
|
||||
for (let i = 23; i >= 0; i--) {
|
||||
const d = new Date(nowDate.getTime() - i * 3600 * 1000);
|
||||
const anchor = new Date(
|
||||
Date.UTC(d.getUTCFullYear(), d.getUTCMonth(), d.getUTCDate(), d.getUTCHours())
|
||||
);
|
||||
const y = anchor.getUTCFullYear();
|
||||
const m = String(anchor.getUTCMonth() + 1).padStart(2, "0");
|
||||
const day = String(anchor.getUTCDate()).padStart(2, "0");
|
||||
const h = String(anchor.getUTCHours()).padStart(2, "0");
|
||||
keys.push(`${ERROR_LOG_HOURLY_PREFIX}${y}${m}${day}${h}`);
|
||||
if (errorLogActive < ERROR_LOG_CONCURRENCY) {
|
||||
errorLogActive++;
|
||||
try {
|
||||
const client = await getErrorLogClient();
|
||||
if (client) {
|
||||
const nowDate = new Date();
|
||||
const keys: string[] = [];
|
||||
for (let i = 23; i >= 0; i--) {
|
||||
const d = new Date(nowDate.getTime() - i * 3600 * 1000);
|
||||
const anchor = new Date(
|
||||
Date.UTC(d.getUTCFullYear(), d.getUTCMonth(), d.getUTCDate(), d.getUTCHours())
|
||||
);
|
||||
const y = anchor.getUTCFullYear();
|
||||
const m = String(anchor.getUTCMonth() + 1).padStart(2, "0");
|
||||
const day = String(anchor.getUTCDate()).padStart(2, "0");
|
||||
const h = String(anchor.getUTCHours()).padStart(2, "0");
|
||||
keys.push(`${ERROR_LOG_HOURLY_PREFIX}${y}${m}${day}${h}`);
|
||||
}
|
||||
const pipe = client.multi();
|
||||
for (const k of keys) pipe.hGetAll(k);
|
||||
const results = (await pipe.exec()) as unknown as Record<string, string>[];
|
||||
let total = 0;
|
||||
const sev = { error: 0, warn: 0, info: 0 };
|
||||
for (const h of results) {
|
||||
const flat = h || {};
|
||||
total += parseInt(flat.total || "0", 10) || 0;
|
||||
sev.error += parseInt(flat["bucket:error"] || "0", 10) || 0;
|
||||
sev.warn += parseInt(flat["bucket:warn"] || "0", 10) || 0;
|
||||
sev.info += parseInt(flat["bucket:info"] || "0", 10) || 0;
|
||||
}
|
||||
errorStats = { last24h: total, severity: sev };
|
||||
}
|
||||
const pipe = client.multi();
|
||||
for (const k of keys) pipe.hGetAll(k);
|
||||
const results = (await pipe.exec()) as unknown as Record<string, string>[];
|
||||
let total = 0;
|
||||
const sev = { error: 0, warn: 0, info: 0 };
|
||||
for (const h of results) {
|
||||
const flat = h || {};
|
||||
total += parseInt(flat.total || "0", 10) || 0;
|
||||
sev.error += parseInt(flat["bucket:error"] || "0", 10) || 0;
|
||||
sev.warn += parseInt(flat["bucket:warn"] || "0", 10) || 0;
|
||||
sev.info += parseInt(flat["bucket:info"] || "0", 10) || 0;
|
||||
}
|
||||
errorStats = { last24h: total, severity: sev };
|
||||
} catch {
|
||||
// Redis unavailable — keep defaults
|
||||
} finally {
|
||||
errorLogActive--;
|
||||
}
|
||||
} catch {
|
||||
// Redis unavailable — keep defaults
|
||||
}
|
||||
|
||||
// Daily history (last 30 days) from DailyStatsModel
|
||||
|
||||
Reference in new issue
Block a user