feat: add operational incident evidence and recovering alerts

This commit is contained in:
Thomas Durieux committed 2026-11-15 13:45:38 +00:00
1 parent a4f441f097
commit 0f9d8c657a
18 files changed
+558 -123

No files matched your search

+18 -2
View File
@@ -25,7 +25,7 @@ if (args[0] === "inspect") {
if (args.some(a => a.includes("Config.Env"))) finish(state.sharedCache ? "SHARED_CACHE_ENABLED=true\\n" : "");
if (args.some(a => a.includes("Labels"))) finish(state.oldProtocol || "<no value>");
if (args.includes("{{.State.Running}}")) finish(state.stopped && !state.refuseStop ? "false" : "true");
finish("healthy");
finish(state.unhealthy ? "unhealthy" : "healthy");
}
if (args[0] === "compose") {
if (args[1] === "build") { if (state.buildFails) finish("", 1); state.built = true; finish(); }
@@ -45,12 +45,28 @@ finish("unexpected docker call", 2);
const result = spawnSync("bash", [path.join(directory, "scripts/deploy.sh")], {
encoding: "utf8", timeout: 8000,
env: { ...process.env, PATH: directory + path.delimiter + process.env.PATH,
COORDINATED_RESTART: String(options.coordinated ?? 0), FORCE: String(options.force ?? 1), STREAMER_REPLICAS: "4", TASK_DOCKER_STATE: statePath, TASK_DOCKER_LOG: logPath },
APP_RELEASE: "abcdef1", DEPLOY_OPERATION: options.operation || "deploy", HEALTH_TIMEOUT: options.unhealthy ? "1" : "90", MIGRATION_HEALTH_TIMEOUT: options.unhealthy ? "1" : "900", COORDINATED_RESTART: String(options.coordinated ?? 0), FORCE: String(options.force ?? 1), STREAMER_REPLICAS: "4", TASK_DOCKER_STATE: statePath, TASK_DOCKER_LOG: logPath },
});
return { ...result, calls: fs.readFileSync(logPath, "utf8").trim().split("\n").map(line => JSON.parse(line)) };
} finally { fs.rmSync(directory, { recursive: true, force: true }); }
}
it("records finite deployment start, restart and rollback outcome", () => {
const result = deploy({operation:"rollback"});
expect(result.status, result.stderr).equal(0);
const events = result.stdout.split("\n").filter(line=>line.startsWith("{")).map(JSON.parse);
expect(events.map(row=>row.deploymentPhase)).deep.equal(["start","restart","finish"]);
expect(events.at(-1)).include({deploymentOperation:"rollback",deploymentOutcome:"completed",release:"abcdef1",exitCode:0});
expect(new Set(events.map(row=>row.deploymentId)).size).equal(1);
expect(events.at(-1).restartMs).at.least(0);
});
it("records failure when API or streamers fail health checks", () => {
const result=deploy({unhealthy:true});
expect(result.status).not.equal(0);
const events=result.stdout.split("\n").filter(line=>line.startsWith("{")).map(JSON.parse);
expect(events.at(-1)).include({deploymentPhase:"finish",deploymentOutcome:"failed"});
expect(events.at(-1).exitCode).not.equal(0);
});
for (const preflightFails of ["anonymous_github", "streamer"]) it(`keeps working containers when ${preflightFails} key preflight fails`, () => {
const result = deploy({ preflightFails });
expect(result.status).not.to.equal(0);
+20
View File
@@ -0,0 +1,20 @@
const {expect}=require('chai');
const {randomUUID}=require('node:crypto');
require('ts-node/register/transpile-only');
const {incidentGroups}=require('../src/server/admin-log-investigation');
describe('bounded incident evidence',()=>{
const base={timestamp:'2026-11-15T10:00:00Z',service:'api',module:'route',event:'application_log',severity:'error',errorCode:'upstream_error',errorType:'Error',httpStatus:502,release:'abcdef1',errorStack:'at printError (src/server/routes/route-utils.ts:200:7)'};
it('counts affected request UUIDs once without inventing identities for historical records',()=>{
const id=randomUUID();const groups=incidentGroups([base,{...base,requestId:id},{...base,requestId:id},{...base,requestId:randomUUID(),timestamp:'2026-11-15T11:00:00Z'}]);
expect(groups).length(1);expect(groups[0]).include({count:4,affectedRequests:2,uncorrelatedRecords:1,first:base.timestamp,last:'2026-11-15T11:00:00Z',regression:'unknown'});
});
it('compares stable stack functions across line shifts, retaining unknown for missing/partial baselines',()=>{
const shifted={...base,errorStack:'at printError (src/server/routes/route-utils.ts:220:9)',release:'abcdef2'};
expect(incidentGroups([shifted],{records:[base],partial:true})[0].regression).equal('known');
expect(incidentGroups([shifted],{records:[],partial:false})[0].regression).equal('unknown');
const different={...base,errorCode:'preparation_interrupted'};
expect(incidentGroups([different],{records:[base],partial:true})[0].regression).equal('unknown');
expect(incidentGroups([different],{records:[base],partial:false})[0].regression).equal('not_seen_in_baseline');
expect(incidentGroups([base,different])).length(2);
});
});
+16
View File
@@ -111,6 +111,22 @@ describe('structured logging privacy and context', function () {
`],{env:{...process.env,LOG_REDIS_MIRROR:'on'},encoding:'utf8',timeout:10000});
expect(child.status,child.stderr).equal(0);
});
it('reports interval drops once and retries a dropped report without resetting history', () => {
const child=spawnSync(process.execPath,['-r','ts-node/register/transpile-only','-e',`
const assert=require('node:assert/strict');
const {createLogger,reportOutputDrops}=require('./src/core/logger');
const logger=createLogger('process');let blocked=true;
Object.defineProperty(process.stdout,'writableLength',{get:()=>blocked?300000:0});
logger.info('request');logger.info('request');
reportOutputDrops(fields=>logger.info('process heartbeat',fields));
blocked=false;
reportOutputDrops(fields=>{assert.equal(fields.droppedLogsInterval,3);logger.info('process heartbeat',fields);});
reportOutputDrops(fields=>{assert.equal(fields.droppedLogsInterval,0);assert.equal(fields.droppedLogs,3);logger.info('process heartbeat',fields);});
`],{env:{...process.env,LOG_REDIS_MIRROR:'off'},encoding:'utf8',timeout:10000});
expect(child.status,child.stderr).equal(0);
const rows=child.stdout.trim().split('\n').map(JSON.parse);
expect(rows.map(r=>r.droppedLogsInterval)).deep.equal([3,0]);
});
it('writes JSON without Redis and bounds a blocked stdout pipe', () => {
const child = spawnSync(process.execPath, ['-r', 'ts-node/register/transpile-only', '-e', `
const { createLogger, getDroppedOutput } = require('./src/core/logger');