mirror of
https://github.com/tdurieux/anonymous_github.git
synced 2026-11-18 01:54:23 +01:00
feat: add operational incident evidence and recovering alerts
This commit is contained in:
1 parent
a4f441f097
commit
0f9d8c657a
18 files changed
+558
-123
No files matched your search
@@ -25,7 +25,7 @@ if (args[0] === "inspect") {
|
||||
if (args.some(a => a.includes("Config.Env"))) finish(state.sharedCache ? "SHARED_CACHE_ENABLED=true\\n" : "");
|
||||
if (args.some(a => a.includes("Labels"))) finish(state.oldProtocol || "<no value>");
|
||||
if (args.includes("{{.State.Running}}")) finish(state.stopped && !state.refuseStop ? "false" : "true");
|
||||
finish("healthy");
|
||||
finish(state.unhealthy ? "unhealthy" : "healthy");
|
||||
}
|
||||
if (args[0] === "compose") {
|
||||
if (args[1] === "build") { if (state.buildFails) finish("", 1); state.built = true; finish(); }
|
||||
@@ -45,12 +45,28 @@ finish("unexpected docker call", 2);
|
||||
const result = spawnSync("bash", [path.join(directory, "scripts/deploy.sh")], {
|
||||
encoding: "utf8", timeout: 8000,
|
||||
env: { ...process.env, PATH: directory + path.delimiter + process.env.PATH,
|
||||
COORDINATED_RESTART: String(options.coordinated ?? 0), FORCE: String(options.force ?? 1), STREAMER_REPLICAS: "4", TASK_DOCKER_STATE: statePath, TASK_DOCKER_LOG: logPath },
|
||||
APP_RELEASE: "abcdef1", DEPLOY_OPERATION: options.operation || "deploy", HEALTH_TIMEOUT: options.unhealthy ? "1" : "90", MIGRATION_HEALTH_TIMEOUT: options.unhealthy ? "1" : "900", COORDINATED_RESTART: String(options.coordinated ?? 0), FORCE: String(options.force ?? 1), STREAMER_REPLICAS: "4", TASK_DOCKER_STATE: statePath, TASK_DOCKER_LOG: logPath },
|
||||
});
|
||||
return { ...result, calls: fs.readFileSync(logPath, "utf8").trim().split("\n").map(line => JSON.parse(line)) };
|
||||
} finally { fs.rmSync(directory, { recursive: true, force: true }); }
|
||||
}
|
||||
|
||||
it("records finite deployment start, restart and rollback outcome", () => {
|
||||
const result = deploy({operation:"rollback"});
|
||||
expect(result.status, result.stderr).equal(0);
|
||||
const events = result.stdout.split("\n").filter(line=>line.startsWith("{")).map(JSON.parse);
|
||||
expect(events.map(row=>row.deploymentPhase)).deep.equal(["start","restart","finish"]);
|
||||
expect(events.at(-1)).include({deploymentOperation:"rollback",deploymentOutcome:"completed",release:"abcdef1",exitCode:0});
|
||||
expect(new Set(events.map(row=>row.deploymentId)).size).equal(1);
|
||||
expect(events.at(-1).restartMs).at.least(0);
|
||||
});
|
||||
it("records failure when API or streamers fail health checks", () => {
|
||||
const result=deploy({unhealthy:true});
|
||||
expect(result.status).not.equal(0);
|
||||
const events=result.stdout.split("\n").filter(line=>line.startsWith("{")).map(JSON.parse);
|
||||
expect(events.at(-1)).include({deploymentPhase:"finish",deploymentOutcome:"failed"});
|
||||
expect(events.at(-1).exitCode).not.equal(0);
|
||||
});
|
||||
for (const preflightFails of ["anonymous_github", "streamer"]) it(`keeps working containers when ${preflightFails} key preflight fails`, () => {
|
||||
const result = deploy({ preflightFails });
|
||||
expect(result.status).not.to.equal(0);
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
const {expect}=require('chai');
|
||||
const {randomUUID}=require('node:crypto');
|
||||
require('ts-node/register/transpile-only');
|
||||
const {incidentGroups}=require('../src/server/admin-log-investigation');
|
||||
describe('bounded incident evidence',()=>{
|
||||
const base={timestamp:'2026-11-15T10:00:00Z',service:'api',module:'route',event:'application_log',severity:'error',errorCode:'upstream_error',errorType:'Error',httpStatus:502,release:'abcdef1',errorStack:'at printError (src/server/routes/route-utils.ts:200:7)'};
|
||||
it('counts affected request UUIDs once without inventing identities for historical records',()=>{
|
||||
const id=randomUUID();const groups=incidentGroups([base,{...base,requestId:id},{...base,requestId:id},{...base,requestId:randomUUID(),timestamp:'2026-11-15T11:00:00Z'}]);
|
||||
expect(groups).length(1);expect(groups[0]).include({count:4,affectedRequests:2,uncorrelatedRecords:1,first:base.timestamp,last:'2026-11-15T11:00:00Z',regression:'unknown'});
|
||||
});
|
||||
it('compares stable stack functions across line shifts, retaining unknown for missing/partial baselines',()=>{
|
||||
const shifted={...base,errorStack:'at printError (src/server/routes/route-utils.ts:220:9)',release:'abcdef2'};
|
||||
expect(incidentGroups([shifted],{records:[base],partial:true})[0].regression).equal('known');
|
||||
expect(incidentGroups([shifted],{records:[],partial:false})[0].regression).equal('unknown');
|
||||
const different={...base,errorCode:'preparation_interrupted'};
|
||||
expect(incidentGroups([different],{records:[base],partial:true})[0].regression).equal('unknown');
|
||||
expect(incidentGroups([different],{records:[base],partial:false})[0].regression).equal('not_seen_in_baseline');
|
||||
expect(incidentGroups([base,different])).length(2);
|
||||
});
|
||||
});
|
||||
@@ -111,6 +111,22 @@ describe('structured logging privacy and context', function () {
|
||||
`],{env:{...process.env,LOG_REDIS_MIRROR:'on'},encoding:'utf8',timeout:10000});
|
||||
expect(child.status,child.stderr).equal(0);
|
||||
});
|
||||
it('reports interval drops once and retries a dropped report without resetting history', () => {
|
||||
const child=spawnSync(process.execPath,['-r','ts-node/register/transpile-only','-e',`
|
||||
const assert=require('node:assert/strict');
|
||||
const {createLogger,reportOutputDrops}=require('./src/core/logger');
|
||||
const logger=createLogger('process');let blocked=true;
|
||||
Object.defineProperty(process.stdout,'writableLength',{get:()=>blocked?300000:0});
|
||||
logger.info('request');logger.info('request');
|
||||
reportOutputDrops(fields=>logger.info('process heartbeat',fields));
|
||||
blocked=false;
|
||||
reportOutputDrops(fields=>{assert.equal(fields.droppedLogsInterval,3);logger.info('process heartbeat',fields);});
|
||||
reportOutputDrops(fields=>{assert.equal(fields.droppedLogsInterval,0);assert.equal(fields.droppedLogs,3);logger.info('process heartbeat',fields);});
|
||||
`],{env:{...process.env,LOG_REDIS_MIRROR:'off'},encoding:'utf8',timeout:10000});
|
||||
expect(child.status,child.stderr).equal(0);
|
||||
const rows=child.stdout.trim().split('\n').map(JSON.parse);
|
||||
expect(rows.map(r=>r.droppedLogsInterval)).deep.equal([3,0]);
|
||||
});
|
||||
it('writes JSON without Redis and bounds a blocked stdout pipe', () => {
|
||||
const child = spawnSync(process.execPath, ['-r', 'ts-node/register/transpile-only', '-e', `
|
||||
const { createLogger, getDroppedOutput } = require('./src/core/logger');
|
||||
|
||||
Reference in new issue
Block a user