From d99a9587f7948e76e8834d9ed6018407408c2051 Mon Sep 17 00:00:00 2001 From: Thomas Durieux <5577568+tdurieux@users.noreply.github.com> Date: Sun, 15 Nov 2026 13:52:05 +0000 Subject: [PATCH] fix: retain loss evidence and fail interrupted deployments --- deploy/logging/README.md | 6 ++++-- scripts/deploy.sh | 3 +++ src/core/logger.ts | 22 ++++++++++++++++------ test/deploy-owner-alert-migration.test.js | 8 +++++++- test/structured-logging.test.js | 19 ++++++++++++++++++- 5 files changed, 48 insertions(+), 10 deletions(-) diff --git a/deploy/logging/README.md b/deploy/logging/README.md index 6e468db..3282b01 100644 --- a/deploy/logging/README.md +++ b/deploy/logging/README.md @@ -424,10 +424,12 @@ configure `ADMIN_LOG_GRAFANA_URL` to that investigation dashboard to use them. Metrics rules link to the operations dashboard and include the exact PromQL query. No notification receiver is added or enabled. -`ApplicationLogDrops` uses `droppedLogsInterval`, the losses since the last emitted +`ApplicationLogDrops` uses `droppedLogsInterval`, the losses since the last stream-acknowledged process report, so it recovers after losses stop. `droppedLogs` remains cumulative for historical diagnosis. If emitting the report itself drops a record, the -watermark stays unchanged and the next report includes those losses. A dead +watermark stays unchanged and the next report includes those losses. Concurrent +reports before acknowledgement can repeat a delta; it is an alert signal, not an +exact window loss total. A successful write does not prove collector ingestion. A dead process or broken pair of stdout/stderr pipes still needs missing-heartbeat alerts. Older applications without interval fields cannot drive the new interval rule; roll out matching application and alert versions together. Collector/storage diff --git a/scripts/deploy.sh b/scripts/deploy.sh index f17eb83..996f9f7 100755 --- a/scripts/deploy.sh +++ b/scripts/deploy.sh @@ -53,6 +53,9 @@ deployment_finished() { exit "$code" } trap deployment_finished EXIT +trap 'exit 129' HUP +trap 'exit 130' INT +trap 'exit 143' TERM deployment_event start started 0 image_id() { docker image inspect --format '{{.Id}}' "$1" 2>/dev/null || true; } diff --git a/src/core/logger.ts b/src/core/logger.ts index d222e93..1d0a990 100644 --- a/src/core/logger.ts +++ b/src/core/logger.ts @@ -90,15 +90,18 @@ let droppedOutput = 0; let outputBroken = false; let errorOutputBroken = false; // A closed log reader must not become an uncaught EPIPE on the serving process. -process.stdout.on("error", () => { outputBroken = true; droppedOutput++; }); -process.stderr.on("error", () => { errorOutputBroken = true; droppedOutput++; }); +process.stdout.on("error", () => { outputBroken = true; }); +process.stderr.on("error", () => { errorOutputBroken = true; }); export function getDroppedOutput() { return droppedOutput; } let reportedDrops = 0; -/** Advance the interval only when emitting the report did not itself drop a record. */ +let dropReportWatermark: number | undefined; +/** Advance only after the output stream acknowledges the report's write. */ export function reportOutputDrops(report: (fields: { droppedLogs: number; droppedLogsInterval: number }) => void) { const before = droppedOutput; - report({ droppedLogs: before, droppedLogsInterval: Math.max(0, before - reportedDrops) }); - if (droppedOutput === before) reportedDrops = before; + const previous = dropReportWatermark; + dropReportWatermark = before; + try { report({ droppedLogs: before, droppedLogsInterval: Math.max(0, before - reportedDrops) }); } + finally { dropReportWatermark = previous; } } export function getInProcessDropped(): number { return droppedInProcess; @@ -282,7 +285,14 @@ function emit(level: Level, module: string, args: unknown[]) { // building an unbounded userland queue. The next heartbeat reports losses. const output = outputBroken ? process.stderr : process.stdout; if (!(outputBroken && errorOutputBroken) && output.writableLength < 256 * 1024) { - try { output.write(JSON.stringify(record) + "\n"); } + const watermark = dropReportWatermark; + try { output.write(JSON.stringify(record) + "\n", error => { + if (error) { + if (output === process.stdout) outputBroken = true; + else errorOutputBroken = true; + droppedOutput++; + } else if (watermark !== undefined) reportedDrops = Math.max(reportedDrops, watermark); + }); } catch { if (output === process.stdout) outputBroken = true; else errorOutputBroken = true; diff --git a/test/deploy-owner-alert-migration.test.js b/test/deploy-owner-alert-migration.test.js index ac38d6f..768a8e2 100644 --- a/test/deploy-owner-alert-migration.test.js +++ b/test/deploy-owner-alert-migration.test.js @@ -28,7 +28,7 @@ if (args[0] === "inspect") { finish(state.unhealthy ? "unhealthy" : "healthy"); } if (args[0] === "compose") { - if (args[1] === "build") { if (state.buildFails) finish("", 1); state.built = true; finish(); } + if (args[1] === "build") { if (state.signal) { process.kill(process.ppid, state.signal); finish(); } if (state.buildFails) finish("", 1); state.built = true; finish(); } if (args[1] === "stop") { state.stopped = true; finish(); } if (args[1] === "run" && args.at(-1).endsWith("check-github-app-key.js")) { finish("", args.includes(state.preflightFails) ? 1 : 0); } if (args[1] === "run") { if (state.reconcileFails) finish("", 1); state.reconciled = true; finish(); } @@ -60,6 +60,12 @@ finish("unexpected docker call", 2); expect(new Set(events.map(row=>row.deploymentId)).size).equal(1); expect(events.at(-1).restartMs).at.least(0); }); + for (const [signal, code] of [["SIGHUP",129],["SIGINT",130],["SIGTERM",143]]) it(`records ${signal} termination as failed`, () => { + const result=deploy({signal,operation:"rollback"}); + expect(result.status,result.stderr).equal(code); + const events=result.stdout.split("\n").filter(line=>line.startsWith("{")).map(JSON.parse); + expect(events.at(-1)).include({deploymentPhase:"finish",deploymentOutcome:"failed",exitCode:code}); + }); it("records failure when API or streamers fail health checks", () => { const result=deploy({unhealthy:true}); expect(result.status).not.equal(0); diff --git a/test/structured-logging.test.js b/test/structured-logging.test.js index 011da21..9fd165d 100644 --- a/test/structured-logging.test.js +++ b/test/structured-logging.test.js @@ -121,12 +121,29 @@ describe('structured logging privacy and context', function () { reportOutputDrops(fields=>logger.info('process heartbeat',fields)); blocked=false; reportOutputDrops(fields=>{assert.equal(fields.droppedLogsInterval,3);logger.info('process heartbeat',fields);}); - reportOutputDrops(fields=>{assert.equal(fields.droppedLogsInterval,0);assert.equal(fields.droppedLogs,3);logger.info('process heartbeat',fields);}); + setImmediate(()=>reportOutputDrops(fields=>{assert.equal(fields.droppedLogsInterval,0);assert.equal(fields.droppedLogs,3);logger.info('process heartbeat',fields);})); `],{env:{...process.env,LOG_REDIS_MIRROR:'off'},encoding:'utf8',timeout:10000}); expect(child.status,child.stderr).equal(0); const rows=child.stdout.trim().split('\n').map(JSON.parse); expect(rows.map(r=>r.droppedLogsInterval)).deep.equal([3,0]); }); + it('retains earlier losses when a report write fails asynchronously', () => { + const child=spawnSync(process.execPath,['-r','ts-node/register/transpile-only','-e',` + const assert=require('node:assert/strict'); + const {createLogger,reportOutputDrops}=require('./src/core/logger'); + const logger=createLogger('process');let blocked=true; + Object.defineProperty(process.stdout,'writableLength',{get:()=>blocked?300000:0}); + logger.info('request');logger.info('request');blocked=false; + process.stdout.write=(line,callback)=>{setImmediate(()=>callback(new Error('EPIPE')));return true;}; + reportOutputDrops(fields=>logger.info('process heartbeat',fields)); + setImmediate(()=>{ + reportOutputDrops(fields=>{assert.equal(fields.droppedLogsInterval,3);logger.info('process heartbeat',fields);}); + setImmediate(()=>reportOutputDrops(fields=>{assert.equal(fields.droppedLogsInterval,0);logger.info('process heartbeat',fields);})); + }); + `],{env:{...process.env,LOG_REDIS_MIRROR:'off'},encoding:'utf8',timeout:10000}); + expect(child.status,child.stderr).equal(0); + expect(child.stderr.trim().split('\n').filter(line=>line.startsWith('{')).map(JSON.parse).map(r=>r.droppedLogsInterval)).deep.equal([3,0]); + }); it('writes JSON without Redis and bounds a blocked stdout pipe', () => { const child = spawnSync(process.execPath, ['-r', 'ts-node/register/transpile-only', '-e', ` const { createLogger, getDroppedOutput } = require('./src/core/logger');