fix: retain loss evidence and fail interrupted deployments

This commit is contained in:
Thomas Durieux committed 2026-11-15 13:52:05 +00:00
1 parent 0f9d8c657a
commit d99a9587f7
5 files changed
+48 -10

No files matched your search

+4 -2
View File
@@ -424,10 +424,12 @@ configure `ADMIN_LOG_GRAFANA_URL` to that investigation dashboard to use them.
Metrics rules link to the operations dashboard and include the exact PromQL query.
No notification receiver is added or enabled.
`ApplicationLogDrops` uses `droppedLogsInterval`, the losses since the last emitted
`ApplicationLogDrops` uses `droppedLogsInterval`, the losses since the last stream-acknowledged
process report, so it recovers after losses stop. `droppedLogs` remains cumulative
for historical diagnosis. If emitting the report itself drops a record, the
watermark stays unchanged and the next report includes those losses. A dead
watermark stays unchanged and the next report includes those losses. Concurrent
reports before acknowledgement can repeat a delta; it is an alert signal, not an
exact window loss total. A successful write does not prove collector ingestion. A dead
process or broken pair of stdout/stderr pipes still needs missing-heartbeat alerts.
Older applications without interval fields cannot drive the new interval rule;
roll out matching application and alert versions together. Collector/storage
+3
View File
@@ -53,6 +53,9 @@ deployment_finished() {
exit "$code"
}
trap deployment_finished EXIT
trap 'exit 129' HUP
trap 'exit 130' INT
trap 'exit 143' TERM
deployment_event start started 0
image_id() { docker image inspect --format '{{.Id}}' "$1" 2>/dev/null || true; }
+16 -6
View File
@@ -90,15 +90,18 @@ let droppedOutput = 0;
let outputBroken = false;
let errorOutputBroken = false;
// A closed log reader must not become an uncaught EPIPE on the serving process.
process.stdout.on("error", () => { outputBroken = true; droppedOutput++; });
process.stderr.on("error", () => { errorOutputBroken = true; droppedOutput++; });
process.stdout.on("error", () => { outputBroken = true; });
process.stderr.on("error", () => { errorOutputBroken = true; });
export function getDroppedOutput() { return droppedOutput; }
let reportedDrops = 0;
/** Advance the interval only when emitting the report did not itself drop a record. */
let dropReportWatermark: number | undefined;
/** Advance only after the output stream acknowledges the report's write. */
export function reportOutputDrops(report: (fields: { droppedLogs: number; droppedLogsInterval: number }) => void) {
const before = droppedOutput;
report({ droppedLogs: before, droppedLogsInterval: Math.max(0, before - reportedDrops) });
if (droppedOutput === before) reportedDrops = before;
const previous = dropReportWatermark;
dropReportWatermark = before;
try { report({ droppedLogs: before, droppedLogsInterval: Math.max(0, before - reportedDrops) }); }
finally { dropReportWatermark = previous; }
}
export function getInProcessDropped(): number {
return droppedInProcess;
@@ -282,7 +285,14 @@ function emit(level: Level, module: string, args: unknown[]) {
// building an unbounded userland queue. The next heartbeat reports losses.
const output = outputBroken ? process.stderr : process.stdout;
if (!(outputBroken && errorOutputBroken) && output.writableLength < 256 * 1024) {
try { output.write(JSON.stringify(record) + "\n"); }
const watermark = dropReportWatermark;
try { output.write(JSON.stringify(record) + "\n", error => {
if (error) {
if (output === process.stdout) outputBroken = true;
else errorOutputBroken = true;
droppedOutput++;
} else if (watermark !== undefined) reportedDrops = Math.max(reportedDrops, watermark);
}); }
catch {
if (output === process.stdout) outputBroken = true;
else errorOutputBroken = true;
+7 -1
View File
@@ -28,7 +28,7 @@ if (args[0] === "inspect") {
finish(state.unhealthy ? "unhealthy" : "healthy");
}
if (args[0] === "compose") {
if (args[1] === "build") { if (state.buildFails) finish("", 1); state.built = true; finish(); }
if (args[1] === "build") { if (state.signal) { process.kill(process.ppid, state.signal); finish(); } if (state.buildFails) finish("", 1); state.built = true; finish(); }
if (args[1] === "stop") { state.stopped = true; finish(); }
if (args[1] === "run" && args.at(-1).endsWith("check-github-app-key.js")) { finish("", args.includes(state.preflightFails) ? 1 : 0); }
if (args[1] === "run") { if (state.reconcileFails) finish("", 1); state.reconciled = true; finish(); }
@@ -60,6 +60,12 @@ finish("unexpected docker call", 2);
expect(new Set(events.map(row=>row.deploymentId)).size).equal(1);
expect(events.at(-1).restartMs).at.least(0);
});
for (const [signal, code] of [["SIGHUP",129],["SIGINT",130],["SIGTERM",143]]) it(`records ${signal} termination as failed`, () => {
const result=deploy({signal,operation:"rollback"});
expect(result.status,result.stderr).equal(code);
const events=result.stdout.split("\n").filter(line=>line.startsWith("{")).map(JSON.parse);
expect(events.at(-1)).include({deploymentPhase:"finish",deploymentOutcome:"failed",exitCode:code});
});
it("records failure when API or streamers fail health checks", () => {
const result=deploy({unhealthy:true});
expect(result.status).not.equal(0);
+18 -1
View File
@@ -121,12 +121,29 @@ describe('structured logging privacy and context', function () {
reportOutputDrops(fields=>logger.info('process heartbeat',fields));
blocked=false;
reportOutputDrops(fields=>{assert.equal(fields.droppedLogsInterval,3);logger.info('process heartbeat',fields);});
reportOutputDrops(fields=>{assert.equal(fields.droppedLogsInterval,0);assert.equal(fields.droppedLogs,3);logger.info('process heartbeat',fields);});
setImmediate(()=>reportOutputDrops(fields=>{assert.equal(fields.droppedLogsInterval,0);assert.equal(fields.droppedLogs,3);logger.info('process heartbeat',fields);}));
`],{env:{...process.env,LOG_REDIS_MIRROR:'off'},encoding:'utf8',timeout:10000});
expect(child.status,child.stderr).equal(0);
const rows=child.stdout.trim().split('\n').map(JSON.parse);
expect(rows.map(r=>r.droppedLogsInterval)).deep.equal([3,0]);
});
it('retains earlier losses when a report write fails asynchronously', () => {
const child=spawnSync(process.execPath,['-r','ts-node/register/transpile-only','-e',`
const assert=require('node:assert/strict');
const {createLogger,reportOutputDrops}=require('./src/core/logger');
const logger=createLogger('process');let blocked=true;
Object.defineProperty(process.stdout,'writableLength',{get:()=>blocked?300000:0});
logger.info('request');logger.info('request');blocked=false;
process.stdout.write=(line,callback)=>{setImmediate(()=>callback(new Error('EPIPE')));return true;};
reportOutputDrops(fields=>logger.info('process heartbeat',fields));
setImmediate(()=>{
reportOutputDrops(fields=>{assert.equal(fields.droppedLogsInterval,3);logger.info('process heartbeat',fields);});
setImmediate(()=>reportOutputDrops(fields=>{assert.equal(fields.droppedLogsInterval,0);logger.info('process heartbeat',fields);}));
});
`],{env:{...process.env,LOG_REDIS_MIRROR:'off'},encoding:'utf8',timeout:10000});
expect(child.status,child.stderr).equal(0);
expect(child.stderr.trim().split('\n').filter(line=>line.startsWith('{')).map(JSON.parse).map(r=>r.droppedLogsInterval)).deep.equal([3,0]);
});
it('writes JSON without Redis and bounds a blocked stdout pipe', () => {
const child = spawnSync(process.execPath, ['-r', 'ts-node/register/transpile-only', '-e', `
const { createLogger, getDroppedOutput } = require('./src/core/logger');