mirror of
https://github.com/tdurieux/anonymous_github.git
synced 2026-11-17 17:44:09 +01:00
fix: retain loss evidence and fail interrupted deployments
This commit is contained in:
1 parent
0f9d8c657a
commit
d99a9587f7
5 files changed
+48
-10
No files matched your search
@@ -424,10 +424,12 @@ configure `ADMIN_LOG_GRAFANA_URL` to that investigation dashboard to use them.
|
||||
Metrics rules link to the operations dashboard and include the exact PromQL query.
|
||||
No notification receiver is added or enabled.
|
||||
|
||||
`ApplicationLogDrops` uses `droppedLogsInterval`, the losses since the last emitted
|
||||
`ApplicationLogDrops` uses `droppedLogsInterval`, the losses since the last stream-acknowledged
|
||||
process report, so it recovers after losses stop. `droppedLogs` remains cumulative
|
||||
for historical diagnosis. If emitting the report itself drops a record, the
|
||||
watermark stays unchanged and the next report includes those losses. A dead
|
||||
watermark stays unchanged and the next report includes those losses. Concurrent
|
||||
reports before acknowledgement can repeat a delta; it is an alert signal, not an
|
||||
exact window loss total. A successful write does not prove collector ingestion. A dead
|
||||
process or broken pair of stdout/stderr pipes still needs missing-heartbeat alerts.
|
||||
Older applications without interval fields cannot drive the new interval rule;
|
||||
roll out matching application and alert versions together. Collector/storage
|
||||
|
||||
@@ -53,6 +53,9 @@ deployment_finished() {
|
||||
exit "$code"
|
||||
}
|
||||
trap deployment_finished EXIT
|
||||
trap 'exit 129' HUP
|
||||
trap 'exit 130' INT
|
||||
trap 'exit 143' TERM
|
||||
deployment_event start started 0
|
||||
|
||||
image_id() { docker image inspect --format '{{.Id}}' "$1" 2>/dev/null || true; }
|
||||
|
||||
+16
-6
@@ -90,15 +90,18 @@ let droppedOutput = 0;
|
||||
let outputBroken = false;
|
||||
let errorOutputBroken = false;
|
||||
// A closed log reader must not become an uncaught EPIPE on the serving process.
|
||||
process.stdout.on("error", () => { outputBroken = true; droppedOutput++; });
|
||||
process.stderr.on("error", () => { errorOutputBroken = true; droppedOutput++; });
|
||||
process.stdout.on("error", () => { outputBroken = true; });
|
||||
process.stderr.on("error", () => { errorOutputBroken = true; });
|
||||
export function getDroppedOutput() { return droppedOutput; }
|
||||
let reportedDrops = 0;
|
||||
/** Advance the interval only when emitting the report did not itself drop a record. */
|
||||
let dropReportWatermark: number | undefined;
|
||||
/** Advance only after the output stream acknowledges the report's write. */
|
||||
export function reportOutputDrops(report: (fields: { droppedLogs: number; droppedLogsInterval: number }) => void) {
|
||||
const before = droppedOutput;
|
||||
report({ droppedLogs: before, droppedLogsInterval: Math.max(0, before - reportedDrops) });
|
||||
if (droppedOutput === before) reportedDrops = before;
|
||||
const previous = dropReportWatermark;
|
||||
dropReportWatermark = before;
|
||||
try { report({ droppedLogs: before, droppedLogsInterval: Math.max(0, before - reportedDrops) }); }
|
||||
finally { dropReportWatermark = previous; }
|
||||
}
|
||||
export function getInProcessDropped(): number {
|
||||
return droppedInProcess;
|
||||
@@ -282,7 +285,14 @@ function emit(level: Level, module: string, args: unknown[]) {
|
||||
// building an unbounded userland queue. The next heartbeat reports losses.
|
||||
const output = outputBroken ? process.stderr : process.stdout;
|
||||
if (!(outputBroken && errorOutputBroken) && output.writableLength < 256 * 1024) {
|
||||
try { output.write(JSON.stringify(record) + "\n"); }
|
||||
const watermark = dropReportWatermark;
|
||||
try { output.write(JSON.stringify(record) + "\n", error => {
|
||||
if (error) {
|
||||
if (output === process.stdout) outputBroken = true;
|
||||
else errorOutputBroken = true;
|
||||
droppedOutput++;
|
||||
} else if (watermark !== undefined) reportedDrops = Math.max(reportedDrops, watermark);
|
||||
}); }
|
||||
catch {
|
||||
if (output === process.stdout) outputBroken = true;
|
||||
else errorOutputBroken = true;
|
||||
|
||||
@@ -28,7 +28,7 @@ if (args[0] === "inspect") {
|
||||
finish(state.unhealthy ? "unhealthy" : "healthy");
|
||||
}
|
||||
if (args[0] === "compose") {
|
||||
if (args[1] === "build") { if (state.buildFails) finish("", 1); state.built = true; finish(); }
|
||||
if (args[1] === "build") { if (state.signal) { process.kill(process.ppid, state.signal); finish(); } if (state.buildFails) finish("", 1); state.built = true; finish(); }
|
||||
if (args[1] === "stop") { state.stopped = true; finish(); }
|
||||
if (args[1] === "run" && args.at(-1).endsWith("check-github-app-key.js")) { finish("", args.includes(state.preflightFails) ? 1 : 0); }
|
||||
if (args[1] === "run") { if (state.reconcileFails) finish("", 1); state.reconciled = true; finish(); }
|
||||
@@ -60,6 +60,12 @@ finish("unexpected docker call", 2);
|
||||
expect(new Set(events.map(row=>row.deploymentId)).size).equal(1);
|
||||
expect(events.at(-1).restartMs).at.least(0);
|
||||
});
|
||||
for (const [signal, code] of [["SIGHUP",129],["SIGINT",130],["SIGTERM",143]]) it(`records ${signal} termination as failed`, () => {
|
||||
const result=deploy({signal,operation:"rollback"});
|
||||
expect(result.status,result.stderr).equal(code);
|
||||
const events=result.stdout.split("\n").filter(line=>line.startsWith("{")).map(JSON.parse);
|
||||
expect(events.at(-1)).include({deploymentPhase:"finish",deploymentOutcome:"failed",exitCode:code});
|
||||
});
|
||||
it("records failure when API or streamers fail health checks", () => {
|
||||
const result=deploy({unhealthy:true});
|
||||
expect(result.status).not.equal(0);
|
||||
|
||||
@@ -121,12 +121,29 @@ describe('structured logging privacy and context', function () {
|
||||
reportOutputDrops(fields=>logger.info('process heartbeat',fields));
|
||||
blocked=false;
|
||||
reportOutputDrops(fields=>{assert.equal(fields.droppedLogsInterval,3);logger.info('process heartbeat',fields);});
|
||||
reportOutputDrops(fields=>{assert.equal(fields.droppedLogsInterval,0);assert.equal(fields.droppedLogs,3);logger.info('process heartbeat',fields);});
|
||||
setImmediate(()=>reportOutputDrops(fields=>{assert.equal(fields.droppedLogsInterval,0);assert.equal(fields.droppedLogs,3);logger.info('process heartbeat',fields);}));
|
||||
`],{env:{...process.env,LOG_REDIS_MIRROR:'off'},encoding:'utf8',timeout:10000});
|
||||
expect(child.status,child.stderr).equal(0);
|
||||
const rows=child.stdout.trim().split('\n').map(JSON.parse);
|
||||
expect(rows.map(r=>r.droppedLogsInterval)).deep.equal([3,0]);
|
||||
});
|
||||
it('retains earlier losses when a report write fails asynchronously', () => {
|
||||
const child=spawnSync(process.execPath,['-r','ts-node/register/transpile-only','-e',`
|
||||
const assert=require('node:assert/strict');
|
||||
const {createLogger,reportOutputDrops}=require('./src/core/logger');
|
||||
const logger=createLogger('process');let blocked=true;
|
||||
Object.defineProperty(process.stdout,'writableLength',{get:()=>blocked?300000:0});
|
||||
logger.info('request');logger.info('request');blocked=false;
|
||||
process.stdout.write=(line,callback)=>{setImmediate(()=>callback(new Error('EPIPE')));return true;};
|
||||
reportOutputDrops(fields=>logger.info('process heartbeat',fields));
|
||||
setImmediate(()=>{
|
||||
reportOutputDrops(fields=>{assert.equal(fields.droppedLogsInterval,3);logger.info('process heartbeat',fields);});
|
||||
setImmediate(()=>reportOutputDrops(fields=>{assert.equal(fields.droppedLogsInterval,0);logger.info('process heartbeat',fields);}));
|
||||
});
|
||||
`],{env:{...process.env,LOG_REDIS_MIRROR:'off'},encoding:'utf8',timeout:10000});
|
||||
expect(child.status,child.stderr).equal(0);
|
||||
expect(child.stderr.trim().split('\n').filter(line=>line.startsWith('{')).map(JSON.parse).map(r=>r.droppedLogsInterval)).deep.equal([3,0]);
|
||||
});
|
||||
it('writes JSON without Redis and bounds a blocked stdout pipe', () => {
|
||||
const child = spawnSync(process.execPath, ['-r', 'ts-node/register/transpile-only', '-e', `
|
||||
const { createLogger, getDroppedOutput } = require('./src/core/logger');
|
||||
|
||||
Reference in new issue
Block a user