feat: add operational incident evidence and recovering alerts

This commit is contained in:
Thomas Durieux committed 2026-11-15 13:45:38 +00:00
1 parent a4f441f097
commit 0f9d8c657a
18 files changed
+558 -123

No files matched your search

+47 -1
View File
@@ -18,6 +18,43 @@ COORDINATED_RESTART="${COORDINATED_RESTART:-0}"
log() { printf '==> %s\n' "$*"; }
warn() { printf '!! %s\n' "$*" >&2; }
# Local, finite deployment evidence. Collection is opt-in via a pre-created file.
# Logging failure must never change deployment behavior or expose shell commands.
deployment_started=$(date +%s)
restart_started=0
deployment_id=$(cat /proc/sys/kernel/random/uuid 2>/dev/null || true)
[[ $deployment_id =~ ^[a-f0-9-]{36}$ ]] || deployment_id=""
deployment_release=${APP_RELEASE:-unknown}
[[ $deployment_release =~ ^[a-f0-9]{7,40}$ ]] || deployment_release=unknown
deployment_operation=${DEPLOY_OPERATION:-deploy}
[[ $deployment_operation == rollback ]] || deployment_operation=deploy
deployment_outcome=completed
deployment_event() {
local phase="$1" outcome="$2" code="$3" now elapsed restart_ms record
now=$(date +%s); elapsed=$(( (now - deployment_started) * 1000 )); restart_ms=0
if (( restart_started > 0 )); then restart_ms=$(( (now - restart_started) * 1000 )); fi
printf -v record '{"timestamp":"%s","service":"api","severity":"%s","event":"deployment","deploymentId":"%s","release":"%s","deploymentOperation":"%s","deploymentPhase":"%s","deploymentOutcome":"%s","ms":%d,"restartMs":%d,"exitCode":%d}' \
"$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$([[ $code == 0 ]] && echo info || echo error)" "$deployment_id" "$deployment_release" "$deployment_operation" "$phase" "$outcome" "$elapsed" "$restart_ms" "$code"
printf '%s\n' "$record" || true
if [[ -n ${DEPLOY_EVENT_FILE:-} ]]; then
if [[ -f $DEPLOY_EVENT_FILE && ! -L $DEPLOY_EVENT_FILE ]]; then
(printf '%s\n' "$record" >> "$DEPLOY_EVENT_FILE") 2>/dev/null || warn "deployment event file unavailable"
else warn "deployment event file must already exist and must not be a symlink"; fi
fi
return 0
}
restart_event() {
if (( restart_started == 0 )); then restart_started=$(date +%s); deployment_event restart started 0; fi
}
deployment_finished() {
local code=$?
if (( code != 0 )); then deployment_outcome=failed; fi
deployment_event finish "$deployment_outcome" "$code"
exit "$code"
}
trap deployment_finished EXIT
deployment_event start started 0
image_id() { docker image inspect --format '{{.Id}}' "$1" 2>/dev/null || true; }
wait_healthy() {
@@ -67,6 +104,7 @@ if [[ $legacy_alerts == 1 ]]; then
[[ $protocol == recipient-v2 ]] || { warn "replacement image lacks recipient alert migration support"; exit 1; }
TIMEOUT="${MIGRATION_HEALTH_TIMEOUT:-900}"
log "Pausing legacy application writers before recipient alert migration"
restart_event
docker compose stop -t 30 anonymous_github streamer
for app in "${apps[@]}"; do
running="$(docker inspect -f '{{.State.Running}}' "$app")"
@@ -96,6 +134,7 @@ for app in "${apps[@]}"; do
done
if [[ $COORDINATED_RESTART == 1 ]]; then
log "Coordinated application restart for shared cache compatibility"
restart_event
docker compose stop -t 30 anonymous_github streamer
for app in "${apps[@]}"; do
[[ $(docker inspect -f '{{.State.Running}}' "$app") == false ]] || { warn "cache writer $app is still running"; exit 1; }
@@ -116,13 +155,18 @@ fi
mapfile -t olds < <(list_streamers)
if [[ ${#olds[@]} -eq 0 ]]; then
log "Cold start — bringing the whole stack up"
restart_event
docker compose up -d --remove-orphans
api="$(docker compose ps -q anonymous_github)"
[[ -n $api ]] || { warn "API container missing after cold start"; exit 1; }
wait_healthy "$api"
docker compose ps
exit 0
fi
# 3. No-op if image unchanged
if [[ -n $before && $before == "$after" && $FORCE != 1 ]]; then
deployment_outcome=unchanged
log "Image unchanged — nothing to deploy (FORCE=1 to override)"
docker compose ps
exit 0
@@ -152,6 +196,7 @@ if [[ $fail == 1 ]]; then
fi
# Drop olds and reconcile scale.
restart_event
docker rm -f "${olds[@]}" >/dev/null
docker compose up -d --no-deps --no-recreate --scale "streamer=${REPLICAS}" streamer >/dev/null
@@ -160,6 +205,7 @@ docker compose up -d --no-deps --no-recreate --scale "streamer=${REPLICAS}" stre
log "Recreating API"
docker compose up -d --no-deps --remove-orphans anonymous_github
api="$(docker compose ps -q anonymous_github | head -1)"
[[ -n $api ]] && wait_healthy "$api" || true
[[ -n $api ]] || { warn "API container missing after restart"; exit 1; }
wait_healthy "$api"
docker compose ps
+17 -2
View File
@@ -23,10 +23,11 @@ const sleep = ms => new Promise(resolve=>setTimeout(resolve,ms));
const requestId=randomUUID(),jobId=randomUUID();
const records=['api','streamer','worker'].map((service,i)=>({timestamp:new Date(Date.now()-10000+i).toISOString(),event:i===2?'job_failed':'request_completed',severity:'error',service,module:'queue',status:502,errorCode:'upstream_error',release:'abcdef1',requestId,...(i===2?{jobId}:{}),url:'private-owner/private-repository?token=secret',body:'<script>secret</script>'}));
records.push({...records[0],requestId:randomUUID(),timestamp:new Date(Date.now()-5000).toISOString()});
records.push({...records[0], release:'abcdef0', timestamp:new Date(Date.now()-2*60*60*1000).toISOString()});
const ingest = await fetch(process.env.ADMIN_LOG_URL+'/insert/jsonline?_time_field=timestamp', {method:'POST',body:records.map(JSON.stringify).join('\n'),headers:{'Content-Type':'application/stream+json'}});
assert(ingest.ok); await sleep(2000);
const errors=await investigateLogs(logOptions({service:'api',release:'abcdef1'}));
assert.equal(errors.state,'stale'); assert.equal(errors.groups.length,1); assert.equal(errors.groups[0].count,2);
assert.equal(errors.state,'stale'); assert.equal(errors.groups.length,1); assert.equal(errors.groups[0].count,2); assert.equal(errors.groups[0].affectedRequests,2); assert.equal(errors.groups[0].regression,'known'); assert.equal(errors.baseline.partial,false);
assert(!JSON.stringify(errors).includes('private-owner')); assert(!JSON.stringify(errors).includes('secret'));
const timeline=await investigateLogs(logOptions({kind:'jobId',id:jobId},true),true);
assert.equal(timeline.state,'ready'); assert.deepEqual([...new Set(timeline.events.map(row=>row.service))].sort(),['api','streamer','worker']);
@@ -91,7 +92,7 @@ const sleep = ms => new Promise(resolve=>setTimeout(resolve,ms));
const redis=safeRedisLog({ts:diagnostic.timestamp,level:'error',module:'route',raw:[diagnostic]});
for(const row of [traced.events[0],redis.raw[0]]) {assert.equal(row.requestPath,diagnostic.requestPath);assert.equal(row.repoId,'demo-123');assert.equal(row.route,diagnostic.route);assert.equal(row.method,'GET');assert.equal(row.errorStack,projected.errorStack);assert.equal(row.causeType,'RangeError');assert(!JSON.stringify(row).includes(secret));}
const paths=['/api/repo/demo-123/refresh','/api/repo/demo-123/extend','/api/repo/demo-123/coauthors','/api/repo/demo-123','/api/repo/demo-123/zip','/api/repo/demo-123/files','/api/repo/demo-123/files/counts','/api/repo/demo-123/options','/api/repo/demo-123/file/README.md'];
const pathRecords=paths.map((requestPath,i)=>logContext.run({requestId:randomUUID(),requestPath,method:/\/(refresh|extend|coauthors)$/.test(requestPath)?'POST':'GET'},()=>logRecord(i%2?'warn':'info','requests',['request',{status:200,outcome:i%2?'interrupted':'completed'}])));
const pathRecords=paths.map((requestPath,i)=>logContext.run({requestId:randomUUID(),requestPath,method:/\/(refresh|extend|coauthors)$/.test(requestPath)?'POST':'GET'},()=>logRecord(i%2?'warn':'info','requests',['request',{status:200,outcome:i%2?'interrupted':'completed',bytes:7,expectedBytes:100,cancellationOrigin:'unknown',cancellationEvidence:'response_close',hopId:randomUUID(),endpointKind:'public',measuredMs:50,unattributedMs:20,overlapMs:10,stages:{database:10,transfer:40},stageCounts:{database:2},stageOpen:{database:0}}])));
const pathSources=pathRecords.map(record=>({...rawSource,message:JSON.stringify({attrs:{tag:'api'},log:JSON.stringify(record)})}));
const pathOutput=execFileSync('docker',['run','--rm','-i','--network','none','--memory','256m',vectorImage,'vrl','-i','/dev/stdin',fs.readFileSync('deploy/logging/sanitize.vrl','utf8')+'\nencode_json(.)'],{encoding:'utf8',input:pathSources.map(JSON.stringify).join('\n')+'\n'}).trim().split('\n').map(line=>JSON.parse(JSON.parse(line)));
assert.deepEqual(pathOutput.map(x=>x.requestPath),paths);
@@ -101,9 +102,23 @@ const sleep = ms => new Promise(resolve=>setTimeout(resolve,ms));
const result=await investigateLogs(logOptions({kind:'requestId',id:pathRecords[i].requestId},true),true);
assert.equal(result.events[0].requestPath,paths[i]);
assert.equal(result.events[0].repoId,'demo-123');
assert.equal(result.events[0].bytes,7);assert.equal(result.events[0].expectedBytes,100);
assert.equal(result.events[0].cancellationOrigin,'unknown');assert.equal(result.events[0].cancellationEvidence,'response_close');
assert.equal(result.events[0].hopId,pathRecords[i].hopId);assert.equal(result.events[0].endpointKind,'public');
assert.equal(result.events[0].stage_database_ms,10);assert.equal(result.events[0].stage_database_count,2);
assert.equal(result.events[0].measuredMs,50);assert.equal(result.events[0].unattributedMs,20);assert.equal(result.events[0].overlapMs,10);
}
console.log('PASS successful and interrupted anonymous endpoint paths through collector and central API');
console.log('PASS synthetic exception logger → pinned collector → VictoriaLogs/Redis safe API with normalized route and bounded frames');
const deploymentId=randomUUID();
const deployment={timestamp:new Date().toISOString(),event:'deployment',severity:'error',release:'abcdef3',deploymentId,
deploymentPhase:'finish',deploymentOperation:'rollback',deploymentOutcome:'failed',restartMs:3000,ms:5000,exitCode:1,message:secret,url:secret};
const deployOutput=execFileSync('docker',['run','--rm','-i','--network','none','--memory','256m',vectorImage,'vrl','-i','/dev/stdin',fs.readFileSync('deploy/logging/sanitize.vrl','utf8')+'\nencode_json(.)'],{encoding:'utf8',input:JSON.stringify({file:'/host/nginx/anonymous.deploy.json',message:JSON.stringify(deployment)})+'\n'}).trim().split('\n').map(line=>JSON.parse(JSON.parse(line)))[0];
assert.equal(deployOutput.event,'deployment');assert.equal(deployOutput.deploymentId,deploymentId);assert(!JSON.stringify(deployOutput).includes(secret));
assert((await fetch(process.env.ADMIN_LOG_URL+'/insert/jsonline?_time_field=timestamp',{method:'POST',body:JSON.stringify(deployOutput),headers:{'Content-Type':'application/stream+json'}})).ok);await sleep(2000);
const deployView=await investigateLogs(logOptions({release:'abcdef3'}));
assert.equal(deployView.groups[0].sample.deploymentId,deploymentId);assert.equal(deployView.groups[0].sample.deploymentOutcome,'failed');assert.equal(deployView.groups[0].sample.restartMs,3000);
console.log('PASS deployment evidence, request timings/bytes and release baseline through pinned collector and central API');
docker('stop',name);
const unavailable=await investigateLogs(logOptions({})); assert.equal(unavailable.state,'unreachable');
delete process.env.ADMIN_LOG_URL;
+102
View File
@@ -0,0 +1,102 @@
#!/usr/bin/env python3
"""Pinned, isolated alert firing/recovery. No production access or notifications.
Log tests accelerate windows/for/interval only; expressions and numeric thresholds
are otherwise shipped rules. PromQL tests use promtool's simulated clock.
"""
import json, pathlib, re, subprocess, tempfile, time, urllib.request, urllib.parse, uuid
import yaml
ROOT = pathlib.Path(__file__).resolve().parents[1]
CONFIG = ROOT / 'deploy/logging'
images = yaml.safe_load((CONFIG/'compose.yml').read_text())['services']
def run(*args):
result=subprocess.run(args, capture_output=True, text=True)
if result.returncode:raise RuntimeError(result.stdout+'\n'+result.stderr)
return result.stdout.strip()
def request(url, data=None):
return json.loads(urllib.request.urlopen(urllib.request.Request(url, data=data,
headers={'Content-Type':'application/stream+json'} if data else {}), timeout=5).read())
def wait(fn, seconds=30):
end=time.monotonic()+seconds; last=None
while time.monotonic()<end:
try:
result=fn()
if result:return result
except Exception as error:last=error
time.sleep(.25)
raise AssertionError('condition did not become true: '+str(last))
with tempfile.TemporaryDirectory(prefix='operational-alerts-') as tmp:
tmp=pathlib.Path(tmp); tmp.chmod(0o755); project='operational-alerts-'+uuid.uuid4().hex[:10]
rules=yaml.safe_load((CONFIG/'log-alerts.yml').read_text())
for group in rules['groups']:
group['interval']='1s'
for rule in group['rules']:
rule['expr']=re.sub(r'_time:\d+m', '_time:5s', rule['expr']);rule['for']='2s'
(tmp/'rules.yml').write_text(yaml.safe_dump(rules))
compose={'services':{
'logs':{'image':images['victorialogs']['image'],'mem_limit':'512m','command':['-storageDataPath=/storage','-storage.minFreeDiskSpaceBytes=1MB','-memory.allowedBytes=256MiB'], 'ports':['127.0.0.1::9428']},
'alerts':{'image':images['vmalert']['image'],'command':['-rule=/rules.yml','-datasource.url=http://logs:9428','-notifier.blackhole','-rule.evalDelay=0s'], 'volumes':[str(tmp/'rules.yml')+':/rules.yml:ro'],'ports':['127.0.0.1::8880']},
},'networks':{'default':{'internal':False}}}
(tmp/'compose.yml').write_text(yaml.safe_dump(compose))
cmd=['docker','compose','-p',project,'-f',str(tmp/'compose.yml')]
try:
run(*cmd,'up','-d')
logs='http://'+run(*cmd,'port','logs','9428');alerts='http://'+run(*cmd,'port','alerts','8880')
def states():
groups=request(alerts+'/api/v1/rules')['data']['groups']
return {r['name']:r['state'] for g in groups for r in g['rules']}
wait(states)
def emit(rows):
data=('\n'.join(json.dumps({**r,"message":r.get("event", "fixture")}) for r in rows)+'\n').encode()
urllib.request.urlopen(urllib.request.Request(logs+'/insert/jsonline?_msg_field=message',data=data,headers={'Content-Type':'application/stream+json'}),timeout=5).read()
def healthy():
rows=[{'service':'api','instance':'api','event':'process_heartbeat','memoryRatio':.2,'droppedLogsInterval':0}]
rows += [{'service':'streamer','instance':str(i),'event':'process_heartbeat','memoryRatio':.2,'droppedLogsInterval':0} for i in range(4)]
rows += [{'service':'api','queue':q,'event':'queue_heartbeat','oldestWaitingMs':0} for q in ['download','remove','cache']]
return rows
# Threshold violations, then missing telemetry, then healthy recovery.
target={'SustainedServerErrors','ApplicationRestartBurst','ApplicationMemoryPressure','QueueStalled','QueueWaitingTooLong','WebhookFailures','ApplicationLogDrops'}
def failures():
rows=healthy()
rows += [{'service':'api','event':'request_completed','outcome':'completed','status':503,'route':'/github/app/webhook'} for _ in range(12)]
rows += [{'service':'api','event':'process_started'} for _ in range(5)]
rows += [{'service':'api','event':'process_heartbeat','instance':'api','memoryRatio':.95,'droppedLogsInterval':2}]
rows += [{'service':'api','event':'job_stalled','queue':'download'},{'service':'api','event':'queue_heartbeat','queue':'download','oldestWaitingMs':700000}]
return rows
seen=set()
for _ in range(16):
emit(failures());time.sleep(.5);seen.update(n for n,s in states().items() if s=='firing')
assert target <= seen,(target-seen,states())
print('PASS real vmalert firing for 5xx, restarts, memory, stalls, waiting age, webhook and interval drops',flush=True)
missing={'ApiTelemetryMissing','StreamerTelemetryMissing','QueueTelemetryMissing'}
wait(lambda: missing <= {n for n,s in states().items() if s=='firing'},20)
print('PASS real missing-heartbeat alerts fire',flush=True)
for _ in range(16):emit(healthy());time.sleep(.5)
assert all(s=='inactive' for s in states().values()),states()
print('PASS all log alerts recover; no external notifier configured',flush=True)
except Exception:
print(run(*cmd,'logs','--tail','25'),flush=True);raise
finally:run(*cmd,'down','-v','--remove-orphans')
metric_rules=yaml.safe_load((CONFIG/'metrics-alerts.yml').read_text())
(tmp/'metrics.yml').write_text(yaml.safe_dump(metric_rules))
series=[
{'series':'up{job="collector",instance="vector:9598"}','values':'0+0x6 1+0x14'},
{'series':'vector_buffer_discarded_events_total','values':'0+1x6 6+0x14'},
{'series':'vector_component_discarded_events_total{intentional="false"}','values':'0+1x6 6+0x14'},
{'series':'vector_component_errors_total','values':'0+1x6 6+0x14'},
{'series':'vl_data_size_bytes','values':str(9*1024**3)+'+0x6 1000+0x14'},
{'series':'vl_free_disk_space_bytes','values':str(10*1024**3)+'+0x21'},
{'series':'vl_rows_dropped_total','values':'0+1x6 6+0x14'},
{'series':'vmalert_execution_errors_total','values':'0+1x6 6+0x14'},
]
checks=[]
for group in metric_rules['groups']:
for rule in group['rules']:
labels=dict(rule['labels'])
if rule['alert']=='LoggingTelemetryMissing':labels.update(job='collector',instance='vector:9598')
checks += [{'eval_time':'6m','alertname':rule['alert'],'exp_alerts':[{'exp_labels':labels,'exp_annotations':rule['annotations']}]},
{'eval_time':'15m','alertname':rule['alert'],'exp_alerts':[]},
{'eval_time':'1m','alertname':rule['alert'],'exp_alerts':[]}]
tests={'rule_files':['/fixtures/metrics.yml'],'evaluation_interval':'1m','tests':[{'interval':'1m','input_series':series,'alert_rule_test':checks}]}
(tmp/'tests.yml').write_text(yaml.safe_dump(tests))
print(run('docker','run','--rm','--network','none','--entrypoint','/bin/promtool','-v',str(tmp)+':/fixtures:ro',images['prometheus']['image'],'test','rules','/fixtures/tests.yml'),flush=True)
print('PASS pinned Prometheus firing, sustained thresholds and recovery for every infrastructure alert',flush=True)