fix: deduplicate owner access alerts per recipient (#896)

* fix: prevent duplicate access alerts to the same recipient

* fix: address recipient alert recovery review findings

* fix: reconcile unsent alerts and wait for startup migration

* fix: preserve legacy recipient alerts through upgrades

* fix: reconcile alert results and legacy recovery races

* fix: fence dispatch and clean cancelled alert metadata

* fix: fence alert preparation and retain in-flight recovery

* fix: atomically commit alert dispatch and retry deferred recovery

* fix: keep recovery lookups from blocking upstream file reads
This commit is contained in:
Thomas Durieux authored and GitHub committed 2026-10-13 13:55:10 +01:00
1 parent eb9ac7e13d
commit 445a479339
20 files changed
+1724 -89

No files matched your search

+38
View File
@@ -1,6 +1,7 @@
#!/usr/bin/env bash
# Zero-downtime redeploy:
# build → blue/green streamer swap → recreate API.
# Upgrading legacy owner alerts pauses application containers before backfill.
# Skips everything if the rebuilt image hasn't changed (FORCE=1 to override).
set -euo pipefail
cd "$(dirname "$0")/.."
@@ -35,12 +36,49 @@ list_streamers() {
docker compose ps --status running --format '{{.Name}}' streamer 2>/dev/null
}
# Capture application writers, including stopped containers from a rollback.
# Legacy instances do not participate in recipient claims, so they must stop
# before a new instance marks the one-time claim migration complete.
application_ids="$(docker compose ps --all -q anonymous_github streamer)"
apps=()
if [[ -n $application_ids ]]; then mapfile -t apps <<<"$application_ids"; fi
legacy_alerts=0
for app in "${apps[@]}"; do
protocol="$(docker inspect -f '{{index .Config.Labels "science.4open.owner-alert-protocol"}}' "$app")"
[[ $protocol == recipient-v1 ]] || legacy_alerts=1
done
# 1. Build
log "Building image"
before="$(image_id "$IMAGE")"
docker compose build anonymous_github
after="$(image_id "$IMAGE")"
if [[ $legacy_alerts == 1 ]]; then
protocol="$(docker image inspect -f '{{index .Config.Labels "science.4open.owner-alert-protocol"}}' "$IMAGE")"
[[ $protocol == recipient-v1 ]] || { warn "replacement image lacks recipient alert migration support"; exit 1; }
TIMEOUT="${MIGRATION_HEALTH_TIMEOUT:-900}"
log "Pausing legacy application writers before recipient alert migration"
docker compose stop -t 30 anonymous_github streamer
for app in "${apps[@]}"; do
running="$(docker inspect -f '{{.State.Running}}' "$app")"
[[ $running == false ]] || { warn "legacy writer $app is still running; aborting migration"; exit 1; }
done
log "Preparing claim reconciliation, including claims written after rollback"
docker compose run --rm --no-deps --entrypoint node anonymous_github ./build/scripts/reconcile-legacy-owner-alerts.js
log "Starting ${REPLICAS} streamers after legacy writers stopped"
docker compose up -d --no-deps --force-recreate --scale "streamer=${REPLICAS}" streamer
mapfile -t news < <(list_streamers)
[[ ${#news[@]} == "$REPLICAS" ]] || { warn "streamer replica count does not match ${REPLICAS}"; exit 1; }
for app in "${news[@]}"; do wait_healthy "$app"; done
docker compose up -d --no-deps --force-recreate anonymous_github
api="$(docker compose ps -q anonymous_github)"
[[ -n $api ]] || { warn "API container missing after migration"; exit 1; }
wait_healthy "$api"
docker compose ps
exit 0
fi
# 2. Cold start
mapfile -t olds < <(list_streamers)
if [[ ${#olds[@]} -eq 0 ]]; then