mirror of
https://github.com/tdurieux/anonymous_github.git
synced 2026-10-19 22:27:20 +02:00
* fix: retain recipient suppression after access recovery * fix: migrate cold starts and fence resource deactivation * docs: stop every writer before alert reconciliation
133 lines
5.0 KiB
Bash
Executable File
133 lines
5.0 KiB
Bash
Executable File
#!/usr/bin/env bash
|
||
# Zero-downtime redeploy:
|
||
# build → blue/green streamer swap → recreate API.
|
||
# Upgrading legacy owner alerts pauses application containers before backfill.
|
||
# Skips everything if the rebuilt image hasn't changed (FORCE=1 to override).
|
||
set -euo pipefail
|
||
cd "$(dirname "$0")/.."
|
||
|
||
export DOCKER_BUILDKIT=1
|
||
export COMPOSE_DOCKER_CLI_BUILD=1
|
||
|
||
IMAGE="${IMAGE:-tdurieux/anonymous_github:v2}"
|
||
REPLICAS="${STREAMER_REPLICAS:-4}"
|
||
TIMEOUT="${HEALTH_TIMEOUT:-90}"
|
||
FORCE="${FORCE:-0}"
|
||
|
||
log() { printf '==> %s\n' "$*"; }
|
||
warn() { printf '!! %s\n' "$*" >&2; }
|
||
|
||
image_id() { docker image inspect --format '{{.Id}}' "$1" 2>/dev/null || true; }
|
||
|
||
wait_healthy() {
|
||
local name="$1" waited=0 status
|
||
while (( waited < TIMEOUT )); do
|
||
status="$(docker inspect -f \
|
||
'{{if .State.Health}}{{.State.Health.Status}}{{else if .State.Running}}running{{else}}stopped{{end}}' \
|
||
"$name" 2>/dev/null || echo missing)"
|
||
[[ $status == healthy || $status == running ]] && return 0
|
||
sleep 2; waited=$((waited + 2))
|
||
done
|
||
warn "$name not healthy after ${TIMEOUT}s (status=$status)"
|
||
return 1
|
||
}
|
||
|
||
list_streamers() {
|
||
docker compose ps --status running --format '{{.Name}}' streamer 2>/dev/null
|
||
}
|
||
|
||
# Capture application writers, including stopped containers from a rollback.
|
||
# Older instances can release recipient guards, so they must stop before
|
||
# a new instance marks permanent-suppression migration complete.
|
||
application_ids="$(docker compose ps --all -q anonymous_github streamer)"
|
||
apps=()
|
||
if [[ -n $application_ids ]]; then mapfile -t apps <<<"$application_ids"; fi
|
||
legacy_alerts=0
|
||
for app in "${apps[@]}"; do
|
||
protocol="$(docker inspect -f '{{index .Config.Labels "science.4open.owner-alert-protocol"}}' "$app")"
|
||
[[ $protocol == recipient-v2 ]] || legacy_alerts=1
|
||
done
|
||
|
||
# 1. Build
|
||
log "Building image"
|
||
before="$(image_id "$IMAGE")"
|
||
docker compose build anonymous_github
|
||
after="$(image_id "$IMAGE")"
|
||
|
||
if [[ $legacy_alerts == 1 ]]; then
|
||
protocol="$(docker image inspect -f '{{index .Config.Labels "science.4open.owner-alert-protocol"}}' "$IMAGE")"
|
||
[[ $protocol == recipient-v2 ]] || { warn "replacement image lacks recipient alert migration support"; exit 1; }
|
||
TIMEOUT="${MIGRATION_HEALTH_TIMEOUT:-900}"
|
||
log "Pausing legacy application writers before recipient alert migration"
|
||
docker compose stop -t 30 anonymous_github streamer
|
||
for app in "${apps[@]}"; do
|
||
running="$(docker inspect -f '{{.State.Running}}' "$app")"
|
||
[[ $running == false ]] || { warn "legacy writer $app is still running; aborting migration"; exit 1; }
|
||
done
|
||
log "Preparing claim reconciliation, including claims written after rollback"
|
||
docker compose run --rm --no-deps --entrypoint node anonymous_github ./build/scripts/reconcile-legacy-owner-alerts.js
|
||
log "Starting ${REPLICAS} streamers after legacy writers stopped"
|
||
docker compose up -d --no-deps --force-recreate --scale "streamer=${REPLICAS}" streamer
|
||
mapfile -t news < <(list_streamers)
|
||
[[ ${#news[@]} == "$REPLICAS" ]] || { warn "streamer replica count does not match ${REPLICAS}"; exit 1; }
|
||
for app in "${news[@]}"; do wait_healthy "$app"; done
|
||
docker compose up -d --no-deps --force-recreate anonymous_github
|
||
api="$(docker compose ps -q anonymous_github)"
|
||
[[ -n $api ]] || { warn "API container missing after migration"; exit 1; }
|
||
wait_healthy "$api"
|
||
docker compose ps
|
||
exit 0
|
||
fi
|
||
|
||
# 2. Cold start
|
||
mapfile -t olds < <(list_streamers)
|
||
if [[ ${#olds[@]} -eq 0 ]]; then
|
||
log "Cold start — bringing the whole stack up"
|
||
docker compose up -d --remove-orphans
|
||
docker compose ps
|
||
exit 0
|
||
fi
|
||
|
||
# 3. No-op if image unchanged
|
||
if [[ -n $before && $before == "$after" && $FORCE != 1 ]]; then
|
||
log "Image unchanged — nothing to deploy (FORCE=1 to override)"
|
||
docker compose ps
|
||
exit 0
|
||
fi
|
||
|
||
# 4. Blue/green streamer swap
|
||
log "Swapping streamer (${#olds[@]} old → ${REPLICAS} new)"
|
||
docker compose up -d --no-deps --no-recreate \
|
||
--scale "streamer=$((${#olds[@]} + REPLICAS))" streamer >/dev/null
|
||
|
||
# Anything not in $olds is new.
|
||
declare -A was_old=()
|
||
for o in "${olds[@]}"; do was_old[$o]=1; done
|
||
news=()
|
||
while IFS= read -r n; do
|
||
[[ -n $n && -z ${was_old[$n]:-} ]] && news+=("$n")
|
||
done < <(list_streamers)
|
||
|
||
# Wait for all new replicas to be healthy in parallel.
|
||
pids=()
|
||
for n in "${news[@]}"; do wait_healthy "$n" & pids+=($!); done
|
||
fail=0
|
||
for p in "${pids[@]}"; do wait "$p" || fail=1; done
|
||
if [[ $fail == 1 ]]; then
|
||
warn "new replicas unhealthy — keeping olds in place"
|
||
exit 1
|
||
fi
|
||
|
||
# Drop olds and reconcile scale.
|
||
docker rm -f "${olds[@]}" >/dev/null
|
||
docker compose up -d --no-deps --no-recreate --scale "streamer=${REPLICAS}" streamer >/dev/null
|
||
|
||
# 5. Recreate API. --no-deps: we already manage every dependency above,
|
||
# skipping compose's own depends_on health re-poll (~5–10s).
|
||
log "Recreating API"
|
||
docker compose up -d --no-deps --remove-orphans anonymous_github
|
||
api="$(docker compose ps -q anonymous_github | head -1)"
|
||
[[ -n $api ]] && wait_healthy "$api" || true
|
||
|
||
docker compose ps
|