#!/usr/bin/env bash # Zero-downtime redeploy: # build → blue/green streamer swap → recreate API. # Upgrading legacy owner alerts pauses application containers before backfill. # Skips everything if the rebuilt image hasn't changed (FORCE=1 to override). set -euo pipefail cd "$(dirname "$0")/.." export DOCKER_BUILDKIT=1 export COMPOSE_DOCKER_CLI_BUILD=1 IMAGE="${IMAGE:-tdurieux/anonymous_github:v2}" REPLICAS="${STREAMER_REPLICAS:-4}" TIMEOUT="${HEALTH_TIMEOUT:-90}" FORCE="${FORCE:-0}" COORDINATED_RESTART="${COORDINATED_RESTART:-0}" log() { printf '==> %s\n' "$*"; } warn() { printf '!! %s\n' "$*" >&2; } # Local, finite deployment evidence. Collection is opt-in via a pre-created file. # Logging failure must never change deployment behavior or expose shell commands. deployment_started=$(date +%s) restart_started=0 deployment_id=$(cat /proc/sys/kernel/random/uuid 2>/dev/null || true) [[ $deployment_id =~ ^[a-f0-9-]{36}$ ]] || deployment_id="" deployment_release=${APP_RELEASE:-unknown} [[ $deployment_release =~ ^[a-f0-9]{7,40}$ ]] || deployment_release=unknown deployment_operation=${DEPLOY_OPERATION:-deploy} [[ $deployment_operation == rollback ]] || deployment_operation=deploy deployment_outcome=completed deployment_event() { local phase="$1" outcome="$2" code="$3" now elapsed restart_ms record now=$(date +%s); elapsed=$(( (now - deployment_started) * 1000 )); restart_ms=0 if (( restart_started > 0 )); then restart_ms=$(( (now - restart_started) * 1000 )); fi printf -v record '{"timestamp":"%s","service":"api","severity":"%s","event":"deployment","deploymentId":"%s","release":"%s","deploymentOperation":"%s","deploymentPhase":"%s","deploymentOutcome":"%s","ms":%d,"restartMs":%d,"exitCode":%d}' \ "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$([[ $code == 0 ]] && echo info || echo error)" "$deployment_id" "$deployment_release" "$deployment_operation" "$phase" "$outcome" "$elapsed" "$restart_ms" "$code" printf '%s\n' "$record" || true if [[ -n ${DEPLOY_EVENT_FILE:-} ]]; then if [[ -f $DEPLOY_EVENT_FILE && ! -L $DEPLOY_EVENT_FILE ]]; then (printf '%s\n' "$record" >> "$DEPLOY_EVENT_FILE") 2>/dev/null || warn "deployment event file unavailable" else warn "deployment event file must already exist and must not be a symlink"; fi fi return 0 } restart_event() { if (( restart_started == 0 )); then restart_started=$(date +%s); deployment_event restart started 0; fi } deployment_finished() { local code=$? if (( code != 0 )); then deployment_outcome=failed; fi deployment_event finish "$deployment_outcome" "$code" exit "$code" } trap deployment_finished EXIT deployment_event start started 0 image_id() { docker image inspect --format '{{.Id}}' "$1" 2>/dev/null || true; } wait_healthy() { local name="$1" waited=0 status while (( waited < TIMEOUT )); do status="$(docker inspect -f \ '{{if .State.Health}}{{.State.Health.Status}}{{else if .State.Running}}running{{else}}stopped{{end}}' \ "$name" 2>/dev/null || echo missing)" [[ $status == healthy || $status == running ]] && return 0 sleep 2; waited=$((waited + 2)) done warn "$name not healthy after ${TIMEOUT}s (status=$status)" return 1 } list_streamers() { docker compose ps --status running --format '{{.Name}}' streamer 2>/dev/null } # Capture application writers, including stopped containers from a rollback. # Older instances can release recipient guards, so they must stop before # a new instance marks permanent-suppression migration complete. application_ids="$(docker compose ps --all -q anonymous_github streamer)" apps=() if [[ -n $application_ids ]]; then mapfile -t apps <<<"$application_ids"; fi legacy_alerts=0 for app in "${apps[@]}"; do protocol="$(docker inspect -f '{{index .Config.Labels "science.4open.owner-alert-protocol"}}' "$app")" [[ $protocol == recipient-v2 ]] || legacy_alerts=1 done # 1. Build log "Building image" before="$(image_id "$IMAGE")" docker compose build anonymous_github after="$(image_id "$IMAGE")" # Validate replacement configuration before stopping any working container. # A healthy HTTP endpoint does not prove that the GitHub App key is mounted. for service in anonymous_github streamer; do log "Checking GitHub App configuration in ${service}" docker compose run --rm --no-deps --entrypoint node "$service" ./build/scripts/check-github-app-key.js done if [[ $legacy_alerts == 1 ]]; then protocol="$(docker image inspect -f '{{index .Config.Labels "science.4open.owner-alert-protocol"}}' "$IMAGE")" [[ $protocol == recipient-v2 ]] || { warn "replacement image lacks recipient alert migration support"; exit 1; } TIMEOUT="${MIGRATION_HEALTH_TIMEOUT:-900}" log "Pausing legacy application writers before recipient alert migration" restart_event docker compose stop -t 30 anonymous_github streamer for app in "${apps[@]}"; do running="$(docker inspect -f '{{.State.Running}}' "$app")" [[ $running == false ]] || { warn "legacy writer $app is still running; aborting migration"; exit 1; } done log "Preparing claim reconciliation, including claims written after rollback" docker compose run --rm --no-deps --entrypoint node anonymous_github ./build/scripts/reconcile-legacy-owner-alerts.js log "Starting ${REPLICAS} streamers after legacy writers stopped" docker compose up -d --no-deps --force-recreate --scale "streamer=${REPLICAS}" streamer mapfile -t news < <(list_streamers) [[ ${#news[@]} == "$REPLICAS" ]] || { warn "streamer replica count does not match ${REPLICAS}"; exit 1; } for app in "${news[@]}"; do wait_healthy "$app"; done docker compose up -d --no-deps --force-recreate anonymous_github api="$(docker compose ps -q anonymous_github)" [[ -n $api ]] || { warn "API container missing after migration"; exit 1; } wait_healthy "$api" docker compose ps exit 0 fi # Cache index upgrades and budget changes must not overlap old/new writers. # Set this explicitly when enabling shared caching for the first time. for app in "${apps[@]}"; do if docker inspect -f '{{range .Config.Env}}{{println .}}{{end}}' "$app" | grep -qx 'SHARED_CACHE_ENABLED=true'; then COORDINATED_RESTART=1 fi done if [[ $COORDINATED_RESTART == 1 ]]; then log "Coordinated application restart for shared cache compatibility" restart_event docker compose stop -t 30 anonymous_github streamer for app in "${apps[@]}"; do [[ $(docker inspect -f '{{.State.Running}}' "$app") == false ]] || { warn "cache writer $app is still running"; exit 1; } done docker compose up -d --no-deps --force-recreate --scale "streamer=${REPLICAS}" streamer mapfile -t news < <(list_streamers) [[ ${#news[@]} == "$REPLICAS" ]] || { warn "streamer replica count does not match ${REPLICAS}"; exit 1; } for app in "${news[@]}"; do wait_healthy "$app"; done docker compose up -d --no-deps --force-recreate anonymous_github api="$(docker compose ps -q anonymous_github)" [[ -n $api ]] || { warn "API container missing after coordinated restart"; exit 1; } wait_healthy "$api" docker compose ps exit 0 fi # 2. Cold start mapfile -t olds < <(list_streamers) if [[ ${#olds[@]} -eq 0 ]]; then log "Cold start — bringing the whole stack up" restart_event docker compose up -d --remove-orphans api="$(docker compose ps -q anonymous_github)" [[ -n $api ]] || { warn "API container missing after cold start"; exit 1; } wait_healthy "$api" docker compose ps exit 0 fi # 3. No-op if image unchanged if [[ -n $before && $before == "$after" && $FORCE != 1 ]]; then deployment_outcome=unchanged log "Image unchanged — nothing to deploy (FORCE=1 to override)" docker compose ps exit 0 fi # 4. Blue/green streamer swap log "Swapping streamer (${#olds[@]} old → ${REPLICAS} new)" docker compose up -d --no-deps --no-recreate \ --scale "streamer=$((${#olds[@]} + REPLICAS))" streamer >/dev/null # Anything not in $olds is new. declare -A was_old=() for o in "${olds[@]}"; do was_old[$o]=1; done news=() while IFS= read -r n; do [[ -n $n && -z ${was_old[$n]:-} ]] && news+=("$n") done < <(list_streamers) # Wait for all new replicas to be healthy in parallel. pids=() for n in "${news[@]}"; do wait_healthy "$n" & pids+=($!); done fail=0 for p in "${pids[@]}"; do wait "$p" || fail=1; done if [[ $fail == 1 ]]; then warn "new replicas unhealthy — keeping olds in place" exit 1 fi # Drop olds and reconcile scale. restart_event docker rm -f "${olds[@]}" >/dev/null docker compose up -d --no-deps --no-recreate --scale "streamer=${REPLICAS}" streamer >/dev/null # 5. Recreate API. --no-deps: we already manage every dependency above, # skipping compose's own depends_on health re-poll (~5–10s). log "Recreating API" docker compose up -d --no-deps --remove-orphans anonymous_github api="$(docker compose ps -q anonymous_github | head -1)" [[ -n $api ]] || { warn "API container missing after restart"; exit 1; } wait_healthy "$api" docker compose ps