mirror of
https://github.com/tdurieux/anonymous_github.git
synced 2026-11-17 17:44:09 +01:00
215 lines
9.0 KiB
Bash
Executable File
215 lines
9.0 KiB
Bash
Executable File
#!/usr/bin/env bash
|
||
# Zero-downtime redeploy:
|
||
# build → blue/green streamer swap → recreate API.
|
||
# Upgrading legacy owner alerts pauses application containers before backfill.
|
||
# Skips everything if the rebuilt image hasn't changed (FORCE=1 to override).
|
||
set -euo pipefail
|
||
cd "$(dirname "$0")/.."
|
||
|
||
export DOCKER_BUILDKIT=1
|
||
export COMPOSE_DOCKER_CLI_BUILD=1
|
||
|
||
IMAGE="${IMAGE:-tdurieux/anonymous_github:v2}"
|
||
REPLICAS="${STREAMER_REPLICAS:-4}"
|
||
TIMEOUT="${HEALTH_TIMEOUT:-90}"
|
||
FORCE="${FORCE:-0}"
|
||
COORDINATED_RESTART="${COORDINATED_RESTART:-0}"
|
||
|
||
log() { printf '==> %s\n' "$*"; }
|
||
warn() { printf '!! %s\n' "$*" >&2; }
|
||
|
||
# Local, finite deployment evidence. Collection is opt-in via a pre-created file.
|
||
# Logging failure must never change deployment behavior or expose shell commands.
|
||
deployment_started=$(date +%s)
|
||
restart_started=0
|
||
deployment_id=$(cat /proc/sys/kernel/random/uuid 2>/dev/null || true)
|
||
[[ $deployment_id =~ ^[a-f0-9-]{36}$ ]] || deployment_id=""
|
||
deployment_release=${APP_RELEASE:-unknown}
|
||
[[ $deployment_release =~ ^[a-f0-9]{7,40}$ ]] || deployment_release=unknown
|
||
deployment_operation=${DEPLOY_OPERATION:-deploy}
|
||
[[ $deployment_operation == rollback ]] || deployment_operation=deploy
|
||
deployment_outcome=completed
|
||
deployment_event() {
|
||
local phase="$1" outcome="$2" code="$3" now elapsed restart_ms record
|
||
now=$(date +%s); elapsed=$(( (now - deployment_started) * 1000 )); restart_ms=0
|
||
if (( restart_started > 0 )); then restart_ms=$(( (now - restart_started) * 1000 )); fi
|
||
printf -v record '{"timestamp":"%s","service":"api","severity":"%s","event":"deployment","deploymentId":"%s","release":"%s","deploymentOperation":"%s","deploymentPhase":"%s","deploymentOutcome":"%s","ms":%d,"restartMs":%d,"exitCode":%d}' \
|
||
"$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$([[ $code == 0 ]] && echo info || echo error)" "$deployment_id" "$deployment_release" "$deployment_operation" "$phase" "$outcome" "$elapsed" "$restart_ms" "$code"
|
||
printf '%s\n' "$record" || true
|
||
if [[ -n ${DEPLOY_EVENT_FILE:-} ]]; then
|
||
if [[ -f $DEPLOY_EVENT_FILE && ! -L $DEPLOY_EVENT_FILE ]]; then
|
||
(printf '%s\n' "$record" >> "$DEPLOY_EVENT_FILE") 2>/dev/null || warn "deployment event file unavailable"
|
||
else warn "deployment event file must already exist and must not be a symlink"; fi
|
||
fi
|
||
return 0
|
||
}
|
||
restart_event() {
|
||
if (( restart_started == 0 )); then restart_started=$(date +%s); deployment_event restart started 0; fi
|
||
}
|
||
deployment_finished() {
|
||
local code=$?
|
||
if (( code != 0 )); then deployment_outcome=failed; fi
|
||
deployment_event finish "$deployment_outcome" "$code"
|
||
exit "$code"
|
||
}
|
||
trap deployment_finished EXIT
|
||
trap 'exit 129' HUP
|
||
trap 'exit 130' INT
|
||
trap 'exit 143' TERM
|
||
deployment_event start started 0
|
||
|
||
image_id() { docker image inspect --format '{{.Id}}' "$1" 2>/dev/null || true; }
|
||
|
||
wait_healthy() {
|
||
local name="$1" waited=0 status
|
||
while (( waited < TIMEOUT )); do
|
||
status="$(docker inspect -f \
|
||
'{{if .State.Health}}{{.State.Health.Status}}{{else if .State.Running}}running{{else}}stopped{{end}}' \
|
||
"$name" 2>/dev/null || echo missing)"
|
||
[[ $status == healthy || $status == running ]] && return 0
|
||
sleep 2; waited=$((waited + 2))
|
||
done
|
||
warn "$name not healthy after ${TIMEOUT}s (status=$status)"
|
||
return 1
|
||
}
|
||
|
||
list_streamers() {
|
||
docker compose ps --status running --format '{{.Name}}' streamer 2>/dev/null
|
||
}
|
||
|
||
# Capture application writers, including stopped containers from a rollback.
|
||
# Older instances can release recipient guards, so they must stop before
|
||
# a new instance marks permanent-suppression migration complete.
|
||
application_ids="$(docker compose ps --all -q anonymous_github streamer)"
|
||
apps=()
|
||
if [[ -n $application_ids ]]; then mapfile -t apps <<<"$application_ids"; fi
|
||
legacy_alerts=0
|
||
for app in "${apps[@]}"; do
|
||
protocol="$(docker inspect -f '{{index .Config.Labels "science.4open.owner-alert-protocol"}}' "$app")"
|
||
[[ $protocol == recipient-v2 ]] || legacy_alerts=1
|
||
done
|
||
|
||
# 1. Build
|
||
log "Building image"
|
||
before="$(image_id "$IMAGE")"
|
||
docker compose build anonymous_github
|
||
after="$(image_id "$IMAGE")"
|
||
|
||
# Validate replacement configuration before stopping any working container.
|
||
# A healthy HTTP endpoint does not prove that the GitHub App key is mounted.
|
||
for service in anonymous_github streamer; do
|
||
log "Checking GitHub App configuration in ${service}"
|
||
docker compose run --rm --no-deps --entrypoint node "$service" ./build/scripts/check-github-app-key.js
|
||
done
|
||
|
||
if [[ $legacy_alerts == 1 ]]; then
|
||
protocol="$(docker image inspect -f '{{index .Config.Labels "science.4open.owner-alert-protocol"}}' "$IMAGE")"
|
||
[[ $protocol == recipient-v2 ]] || { warn "replacement image lacks recipient alert migration support"; exit 1; }
|
||
TIMEOUT="${MIGRATION_HEALTH_TIMEOUT:-900}"
|
||
log "Pausing legacy application writers before recipient alert migration"
|
||
restart_event
|
||
docker compose stop -t 30 anonymous_github streamer
|
||
for app in "${apps[@]}"; do
|
||
running="$(docker inspect -f '{{.State.Running}}' "$app")"
|
||
[[ $running == false ]] || { warn "legacy writer $app is still running; aborting migration"; exit 1; }
|
||
done
|
||
log "Preparing claim reconciliation, including claims written after rollback"
|
||
docker compose run --rm --no-deps --entrypoint node anonymous_github ./build/scripts/reconcile-legacy-owner-alerts.js
|
||
log "Starting ${REPLICAS} streamers after legacy writers stopped"
|
||
docker compose up -d --no-deps --force-recreate --scale "streamer=${REPLICAS}" streamer
|
||
mapfile -t news < <(list_streamers)
|
||
[[ ${#news[@]} == "$REPLICAS" ]] || { warn "streamer replica count does not match ${REPLICAS}"; exit 1; }
|
||
for app in "${news[@]}"; do wait_healthy "$app"; done
|
||
docker compose up -d --no-deps --force-recreate anonymous_github
|
||
api="$(docker compose ps -q anonymous_github)"
|
||
[[ -n $api ]] || { warn "API container missing after migration"; exit 1; }
|
||
wait_healthy "$api"
|
||
docker compose ps
|
||
exit 0
|
||
fi
|
||
|
||
# Cache index upgrades and budget changes must not overlap old/new writers.
|
||
# Set this explicitly when enabling shared caching for the first time.
|
||
for app in "${apps[@]}"; do
|
||
if docker inspect -f '{{range .Config.Env}}{{println .}}{{end}}' "$app" | grep -qx 'SHARED_CACHE_ENABLED=true'; then
|
||
COORDINATED_RESTART=1
|
||
fi
|
||
done
|
||
if [[ $COORDINATED_RESTART == 1 ]]; then
|
||
log "Coordinated application restart for shared cache compatibility"
|
||
restart_event
|
||
docker compose stop -t 30 anonymous_github streamer
|
||
for app in "${apps[@]}"; do
|
||
[[ $(docker inspect -f '{{.State.Running}}' "$app") == false ]] || { warn "cache writer $app is still running"; exit 1; }
|
||
done
|
||
docker compose up -d --no-deps --force-recreate --scale "streamer=${REPLICAS}" streamer
|
||
mapfile -t news < <(list_streamers)
|
||
[[ ${#news[@]} == "$REPLICAS" ]] || { warn "streamer replica count does not match ${REPLICAS}"; exit 1; }
|
||
for app in "${news[@]}"; do wait_healthy "$app"; done
|
||
docker compose up -d --no-deps --force-recreate anonymous_github
|
||
api="$(docker compose ps -q anonymous_github)"
|
||
[[ -n $api ]] || { warn "API container missing after coordinated restart"; exit 1; }
|
||
wait_healthy "$api"
|
||
docker compose ps
|
||
exit 0
|
||
fi
|
||
|
||
# 2. Cold start
|
||
mapfile -t olds < <(list_streamers)
|
||
if [[ ${#olds[@]} -eq 0 ]]; then
|
||
log "Cold start — bringing the whole stack up"
|
||
restart_event
|
||
docker compose up -d --remove-orphans
|
||
api="$(docker compose ps -q anonymous_github)"
|
||
[[ -n $api ]] || { warn "API container missing after cold start"; exit 1; }
|
||
wait_healthy "$api"
|
||
docker compose ps
|
||
exit 0
|
||
fi
|
||
|
||
# 3. No-op if image unchanged
|
||
if [[ -n $before && $before == "$after" && $FORCE != 1 ]]; then
|
||
deployment_outcome=unchanged
|
||
log "Image unchanged — nothing to deploy (FORCE=1 to override)"
|
||
docker compose ps
|
||
exit 0
|
||
fi
|
||
|
||
# 4. Blue/green streamer swap
|
||
log "Swapping streamer (${#olds[@]} old → ${REPLICAS} new)"
|
||
docker compose up -d --no-deps --no-recreate \
|
||
--scale "streamer=$((${#olds[@]} + REPLICAS))" streamer >/dev/null
|
||
|
||
# Anything not in $olds is new.
|
||
declare -A was_old=()
|
||
for o in "${olds[@]}"; do was_old[$o]=1; done
|
||
news=()
|
||
while IFS= read -r n; do
|
||
[[ -n $n && -z ${was_old[$n]:-} ]] && news+=("$n")
|
||
done < <(list_streamers)
|
||
|
||
# Wait for all new replicas to be healthy in parallel.
|
||
pids=()
|
||
for n in "${news[@]}"; do wait_healthy "$n" & pids+=($!); done
|
||
fail=0
|
||
for p in "${pids[@]}"; do wait "$p" || fail=1; done
|
||
if [[ $fail == 1 ]]; then
|
||
warn "new replicas unhealthy — keeping olds in place"
|
||
exit 1
|
||
fi
|
||
|
||
# Drop olds and reconcile scale.
|
||
restart_event
|
||
docker rm -f "${olds[@]}" >/dev/null
|
||
docker compose up -d --no-deps --no-recreate --scale "streamer=${REPLICAS}" streamer >/dev/null
|
||
|
||
# 5. Recreate API. --no-deps: we already manage every dependency above,
|
||
# skipping compose's own depends_on health re-poll (~5–10s).
|
||
log "Recreating API"
|
||
docker compose up -d --no-deps --remove-orphans anonymous_github
|
||
api="$(docker compose ps -q anonymous_github | head -1)"
|
||
[[ -n $api ]] || { warn "API container missing after restart"; exit 1; }
|
||
wait_healthy "$api"
|
||
|
||
docker compose ps
|