Files
anonymous_github/scripts/deploy.sh
T

215 lines
9.0 KiB
Bash
Executable File
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env bash
# Zero-downtime redeploy:
# build → blue/green streamer swap → recreate API.
# Upgrading legacy owner alerts pauses application containers before backfill.
# Skips everything if the rebuilt image hasn't changed (FORCE=1 to override).
set -euo pipefail
cd "$(dirname "$0")/.."
export DOCKER_BUILDKIT=1
export COMPOSE_DOCKER_CLI_BUILD=1
IMAGE="${IMAGE:-tdurieux/anonymous_github:v2}"
REPLICAS="${STREAMER_REPLICAS:-4}"
TIMEOUT="${HEALTH_TIMEOUT:-90}"
FORCE="${FORCE:-0}"
COORDINATED_RESTART="${COORDINATED_RESTART:-0}"
log() { printf '==> %s\n' "$*"; }
warn() { printf '!! %s\n' "$*" >&2; }
# Local, finite deployment evidence. Collection is opt-in via a pre-created file.
# Logging failure must never change deployment behavior or expose shell commands.
deployment_started=$(date +%s)
restart_started=0
deployment_id=$(cat /proc/sys/kernel/random/uuid 2>/dev/null || true)
[[ $deployment_id =~ ^[a-f0-9-]{36}$ ]] || deployment_id=""
deployment_release=${APP_RELEASE:-unknown}
[[ $deployment_release =~ ^[a-f0-9]{7,40}$ ]] || deployment_release=unknown
deployment_operation=${DEPLOY_OPERATION:-deploy}
[[ $deployment_operation == rollback ]] || deployment_operation=deploy
deployment_outcome=completed
deployment_event() {
local phase="$1" outcome="$2" code="$3" now elapsed restart_ms record
now=$(date +%s); elapsed=$(( (now - deployment_started) * 1000 )); restart_ms=0
if (( restart_started > 0 )); then restart_ms=$(( (now - restart_started) * 1000 )); fi
printf -v record '{"timestamp":"%s","service":"api","severity":"%s","event":"deployment","deploymentId":"%s","release":"%s","deploymentOperation":"%s","deploymentPhase":"%s","deploymentOutcome":"%s","ms":%d,"restartMs":%d,"exitCode":%d}' \
"$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$([[ $code == 0 ]] && echo info || echo error)" "$deployment_id" "$deployment_release" "$deployment_operation" "$phase" "$outcome" "$elapsed" "$restart_ms" "$code"
printf '%s\n' "$record" || true
if [[ -n ${DEPLOY_EVENT_FILE:-} ]]; then
if [[ -f $DEPLOY_EVENT_FILE && ! -L $DEPLOY_EVENT_FILE ]]; then
(printf '%s\n' "$record" >> "$DEPLOY_EVENT_FILE") 2>/dev/null || warn "deployment event file unavailable"
else warn "deployment event file must already exist and must not be a symlink"; fi
fi
return 0
}
restart_event() {
if (( restart_started == 0 )); then restart_started=$(date +%s); deployment_event restart started 0; fi
}
deployment_finished() {
local code=$?
if (( code != 0 )); then deployment_outcome=failed; fi
deployment_event finish "$deployment_outcome" "$code"
exit "$code"
}
trap deployment_finished EXIT
trap 'exit 129' HUP
trap 'exit 130' INT
trap 'exit 143' TERM
deployment_event start started 0
image_id() { docker image inspect --format '{{.Id}}' "$1" 2>/dev/null || true; }
wait_healthy() {
local name="$1" waited=0 status
while (( waited < TIMEOUT )); do
status="$(docker inspect -f \
'{{if .State.Health}}{{.State.Health.Status}}{{else if .State.Running}}running{{else}}stopped{{end}}' \
"$name" 2>/dev/null || echo missing)"
[[ $status == healthy || $status == running ]] && return 0
sleep 2; waited=$((waited + 2))
done
warn "$name not healthy after ${TIMEOUT}s (status=$status)"
return 1
}
list_streamers() {
docker compose ps --status running --format '{{.Name}}' streamer 2>/dev/null
}
# Capture application writers, including stopped containers from a rollback.
# Older instances can release recipient guards, so they must stop before
# a new instance marks permanent-suppression migration complete.
application_ids="$(docker compose ps --all -q anonymous_github streamer)"
apps=()
if [[ -n $application_ids ]]; then mapfile -t apps <<<"$application_ids"; fi
legacy_alerts=0
for app in "${apps[@]}"; do
protocol="$(docker inspect -f '{{index .Config.Labels "science.4open.owner-alert-protocol"}}' "$app")"
[[ $protocol == recipient-v2 ]] || legacy_alerts=1
done
# 1. Build
log "Building image"
before="$(image_id "$IMAGE")"
docker compose build anonymous_github
after="$(image_id "$IMAGE")"
# Validate replacement configuration before stopping any working container.
# A healthy HTTP endpoint does not prove that the GitHub App key is mounted.
for service in anonymous_github streamer; do
log "Checking GitHub App configuration in ${service}"
docker compose run --rm --no-deps --entrypoint node "$service" ./build/scripts/check-github-app-key.js
done
if [[ $legacy_alerts == 1 ]]; then
protocol="$(docker image inspect -f '{{index .Config.Labels "science.4open.owner-alert-protocol"}}' "$IMAGE")"
[[ $protocol == recipient-v2 ]] || { warn "replacement image lacks recipient alert migration support"; exit 1; }
TIMEOUT="${MIGRATION_HEALTH_TIMEOUT:-900}"
log "Pausing legacy application writers before recipient alert migration"
restart_event
docker compose stop -t 30 anonymous_github streamer
for app in "${apps[@]}"; do
running="$(docker inspect -f '{{.State.Running}}' "$app")"
[[ $running == false ]] || { warn "legacy writer $app is still running; aborting migration"; exit 1; }
done
log "Preparing claim reconciliation, including claims written after rollback"
docker compose run --rm --no-deps --entrypoint node anonymous_github ./build/scripts/reconcile-legacy-owner-alerts.js
log "Starting ${REPLICAS} streamers after legacy writers stopped"
docker compose up -d --no-deps --force-recreate --scale "streamer=${REPLICAS}" streamer
mapfile -t news < <(list_streamers)
[[ ${#news[@]} == "$REPLICAS" ]] || { warn "streamer replica count does not match ${REPLICAS}"; exit 1; }
for app in "${news[@]}"; do wait_healthy "$app"; done
docker compose up -d --no-deps --force-recreate anonymous_github
api="$(docker compose ps -q anonymous_github)"
[[ -n $api ]] || { warn "API container missing after migration"; exit 1; }
wait_healthy "$api"
docker compose ps
exit 0
fi
# Cache index upgrades and budget changes must not overlap old/new writers.
# Set this explicitly when enabling shared caching for the first time.
for app in "${apps[@]}"; do
if docker inspect -f '{{range .Config.Env}}{{println .}}{{end}}' "$app" | grep -qx 'SHARED_CACHE_ENABLED=true'; then
COORDINATED_RESTART=1
fi
done
if [[ $COORDINATED_RESTART == 1 ]]; then
log "Coordinated application restart for shared cache compatibility"
restart_event
docker compose stop -t 30 anonymous_github streamer
for app in "${apps[@]}"; do
[[ $(docker inspect -f '{{.State.Running}}' "$app") == false ]] || { warn "cache writer $app is still running"; exit 1; }
done
docker compose up -d --no-deps --force-recreate --scale "streamer=${REPLICAS}" streamer
mapfile -t news < <(list_streamers)
[[ ${#news[@]} == "$REPLICAS" ]] || { warn "streamer replica count does not match ${REPLICAS}"; exit 1; }
for app in "${news[@]}"; do wait_healthy "$app"; done
docker compose up -d --no-deps --force-recreate anonymous_github
api="$(docker compose ps -q anonymous_github)"
[[ -n $api ]] || { warn "API container missing after coordinated restart"; exit 1; }
wait_healthy "$api"
docker compose ps
exit 0
fi
# 2. Cold start
mapfile -t olds < <(list_streamers)
if [[ ${#olds[@]} -eq 0 ]]; then
log "Cold start — bringing the whole stack up"
restart_event
docker compose up -d --remove-orphans
api="$(docker compose ps -q anonymous_github)"
[[ -n $api ]] || { warn "API container missing after cold start"; exit 1; }
wait_healthy "$api"
docker compose ps
exit 0
fi
# 3. No-op if image unchanged
if [[ -n $before && $before == "$after" && $FORCE != 1 ]]; then
deployment_outcome=unchanged
log "Image unchanged — nothing to deploy (FORCE=1 to override)"
docker compose ps
exit 0
fi
# 4. Blue/green streamer swap
log "Swapping streamer (${#olds[@]} old → ${REPLICAS} new)"
docker compose up -d --no-deps --no-recreate \
--scale "streamer=$((${#olds[@]} + REPLICAS))" streamer >/dev/null
# Anything not in $olds is new.
declare -A was_old=()
for o in "${olds[@]}"; do was_old[$o]=1; done
news=()
while IFS= read -r n; do
[[ -n $n && -z ${was_old[$n]:-} ]] && news+=("$n")
done < <(list_streamers)
# Wait for all new replicas to be healthy in parallel.
pids=()
for n in "${news[@]}"; do wait_healthy "$n" & pids+=($!); done
fail=0
for p in "${pids[@]}"; do wait "$p" || fail=1; done
if [[ $fail == 1 ]]; then
warn "new replicas unhealthy — keeping olds in place"
exit 1
fi
# Drop olds and reconcile scale.
restart_event
docker rm -f "${olds[@]}" >/dev/null
docker compose up -d --no-deps --no-recreate --scale "streamer=${REPLICAS}" streamer >/dev/null
# 5. Recreate API. --no-deps: we already manage every dependency above,
# skipping compose's own depends_on health re-poll (~5–10s).
log "Recreating API"
docker compose up -d --no-deps --remove-orphans anonymous_github
api="$(docker compose ps -q anonymous_github | head -1)"
[[ -n $api ]] || { warn "API container missing after restart"; exit 1; }
wait_healthy "$api"
docker compose ps