From c32125fc2edb5a36bc2099e973d7ebd288cf6c06 Mon Sep 17 00:00:00 2001 From: Thomas Durieux <5577568+tdurieux@users.noreply.github.com> Date: Tue, 3 Nov 2026 08:33:12 +0000 Subject: [PATCH] fix: coordinate deployments when shared cache writers are active --- docs/shared-cache.md | 4 ++++ scripts/deploy.sh | 26 +++++++++++++++++++++++ test/deploy-owner-alert-migration.test.js | 17 ++++++++++++++- 3 files changed, 46 insertions(+), 1 deletion(-) diff --git a/docs/shared-cache.md b/docs/shared-cache.md index 34dfa8f..c07e7bf 100644 --- a/docs/shared-cache.md +++ b/docs/shared-cache.md @@ -171,6 +171,10 @@ History and decisions are capped at 100,000 rows each, aliases at 200,000. Expir history/decision cleanup removes at most 256 rows per pass. Access history has 30-second resolution. Stop all cache processes before upgrading an existing index and restart them on the same version; do not mix old and new writers. +`scripts/deploy.sh` coordinates restarts when existing containers have shared caching +enabled. For first activation or an explicit budget/configuration change, run +`COORDINATED_RESTART=1 bash scripts/deploy.sh`. It stops and verifies every old +application writer before starting replacements. This briefly interrupts service. ### Performance and validation diff --git a/scripts/deploy.sh b/scripts/deploy.sh index dab0c01..697af85 100755 --- a/scripts/deploy.sh +++ b/scripts/deploy.sh @@ -13,6 +13,7 @@ IMAGE="${IMAGE:-tdurieux/anonymous_github:v2}" REPLICAS="${STREAMER_REPLICAS:-4}" TIMEOUT="${HEALTH_TIMEOUT:-90}" FORCE="${FORCE:-0}" +COORDINATED_RESTART="${COORDINATED_RESTART:-0}" log() { printf '==> %s\n' "$*"; } warn() { printf '!! %s\n' "$*" >&2; } @@ -79,6 +80,31 @@ if [[ $legacy_alerts == 1 ]]; then exit 0 fi +# Cache index upgrades and budget changes must not overlap old/new writers. +# Set this explicitly when enabling shared caching for the first time. +for app in "${apps[@]}"; do + if docker inspect -f '{{range .Config.Env}}{{println .}}{{end}}' "$app" | grep -qx 'SHARED_CACHE_ENABLED=true'; then + COORDINATED_RESTART=1 + fi +done +if [[ $COORDINATED_RESTART == 1 ]]; then + log "Coordinated application restart for shared cache compatibility" + docker compose stop -t 30 anonymous_github streamer + for app in "${apps[@]}"; do + [[ $(docker inspect -f '{{.State.Running}}' "$app") == false ]] || { warn "cache writer $app is still running"; exit 1; } + done + docker compose up -d --no-deps --force-recreate --scale "streamer=${REPLICAS}" streamer + mapfile -t news < <(list_streamers) + [[ ${#news[@]} == "$REPLICAS" ]] || { warn "streamer replica count does not match ${REPLICAS}"; exit 1; } + for app in "${news[@]}"; do wait_healthy "$app"; done + docker compose up -d --no-deps --force-recreate anonymous_github + api="$(docker compose ps -q anonymous_github)" + [[ -n $api ]] || { warn "API container missing after coordinated restart"; exit 1; } + wait_healthy "$api" + docker compose ps + exit 0 +fi + # 2. Cold start mapfile -t olds < <(list_streamers) if [[ ${#olds[@]} -eq 0 ]]; then diff --git a/test/deploy-owner-alert-migration.test.js b/test/deploy-owner-alert-migration.test.js index 9274526..1e09e8f 100644 --- a/test/deploy-owner-alert-migration.test.js +++ b/test/deploy-owner-alert-migration.test.js @@ -22,6 +22,7 @@ if (args[0] === "image" && args[1] === "inspect") { finish(state.built ? "sha:new" : "sha:old"); } if (args[0] === "inspect") { + if (args.some(a => a.includes("Config.Env"))) finish(state.sharedCache ? "SHARED_CACHE_ENABLED=true\\n" : ""); if (args.some(a => a.includes("Labels"))) finish(state.oldProtocol || ""); if (args.includes("{{.State.Running}}")) finish(state.stopped && !state.refuseStop ? "false" : "true"); finish("healthy"); @@ -43,12 +44,26 @@ finish("unexpected docker call", 2); const result = spawnSync("bash", [path.join(directory, "scripts/deploy.sh")], { encoding: "utf8", timeout: 8000, env: { ...process.env, PATH: directory + path.delimiter + process.env.PATH, - FORCE: String(options.force ?? 1), STREAMER_REPLICAS: "4", TASK_DOCKER_STATE: statePath, TASK_DOCKER_LOG: logPath }, + COORDINATED_RESTART: String(options.coordinated ?? 0), FORCE: String(options.force ?? 1), STREAMER_REPLICAS: "4", TASK_DOCKER_STATE: statePath, TASK_DOCKER_LOG: logPath }, }); return { ...result, calls: fs.readFileSync(logPath, "utf8").trim().split("\n").map(line => JSON.parse(line)) }; } finally { fs.rmSync(directory, { recursive: true, force: true }); } } + for (const sharedCache of [true, false]) it(`coordinates cache rollout with ${sharedCache ? "existing" : "new"} caching`, () => { + const result = deploy({ oldProtocol: "recipient-v2", sharedCache, coordinated: sharedCache ? 0 : 1 }); + expect(result.status, result.stderr).to.equal(0); + const stop = result.calls.findIndex(args => args[0] === "compose" && args[1] === "stop"); + const up = result.calls.findIndex(args => args[0] === "compose" && args[1] === "up"); + expect(stop).to.be.greaterThan(-1); expect(up).to.be.greaterThan(stop); + expect(result.calls[up]).to.include.members(["--force-recreate", "streamer=4"]); + expect(result.calls.some(args => args.includes("streamer=8"))).to.equal(false); + }); + it("does not start new cache writers when an old writer fails to stop", () => { + const result = deploy({ oldProtocol: "recipient-v2", coordinated: 1, refuseStop: true }); + expect(result.status).not.to.equal(0); + expect(result.calls.some(args => args[0] === "compose" && args[1] === "up")).to.equal(false); + }); it("stops and verifies every legacy writer before starting any migration-capable instance", () => { const result = deploy(); expect(result.status, result.stderr).to.equal(0);