From e54b78c7e6d8be7576fb26358980c76d9520e357 Mon Sep 17 00:00:00 2001 From: Thomas Durieux <5577568+tdurieux@users.noreply.github.com> Date: Wed, 29 Jul 2026 18:10:13 -0700 Subject: [PATCH] Add a reliable MongoDB replica setup runbook (#766) * docs: clarify MongoDB Tailscale bind addresses * fix: make MongoDB replica host mapping deterministic * docs: add complete MongoDB replica runbook * docs: support different replica server paths --- docker-compose.replica-primary.yml | 7 + docker-compose.replica-secondary.yml | 7 + docs/mongodb-replication.md | 820 ++++++++++++++++++++++----- scripts/mongodb-replica.sh | 69 ++- 4 files changed, 760 insertions(+), 143 deletions(-) diff --git a/docker-compose.replica-primary.yml b/docker-compose.replica-primary.yml index 2805c5f..6081e17 100644 --- a/docker-compose.replica-primary.yml +++ b/docker-compose.replica-primary.yml @@ -1,5 +1,6 @@ services: mongodb: + hostname: ${MONGO_PRIMARY_HOSTNAME:-mongo-primary} entrypoint: - /bin/bash - /opt/anonymous-github/mongodb-replica-entrypoint.sh @@ -12,6 +13,12 @@ services: - --bind_ip_all environment: MONGO_REPLICA_KEYFILE_PATH: /run/secrets/mongo-replica-keyfile + extra_hosts: + - "${MONGO_SECONDARY_HOSTNAME:-mongo-secondary}:${MONGO_SECONDARY_ADDRESS:?Set MONGO_SECONDARY_ADDRESS to the secondary server Tailscale IP}" + networks: + default: + aliases: + - ${MONGO_PRIMARY_HOSTNAME:-mongo-primary} volumes: - ${MONGO_REPLICA_KEYFILE:-./secrets/mongo-replica-keyfile}:/run/secrets/mongo-replica-keyfile:ro - ./scripts/mongodb-replica-entrypoint.sh:/opt/anonymous-github/mongodb-replica-entrypoint.sh:ro diff --git a/docker-compose.replica-secondary.yml b/docker-compose.replica-secondary.yml index c32f9a0..187a6df 100644 --- a/docker-compose.replica-secondary.yml +++ b/docker-compose.replica-secondary.yml @@ -3,6 +3,7 @@ name: anonymous-github-mongo-secondary services: mongodb-secondary: image: ${MONGO_IMAGE:-mongo:latest} + hostname: ${MONGO_SECONDARY_HOSTNAME:-mongo-secondary} restart: unless-stopped entrypoint: - /bin/bash @@ -16,6 +17,12 @@ services: - --bind_ip_all environment: MONGO_REPLICA_KEYFILE_PATH: /run/secrets/mongo-replica-keyfile + extra_hosts: + - "${MONGO_PRIMARY_HOSTNAME:-mongo-primary}:${MONGO_PRIMARY_ADDRESS:?Set MONGO_PRIMARY_ADDRESS to the primary server Tailscale IP}" + networks: + default: + aliases: + - ${MONGO_SECONDARY_HOSTNAME:-mongo-secondary} volumes: - mongodb_secondary_data:/data/db - ${MONGO_REPLICA_KEYFILE:-./secrets/mongo-replica-keyfile}:/run/secrets/mongo-replica-keyfile:ro diff --git a/docs/mongodb-replication.md b/docs/mongodb-replication.md index 6bcda60..84ac131 100644 --- a/docs/mongodb-replication.md +++ b/docs/mongodb-replication.md @@ -1,200 +1,736 @@ -# Optional remote MongoDB replica +# Remote MongoDB recovery replica -This deployment option converts the existing standalone MongoDB container into -a replica set named `rs0` and adds a remote, hidden, non-voting secondary. -Normal single-node deployments continue to use `docker-compose.yml` alone. +This guide converts the existing production MongoDB container into a replica +set named `rs0` and adds a MongoDB container on a second server. The servers +communicate over Tailscale; MongoDB is not exposed on a public interface. -The remote member is designed as a recovery and backup source: +Read the complete guide before changing production. -- It cannot become primary. -- Its availability does not affect production majority writes. -- It is hidden from application traffic. -- It is delayed by one hour by default, providing a short recovery window for - accidental deletes or a bad migration. +## What this creates -Replication is not a backup. Destructive changes eventually reach every -member. Continue making encrypted, off-host `mongodump` archives from the +| Member | Role | Votes | Priority | Hidden | Default delay | +| --- | --- | ---: | ---: | --- | ---: | +| `mongo-primary` | Production primary | 1 | 1 | No | 0 | +| `mongo-secondary` | Recovery copy | 0 | 0 | Yes | 1 hour | + +The secondary cannot become primary. Losing the secondary or its network +connection therefore does not stop production writes. This two-server layout +is for recovery and for taking backups from a remote copy; it is not automatic +failover. + +Replication is also not a backup. Deletes and corrupt data eventually +replicate. Continue making versioned, encrypted, off-server `mongodump` +archives. + +## How names and addresses are used + +MongoDB replica-set members are advertised with stable hostnames: + +- `mongo-primary:27017` +- `mongo-secondary:27017` + +Do not advertise the Tailscale IPs directly in `rs.initiate()` or `rs.add()`. +MongoDB expects stable hostnames, and recent MongoDB versions reject +IP-only replica-set configurations. + +The Compose overlays make the two hostnames deterministic: + +- Inside the primary Docker network, `mongo-primary` resolves directly to the + primary MongoDB container. +- Inside the secondary Docker network, `mongo-secondary` resolves directly to + the secondary MongoDB container. +- Cross-server names are mapped to the other server's Tailscale IP with + Compose `extra_hosts`. + +This mapping is important. If `mongo-primary` does not map back to the primary +container, `rs.initiate()` fails with: + +```text +No host described in new configuration ... maps to this node +``` + +## Example values + +The guide uses these examples: + +| Setting | Example | +| --- | --- | +| Primary SSH user | `ubuntu` | +| Primary repository path | `/home/ubuntu/git/anonymous_github` | +| Secondary SSH user | `deploy` | +| Secondary repository path | `/srv/anonymous_github` | +| Primary Tailscale machine name | `mongo-primary` | +| Primary Tailscale IPv4 | `100.64.10.20` | +| Secondary Tailscale machine name | `mongo-secondary` | +| Secondary Tailscale IPv4 | `100.64.10.30` | + +The users and absolute repository paths may be different on the two servers. +Run each command from that server's own checkout. Replace every example user, +IP, and path with the value from your server. Never enter the literal value +`100.x.x.x`. + +`MONGO_BIND_ADDRESS` always contains the Tailscale IP of the server whose +`.env` file you are editing: + +```text +primary .env -> MONGO_BIND_ADDRESS= +secondary .env -> MONGO_BIND_ADDRESS= +``` + +## Prerequisites + +Before starting, confirm all of the following: + +- The production standalone MongoDB is healthy. +- You have a recent, verified `mongodump` backup. +- The repository is installed on both servers; the absolute paths and owners + may be different. +- Both servers run the same repository revision. +- Both servers run the exact same MongoDB version. +- Tailscale is connected on both servers. +- TCP port `27017` is not publicly exposed. +- You have a maintenance window for restarting MongoDB and the application. + +Do not delete or copy a live WiredTiger data directory. The existing primary +volume stays in place, and MongoDB initial-syncs the secondary after it is +added. + +## 1. Record the current production state + +Run on the primary: + +```bash +cd /home/ubuntu/git/anonymous_github + +docker compose -f docker-compose.yml ps +docker compose -f docker-compose.yml exec -T mongodb mongod --version +git rev-parse HEAD +``` + +Pin the current MongoDB version in `.env`. Do not leave production on +`mongo:latest` while changing the topology: + +```env +MONGO_IMAGE=mongo: +``` + +Save a protected copy of the current configuration: + +```bash +cp .env .env.before-replica +chmod 600 .env.before-replica +``` + +This file contains secrets. Do not commit or copy it to an untrusted system. + +## 2. Back up the standalone database + +Stop application writers for a consistent pre-conversion backup: + +```bash +docker compose -f docker-compose.yml stop \ + anonymous_github streamer mongodb-backup +``` + +Create an archive: + +```bash +mkdir -p db_backups/manual +BACKUP="db_backups/manual/production-before-replica-$(date -u +%Y%m%dT%H%M%SZ).archive.gz" + +docker compose -f docker-compose.yml exec -T mongodb sh -eu -c \ + 'mongodump \ + --username="$MONGO_INITDB_ROOT_USERNAME" \ + --password="$MONGO_INITDB_ROOT_PASSWORD" \ + --authenticationDatabase=admin \ + --db=production \ + --archive \ + --gzip' > "$BACKUP" + +test -s "$BACKUP" +chmod 600 "$BACKUP" +ls -lh "$BACKUP" +``` + +Keep the application stopped until the replica primary is working. If you +restart it temporarily, stop it again before step 9. + +## 3. Connect and name both Tailscale servers + +On the primary: + +```bash +sudo tailscale set --hostname=mongo-primary +tailscale ip -4 +tailscale status +``` + +On the secondary: + +```bash +sudo tailscale set --hostname=mongo-secondary +tailscale ip -4 +tailscale status +``` + +Record both IPv4 addresses. For the example topology: + +```text +mongo-primary 100.64.10.20 +mongo-secondary 100.64.10.30 +``` + +Verify connectivity in both directions: + +```bash +# Run on the primary. +tailscale ping mongo-secondary + +# Run on the secondary. +tailscale ping mongo-primary +``` + +If the short names do not work, fix Tailscale/MagicDNS first. You may test with +the Tailscale IPs, but the MongoDB member names used later remain +`mongo-primary` and `mongo-secondary`. + +Restrict your Tailscale policy so only the required servers can reach TCP +`27017`. Do not publish MongoDB on `0.0.0.0` or a public cloud interface. + +## 4. Install the same repository revision on the secondary + +On the primary: + +```bash +cd /home/ubuntu/git/anonymous_github +git rev-parse HEAD +``` + +On the secondary, use its own repository path: + +```bash +cd /srv/anonymous_github +git rev-parse HEAD +``` + +The commit IDs must match. The following files must exist: + +```bash +test -f docker-compose.replica-primary.yml +test -f docker-compose.replica-secondary.yml +test -x scripts/mongodb-replica.sh +test -x scripts/mongodb-replica-entrypoint.sh +``` + +Do not start a second copy of the Anonymous GitHub application on the +secondary. Only the `mongodb-secondary` service is used there. + +## 5. Generate and copy the shared MongoDB keyfile + +Generate the key once on the primary: + +```bash +cd /home/ubuntu/git/anonymous_github +./scripts/mongodb-replica.sh generate-key +``` + +If the file already exists, reuse it. Do not generate a different key on the secondary. -## Requirements +Create the destination directory: -1. Take and verify a `mongodump` backup before converting production. -2. Install this repository on both servers. -3. Connect the servers through a trusted private network such as WireGuard, - Tailscale, a private VLAN, or a cloud private network. -4. Create stable DNS names that resolve from both servers and from the - application container, for example: +```bash +ssh deploy@mongo-secondary \ + 'mkdir -p /srv/anonymous_github/secrets && + chmod 700 /srv/anonymous_github/secrets' +``` - - `mongo-primary.internal` - - `mongo-secondary.internal` +Copy the key over Tailscale: -5. Permit TCP port `27017` only between the required private-network hosts. - Do not expose MongoDB to the public internet. -6. Run the exact same MongoDB version on both servers. Find the current - production version with: +```bash +scp secrets/mongo-replica-keyfile \ + deploy@mongo-secondary:/srv/anonymous_github/secrets/mongo-replica-keyfile +``` - ```bash - docker compose exec mongodb mongod --version - ``` +On the secondary: - Set the matching image in both servers' `.env` files: +```bash +chmod 600 /srv/anonymous_github/secrets/mongo-replica-keyfile +``` - ```env - MONGO_IMAGE=mongo: - ``` +Verify that both servers have the exact same file: -## 1. Generate the shared member key +```bash +# Run on the primary. +sha256sum /home/ubuntu/git/anonymous_github/secrets/mongo-replica-keyfile -On the primary server: +# Run on the secondary. +sha256sum /srv/anonymous_github/secrets/mongo-replica-keyfile +``` + +The hashes must match. The keyfile is ignored by Git; never commit it. + +The relative setting below works on both servers even though their absolute +paths and Unix users differ: + +```env +MONGO_REPLICA_KEYFILE=./secrets/mongo-replica-keyfile +``` + +It is resolved from the repository/Compose project on each server. Each local +user only needs to own and read the key in their own checkout. + +## 6. Configure the primary `.env` + +On the primary, add the following values using the actual IPs: + +```env +MONGO_IMAGE=mongo: + +MONGO_PRIMARY_HOSTNAME=mongo-primary +MONGO_PRIMARY_ADDRESS=100.64.10.20 +MONGO_SECONDARY_HOSTNAME=mongo-secondary +MONGO_SECONDARY_ADDRESS=100.64.10.30 + +# This is the primary server's own Tailscale address. +MONGO_BIND_ADDRESS=100.64.10.20 +MONGO_REPLICA_KEYFILE=./secrets/mongo-replica-keyfile +``` + +Do not set `COMPOSE_FILE` yet. Do not switch `MONGODB_URI` to a replica-set +URI yet. Those two values are enabled only after replication is healthy. + +Keep the existing `DB_USERNAME` and `DB_PASSWORD` values unchanged. + +Validate the rendered primary configuration: + +```bash +docker compose \ + -f docker-compose.yml \ + -f docker-compose.replica-primary.yml \ + config -q +``` + +## 7. Configure and start the secondary + +Create `.env` on the secondary: + +```env +MONGO_IMAGE=mongo: + +MONGO_PRIMARY_HOSTNAME=mongo-primary +MONGO_PRIMARY_ADDRESS=100.64.10.20 +MONGO_SECONDARY_HOSTNAME=mongo-secondary +MONGO_SECONDARY_ADDRESS=100.64.10.30 + +# This is the secondary server's own Tailscale address. +MONGO_BIND_ADDRESS=100.64.10.30 +MONGO_REPLICA_KEYFILE=./secrets/mongo-replica-keyfile +``` + +Validate and start it: + +```bash +cd /srv/anonymous_github + +docker compose -f docker-compose.replica-secondary.yml config -q +./scripts/mongodb-replica.sh secondary-up mongo-secondary:27017 +``` + +The command verifies that `mongo-secondary` resolves to the secondary +container before continuing. + +Check both mappings inside the secondary container: + +```bash +docker compose -f docker-compose.replica-secondary.yml \ + exec -T mongodb-secondary getent hosts \ + mongo-primary mongo-secondary +``` + +Expected: + +- `mongo-primary` resolves to the primary Tailscale IP (`100.64.10.20` in the + example). +- `mongo-secondary` resolves to a Docker/container address, normally + `172.x.x.x`. + +If either mapping is wrong, stop here and fix `.env`. + +## 8. Verify the secondary network + +On the primary, verify that the Tailscale peer is reachable: + +```bash +tailscale ping mongo-secondary +``` + +On the secondary, inspect the published port: + +```bash +docker compose -f docker-compose.replica-secondary.yml ps +``` + +The port display must use the secondary's Tailscale IP, for example +`100.64.10.30:27017->27017/tcp`. It must not use `0.0.0.0`. + +## 9. Convert the production MongoDB to a replica primary + +On the primary, make sure application writers are stopped: + +```bash +cd /home/ubuntu/git/anonymous_github + +docker compose -f docker-compose.yml stop \ + anonymous_github streamer mongodb-backup +``` + +Start MongoDB through the replica overlay and initialize `rs0`: + +```bash +./scripts/mongodb-replica.sh primary-up mongo-primary:27017 +``` + +This command: + +1. Recreates only the primary MongoDB container with `--replSet rs0`. +2. Enables member authentication with the shared keyfile. +3. Waits for MongoDB to answer. +4. Verifies `mongo-primary` from inside the container. +5. Runs `rs.initiate()` only if the replica set is not already initialized. +6. Waits until `rs0` has elected a writable primary. + +Verify both mappings inside the primary container: + +```bash +docker compose \ + -f docker-compose.yml \ + -f docker-compose.replica-primary.yml \ + exec -T mongodb getent hosts \ + mongo-primary mongo-secondary +``` + +Expected: + +- `mongo-primary` resolves to a Docker/container address, normally + `172.x.x.x`. +- `mongo-secondary` resolves to the secondary Tailscale IP + (`100.64.10.30` in the example). + +Confirm the primary state: + +```bash +./scripts/mongodb-replica.sh status +``` + +The only member should initially report `PRIMARY`. + +## 10. Add the remote secondary + +Use the default one-hour delay: + +```bash +./scripts/mongodb-replica.sh add-secondary \ + mongo-secondary:27017 +``` + +For a current, non-delayed recovery copy, explicitly pass `0`: + +```bash +./scripts/mongodb-replica.sh add-secondary \ + mongo-secondary:27017 0 +``` + +The command first checks that the primary container can resolve and reach +`mongo-secondary`. It then adds the member as: + +```text +priority: 0 +votes: 0 +hidden: true +secondaryDelaySecs: 3600 +``` + +Watch the initial sync: + +```bash +./scripts/mongodb-replica.sh status +``` + +`STARTUP2` or `RECOVERING` can appear during initial sync. Continue only when: + +```text +mongo-primary:27017 PRIMARY +mongo-secondary:27017 SECONDARY +``` + +Large databases can take a long time to initial-sync. + +## 11. Switch the application to the replica-set URI + +Only after both members are healthy, add these values to the primary `.env`: + +```env +COMPOSE_FILE=docker-compose.yml:docker-compose.replica-primary.yml + +MONGODB_URI="mongodb://:@mongo-primary:27017/production?authSource=admin&replicaSet=rs0&retryWrites=true&w=majority" +``` + +MongoDB usernames and passwords must be URL-encoded if they contain characters +such as `@`, `:`, `/`, `?`, `#`, or `%`. + +`COMPOSE_FILE` makes future ordinary `docker compose` commands include the +primary replica overlay. Without it, a later `docker compose up -d` could +recreate MongoDB without `--replSet`. + +Restart and verify the application: + +```bash +docker compose up -d redis streamer anonymous_github +docker compose ps +docker compose logs --tail=100 anonymous_github +``` + +Verify replica status again: + +```bash +./scripts/mongodb-replica.sh status +``` + +## Routine operations + +Check status from the primary: + +```bash +./scripts/mongodb-replica.sh status +``` + +Check logs: + +```bash +# Primary +docker compose logs --tail=100 mongodb + +# Secondary +docker compose -f docker-compose.replica-secondary.yml \ + logs --tail=100 mongodb-secondary +``` + +If an accidental delete occurs and the secondary is delayed, stop the +secondary before the one-hour window expires: + +```bash +docker compose -f docker-compose.replica-secondary.yml \ + stop mongodb-secondary +``` + +Take a logical dump or snapshot of the stopped recovery copy before attempting +repair. + +Do not turn the secondary into a second voting member. A two-voter set requires +both servers for a majority and can make production unwritable during a +network outage. Automatic failover requires three voting, data-bearing +members. + +## Complete rollback to standalone MongoDB + +Use this procedure if conversion fails or you decide not to use replication. +It keeps the existing primary data volume. + +On the primary, remove or comment out: + +```env +COMPOSE_FILE=docker-compose.yml:docker-compose.replica-primary.yml +MONGODB_URI="mongodb://...replicaSet=rs0..." +MONGO_REPLICA_KEYFILE=./secrets/mongo-replica-keyfile +``` + +Set: + +```env +MONGODB_URI= +DB_HOSTNAME=mongodb +MONGO_BIND_ADDRESS=127.0.0.1 +``` + +Keep the pinned `MONGO_IMAGE` and the real `DB_USERNAME`/`DB_PASSWORD`. + +Recreate MongoDB from only the base Compose file: + +```bash +docker compose -f docker-compose.yml stop \ + anonymous_github streamer mongodb + +docker compose -f docker-compose.yml \ + up -d --force-recreate mongodb +``` + +Verify the running command does not contain `--replSet`: + +```bash +docker compose -f docker-compose.yml \ + exec -T mongodb sh -c \ + 'tr "\000" " " < /proc/1/cmdline; echo' +``` + +Verify database access: + +```bash +docker compose -f docker-compose.yml exec -T mongodb sh -eu -c \ + 'mongosh --quiet \ + --username="$MONGO_INITDB_ROOT_USERNAME" \ + --password="$MONGO_INITDB_ROOT_PASSWORD" \ + --authenticationDatabase=admin \ + --eval "db.adminCommand({ ping: 1 })"' +``` + +Restart the standalone application: + +```bash +docker compose -f docker-compose.yml \ + up -d redis streamer anonymous_github +``` + +Stop the remote secondary: + +```bash +docker compose -f docker-compose.replica-secondary.yml \ + stop mongodb-secondary +``` + +Do not delete either MongoDB volume, the `local` database, or the shared +keyfile during rollback. + +## Troubleshooting + +### `Replica keyfile not found` + +On the primary: ```bash ./scripts/mongodb-replica.sh generate-key ``` -This creates `secrets/mongo-replica-keyfile` without overwriting an existing -key. Copy that exact file securely to the same repository-relative path on the -secondary server: +Copy that same file to the `secrets` directory inside the secondary server's +own repository checkout, for example: + +```text +/srv/anonymous_github/secrets/mongo-replica-keyfile +``` + +Do not generate two independent keys. Compare both files with `sha256sum`. + +### `MongoServerError: not running with --replSet` + +The primary is still using the base Compose configuration. Run: ```bash -scp secrets/mongo-replica-keyfile \ - secondary-server:/path/to/anonymous_github/secrets/mongo-replica-keyfile +./scripts/mongodb-replica.sh primary-up mongo-primary:27017 ``` -Keep this key outside source control and backups that are accessible to -untrusted users. Every replica-set member must share the same key. +Do not run `add-secondary` before `primary-up` succeeds. -## 2. Configure production - -Add these values to the primary server's `.env`: - -```env -# Use the exact version already running in production. -MONGO_IMAGE=mongo: - -# The primary server's private/VPN interface. -MONGO_BIND_ADDRESS= -MONGO_REPLICA_KEYFILE=./secrets/mongo-replica-keyfile - -# Make the replica overlay the default for future Compose commands. -COMPOSE_FILE=docker-compose.yml:docker-compose.replica-primary.yml - -# URL-encode special characters in the username and password. -MONGODB_URI=mongodb://:@mongo-primary.internal:27017/production?authSource=admin&replicaSet=rs0&retryWrites=true&w=majority -``` - -Stop application writers, leaving MongoDB available: +Inspect the primary process: ```bash -docker compose stop anonymous_github streamer +docker compose \ + -f docker-compose.yml \ + -f docker-compose.replica-primary.yml \ + exec -T mongodb sh -c \ + 'tr "\000" " " < /proc/1/cmdline; echo' ``` -Start MongoDB with replica-set support and initialize the existing database as -the primary: +It must contain `--replSet rs0`. + +### `No host described ... maps to this node` + +`mongo-primary` does not resolve back to the primary MongoDB container. +Confirm that the current Compose overlays include `hostname`, `networks` +aliases, and `extra_hosts`, then run: ```bash -./scripts/mongodb-replica.sh primary-up \ - mongo-primary.internal:27017 +docker compose \ + -f docker-compose.yml \ + -f docker-compose.replica-primary.yml \ + config + +docker compose \ + -f docker-compose.yml \ + -f docker-compose.replica-primary.yml \ + exec -T mongodb getent hosts mongo-primary ``` -`primary-up` is idempotent. If `rs0` is already initialized, it reports that -state instead of replacing the replica-set configuration. +The result must be the primary container address, normally `172.x.x.x`, not +the host's `100.x.x.x` Tailscale address. -## 3. Start the remote secondary - -On the secondary server, create a minimal `.env`: - -```env -MONGO_IMAGE=mongo: -MONGO_BIND_ADDRESS= -MONGO_REPLICA_KEYFILE=./secrets/mongo-replica-keyfile -``` - -Start its empty MongoDB data volume: +After correcting `.env`, recreate the primary container and retry: ```bash -./scripts/mongodb-replica.sh secondary-up +docker compose \ + -f docker-compose.yml \ + -f docker-compose.replica-primary.yml \ + up -d --force-recreate mongodb + +./scripts/mongodb-replica.sh primary-up mongo-primary:27017 ``` -Do not copy the primary's live WiredTiger volume. MongoDB performs the initial -sync after the member is added. +If a bad replica configuration was already accepted rather than rejected, use +the standalone rollback procedure before attempting any forced +`rs.reconfig()`. Forced replica reconfiguration can cause rollback or data +loss. -## 4. Add and verify the secondary +### The primary cannot reach the secondary -Back on the primary server, add the remote member with the default one-hour -delay: +Check the host network: ```bash -./scripts/mongodb-replica.sh add-secondary \ - mongo-secondary.internal:27017 +tailscale ping mongo-secondary +tailscale status ``` -To keep the remote copy current instead, explicitly set a zero-second delay: +Check the mapping from the primary container: ```bash -./scripts/mongodb-replica.sh add-secondary \ - mongo-secondary.internal:27017 0 +docker compose \ + -f docker-compose.yml \ + -f docker-compose.replica-primary.yml \ + exec -T mongodb getent hosts mongo-secondary ``` -Check initial-sync and replication state: +It must return the secondary's Tailscale IP. + +Check the secondary: ```bash -./scripts/mongodb-replica.sh status +docker compose -f docker-compose.replica-secondary.yml ps +docker compose -f docker-compose.replica-secondary.yml \ + logs --tail=100 mongodb-secondary ``` -Wait until the remote member reports `SECONDARY`, then restart the application: +Confirm the Tailscale ACL and host firewall allow TCP `27017` from the primary +to the secondary. -```bash -docker compose up -d anonymous_github streamer redis -``` +### The application cannot connect after conversion -## Operations +Check: -Use the primary overlay for every future operation on the production stack. -Keeping `COMPOSE_FILE` in `.env` makes ordinary commands such as -`docker compose up -d` and `docker compose logs mongodb` use it automatically. +1. `MONGODB_URI` uses `mongo-primary:27017`. +2. It includes `replicaSet=rs0` and `authSource=admin`. +3. Credentials are URL-encoded. +4. `COMPOSE_FILE` includes the primary overlay. +5. The application container resolves the alias: -Check replica status: + ```bash + docker compose exec -T anonymous_github getent hosts mongo-primary + ``` -```bash -./scripts/mongodb-replica.sh status -``` +6. `./scripts/mongodb-replica.sh status` reports a `PRIMARY`. -If a destructive production operation occurs and the secondary is delayed, -stop the secondary container immediately before the delay window elapses: +## References -```bash -docker compose -f docker-compose.replica-secondary.yml stop mongodb-secondary -``` - -Then take a copy or logical dump of the delayed data before attempting -recovery. - -For automatic failover, use three voting data-bearing members instead of -turning this two-server recovery topology into a two-voter replica set. Two -voters require both servers to acknowledge a majority and can make production -unwritable during a network outage. - -## Troubleshooting - -- All members must use the same replica-set name (`rs0`), MongoDB version, and - shared keyfile. -- The hostnames stored in `rs.conf()` must resolve from every member. -- The application container must also resolve `mongo-primary.internal`. Use - private DNS or a Compose `extra_hosts` entry if host DNS is not propagated - into Docker. -- Check container logs with: - - ```bash - docker compose logs mongodb - docker compose -f docker-compose.replica-secondary.yml logs mongodb-secondary - ``` - -- Re-run `primary-up`, `add-secondary`, or `status` safely; the management - operations do not replace existing replica-set members. - -## MongoDB references - -- [Convert a standalone server to a replica set](https://www.mongodb.com/docs/manual/tutorial/convert-standalone-to-replica-set/) -- [Deploy a replica set with member authentication](https://www.mongodb.com/docs/v8.0/tutorial/deploy-replica-set-with-keyfile-access-control/) -- [Configure a delayed member](https://www.mongodb.com/docs/manual/tutorial/configure-a-delayed-replica-set-member/) -- [Hidden replica-set members](https://www.mongodb.com/docs/manual/core/replica-set-hidden-member/) +- [MongoDB: convert a standalone server to a replica set](https://www.mongodb.com/docs/manual/tutorial/convert-standalone-to-replica-set/) +- [MongoDB: deploy a replica set with keyfile authentication](https://www.mongodb.com/docs/v8.0/tutorial/deploy-replica-set-with-keyfile-access-control/) +- [MongoDB: use resolvable hostnames for replica members](https://www.mongodb.com/docs/manual/tutorial/change-hostnames-in-a-replica-set/) +- [MongoDB: configure a delayed member](https://www.mongodb.com/docs/manual/tutorial/configure-a-delayed-replica-set-member/) +- [MongoDB: hidden replica-set members](https://www.mongodb.com/docs/manual/core/replica-set-hidden-member/) +- [Tailscale: MagicDNS](https://tailscale.com/docs/features/magicdns) diff --git a/scripts/mongodb-replica.sh b/scripts/mongodb-replica.sh index 1e2bd7a..c3fe660 100755 --- a/scripts/mongodb-replica.sh +++ b/scripts/mongodb-replica.sh @@ -25,7 +25,7 @@ usage() { Usage: ./scripts/mongodb-replica.sh generate-key ./scripts/mongodb-replica.sh primary-up - ./scripts/mongodb-replica.sh secondary-up + ./scripts/mongodb-replica.sh secondary-up ./scripts/mongodb-replica.sh add-secondary [delay-seconds] ./scripts/mongodb-replica.sh status @@ -96,6 +96,66 @@ wait_for_primary_container() { exit 1 } +wait_for_replica_primary() { + local attempt + for attempt in $(seq 1 60); do + if mongo_eval \ + "if (!db.hello().isWritablePrimary) throw new Error('not primary yet');" \ + >/dev/null 2>&1; then + return + fi + sleep 2 + done + echo "Replica set rs0 did not elect a writable primary within 120 seconds" >&2 + echo "Check ./scripts/mongodb-replica.sh status and the MongoDB container logs." >&2 + exit 1 +} + +wait_for_secondary_container() { + local attempt + for attempt in $(seq 1 60); do + if secondary_compose exec -T mongodb-secondary mongosh --quiet \ + --eval "db.adminCommand({ ping: 1 }).ok" >/dev/null 2>&1; then + return + fi + sleep 2 + done + echo "Secondary MongoDB did not become ready within 120 seconds" >&2 + exit 1 +} + +check_primary_endpoint() { + local endpoint="$1" + primary_compose exec -T -e "MONGO_RS_ENDPOINT=$endpoint" mongodb sh -eu -c ' + host="${MONGO_RS_ENDPOINT%:*}" + port="${MONGO_RS_ENDPOINT##*:}" + echo "Checking $host resolves inside the primary MongoDB container:" + getent hosts "$host" + mongosh --quiet --host "$host" --port "$port" \ + --eval "db.adminCommand({ ping: 1 }).ok" >/dev/null + ' || { + echo "The endpoint $endpoint does not resolve to a reachable MongoDB instance from the primary container." >&2 + echo "Check MONGO_PRIMARY_HOSTNAME, MONGO_PRIMARY_ADDRESS, MONGO_SECONDARY_HOSTNAME, and MONGO_SECONDARY_ADDRESS." >&2 + exit 1 + } +} + +check_secondary_endpoint() { + local endpoint="$1" + secondary_compose exec -T -e "MONGO_RS_ENDPOINT=$endpoint" mongodb-secondary sh -eu -c ' + host="${MONGO_RS_ENDPOINT%:*}" + port="${MONGO_RS_ENDPOINT##*:}" + echo "Checking $host resolves inside the secondary MongoDB container:" + getent hosts "$host" + mongosh --quiet --host "$host" --port "$port" \ + --eval "db.adminCommand({ ping: 1 }).ok" >/dev/null + ' || { + echo "The endpoint $endpoint does not resolve to the secondary MongoDB container." >&2 + echo "Check MONGO_SECONDARY_HOSTNAME and the secondary Compose configuration." >&2 + exit 1 + } +} + command="${1:-}" case "$command" in generate-key) @@ -117,6 +177,7 @@ case "$command" in require_keyfile primary_compose up -d mongodb wait_for_primary_container + check_primary_endpoint "$primary_host" mongo_eval " try { const status = rs.status(); @@ -129,11 +190,16 @@ case "$command" in })); } " + wait_for_replica_primary ;; secondary-up) + secondary_host="${2:-}" + validate_host "$secondary_host" require_keyfile secondary_compose up -d mongodb-secondary + wait_for_secondary_container + check_secondary_endpoint "$secondary_host" ;; add-secondary) @@ -141,6 +207,7 @@ case "$command" in delay="${3:-3600}" validate_host "$secondary_host" validate_delay "$delay" + check_primary_endpoint "$secondary_host" mongo_eval " const host = '$secondary_host'; const existing = rs.conf().members.find((member) => member.host === host);