Add a reliable MongoDB replica setup runbook (#766)

* docs: clarify MongoDB Tailscale bind addresses * fix: make MongoDB replica host mapping deterministic * docs: add complete MongoDB replica runbook * docs: support different replica server paths
2026-08-14 07:50:34 +02:00 · 2026-07-30 03:10:13 +02:00
parent debd83c079
commit e54b78c7e6
4 changed files with 760 additions and 143 deletions
@@ -1,5 +1,6 @@
 services:
  mongodb:
+    hostname: ${MONGO_PRIMARY_HOSTNAME:-mongo-primary}
    entrypoint:
      - /bin/bash
      - /opt/anonymous-github/mongodb-replica-entrypoint.sh
@@ -12,6 +13,12 @@ services:
      - --bind_ip_all
    environment:
      MONGO_REPLICA_KEYFILE_PATH: /run/secrets/mongo-replica-keyfile
+    extra_hosts:
+      - "${MONGO_SECONDARY_HOSTNAME:-mongo-secondary}:${MONGO_SECONDARY_ADDRESS:?Set MONGO_SECONDARY_ADDRESS to the secondary server Tailscale IP}"
+    networks:
+      default:
+        aliases:
+          - ${MONGO_PRIMARY_HOSTNAME:-mongo-primary}
    volumes:
      - ${MONGO_REPLICA_KEYFILE:-./secrets/mongo-replica-keyfile}:/run/secrets/mongo-replica-keyfile:ro
      - ./scripts/mongodb-replica-entrypoint.sh:/opt/anonymous-github/mongodb-replica-entrypoint.sh:ro
@@ -3,6 +3,7 @@ name: anonymous-github-mongo-secondary
 services:
  mongodb-secondary:
    image: ${MONGO_IMAGE:-mongo:latest}
+    hostname: ${MONGO_SECONDARY_HOSTNAME:-mongo-secondary}
    restart: unless-stopped
    entrypoint:
      - /bin/bash
@@ -16,6 +17,12 @@ services:
      - --bind_ip_all
    environment:
      MONGO_REPLICA_KEYFILE_PATH: /run/secrets/mongo-replica-keyfile
+    extra_hosts:
+      - "${MONGO_PRIMARY_HOSTNAME:-mongo-primary}:${MONGO_PRIMARY_ADDRESS:?Set MONGO_PRIMARY_ADDRESS to the primary server Tailscale IP}"
+    networks:
+      default:
+        aliases:
+          - ${MONGO_SECONDARY_HOSTNAME:-mongo-secondary}
    volumes:
      - mongodb_secondary_data:/data/db
      - ${MONGO_REPLICA_KEYFILE:-./secrets/mongo-replica-keyfile}:/run/secrets/mongo-replica-keyfile:ro
@@ -1,200 +1,736 @@
-# Optional remote MongoDB replica
+# Remote MongoDB recovery replica

-This deployment option converts the existing standalone MongoDB container into
-a replica set named `rs0` and adds a remote, hidden, non-voting secondary.
-Normal single-node deployments continue to use `docker-compose.yml` alone.
+This guide converts the existing production MongoDB container into a replica
+set named `rs0` and adds a MongoDB container on a second server. The servers
+communicate over Tailscale; MongoDB is not exposed on a public interface.

-The remote member is designed as a recovery and backup source:
+Read the complete guide before changing production.

- It cannot become primary.
- Its availability does not affect production majority writes.
- It is hidden from application traffic.
- It is delayed by one hour by default, providing a short recovery window for
-  accidental deletes or a bad migration.
+## What this creates

-Replication is not a backup. Destructive changes eventually reach every
-member. Continue making encrypted, off-host `mongodump` archives from the
+| Member | Role | Votes | Priority | Hidden | Default delay |
+| --- | --- | ---: | ---: | --- | ---: |
+| `mongo-primary` | Production primary | 1 | 1 | No | 0 |
+| `mongo-secondary` | Recovery copy | 0 | 0 | Yes | 1 hour |
+
+The secondary cannot become primary. Losing the secondary or its network
+connection therefore does not stop production writes. This two-server layout
+is for recovery and for taking backups from a remote copy; it is not automatic
+failover.
+
+Replication is also not a backup. Deletes and corrupt data eventually
+replicate. Continue making versioned, encrypted, off-server `mongodump`
+archives.
+
+## How names and addresses are used
+
+MongoDB replica-set members are advertised with stable hostnames:
+
+- `mongo-primary:27017`
+- `mongo-secondary:27017`
+
+Do not advertise the Tailscale IPs directly in `rs.initiate()` or `rs.add()`.
+MongoDB expects stable hostnames, and recent MongoDB versions reject
+IP-only replica-set configurations.
+
+The Compose overlays make the two hostnames deterministic:
+
+- Inside the primary Docker network, `mongo-primary` resolves directly to the
+  primary MongoDB container.
+- Inside the secondary Docker network, `mongo-secondary` resolves directly to
+  the secondary MongoDB container.
+- Cross-server names are mapped to the other server's Tailscale IP with
+  Compose `extra_hosts`.
+
+This mapping is important. If `mongo-primary` does not map back to the primary
+container, `rs.initiate()` fails with:
+
+```text
+No host described in new configuration ... maps to this node
+```
+
+## Example values
+
+The guide uses these examples:
+
+| Setting | Example |
+| --- | --- |
+| Primary SSH user | `ubuntu` |
+| Primary repository path | `/home/ubuntu/git/anonymous_github` |
+| Secondary SSH user | `deploy` |
+| Secondary repository path | `/srv/anonymous_github` |
+| Primary Tailscale machine name | `mongo-primary` |
+| Primary Tailscale IPv4 | `100.64.10.20` |
+| Secondary Tailscale machine name | `mongo-secondary` |
+| Secondary Tailscale IPv4 | `100.64.10.30` |
+
+The users and absolute repository paths may be different on the two servers.
+Run each command from that server's own checkout. Replace every example user,
+IP, and path with the value from your server. Never enter the literal value
+`100.x.x.x`.
+
+`MONGO_BIND_ADDRESS` always contains the Tailscale IP of the server whose
+`.env` file you are editing:
+
+```text
+primary .env   -> MONGO_BIND_ADDRESS=<primary Tailscale IP>
+secondary .env -> MONGO_BIND_ADDRESS=<secondary Tailscale IP>
+```
+
+## Prerequisites
+
+Before starting, confirm all of the following:
+
+- The production standalone MongoDB is healthy.
+- You have a recent, verified `mongodump` backup.
+- The repository is installed on both servers; the absolute paths and owners
+  may be different.
+- Both servers run the same repository revision.
+- Both servers run the exact same MongoDB version.
+- Tailscale is connected on both servers.
+- TCP port `27017` is not publicly exposed.
+- You have a maintenance window for restarting MongoDB and the application.
+
+Do not delete or copy a live WiredTiger data directory. The existing primary
+volume stays in place, and MongoDB initial-syncs the secondary after it is
+added.
+
+## 1. Record the current production state
+
+Run on the primary:
+
+```bash
+cd /home/ubuntu/git/anonymous_github
+
+docker compose -f docker-compose.yml ps
+docker compose -f docker-compose.yml exec -T mongodb mongod --version
+git rev-parse HEAD
+```
+
+Pin the current MongoDB version in `.env`. Do not leave production on
+`mongo:latest` while changing the topology:
+
+```env
+MONGO_IMAGE=mongo:<exact-version-running-now>
+```
+
+Save a protected copy of the current configuration:
+
+```bash
+cp .env .env.before-replica
+chmod 600 .env.before-replica
+```
+
+This file contains secrets. Do not commit or copy it to an untrusted system.
+
+## 2. Back up the standalone database
+
+Stop application writers for a consistent pre-conversion backup:
+
+```bash
+docker compose -f docker-compose.yml stop \
+  anonymous_github streamer mongodb-backup
+```
+
+Create an archive:
+
+```bash
+mkdir -p db_backups/manual
+BACKUP="db_backups/manual/production-before-replica-$(date -u +%Y%m%dT%H%M%SZ).archive.gz"
+
+docker compose -f docker-compose.yml exec -T mongodb sh -eu -c \
+  'mongodump \
+    --username="$MONGO_INITDB_ROOT_USERNAME" \
+    --password="$MONGO_INITDB_ROOT_PASSWORD" \
+    --authenticationDatabase=admin \
+    --db=production \
+    --archive \
+    --gzip' > "$BACKUP"
+
+test -s "$BACKUP"
+chmod 600 "$BACKUP"
+ls -lh "$BACKUP"
+```
+
+Keep the application stopped until the replica primary is working. If you
+restart it temporarily, stop it again before step 9.
+
+## 3. Connect and name both Tailscale servers
+
+On the primary:
+
+```bash
+sudo tailscale set --hostname=mongo-primary
+tailscale ip -4
+tailscale status
+```
+
+On the secondary:
+
+```bash
+sudo tailscale set --hostname=mongo-secondary
+tailscale ip -4
+tailscale status
+```
+
+Record both IPv4 addresses. For the example topology:
+
+```text
+mongo-primary   100.64.10.20
+mongo-secondary 100.64.10.30
+```
+
+Verify connectivity in both directions:
+
+```bash
+# Run on the primary.
+tailscale ping mongo-secondary
+
+# Run on the secondary.
+tailscale ping mongo-primary
+```
+
+If the short names do not work, fix Tailscale/MagicDNS first. You may test with
+the Tailscale IPs, but the MongoDB member names used later remain
+`mongo-primary` and `mongo-secondary`.
+
+Restrict your Tailscale policy so only the required servers can reach TCP
+`27017`. Do not publish MongoDB on `0.0.0.0` or a public cloud interface.
+
+## 4. Install the same repository revision on the secondary
+
+On the primary:
+
+```bash
+cd /home/ubuntu/git/anonymous_github
+git rev-parse HEAD
+```
+
+On the secondary, use its own repository path:
+
+```bash
+cd /srv/anonymous_github
+git rev-parse HEAD
+```
+
+The commit IDs must match. The following files must exist:
+
+```bash
+test -f docker-compose.replica-primary.yml
+test -f docker-compose.replica-secondary.yml
+test -x scripts/mongodb-replica.sh
+test -x scripts/mongodb-replica-entrypoint.sh
+```
+
+Do not start a second copy of the Anonymous GitHub application on the
+secondary. Only the `mongodb-secondary` service is used there.
+
+## 5. Generate and copy the shared MongoDB keyfile
+
+Generate the key once on the primary:
+
+```bash
+cd /home/ubuntu/git/anonymous_github
+./scripts/mongodb-replica.sh generate-key
+```
+
+If the file already exists, reuse it. Do not generate a different key on the
 secondary.

-## Requirements
+Create the destination directory:

-1. Take and verify a `mongodump` backup before converting production.
-2. Install this repository on both servers.
-3. Connect the servers through a trusted private network such as WireGuard,
-   Tailscale, a private VLAN, or a cloud private network.
-4. Create stable DNS names that resolve from both servers and from the
-   application container, for example:
+```bash
+ssh deploy@mongo-secondary \
+  'mkdir -p /srv/anonymous_github/secrets &&
+   chmod 700 /srv/anonymous_github/secrets'
+```

-   - `mongo-primary.internal`
-   - `mongo-secondary.internal`
+Copy the key over Tailscale:

-5. Permit TCP port `27017` only between the required private-network hosts.
-   Do not expose MongoDB to the public internet.
-6. Run the exact same MongoDB version on both servers. Find the current
-   production version with:
+```bash
+scp secrets/mongo-replica-keyfile \
+  deploy@mongo-secondary:/srv/anonymous_github/secrets/mongo-replica-keyfile
+```

-   ```bash
-   docker compose exec mongodb mongod --version
-   ```
+On the secondary:

-   Set the matching image in both servers' `.env` files:
+```bash
+chmod 600 /srv/anonymous_github/secrets/mongo-replica-keyfile
+```

-   ```env
-   MONGO_IMAGE=mongo:<exact-version>
-   ```
+Verify that both servers have the exact same file:

-## 1. Generate the shared member key
+```bash
+# Run on the primary.
+sha256sum /home/ubuntu/git/anonymous_github/secrets/mongo-replica-keyfile

-On the primary server:
+# Run on the secondary.
+sha256sum /srv/anonymous_github/secrets/mongo-replica-keyfile
+```
+
+The hashes must match. The keyfile is ignored by Git; never commit it.
+
+The relative setting below works on both servers even though their absolute
+paths and Unix users differ:
+
+```env
+MONGO_REPLICA_KEYFILE=./secrets/mongo-replica-keyfile
+```
+
+It is resolved from the repository/Compose project on each server. Each local
+user only needs to own and read the key in their own checkout.
+
+## 6. Configure the primary `.env`
+
+On the primary, add the following values using the actual IPs:
+
+```env
+MONGO_IMAGE=mongo:<exact-version-running-now>
+
+MONGO_PRIMARY_HOSTNAME=mongo-primary
+MONGO_PRIMARY_ADDRESS=100.64.10.20
+MONGO_SECONDARY_HOSTNAME=mongo-secondary
+MONGO_SECONDARY_ADDRESS=100.64.10.30
+
+# This is the primary server's own Tailscale address.
+MONGO_BIND_ADDRESS=100.64.10.20
+MONGO_REPLICA_KEYFILE=./secrets/mongo-replica-keyfile
+```
+
+Do not set `COMPOSE_FILE` yet. Do not switch `MONGODB_URI` to a replica-set
+URI yet. Those two values are enabled only after replication is healthy.
+
+Keep the existing `DB_USERNAME` and `DB_PASSWORD` values unchanged.
+
+Validate the rendered primary configuration:
+
+```bash
+docker compose \
+  -f docker-compose.yml \
+  -f docker-compose.replica-primary.yml \
+  config -q
+```
+
+## 7. Configure and start the secondary
+
+Create `.env` on the secondary:
+
+```env
+MONGO_IMAGE=mongo:<same-exact-version-as-primary>
+
+MONGO_PRIMARY_HOSTNAME=mongo-primary
+MONGO_PRIMARY_ADDRESS=100.64.10.20
+MONGO_SECONDARY_HOSTNAME=mongo-secondary
+MONGO_SECONDARY_ADDRESS=100.64.10.30
+
+# This is the secondary server's own Tailscale address.
+MONGO_BIND_ADDRESS=100.64.10.30
+MONGO_REPLICA_KEYFILE=./secrets/mongo-replica-keyfile
+```
+
+Validate and start it:
+
+```bash
+cd /srv/anonymous_github
+
+docker compose -f docker-compose.replica-secondary.yml config -q
+./scripts/mongodb-replica.sh secondary-up mongo-secondary:27017
+```
+
+The command verifies that `mongo-secondary` resolves to the secondary
+container before continuing.
+
+Check both mappings inside the secondary container:
+
+```bash
+docker compose -f docker-compose.replica-secondary.yml \
+  exec -T mongodb-secondary getent hosts \
+  mongo-primary mongo-secondary
+```
+
+Expected:
+
+- `mongo-primary` resolves to the primary Tailscale IP (`100.64.10.20` in the
+  example).
+- `mongo-secondary` resolves to a Docker/container address, normally
+  `172.x.x.x`.
+
+If either mapping is wrong, stop here and fix `.env`.
+
+## 8. Verify the secondary network
+
+On the primary, verify that the Tailscale peer is reachable:
+
+```bash
+tailscale ping mongo-secondary
+```
+
+On the secondary, inspect the published port:
+
+```bash
+docker compose -f docker-compose.replica-secondary.yml ps
+```
+
+The port display must use the secondary's Tailscale IP, for example
+`100.64.10.30:27017->27017/tcp`. It must not use `0.0.0.0`.
+
+## 9. Convert the production MongoDB to a replica primary
+
+On the primary, make sure application writers are stopped:
+
+```bash
+cd /home/ubuntu/git/anonymous_github
+
+docker compose -f docker-compose.yml stop \
+  anonymous_github streamer mongodb-backup
+```
+
+Start MongoDB through the replica overlay and initialize `rs0`:
+
+```bash
+./scripts/mongodb-replica.sh primary-up mongo-primary:27017
+```
+
+This command:
+
+1. Recreates only the primary MongoDB container with `--replSet rs0`.
+2. Enables member authentication with the shared keyfile.
+3. Waits for MongoDB to answer.
+4. Verifies `mongo-primary` from inside the container.
+5. Runs `rs.initiate()` only if the replica set is not already initialized.
+6. Waits until `rs0` has elected a writable primary.
+
+Verify both mappings inside the primary container:
+
+```bash
+docker compose \
+  -f docker-compose.yml \
+  -f docker-compose.replica-primary.yml \
+  exec -T mongodb getent hosts \
+  mongo-primary mongo-secondary
+```
+
+Expected:
+
+- `mongo-primary` resolves to a Docker/container address, normally
+  `172.x.x.x`.
+- `mongo-secondary` resolves to the secondary Tailscale IP
+  (`100.64.10.30` in the example).
+
+Confirm the primary state:
+
+```bash
+./scripts/mongodb-replica.sh status
+```
+
+The only member should initially report `PRIMARY`.
+
+## 10. Add the remote secondary
+
+Use the default one-hour delay:
+
+```bash
+./scripts/mongodb-replica.sh add-secondary \
+  mongo-secondary:27017
+```
+
+For a current, non-delayed recovery copy, explicitly pass `0`:
+
+```bash
+./scripts/mongodb-replica.sh add-secondary \
+  mongo-secondary:27017 0
+```
+
+The command first checks that the primary container can resolve and reach
+`mongo-secondary`. It then adds the member as:
+
+```text
+priority: 0
+votes: 0
+hidden: true
+secondaryDelaySecs: 3600
+```
+
+Watch the initial sync:
+
+```bash
+./scripts/mongodb-replica.sh status
+```
+
+`STARTUP2` or `RECOVERING` can appear during initial sync. Continue only when:
+
+```text
+mongo-primary:27017   PRIMARY
+mongo-secondary:27017 SECONDARY
+```
+
+Large databases can take a long time to initial-sync.
+
+## 11. Switch the application to the replica-set URI
+
+Only after both members are healthy, add these values to the primary `.env`:
+
+```env
+COMPOSE_FILE=docker-compose.yml:docker-compose.replica-primary.yml
+
+MONGODB_URI="mongodb://<url-encoded-user>:<url-encoded-password>@mongo-primary:27017/production?authSource=admin&replicaSet=rs0&retryWrites=true&w=majority"
+```
+
+MongoDB usernames and passwords must be URL-encoded if they contain characters
+such as `@`, `:`, `/`, `?`, `#`, or `%`.
+
+`COMPOSE_FILE` makes future ordinary `docker compose` commands include the
+primary replica overlay. Without it, a later `docker compose up -d` could
+recreate MongoDB without `--replSet`.
+
+Restart and verify the application:
+
+```bash
+docker compose up -d redis streamer anonymous_github
+docker compose ps
+docker compose logs --tail=100 anonymous_github
+```
+
+Verify replica status again:
+
+```bash
+./scripts/mongodb-replica.sh status
+```
+
+## Routine operations
+
+Check status from the primary:
+
+```bash
+./scripts/mongodb-replica.sh status
+```
+
+Check logs:
+
+```bash
+# Primary
+docker compose logs --tail=100 mongodb
+
+# Secondary
+docker compose -f docker-compose.replica-secondary.yml \
+  logs --tail=100 mongodb-secondary
+```
+
+If an accidental delete occurs and the secondary is delayed, stop the
+secondary before the one-hour window expires:
+
+```bash
+docker compose -f docker-compose.replica-secondary.yml \
+  stop mongodb-secondary
+```
+
+Take a logical dump or snapshot of the stopped recovery copy before attempting
+repair.
+
+Do not turn the secondary into a second voting member. A two-voter set requires
+both servers for a majority and can make production unwritable during a
+network outage. Automatic failover requires three voting, data-bearing
+members.
+
+## Complete rollback to standalone MongoDB
+
+Use this procedure if conversion fails or you decide not to use replication.
+It keeps the existing primary data volume.
+
+On the primary, remove or comment out:
+
+```env
+COMPOSE_FILE=docker-compose.yml:docker-compose.replica-primary.yml
+MONGODB_URI="mongodb://...replicaSet=rs0..."
+MONGO_REPLICA_KEYFILE=./secrets/mongo-replica-keyfile
+```
+
+Set:
+
+```env
+MONGODB_URI=
+DB_HOSTNAME=mongodb
+MONGO_BIND_ADDRESS=127.0.0.1
+```
+
+Keep the pinned `MONGO_IMAGE` and the real `DB_USERNAME`/`DB_PASSWORD`.
+
+Recreate MongoDB from only the base Compose file:
+
+```bash
+docker compose -f docker-compose.yml stop \
+  anonymous_github streamer mongodb
+
+docker compose -f docker-compose.yml \
+  up -d --force-recreate mongodb
+```
+
+Verify the running command does not contain `--replSet`:
+
+```bash
+docker compose -f docker-compose.yml \
+  exec -T mongodb sh -c \
+  'tr "\000" " " < /proc/1/cmdline; echo'
+```
+
+Verify database access:
+
+```bash
+docker compose -f docker-compose.yml exec -T mongodb sh -eu -c \
+  'mongosh --quiet \
+    --username="$MONGO_INITDB_ROOT_USERNAME" \
+    --password="$MONGO_INITDB_ROOT_PASSWORD" \
+    --authenticationDatabase=admin \
+    --eval "db.adminCommand({ ping: 1 })"'
+```
+
+Restart the standalone application:
+
+```bash
+docker compose -f docker-compose.yml \
+  up -d redis streamer anonymous_github
+```
+
+Stop the remote secondary:
+
+```bash
+docker compose -f docker-compose.replica-secondary.yml \
+  stop mongodb-secondary
+```
+
+Do not delete either MongoDB volume, the `local` database, or the shared
+keyfile during rollback.
+
+## Troubleshooting
+
+### `Replica keyfile not found`
+
+On the primary:

 ```bash
 ./scripts/mongodb-replica.sh generate-key
 ```

-This creates `secrets/mongo-replica-keyfile` without overwriting an existing
-key. Copy that exact file securely to the same repository-relative path on the
-secondary server:
+Copy that same file to the `secrets` directory inside the secondary server's
+own repository checkout, for example:
+
+```text
+/srv/anonymous_github/secrets/mongo-replica-keyfile
+```
+
+Do not generate two independent keys. Compare both files with `sha256sum`.
+
+### `MongoServerError: not running with --replSet`
+
+The primary is still using the base Compose configuration. Run:

 ```bash
-scp secrets/mongo-replica-keyfile \
-  secondary-server:/path/to/anonymous_github/secrets/mongo-replica-keyfile
+./scripts/mongodb-replica.sh primary-up mongo-primary:27017
 ```

-Keep this key outside source control and backups that are accessible to
-untrusted users. Every replica-set member must share the same key.
+Do not run `add-secondary` before `primary-up` succeeds.

-## 2. Configure production
-
-Add these values to the primary server's `.env`:
-
-```env
-# Use the exact version already running in production.
-MONGO_IMAGE=mongo:<exact-version>
-
-# The primary server's private/VPN interface.
-MONGO_BIND_ADDRESS=<primary-private-ip>
-MONGO_REPLICA_KEYFILE=./secrets/mongo-replica-keyfile
-
-# Make the replica overlay the default for future Compose commands.
-COMPOSE_FILE=docker-compose.yml:docker-compose.replica-primary.yml
-
-# URL-encode special characters in the username and password.
-MONGODB_URI=mongodb://<user>:<password>@mongo-primary.internal:27017/production?authSource=admin&replicaSet=rs0&retryWrites=true&w=majority
-```
-
-Stop application writers, leaving MongoDB available:
+Inspect the primary process:

 ```bash
-docker compose stop anonymous_github streamer
+docker compose \
+  -f docker-compose.yml \
+  -f docker-compose.replica-primary.yml \
+  exec -T mongodb sh -c \
+  'tr "\000" " " < /proc/1/cmdline; echo'
 ```

-Start MongoDB with replica-set support and initialize the existing database as
-the primary:
+It must contain `--replSet rs0`.
+
+### `No host described ... maps to this node`
+
+`mongo-primary` does not resolve back to the primary MongoDB container.
+Confirm that the current Compose overlays include `hostname`, `networks`
+aliases, and `extra_hosts`, then run:

 ```bash
-./scripts/mongodb-replica.sh primary-up \
-  mongo-primary.internal:27017
+docker compose \
+  -f docker-compose.yml \
+  -f docker-compose.replica-primary.yml \
+  config
+
+docker compose \
+  -f docker-compose.yml \
+  -f docker-compose.replica-primary.yml \
+  exec -T mongodb getent hosts mongo-primary
 ```

-`primary-up` is idempotent. If `rs0` is already initialized, it reports that
-state instead of replacing the replica-set configuration.
+The result must be the primary container address, normally `172.x.x.x`, not
+the host's `100.x.x.x` Tailscale address.

-## 3. Start the remote secondary
-
-On the secondary server, create a minimal `.env`:
-
-```env
-MONGO_IMAGE=mongo:<same-exact-version-as-primary>
-MONGO_BIND_ADDRESS=<secondary-private-ip>
-MONGO_REPLICA_KEYFILE=./secrets/mongo-replica-keyfile
-```
-
-Start its empty MongoDB data volume:
+After correcting `.env`, recreate the primary container and retry:

 ```bash
-./scripts/mongodb-replica.sh secondary-up
+docker compose \
+  -f docker-compose.yml \
+  -f docker-compose.replica-primary.yml \
+  up -d --force-recreate mongodb
+
+./scripts/mongodb-replica.sh primary-up mongo-primary:27017
 ```

-Do not copy the primary's live WiredTiger volume. MongoDB performs the initial
-sync after the member is added.
+If a bad replica configuration was already accepted rather than rejected, use
+the standalone rollback procedure before attempting any forced
+`rs.reconfig()`. Forced replica reconfiguration can cause rollback or data
+loss.

-## 4. Add and verify the secondary
+### The primary cannot reach the secondary

-Back on the primary server, add the remote member with the default one-hour
-delay:
+Check the host network:

 ```bash
-./scripts/mongodb-replica.sh add-secondary \
-  mongo-secondary.internal:27017
+tailscale ping mongo-secondary
+tailscale status
 ```

-To keep the remote copy current instead, explicitly set a zero-second delay:
+Check the mapping from the primary container:

 ```bash
-./scripts/mongodb-replica.sh add-secondary \
-  mongo-secondary.internal:27017 0
+docker compose \
+  -f docker-compose.yml \
+  -f docker-compose.replica-primary.yml \
+  exec -T mongodb getent hosts mongo-secondary
 ```

-Check initial-sync and replication state:
+It must return the secondary's Tailscale IP.
+
+Check the secondary:

 ```bash
-./scripts/mongodb-replica.sh status
+docker compose -f docker-compose.replica-secondary.yml ps
+docker compose -f docker-compose.replica-secondary.yml \
+  logs --tail=100 mongodb-secondary
 ```

-Wait until the remote member reports `SECONDARY`, then restart the application:
+Confirm the Tailscale ACL and host firewall allow TCP `27017` from the primary
+to the secondary.

-```bash
-docker compose up -d anonymous_github streamer redis
-```
+### The application cannot connect after conversion

-## Operations
+Check:

-Use the primary overlay for every future operation on the production stack.
-Keeping `COMPOSE_FILE` in `.env` makes ordinary commands such as
-`docker compose up -d` and `docker compose logs mongodb` use it automatically.
+1. `MONGODB_URI` uses `mongo-primary:27017`.
+2. It includes `replicaSet=rs0` and `authSource=admin`.
+3. Credentials are URL-encoded.
+4. `COMPOSE_FILE` includes the primary overlay.
+5. The application container resolves the alias:

-Check replica status:
+   ```bash
+   docker compose exec -T anonymous_github getent hosts mongo-primary
+   ```

-```bash
-./scripts/mongodb-replica.sh status
-```
+6. `./scripts/mongodb-replica.sh status` reports a `PRIMARY`.

-If a destructive production operation occurs and the secondary is delayed,
-stop the secondary container immediately before the delay window elapses:
+## References

-```bash
-docker compose -f docker-compose.replica-secondary.yml stop mongodb-secondary
-```
-
-Then take a copy or logical dump of the delayed data before attempting
-recovery.
-
-For automatic failover, use three voting data-bearing members instead of
-turning this two-server recovery topology into a two-voter replica set. Two
-voters require both servers to acknowledge a majority and can make production
-unwritable during a network outage.
-
-## Troubleshooting
-
- All members must use the same replica-set name (`rs0`), MongoDB version, and
-  shared keyfile.
- The hostnames stored in `rs.conf()` must resolve from every member.
- The application container must also resolve `mongo-primary.internal`. Use
-  private DNS or a Compose `extra_hosts` entry if host DNS is not propagated
-  into Docker.
- Check container logs with:
-
-  ```bash
-  docker compose logs mongodb
-  docker compose -f docker-compose.replica-secondary.yml logs mongodb-secondary
-  ```
-
- Re-run `primary-up`, `add-secondary`, or `status` safely; the management
-  operations do not replace existing replica-set members.
-
-## MongoDB references
-
- [Convert a standalone server to a replica set](https://www.mongodb.com/docs/manual/tutorial/convert-standalone-to-replica-set/)
- [Deploy a replica set with member authentication](https://www.mongodb.com/docs/v8.0/tutorial/deploy-replica-set-with-keyfile-access-control/)
- [Configure a delayed member](https://www.mongodb.com/docs/manual/tutorial/configure-a-delayed-replica-set-member/)
- [Hidden replica-set members](https://www.mongodb.com/docs/manual/core/replica-set-hidden-member/)
+- [MongoDB: convert a standalone server to a replica set](https://www.mongodb.com/docs/manual/tutorial/convert-standalone-to-replica-set/)
+- [MongoDB: deploy a replica set with keyfile authentication](https://www.mongodb.com/docs/v8.0/tutorial/deploy-replica-set-with-keyfile-access-control/)
+- [MongoDB: use resolvable hostnames for replica members](https://www.mongodb.com/docs/manual/tutorial/change-hostnames-in-a-replica-set/)
+- [MongoDB: configure a delayed member](https://www.mongodb.com/docs/manual/tutorial/configure-a-delayed-replica-set-member/)
+- [MongoDB: hidden replica-set members](https://www.mongodb.com/docs/manual/core/replica-set-hidden-member/)
+- [Tailscale: MagicDNS](https://tailscale.com/docs/features/magicdns)
@@ -25,7 +25,7 @@ usage() {
 Usage:
  ./scripts/mongodb-replica.sh generate-key
  ./scripts/mongodb-replica.sh primary-up <primary-hostname:port>
-  ./scripts/mongodb-replica.sh secondary-up
+  ./scripts/mongodb-replica.sh secondary-up <secondary-hostname:port>
  ./scripts/mongodb-replica.sh add-secondary <secondary-hostname:port> [delay-seconds]
  ./scripts/mongodb-replica.sh status

@@ -96,6 +96,66 @@ wait_for_primary_container() {
  exit 1
 }

+wait_for_replica_primary() {
+  local attempt
+  for attempt in $(seq 1 60); do
+    if mongo_eval \
+      "if (!db.hello().isWritablePrimary) throw new Error('not primary yet');" \
+      >/dev/null 2>&1; then
+      return
+    fi
+    sleep 2
+  done
+  echo "Replica set rs0 did not elect a writable primary within 120 seconds" >&2
+  echo "Check ./scripts/mongodb-replica.sh status and the MongoDB container logs." >&2
+  exit 1
+}
+
+wait_for_secondary_container() {
+  local attempt
+  for attempt in $(seq 1 60); do
+    if secondary_compose exec -T mongodb-secondary mongosh --quiet \
+      --eval "db.adminCommand({ ping: 1 }).ok" >/dev/null 2>&1; then
+      return
+    fi
+    sleep 2
+  done
+  echo "Secondary MongoDB did not become ready within 120 seconds" >&2
+  exit 1
+}
+
+check_primary_endpoint() {
+  local endpoint="$1"
+  primary_compose exec -T -e "MONGO_RS_ENDPOINT=$endpoint" mongodb sh -eu -c '
+    host="${MONGO_RS_ENDPOINT%:*}"
+    port="${MONGO_RS_ENDPOINT##*:}"
+    echo "Checking $host resolves inside the primary MongoDB container:"
+    getent hosts "$host"
+    mongosh --quiet --host "$host" --port "$port" \
+      --eval "db.adminCommand({ ping: 1 }).ok" >/dev/null
+  ' || {
+    echo "The endpoint $endpoint does not resolve to a reachable MongoDB instance from the primary container." >&2
+    echo "Check MONGO_PRIMARY_HOSTNAME, MONGO_PRIMARY_ADDRESS, MONGO_SECONDARY_HOSTNAME, and MONGO_SECONDARY_ADDRESS." >&2
+    exit 1
+  }
+}
+
+check_secondary_endpoint() {
+  local endpoint="$1"
+  secondary_compose exec -T -e "MONGO_RS_ENDPOINT=$endpoint" mongodb-secondary sh -eu -c '
+    host="${MONGO_RS_ENDPOINT%:*}"
+    port="${MONGO_RS_ENDPOINT##*:}"
+    echo "Checking $host resolves inside the secondary MongoDB container:"
+    getent hosts "$host"
+    mongosh --quiet --host "$host" --port "$port" \
+      --eval "db.adminCommand({ ping: 1 }).ok" >/dev/null
+  ' || {
+    echo "The endpoint $endpoint does not resolve to the secondary MongoDB container." >&2
+    echo "Check MONGO_SECONDARY_HOSTNAME and the secondary Compose configuration." >&2
+    exit 1
+  }
+}
+
 command="${1:-}"
 case "$command" in
  generate-key)
@@ -117,6 +177,7 @@ case "$command" in
    require_keyfile
    primary_compose up -d mongodb
    wait_for_primary_container
+    check_primary_endpoint "$primary_host"
    mongo_eval "
      try {
        const status = rs.status();
@@ -129,11 +190,16 @@ case "$command" in
        }));
      }
    "
+    wait_for_replica_primary
    ;;

  secondary-up)
+    secondary_host="${2:-}"
+    validate_host "$secondary_host"
    require_keyfile
    secondary_compose up -d mongodb-secondary
+    wait_for_secondary_container
+    check_secondary_endpoint "$secondary_host"
    ;;

  add-secondary)
@@ -141,6 +207,7 @@ case "$command" in
    delay="${3:-3600}"
    validate_host "$secondary_host"
    validate_delay "$delay"
+    check_primary_endpoint "$secondary_host"
    mongo_eval "
      const host = '$secondary_host';
      const existing = rs.conf().members.find((member) => member.host === host);