From 37b12c5feef1cdd3e827f4b659cacec0ec3807a7 Mon Sep 17 00:00:00 2001 From: thepra Date: Wed, 7 Oct 2026 11:21:13 +0200 Subject: [PATCH] Mongo is a one-member replica set Owner decision 2026-10-07: production's mongod becomes a one-member replica set (rs0), so that a backup can read every collection at one instant (a snapshot read session), which a standalone mongod can't. The unit runs mongod with --replSet rs0 and a 990 MB oplog; setup.sh converts it once, idempotently (PrivaPub stopped, mongod restarted, rs.initiate, the primary awaited, PrivaPub started); every connection string says directConnection=true, which works against the standalone too, so this code can deploy before the conversion. The CI's throwaway mongod and the pasture's are replica sets as well, so the snapshot path is what the tests exercise; MongoTopology tells which a mongod is, and TopologyTests holds the test mongod to it. The suite passes on it (906). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01LsXgEaXee4GCU1hwYgPJXw --- .gitea/workflows/deploy.yml | 2 +- CLAUDE.md | 7 +++++- .../Infrastructure/TopologyTests.cs | 21 +++++++++++++++++ PrivaPub/Infrastructure/Data/MongoTopology.cs | 23 +++++++++++++++++++ PrivaPub/appsettings.Production.json | 4 ++-- deploy/max/setup.sh | 14 +++++++++++ deploy/systemd/privapub-mongod.service | 3 ++- tools/ci/with-test-mongod.sh | 19 +++++++++++---- tools/pasture/appsettings.Pasture.json | 2 +- tools/pasture/lib/pasture.sh | 6 ++++- 10 files changed, 90 insertions(+), 11 deletions(-) create mode 100644 PrivaPub.Tests/Infrastructure/TopologyTests.cs create mode 100644 PrivaPub/Infrastructure/Data/MongoTopology.cs diff --git a/.gitea/workflows/deploy.yml b/.gitea/workflows/deploy.yml index c10982a..4b6828f 100644 --- a/.gitea/workflows/deploy.yml +++ b/.gitea/workflows/deploy.yml @@ -12,7 +12,7 @@ env: BACKUPS: /var/backups/privapub.thepra.dev LOCAL_URL: http://127.0.0.1:6970 PUBLIC_URL: https://privapub.thepra.dev - MONGO_URI: mongodb://127.0.0.1:27022 + MONGO_URI: mongodb://127.0.0.1:27022/?directConnection=true MONGO_DB: PrivaPub jobs: diff --git a/CLAUDE.md b/CLAUDE.md index aa70ede..a6b2c18 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -480,7 +480,10 @@ group www-data and reaches the private mongod; `sudo -u www-data` works too. `_NNN_` order, each runs once), then `Indexes.Create()`. A new entity needs nothing; a new unique index needs a dedupe migration before it. - Production runs its **own mongod** on 127.0.0.1:27022 (unit `privapub-mongod`, no auth, data in - `/var/lib/privapub/mongo`). The box's shared mongod needs credentials nobody here has. + `/var/lib/privapub/mongo`). The box's shared mongod needs credentials nobody here has. It is a one-member replica set + (`--replSet rs0`, a 990 MB oplog; owner decision 2026-10-07), so a backup reads every collection at one instant; + `setup.sh` converts it once (PrivaPub stopped, mongod restarted, `rs.initiate`), and every connection string says + `directConnection=true`, which also works against a standalone. The pasture's mongo is one too. ## Code style @@ -505,6 +508,8 @@ group www-data and reaches the private mongod; `sudo -u www-data` works too. `PrivaPub.Tests` (xUnit v3). Unit tests need nothing; tests marked `Category=Integration` need a mongod and skip without `PRIVAPUB_TEST_MONGOD=1`. CI (`build.yml` and `deploy.yml`) runs all of them through `tools/ci/with-test-mongod.sh`, which starts a throwaway mongod on a random localhost port and deletes it afterwards. +Like production's, it is a one-member replica set (`rs0`, reached with `directConnection=true`), so backups' snapshot +reads are what the tests exercise (`TopologyTests`). The box's own mongods are production, so `MongoFixture` refuses port 27022, a data directory under `/var/lib/privapub`, and in CI anything but the wrapper's mongod. With `PRIVAPUB_TEST_REQUIRE_MONGOD=1`, which the wrapper sets, a missing mongod fails the run instead of silently skipping half the tests. diff --git a/PrivaPub.Tests/Infrastructure/TopologyTests.cs b/PrivaPub.Tests/Infrastructure/TopologyTests.cs new file mode 100644 index 0000000..1ad10ca --- /dev/null +++ b/PrivaPub.Tests/Infrastructure/TopologyTests.cs @@ -0,0 +1,21 @@ +using MongoDB.Entities; + +using PrivaPub.Infrastructure.Data; +using PrivaPub.Tests.Support; + +namespace PrivaPub.Tests.Infrastructure +{ + [Trait("Category", "Integration")] + public sealed class TopologyTests + { + // tools/ci/with-test-mongod.sh starts a one-member replica set, as production's mongod is: the backup's snapshot reads + // are what the tests exercise + [Fact] + public async Task The_test_mongod_is_a_replica_set_like_productions() + { + Assert.SkipUnless(MongoFixture.Enabled, MongoFixture.Skip); + Assert.SkipUnless(MongoFixture.Connection.Contains("directConnection=true"), "a mongod not started by tools/ci/with-test-mongod.sh"); + Assert.True(await MongoTopology.IsReplicaSet(DB.Default.Database(), TestContext.Current.CancellationToken)); + } + } +} diff --git a/PrivaPub/Infrastructure/Data/MongoTopology.cs b/PrivaPub/Infrastructure/Data/MongoTopology.cs new file mode 100644 index 0000000..3750d6e --- /dev/null +++ b/PrivaPub/Infrastructure/Data/MongoTopology.cs @@ -0,0 +1,23 @@ +using MongoDB.Bson; +using MongoDB.Driver; + +namespace PrivaPub.Infrastructure.Data +{ + // Whether the mongod is a replica set member (production's is a one-member set, rs0): only then can a read session + // hold one point in time across collections (a snapshot), which a backup needs to be consistent. + public static class MongoTopology + { + public static async Task IsReplicaSet(IMongoDatabase database, CancellationToken token) + { + try + { + var hello = await database.Client.GetDatabase("admin").RunCommandAsync(new BsonDocument("hello", 1), cancellationToken: token); + return hello.Contains("setName"); + } + catch (MongoCommandException) + { + return false; + } + } + } +} diff --git a/PrivaPub/appsettings.Production.json b/PrivaPub/appsettings.Production.json index fb14057..e83599c 100644 --- a/PrivaPub/appsettings.Production.json +++ b/PrivaPub/appsettings.Production.json @@ -28,7 +28,7 @@ "MongoSettings": { "Database": "PrivaPub", "LogsDatabase": "logs", - "ConnectionString": "mongodb://127.0.0.1:27022" + "ConnectionString": "mongodb://127.0.0.1:27022/?directConnection=true" }, "AppConfiguration": { "Version": "0.0.0", @@ -74,7 +74,7 @@ { "Name": "MongoDBCapped", "Args": { - "databaseUrl": "mongodb://127.0.0.1:27022/logs", + "databaseUrl": "mongodb://127.0.0.1:27022/logs?directConnection=true", "collectionName": "PrivaPub", "cappedMaxSizeMb": "1024", "cappedMaxDocuments": "10000" diff --git a/deploy/max/setup.sh b/deploy/max/setup.sh index a028224..3358f8d 100755 --- a/deploy/max/setup.sh +++ b/deploy/max/setup.sh @@ -13,6 +13,8 @@ echo "== directories" install -d -o "$RUNNER" -g www-data -m 755 /var/www/$HOST install -d -o "$RUNNER" -g "$RUNNER" -m 750 /var/backups/$HOST install -d -o www-data -g www-data -m 750 /var/lib/privapub /var/lib/privapub/mongo +# backups: the service writes them, and so does the deploy (as $RUNNER, a member of www-data) before each deploy +install -d -o www-data -g www-data -m 2770 /var/lib/privapub/backups echo "== sudoers" SUDOERS=/etc/sudoers.d/$RUNNER @@ -28,6 +30,18 @@ systemctl enable --now privapub-mongod >/dev/null systemctl enable $UNIT >/dev/null systemctl is-active privapub-mongod +echo "== replica set" +# once: mongod restarted with --replSet (PrivaPub stopped meanwhile), the one-member set rs0 made, its primary awaited +if [ "$(mongosh --quiet --port 27022 --eval 'db.hello().setName' 2>/dev/null)" != "rs0" ]; then + systemctl stop $UNIT + systemctl restart privapub-mongod + for _ in $(seq 1 60); do mongosh --quiet --port 27022 --eval 'db.hello()' >/dev/null 2>&1 && break; sleep 1; done + mongosh --quiet --port 27022 --eval "rs.initiate({_id: 'rs0', members: [{_id: 0, host: '127.0.0.1:27022'}]})" >/dev/null + for _ in $(seq 1 60); do [ "$(mongosh --quiet --port 27022 --eval 'db.hello().isWritablePrimary')" = "true" ] && break; sleep 1; done + systemctl start $UNIT +fi +echo "replica set: $(mongosh --quiet --port 27022 --eval 'db.hello().setName')" + echo "== nginx snippet and bootstrap vhost" install -m 644 "$SRC/nginx/privapub-headers.conf" /etc/nginx/snippets/privapub-headers.conf if [ -f /root/.acme.sh/${HOST}_ecc/fullchain.cer ]; then diff --git a/deploy/systemd/privapub-mongod.service b/deploy/systemd/privapub-mongod.service index aba0258..1e13e2a 100644 --- a/deploy/systemd/privapub-mongod.service +++ b/deploy/systemd/privapub-mongod.service @@ -6,7 +6,8 @@ After=network.target Type=exec User=www-data Group=www-data -ExecStart=/usr/bin/mongod --dbpath /var/lib/privapub/mongo --port 27022 --bind_ip 127.0.0.1 --noauth --wiredTigerCacheSizeGB 0.25 --quiet +# a one-member replica set (rs0), so a backup reads every collection at one instant; its oplog kept small +ExecStart=/usr/bin/mongod --dbpath /var/lib/privapub/mongo --port 27022 --bind_ip 127.0.0.1 --noauth --wiredTigerCacheSizeGB 0.25 --replSet rs0 --oplogSizeMB 990 --quiet Restart=always RestartSec=5 LimitNOFILE=64000 diff --git a/tools/ci/with-test-mongod.sh b/tools/ci/with-test-mongod.sh index b07825e..5c907fc 100755 --- a/tools/ci/with-test-mongod.sh +++ b/tools/ci/with-test-mongod.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash # Runs a command (normally `dotnet test`) with a throwaway mongod: a fresh data directory on a random localhost port, # stopped and deleted when the command ends. The box's own mongods (27017 shared, 27022 PrivaPub's) are never touched. +# Like production's, it is a one-member replica set (rs0), so the backup's snapshot reads are what the tests exercise. # usage: tools/ci/with-test-mongod.sh dotnet test PrivaPub.sln -c Release set -euo pipefail @@ -39,11 +40,11 @@ trap cleanup EXIT INT TERM if command -v mongod >/dev/null 2>&1; then mkdir -p "$dir/db" timeout 7200 mongod --dbpath "$dir/db" --port "$port" --bind_ip 127.0.0.1 --noauth \ - --wiredTigerCacheSizeGB 0.25 --quiet --logpath "$dir/mongod.log" & + --wiredTigerCacheSizeGB 0.25 --replSet rs0 --quiet --logpath "$dir/mongod.log" & echo $! > "$dir/pid" how="mongod $(mongod --version | head -1)" elif command -v podman >/dev/null 2>&1; then - podman run -d --rm -p "127.0.0.1:$port:27017" docker.io/library/mongo:8 --quiet > "$dir/container" + podman run -d --rm -p "127.0.0.1:$port:27017" docker.io/library/mongo:8 --quiet --replSet rs0 > "$dir/container" how="podman mongo:8" else echo "::error::neither mongod nor podman is available for the test mongod"; exit 1 @@ -51,7 +52,17 @@ fi for _ in $(seq 1 60); do listening "$port" && break; sleep 0.5; done listening "$port" || { echo "::error::the test mongod did not start"; cat "$dir/mongod.log" 2>/dev/null | tail -20; exit 1; } -echo "test mongod: $how on 127.0.0.1:$port" -export PRIVAPUB_TEST_MONGOD=1 PRIVAPUB_TEST_REQUIRE_MONGOD=1 PRIVAPUB_TEST_MONGO="mongodb://127.0.0.1:$port" +# the one-member replica set, then wait until it is its primary +shell() { + if [ -f "$dir/container" ]; then podman exec "$(cat "$dir/container")" mongosh --quiet --eval "$1" + else mongosh --quiet --port "$port" --eval "$1"; fi +} +member=$([ -f "$dir/container" ] && echo "127.0.0.1:27017" || echo "127.0.0.1:$port") +shell "rs.initiate({_id: 'rs0', members: [{_id: 0, host: '$member'}]})" >/dev/null +for _ in $(seq 1 60); do [ "$(shell 'db.hello().isWritablePrimary' 2>/dev/null)" = "true" ] && break; sleep 0.5; done +[ "$(shell 'db.hello().isWritablePrimary' 2>/dev/null)" = "true" ] || { echo "::error::the test mongod never became its replica set's primary"; exit 1; } +echo "test mongod: $how on 127.0.0.1:$port, replica set rs0" + +export PRIVAPUB_TEST_MONGOD=1 PRIVAPUB_TEST_REQUIRE_MONGOD=1 PRIVAPUB_TEST_MONGO="mongodb://127.0.0.1:$port/?directConnection=true" "$@" diff --git a/tools/pasture/appsettings.Pasture.json b/tools/pasture/appsettings.Pasture.json index eaf8857..6a3bee2 100644 --- a/tools/pasture/appsettings.Pasture.json +++ b/tools/pasture/appsettings.Pasture.json @@ -2,7 +2,7 @@ "MongoSettings": { "Database": "PrivaPub", "LogsDatabase": "logs", - "ConnectionString": "mongodb://mongo:27017" + "ConnectionString": "mongodb://mongo:27017/?directConnection=true" }, "AppConfiguration": { "Version": "0.0.0", diff --git a/tools/pasture/lib/pasture.sh b/tools/pasture/lib/pasture.sh index 7df3926..315068a 100644 --- a/tools/pasture/lib/pasture.sh +++ b/tools/pasture/lib/pasture.sh @@ -21,7 +21,11 @@ pasture_base_up() { podman volume exists pasture-caddy-data || podman volume create pasture-caddy-data >/dev/null # (PrivaPub's uploads live in a volume: a container made again must not lose them) podman volume exists pasture-privapub-media || podman volume create --label pasture=1 pasture-privapub-media >/dev/null - podman run -d --replace --name pasture-mongo --network $net --network-alias mongo docker.io/library/mongo:8 --quiet >/dev/null + # a one-member replica set, as production's, so backups read at one instant + podman run -d --replace --name pasture-mongo --network $net --network-alias mongo docker.io/library/mongo:8 --quiet --replSet rs0 >/dev/null + for _ in $(seq 1 60); do podman exec pasture-mongo mongosh --quiet --eval 'db.hello()' >/dev/null 2>&1 && break; sleep 1; done + podman exec pasture-mongo mongosh --quiet --eval "rs.initiate({_id: 'rs0', members: [{_id: 0, host: 'mongo:27017'}]})" >/dev/null + for _ in $(seq 1 60); do [ "$(podman exec pasture-mongo mongosh --quiet --eval 'db.hello().isWritablePrimary')" = "true" ] && break; sleep 1; done caddy_up local extra=() for setting in ${PRIVAPUB_ENV:-}; do extra+=(-e "$setting"); done