The server backs itself up: every collection byte for byte, the media linked

A backup is a directory <stamp>-<kind> under Backups:Root (/var/lib/privapub/backups, 2770, files 0640), written as
.partial and renamed once whole: a manifest (host, build, newest migration, each collection's count, size, sha256 and
indexes, what was left out and why, the media list), each collection as gzipped canonical Extended JSON read raw, and
hard links to the files of untrashed media rows (copies where a link can't be made). On a replica set every collection
is read in one snapshot session. Never in a backup: the statistics salt, jobs, recovery codes, sessions, the
maintenance lock and the configuration's copy with its SMTP password.

One backup or restore at a time (MaintenanceLock, a heartbeat document), and the janitor purges nothing meanwhile.
BackupScheduler backs up nightly at 03:30 UTC (or at once after missing a night); rotation keeps 7 daily, 4 weekly,
3 pre-deploy and 3 pre-restore backups. CLI: admin backup [--kind] [--db-only], admin backups, admin backup verify;
these run before migrations, so the deploy's own pre-deploy backup, which replaces mongodump, is of the database as the
live build left it. EntityMaps.Warm runs once under a lock, since test hosts now boot side by side.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01LsXgEaXee4GCU1hwYgPJXw

.gitea/workflows/deploy.yml

@@ -12,8 +12,7 @@ env:
   BACKUPS: /var/backups/privapub.thepra.dev
   LOCAL_URL: http://127.0.0.1:6970
   PUBLIC_URL: https://privapub.thepra.dev
-  MONGO_URI: mongodb://127.0.0.1:27022/?directConnection=true
+  SERVER_BACKUPS: /var/lib/privapub/backups
-  MONGO_DB: PrivaPub
 
 jobs:
   site:
@@ -57,19 +56,24 @@ jobs:
           echo "SNAPSHOT=$BACKUPS/site-$STAMP" >> "$GITHUB_ENV"
           ls -1dt "$BACKUPS"/site-* 2>/dev/null | tail -n +4 | xargs -r rm -rf || true
 
-      # without the statistics salt, which must not outlive its day (owner rule: it is destroyed at each rollup, and a dump
+      # The server's own backup (PrivaPub admin backup, owner decision 2026-10-07) of the database as the live build left it:
-      # kept for days would let the day's actor hashes be reversed)
+      # the new build's command runs before its migrations. The media are listed, not linked (the runner may not link
-      - name: Dump the database
+      # www-data's files; the nightly backups hold them). Never the statistics salt, which must not outlive its day (owner
+      # rule), nor the configuration's copy with its secrets. No deploy while a restore waits for its boot.
+      - name: Back up the database
         run: |
-          DUMP="$BACKUPS/mongo-$(date +%Y%m%d-%H%M%S).archive.gz"
+          [ ! -e "$SERVER_BACKUPS/restore.json" ] || { echo "::error::a restore is pending or running ($SERVER_BACKUPS/restore.json): deploy after it"; exit 1; }
-          mongodump --quiet --uri "$MONGO_URI" --db "$MONGO_DB" --excludeCollection=InteractionSalt --gzip --archive="$DUMP"
+          out=$(cd "$GITHUB_WORKSPACE/publish" && ASPNETCORE_ENVIRONMENT=Production ./PrivaPub admin backup --kind pre-deploy --db-only)
-          if mongorestore --gzip --archive="$DUMP" --dryRun -v 2>&1 | grep -q "InteractionSalt"; then
+          echo "$out"
-            rm -f "$DUMP"
+          id=$(echo "$out" | grep -oE '^[0-9]{8}-[0-9]{6}-pre-deploy' | tail -1)
-            echo "::error::the dump holds the statistics salt"
+          [ -d "$SERVER_BACKUPS/$id" ] || { echo "::error::the backup was not made"; exit 1; }
+          (cd "$GITHUB_WORKSPACE/publish" && ASPNETCORE_ENVIRONMENT=Production ./PrivaPub admin backup verify "$id")
+          if [ -e "$SERVER_BACKUPS/$id/db/InteractionSalt.jsonl.gz" ]; then
+            echo "::error::the backup holds the statistics salt"
             exit 1
           fi
-          echo "::notice::database dumped to $DUMP ($(du -h "$DUMP" | cut -f1))"
+          echo "BACKUP_ID=$id" >> "$GITHUB_ENV"
-          ls -1t "$BACKUPS"/mongo-*.archive.gz 2>/dev/null | tail -n +8 | xargs -r rm -f || true
+          echo "::notice::database backed up as $id"
 
       - name: Stop, sync, start
         run: |

.gitignore

@@ -257,6 +257,7 @@ Generated_Code/
 # because we have git ;-)
 _UpgradeReport_Files/
 Backup*/
+!PrivaPub/Infrastructure/Backup/
 UpgradeLog*.XML
 UpgradeLog*.htm
 ServiceFabricBackup/
@@ -401,6 +402,9 @@ FodyWeavers.xsd
 
 PrivaPub/media-store/
 PrivaPub/media-store-proxy/
+PrivaPub/media-store-trash/
+PrivaPub/media-store-incoming/
+PrivaPub/media-store-backups/
 tools/pasture/.publish/
 tools/pasture/.flood/
 .claude/worktrees/

CLAUDE.md

@@ -484,6 +484,26 @@ group www-data and reaches the private mongod; `sudo -u www-data` works too.
   (`--replSet rs0`, a 990 MB oplog; owner decision 2026-10-07), so a backup reads every collection at one instant;
   `setup.sh` converts it once (PrivaPub stopped, mongod restarted, `rs.initiate`), and every connection string says
   `directConnection=true`, which also works against a standalone. The pasture's mongo is one too.
+- **Backups** (`Infrastructure/Backup/`; owner decisions 2026-10-07: the whole server, plain files protected by their
+  permissions, run from the CLI, nightly and from the administrator's page). Each backup is a directory
+  `<yyyyMMdd-HHmmss>-<kind>` under `Backups:Root` (`/var/lib/privapub/backups`, www-data 2770, files 0640), written as
+  `.partial` and renamed once whole:
+  - `manifest.json` (`ArchiveManifest`): host, build, newest migration, whether it was read at one instant, each
+    collection's count, size, sha256 and indexes, what was left out and why, and the media list;
+  - `db/<collection>.jsonl.gz`: each document as one line of canonical Extended JSON, read raw, so every BSON type comes
+    back byte for byte; on a replica set every collection is read in one snapshot session
+    (`minSnapshotHistoryWindowInSeconds` is raised to an hour only while it reads);
+  - `media/`: hard links to the files of untrashed `MediaAttachment` rows (no disk spent; a deleted file stays until the
+    backup rotates out), copies where a link can't be made. `--db-only` lists them instead.
+
+  **Never in a backup** (`ServerBackup.Excluded`): `InteractionSalt` (owner rule), `Job` and `Delivery` (work in flight),
+  `EmailRecovery`, `openiddict.tokens` and `.authorizations` (sessions), `MaintenanceLock`, and `AppConfiguration`
+  (its SMTP password; it is made again from appsettings at boot). One backup or restore runs at a time
+  (`MaintenanceLock`, a heartbeat document; a holder silent for 2 minutes is taken over), and the media janitor purges
+  nothing meanwhile. `BackupScheduler` backs up at `Backups:NightlyAt` (03:30 UTC), or at once when it missed the night;
+  rotation keeps 7 daily and 4 weekly nightly backups, 3 pre-deploy, 3 pre-restore, and manual and uploaded ones until
+  deleted. CLI: `PrivaPub admin backup [--kind manual|pre-deploy] [--db-only]`, `admin backups`, `admin backup verify
+  <id>`; these run before migrations, so the deploy's backup is of the database as the live build left it.
 
 ## Code style
 
@@ -840,8 +860,9 @@ the owner's GoToSocial account.**
 ## Deploy
 
 - **CI/CD:** push to `master` runs `build.yml` (build + tests) on the instance-wide `build` runner. A `v*` tag runs
-  `deploy.yml`: tests, self-contained linux-x64 publish, snapshot and `mongodump` to `/var/backups/privapub.thepra.dev`
+  `deploy.yml`: tests, self-contained linux-x64 publish, a snapshot of the site to `/var/backups/privapub.thepra.dev`,
-  (never the statistics salt: `InteractionSalt` is excluded and the dump is checked for it),
+  the server's own pre-deploy backup (`PrivaPub admin backup --kind pre-deploy --db-only`, run from the new build before
+  its migrations, verified, and checked to hold no statistics salt; no deploy while `restore.json` waits),
   stop → rsync → start, a `127.0.0.1:6970/build.json` health loop with rollback, then public checks (actor, NodeInfo,
   Swagger 404, inbox junk 400, unsigned 401) and `tools/smoke/mastodon-api.sh` (app registration, client credentials,
   discovery, instance, public timeline). Then it checks that what production should be running is running:

PrivaPub.Tests/Infrastructure/BackupTests.cs

@@ -0,0 +1,286 @@
+using MongoDB.Bson;
+using MongoDB.Bson.IO;
+using MongoDB.Driver;
+using MongoDB.Entities;
+
+using PrivaPub.Infrastructure.Backup;
+using PrivaPub.Infrastructure.Cli;
+using PrivaPub.Tests.Support;
+using PrivaPub.Tests.Support.Host;
+
+using System.IO.Compression;
+
+namespace PrivaPub.Tests.Infrastructure
+{
+	// The server's backup: every collection as it was, byte for byte, read at one instant on a replica set, without what
+	// belongs to the moment (the statistics salt above all), with the media rows' files linked, checkable and rotated.
+	// Each test backs up a database of its own; alone, since the maintenance lock is the server's one.
+	[Trait("Category", "Integration")]
+	[Xunit.Collection(nameof(Exclusive))]
+	public sealed class BackupTests : IAsyncLifetime
+	{
+		readonly string _scratch = Path.Combine(Path.GetTempPath(), $"privapub-backup-tests-{Guid.NewGuid():N}");
+		IMongoDatabase _database;
+
+		string Backups => Path.Combine(_scratch, "backups");
+		string Media => Path.Combine(_scratch, "media");
+		string Trash => Path.Combine(_scratch, "media-trash");
+
+		public async ValueTask InitializeAsync()
+		{
+			Assert.SkipUnless(MongoFixture.Enabled, MongoFixture.Skip);
+			await PrivaPubHost.Shared();
+			_database = DB.Default.Database().Client.GetDatabase($"PrivaPubBackup_{Guid.NewGuid():N}");
+			Directory.CreateDirectory(Media);
+			Directory.CreateDirectory(Trash);
+		}
+
+		public async ValueTask DisposeAsync()
+		{
+			if (_database != default)
+				await _database.Client.DropDatabaseAsync(_database.DatabaseNamespace.DatabaseName);
+			if (Directory.Exists(_scratch))
+				Directory.Delete(_scratch, recursive: true);
+		}
+
+		BackupContext Context(BackupOptions options = default) => new(_database, Backups, Media, Trash, "https://privapub.test", options ?? new BackupOptions());
+
+		static CancellationToken Token => TestContext.Current.CancellationToken;
+
+		static List<BsonDocument> Read(string file)
+		{
+			using var gzip = new GZipStream(File.OpenRead(file), CompressionMode.Decompress);
+			using var reader = new StreamReader(gzip);
+			var documents = new List<BsonDocument>();
+			while (reader.ReadLine() is { } line)
+				documents.Add(BsonDocument.Parse(line));
+			return documents;
+		}
+
+		async Task<List<byte[]>> Raw(string collection) =>
+			(await (await _database.GetCollection<RawBsonDocument>(collection).FindAsync(FilterDefinition<RawBsonDocument>.Empty, cancellationToken: Token)).ToListAsync(Token))
+				.Select(d =>
+				{
+					var bytes = new byte[d.Slice.Length];
+					d.Slice.GetBytes(0, bytes, 0, bytes.Length);
+					return bytes;
+				})
+				.ToList();
+
+		[Fact]
+		public async Task Every_document_comes_back_byte_for_byte_and_the_salt_never_leaves()
+		{
+			var odd = new BsonDocument
+			{
+				{ "_id", ObjectId.GenerateNewId() },
+				{ "Guid", new BsonBinaryData(Guid.NewGuid(), GuidRepresentation.Standard) },
+				{ "OldGuid", new BsonBinaryData(new byte[16], BsonBinarySubType.UuidLegacy) },
+				{ "Money", new BsonDecimal128(Decimal128.Parse("12345.678900")) },
+				{ "Long", new BsonInt64(long.MaxValue) },
+				{ "Int", 7 },
+				{ "Double", -0.0 },
+				{ "NaN", double.NaN },
+				{ "At", new BsonDateTime(new DateTime(2026, 10, 7, 3, 30, 0, 123, DateTimeKind.Utc)) },
+				{ "Stamp", new BsonTimestamp(1, 2) },
+				{ "Null", BsonNull.Value },
+				{ "Regex", new BsonRegularExpression("^a.b$", "i") },
+				{ "Nested", new BsonDocument { { "z", 1 }, { "a", new BsonArray { 1, "two", new BsonDocument("three", 3) } } } },
+				{ "Unicode", "città 🌍 \u0000 end" }
+			};
+			// an ObjectRecord as big as a remote's long post gets
+			var big = new BsonDocument { { "_id", ObjectId.GenerateNewId() }, { "Json", new string('x', 10 * 1024 * 1024) } };
+			await _database.GetCollection<BsonDocument>("Odd").InsertOneAsync(odd, cancellationToken: Token);
+			await _database.GetCollection<BsonDocument>("ObjectRecord").InsertOneAsync(big, cancellationToken: Token);
+			await _database.GetCollection<BsonDocument>("Odd").Indexes.CreateOneAsync(
+				new CreateIndexModel<BsonDocument>(Builders<BsonDocument>.IndexKeys.Ascending("At"), new CreateIndexOptions { Name = "at", Unique = true }), cancellationToken: Token);
+			await _database.GetCollection<BsonDocument>("InteractionSalt").InsertOneAsync(new BsonDocument("Salt", "never in a backup"), cancellationToken: Token);
+			await _database.GetCollection<BsonDocument>("Job").InsertOneAsync(new BsonDocument("Kind", "Deliver"), cancellationToken: Token);
+			await _database.GetCollection<BsonDocument>("_migration_history_").InsertManyAsync([
+				new BsonDocument { { "Number", 15 }, { "Name", "older" } }, new BsonDocument { { "Number", 16 }, { "Name", "focal points are finite" } }], cancellationToken: Token);
+
+			var (backup, error) = await ServerBackup.Create(Context(), "manual", dbOnly: false, Token);
+
+			Assert.Null(error);
+			var directory = Path.Combine(Backups, backup.Id);
+			Assert.False(Directory.Exists(directory + ServerBackup.PartialSuffix));
+			Assert.Equal(await Raw("Odd"), Read(Path.Combine(directory, "db", "Odd.jsonl.gz")).Select(d => d.ToBson()).ToList());
+			Assert.Equal(await Raw("ObjectRecord"), Read(Path.Combine(directory, "db", "ObjectRecord.jsonl.gz")).Select(d => d.ToBson()).ToList());
+
+			Assert.False(File.Exists(Path.Combine(directory, "db", "InteractionSalt.jsonl.gz")));
+			Assert.False(File.Exists(Path.Combine(directory, "db", "Job.jsonl.gz")));
+			foreach (var file in Directory.EnumerateFiles(directory, "*", SearchOption.AllDirectories))
+				Assert.DoesNotContain("never in a backup", await ReadAll(file));
+			Assert.Contains(backup.Manifest.Excluded, e => e.Name == "InteractionSalt");
+
+			var odds = backup.Manifest.Collections.Single(c => c.Name == "Odd");
+			Assert.Equal(1, odds.Count);
+			Assert.Contains(odds.Indexes, i => BsonDocument.Parse(i)["name"] == "at" && BsonDocument.Parse(i)["unique"].ToBoolean());
+			Assert.Equal((16, "focal points are finite"), (backup.Manifest.MigrationNumber, backup.Manifest.NewestMigration));
+			Assert.Equal("https://privapub.test", backup.Manifest.Host);
+			Assert.Equal(MongoFixture.Connection.Contains("directConnection=true"), backup.Manifest.Consistent);
+			Assert.Empty(await ServerBackup.Verify(Backups, backup.Id, Token));
+			if (!OperatingSystem.IsWindows())
+			{
+				Assert.Equal(UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.GroupRead, File.GetUnixFileMode(Path.Combine(directory, "db", "Odd.jsonl.gz")));
+				Assert.False(File.GetUnixFileMode(directory).HasFlag(UnixFileMode.OtherRead));
+			}
+		}
+
+		static async Task<string> ReadAll(string file)
+		{
+			if (!file.EndsWith(".gz", StringComparison.Ordinal))
+				return await File.ReadAllTextAsync(file, Token);
+			await using var gzip = new GZipStream(File.OpenRead(file), CompressionMode.Decompress);
+			using var reader = new StreamReader(gzip);
+			return await reader.ReadToEndAsync(Token);
+		}
+
+		[Fact]
+		public async Task Media_rows_files_are_linked_from_the_live_directory_or_the_trash()
+		{
+			Directory.CreateDirectory(Path.Combine(Media, "2026", "10"));
+			await File.WriteAllTextAsync(Path.Combine(Media, "2026", "10", "live.jpg"), "live", Token);
+			await File.WriteAllTextAsync(Path.Combine(Media, "2026", "10", "live-preview.jpg"), "preview", Token);
+			Directory.CreateDirectory(Path.Combine(Trash, "2026", "10"));
+			await File.WriteAllTextAsync(Path.Combine(Trash, "2026", "10", "moving.jpg"), "moving", Token);
+			await File.WriteAllTextAsync(Path.Combine(Media, "2026", "10", "trashed.jpg"), "trashed", Token);
+			await _database.GetCollection<BsonDocument>("MediaAttachment").InsertManyAsync([
+				new BsonDocument { { "FilePath", "2026/10/live.jpg" }, { "PreviewPath", "2026/10/live-preview.jpg" }, { "TrashedAt", BsonNull.Value } },
+				new BsonDocument { { "FilePath", "2026/10/moving.jpg" } },//a file the janitor moved while its row was being trashed
+				new BsonDocument { { "FilePath", "2026/10/gone.jpg" } },
+				new BsonDocument { { "FilePath", "2026/10/trashed.jpg" }, { "TrashedAt", DateTime.UtcNow } },
+				new BsonDocument { { "FilePath", "../../etc/passwd" } }], cancellationToken: Token);
+
+			var (backup, _) = await ServerBackup.Create(Context(), "manual", dbOnly: false, Token);
+
+			var media = Path.Combine(Backups, backup.Id, "media");
+			Assert.Equal("live", await File.ReadAllTextAsync(Path.Combine(media, "2026", "10", "live.jpg"), Token));
+			Assert.Equal("preview", await File.ReadAllTextAsync(Path.Combine(media, "2026", "10", "live-preview.jpg"), Token));
+			Assert.Equal("moving", await File.ReadAllTextAsync(Path.Combine(media, "2026", "10", "moving.jpg"), Token));
+			Assert.False(File.Exists(Path.Combine(media, "2026", "10", "trashed.jpg")));
+			Assert.Equal(["2026/10/gone.jpg", "2026/10/live-preview.jpg", "2026/10/live.jpg", "2026/10/moving.jpg"], backup.Manifest.Media.List);
+			Assert.Equal((3, 1), (backup.Manifest.Media.Files, backup.Manifest.Media.Missing));
+
+			// a link, not a copy: the live file deleted, the backup's stays
+			File.Delete(Path.Combine(Media, "2026", "10", "live.jpg"));
+			Assert.Equal("live", await File.ReadAllTextAsync(Path.Combine(media, "2026", "10", "live.jpg"), Token));
+
+			// db-only lists them and links none
+			var (listed, _) = await ServerBackup.Create(Context(), "pre-deploy", dbOnly: true, Token);
+			Assert.False(Directory.Exists(Path.Combine(Backups, listed.Id, "media")));
+			Assert.Equal(backup.Manifest.Media.List, listed.Manifest.Media.List);
+			Assert.Empty(await ServerBackup.Verify(Backups, listed.Id, Token));
+		}
+
+		[Fact]
+		public async Task Verify_finds_an_altered_or_missing_file()
+		{
+			await _database.GetCollection<BsonDocument>("Post").InsertOneAsync(new BsonDocument("Content", "hello"), cancellationToken: Token);
+			await _database.GetCollection<BsonDocument>("Avatar").InsertOneAsync(new BsonDocument("UserName", "someone"), cancellationToken: Token);
+			var (backup, _) = await ServerBackup.Create(Context(), "manual", dbOnly: false, Token);
+			var db = Path.Combine(Backups, backup.Id, "db");
+
+			await File.AppendAllTextAsync(Path.Combine(db, "Post.jsonl.gz"), "tampered", Token);
+			File.Delete(Path.Combine(db, "Avatar.jsonl.gz"));
+
+			Assert.Equal(["Avatar: missing", "Post: altered"], (await ServerBackup.Verify(Backups, backup.Id, Token)).Order());
+			Assert.Equal(["no such backup, or no manifest"], await ServerBackup.Verify(Backups, "../" + backup.Id, Token));
+		}
+
+		[Fact]
+		public async Task One_backup_at_a_time()
+		{
+			await using (var held = await MaintenanceLock.Take("restore", Token))
+			{
+				Assert.NotNull(held);
+				var (backup, error) = await ServerBackup.Create(Context(), "manual", dbOnly: true, Token);
+				Assert.Null(backup);
+				Assert.Contains("already running", error);
+				Assert.Equal("restore", (await MaintenanceLock.Current(Token)).What);
+			}
+			Assert.Null(await MaintenanceLock.Current(Token));
+			Assert.NotNull((await ServerBackup.Create(Context(), "manual", dbOnly: true, Token)).Backup);
+		}
+
+		[Fact]
+		public async Task A_holder_that_died_is_taken_over()
+		{
+			await using var dead = await MaintenanceLock.Take("backup", Token);
+			await DB.Default.Update<MaintenanceLock>().Match(l => l.ID == MaintenanceLock.Name)
+				.Modify(l => l.Heartbeat, DateTime.UtcNow.AddMinutes(-3)).ExecuteAsync(Token);
+
+			Assert.Null(await MaintenanceLock.Current(Token));
+			await using var alive = await MaintenanceLock.Take("restore", Token);
+			Assert.NotNull(alive);
+			Assert.Equal("restore", (await MaintenanceLock.Current(Token)).What);
+		}
+
+		// the newest 7 nightly, the newest of each of the 4 weeks before, the newest 3 pre-deploy; manual ones kept
+		[Fact]
+		public void Rotation_keeps_days_weeks_and_the_last_deploys()
+		{
+			var today = new DateTime(2026, 10, 7, 3, 30, 0, DateTimeKind.Utc);
+			for (var day = 0; day < 60; day++)
+				Fake("nightly", today.AddDays(-day));
+			for (var deploy = 0; deploy < 5; deploy++)
+				Fake("pre-deploy", today.AddDays(-deploy).AddHours(5));
+			Fake("manual", today.AddYears(-1));
+			Directory.CreateDirectory(Path.Combine(Backups, "20260901-000000-nightly" + ServerBackup.PartialSuffix));
+			Directory.SetLastWriteTimeUtc(Path.Combine(Backups, "20260901-000000-nightly" + ServerBackup.PartialSuffix), today.AddDays(-30));
+
+			ServerBackup.Retain(Backups, new BackupOptions());
+
+			var kept = ServerBackup.List(Backups);
+			var nightly = kept.Where(b => b.Kind == "nightly").Select(b => b.CreatedAt).ToList();
+			Assert.Equal(11, nightly.Count);
+			Assert.Equal(Enumerable.Range(0, 7).Select(d => today.AddDays(-d)), nightly.Take(7));
+			Assert.Equal(4, nightly.Skip(7).Select(d => System.Globalization.ISOWeek.GetWeekOfYear(d)).Distinct().Count());
+			Assert.Equal(3, kept.Count(b => b.Kind == "pre-deploy"));
+			Assert.Single(kept, b => b.Kind == "manual");
+			Assert.Empty(Directory.EnumerateDirectories(Backups, "*" + ServerBackup.PartialSuffix));
+		}
+
+		void Fake(string kind, DateTime at)
+		{
+			var directory = Path.Combine(Backups, $"{at:yyyyMMdd-HHmmss}-{kind}");
+			Directory.CreateDirectory(Path.Combine(directory, "db"));
+			new ArchiveManifest { Kind = kind, CreatedAt = at }.Write(directory);
+		}
+
+		[Theory]
+		[InlineData("2026-10-07T03:29:00", "2026-10-06T03:31:00", false)]//yesterday's is the last one due
+		[InlineData("2026-10-07T03:31:00", "2026-10-06T03:31:00", true)]
+		[InlineData("2026-10-07T03:31:00", "2026-10-07T03:30:30", false)]
+		[InlineData("2026-10-07T01:00:00", "2026-10-05T03:31:00", true)]//missed last night: at once
+		public void A_nightly_backup_is_due_once_a_night(string now, string last, bool due) =>
+			Assert.Equal(due, BackupScheduler.IsDue(DateTime.Parse(now, null, System.Globalization.DateTimeStyles.AdjustToUniversal),
+				new TimeOnly(3, 30), [DateTime.Parse(last, null, System.Globalization.DateTimeStyles.AdjustToUniversal)]));
+
+		// the deploy's: the server's own database, through the configuration, without media links
+		[Fact]
+		public async Task The_deploy_backs_up_from_the_command_line()
+		{
+			var host = await PrivaPubHost.Shared();
+			var output = new StringWriter();
+
+			Assert.Equal(0, await AdminCommands.Run(["backup", "--kind", "pre-deploy", "--db-only"], host.Services, output: output));
+			var id = output.ToString().Split(':')[0];
+			Assert.EndsWith("-pre-deploy", id);
+			Assert.Equal(2, await AdminCommands.Run(["backup", "--kind", "nightly"], host.Services, output: TextWriter.Null));
+
+			output = new StringWriter();
+			Assert.Equal(0, await AdminCommands.Run(["backups"], host.Services, output: output));
+			Assert.StartsWith(id + "\tpre-deploy", output.ToString());
+			output = new StringWriter();
+			Assert.Equal(0, await AdminCommands.Run(["backup", "verify", id], host.Services, output: output));
+			Assert.Contains("whole", output.ToString());
+
+			var backups = host.Get<Backups>();
+			var manifest = backups.Find(id).Manifest;
+			Assert.Contains(manifest.Collections, c => c.Name == "Avatar");
+			Assert.DoesNotContain(manifest.Collections, c => c.Name is "InteractionSalt" or "Job" or "MaintenanceLock" or "openiddict.tokens" or "AppConfiguration");
+			Assert.Equal(ServerBackup.CodeMigration(), manifest.MigrationNumber);
+			Assert.True(backups.Delete(id));
+		}
+	}
+}

PrivaPub.Tests/Support/Host/PrivaPubHost.cs

@@ -24,7 +24,7 @@ namespace PrivaPub.Tests.Support.Host
 
 		static readonly SemaphoreSlim Boot = new(1, 1);
 		static readonly Type[] Unwanted = { typeof(JobWorker), typeof(MediaJanitor), typeof(OAuthPruner), typeof(StatisticsSchedule),
-			typeof(PrivaPub.Domain.Social.FollowResender) };
+			typeof(PrivaPub.Domain.Social.FollowResender), typeof(PrivaPub.Infrastructure.Backup.BackupScheduler) };
 		static PrivaPubHost _shared;
 
 		readonly string _mediaRoot = Path.Combine(Path.GetTempPath(), $"privapub-tests-{Guid.NewGuid():N}");
@@ -78,6 +78,8 @@ namespace PrivaPub.Tests.Support.Host
 			["Statistics:Geo:AutoUpdate"] = "false",
 			["Statistics:Cdn:AutoUpdate"] = "false",
 			["Media:Root"] = _mediaRoot,
+			["Backups:Root"] = _mediaRoot + "-backups",
+			["Backups:Nightly"] = "false",
 			["RateLimits:UploadsBurst"] = "1000",
 			["RateLimits:ProxyBurst"] = "100000",
 			["Logging:LogLevel:Default"] = "Warning",
@@ -117,8 +119,11 @@ namespace PrivaPub.Tests.Support.Host
 		protected override void Dispose(bool disposing)
 		{
 			base.Dispose(disposing);
-			if (disposing && Directory.Exists(_mediaRoot))
+			if (!disposing)
-				Directory.Delete(_mediaRoot, recursive: true);
+				return;
+			foreach (var directory in new[] { _mediaRoot, _mediaRoot + "-backups", _mediaRoot + "-trash", _mediaRoot + "-incoming", _mediaRoot + "-proxy" })
+				if (Directory.Exists(directory))
+					Directory.Delete(directory, recursive: true);
 		}
 
 		sealed class ClientAddressFilter : IStartupFilter

PrivaPub/Domain/Media/MediaProxy.cs

@@ -361,7 +361,8 @@ namespace PrivaPub.Domain.Media
 		// one pass:
 		// - uploads never posted for a day (and not waiting for a scheduled post, nor a profile picture) go to the trash;
 		// - trashed files still served (a crash between the mark and the move) are moved out;
-		// - trashed files past their grace are deleted, with their rows;
+		// - trashed files past their grace are deleted, with their rows, unless a backup or a restore is running (one may be
+		//   reading the trash);
 		// - the proxy cache is trimmed to its size, oldest first
 		public async Task Sweep(CancellationToken token)
 		{
@@ -371,10 +372,11 @@ namespace PrivaPub.Domain.Media
 
 			var trashed = await DB.Default.Find<MediaAttachment>().Match(m => m.TrashedAt != null).Limit(2000).ExecuteAsync(token);
 			var purgeBefore = DateTime.UtcNow - TrashGrace;
+			var maintenance = await Infrastructure.Backup.MaintenanceLock.Current(token) != default;
 			var purged = 0;
 			foreach (var row in trashed)
 			{
-				if (row.TrashedAt < purgeBefore)
+				if (row.TrashedAt < purgeBefore && !maintenance)
 				{
 					await _media.Purge(row, token);
 					purged++;

PrivaPub/Infrastructure/Backup/ArchiveManifest.cs

@@ -0,0 +1,65 @@
+using System.Text.Json;
+using System.Text.Json.Serialization;
+
+namespace PrivaPub.Infrastructure.Backup
+{
+	// What a backup holds (manifest.json at its root): enough to check it is whole, to restore it on this host only, and to
+	// rebuild what Mongo had (each collection's indexes).
+	public sealed class ArchiveManifest
+	{
+		public const int CurrentFormat = 1;
+		public const string FileName = "manifest.json";
+
+		public int Format { get; set; } = CurrentFormat;
+		public string Kind { get; set; }//nightly, manual, pre-deploy, pre-restore, uploaded
+		public DateTime CreatedAt { get; set; } = DateTime.UtcNow;
+		public string Host { get; set; }//BackendBaseAddress: URIs and media URLs are stored whole, so only this host can restore it
+		public string AppCommit { get; set; }
+		public string AppRef { get; set; }
+		public string NewestMigration { get; set; }
+		public int MigrationNumber { get; set; }
+		public bool Consistent { get; set; }//read at one instant (a snapshot session on a replica set)
+		public bool DbOnly { get; set; }//no media files, only their list (the deploy's, which can't link www-data's files)
+		public List<CollectionEntry> Collections { get; set; } = [];
+		public List<ExcludedEntry> Excluded { get; set; } = [];
+		public MediaEntry Media { get; set; } = new();
+
+		public sealed class CollectionEntry
+		{
+			public string Name { get; set; }
+			public long Count { get; set; }
+			public long Bytes { get; set; }
+			public string Sha256 { get; set; }
+			public List<string> Indexes { get; set; } = [];//each as canonical Extended JSON
+		}
+
+		public sealed class ExcludedEntry
+		{
+			public string Name { get; set; }
+			public string Why { get; set; }
+		}
+
+		public sealed class MediaEntry
+		{
+			public int Files { get; set; }
+			public long Bytes { get; set; }
+			public int Missing { get; set; }
+			public List<string> List { get; set; } = [];
+		}
+
+		static readonly JsonSerializerOptions Json = new() { WriteIndented = true, PropertyNamingPolicy = JsonNamingPolicy.CamelCase, DefaultIgnoreCondition = JsonIgnoreCondition.Never };
+
+		public static ArchiveManifest Read(string directory)
+		{
+			var path = Path.Combine(directory, FileName);
+			return File.Exists(path) ? JsonSerializer.Deserialize<ArchiveManifest>(File.ReadAllText(path), Json) : default;
+		}
+
+		public void Write(string directory)
+		{
+			using var file = new FileStream(Path.Combine(directory, FileName), FileMode.Create, FileAccess.Write);
+			JsonSerializer.Serialize(file, this, Json);
+			file.Flush(flushToDisk: true);
+		}
+	}
+}

PrivaPub/Infrastructure/Backup/BackupOptions.cs

@@ -0,0 +1,13 @@
+namespace PrivaPub.Infrastructure.Backup
+{
+	public class BackupOptions
+	{
+		public string Root { get; set; }//where backups are kept (/var/lib/privapub/backups); <media root>-backups by default
+		public string NightlyAt { get; set; } = "03:30";//UTC
+		public int KeepDaily { get; set; } = 7;
+		public int KeepWeekly { get; set; } = 4;
+		public int KeepPreDeploy { get; set; } = 3;
+		public int KeepPreRestore { get; set; } = 3;
+		public bool Nightly { get; set; } = true;
+	}
+}

PrivaPub/Infrastructure/Backup/Backups.cs

@@ -0,0 +1,89 @@
+using Microsoft.Extensions.Options;
+
+using MongoDB.Entities;
+
+using PrivaPub.Domain.Media;
+using PrivaPub.Models;
+
+namespace PrivaPub.Infrastructure.Backup
+{
+	// The server's backups, wherever they are started from: the CLI, the nightly schedule, the administrator's page.
+	public class Backups(IOptionsMonitor<BackupOptions> options, IOptionsMonitor<AppConfiguration> app, IMediaService media)
+	{
+		/// <summary>Where backups are kept: Backups:Root (production's /var/lib/privapub/backups), else beside the media root.</summary>
+		public string Root => Path.GetFullPath(options.CurrentValue.Root ?? media.Root.TrimEnd(Path.DirectorySeparatorChar) + "-backups");
+
+		public BackupOptions Options => options.CurrentValue;
+
+		public string Host => app.CurrentValue.BackendBaseAddress?.TrimEnd('/');
+
+		public BackupContext Context() => new(DB.Default.Database(), Root, media.Root, media.TrashRoot, Host, options.CurrentValue);
+
+		public Task<(BackupInfo Backup, string Error)> Create(string kind, bool dbOnly, CancellationToken token) =>
+			ServerBackup.Create(Context(), kind, dbOnly, token);
+
+		public List<BackupInfo> List() => ServerBackup.List(Root);
+
+		public BackupInfo Find(string id) => ServerBackup.Find(Root, id);
+
+		public Task<List<string>> Verify(string id, CancellationToken token) => ServerBackup.Verify(Root, id, token);
+
+		public bool Delete(string id) => ServerBackup.Delete(Root, id);
+	}
+
+	// A nightly backup at Backups:NightlyAt (UTC), or as soon as the service is up after missing it; the old ones rotated
+	// out as each is made.
+	public class BackupScheduler(Backups backups, ILogger<BackupScheduler> logger) : BackgroundService
+	{
+		static readonly TimeSpan Interval = TimeSpan.FromMinutes(10);
+
+		protected override async Task ExecuteAsync(CancellationToken stoppingToken)
+		{
+			while (!stoppingToken.IsCancellationRequested)
+			{
+				try
+				{
+					await Task.Delay(Interval, stoppingToken);
+				}
+				catch (OperationCanceledException)
+				{
+					return;
+				}
+				try
+				{
+					await RunIfDue(DateTime.UtcNow, stoppingToken);
+				}
+				catch (Exception ex) when (ex is not OperationCanceledException)
+				{
+					logger.LogError(ex, "The nightly backup failed");
+				}
+			}
+		}
+
+		/// <summary>Backs up when the day's time has come and there is no nightly backup since: whether it did.</summary>
+		public async Task<bool> RunIfDue(DateTime now, CancellationToken token)
+		{
+			var options = backups.Options;
+			if (!options.Nightly || !TimeOnly.TryParse(options.NightlyAt, System.Globalization.CultureInfo.InvariantCulture, out var at)
+				|| !IsDue(now, at, backups.List().Where(b => b.Kind == "nightly").Select(b => b.CreatedAt)))
+				return false;
+			var (made, error) = await backups.Create("nightly", dbOnly: false, token);
+			if (made == default)
+			{
+				logger.LogWarning("The nightly backup was not made: {Error}", error);
+				return false;
+			}
+			logger.LogInformation("Nightly backup {Id}: {Collections} collections, {Files} media files", made.Id, made.Manifest.Collections.Count, made.Manifest.Media.Files);
+			return true;
+		}
+
+		/// <summary>Whether the last time of day for a nightly backup has passed with no nightly backup made since.</summary>
+		public static bool IsDue(DateTime now, TimeOnly at, IEnumerable<DateTime> nightlies)
+		{
+			var due = now.Date + at.ToTimeSpan();
+			if (now < due)
+				due = due.AddDays(-1);
+			return !nightlies.Any(made => made >= due);
+		}
+	}
+}

PrivaPub/Infrastructure/Backup/HardLink.cs

@@ -0,0 +1,35 @@
+using System.Runtime.InteropServices;
+
+namespace PrivaPub.Infrastructure.Backup
+{
+	// A second name for a file (.NET has no API for it): a backup links the live media, which costs no disk, since names are
+	// random and files never change; deleting the live one leaves the backup's.
+	static class HardLink
+	{
+		// strings go to libc as UTF-8 on Unix
+		[DllImport("libc", EntryPoint = "link", SetLastError = true, CharSet = CharSet.Ansi)]
+		static extern int Link(string existing, string created);
+
+		/// <summary>Links, or copies where a link can't be made (another disk, a filesystem refusing it).</summary>
+		public static bool LinkOrCopy(string existing, string created)
+		{
+			Directory.CreateDirectory(Path.GetDirectoryName(created)!);
+			if (OperatingSystem.IsLinux())
+			{
+				try
+				{
+					if (Link(existing, created) == 0)
+						return true;
+				}
+				catch (DllNotFoundException)
+				{
+				}
+				catch (EntryPointNotFoundException)
+				{
+				}
+			}
+			File.Copy(existing, created, overwrite: true);
+			return false;
+		}
+	}
+}

PrivaPub/Infrastructure/Backup/MaintenanceLock.cs

@@ -0,0 +1,75 @@
+using MongoDB.Driver;
+using MongoDB.Entities;
+
+namespace PrivaPub.Infrastructure.Backup
+{
+	// One backup or restore at a time, whoever starts it (the CLI, the nightly schedule, the administrator's page): a
+	// document held with a heartbeat. A holder that died (no heartbeat for two minutes) is taken over. The collection is
+	// never backed up, nor dropped by a restore.
+	public class MaintenanceLock : Entity
+	{
+		public const string Name = "maintenance";
+		static readonly TimeSpan Stale = TimeSpan.FromMinutes(2);
+		static readonly TimeSpan Beat = TimeSpan.FromSeconds(30);
+
+		public string Owner { get; set; }
+		public string What { get; set; }
+		public DateTime Since { get; set; }
+		public DateTime Heartbeat { get; set; }
+
+		/// <summary>Takes the lock for what is said, or null when someone else holds it; disposing the result frees it.</summary>
+		public static async Task<Held> Take(string what, CancellationToken token)
+		{
+			var owner = $"{Environment.MachineName}:{Environment.ProcessId}:{Guid.NewGuid():N}";
+			var now = DateTime.UtcNow;
+			var stale = now - Stale;
+			try
+			{
+				var taken = await DB.Default.UpdateAndGet<MaintenanceLock>()
+					.Match(l => l.ID == Name && (l.Heartbeat < stale || l.Owner == null))
+					.Modify(l => l.Owner, owner)
+					.Modify(l => l.What, what)
+					.Modify(l => l.Since, now)
+					.Modify(l => l.Heartbeat, now)
+					.Option(o => o.IsUpsert = true)
+					.ExecuteAsync(token);
+				return taken?.Owner == owner ? new Held(owner) : default;
+			}
+			catch (MongoCommandException ex) when (ex.Code == 11000)
+			{
+				return default;//held: the upsert met the live lock's id
+			}
+			catch (MongoWriteException ex) when (ex.WriteError?.Category == ServerErrorCategory.DuplicateKey)
+			{
+				return default;
+			}
+		}
+
+		/// <summary>Who holds it now and for what, or null.</summary>
+		public static async Task<MaintenanceLock> Current(CancellationToken token)
+		{
+			var held = await DB.Default.Find<MaintenanceLock>().Match(l => l.ID == Name).ExecuteFirstAsync(token);
+			return held?.Owner != null && held.Heartbeat >= DateTime.UtcNow - Stale ? held : default;
+		}
+
+		public sealed class Held : IAsyncDisposable
+		{
+			readonly string _owner;
+			readonly Timer _heartbeat;
+
+			public Held(string owner)
+			{
+				_owner = owner;
+				_heartbeat = new Timer(_ => _ = DB.Default.Update<MaintenanceLock>().Match(l => l.ID == Name && l.Owner == owner)
+					.Modify(l => l.Heartbeat, DateTime.UtcNow).ExecuteAsync(), default, Beat, Beat);
+			}
+
+			public async ValueTask DisposeAsync()
+			{
+				await _heartbeat.DisposeAsync();
+				await DB.Default.Update<MaintenanceLock>().Match(l => l.ID == Name && l.Owner == _owner)
+					.Modify(l => l.Owner, null).Modify(l => l.Heartbeat, DateTime.MinValue).ExecuteAsync();
+			}
+		}
+	}
+}

PrivaPub/Infrastructure/Backup/RestoreMarker.cs

@@ -0,0 +1,56 @@
+using System.Text.Json;
+
+namespace PrivaPub.Infrastructure.Backup
+{
+	// A restore asked for (by the administrator's page or the CLI) and carried out at the next boot, before anything
+	// else (Program, MaintenanceGate): restore.json in the backups' directory, written whole or not at all.
+	public sealed class RestoreMarker
+	{
+		public const string FileName = "restore.json";
+		public const int MaxAttempts = 3;
+
+		public string Backup { get; set; }//the backup restored
+		public string PreRestore { get; set; }//the backup taken just before, what the protective merge reads
+		public string State { get; set; } = "pending";//pending, then running
+		public int Attempts { get; set; }
+		public DateTime RequestedAt { get; set; } = DateTime.UtcNow;
+		public string Error { get; set; }
+
+		public static string PathIn(string backupsRoot) => Path.Combine(backupsRoot, FileName);
+
+		public static RestoreMarker Read(string backupsRoot)
+		{
+			var path = PathIn(backupsRoot);
+			if (!File.Exists(path))
+				return default;
+			try
+			{
+				return JsonSerializer.Deserialize<RestoreMarker>(File.ReadAllText(path));
+			}
+			catch (JsonException)
+			{
+				return default;
+			}
+		}
+
+		public void Write(string backupsRoot)
+		{
+			Directory.CreateDirectory(backupsRoot);
+			var path = PathIn(backupsRoot);
+			var part = path + ".part";
+			using (var file = new FileStream(part, FileMode.Create, FileAccess.Write))
+			{
+				JsonSerializer.Serialize(file, this);
+				file.Flush(flushToDisk: true);
+			}
+			File.Move(part, path, overwrite: true);
+		}
+
+		public static void Clear(string backupsRoot)
+		{
+			var path = PathIn(backupsRoot);
+			if (File.Exists(path))
+				File.Delete(path);
+		}
+	}
+}

PrivaPub/Infrastructure/Backup/ServerBackup.cs

@@ -0,0 +1,340 @@
+using MongoDB.Bson;
+using MongoDB.Bson.IO;
+using MongoDB.Driver;
+
+using PrivaPub.Infrastructure.Data;
+
+using System.Globalization;
+using System.IO.Compression;
+using System.Security.Cryptography;
+using System.Text;
+
+namespace PrivaPub.Infrastructure.Backup
+{
+	// What a backup needs to know: where things are, and whose host it is.
+	public sealed record BackupContext(IMongoDatabase Database, string BackupsRoot, string MediaRoot, string TrashRoot, string Host, BackupOptions Options);
+
+	public sealed record BackupInfo(string Id, string Kind, DateTime CreatedAt, long Bytes, ArchiveManifest Manifest);
+
+	// The whole server, backed up as files (owner decision 2026-10-07: a whole-server backup, plain, protected by file
+	// permissions). Each backup is a directory <stamp>-<kind> in the backups' root:
+	// - manifest.json: what it holds (ArchiveManifest);
+	// - db/<collection>.jsonl.gz: every document of each collection, one per line, in canonical Extended JSON (every BSON
+	//   type kept as it was), all read at one instant when Mongo is a replica set (a snapshot session);
+	// - media/<path>: the media files rows hold, as hard links to the live ones (no disk spent; a deleted file stays here
+	//   until the backup is rotated out), copied where a link can't be made; a db-only backup lists them instead.
+	// It is written as <id>.partial and renamed once whole. Left out: what belongs to the moment (Excluded). Everything is
+	// readable by the service's user and group only: it holds every persona's private key and private posts.
+	public static class ServerBackup
+	{
+		public const string PartialSuffix = ".partial";
+
+		public static readonly IReadOnlyDictionary<string, string> Excluded = new Dictionary<string, string>
+		{
+			["InteractionSalt"] = "the day's statistics salt never outlives its day (owner rule)",
+			["Job"] = "work in flight: deliveries and inbox processing belong to the moment",
+			["Delivery"] = "work in flight, from before jobs",
+			["EmailRecovery"] = "recovery codes live an hour",
+			["openiddict.tokens"] = "sessions: a restore ends every one",
+			["openiddict.authorizations"] = "sessions: a restore ends every one",
+			[nameof(MaintenanceLock)] = "who is backing up or restoring now",
+			["AppConfiguration"] = "the configuration's copy holds the SMTP password, and is made again from appsettings at boot"
+		};
+
+		static readonly JsonWriterSettings Json = new() { OutputMode = JsonOutputMode.CanonicalExtendedJson, Indent = false };
+
+		// 2770: the service (www-data) and the deploy (in www-data's group) each read and rotate what the other wrote, and new
+		// files take the directory's group
+		const UnixFileMode DirectoryMode = UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute
+			| UnixFileMode.GroupRead | UnixFileMode.GroupWrite | UnixFileMode.GroupExecute | UnixFileMode.SetGroup;
+		const UnixFileMode FileMode0640 = UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.GroupRead;
+
+		/// <summary>Takes the maintenance lock and backs the server up: the backup, or why not.</summary>
+		public static async Task<(BackupInfo Backup, string Error)> Create(BackupContext context, string kind, bool dbOnly, CancellationToken token)
+		{
+			await using var held = await MaintenanceLock.Take("backup", token);
+			if (held == default)
+				return (default, "a backup or a restore is already running");
+			return await CreateHeld(context, kind, dbOnly, token);
+		}
+
+		/// <summary>Backs the server up with the maintenance lock already held (a restore's own backup, taken first).</summary>
+		public static async Task<(BackupInfo Backup, string Error)> CreateHeld(BackupContext context, string kind, bool dbOnly, CancellationToken token)
+		{
+			var database = context.Database;
+			var names = (await (await database.ListCollectionNamesAsync(cancellationToken: token)).ToListAsync(token))
+				.Where(n => !n.StartsWith("system.", StringComparison.Ordinal))
+				.OrderBy(n => n, StringComparer.Ordinal)
+				.ToList();
+			MakeDirectory(context.BackupsRoot);
+			var stats = await database.RunCommandAsync<BsonDocument>(new BsonDocument("dbStats", 1), cancellationToken: token);
+			var needed = (long)(stats.GetValue("dataSize", 0).ToDouble() * 1.1);
+			if (new DriveInfo(Path.GetFullPath(context.BackupsRoot)).AvailableFreeSpace < needed)
+				return (default, $"not enough free disk for a backup: about {needed / 1024 / 1024} MB needed");
+
+			var id = $"{DateTime.UtcNow.ToString("yyyyMMdd-HHmmss", CultureInfo.InvariantCulture)}-{kind}";
+			var partial = Path.Combine(context.BackupsRoot, id + PartialSuffix);
+			MakeDirectory(partial);
+			MakeDirectory(Path.Combine(partial, "db"));
+			try
+			{
+				var manifest = new ArchiveManifest
+				{
+					Kind = kind,
+					Host = context.Host,
+					AppCommit = BuildInfo.Commit,
+					AppRef = BuildInfo.Ref,
+					DbOnly = dbOnly
+				};
+				var replica = await MongoTopology.IsReplicaSet(database, token);
+				var media = new SortedSet<string>(StringComparer.Ordinal);
+				using var session = replica ? await database.Client.StartSessionAsync(new ClientSessionOptions { Snapshot = true }, token) : default;
+				var window = replica ? await RaiseSnapshotWindow(database, token) : default(int?);
+				try
+				{
+					foreach (var name in names)
+					{
+						if (Excluded.TryGetValue(name, out var why))
+						{
+							manifest.Excluded.Add(new ArchiveManifest.ExcludedEntry { Name = name, Why = why });
+							continue;
+						}
+						manifest.Collections.Add(await Dump(database, session, name, partial, name == "MediaAttachment" ? media : default, token));
+					}
+					(manifest.NewestMigration, manifest.MigrationNumber) = await NewestMigration(database, session, token);
+				}
+				finally
+				{
+					if (window is { } previous)
+						await SetSnapshotWindow(database, previous, CancellationToken.None);
+				}
+				manifest.Consistent = replica;
+
+				manifest.Media.List = [.. media];
+				if (!dbOnly)
+					foreach (var relative in media)
+					{
+						var source = new[] { context.MediaRoot, context.TrashRoot }.Select(root => Inside(root, relative)).FirstOrDefault(File.Exists);
+						if (source == default)
+						{
+							manifest.Media.Missing++;
+							continue;
+						}
+						HardLink.LinkOrCopy(source, Inside(Path.Combine(partial, "media"), relative));
+						manifest.Media.Files++;
+						manifest.Media.Bytes += new FileInfo(source).Length;
+					}
+
+				manifest.Write(partial);
+				Directory.Move(partial, Path.Combine(context.BackupsRoot, id));
+				Retain(context.BackupsRoot, context.Options);
+				return (Info(context.BackupsRoot, id), default);
+			}
+			catch
+			{
+				if (Directory.Exists(partial))
+					Directory.Delete(partial, recursive: true);
+				throw;
+			}
+		}
+
+		// a collection read as raw BSON in batches, each document a line of canonical Extended JSON, gzipped; the media rows'
+		// files collected on the way
+		static async Task<ArchiveManifest.CollectionEntry> Dump(IMongoDatabase database, IClientSessionHandle session, string name, string directory,
+			ISet<string> media, CancellationToken token)
+		{
+			var collection = database.GetCollection<RawBsonDocument>(name);
+			var path = Path.Combine(directory, "db", name + ".jsonl.gz");
+			var count = 0L;
+			await using (var file = NewFile(path))
+			await using (var gzip = new GZipStream(file, CompressionLevel.Fastest))
+			await using (var writer = new StreamWriter(gzip, new UTF8Encoding(false)))
+			{
+				var options = new FindOptions<RawBsonDocument> { BatchSize = 1000 };
+				using var cursor = session == default
+					? await collection.FindAsync(FilterDefinition<RawBsonDocument>.Empty, options, token)
+					: await collection.FindAsync(session, FilterDefinition<RawBsonDocument>.Empty, options, token);
+				while (await cursor.MoveNextAsync(token))
+					foreach (var document in cursor.Current)
+						using (document)
+						{
+							await writer.WriteLineAsync(document.ToJson(Json));
+							count++;
+							if (media != default)
+								Collect(document, media);
+						}
+			}
+			var indexes = await (await collection.Indexes.ListAsync(token)).ToListAsync(token);
+			return new ArchiveManifest.CollectionEntry
+			{
+				Name = name,
+				Count = count,
+				Bytes = new FileInfo(path).Length,
+				Sha256 = await Hash(path, token),
+				Indexes = [.. indexes.Select(i => i.ToJson(Json))]
+			};
+		}
+
+		// a media row's files, unless it is trashed
+		static void Collect(RawBsonDocument row, ISet<string> media)
+		{
+			if (row.TryGetValue("TrashedAt", out var trashed) && !trashed.IsBsonNull)
+				return;
+			foreach (var field in new[] { "FilePath", "PreviewPath" })
+				if (row.TryGetValue(field, out var value) && value.IsString && Safe(value.AsString))
+					media.Add(value.AsString);
+		}
+
+		// MongoDB.Entities records each migration run with its class's number (_016_... is 16) and its name in words
+		static async Task<(string Name, int Number)> NewestMigration(IMongoDatabase database, IClientSessionHandle session, CancellationToken token)
+		{
+			var history = database.GetCollection<BsonDocument>("_migration_history_");
+			var options = new FindOptions<BsonDocument> { Sort = new BsonDocument("Number", -1), Limit = 1 };
+			var newest = session == default
+				? await (await history.FindAsync(FilterDefinition<BsonDocument>.Empty, options, token)).FirstOrDefaultAsync(token)
+				: await (await history.FindAsync(session, FilterDefinition<BsonDocument>.Empty, options, token)).FirstOrDefaultAsync(token);
+			return newest == default ? default : (newest.GetValue("Name", BsonNull.Value).ToString(), newest.GetValue("Number", 0).ToInt32());
+		}
+
+		/// <summary>The newest migration this build has: a backup made by a newer one can't be restored by it.</summary>
+		public static int CodeMigration() => typeof(ServerBackup).Assembly.GetTypes()
+			.Where(t => typeof(MongoDB.Entities.IMigration).IsAssignableFrom(t) && !t.IsAbstract)
+			.Select(t => int.TryParse(new string(t.Name.TrimStart('_').TakeWhile(char.IsDigit).ToArray()), out var number) ? number : 0)
+			.DefaultIfEmpty(0)
+			.Max();
+
+		// how long a snapshot stays readable: raised only while a backup reads (a longer window keeps old versions in the
+		// small cache all day); the value it had, to set back
+		static async Task<int?> RaiseSnapshotWindow(IMongoDatabase database, CancellationToken token)
+		{
+			var admin = database.Client.GetDatabase("admin");
+			try
+			{
+				var current = await admin.RunCommandAsync<BsonDocument>(new BsonDocument { { "getParameter", 1 }, { "minSnapshotHistoryWindowInSeconds", 1 } }, cancellationToken: token);
+				var previous = current.GetValue("minSnapshotHistoryWindowInSeconds", 300).ToInt32();
+				await SetSnapshotWindow(database, Math.Max(previous, 3600), token);
+				return previous;
+			}
+			catch (MongoCommandException)
+			{
+				return default;
+			}
+		}
+
+		static async Task SetSnapshotWindow(IMongoDatabase database, int seconds, CancellationToken token) =>
+			await database.Client.GetDatabase("admin").RunCommandAsync<BsonDocument>(
+				new BsonDocument { { "setParameter", 1 }, { "minSnapshotHistoryWindowInSeconds", seconds } }, cancellationToken: token);
+
+		/// <summary>The backups kept, newest first (not one still being written).</summary>
+		public static List<BackupInfo> List(string backupsRoot)
+		{
+			if (!Directory.Exists(backupsRoot))
+				return [];
+			return Directory.EnumerateDirectories(backupsRoot)
+				.Select(Path.GetFileName)
+				.Where(name => !name.EndsWith(PartialSuffix, StringComparison.Ordinal) && File.Exists(Path.Combine(backupsRoot, name, ArchiveManifest.FileName)))
+				.Select(name => Info(backupsRoot, name))
+				.OrderByDescending(b => b.CreatedAt)
+				.ToList();
+		}
+
+		public static BackupInfo Find(string backupsRoot, string id) =>
+			Safe(id) && !id.Contains('/') && Directory.Exists(Path.Combine(backupsRoot, id)) && !id.EndsWith(PartialSuffix, StringComparison.Ordinal)
+				? Info(backupsRoot, id)
+				: default;
+
+		static BackupInfo Info(string backupsRoot, string id)
+		{
+			var directory = Path.Combine(backupsRoot, id);
+			var manifest = ArchiveManifest.Read(directory);
+			var bytes = Directory.EnumerateFiles(Path.Combine(directory, "db")).Sum(f => new FileInfo(f).Length);
+			return new BackupInfo(id, manifest?.Kind, manifest?.CreatedAt ?? Directory.GetCreationTimeUtc(directory), bytes, manifest);
+		}
+
+		/// <summary>Whether every file of a backup is what its manifest says: the problems found (none when whole).</summary>
+		public static async Task<List<string>> Verify(string backupsRoot, string id, CancellationToken token)
+		{
+			var problems = new List<string>();
+			var backup = Find(backupsRoot, id);
+			if (backup?.Manifest == default)
+				return ["no such backup, or no manifest"];
+			var directory = Path.Combine(backupsRoot, id);
+			foreach (var collection in backup.Manifest.Collections)
+			{
+				var path = Path.Combine(directory, "db", collection.Name + ".jsonl.gz");
+				if (!File.Exists(path))
+					problems.Add($"{collection.Name}: missing");
+				else if (await Hash(path, token) != collection.Sha256)
+					problems.Add($"{collection.Name}: altered");
+			}
+			if (!backup.Manifest.DbOnly)
+				foreach (var relative in backup.Manifest.Media.List.Where(r => !File.Exists(Inside(Path.Combine(directory, "media"), r))))
+					problems.Add($"media {relative}: missing");
+			return problems;
+		}
+
+		/// <summary>Deletes a backup (its media links go; the live files stay).</summary>
+		public static bool Delete(string backupsRoot, string id)
+		{
+			if (Find(backupsRoot, id) == default)
+				return false;
+			Directory.Delete(Path.Combine(backupsRoot, id), recursive: true);
+			return true;
+		}
+
+		// what is kept: the newest nightly ones for KeepDaily days and the newest of each of KeepWeekly weeks before, the
+		// newest pre-deploy and pre-restore ones; manual and uploaded ones until deleted; a backup that died writing goes
+		public static void Retain(string backupsRoot, BackupOptions options)
+		{
+			var all = List(backupsRoot);
+			var nightly = all.Where(b => b.Kind == "nightly").OrderByDescending(b => b.CreatedAt).ToList();
+			var kept = nightly.Take(options.KeepDaily).ToHashSet();
+			foreach (var week in nightly.Skip(options.KeepDaily).GroupBy(b => (ISOWeek.GetYear(b.CreatedAt), ISOWeek.GetWeekOfYear(b.CreatedAt))).Take(options.KeepWeekly))
+				kept.Add(week.First());
+			var doomed = nightly.Where(b => !kept.Contains(b))
+				.Concat(all.Where(b => b.Kind == "pre-deploy").OrderByDescending(b => b.CreatedAt).Skip(options.KeepPreDeploy))
+				.Concat(all.Where(b => b.Kind == "pre-restore").OrderByDescending(b => b.CreatedAt).Skip(options.KeepPreRestore));
+			var stale = Directory.EnumerateDirectories(backupsRoot, "*" + PartialSuffix).Where(p => Directory.GetLastWriteTimeUtc(p) < DateTime.UtcNow.AddDays(-1));
+			foreach (var directory in doomed.Select(b => Path.Combine(backupsRoot, b.Id)).Concat(stale).ToList())
+				try
+				{
+					Directory.Delete(directory, recursive: true);
+				}
+				catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
+				{
+					//another user's backup this one may not delete: the next rotation tries again
+				}
+		}
+
+		// a relative path from a database row or a manifest, never outside its root
+		static bool Safe(string relative) =>
+			!string.IsNullOrEmpty(relative) && !Path.IsPathRooted(relative) && !relative.Split('/', '\\').Any(part => part is ".." or ".");
+
+		public static string Inside(string root, string relative)
+		{
+			var full = Path.GetFullPath(Path.Combine(root, relative));
+			if (!Safe(relative) || !full.StartsWith(Path.GetFullPath(root) + Path.DirectorySeparatorChar, StringComparison.Ordinal))
+				throw new InvalidOperationException($"{relative} is not inside {root}");
+			return full;
+		}
+
+		static async Task<string> Hash(string path, CancellationToken token)
+		{
+			await using var file = File.OpenRead(path);
+			return Convert.ToHexStringLower(await SHA256.HashDataAsync(file, token));
+		}
+
+		static FileStream NewFile(string path) => OperatingSystem.IsWindows()
+			? new FileStream(path, FileMode.CreateNew, FileAccess.Write)
+			: new FileStream(path, new FileStreamOptions { Mode = FileMode.CreateNew, Access = FileAccess.Write, UnixCreateMode = FileMode0640 });
+
+		// set after creating, since the umask would take the group's write away
+		public static void MakeDirectory(string path)
+		{
+			var existed = Directory.Exists(path);
+			Directory.CreateDirectory(path);
+			if (!existed && !OperatingSystem.IsWindows())
+				File.SetUnixFileMode(path, DirectoryMode);
+		}
+	}
+}

PrivaPub/Infrastructure/Cli/AdminCommands.cs

@@ -24,8 +24,16 @@ namespace PrivaPub.Infrastructure.Cli
 			       PrivaPub admin smoke <persona>                 prints "<login> <password>" for the deploy's signed-in check
 			       PrivaPub admin media audit [--fix]             media files against what holds them; --fix (as www-data) trashes
 			                                                      what nothing holds and gives today's pictures their rows
+			       PrivaPub admin backup [--kind manual|pre-deploy] [--db-only]
+			                                                      backs the server up; --db-only lists the media without
+			                                                      linking them (the deploy's, which can't link www-data's files)
+			       PrivaPub admin backups                         the backups kept, newest first
+			       PrivaPub admin backup verify <id>              whether a backup's files are what its manifest says
 			""";
 
+		/// <summary>Whether a command runs before migrations: backups are of the database as it was.</summary>
+		public static bool BeforeMigrations(string[] args) => args is ["backup", ..] or ["backups"];
+
 		public static async Task<int> Run(string[] args, IServiceProvider services, TextReader input = default, TextWriter output = default)
 		{
 			input ??= Console.In;
@@ -40,12 +48,75 @@ namespace PrivaPub.Infrastructure.Cli
 					return await Smoke(services, persona, output);
 				case ["media", "audit", .. var flags] when flags.All(f => f == "--fix"):
 					return await AuditMedia(services, flags.Contains("--fix"), output);
+				case ["backup", "verify", var id]:
+					return await VerifyBackup(services, id, output);
+				case ["backup", .. var flags] when BackupFlags(flags, out var kind, out var dbOnly):
+					return await Backup(services, kind, dbOnly, output);
+				case ["backups"]:
+					return ListBackups(services, output);
 				default:
 					Console.Error.WriteLine(Usage);
 					return 2;
 			}
 		}
 
+		static readonly string[] CommandKinds = ["manual", "pre-deploy"];
+
+		static bool BackupFlags(string[] flags, out string kind, out bool dbOnly)
+		{
+			kind = "manual";
+			dbOnly = false;
+			for (var i = 0; i < flags.Length; i++)
+				switch (flags[i])
+				{
+					case "--db-only":
+						dbOnly = true;
+						break;
+					case "--kind" when i + 1 < flags.Length && CommandKinds.Contains(flags[i + 1]):
+						kind = flags[++i];
+						break;
+					default:
+						return false;
+				}
+			return true;
+		}
+
+		static async Task<int> Backup(IServiceProvider services, string kind, bool dbOnly, TextWriter output)
+		{
+			var (made, error) = await services.GetRequiredService<Backup.Backups>().Create(kind, dbOnly, CancellationToken.None);
+			if (made == default)
+			{
+				Console.Error.WriteLine(error);
+				return 1;
+			}
+			var manifest = made.Manifest;
+			output.WriteLine($"{made.Id}: {manifest.Collections.Count} collections, {manifest.Collections.Sum(c => c.Count)} documents, {made.Bytes / 1024} KiB");
+			output.WriteLine(dbOnly
+				? $"{manifest.Media.List.Count} media files listed, not linked"
+				: $"{manifest.Media.Files} media files, {manifest.Media.Bytes / 1024 / 1024} MiB, {manifest.Media.Missing} missing");
+			if (!manifest.Consistent)
+				output.WriteLine("not read at one instant: Mongo is not a replica set");
+			return 0;
+		}
+
+		static int ListBackups(IServiceProvider services, TextWriter output)
+		{
+			foreach (var backup in services.GetRequiredService<Backup.Backups>().List())
+				output.WriteLine($"{backup.Id}\t{backup.Kind}\t{backup.CreatedAt:u}\t{backup.Bytes / 1024} KiB\t{backup.Manifest?.Media.Files ?? 0} media{(backup.Manifest?.DbOnly == true ? " (listed)" : string.Empty)}");
+			return 0;
+		}
+
+		static async Task<int> VerifyBackup(IServiceProvider services, string id, TextWriter output)
+		{
+			var problems = await services.GetRequiredService<Backup.Backups>().Verify(id, CancellationToken.None);
+			foreach (var problem in problems)
+				output.WriteLine(problem);
+			if (problems.Count > 0)
+				return 1;
+			output.WriteLine($"{id}: whole");
+			return 0;
+		}
+
 		static async Task<int> AuditMedia(IServiceProvider services, bool fix, TextWriter output)
 		{
 			var report = await Domain.Media.MediaAudit.Run(services.GetRequiredService<Domain.Media.IMediaService>(), fix, CancellationToken.None);

PrivaPub/Infrastructure/Data/EntityMaps.cs

@@ -4,7 +4,22 @@ namespace PrivaPub.Infrastructure.Data
 {
 	public static class EntityMaps
 	{
+		static readonly Lock Gate = new();
+		static bool _warm;
+
+		// once per process, one map at a time: test hosts boot side by side
 		public static void Warm()
+		{
+			lock (Gate)
+			{
+				if (_warm)
+					return;
+				Map();
+				_warm = true;
+			}
+		}
+
+		static void Map()
 		{
 			var collection = typeof(DB).GetMethods()
 				.Single(m => m.Name == nameof(DB.Collection) && m.IsGenericMethodDefinition && m.GetParameters().Length == 0);

PrivaPub/Program.cs

@@ -68,6 +68,9 @@ try
 			.AddSingleton<PrivaPub.Domain.Media.IMediaService, PrivaPub.Domain.Media.MediaService>()
 			.AddSingleton<PrivaPub.Domain.Media.IMediaProxy, PrivaPub.Domain.Media.MediaProxy>()
 			.AddHostedService<PrivaPub.Domain.Media.MediaJanitor>()
+			.Configure<PrivaPub.Infrastructure.Backup.BackupOptions>(builder.Configuration.GetSection("Backups"))
+			.AddSingleton<PrivaPub.Infrastructure.Backup.Backups>()
+			.AddHostedService<PrivaPub.Infrastructure.Backup.BackupScheduler>()
 			.PrivaPubMiddlewareConfiguration();
 	}
 	catch (Exception ex)
@@ -86,8 +89,6 @@ try
 		var mongoSettings = builder.Configuration.GetSection(nameof(MongoSettings)).Get<MongoSettings>();
 		await DB.InitAsync(mongoSettings.Database, MongoClientSettings.FromConnectionString(mongoSettings.ConnectionString));
 		EntityMaps.Warm();
-		await DB.Default.MigrateAsync<Program>();
-		await Indexes.Create();
 	}
 	catch (Exception ex)
 	{
@@ -108,6 +109,25 @@ try
 
 	// Commands get every service but start nothing: no Kestrel, no hosted services, no media directory. The deploy runs
 	// them as its own user, which can read the configuration and reach the private mongod but owns no www-data directory.
+	// Backups are taken before migrations: the deploy's is of the database as the live build left it.
+	if (args is ["admin", .. var command] && AdminCommands.BeforeMigrations(command))
+	{
+		using var scope = app.Services.CreateScope();
+		Environment.ExitCode = await AdminCommands.Run(command, scope.ServiceProvider);
+		return;
+	}
+
+	try
+	{
+		await DB.Default.MigrateAsync<Program>();
+		await Indexes.Create();
+	}
+	catch (Exception ex)
+	{
+		Log.ForContext<Program>().Fatal(ex, $"{nameof(Program)}.{nameof(Program)}() DB Migrations");
+		throw;
+	}
+
 	if (args is ["admin", ..])
 	{
 		using var scope = app.Services.CreateScope();

PrivaPub/appsettings.Development.json

@@ -4,6 +4,9 @@
     "LogsDatabase": "logs",
     "ConnectionString": "mongodb://localhost:27017"
   },
+  "Backups": {
+    "Nightly": false
+  },
   "AppConfiguration": {
     "RequiresFirstTimeSetup": null,
     "Version": "0.0.0",

PrivaPub/appsettings.Production.json

@@ -2,6 +2,9 @@
 	"Media": {
 		"Root": "/var/lib/privapub/media"
 	},
+	"Backups": {
+		"Root": "/var/lib/privapub/backups"
+	},
 	"Registrations": {
 		"Mode": "Invitations"
 	},

deploy/max/setup.sh

@@ -13,7 +13,9 @@ echo "== directories"
 install -d -o "$RUNNER" -g www-data -m 755 /var/www/$HOST
 install -d -o "$RUNNER" -g "$RUNNER" -m 750 /var/backups/$HOST
 install -d -o www-data -g www-data -m 750 /var/lib/privapub /var/lib/privapub/mongo
-# backups: the service writes them, and so does the deploy (as $RUNNER, a member of www-data) before each deploy
+# backups: the service writes them, and so does the deploy (as $RUNNER, a member of www-data) before each deploy; a runner
+# just added to the group gets it when the runner restarts
+id -nG "$RUNNER" | grep -qw www-data || usermod -aG www-data "$RUNNER"
 install -d -o www-data -g www-data -m 2770 /var/lib/privapub/backups
 
 echo "== sudoers"

tools/pasture/appsettings.Pasture.json

@@ -26,6 +26,7 @@
 		"Relays": [ "https://relay.test/actor", "https://aoderelay.test/actor" ]
 	},
 	"Media": { "Root": "/tmp/privapub-media" },
+	"Backups": { "Root": "/tmp/privapub-backups", "Nightly": false },
 	"RateLimits": { "AccountsPerMinute": 1000, "UploadsBurst": 1000, "UploadsPerMinute": 1000, "ProxyBurst": 100000, "ProxyPerMinute": 100000 },
 	"Registrations": { "Mode": "Open" },
 	"Statistics": { "Geo": { "AutoUpdate": false } },