From 12bb75809f495f9a385ceb92929934ff003bc695 Mon Sep 17 00:00:00 2001 From: thepra Date: Wed, 7 Oct 2026 11:40:06 +0200 Subject: [PATCH] The server backs itself up: every collection byte for byte, the media linked A backup is a directory - under Backups:Root (/var/lib/privapub/backups, 2770, files 0640), written as .partial and renamed once whole: a manifest (host, build, newest migration, each collection's count, size, sha256 and indexes, what was left out and why, the media list), each collection as gzipped canonical Extended JSON read raw, and hard links to the files of untrashed media rows (copies where a link can't be made). On a replica set every collection is read in one snapshot session. Never in a backup: the statistics salt, jobs, recovery codes, sessions, the maintenance lock and the configuration's copy with its SMTP password. One backup or restore at a time (MaintenanceLock, a heartbeat document), and the janitor purges nothing meanwhile. BackupScheduler backs up nightly at 03:30 UTC (or at once after missing a night); rotation keeps 7 daily, 4 weekly, 3 pre-deploy and 3 pre-restore backups. CLI: admin backup [--kind] [--db-only], admin backups, admin backup verify; these run before migrations, so the deploy's own pre-deploy backup, which replaces mongodump, is of the database as the live build left it. EntityMaps.Warm runs once under a lock, since test hosts now boot side by side. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01LsXgEaXee4GCU1hwYgPJXw --- .gitea/workflows/deploy.yml | 28 +- .gitignore | 4 + CLAUDE.md | 25 +- PrivaPub.Tests/Infrastructure/BackupTests.cs | 286 +++++++++++++++ PrivaPub.Tests/Support/Host/PrivaPubHost.cs | 11 +- PrivaPub/Domain/Media/MediaProxy.cs | 6 +- .../Infrastructure/Backup/ArchiveManifest.cs | 65 ++++ .../Infrastructure/Backup/BackupOptions.cs | 13 + PrivaPub/Infrastructure/Backup/Backups.cs | 89 +++++ PrivaPub/Infrastructure/Backup/HardLink.cs | 35 ++ .../Infrastructure/Backup/MaintenanceLock.cs | 75 ++++ .../Infrastructure/Backup/RestoreMarker.cs | 56 +++ .../Infrastructure/Backup/ServerBackup.cs | 340 ++++++++++++++++++ PrivaPub/Infrastructure/Cli/AdminCommands.cs | 71 ++++ PrivaPub/Infrastructure/Data/EntityMaps.cs | 15 + PrivaPub/Program.cs | 24 +- PrivaPub/appsettings.Development.json | 3 + PrivaPub/appsettings.Production.json | 3 + deploy/max/setup.sh | 4 +- tools/pasture/appsettings.Pasture.json | 1 + 20 files changed, 1132 insertions(+), 22 deletions(-) create mode 100644 PrivaPub.Tests/Infrastructure/BackupTests.cs create mode 100644 PrivaPub/Infrastructure/Backup/ArchiveManifest.cs create mode 100644 PrivaPub/Infrastructure/Backup/BackupOptions.cs create mode 100644 PrivaPub/Infrastructure/Backup/Backups.cs create mode 100644 PrivaPub/Infrastructure/Backup/HardLink.cs create mode 100644 PrivaPub/Infrastructure/Backup/MaintenanceLock.cs create mode 100644 PrivaPub/Infrastructure/Backup/RestoreMarker.cs create mode 100644 PrivaPub/Infrastructure/Backup/ServerBackup.cs diff --git a/.gitea/workflows/deploy.yml b/.gitea/workflows/deploy.yml index 4b6828f..b819dc5 100644 --- a/.gitea/workflows/deploy.yml +++ b/.gitea/workflows/deploy.yml @@ -12,8 +12,7 @@ env: BACKUPS: /var/backups/privapub.thepra.dev LOCAL_URL: http://127.0.0.1:6970 PUBLIC_URL: https://privapub.thepra.dev - MONGO_URI: mongodb://127.0.0.1:27022/?directConnection=true - MONGO_DB: PrivaPub + SERVER_BACKUPS: /var/lib/privapub/backups jobs: site: @@ -57,19 +56,24 @@ jobs: echo "SNAPSHOT=$BACKUPS/site-$STAMP" >> "$GITHUB_ENV" ls -1dt "$BACKUPS"/site-* 2>/dev/null | tail -n +4 | xargs -r rm -rf || true - # without the statistics salt, which must not outlive its day (owner rule: it is destroyed at each rollup, and a dump - # kept for days would let the day's actor hashes be reversed) - - name: Dump the database + # The server's own backup (PrivaPub admin backup, owner decision 2026-10-07) of the database as the live build left it: + # the new build's command runs before its migrations. The media are listed, not linked (the runner may not link + # www-data's files; the nightly backups hold them). Never the statistics salt, which must not outlive its day (owner + # rule), nor the configuration's copy with its secrets. No deploy while a restore waits for its boot. + - name: Back up the database run: | - DUMP="$BACKUPS/mongo-$(date +%Y%m%d-%H%M%S).archive.gz" - mongodump --quiet --uri "$MONGO_URI" --db "$MONGO_DB" --excludeCollection=InteractionSalt --gzip --archive="$DUMP" - if mongorestore --gzip --archive="$DUMP" --dryRun -v 2>&1 | grep -q "InteractionSalt"; then - rm -f "$DUMP" - echo "::error::the dump holds the statistics salt" + [ ! -e "$SERVER_BACKUPS/restore.json" ] || { echo "::error::a restore is pending or running ($SERVER_BACKUPS/restore.json): deploy after it"; exit 1; } + out=$(cd "$GITHUB_WORKSPACE/publish" && ASPNETCORE_ENVIRONMENT=Production ./PrivaPub admin backup --kind pre-deploy --db-only) + echo "$out" + id=$(echo "$out" | grep -oE '^[0-9]{8}-[0-9]{6}-pre-deploy' | tail -1) + [ -d "$SERVER_BACKUPS/$id" ] || { echo "::error::the backup was not made"; exit 1; } + (cd "$GITHUB_WORKSPACE/publish" && ASPNETCORE_ENVIRONMENT=Production ./PrivaPub admin backup verify "$id") + if [ -e "$SERVER_BACKUPS/$id/db/InteractionSalt.jsonl.gz" ]; then + echo "::error::the backup holds the statistics salt" exit 1 fi - echo "::notice::database dumped to $DUMP ($(du -h "$DUMP" | cut -f1))" - ls -1t "$BACKUPS"/mongo-*.archive.gz 2>/dev/null | tail -n +8 | xargs -r rm -f || true + echo "BACKUP_ID=$id" >> "$GITHUB_ENV" + echo "::notice::database backed up as $id" - name: Stop, sync, start run: | diff --git a/.gitignore b/.gitignore index b038b6f..4479eae 100644 --- a/.gitignore +++ b/.gitignore @@ -257,6 +257,7 @@ Generated_Code/ # because we have git ;-) _UpgradeReport_Files/ Backup*/ +!PrivaPub/Infrastructure/Backup/ UpgradeLog*.XML UpgradeLog*.htm ServiceFabricBackup/ @@ -401,6 +402,9 @@ FodyWeavers.xsd PrivaPub/media-store/ PrivaPub/media-store-proxy/ +PrivaPub/media-store-trash/ +PrivaPub/media-store-incoming/ +PrivaPub/media-store-backups/ tools/pasture/.publish/ tools/pasture/.flood/ .claude/worktrees/ diff --git a/CLAUDE.md b/CLAUDE.md index a6b2c18..3112b29 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -484,6 +484,26 @@ group www-data and reaches the private mongod; `sudo -u www-data` works too. (`--replSet rs0`, a 990 MB oplog; owner decision 2026-10-07), so a backup reads every collection at one instant; `setup.sh` converts it once (PrivaPub stopped, mongod restarted, `rs.initiate`), and every connection string says `directConnection=true`, which also works against a standalone. The pasture's mongo is one too. +- **Backups** (`Infrastructure/Backup/`; owner decisions 2026-10-07: the whole server, plain files protected by their + permissions, run from the CLI, nightly and from the administrator's page). Each backup is a directory + `-` under `Backups:Root` (`/var/lib/privapub/backups`, www-data 2770, files 0640), written as + `.partial` and renamed once whole: + - `manifest.json` (`ArchiveManifest`): host, build, newest migration, whether it was read at one instant, each + collection's count, size, sha256 and indexes, what was left out and why, and the media list; + - `db/.jsonl.gz`: each document as one line of canonical Extended JSON, read raw, so every BSON type comes + back byte for byte; on a replica set every collection is read in one snapshot session + (`minSnapshotHistoryWindowInSeconds` is raised to an hour only while it reads); + - `media/`: hard links to the files of untrashed `MediaAttachment` rows (no disk spent; a deleted file stays until the + backup rotates out), copies where a link can't be made. `--db-only` lists them instead. + + **Never in a backup** (`ServerBackup.Excluded`): `InteractionSalt` (owner rule), `Job` and `Delivery` (work in flight), + `EmailRecovery`, `openiddict.tokens` and `.authorizations` (sessions), `MaintenanceLock`, and `AppConfiguration` + (its SMTP password; it is made again from appsettings at boot). One backup or restore runs at a time + (`MaintenanceLock`, a heartbeat document; a holder silent for 2 minutes is taken over), and the media janitor purges + nothing meanwhile. `BackupScheduler` backs up at `Backups:NightlyAt` (03:30 UTC), or at once when it missed the night; + rotation keeps 7 daily and 4 weekly nightly backups, 3 pre-deploy, 3 pre-restore, and manual and uploaded ones until + deleted. CLI: `PrivaPub admin backup [--kind manual|pre-deploy] [--db-only]`, `admin backups`, `admin backup verify + `; these run before migrations, so the deploy's backup is of the database as the live build left it. ## Code style @@ -840,8 +860,9 @@ the owner's GoToSocial account.** ## Deploy - **CI/CD:** push to `master` runs `build.yml` (build + tests) on the instance-wide `build` runner. A `v*` tag runs - `deploy.yml`: tests, self-contained linux-x64 publish, snapshot and `mongodump` to `/var/backups/privapub.thepra.dev` - (never the statistics salt: `InteractionSalt` is excluded and the dump is checked for it), + `deploy.yml`: tests, self-contained linux-x64 publish, a snapshot of the site to `/var/backups/privapub.thepra.dev`, + the server's own pre-deploy backup (`PrivaPub admin backup --kind pre-deploy --db-only`, run from the new build before + its migrations, verified, and checked to hold no statistics salt; no deploy while `restore.json` waits), stop → rsync → start, a `127.0.0.1:6970/build.json` health loop with rollback, then public checks (actor, NodeInfo, Swagger 404, inbox junk 400, unsigned 401) and `tools/smoke/mastodon-api.sh` (app registration, client credentials, discovery, instance, public timeline). Then it checks that what production should be running is running: diff --git a/PrivaPub.Tests/Infrastructure/BackupTests.cs b/PrivaPub.Tests/Infrastructure/BackupTests.cs new file mode 100644 index 0000000..f779d28 --- /dev/null +++ b/PrivaPub.Tests/Infrastructure/BackupTests.cs @@ -0,0 +1,286 @@ +using MongoDB.Bson; +using MongoDB.Bson.IO; +using MongoDB.Driver; +using MongoDB.Entities; + +using PrivaPub.Infrastructure.Backup; +using PrivaPub.Infrastructure.Cli; +using PrivaPub.Tests.Support; +using PrivaPub.Tests.Support.Host; + +using System.IO.Compression; + +namespace PrivaPub.Tests.Infrastructure +{ + // The server's backup: every collection as it was, byte for byte, read at one instant on a replica set, without what + // belongs to the moment (the statistics salt above all), with the media rows' files linked, checkable and rotated. + // Each test backs up a database of its own; alone, since the maintenance lock is the server's one. + [Trait("Category", "Integration")] + [Xunit.Collection(nameof(Exclusive))] + public sealed class BackupTests : IAsyncLifetime + { + readonly string _scratch = Path.Combine(Path.GetTempPath(), $"privapub-backup-tests-{Guid.NewGuid():N}"); + IMongoDatabase _database; + + string Backups => Path.Combine(_scratch, "backups"); + string Media => Path.Combine(_scratch, "media"); + string Trash => Path.Combine(_scratch, "media-trash"); + + public async ValueTask InitializeAsync() + { + Assert.SkipUnless(MongoFixture.Enabled, MongoFixture.Skip); + await PrivaPubHost.Shared(); + _database = DB.Default.Database().Client.GetDatabase($"PrivaPubBackup_{Guid.NewGuid():N}"); + Directory.CreateDirectory(Media); + Directory.CreateDirectory(Trash); + } + + public async ValueTask DisposeAsync() + { + if (_database != default) + await _database.Client.DropDatabaseAsync(_database.DatabaseNamespace.DatabaseName); + if (Directory.Exists(_scratch)) + Directory.Delete(_scratch, recursive: true); + } + + BackupContext Context(BackupOptions options = default) => new(_database, Backups, Media, Trash, "https://privapub.test", options ?? new BackupOptions()); + + static CancellationToken Token => TestContext.Current.CancellationToken; + + static List Read(string file) + { + using var gzip = new GZipStream(File.OpenRead(file), CompressionMode.Decompress); + using var reader = new StreamReader(gzip); + var documents = new List(); + while (reader.ReadLine() is { } line) + documents.Add(BsonDocument.Parse(line)); + return documents; + } + + async Task> Raw(string collection) => + (await (await _database.GetCollection(collection).FindAsync(FilterDefinition.Empty, cancellationToken: Token)).ToListAsync(Token)) + .Select(d => + { + var bytes = new byte[d.Slice.Length]; + d.Slice.GetBytes(0, bytes, 0, bytes.Length); + return bytes; + }) + .ToList(); + + [Fact] + public async Task Every_document_comes_back_byte_for_byte_and_the_salt_never_leaves() + { + var odd = new BsonDocument + { + { "_id", ObjectId.GenerateNewId() }, + { "Guid", new BsonBinaryData(Guid.NewGuid(), GuidRepresentation.Standard) }, + { "OldGuid", new BsonBinaryData(new byte[16], BsonBinarySubType.UuidLegacy) }, + { "Money", new BsonDecimal128(Decimal128.Parse("12345.678900")) }, + { "Long", new BsonInt64(long.MaxValue) }, + { "Int", 7 }, + { "Double", -0.0 }, + { "NaN", double.NaN }, + { "At", new BsonDateTime(new DateTime(2026, 10, 7, 3, 30, 0, 123, DateTimeKind.Utc)) }, + { "Stamp", new BsonTimestamp(1, 2) }, + { "Null", BsonNull.Value }, + { "Regex", new BsonRegularExpression("^a.b$", "i") }, + { "Nested", new BsonDocument { { "z", 1 }, { "a", new BsonArray { 1, "two", new BsonDocument("three", 3) } } } }, + { "Unicode", "città 🌍 \u0000 end" } + }; + // an ObjectRecord as big as a remote's long post gets + var big = new BsonDocument { { "_id", ObjectId.GenerateNewId() }, { "Json", new string('x', 10 * 1024 * 1024) } }; + await _database.GetCollection("Odd").InsertOneAsync(odd, cancellationToken: Token); + await _database.GetCollection("ObjectRecord").InsertOneAsync(big, cancellationToken: Token); + await _database.GetCollection("Odd").Indexes.CreateOneAsync( + new CreateIndexModel(Builders.IndexKeys.Ascending("At"), new CreateIndexOptions { Name = "at", Unique = true }), cancellationToken: Token); + await _database.GetCollection("InteractionSalt").InsertOneAsync(new BsonDocument("Salt", "never in a backup"), cancellationToken: Token); + await _database.GetCollection("Job").InsertOneAsync(new BsonDocument("Kind", "Deliver"), cancellationToken: Token); + await _database.GetCollection("_migration_history_").InsertManyAsync([ + new BsonDocument { { "Number", 15 }, { "Name", "older" } }, new BsonDocument { { "Number", 16 }, { "Name", "focal points are finite" } }], cancellationToken: Token); + + var (backup, error) = await ServerBackup.Create(Context(), "manual", dbOnly: false, Token); + + Assert.Null(error); + var directory = Path.Combine(Backups, backup.Id); + Assert.False(Directory.Exists(directory + ServerBackup.PartialSuffix)); + Assert.Equal(await Raw("Odd"), Read(Path.Combine(directory, "db", "Odd.jsonl.gz")).Select(d => d.ToBson()).ToList()); + Assert.Equal(await Raw("ObjectRecord"), Read(Path.Combine(directory, "db", "ObjectRecord.jsonl.gz")).Select(d => d.ToBson()).ToList()); + + Assert.False(File.Exists(Path.Combine(directory, "db", "InteractionSalt.jsonl.gz"))); + Assert.False(File.Exists(Path.Combine(directory, "db", "Job.jsonl.gz"))); + foreach (var file in Directory.EnumerateFiles(directory, "*", SearchOption.AllDirectories)) + Assert.DoesNotContain("never in a backup", await ReadAll(file)); + Assert.Contains(backup.Manifest.Excluded, e => e.Name == "InteractionSalt"); + + var odds = backup.Manifest.Collections.Single(c => c.Name == "Odd"); + Assert.Equal(1, odds.Count); + Assert.Contains(odds.Indexes, i => BsonDocument.Parse(i)["name"] == "at" && BsonDocument.Parse(i)["unique"].ToBoolean()); + Assert.Equal((16, "focal points are finite"), (backup.Manifest.MigrationNumber, backup.Manifest.NewestMigration)); + Assert.Equal("https://privapub.test", backup.Manifest.Host); + Assert.Equal(MongoFixture.Connection.Contains("directConnection=true"), backup.Manifest.Consistent); + Assert.Empty(await ServerBackup.Verify(Backups, backup.Id, Token)); + if (!OperatingSystem.IsWindows()) + { + Assert.Equal(UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.GroupRead, File.GetUnixFileMode(Path.Combine(directory, "db", "Odd.jsonl.gz"))); + Assert.False(File.GetUnixFileMode(directory).HasFlag(UnixFileMode.OtherRead)); + } + } + + static async Task ReadAll(string file) + { + if (!file.EndsWith(".gz", StringComparison.Ordinal)) + return await File.ReadAllTextAsync(file, Token); + await using var gzip = new GZipStream(File.OpenRead(file), CompressionMode.Decompress); + using var reader = new StreamReader(gzip); + return await reader.ReadToEndAsync(Token); + } + + [Fact] + public async Task Media_rows_files_are_linked_from_the_live_directory_or_the_trash() + { + Directory.CreateDirectory(Path.Combine(Media, "2026", "10")); + await File.WriteAllTextAsync(Path.Combine(Media, "2026", "10", "live.jpg"), "live", Token); + await File.WriteAllTextAsync(Path.Combine(Media, "2026", "10", "live-preview.jpg"), "preview", Token); + Directory.CreateDirectory(Path.Combine(Trash, "2026", "10")); + await File.WriteAllTextAsync(Path.Combine(Trash, "2026", "10", "moving.jpg"), "moving", Token); + await File.WriteAllTextAsync(Path.Combine(Media, "2026", "10", "trashed.jpg"), "trashed", Token); + await _database.GetCollection("MediaAttachment").InsertManyAsync([ + new BsonDocument { { "FilePath", "2026/10/live.jpg" }, { "PreviewPath", "2026/10/live-preview.jpg" }, { "TrashedAt", BsonNull.Value } }, + new BsonDocument { { "FilePath", "2026/10/moving.jpg" } },//a file the janitor moved while its row was being trashed + new BsonDocument { { "FilePath", "2026/10/gone.jpg" } }, + new BsonDocument { { "FilePath", "2026/10/trashed.jpg" }, { "TrashedAt", DateTime.UtcNow } }, + new BsonDocument { { "FilePath", "../../etc/passwd" } }], cancellationToken: Token); + + var (backup, _) = await ServerBackup.Create(Context(), "manual", dbOnly: false, Token); + + var media = Path.Combine(Backups, backup.Id, "media"); + Assert.Equal("live", await File.ReadAllTextAsync(Path.Combine(media, "2026", "10", "live.jpg"), Token)); + Assert.Equal("preview", await File.ReadAllTextAsync(Path.Combine(media, "2026", "10", "live-preview.jpg"), Token)); + Assert.Equal("moving", await File.ReadAllTextAsync(Path.Combine(media, "2026", "10", "moving.jpg"), Token)); + Assert.False(File.Exists(Path.Combine(media, "2026", "10", "trashed.jpg"))); + Assert.Equal(["2026/10/gone.jpg", "2026/10/live-preview.jpg", "2026/10/live.jpg", "2026/10/moving.jpg"], backup.Manifest.Media.List); + Assert.Equal((3, 1), (backup.Manifest.Media.Files, backup.Manifest.Media.Missing)); + + // a link, not a copy: the live file deleted, the backup's stays + File.Delete(Path.Combine(Media, "2026", "10", "live.jpg")); + Assert.Equal("live", await File.ReadAllTextAsync(Path.Combine(media, "2026", "10", "live.jpg"), Token)); + + // db-only lists them and links none + var (listed, _) = await ServerBackup.Create(Context(), "pre-deploy", dbOnly: true, Token); + Assert.False(Directory.Exists(Path.Combine(Backups, listed.Id, "media"))); + Assert.Equal(backup.Manifest.Media.List, listed.Manifest.Media.List); + Assert.Empty(await ServerBackup.Verify(Backups, listed.Id, Token)); + } + + [Fact] + public async Task Verify_finds_an_altered_or_missing_file() + { + await _database.GetCollection("Post").InsertOneAsync(new BsonDocument("Content", "hello"), cancellationToken: Token); + await _database.GetCollection("Avatar").InsertOneAsync(new BsonDocument("UserName", "someone"), cancellationToken: Token); + var (backup, _) = await ServerBackup.Create(Context(), "manual", dbOnly: false, Token); + var db = Path.Combine(Backups, backup.Id, "db"); + + await File.AppendAllTextAsync(Path.Combine(db, "Post.jsonl.gz"), "tampered", Token); + File.Delete(Path.Combine(db, "Avatar.jsonl.gz")); + + Assert.Equal(["Avatar: missing", "Post: altered"], (await ServerBackup.Verify(Backups, backup.Id, Token)).Order()); + Assert.Equal(["no such backup, or no manifest"], await ServerBackup.Verify(Backups, "../" + backup.Id, Token)); + } + + [Fact] + public async Task One_backup_at_a_time() + { + await using (var held = await MaintenanceLock.Take("restore", Token)) + { + Assert.NotNull(held); + var (backup, error) = await ServerBackup.Create(Context(), "manual", dbOnly: true, Token); + Assert.Null(backup); + Assert.Contains("already running", error); + Assert.Equal("restore", (await MaintenanceLock.Current(Token)).What); + } + Assert.Null(await MaintenanceLock.Current(Token)); + Assert.NotNull((await ServerBackup.Create(Context(), "manual", dbOnly: true, Token)).Backup); + } + + [Fact] + public async Task A_holder_that_died_is_taken_over() + { + await using var dead = await MaintenanceLock.Take("backup", Token); + await DB.Default.Update().Match(l => l.ID == MaintenanceLock.Name) + .Modify(l => l.Heartbeat, DateTime.UtcNow.AddMinutes(-3)).ExecuteAsync(Token); + + Assert.Null(await MaintenanceLock.Current(Token)); + await using var alive = await MaintenanceLock.Take("restore", Token); + Assert.NotNull(alive); + Assert.Equal("restore", (await MaintenanceLock.Current(Token)).What); + } + + // the newest 7 nightly, the newest of each of the 4 weeks before, the newest 3 pre-deploy; manual ones kept + [Fact] + public void Rotation_keeps_days_weeks_and_the_last_deploys() + { + var today = new DateTime(2026, 10, 7, 3, 30, 0, DateTimeKind.Utc); + for (var day = 0; day < 60; day++) + Fake("nightly", today.AddDays(-day)); + for (var deploy = 0; deploy < 5; deploy++) + Fake("pre-deploy", today.AddDays(-deploy).AddHours(5)); + Fake("manual", today.AddYears(-1)); + Directory.CreateDirectory(Path.Combine(Backups, "20260901-000000-nightly" + ServerBackup.PartialSuffix)); + Directory.SetLastWriteTimeUtc(Path.Combine(Backups, "20260901-000000-nightly" + ServerBackup.PartialSuffix), today.AddDays(-30)); + + ServerBackup.Retain(Backups, new BackupOptions()); + + var kept = ServerBackup.List(Backups); + var nightly = kept.Where(b => b.Kind == "nightly").Select(b => b.CreatedAt).ToList(); + Assert.Equal(11, nightly.Count); + Assert.Equal(Enumerable.Range(0, 7).Select(d => today.AddDays(-d)), nightly.Take(7)); + Assert.Equal(4, nightly.Skip(7).Select(d => System.Globalization.ISOWeek.GetWeekOfYear(d)).Distinct().Count()); + Assert.Equal(3, kept.Count(b => b.Kind == "pre-deploy")); + Assert.Single(kept, b => b.Kind == "manual"); + Assert.Empty(Directory.EnumerateDirectories(Backups, "*" + ServerBackup.PartialSuffix)); + } + + void Fake(string kind, DateTime at) + { + var directory = Path.Combine(Backups, $"{at:yyyyMMdd-HHmmss}-{kind}"); + Directory.CreateDirectory(Path.Combine(directory, "db")); + new ArchiveManifest { Kind = kind, CreatedAt = at }.Write(directory); + } + + [Theory] + [InlineData("2026-10-07T03:29:00", "2026-10-06T03:31:00", false)]//yesterday's is the last one due + [InlineData("2026-10-07T03:31:00", "2026-10-06T03:31:00", true)] + [InlineData("2026-10-07T03:31:00", "2026-10-07T03:30:30", false)] + [InlineData("2026-10-07T01:00:00", "2026-10-05T03:31:00", true)]//missed last night: at once + public void A_nightly_backup_is_due_once_a_night(string now, string last, bool due) => + Assert.Equal(due, BackupScheduler.IsDue(DateTime.Parse(now, null, System.Globalization.DateTimeStyles.AdjustToUniversal), + new TimeOnly(3, 30), [DateTime.Parse(last, null, System.Globalization.DateTimeStyles.AdjustToUniversal)])); + + // the deploy's: the server's own database, through the configuration, without media links + [Fact] + public async Task The_deploy_backs_up_from_the_command_line() + { + var host = await PrivaPubHost.Shared(); + var output = new StringWriter(); + + Assert.Equal(0, await AdminCommands.Run(["backup", "--kind", "pre-deploy", "--db-only"], host.Services, output: output)); + var id = output.ToString().Split(':')[0]; + Assert.EndsWith("-pre-deploy", id); + Assert.Equal(2, await AdminCommands.Run(["backup", "--kind", "nightly"], host.Services, output: TextWriter.Null)); + + output = new StringWriter(); + Assert.Equal(0, await AdminCommands.Run(["backups"], host.Services, output: output)); + Assert.StartsWith(id + "\tpre-deploy", output.ToString()); + output = new StringWriter(); + Assert.Equal(0, await AdminCommands.Run(["backup", "verify", id], host.Services, output: output)); + Assert.Contains("whole", output.ToString()); + + var backups = host.Get(); + var manifest = backups.Find(id).Manifest; + Assert.Contains(manifest.Collections, c => c.Name == "Avatar"); + Assert.DoesNotContain(manifest.Collections, c => c.Name is "InteractionSalt" or "Job" or "MaintenanceLock" or "openiddict.tokens" or "AppConfiguration"); + Assert.Equal(ServerBackup.CodeMigration(), manifest.MigrationNumber); + Assert.True(backups.Delete(id)); + } + } +} diff --git a/PrivaPub.Tests/Support/Host/PrivaPubHost.cs b/PrivaPub.Tests/Support/Host/PrivaPubHost.cs index a6e55f4..9873700 100644 --- a/PrivaPub.Tests/Support/Host/PrivaPubHost.cs +++ b/PrivaPub.Tests/Support/Host/PrivaPubHost.cs @@ -24,7 +24,7 @@ namespace PrivaPub.Tests.Support.Host static readonly SemaphoreSlim Boot = new(1, 1); static readonly Type[] Unwanted = { typeof(JobWorker), typeof(MediaJanitor), typeof(OAuthPruner), typeof(StatisticsSchedule), - typeof(PrivaPub.Domain.Social.FollowResender) }; + typeof(PrivaPub.Domain.Social.FollowResender), typeof(PrivaPub.Infrastructure.Backup.BackupScheduler) }; static PrivaPubHost _shared; readonly string _mediaRoot = Path.Combine(Path.GetTempPath(), $"privapub-tests-{Guid.NewGuid():N}"); @@ -78,6 +78,8 @@ namespace PrivaPub.Tests.Support.Host ["Statistics:Geo:AutoUpdate"] = "false", ["Statistics:Cdn:AutoUpdate"] = "false", ["Media:Root"] = _mediaRoot, + ["Backups:Root"] = _mediaRoot + "-backups", + ["Backups:Nightly"] = "false", ["RateLimits:UploadsBurst"] = "1000", ["RateLimits:ProxyBurst"] = "100000", ["Logging:LogLevel:Default"] = "Warning", @@ -117,8 +119,11 @@ namespace PrivaPub.Tests.Support.Host protected override void Dispose(bool disposing) { base.Dispose(disposing); - if (disposing && Directory.Exists(_mediaRoot)) - Directory.Delete(_mediaRoot, recursive: true); + if (!disposing) + return; + foreach (var directory in new[] { _mediaRoot, _mediaRoot + "-backups", _mediaRoot + "-trash", _mediaRoot + "-incoming", _mediaRoot + "-proxy" }) + if (Directory.Exists(directory)) + Directory.Delete(directory, recursive: true); } sealed class ClientAddressFilter : IStartupFilter diff --git a/PrivaPub/Domain/Media/MediaProxy.cs b/PrivaPub/Domain/Media/MediaProxy.cs index 263c3b5..b911065 100644 --- a/PrivaPub/Domain/Media/MediaProxy.cs +++ b/PrivaPub/Domain/Media/MediaProxy.cs @@ -361,7 +361,8 @@ namespace PrivaPub.Domain.Media // one pass: // - uploads never posted for a day (and not waiting for a scheduled post, nor a profile picture) go to the trash; // - trashed files still served (a crash between the mark and the move) are moved out; - // - trashed files past their grace are deleted, with their rows; + // - trashed files past their grace are deleted, with their rows, unless a backup or a restore is running (one may be + // reading the trash); // - the proxy cache is trimmed to its size, oldest first public async Task Sweep(CancellationToken token) { @@ -371,10 +372,11 @@ namespace PrivaPub.Domain.Media var trashed = await DB.Default.Find().Match(m => m.TrashedAt != null).Limit(2000).ExecuteAsync(token); var purgeBefore = DateTime.UtcNow - TrashGrace; + var maintenance = await Infrastructure.Backup.MaintenanceLock.Current(token) != default; var purged = 0; foreach (var row in trashed) { - if (row.TrashedAt < purgeBefore) + if (row.TrashedAt < purgeBefore && !maintenance) { await _media.Purge(row, token); purged++; diff --git a/PrivaPub/Infrastructure/Backup/ArchiveManifest.cs b/PrivaPub/Infrastructure/Backup/ArchiveManifest.cs new file mode 100644 index 0000000..cbc2b42 --- /dev/null +++ b/PrivaPub/Infrastructure/Backup/ArchiveManifest.cs @@ -0,0 +1,65 @@ +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace PrivaPub.Infrastructure.Backup +{ + // What a backup holds (manifest.json at its root): enough to check it is whole, to restore it on this host only, and to + // rebuild what Mongo had (each collection's indexes). + public sealed class ArchiveManifest + { + public const int CurrentFormat = 1; + public const string FileName = "manifest.json"; + + public int Format { get; set; } = CurrentFormat; + public string Kind { get; set; }//nightly, manual, pre-deploy, pre-restore, uploaded + public DateTime CreatedAt { get; set; } = DateTime.UtcNow; + public string Host { get; set; }//BackendBaseAddress: URIs and media URLs are stored whole, so only this host can restore it + public string AppCommit { get; set; } + public string AppRef { get; set; } + public string NewestMigration { get; set; } + public int MigrationNumber { get; set; } + public bool Consistent { get; set; }//read at one instant (a snapshot session on a replica set) + public bool DbOnly { get; set; }//no media files, only their list (the deploy's, which can't link www-data's files) + public List Collections { get; set; } = []; + public List Excluded { get; set; } = []; + public MediaEntry Media { get; set; } = new(); + + public sealed class CollectionEntry + { + public string Name { get; set; } + public long Count { get; set; } + public long Bytes { get; set; } + public string Sha256 { get; set; } + public List Indexes { get; set; } = [];//each as canonical Extended JSON + } + + public sealed class ExcludedEntry + { + public string Name { get; set; } + public string Why { get; set; } + } + + public sealed class MediaEntry + { + public int Files { get; set; } + public long Bytes { get; set; } + public int Missing { get; set; } + public List List { get; set; } = []; + } + + static readonly JsonSerializerOptions Json = new() { WriteIndented = true, PropertyNamingPolicy = JsonNamingPolicy.CamelCase, DefaultIgnoreCondition = JsonIgnoreCondition.Never }; + + public static ArchiveManifest Read(string directory) + { + var path = Path.Combine(directory, FileName); + return File.Exists(path) ? JsonSerializer.Deserialize(File.ReadAllText(path), Json) : default; + } + + public void Write(string directory) + { + using var file = new FileStream(Path.Combine(directory, FileName), FileMode.Create, FileAccess.Write); + JsonSerializer.Serialize(file, this, Json); + file.Flush(flushToDisk: true); + } + } +} diff --git a/PrivaPub/Infrastructure/Backup/BackupOptions.cs b/PrivaPub/Infrastructure/Backup/BackupOptions.cs new file mode 100644 index 0000000..3522858 --- /dev/null +++ b/PrivaPub/Infrastructure/Backup/BackupOptions.cs @@ -0,0 +1,13 @@ +namespace PrivaPub.Infrastructure.Backup +{ + public class BackupOptions + { + public string Root { get; set; }//where backups are kept (/var/lib/privapub/backups); -backups by default + public string NightlyAt { get; set; } = "03:30";//UTC + public int KeepDaily { get; set; } = 7; + public int KeepWeekly { get; set; } = 4; + public int KeepPreDeploy { get; set; } = 3; + public int KeepPreRestore { get; set; } = 3; + public bool Nightly { get; set; } = true; + } +} diff --git a/PrivaPub/Infrastructure/Backup/Backups.cs b/PrivaPub/Infrastructure/Backup/Backups.cs new file mode 100644 index 0000000..97d0c9b --- /dev/null +++ b/PrivaPub/Infrastructure/Backup/Backups.cs @@ -0,0 +1,89 @@ +using Microsoft.Extensions.Options; + +using MongoDB.Entities; + +using PrivaPub.Domain.Media; +using PrivaPub.Models; + +namespace PrivaPub.Infrastructure.Backup +{ + // The server's backups, wherever they are started from: the CLI, the nightly schedule, the administrator's page. + public class Backups(IOptionsMonitor options, IOptionsMonitor app, IMediaService media) + { + /// Where backups are kept: Backups:Root (production's /var/lib/privapub/backups), else beside the media root. + public string Root => Path.GetFullPath(options.CurrentValue.Root ?? media.Root.TrimEnd(Path.DirectorySeparatorChar) + "-backups"); + + public BackupOptions Options => options.CurrentValue; + + public string Host => app.CurrentValue.BackendBaseAddress?.TrimEnd('/'); + + public BackupContext Context() => new(DB.Default.Database(), Root, media.Root, media.TrashRoot, Host, options.CurrentValue); + + public Task<(BackupInfo Backup, string Error)> Create(string kind, bool dbOnly, CancellationToken token) => + ServerBackup.Create(Context(), kind, dbOnly, token); + + public List List() => ServerBackup.List(Root); + + public BackupInfo Find(string id) => ServerBackup.Find(Root, id); + + public Task> Verify(string id, CancellationToken token) => ServerBackup.Verify(Root, id, token); + + public bool Delete(string id) => ServerBackup.Delete(Root, id); + } + + // A nightly backup at Backups:NightlyAt (UTC), or as soon as the service is up after missing it; the old ones rotated + // out as each is made. + public class BackupScheduler(Backups backups, ILogger logger) : BackgroundService + { + static readonly TimeSpan Interval = TimeSpan.FromMinutes(10); + + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + while (!stoppingToken.IsCancellationRequested) + { + try + { + await Task.Delay(Interval, stoppingToken); + } + catch (OperationCanceledException) + { + return; + } + try + { + await RunIfDue(DateTime.UtcNow, stoppingToken); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + logger.LogError(ex, "The nightly backup failed"); + } + } + } + + /// Backs up when the day's time has come and there is no nightly backup since: whether it did. + public async Task RunIfDue(DateTime now, CancellationToken token) + { + var options = backups.Options; + if (!options.Nightly || !TimeOnly.TryParse(options.NightlyAt, System.Globalization.CultureInfo.InvariantCulture, out var at) + || !IsDue(now, at, backups.List().Where(b => b.Kind == "nightly").Select(b => b.CreatedAt))) + return false; + var (made, error) = await backups.Create("nightly", dbOnly: false, token); + if (made == default) + { + logger.LogWarning("The nightly backup was not made: {Error}", error); + return false; + } + logger.LogInformation("Nightly backup {Id}: {Collections} collections, {Files} media files", made.Id, made.Manifest.Collections.Count, made.Manifest.Media.Files); + return true; + } + + /// Whether the last time of day for a nightly backup has passed with no nightly backup made since. + public static bool IsDue(DateTime now, TimeOnly at, IEnumerable nightlies) + { + var due = now.Date + at.ToTimeSpan(); + if (now < due) + due = due.AddDays(-1); + return !nightlies.Any(made => made >= due); + } + } +} diff --git a/PrivaPub/Infrastructure/Backup/HardLink.cs b/PrivaPub/Infrastructure/Backup/HardLink.cs new file mode 100644 index 0000000..c225353 --- /dev/null +++ b/PrivaPub/Infrastructure/Backup/HardLink.cs @@ -0,0 +1,35 @@ +using System.Runtime.InteropServices; + +namespace PrivaPub.Infrastructure.Backup +{ + // A second name for a file (.NET has no API for it): a backup links the live media, which costs no disk, since names are + // random and files never change; deleting the live one leaves the backup's. + static class HardLink + { + // strings go to libc as UTF-8 on Unix + [DllImport("libc", EntryPoint = "link", SetLastError = true, CharSet = CharSet.Ansi)] + static extern int Link(string existing, string created); + + /// Links, or copies where a link can't be made (another disk, a filesystem refusing it). + public static bool LinkOrCopy(string existing, string created) + { + Directory.CreateDirectory(Path.GetDirectoryName(created)!); + if (OperatingSystem.IsLinux()) + { + try + { + if (Link(existing, created) == 0) + return true; + } + catch (DllNotFoundException) + { + } + catch (EntryPointNotFoundException) + { + } + } + File.Copy(existing, created, overwrite: true); + return false; + } + } +} diff --git a/PrivaPub/Infrastructure/Backup/MaintenanceLock.cs b/PrivaPub/Infrastructure/Backup/MaintenanceLock.cs new file mode 100644 index 0000000..71e9b46 --- /dev/null +++ b/PrivaPub/Infrastructure/Backup/MaintenanceLock.cs @@ -0,0 +1,75 @@ +using MongoDB.Driver; +using MongoDB.Entities; + +namespace PrivaPub.Infrastructure.Backup +{ + // One backup or restore at a time, whoever starts it (the CLI, the nightly schedule, the administrator's page): a + // document held with a heartbeat. A holder that died (no heartbeat for two minutes) is taken over. The collection is + // never backed up, nor dropped by a restore. + public class MaintenanceLock : Entity + { + public const string Name = "maintenance"; + static readonly TimeSpan Stale = TimeSpan.FromMinutes(2); + static readonly TimeSpan Beat = TimeSpan.FromSeconds(30); + + public string Owner { get; set; } + public string What { get; set; } + public DateTime Since { get; set; } + public DateTime Heartbeat { get; set; } + + /// Takes the lock for what is said, or null when someone else holds it; disposing the result frees it. + public static async Task Take(string what, CancellationToken token) + { + var owner = $"{Environment.MachineName}:{Environment.ProcessId}:{Guid.NewGuid():N}"; + var now = DateTime.UtcNow; + var stale = now - Stale; + try + { + var taken = await DB.Default.UpdateAndGet() + .Match(l => l.ID == Name && (l.Heartbeat < stale || l.Owner == null)) + .Modify(l => l.Owner, owner) + .Modify(l => l.What, what) + .Modify(l => l.Since, now) + .Modify(l => l.Heartbeat, now) + .Option(o => o.IsUpsert = true) + .ExecuteAsync(token); + return taken?.Owner == owner ? new Held(owner) : default; + } + catch (MongoCommandException ex) when (ex.Code == 11000) + { + return default;//held: the upsert met the live lock's id + } + catch (MongoWriteException ex) when (ex.WriteError?.Category == ServerErrorCategory.DuplicateKey) + { + return default; + } + } + + /// Who holds it now and for what, or null. + public static async Task Current(CancellationToken token) + { + var held = await DB.Default.Find().Match(l => l.ID == Name).ExecuteFirstAsync(token); + return held?.Owner != null && held.Heartbeat >= DateTime.UtcNow - Stale ? held : default; + } + + public sealed class Held : IAsyncDisposable + { + readonly string _owner; + readonly Timer _heartbeat; + + public Held(string owner) + { + _owner = owner; + _heartbeat = new Timer(_ => _ = DB.Default.Update().Match(l => l.ID == Name && l.Owner == owner) + .Modify(l => l.Heartbeat, DateTime.UtcNow).ExecuteAsync(), default, Beat, Beat); + } + + public async ValueTask DisposeAsync() + { + await _heartbeat.DisposeAsync(); + await DB.Default.Update().Match(l => l.ID == Name && l.Owner == _owner) + .Modify(l => l.Owner, null).Modify(l => l.Heartbeat, DateTime.MinValue).ExecuteAsync(); + } + } + } +} diff --git a/PrivaPub/Infrastructure/Backup/RestoreMarker.cs b/PrivaPub/Infrastructure/Backup/RestoreMarker.cs new file mode 100644 index 0000000..ac9abc7 --- /dev/null +++ b/PrivaPub/Infrastructure/Backup/RestoreMarker.cs @@ -0,0 +1,56 @@ +using System.Text.Json; + +namespace PrivaPub.Infrastructure.Backup +{ + // A restore asked for (by the administrator's page or the CLI) and carried out at the next boot, before anything + // else (Program, MaintenanceGate): restore.json in the backups' directory, written whole or not at all. + public sealed class RestoreMarker + { + public const string FileName = "restore.json"; + public const int MaxAttempts = 3; + + public string Backup { get; set; }//the backup restored + public string PreRestore { get; set; }//the backup taken just before, what the protective merge reads + public string State { get; set; } = "pending";//pending, then running + public int Attempts { get; set; } + public DateTime RequestedAt { get; set; } = DateTime.UtcNow; + public string Error { get; set; } + + public static string PathIn(string backupsRoot) => Path.Combine(backupsRoot, FileName); + + public static RestoreMarker Read(string backupsRoot) + { + var path = PathIn(backupsRoot); + if (!File.Exists(path)) + return default; + try + { + return JsonSerializer.Deserialize(File.ReadAllText(path)); + } + catch (JsonException) + { + return default; + } + } + + public void Write(string backupsRoot) + { + Directory.CreateDirectory(backupsRoot); + var path = PathIn(backupsRoot); + var part = path + ".part"; + using (var file = new FileStream(part, FileMode.Create, FileAccess.Write)) + { + JsonSerializer.Serialize(file, this); + file.Flush(flushToDisk: true); + } + File.Move(part, path, overwrite: true); + } + + public static void Clear(string backupsRoot) + { + var path = PathIn(backupsRoot); + if (File.Exists(path)) + File.Delete(path); + } + } +} diff --git a/PrivaPub/Infrastructure/Backup/ServerBackup.cs b/PrivaPub/Infrastructure/Backup/ServerBackup.cs new file mode 100644 index 0000000..5a3f8ec --- /dev/null +++ b/PrivaPub/Infrastructure/Backup/ServerBackup.cs @@ -0,0 +1,340 @@ +using MongoDB.Bson; +using MongoDB.Bson.IO; +using MongoDB.Driver; + +using PrivaPub.Infrastructure.Data; + +using System.Globalization; +using System.IO.Compression; +using System.Security.Cryptography; +using System.Text; + +namespace PrivaPub.Infrastructure.Backup +{ + // What a backup needs to know: where things are, and whose host it is. + public sealed record BackupContext(IMongoDatabase Database, string BackupsRoot, string MediaRoot, string TrashRoot, string Host, BackupOptions Options); + + public sealed record BackupInfo(string Id, string Kind, DateTime CreatedAt, long Bytes, ArchiveManifest Manifest); + + // The whole server, backed up as files (owner decision 2026-10-07: a whole-server backup, plain, protected by file + // permissions). Each backup is a directory - in the backups' root: + // - manifest.json: what it holds (ArchiveManifest); + // - db/.jsonl.gz: every document of each collection, one per line, in canonical Extended JSON (every BSON + // type kept as it was), all read at one instant when Mongo is a replica set (a snapshot session); + // - media/: the media files rows hold, as hard links to the live ones (no disk spent; a deleted file stays here + // until the backup is rotated out), copied where a link can't be made; a db-only backup lists them instead. + // It is written as .partial and renamed once whole. Left out: what belongs to the moment (Excluded). Everything is + // readable by the service's user and group only: it holds every persona's private key and private posts. + public static class ServerBackup + { + public const string PartialSuffix = ".partial"; + + public static readonly IReadOnlyDictionary Excluded = new Dictionary + { + ["InteractionSalt"] = "the day's statistics salt never outlives its day (owner rule)", + ["Job"] = "work in flight: deliveries and inbox processing belong to the moment", + ["Delivery"] = "work in flight, from before jobs", + ["EmailRecovery"] = "recovery codes live an hour", + ["openiddict.tokens"] = "sessions: a restore ends every one", + ["openiddict.authorizations"] = "sessions: a restore ends every one", + [nameof(MaintenanceLock)] = "who is backing up or restoring now", + ["AppConfiguration"] = "the configuration's copy holds the SMTP password, and is made again from appsettings at boot" + }; + + static readonly JsonWriterSettings Json = new() { OutputMode = JsonOutputMode.CanonicalExtendedJson, Indent = false }; + + // 2770: the service (www-data) and the deploy (in www-data's group) each read and rotate what the other wrote, and new + // files take the directory's group + const UnixFileMode DirectoryMode = UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute + | UnixFileMode.GroupRead | UnixFileMode.GroupWrite | UnixFileMode.GroupExecute | UnixFileMode.SetGroup; + const UnixFileMode FileMode0640 = UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.GroupRead; + + /// Takes the maintenance lock and backs the server up: the backup, or why not. + public static async Task<(BackupInfo Backup, string Error)> Create(BackupContext context, string kind, bool dbOnly, CancellationToken token) + { + await using var held = await MaintenanceLock.Take("backup", token); + if (held == default) + return (default, "a backup or a restore is already running"); + return await CreateHeld(context, kind, dbOnly, token); + } + + /// Backs the server up with the maintenance lock already held (a restore's own backup, taken first). + public static async Task<(BackupInfo Backup, string Error)> CreateHeld(BackupContext context, string kind, bool dbOnly, CancellationToken token) + { + var database = context.Database; + var names = (await (await database.ListCollectionNamesAsync(cancellationToken: token)).ToListAsync(token)) + .Where(n => !n.StartsWith("system.", StringComparison.Ordinal)) + .OrderBy(n => n, StringComparer.Ordinal) + .ToList(); + MakeDirectory(context.BackupsRoot); + var stats = await database.RunCommandAsync(new BsonDocument("dbStats", 1), cancellationToken: token); + var needed = (long)(stats.GetValue("dataSize", 0).ToDouble() * 1.1); + if (new DriveInfo(Path.GetFullPath(context.BackupsRoot)).AvailableFreeSpace < needed) + return (default, $"not enough free disk for a backup: about {needed / 1024 / 1024} MB needed"); + + var id = $"{DateTime.UtcNow.ToString("yyyyMMdd-HHmmss", CultureInfo.InvariantCulture)}-{kind}"; + var partial = Path.Combine(context.BackupsRoot, id + PartialSuffix); + MakeDirectory(partial); + MakeDirectory(Path.Combine(partial, "db")); + try + { + var manifest = new ArchiveManifest + { + Kind = kind, + Host = context.Host, + AppCommit = BuildInfo.Commit, + AppRef = BuildInfo.Ref, + DbOnly = dbOnly + }; + var replica = await MongoTopology.IsReplicaSet(database, token); + var media = new SortedSet(StringComparer.Ordinal); + using var session = replica ? await database.Client.StartSessionAsync(new ClientSessionOptions { Snapshot = true }, token) : default; + var window = replica ? await RaiseSnapshotWindow(database, token) : default(int?); + try + { + foreach (var name in names) + { + if (Excluded.TryGetValue(name, out var why)) + { + manifest.Excluded.Add(new ArchiveManifest.ExcludedEntry { Name = name, Why = why }); + continue; + } + manifest.Collections.Add(await Dump(database, session, name, partial, name == "MediaAttachment" ? media : default, token)); + } + (manifest.NewestMigration, manifest.MigrationNumber) = await NewestMigration(database, session, token); + } + finally + { + if (window is { } previous) + await SetSnapshotWindow(database, previous, CancellationToken.None); + } + manifest.Consistent = replica; + + manifest.Media.List = [.. media]; + if (!dbOnly) + foreach (var relative in media) + { + var source = new[] { context.MediaRoot, context.TrashRoot }.Select(root => Inside(root, relative)).FirstOrDefault(File.Exists); + if (source == default) + { + manifest.Media.Missing++; + continue; + } + HardLink.LinkOrCopy(source, Inside(Path.Combine(partial, "media"), relative)); + manifest.Media.Files++; + manifest.Media.Bytes += new FileInfo(source).Length; + } + + manifest.Write(partial); + Directory.Move(partial, Path.Combine(context.BackupsRoot, id)); + Retain(context.BackupsRoot, context.Options); + return (Info(context.BackupsRoot, id), default); + } + catch + { + if (Directory.Exists(partial)) + Directory.Delete(partial, recursive: true); + throw; + } + } + + // a collection read as raw BSON in batches, each document a line of canonical Extended JSON, gzipped; the media rows' + // files collected on the way + static async Task Dump(IMongoDatabase database, IClientSessionHandle session, string name, string directory, + ISet media, CancellationToken token) + { + var collection = database.GetCollection(name); + var path = Path.Combine(directory, "db", name + ".jsonl.gz"); + var count = 0L; + await using (var file = NewFile(path)) + await using (var gzip = new GZipStream(file, CompressionLevel.Fastest)) + await using (var writer = new StreamWriter(gzip, new UTF8Encoding(false))) + { + var options = new FindOptions { BatchSize = 1000 }; + using var cursor = session == default + ? await collection.FindAsync(FilterDefinition.Empty, options, token) + : await collection.FindAsync(session, FilterDefinition.Empty, options, token); + while (await cursor.MoveNextAsync(token)) + foreach (var document in cursor.Current) + using (document) + { + await writer.WriteLineAsync(document.ToJson(Json)); + count++; + if (media != default) + Collect(document, media); + } + } + var indexes = await (await collection.Indexes.ListAsync(token)).ToListAsync(token); + return new ArchiveManifest.CollectionEntry + { + Name = name, + Count = count, + Bytes = new FileInfo(path).Length, + Sha256 = await Hash(path, token), + Indexes = [.. indexes.Select(i => i.ToJson(Json))] + }; + } + + // a media row's files, unless it is trashed + static void Collect(RawBsonDocument row, ISet media) + { + if (row.TryGetValue("TrashedAt", out var trashed) && !trashed.IsBsonNull) + return; + foreach (var field in new[] { "FilePath", "PreviewPath" }) + if (row.TryGetValue(field, out var value) && value.IsString && Safe(value.AsString)) + media.Add(value.AsString); + } + + // MongoDB.Entities records each migration run with its class's number (_016_... is 16) and its name in words + static async Task<(string Name, int Number)> NewestMigration(IMongoDatabase database, IClientSessionHandle session, CancellationToken token) + { + var history = database.GetCollection("_migration_history_"); + var options = new FindOptions { Sort = new BsonDocument("Number", -1), Limit = 1 }; + var newest = session == default + ? await (await history.FindAsync(FilterDefinition.Empty, options, token)).FirstOrDefaultAsync(token) + : await (await history.FindAsync(session, FilterDefinition.Empty, options, token)).FirstOrDefaultAsync(token); + return newest == default ? default : (newest.GetValue("Name", BsonNull.Value).ToString(), newest.GetValue("Number", 0).ToInt32()); + } + + /// The newest migration this build has: a backup made by a newer one can't be restored by it. + public static int CodeMigration() => typeof(ServerBackup).Assembly.GetTypes() + .Where(t => typeof(MongoDB.Entities.IMigration).IsAssignableFrom(t) && !t.IsAbstract) + .Select(t => int.TryParse(new string(t.Name.TrimStart('_').TakeWhile(char.IsDigit).ToArray()), out var number) ? number : 0) + .DefaultIfEmpty(0) + .Max(); + + // how long a snapshot stays readable: raised only while a backup reads (a longer window keeps old versions in the + // small cache all day); the value it had, to set back + static async Task RaiseSnapshotWindow(IMongoDatabase database, CancellationToken token) + { + var admin = database.Client.GetDatabase("admin"); + try + { + var current = await admin.RunCommandAsync(new BsonDocument { { "getParameter", 1 }, { "minSnapshotHistoryWindowInSeconds", 1 } }, cancellationToken: token); + var previous = current.GetValue("minSnapshotHistoryWindowInSeconds", 300).ToInt32(); + await SetSnapshotWindow(database, Math.Max(previous, 3600), token); + return previous; + } + catch (MongoCommandException) + { + return default; + } + } + + static async Task SetSnapshotWindow(IMongoDatabase database, int seconds, CancellationToken token) => + await database.Client.GetDatabase("admin").RunCommandAsync( + new BsonDocument { { "setParameter", 1 }, { "minSnapshotHistoryWindowInSeconds", seconds } }, cancellationToken: token); + + /// The backups kept, newest first (not one still being written). + public static List List(string backupsRoot) + { + if (!Directory.Exists(backupsRoot)) + return []; + return Directory.EnumerateDirectories(backupsRoot) + .Select(Path.GetFileName) + .Where(name => !name.EndsWith(PartialSuffix, StringComparison.Ordinal) && File.Exists(Path.Combine(backupsRoot, name, ArchiveManifest.FileName))) + .Select(name => Info(backupsRoot, name)) + .OrderByDescending(b => b.CreatedAt) + .ToList(); + } + + public static BackupInfo Find(string backupsRoot, string id) => + Safe(id) && !id.Contains('/') && Directory.Exists(Path.Combine(backupsRoot, id)) && !id.EndsWith(PartialSuffix, StringComparison.Ordinal) + ? Info(backupsRoot, id) + : default; + + static BackupInfo Info(string backupsRoot, string id) + { + var directory = Path.Combine(backupsRoot, id); + var manifest = ArchiveManifest.Read(directory); + var bytes = Directory.EnumerateFiles(Path.Combine(directory, "db")).Sum(f => new FileInfo(f).Length); + return new BackupInfo(id, manifest?.Kind, manifest?.CreatedAt ?? Directory.GetCreationTimeUtc(directory), bytes, manifest); + } + + /// Whether every file of a backup is what its manifest says: the problems found (none when whole). + public static async Task> Verify(string backupsRoot, string id, CancellationToken token) + { + var problems = new List(); + var backup = Find(backupsRoot, id); + if (backup?.Manifest == default) + return ["no such backup, or no manifest"]; + var directory = Path.Combine(backupsRoot, id); + foreach (var collection in backup.Manifest.Collections) + { + var path = Path.Combine(directory, "db", collection.Name + ".jsonl.gz"); + if (!File.Exists(path)) + problems.Add($"{collection.Name}: missing"); + else if (await Hash(path, token) != collection.Sha256) + problems.Add($"{collection.Name}: altered"); + } + if (!backup.Manifest.DbOnly) + foreach (var relative in backup.Manifest.Media.List.Where(r => !File.Exists(Inside(Path.Combine(directory, "media"), r)))) + problems.Add($"media {relative}: missing"); + return problems; + } + + /// Deletes a backup (its media links go; the live files stay). + public static bool Delete(string backupsRoot, string id) + { + if (Find(backupsRoot, id) == default) + return false; + Directory.Delete(Path.Combine(backupsRoot, id), recursive: true); + return true; + } + + // what is kept: the newest nightly ones for KeepDaily days and the newest of each of KeepWeekly weeks before, the + // newest pre-deploy and pre-restore ones; manual and uploaded ones until deleted; a backup that died writing goes + public static void Retain(string backupsRoot, BackupOptions options) + { + var all = List(backupsRoot); + var nightly = all.Where(b => b.Kind == "nightly").OrderByDescending(b => b.CreatedAt).ToList(); + var kept = nightly.Take(options.KeepDaily).ToHashSet(); + foreach (var week in nightly.Skip(options.KeepDaily).GroupBy(b => (ISOWeek.GetYear(b.CreatedAt), ISOWeek.GetWeekOfYear(b.CreatedAt))).Take(options.KeepWeekly)) + kept.Add(week.First()); + var doomed = nightly.Where(b => !kept.Contains(b)) + .Concat(all.Where(b => b.Kind == "pre-deploy").OrderByDescending(b => b.CreatedAt).Skip(options.KeepPreDeploy)) + .Concat(all.Where(b => b.Kind == "pre-restore").OrderByDescending(b => b.CreatedAt).Skip(options.KeepPreRestore)); + var stale = Directory.EnumerateDirectories(backupsRoot, "*" + PartialSuffix).Where(p => Directory.GetLastWriteTimeUtc(p) < DateTime.UtcNow.AddDays(-1)); + foreach (var directory in doomed.Select(b => Path.Combine(backupsRoot, b.Id)).Concat(stale).ToList()) + try + { + Directory.Delete(directory, recursive: true); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + //another user's backup this one may not delete: the next rotation tries again + } + } + + // a relative path from a database row or a manifest, never outside its root + static bool Safe(string relative) => + !string.IsNullOrEmpty(relative) && !Path.IsPathRooted(relative) && !relative.Split('/', '\\').Any(part => part is ".." or "."); + + public static string Inside(string root, string relative) + { + var full = Path.GetFullPath(Path.Combine(root, relative)); + if (!Safe(relative) || !full.StartsWith(Path.GetFullPath(root) + Path.DirectorySeparatorChar, StringComparison.Ordinal)) + throw new InvalidOperationException($"{relative} is not inside {root}"); + return full; + } + + static async Task Hash(string path, CancellationToken token) + { + await using var file = File.OpenRead(path); + return Convert.ToHexStringLower(await SHA256.HashDataAsync(file, token)); + } + + static FileStream NewFile(string path) => OperatingSystem.IsWindows() + ? new FileStream(path, FileMode.CreateNew, FileAccess.Write) + : new FileStream(path, new FileStreamOptions { Mode = FileMode.CreateNew, Access = FileAccess.Write, UnixCreateMode = FileMode0640 }); + + // set after creating, since the umask would take the group's write away + public static void MakeDirectory(string path) + { + var existed = Directory.Exists(path); + Directory.CreateDirectory(path); + if (!existed && !OperatingSystem.IsWindows()) + File.SetUnixFileMode(path, DirectoryMode); + } + } +} diff --git a/PrivaPub/Infrastructure/Cli/AdminCommands.cs b/PrivaPub/Infrastructure/Cli/AdminCommands.cs index 2827a97..c875532 100644 --- a/PrivaPub/Infrastructure/Cli/AdminCommands.cs +++ b/PrivaPub/Infrastructure/Cli/AdminCommands.cs @@ -24,8 +24,16 @@ namespace PrivaPub.Infrastructure.Cli PrivaPub admin smoke prints " " for the deploy's signed-in check PrivaPub admin media audit [--fix] media files against what holds them; --fix (as www-data) trashes what nothing holds and gives today's pictures their rows + PrivaPub admin backup [--kind manual|pre-deploy] [--db-only] + backs the server up; --db-only lists the media without + linking them (the deploy's, which can't link www-data's files) + PrivaPub admin backups the backups kept, newest first + PrivaPub admin backup verify whether a backup's files are what its manifest says """; + /// Whether a command runs before migrations: backups are of the database as it was. + public static bool BeforeMigrations(string[] args) => args is ["backup", ..] or ["backups"]; + public static async Task Run(string[] args, IServiceProvider services, TextReader input = default, TextWriter output = default) { input ??= Console.In; @@ -40,12 +48,75 @@ namespace PrivaPub.Infrastructure.Cli return await Smoke(services, persona, output); case ["media", "audit", .. var flags] when flags.All(f => f == "--fix"): return await AuditMedia(services, flags.Contains("--fix"), output); + case ["backup", "verify", var id]: + return await VerifyBackup(services, id, output); + case ["backup", .. var flags] when BackupFlags(flags, out var kind, out var dbOnly): + return await Backup(services, kind, dbOnly, output); + case ["backups"]: + return ListBackups(services, output); default: Console.Error.WriteLine(Usage); return 2; } } + static readonly string[] CommandKinds = ["manual", "pre-deploy"]; + + static bool BackupFlags(string[] flags, out string kind, out bool dbOnly) + { + kind = "manual"; + dbOnly = false; + for (var i = 0; i < flags.Length; i++) + switch (flags[i]) + { + case "--db-only": + dbOnly = true; + break; + case "--kind" when i + 1 < flags.Length && CommandKinds.Contains(flags[i + 1]): + kind = flags[++i]; + break; + default: + return false; + } + return true; + } + + static async Task Backup(IServiceProvider services, string kind, bool dbOnly, TextWriter output) + { + var (made, error) = await services.GetRequiredService().Create(kind, dbOnly, CancellationToken.None); + if (made == default) + { + Console.Error.WriteLine(error); + return 1; + } + var manifest = made.Manifest; + output.WriteLine($"{made.Id}: {manifest.Collections.Count} collections, {manifest.Collections.Sum(c => c.Count)} documents, {made.Bytes / 1024} KiB"); + output.WriteLine(dbOnly + ? $"{manifest.Media.List.Count} media files listed, not linked" + : $"{manifest.Media.Files} media files, {manifest.Media.Bytes / 1024 / 1024} MiB, {manifest.Media.Missing} missing"); + if (!manifest.Consistent) + output.WriteLine("not read at one instant: Mongo is not a replica set"); + return 0; + } + + static int ListBackups(IServiceProvider services, TextWriter output) + { + foreach (var backup in services.GetRequiredService().List()) + output.WriteLine($"{backup.Id}\t{backup.Kind}\t{backup.CreatedAt:u}\t{backup.Bytes / 1024} KiB\t{backup.Manifest?.Media.Files ?? 0} media{(backup.Manifest?.DbOnly == true ? " (listed)" : string.Empty)}"); + return 0; + } + + static async Task VerifyBackup(IServiceProvider services, string id, TextWriter output) + { + var problems = await services.GetRequiredService().Verify(id, CancellationToken.None); + foreach (var problem in problems) + output.WriteLine(problem); + if (problems.Count > 0) + return 1; + output.WriteLine($"{id}: whole"); + return 0; + } + static async Task AuditMedia(IServiceProvider services, bool fix, TextWriter output) { var report = await Domain.Media.MediaAudit.Run(services.GetRequiredService(), fix, CancellationToken.None); diff --git a/PrivaPub/Infrastructure/Data/EntityMaps.cs b/PrivaPub/Infrastructure/Data/EntityMaps.cs index bbad55c..a75c955 100644 --- a/PrivaPub/Infrastructure/Data/EntityMaps.cs +++ b/PrivaPub/Infrastructure/Data/EntityMaps.cs @@ -4,7 +4,22 @@ namespace PrivaPub.Infrastructure.Data { public static class EntityMaps { + static readonly Lock Gate = new(); + static bool _warm; + + // once per process, one map at a time: test hosts boot side by side public static void Warm() + { + lock (Gate) + { + if (_warm) + return; + Map(); + _warm = true; + } + } + + static void Map() { var collection = typeof(DB).GetMethods() .Single(m => m.Name == nameof(DB.Collection) && m.IsGenericMethodDefinition && m.GetParameters().Length == 0); diff --git a/PrivaPub/Program.cs b/PrivaPub/Program.cs index 0ac26cb..c85c288 100644 --- a/PrivaPub/Program.cs +++ b/PrivaPub/Program.cs @@ -68,6 +68,9 @@ try .AddSingleton() .AddSingleton() .AddHostedService() + .Configure(builder.Configuration.GetSection("Backups")) + .AddSingleton() + .AddHostedService() .PrivaPubMiddlewareConfiguration(); } catch (Exception ex) @@ -86,8 +89,6 @@ try var mongoSettings = builder.Configuration.GetSection(nameof(MongoSettings)).Get(); await DB.InitAsync(mongoSettings.Database, MongoClientSettings.FromConnectionString(mongoSettings.ConnectionString)); EntityMaps.Warm(); - await DB.Default.MigrateAsync(); - await Indexes.Create(); } catch (Exception ex) { @@ -108,6 +109,25 @@ try // Commands get every service but start nothing: no Kestrel, no hosted services, no media directory. The deploy runs // them as its own user, which can read the configuration and reach the private mongod but owns no www-data directory. + // Backups are taken before migrations: the deploy's is of the database as the live build left it. + if (args is ["admin", .. var command] && AdminCommands.BeforeMigrations(command)) + { + using var scope = app.Services.CreateScope(); + Environment.ExitCode = await AdminCommands.Run(command, scope.ServiceProvider); + return; + } + + try + { + await DB.Default.MigrateAsync(); + await Indexes.Create(); + } + catch (Exception ex) + { + Log.ForContext().Fatal(ex, $"{nameof(Program)}.{nameof(Program)}() DB Migrations"); + throw; + } + if (args is ["admin", ..]) { using var scope = app.Services.CreateScope(); diff --git a/PrivaPub/appsettings.Development.json b/PrivaPub/appsettings.Development.json index c0ee1c0..2cbaca3 100644 --- a/PrivaPub/appsettings.Development.json +++ b/PrivaPub/appsettings.Development.json @@ -4,6 +4,9 @@ "LogsDatabase": "logs", "ConnectionString": "mongodb://localhost:27017" }, + "Backups": { + "Nightly": false + }, "AppConfiguration": { "RequiresFirstTimeSetup": null, "Version": "0.0.0", diff --git a/PrivaPub/appsettings.Production.json b/PrivaPub/appsettings.Production.json index e83599c..74c401f 100644 --- a/PrivaPub/appsettings.Production.json +++ b/PrivaPub/appsettings.Production.json @@ -2,6 +2,9 @@ "Media": { "Root": "/var/lib/privapub/media" }, + "Backups": { + "Root": "/var/lib/privapub/backups" + }, "Registrations": { "Mode": "Invitations" }, diff --git a/deploy/max/setup.sh b/deploy/max/setup.sh index 3358f8d..1c66806 100755 --- a/deploy/max/setup.sh +++ b/deploy/max/setup.sh @@ -13,7 +13,9 @@ echo "== directories" install -d -o "$RUNNER" -g www-data -m 755 /var/www/$HOST install -d -o "$RUNNER" -g "$RUNNER" -m 750 /var/backups/$HOST install -d -o www-data -g www-data -m 750 /var/lib/privapub /var/lib/privapub/mongo -# backups: the service writes them, and so does the deploy (as $RUNNER, a member of www-data) before each deploy +# backups: the service writes them, and so does the deploy (as $RUNNER, a member of www-data) before each deploy; a runner +# just added to the group gets it when the runner restarts +id -nG "$RUNNER" | grep -qw www-data || usermod -aG www-data "$RUNNER" install -d -o www-data -g www-data -m 2770 /var/lib/privapub/backups echo "== sudoers" diff --git a/tools/pasture/appsettings.Pasture.json b/tools/pasture/appsettings.Pasture.json index 6a3bee2..61b2395 100644 --- a/tools/pasture/appsettings.Pasture.json +++ b/tools/pasture/appsettings.Pasture.json @@ -26,6 +26,7 @@ "Relays": [ "https://relay.test/actor", "https://aoderelay.test/actor" ] }, "Media": { "Root": "/tmp/privapub-media" }, + "Backups": { "Root": "/tmp/privapub-backups", "Nightly": false }, "RateLimits": { "AccountsPerMinute": 1000, "UploadsBurst": 1000, "UploadsPerMinute": 1000, "ProxyBurst": 100000, "ProxyPerMinute": 100000 }, "Registrations": { "Mode": "Open" }, "Statistics": { "Geo": { "AutoUpdate": false } },