diff --git a/.gitea/workflows/deploy.yml b/.gitea/workflows/deploy.yml index 4b6828f..b819dc5 100644 --- a/.gitea/workflows/deploy.yml +++ b/.gitea/workflows/deploy.yml @@ -12,8 +12,7 @@ env: BACKUPS: /var/backups/privapub.thepra.dev LOCAL_URL: http://127.0.0.1:6970 PUBLIC_URL: https://privapub.thepra.dev - MONGO_URI: mongodb://127.0.0.1:27022/?directConnection=true - MONGO_DB: PrivaPub + SERVER_BACKUPS: /var/lib/privapub/backups jobs: site: @@ -57,19 +56,24 @@ jobs: echo "SNAPSHOT=$BACKUPS/site-$STAMP" >> "$GITHUB_ENV" ls -1dt "$BACKUPS"/site-* 2>/dev/null | tail -n +4 | xargs -r rm -rf || true - # without the statistics salt, which must not outlive its day (owner rule: it is destroyed at each rollup, and a dump - # kept for days would let the day's actor hashes be reversed) - - name: Dump the database + # The server's own backup (PrivaPub admin backup, owner decision 2026-10-07) of the database as the live build left it: + # the new build's command runs before its migrations. The media are listed, not linked (the runner may not link + # www-data's files; the nightly backups hold them). Never the statistics salt, which must not outlive its day (owner + # rule), nor the configuration's copy with its secrets. No deploy while a restore waits for its boot. + - name: Back up the database run: | - DUMP="$BACKUPS/mongo-$(date +%Y%m%d-%H%M%S).archive.gz" - mongodump --quiet --uri "$MONGO_URI" --db "$MONGO_DB" --excludeCollection=InteractionSalt --gzip --archive="$DUMP" - if mongorestore --gzip --archive="$DUMP" --dryRun -v 2>&1 | grep -q "InteractionSalt"; then - rm -f "$DUMP" - echo "::error::the dump holds the statistics salt" + [ ! -e "$SERVER_BACKUPS/restore.json" ] || { echo "::error::a restore is pending or running ($SERVER_BACKUPS/restore.json): deploy after it"; exit 1; } + out=$(cd "$GITHUB_WORKSPACE/publish" && ASPNETCORE_ENVIRONMENT=Production ./PrivaPub admin backup --kind pre-deploy --db-only) + echo "$out" + id=$(echo "$out" | grep -oE '^[0-9]{8}-[0-9]{6}-pre-deploy' | tail -1) + [ -d "$SERVER_BACKUPS/$id" ] || { echo "::error::the backup was not made"; exit 1; } + (cd "$GITHUB_WORKSPACE/publish" && ASPNETCORE_ENVIRONMENT=Production ./PrivaPub admin backup verify "$id") + if [ -e "$SERVER_BACKUPS/$id/db/InteractionSalt.jsonl.gz" ]; then + echo "::error::the backup holds the statistics salt" exit 1 fi - echo "::notice::database dumped to $DUMP ($(du -h "$DUMP" | cut -f1))" - ls -1t "$BACKUPS"/mongo-*.archive.gz 2>/dev/null | tail -n +8 | xargs -r rm -f || true + echo "BACKUP_ID=$id" >> "$GITHUB_ENV" + echo "::notice::database backed up as $id" - name: Stop, sync, start run: | diff --git a/.gitignore b/.gitignore index b038b6f..4479eae 100644 --- a/.gitignore +++ b/.gitignore @@ -257,6 +257,7 @@ Generated_Code/ # because we have git ;-) _UpgradeReport_Files/ Backup*/ +!PrivaPub/Infrastructure/Backup/ UpgradeLog*.XML UpgradeLog*.htm ServiceFabricBackup/ @@ -401,6 +402,9 @@ FodyWeavers.xsd PrivaPub/media-store/ PrivaPub/media-store-proxy/ +PrivaPub/media-store-trash/ +PrivaPub/media-store-incoming/ +PrivaPub/media-store-backups/ tools/pasture/.publish/ tools/pasture/.flood/ .claude/worktrees/ diff --git a/CLAUDE.md b/CLAUDE.md index a6b2c18..3112b29 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -484,6 +484,26 @@ group www-data and reaches the private mongod; `sudo -u www-data` works too. (`--replSet rs0`, a 990 MB oplog; owner decision 2026-10-07), so a backup reads every collection at one instant; `setup.sh` converts it once (PrivaPub stopped, mongod restarted, `rs.initiate`), and every connection string says `directConnection=true`, which also works against a standalone. The pasture's mongo is one too. +- **Backups** (`Infrastructure/Backup/`; owner decisions 2026-10-07: the whole server, plain files protected by their + permissions, run from the CLI, nightly and from the administrator's page). Each backup is a directory + `-` under `Backups:Root` (`/var/lib/privapub/backups`, www-data 2770, files 0640), written as + `.partial` and renamed once whole: + - `manifest.json` (`ArchiveManifest`): host, build, newest migration, whether it was read at one instant, each + collection's count, size, sha256 and indexes, what was left out and why, and the media list; + - `db/.jsonl.gz`: each document as one line of canonical Extended JSON, read raw, so every BSON type comes + back byte for byte; on a replica set every collection is read in one snapshot session + (`minSnapshotHistoryWindowInSeconds` is raised to an hour only while it reads); + - `media/`: hard links to the files of untrashed `MediaAttachment` rows (no disk spent; a deleted file stays until the + backup rotates out), copies where a link can't be made. `--db-only` lists them instead. + + **Never in a backup** (`ServerBackup.Excluded`): `InteractionSalt` (owner rule), `Job` and `Delivery` (work in flight), + `EmailRecovery`, `openiddict.tokens` and `.authorizations` (sessions), `MaintenanceLock`, and `AppConfiguration` + (its SMTP password; it is made again from appsettings at boot). One backup or restore runs at a time + (`MaintenanceLock`, a heartbeat document; a holder silent for 2 minutes is taken over), and the media janitor purges + nothing meanwhile. `BackupScheduler` backs up at `Backups:NightlyAt` (03:30 UTC), or at once when it missed the night; + rotation keeps 7 daily and 4 weekly nightly backups, 3 pre-deploy, 3 pre-restore, and manual and uploaded ones until + deleted. CLI: `PrivaPub admin backup [--kind manual|pre-deploy] [--db-only]`, `admin backups`, `admin backup verify + `; these run before migrations, so the deploy's backup is of the database as the live build left it. ## Code style @@ -840,8 +860,9 @@ the owner's GoToSocial account.** ## Deploy - **CI/CD:** push to `master` runs `build.yml` (build + tests) on the instance-wide `build` runner. A `v*` tag runs - `deploy.yml`: tests, self-contained linux-x64 publish, snapshot and `mongodump` to `/var/backups/privapub.thepra.dev` - (never the statistics salt: `InteractionSalt` is excluded and the dump is checked for it), + `deploy.yml`: tests, self-contained linux-x64 publish, a snapshot of the site to `/var/backups/privapub.thepra.dev`, + the server's own pre-deploy backup (`PrivaPub admin backup --kind pre-deploy --db-only`, run from the new build before + its migrations, verified, and checked to hold no statistics salt; no deploy while `restore.json` waits), stop → rsync → start, a `127.0.0.1:6970/build.json` health loop with rollback, then public checks (actor, NodeInfo, Swagger 404, inbox junk 400, unsigned 401) and `tools/smoke/mastodon-api.sh` (app registration, client credentials, discovery, instance, public timeline). Then it checks that what production should be running is running: diff --git a/PrivaPub.Tests/Infrastructure/BackupTests.cs b/PrivaPub.Tests/Infrastructure/BackupTests.cs new file mode 100644 index 0000000..f779d28 --- /dev/null +++ b/PrivaPub.Tests/Infrastructure/BackupTests.cs @@ -0,0 +1,286 @@ +using MongoDB.Bson; +using MongoDB.Bson.IO; +using MongoDB.Driver; +using MongoDB.Entities; + +using PrivaPub.Infrastructure.Backup; +using PrivaPub.Infrastructure.Cli; +using PrivaPub.Tests.Support; +using PrivaPub.Tests.Support.Host; + +using System.IO.Compression; + +namespace PrivaPub.Tests.Infrastructure +{ + // The server's backup: every collection as it was, byte for byte, read at one instant on a replica set, without what + // belongs to the moment (the statistics salt above all), with the media rows' files linked, checkable and rotated. + // Each test backs up a database of its own; alone, since the maintenance lock is the server's one. + [Trait("Category", "Integration")] + [Xunit.Collection(nameof(Exclusive))] + public sealed class BackupTests : IAsyncLifetime + { + readonly string _scratch = Path.Combine(Path.GetTempPath(), $"privapub-backup-tests-{Guid.NewGuid():N}"); + IMongoDatabase _database; + + string Backups => Path.Combine(_scratch, "backups"); + string Media => Path.Combine(_scratch, "media"); + string Trash => Path.Combine(_scratch, "media-trash"); + + public async ValueTask InitializeAsync() + { + Assert.SkipUnless(MongoFixture.Enabled, MongoFixture.Skip); + await PrivaPubHost.Shared(); + _database = DB.Default.Database().Client.GetDatabase($"PrivaPubBackup_{Guid.NewGuid():N}"); + Directory.CreateDirectory(Media); + Directory.CreateDirectory(Trash); + } + + public async ValueTask DisposeAsync() + { + if (_database != default) + await _database.Client.DropDatabaseAsync(_database.DatabaseNamespace.DatabaseName); + if (Directory.Exists(_scratch)) + Directory.Delete(_scratch, recursive: true); + } + + BackupContext Context(BackupOptions options = default) => new(_database, Backups, Media, Trash, "https://privapub.test", options ?? new BackupOptions()); + + static CancellationToken Token => TestContext.Current.CancellationToken; + + static List Read(string file) + { + using var gzip = new GZipStream(File.OpenRead(file), CompressionMode.Decompress); + using var reader = new StreamReader(gzip); + var documents = new List(); + while (reader.ReadLine() is { } line) + documents.Add(BsonDocument.Parse(line)); + return documents; + } + + async Task> Raw(string collection) => + (await (await _database.GetCollection(collection).FindAsync(FilterDefinition.Empty, cancellationToken: Token)).ToListAsync(Token)) + .Select(d => + { + var bytes = new byte[d.Slice.Length]; + d.Slice.GetBytes(0, bytes, 0, bytes.Length); + return bytes; + }) + .ToList(); + + [Fact] + public async Task Every_document_comes_back_byte_for_byte_and_the_salt_never_leaves() + { + var odd = new BsonDocument + { + { "_id", ObjectId.GenerateNewId() }, + { "Guid", new BsonBinaryData(Guid.NewGuid(), GuidRepresentation.Standard) }, + { "OldGuid", new BsonBinaryData(new byte[16], BsonBinarySubType.UuidLegacy) }, + { "Money", new BsonDecimal128(Decimal128.Parse("12345.678900")) }, + { "Long", new BsonInt64(long.MaxValue) }, + { "Int", 7 }, + { "Double", -0.0 }, + { "NaN", double.NaN }, + { "At", new BsonDateTime(new DateTime(2026, 10, 7, 3, 30, 0, 123, DateTimeKind.Utc)) }, + { "Stamp", new BsonTimestamp(1, 2) }, + { "Null", BsonNull.Value }, + { "Regex", new BsonRegularExpression("^a.b$", "i") }, + { "Nested", new BsonDocument { { "z", 1 }, { "a", new BsonArray { 1, "two", new BsonDocument("three", 3) } } } }, + { "Unicode", "città 🌍 \u0000 end" } + }; + // an ObjectRecord as big as a remote's long post gets + var big = new BsonDocument { { "_id", ObjectId.GenerateNewId() }, { "Json", new string('x', 10 * 1024 * 1024) } }; + await _database.GetCollection("Odd").InsertOneAsync(odd, cancellationToken: Token); + await _database.GetCollection("ObjectRecord").InsertOneAsync(big, cancellationToken: Token); + await _database.GetCollection("Odd").Indexes.CreateOneAsync( + new CreateIndexModel(Builders.IndexKeys.Ascending("At"), new CreateIndexOptions { Name = "at", Unique = true }), cancellationToken: Token); + await _database.GetCollection("InteractionSalt").InsertOneAsync(new BsonDocument("Salt", "never in a backup"), cancellationToken: Token); + await _database.GetCollection("Job").InsertOneAsync(new BsonDocument("Kind", "Deliver"), cancellationToken: Token); + await _database.GetCollection("_migration_history_").InsertManyAsync([ + new BsonDocument { { "Number", 15 }, { "Name", "older" } }, new BsonDocument { { "Number", 16 }, { "Name", "focal points are finite" } }], cancellationToken: Token); + + var (backup, error) = await ServerBackup.Create(Context(), "manual", dbOnly: false, Token); + + Assert.Null(error); + var directory = Path.Combine(Backups, backup.Id); + Assert.False(Directory.Exists(directory + ServerBackup.PartialSuffix)); + Assert.Equal(await Raw("Odd"), Read(Path.Combine(directory, "db", "Odd.jsonl.gz")).Select(d => d.ToBson()).ToList()); + Assert.Equal(await Raw("ObjectRecord"), Read(Path.Combine(directory, "db", "ObjectRecord.jsonl.gz")).Select(d => d.ToBson()).ToList()); + + Assert.False(File.Exists(Path.Combine(directory, "db", "InteractionSalt.jsonl.gz"))); + Assert.False(File.Exists(Path.Combine(directory, "db", "Job.jsonl.gz"))); + foreach (var file in Directory.EnumerateFiles(directory, "*", SearchOption.AllDirectories)) + Assert.DoesNotContain("never in a backup", await ReadAll(file)); + Assert.Contains(backup.Manifest.Excluded, e => e.Name == "InteractionSalt"); + + var odds = backup.Manifest.Collections.Single(c => c.Name == "Odd"); + Assert.Equal(1, odds.Count); + Assert.Contains(odds.Indexes, i => BsonDocument.Parse(i)["name"] == "at" && BsonDocument.Parse(i)["unique"].ToBoolean()); + Assert.Equal((16, "focal points are finite"), (backup.Manifest.MigrationNumber, backup.Manifest.NewestMigration)); + Assert.Equal("https://privapub.test", backup.Manifest.Host); + Assert.Equal(MongoFixture.Connection.Contains("directConnection=true"), backup.Manifest.Consistent); + Assert.Empty(await ServerBackup.Verify(Backups, backup.Id, Token)); + if (!OperatingSystem.IsWindows()) + { + Assert.Equal(UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.GroupRead, File.GetUnixFileMode(Path.Combine(directory, "db", "Odd.jsonl.gz"))); + Assert.False(File.GetUnixFileMode(directory).HasFlag(UnixFileMode.OtherRead)); + } + } + + static async Task ReadAll(string file) + { + if (!file.EndsWith(".gz", StringComparison.Ordinal)) + return await File.ReadAllTextAsync(file, Token); + await using var gzip = new GZipStream(File.OpenRead(file), CompressionMode.Decompress); + using var reader = new StreamReader(gzip); + return await reader.ReadToEndAsync(Token); + } + + [Fact] + public async Task Media_rows_files_are_linked_from_the_live_directory_or_the_trash() + { + Directory.CreateDirectory(Path.Combine(Media, "2026", "10")); + await File.WriteAllTextAsync(Path.Combine(Media, "2026", "10", "live.jpg"), "live", Token); + await File.WriteAllTextAsync(Path.Combine(Media, "2026", "10", "live-preview.jpg"), "preview", Token); + Directory.CreateDirectory(Path.Combine(Trash, "2026", "10")); + await File.WriteAllTextAsync(Path.Combine(Trash, "2026", "10", "moving.jpg"), "moving", Token); + await File.WriteAllTextAsync(Path.Combine(Media, "2026", "10", "trashed.jpg"), "trashed", Token); + await _database.GetCollection("MediaAttachment").InsertManyAsync([ + new BsonDocument { { "FilePath", "2026/10/live.jpg" }, { "PreviewPath", "2026/10/live-preview.jpg" }, { "TrashedAt", BsonNull.Value } }, + new BsonDocument { { "FilePath", "2026/10/moving.jpg" } },//a file the janitor moved while its row was being trashed + new BsonDocument { { "FilePath", "2026/10/gone.jpg" } }, + new BsonDocument { { "FilePath", "2026/10/trashed.jpg" }, { "TrashedAt", DateTime.UtcNow } }, + new BsonDocument { { "FilePath", "../../etc/passwd" } }], cancellationToken: Token); + + var (backup, _) = await ServerBackup.Create(Context(), "manual", dbOnly: false, Token); + + var media = Path.Combine(Backups, backup.Id, "media"); + Assert.Equal("live", await File.ReadAllTextAsync(Path.Combine(media, "2026", "10", "live.jpg"), Token)); + Assert.Equal("preview", await File.ReadAllTextAsync(Path.Combine(media, "2026", "10", "live-preview.jpg"), Token)); + Assert.Equal("moving", await File.ReadAllTextAsync(Path.Combine(media, "2026", "10", "moving.jpg"), Token)); + Assert.False(File.Exists(Path.Combine(media, "2026", "10", "trashed.jpg"))); + Assert.Equal(["2026/10/gone.jpg", "2026/10/live-preview.jpg", "2026/10/live.jpg", "2026/10/moving.jpg"], backup.Manifest.Media.List); + Assert.Equal((3, 1), (backup.Manifest.Media.Files, backup.Manifest.Media.Missing)); + + // a link, not a copy: the live file deleted, the backup's stays + File.Delete(Path.Combine(Media, "2026", "10", "live.jpg")); + Assert.Equal("live", await File.ReadAllTextAsync(Path.Combine(media, "2026", "10", "live.jpg"), Token)); + + // db-only lists them and links none + var (listed, _) = await ServerBackup.Create(Context(), "pre-deploy", dbOnly: true, Token); + Assert.False(Directory.Exists(Path.Combine(Backups, listed.Id, "media"))); + Assert.Equal(backup.Manifest.Media.List, listed.Manifest.Media.List); + Assert.Empty(await ServerBackup.Verify(Backups, listed.Id, Token)); + } + + [Fact] + public async Task Verify_finds_an_altered_or_missing_file() + { + await _database.GetCollection("Post").InsertOneAsync(new BsonDocument("Content", "hello"), cancellationToken: Token); + await _database.GetCollection("Avatar").InsertOneAsync(new BsonDocument("UserName", "someone"), cancellationToken: Token); + var (backup, _) = await ServerBackup.Create(Context(), "manual", dbOnly: false, Token); + var db = Path.Combine(Backups, backup.Id, "db"); + + await File.AppendAllTextAsync(Path.Combine(db, "Post.jsonl.gz"), "tampered", Token); + File.Delete(Path.Combine(db, "Avatar.jsonl.gz")); + + Assert.Equal(["Avatar: missing", "Post: altered"], (await ServerBackup.Verify(Backups, backup.Id, Token)).Order()); + Assert.Equal(["no such backup, or no manifest"], await ServerBackup.Verify(Backups, "../" + backup.Id, Token)); + } + + [Fact] + public async Task One_backup_at_a_time() + { + await using (var held = await MaintenanceLock.Take("restore", Token)) + { + Assert.NotNull(held); + var (backup, error) = await ServerBackup.Create(Context(), "manual", dbOnly: true, Token); + Assert.Null(backup); + Assert.Contains("already running", error); + Assert.Equal("restore", (await MaintenanceLock.Current(Token)).What); + } + Assert.Null(await MaintenanceLock.Current(Token)); + Assert.NotNull((await ServerBackup.Create(Context(), "manual", dbOnly: true, Token)).Backup); + } + + [Fact] + public async Task A_holder_that_died_is_taken_over() + { + await using var dead = await MaintenanceLock.Take("backup", Token); + await DB.Default.Update().Match(l => l.ID == MaintenanceLock.Name) + .Modify(l => l.Heartbeat, DateTime.UtcNow.AddMinutes(-3)).ExecuteAsync(Token); + + Assert.Null(await MaintenanceLock.Current(Token)); + await using var alive = await MaintenanceLock.Take("restore", Token); + Assert.NotNull(alive); + Assert.Equal("restore", (await MaintenanceLock.Current(Token)).What); + } + + // the newest 7 nightly, the newest of each of the 4 weeks before, the newest 3 pre-deploy; manual ones kept + [Fact] + public void Rotation_keeps_days_weeks_and_the_last_deploys() + { + var today = new DateTime(2026, 10, 7, 3, 30, 0, DateTimeKind.Utc); + for (var day = 0; day < 60; day++) + Fake("nightly", today.AddDays(-day)); + for (var deploy = 0; deploy < 5; deploy++) + Fake("pre-deploy", today.AddDays(-deploy).AddHours(5)); + Fake("manual", today.AddYears(-1)); + Directory.CreateDirectory(Path.Combine(Backups, "20260901-000000-nightly" + ServerBackup.PartialSuffix)); + Directory.SetLastWriteTimeUtc(Path.Combine(Backups, "20260901-000000-nightly" + ServerBackup.PartialSuffix), today.AddDays(-30)); + + ServerBackup.Retain(Backups, new BackupOptions()); + + var kept = ServerBackup.List(Backups); + var nightly = kept.Where(b => b.Kind == "nightly").Select(b => b.CreatedAt).ToList(); + Assert.Equal(11, nightly.Count); + Assert.Equal(Enumerable.Range(0, 7).Select(d => today.AddDays(-d)), nightly.Take(7)); + Assert.Equal(4, nightly.Skip(7).Select(d => System.Globalization.ISOWeek.GetWeekOfYear(d)).Distinct().Count()); + Assert.Equal(3, kept.Count(b => b.Kind == "pre-deploy")); + Assert.Single(kept, b => b.Kind == "manual"); + Assert.Empty(Directory.EnumerateDirectories(Backups, "*" + ServerBackup.PartialSuffix)); + } + + void Fake(string kind, DateTime at) + { + var directory = Path.Combine(Backups, $"{at:yyyyMMdd-HHmmss}-{kind}"); + Directory.CreateDirectory(Path.Combine(directory, "db")); + new ArchiveManifest { Kind = kind, CreatedAt = at }.Write(directory); + } + + [Theory] + [InlineData("2026-10-07T03:29:00", "2026-10-06T03:31:00", false)]//yesterday's is the last one due + [InlineData("2026-10-07T03:31:00", "2026-10-06T03:31:00", true)] + [InlineData("2026-10-07T03:31:00", "2026-10-07T03:30:30", false)] + [InlineData("2026-10-07T01:00:00", "2026-10-05T03:31:00", true)]//missed last night: at once + public void A_nightly_backup_is_due_once_a_night(string now, string last, bool due) => + Assert.Equal(due, BackupScheduler.IsDue(DateTime.Parse(now, null, System.Globalization.DateTimeStyles.AdjustToUniversal), + new TimeOnly(3, 30), [DateTime.Parse(last, null, System.Globalization.DateTimeStyles.AdjustToUniversal)])); + + // the deploy's: the server's own database, through the configuration, without media links + [Fact] + public async Task The_deploy_backs_up_from_the_command_line() + { + var host = await PrivaPubHost.Shared(); + var output = new StringWriter(); + + Assert.Equal(0, await AdminCommands.Run(["backup", "--kind", "pre-deploy", "--db-only"], host.Services, output: output)); + var id = output.ToString().Split(':')[0]; + Assert.EndsWith("-pre-deploy", id); + Assert.Equal(2, await AdminCommands.Run(["backup", "--kind", "nightly"], host.Services, output: TextWriter.Null)); + + output = new StringWriter(); + Assert.Equal(0, await AdminCommands.Run(["backups"], host.Services, output: output)); + Assert.StartsWith(id + "\tpre-deploy", output.ToString()); + output = new StringWriter(); + Assert.Equal(0, await AdminCommands.Run(["backup", "verify", id], host.Services, output: output)); + Assert.Contains("whole", output.ToString()); + + var backups = host.Get(); + var manifest = backups.Find(id).Manifest; + Assert.Contains(manifest.Collections, c => c.Name == "Avatar"); + Assert.DoesNotContain(manifest.Collections, c => c.Name is "InteractionSalt" or "Job" or "MaintenanceLock" or "openiddict.tokens" or "AppConfiguration"); + Assert.Equal(ServerBackup.CodeMigration(), manifest.MigrationNumber); + Assert.True(backups.Delete(id)); + } + } +} diff --git a/PrivaPub.Tests/Support/Host/PrivaPubHost.cs b/PrivaPub.Tests/Support/Host/PrivaPubHost.cs index a6e55f4..9873700 100644 --- a/PrivaPub.Tests/Support/Host/PrivaPubHost.cs +++ b/PrivaPub.Tests/Support/Host/PrivaPubHost.cs @@ -24,7 +24,7 @@ namespace PrivaPub.Tests.Support.Host static readonly SemaphoreSlim Boot = new(1, 1); static readonly Type[] Unwanted = { typeof(JobWorker), typeof(MediaJanitor), typeof(OAuthPruner), typeof(StatisticsSchedule), - typeof(PrivaPub.Domain.Social.FollowResender) }; + typeof(PrivaPub.Domain.Social.FollowResender), typeof(PrivaPub.Infrastructure.Backup.BackupScheduler) }; static PrivaPubHost _shared; readonly string _mediaRoot = Path.Combine(Path.GetTempPath(), $"privapub-tests-{Guid.NewGuid():N}"); @@ -78,6 +78,8 @@ namespace PrivaPub.Tests.Support.Host ["Statistics:Geo:AutoUpdate"] = "false", ["Statistics:Cdn:AutoUpdate"] = "false", ["Media:Root"] = _mediaRoot, + ["Backups:Root"] = _mediaRoot + "-backups", + ["Backups:Nightly"] = "false", ["RateLimits:UploadsBurst"] = "1000", ["RateLimits:ProxyBurst"] = "100000", ["Logging:LogLevel:Default"] = "Warning", @@ -117,8 +119,11 @@ namespace PrivaPub.Tests.Support.Host protected override void Dispose(bool disposing) { base.Dispose(disposing); - if (disposing && Directory.Exists(_mediaRoot)) - Directory.Delete(_mediaRoot, recursive: true); + if (!disposing) + return; + foreach (var directory in new[] { _mediaRoot, _mediaRoot + "-backups", _mediaRoot + "-trash", _mediaRoot + "-incoming", _mediaRoot + "-proxy" }) + if (Directory.Exists(directory)) + Directory.Delete(directory, recursive: true); } sealed class ClientAddressFilter : IStartupFilter diff --git a/PrivaPub/Domain/Media/MediaProxy.cs b/PrivaPub/Domain/Media/MediaProxy.cs index 263c3b5..b911065 100644 --- a/PrivaPub/Domain/Media/MediaProxy.cs +++ b/PrivaPub/Domain/Media/MediaProxy.cs @@ -361,7 +361,8 @@ namespace PrivaPub.Domain.Media // one pass: // - uploads never posted for a day (and not waiting for a scheduled post, nor a profile picture) go to the trash; // - trashed files still served (a crash between the mark and the move) are moved out; - // - trashed files past their grace are deleted, with their rows; + // - trashed files past their grace are deleted, with their rows, unless a backup or a restore is running (one may be + // reading the trash); // - the proxy cache is trimmed to its size, oldest first public async Task Sweep(CancellationToken token) { @@ -371,10 +372,11 @@ namespace PrivaPub.Domain.Media var trashed = await DB.Default.Find().Match(m => m.TrashedAt != null).Limit(2000).ExecuteAsync(token); var purgeBefore = DateTime.UtcNow - TrashGrace; + var maintenance = await Infrastructure.Backup.MaintenanceLock.Current(token) != default; var purged = 0; foreach (var row in trashed) { - if (row.TrashedAt < purgeBefore) + if (row.TrashedAt < purgeBefore && !maintenance) { await _media.Purge(row, token); purged++; diff --git a/PrivaPub/Infrastructure/Backup/ArchiveManifest.cs b/PrivaPub/Infrastructure/Backup/ArchiveManifest.cs new file mode 100644 index 0000000..cbc2b42 --- /dev/null +++ b/PrivaPub/Infrastructure/Backup/ArchiveManifest.cs @@ -0,0 +1,65 @@ +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace PrivaPub.Infrastructure.Backup +{ + // What a backup holds (manifest.json at its root): enough to check it is whole, to restore it on this host only, and to + // rebuild what Mongo had (each collection's indexes). + public sealed class ArchiveManifest + { + public const int CurrentFormat = 1; + public const string FileName = "manifest.json"; + + public int Format { get; set; } = CurrentFormat; + public string Kind { get; set; }//nightly, manual, pre-deploy, pre-restore, uploaded + public DateTime CreatedAt { get; set; } = DateTime.UtcNow; + public string Host { get; set; }//BackendBaseAddress: URIs and media URLs are stored whole, so only this host can restore it + public string AppCommit { get; set; } + public string AppRef { get; set; } + public string NewestMigration { get; set; } + public int MigrationNumber { get; set; } + public bool Consistent { get; set; }//read at one instant (a snapshot session on a replica set) + public bool DbOnly { get; set; }//no media files, only their list (the deploy's, which can't link www-data's files) + public List Collections { get; set; } = []; + public List Excluded { get; set; } = []; + public MediaEntry Media { get; set; } = new(); + + public sealed class CollectionEntry + { + public string Name { get; set; } + public long Count { get; set; } + public long Bytes { get; set; } + public string Sha256 { get; set; } + public List Indexes { get; set; } = [];//each as canonical Extended JSON + } + + public sealed class ExcludedEntry + { + public string Name { get; set; } + public string Why { get; set; } + } + + public sealed class MediaEntry + { + public int Files { get; set; } + public long Bytes { get; set; } + public int Missing { get; set; } + public List List { get; set; } = []; + } + + static readonly JsonSerializerOptions Json = new() { WriteIndented = true, PropertyNamingPolicy = JsonNamingPolicy.CamelCase, DefaultIgnoreCondition = JsonIgnoreCondition.Never }; + + public static ArchiveManifest Read(string directory) + { + var path = Path.Combine(directory, FileName); + return File.Exists(path) ? JsonSerializer.Deserialize(File.ReadAllText(path), Json) : default; + } + + public void Write(string directory) + { + using var file = new FileStream(Path.Combine(directory, FileName), FileMode.Create, FileAccess.Write); + JsonSerializer.Serialize(file, this, Json); + file.Flush(flushToDisk: true); + } + } +} diff --git a/PrivaPub/Infrastructure/Backup/BackupOptions.cs b/PrivaPub/Infrastructure/Backup/BackupOptions.cs new file mode 100644 index 0000000..3522858 --- /dev/null +++ b/PrivaPub/Infrastructure/Backup/BackupOptions.cs @@ -0,0 +1,13 @@ +namespace PrivaPub.Infrastructure.Backup +{ + public class BackupOptions + { + public string Root { get; set; }//where backups are kept (/var/lib/privapub/backups); -backups by default + public string NightlyAt { get; set; } = "03:30";//UTC + public int KeepDaily { get; set; } = 7; + public int KeepWeekly { get; set; } = 4; + public int KeepPreDeploy { get; set; } = 3; + public int KeepPreRestore { get; set; } = 3; + public bool Nightly { get; set; } = true; + } +} diff --git a/PrivaPub/Infrastructure/Backup/Backups.cs b/PrivaPub/Infrastructure/Backup/Backups.cs new file mode 100644 index 0000000..97d0c9b --- /dev/null +++ b/PrivaPub/Infrastructure/Backup/Backups.cs @@ -0,0 +1,89 @@ +using Microsoft.Extensions.Options; + +using MongoDB.Entities; + +using PrivaPub.Domain.Media; +using PrivaPub.Models; + +namespace PrivaPub.Infrastructure.Backup +{ + // The server's backups, wherever they are started from: the CLI, the nightly schedule, the administrator's page. + public class Backups(IOptionsMonitor options, IOptionsMonitor app, IMediaService media) + { + /// Where backups are kept: Backups:Root (production's /var/lib/privapub/backups), else beside the media root. + public string Root => Path.GetFullPath(options.CurrentValue.Root ?? media.Root.TrimEnd(Path.DirectorySeparatorChar) + "-backups"); + + public BackupOptions Options => options.CurrentValue; + + public string Host => app.CurrentValue.BackendBaseAddress?.TrimEnd('/'); + + public BackupContext Context() => new(DB.Default.Database(), Root, media.Root, media.TrashRoot, Host, options.CurrentValue); + + public Task<(BackupInfo Backup, string Error)> Create(string kind, bool dbOnly, CancellationToken token) => + ServerBackup.Create(Context(), kind, dbOnly, token); + + public List List() => ServerBackup.List(Root); + + public BackupInfo Find(string id) => ServerBackup.Find(Root, id); + + public Task> Verify(string id, CancellationToken token) => ServerBackup.Verify(Root, id, token); + + public bool Delete(string id) => ServerBackup.Delete(Root, id); + } + + // A nightly backup at Backups:NightlyAt (UTC), or as soon as the service is up after missing it; the old ones rotated + // out as each is made. + public class BackupScheduler(Backups backups, ILogger logger) : BackgroundService + { + static readonly TimeSpan Interval = TimeSpan.FromMinutes(10); + + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + while (!stoppingToken.IsCancellationRequested) + { + try + { + await Task.Delay(Interval, stoppingToken); + } + catch (OperationCanceledException) + { + return; + } + try + { + await RunIfDue(DateTime.UtcNow, stoppingToken); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + logger.LogError(ex, "The nightly backup failed"); + } + } + } + + /// Backs up when the day's time has come and there is no nightly backup since: whether it did. + public async Task RunIfDue(DateTime now, CancellationToken token) + { + var options = backups.Options; + if (!options.Nightly || !TimeOnly.TryParse(options.NightlyAt, System.Globalization.CultureInfo.InvariantCulture, out var at) + || !IsDue(now, at, backups.List().Where(b => b.Kind == "nightly").Select(b => b.CreatedAt))) + return false; + var (made, error) = await backups.Create("nightly", dbOnly: false, token); + if (made == default) + { + logger.LogWarning("The nightly backup was not made: {Error}", error); + return false; + } + logger.LogInformation("Nightly backup {Id}: {Collections} collections, {Files} media files", made.Id, made.Manifest.Collections.Count, made.Manifest.Media.Files); + return true; + } + + /// Whether the last time of day for a nightly backup has passed with no nightly backup made since. + public static bool IsDue(DateTime now, TimeOnly at, IEnumerable nightlies) + { + var due = now.Date + at.ToTimeSpan(); + if (now < due) + due = due.AddDays(-1); + return !nightlies.Any(made => made >= due); + } + } +} diff --git a/PrivaPub/Infrastructure/Backup/HardLink.cs b/PrivaPub/Infrastructure/Backup/HardLink.cs new file mode 100644 index 0000000..c225353 --- /dev/null +++ b/PrivaPub/Infrastructure/Backup/HardLink.cs @@ -0,0 +1,35 @@ +using System.Runtime.InteropServices; + +namespace PrivaPub.Infrastructure.Backup +{ + // A second name for a file (.NET has no API for it): a backup links the live media, which costs no disk, since names are + // random and files never change; deleting the live one leaves the backup's. + static class HardLink + { + // strings go to libc as UTF-8 on Unix + [DllImport("libc", EntryPoint = "link", SetLastError = true, CharSet = CharSet.Ansi)] + static extern int Link(string existing, string created); + + /// Links, or copies where a link can't be made (another disk, a filesystem refusing it). + public static bool LinkOrCopy(string existing, string created) + { + Directory.CreateDirectory(Path.GetDirectoryName(created)!); + if (OperatingSystem.IsLinux()) + { + try + { + if (Link(existing, created) == 0) + return true; + } + catch (DllNotFoundException) + { + } + catch (EntryPointNotFoundException) + { + } + } + File.Copy(existing, created, overwrite: true); + return false; + } + } +} diff --git a/PrivaPub/Infrastructure/Backup/MaintenanceLock.cs b/PrivaPub/Infrastructure/Backup/MaintenanceLock.cs new file mode 100644 index 0000000..71e9b46 --- /dev/null +++ b/PrivaPub/Infrastructure/Backup/MaintenanceLock.cs @@ -0,0 +1,75 @@ +using MongoDB.Driver; +using MongoDB.Entities; + +namespace PrivaPub.Infrastructure.Backup +{ + // One backup or restore at a time, whoever starts it (the CLI, the nightly schedule, the administrator's page): a + // document held with a heartbeat. A holder that died (no heartbeat for two minutes) is taken over. The collection is + // never backed up, nor dropped by a restore. + public class MaintenanceLock : Entity + { + public const string Name = "maintenance"; + static readonly TimeSpan Stale = TimeSpan.FromMinutes(2); + static readonly TimeSpan Beat = TimeSpan.FromSeconds(30); + + public string Owner { get; set; } + public string What { get; set; } + public DateTime Since { get; set; } + public DateTime Heartbeat { get; set; } + + /// Takes the lock for what is said, or null when someone else holds it; disposing the result frees it. + public static async Task Take(string what, CancellationToken token) + { + var owner = $"{Environment.MachineName}:{Environment.ProcessId}:{Guid.NewGuid():N}"; + var now = DateTime.UtcNow; + var stale = now - Stale; + try + { + var taken = await DB.Default.UpdateAndGet() + .Match(l => l.ID == Name && (l.Heartbeat < stale || l.Owner == null)) + .Modify(l => l.Owner, owner) + .Modify(l => l.What, what) + .Modify(l => l.Since, now) + .Modify(l => l.Heartbeat, now) + .Option(o => o.IsUpsert = true) + .ExecuteAsync(token); + return taken?.Owner == owner ? new Held(owner) : default; + } + catch (MongoCommandException ex) when (ex.Code == 11000) + { + return default;//held: the upsert met the live lock's id + } + catch (MongoWriteException ex) when (ex.WriteError?.Category == ServerErrorCategory.DuplicateKey) + { + return default; + } + } + + /// Who holds it now and for what, or null. + public static async Task Current(CancellationToken token) + { + var held = await DB.Default.Find().Match(l => l.ID == Name).ExecuteFirstAsync(token); + return held?.Owner != null && held.Heartbeat >= DateTime.UtcNow - Stale ? held : default; + } + + public sealed class Held : IAsyncDisposable + { + readonly string _owner; + readonly Timer _heartbeat; + + public Held(string owner) + { + _owner = owner; + _heartbeat = new Timer(_ => _ = DB.Default.Update().Match(l => l.ID == Name && l.Owner == owner) + .Modify(l => l.Heartbeat, DateTime.UtcNow).ExecuteAsync(), default, Beat, Beat); + } + + public async ValueTask DisposeAsync() + { + await _heartbeat.DisposeAsync(); + await DB.Default.Update().Match(l => l.ID == Name && l.Owner == _owner) + .Modify(l => l.Owner, null).Modify(l => l.Heartbeat, DateTime.MinValue).ExecuteAsync(); + } + } + } +} diff --git a/PrivaPub/Infrastructure/Backup/RestoreMarker.cs b/PrivaPub/Infrastructure/Backup/RestoreMarker.cs new file mode 100644 index 0000000..ac9abc7 --- /dev/null +++ b/PrivaPub/Infrastructure/Backup/RestoreMarker.cs @@ -0,0 +1,56 @@ +using System.Text.Json; + +namespace PrivaPub.Infrastructure.Backup +{ + // A restore asked for (by the administrator's page or the CLI) and carried out at the next boot, before anything + // else (Program, MaintenanceGate): restore.json in the backups' directory, written whole or not at all. + public sealed class RestoreMarker + { + public const string FileName = "restore.json"; + public const int MaxAttempts = 3; + + public string Backup { get; set; }//the backup restored + public string PreRestore { get; set; }//the backup taken just before, what the protective merge reads + public string State { get; set; } = "pending";//pending, then running + public int Attempts { get; set; } + public DateTime RequestedAt { get; set; } = DateTime.UtcNow; + public string Error { get; set; } + + public static string PathIn(string backupsRoot) => Path.Combine(backupsRoot, FileName); + + public static RestoreMarker Read(string backupsRoot) + { + var path = PathIn(backupsRoot); + if (!File.Exists(path)) + return default; + try + { + return JsonSerializer.Deserialize(File.ReadAllText(path)); + } + catch (JsonException) + { + return default; + } + } + + public void Write(string backupsRoot) + { + Directory.CreateDirectory(backupsRoot); + var path = PathIn(backupsRoot); + var part = path + ".part"; + using (var file = new FileStream(part, FileMode.Create, FileAccess.Write)) + { + JsonSerializer.Serialize(file, this); + file.Flush(flushToDisk: true); + } + File.Move(part, path, overwrite: true); + } + + public static void Clear(string backupsRoot) + { + var path = PathIn(backupsRoot); + if (File.Exists(path)) + File.Delete(path); + } + } +} diff --git a/PrivaPub/Infrastructure/Backup/ServerBackup.cs b/PrivaPub/Infrastructure/Backup/ServerBackup.cs new file mode 100644 index 0000000..5a3f8ec --- /dev/null +++ b/PrivaPub/Infrastructure/Backup/ServerBackup.cs @@ -0,0 +1,340 @@ +using MongoDB.Bson; +using MongoDB.Bson.IO; +using MongoDB.Driver; + +using PrivaPub.Infrastructure.Data; + +using System.Globalization; +using System.IO.Compression; +using System.Security.Cryptography; +using System.Text; + +namespace PrivaPub.Infrastructure.Backup +{ + // What a backup needs to know: where things are, and whose host it is. + public sealed record BackupContext(IMongoDatabase Database, string BackupsRoot, string MediaRoot, string TrashRoot, string Host, BackupOptions Options); + + public sealed record BackupInfo(string Id, string Kind, DateTime CreatedAt, long Bytes, ArchiveManifest Manifest); + + // The whole server, backed up as files (owner decision 2026-10-07: a whole-server backup, plain, protected by file + // permissions). Each backup is a directory - in the backups' root: + // - manifest.json: what it holds (ArchiveManifest); + // - db/.jsonl.gz: every document of each collection, one per line, in canonical Extended JSON (every BSON + // type kept as it was), all read at one instant when Mongo is a replica set (a snapshot session); + // - media/: the media files rows hold, as hard links to the live ones (no disk spent; a deleted file stays here + // until the backup is rotated out), copied where a link can't be made; a db-only backup lists them instead. + // It is written as .partial and renamed once whole. Left out: what belongs to the moment (Excluded). Everything is + // readable by the service's user and group only: it holds every persona's private key and private posts. + public static class ServerBackup + { + public const string PartialSuffix = ".partial"; + + public static readonly IReadOnlyDictionary Excluded = new Dictionary + { + ["InteractionSalt"] = "the day's statistics salt never outlives its day (owner rule)", + ["Job"] = "work in flight: deliveries and inbox processing belong to the moment", + ["Delivery"] = "work in flight, from before jobs", + ["EmailRecovery"] = "recovery codes live an hour", + ["openiddict.tokens"] = "sessions: a restore ends every one", + ["openiddict.authorizations"] = "sessions: a restore ends every one", + [nameof(MaintenanceLock)] = "who is backing up or restoring now", + ["AppConfiguration"] = "the configuration's copy holds the SMTP password, and is made again from appsettings at boot" + }; + + static readonly JsonWriterSettings Json = new() { OutputMode = JsonOutputMode.CanonicalExtendedJson, Indent = false }; + + // 2770: the service (www-data) and the deploy (in www-data's group) each read and rotate what the other wrote, and new + // files take the directory's group + const UnixFileMode DirectoryMode = UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute + | UnixFileMode.GroupRead | UnixFileMode.GroupWrite | UnixFileMode.GroupExecute | UnixFileMode.SetGroup; + const UnixFileMode FileMode0640 = UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.GroupRead; + + /// Takes the maintenance lock and backs the server up: the backup, or why not. + public static async Task<(BackupInfo Backup, string Error)> Create(BackupContext context, string kind, bool dbOnly, CancellationToken token) + { + await using var held = await MaintenanceLock.Take("backup", token); + if (held == default) + return (default, "a backup or a restore is already running"); + return await CreateHeld(context, kind, dbOnly, token); + } + + /// Backs the server up with the maintenance lock already held (a restore's own backup, taken first). + public static async Task<(BackupInfo Backup, string Error)> CreateHeld(BackupContext context, string kind, bool dbOnly, CancellationToken token) + { + var database = context.Database; + var names = (await (await database.ListCollectionNamesAsync(cancellationToken: token)).ToListAsync(token)) + .Where(n => !n.StartsWith("system.", StringComparison.Ordinal)) + .OrderBy(n => n, StringComparer.Ordinal) + .ToList(); + MakeDirectory(context.BackupsRoot); + var stats = await database.RunCommandAsync(new BsonDocument("dbStats", 1), cancellationToken: token); + var needed = (long)(stats.GetValue("dataSize", 0).ToDouble() * 1.1); + if (new DriveInfo(Path.GetFullPath(context.BackupsRoot)).AvailableFreeSpace < needed) + return (default, $"not enough free disk for a backup: about {needed / 1024 / 1024} MB needed"); + + var id = $"{DateTime.UtcNow.ToString("yyyyMMdd-HHmmss", CultureInfo.InvariantCulture)}-{kind}"; + var partial = Path.Combine(context.BackupsRoot, id + PartialSuffix); + MakeDirectory(partial); + MakeDirectory(Path.Combine(partial, "db")); + try + { + var manifest = new ArchiveManifest + { + Kind = kind, + Host = context.Host, + AppCommit = BuildInfo.Commit, + AppRef = BuildInfo.Ref, + DbOnly = dbOnly + }; + var replica = await MongoTopology.IsReplicaSet(database, token); + var media = new SortedSet(StringComparer.Ordinal); + using var session = replica ? await database.Client.StartSessionAsync(new ClientSessionOptions { Snapshot = true }, token) : default; + var window = replica ? await RaiseSnapshotWindow(database, token) : default(int?); + try + { + foreach (var name in names) + { + if (Excluded.TryGetValue(name, out var why)) + { + manifest.Excluded.Add(new ArchiveManifest.ExcludedEntry { Name = name, Why = why }); + continue; + } + manifest.Collections.Add(await Dump(database, session, name, partial, name == "MediaAttachment" ? media : default, token)); + } + (manifest.NewestMigration, manifest.MigrationNumber) = await NewestMigration(database, session, token); + } + finally + { + if (window is { } previous) + await SetSnapshotWindow(database, previous, CancellationToken.None); + } + manifest.Consistent = replica; + + manifest.Media.List = [.. media]; + if (!dbOnly) + foreach (var relative in media) + { + var source = new[] { context.MediaRoot, context.TrashRoot }.Select(root => Inside(root, relative)).FirstOrDefault(File.Exists); + if (source == default) + { + manifest.Media.Missing++; + continue; + } + HardLink.LinkOrCopy(source, Inside(Path.Combine(partial, "media"), relative)); + manifest.Media.Files++; + manifest.Media.Bytes += new FileInfo(source).Length; + } + + manifest.Write(partial); + Directory.Move(partial, Path.Combine(context.BackupsRoot, id)); + Retain(context.BackupsRoot, context.Options); + return (Info(context.BackupsRoot, id), default); + } + catch + { + if (Directory.Exists(partial)) + Directory.Delete(partial, recursive: true); + throw; + } + } + + // a collection read as raw BSON in batches, each document a line of canonical Extended JSON, gzipped; the media rows' + // files collected on the way + static async Task Dump(IMongoDatabase database, IClientSessionHandle session, string name, string directory, + ISet media, CancellationToken token) + { + var collection = database.GetCollection(name); + var path = Path.Combine(directory, "db", name + ".jsonl.gz"); + var count = 0L; + await using (var file = NewFile(path)) + await using (var gzip = new GZipStream(file, CompressionLevel.Fastest)) + await using (var writer = new StreamWriter(gzip, new UTF8Encoding(false))) + { + var options = new FindOptions { BatchSize = 1000 }; + using var cursor = session == default + ? await collection.FindAsync(FilterDefinition.Empty, options, token) + : await collection.FindAsync(session, FilterDefinition.Empty, options, token); + while (await cursor.MoveNextAsync(token)) + foreach (var document in cursor.Current) + using (document) + { + await writer.WriteLineAsync(document.ToJson(Json)); + count++; + if (media != default) + Collect(document, media); + } + } + var indexes = await (await collection.Indexes.ListAsync(token)).ToListAsync(token); + return new ArchiveManifest.CollectionEntry + { + Name = name, + Count = count, + Bytes = new FileInfo(path).Length, + Sha256 = await Hash(path, token), + Indexes = [.. indexes.Select(i => i.ToJson(Json))] + }; + } + + // a media row's files, unless it is trashed + static void Collect(RawBsonDocument row, ISet media) + { + if (row.TryGetValue("TrashedAt", out var trashed) && !trashed.IsBsonNull) + return; + foreach (var field in new[] { "FilePath", "PreviewPath" }) + if (row.TryGetValue(field, out var value) && value.IsString && Safe(value.AsString)) + media.Add(value.AsString); + } + + // MongoDB.Entities records each migration run with its class's number (_016_... is 16) and its name in words + static async Task<(string Name, int Number)> NewestMigration(IMongoDatabase database, IClientSessionHandle session, CancellationToken token) + { + var history = database.GetCollection("_migration_history_"); + var options = new FindOptions { Sort = new BsonDocument("Number", -1), Limit = 1 }; + var newest = session == default + ? await (await history.FindAsync(FilterDefinition.Empty, options, token)).FirstOrDefaultAsync(token) + : await (await history.FindAsync(session, FilterDefinition.Empty, options, token)).FirstOrDefaultAsync(token); + return newest == default ? default : (newest.GetValue("Name", BsonNull.Value).ToString(), newest.GetValue("Number", 0).ToInt32()); + } + + /// The newest migration this build has: a backup made by a newer one can't be restored by it. + public static int CodeMigration() => typeof(ServerBackup).Assembly.GetTypes() + .Where(t => typeof(MongoDB.Entities.IMigration).IsAssignableFrom(t) && !t.IsAbstract) + .Select(t => int.TryParse(new string(t.Name.TrimStart('_').TakeWhile(char.IsDigit).ToArray()), out var number) ? number : 0) + .DefaultIfEmpty(0) + .Max(); + + // how long a snapshot stays readable: raised only while a backup reads (a longer window keeps old versions in the + // small cache all day); the value it had, to set back + static async Task RaiseSnapshotWindow(IMongoDatabase database, CancellationToken token) + { + var admin = database.Client.GetDatabase("admin"); + try + { + var current = await admin.RunCommandAsync(new BsonDocument { { "getParameter", 1 }, { "minSnapshotHistoryWindowInSeconds", 1 } }, cancellationToken: token); + var previous = current.GetValue("minSnapshotHistoryWindowInSeconds", 300).ToInt32(); + await SetSnapshotWindow(database, Math.Max(previous, 3600), token); + return previous; + } + catch (MongoCommandException) + { + return default; + } + } + + static async Task SetSnapshotWindow(IMongoDatabase database, int seconds, CancellationToken token) => + await database.Client.GetDatabase("admin").RunCommandAsync( + new BsonDocument { { "setParameter", 1 }, { "minSnapshotHistoryWindowInSeconds", seconds } }, cancellationToken: token); + + /// The backups kept, newest first (not one still being written). + public static List List(string backupsRoot) + { + if (!Directory.Exists(backupsRoot)) + return []; + return Directory.EnumerateDirectories(backupsRoot) + .Select(Path.GetFileName) + .Where(name => !name.EndsWith(PartialSuffix, StringComparison.Ordinal) && File.Exists(Path.Combine(backupsRoot, name, ArchiveManifest.FileName))) + .Select(name => Info(backupsRoot, name)) + .OrderByDescending(b => b.CreatedAt) + .ToList(); + } + + public static BackupInfo Find(string backupsRoot, string id) => + Safe(id) && !id.Contains('/') && Directory.Exists(Path.Combine(backupsRoot, id)) && !id.EndsWith(PartialSuffix, StringComparison.Ordinal) + ? Info(backupsRoot, id) + : default; + + static BackupInfo Info(string backupsRoot, string id) + { + var directory = Path.Combine(backupsRoot, id); + var manifest = ArchiveManifest.Read(directory); + var bytes = Directory.EnumerateFiles(Path.Combine(directory, "db")).Sum(f => new FileInfo(f).Length); + return new BackupInfo(id, manifest?.Kind, manifest?.CreatedAt ?? Directory.GetCreationTimeUtc(directory), bytes, manifest); + } + + /// Whether every file of a backup is what its manifest says: the problems found (none when whole). + public static async Task> Verify(string backupsRoot, string id, CancellationToken token) + { + var problems = new List(); + var backup = Find(backupsRoot, id); + if (backup?.Manifest == default) + return ["no such backup, or no manifest"]; + var directory = Path.Combine(backupsRoot, id); + foreach (var collection in backup.Manifest.Collections) + { + var path = Path.Combine(directory, "db", collection.Name + ".jsonl.gz"); + if (!File.Exists(path)) + problems.Add($"{collection.Name}: missing"); + else if (await Hash(path, token) != collection.Sha256) + problems.Add($"{collection.Name}: altered"); + } + if (!backup.Manifest.DbOnly) + foreach (var relative in backup.Manifest.Media.List.Where(r => !File.Exists(Inside(Path.Combine(directory, "media"), r)))) + problems.Add($"media {relative}: missing"); + return problems; + } + + /// Deletes a backup (its media links go; the live files stay). + public static bool Delete(string backupsRoot, string id) + { + if (Find(backupsRoot, id) == default) + return false; + Directory.Delete(Path.Combine(backupsRoot, id), recursive: true); + return true; + } + + // what is kept: the newest nightly ones for KeepDaily days and the newest of each of KeepWeekly weeks before, the + // newest pre-deploy and pre-restore ones; manual and uploaded ones until deleted; a backup that died writing goes + public static void Retain(string backupsRoot, BackupOptions options) + { + var all = List(backupsRoot); + var nightly = all.Where(b => b.Kind == "nightly").OrderByDescending(b => b.CreatedAt).ToList(); + var kept = nightly.Take(options.KeepDaily).ToHashSet(); + foreach (var week in nightly.Skip(options.KeepDaily).GroupBy(b => (ISOWeek.GetYear(b.CreatedAt), ISOWeek.GetWeekOfYear(b.CreatedAt))).Take(options.KeepWeekly)) + kept.Add(week.First()); + var doomed = nightly.Where(b => !kept.Contains(b)) + .Concat(all.Where(b => b.Kind == "pre-deploy").OrderByDescending(b => b.CreatedAt).Skip(options.KeepPreDeploy)) + .Concat(all.Where(b => b.Kind == "pre-restore").OrderByDescending(b => b.CreatedAt).Skip(options.KeepPreRestore)); + var stale = Directory.EnumerateDirectories(backupsRoot, "*" + PartialSuffix).Where(p => Directory.GetLastWriteTimeUtc(p) < DateTime.UtcNow.AddDays(-1)); + foreach (var directory in doomed.Select(b => Path.Combine(backupsRoot, b.Id)).Concat(stale).ToList()) + try + { + Directory.Delete(directory, recursive: true); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + //another user's backup this one may not delete: the next rotation tries again + } + } + + // a relative path from a database row or a manifest, never outside its root + static bool Safe(string relative) => + !string.IsNullOrEmpty(relative) && !Path.IsPathRooted(relative) && !relative.Split('/', '\\').Any(part => part is ".." or "."); + + public static string Inside(string root, string relative) + { + var full = Path.GetFullPath(Path.Combine(root, relative)); + if (!Safe(relative) || !full.StartsWith(Path.GetFullPath(root) + Path.DirectorySeparatorChar, StringComparison.Ordinal)) + throw new InvalidOperationException($"{relative} is not inside {root}"); + return full; + } + + static async Task Hash(string path, CancellationToken token) + { + await using var file = File.OpenRead(path); + return Convert.ToHexStringLower(await SHA256.HashDataAsync(file, token)); + } + + static FileStream NewFile(string path) => OperatingSystem.IsWindows() + ? new FileStream(path, FileMode.CreateNew, FileAccess.Write) + : new FileStream(path, new FileStreamOptions { Mode = FileMode.CreateNew, Access = FileAccess.Write, UnixCreateMode = FileMode0640 }); + + // set after creating, since the umask would take the group's write away + public static void MakeDirectory(string path) + { + var existed = Directory.Exists(path); + Directory.CreateDirectory(path); + if (!existed && !OperatingSystem.IsWindows()) + File.SetUnixFileMode(path, DirectoryMode); + } + } +} diff --git a/PrivaPub/Infrastructure/Cli/AdminCommands.cs b/PrivaPub/Infrastructure/Cli/AdminCommands.cs index 2827a97..c875532 100644 --- a/PrivaPub/Infrastructure/Cli/AdminCommands.cs +++ b/PrivaPub/Infrastructure/Cli/AdminCommands.cs @@ -24,8 +24,16 @@ namespace PrivaPub.Infrastructure.Cli PrivaPub admin smoke prints " " for the deploy's signed-in check PrivaPub admin media audit [--fix] media files against what holds them; --fix (as www-data) trashes what nothing holds and gives today's pictures their rows + PrivaPub admin backup [--kind manual|pre-deploy] [--db-only] + backs the server up; --db-only lists the media without + linking them (the deploy's, which can't link www-data's files) + PrivaPub admin backups the backups kept, newest first + PrivaPub admin backup verify whether a backup's files are what its manifest says """; + /// Whether a command runs before migrations: backups are of the database as it was. + public static bool BeforeMigrations(string[] args) => args is ["backup", ..] or ["backups"]; + public static async Task Run(string[] args, IServiceProvider services, TextReader input = default, TextWriter output = default) { input ??= Console.In; @@ -40,12 +48,75 @@ namespace PrivaPub.Infrastructure.Cli return await Smoke(services, persona, output); case ["media", "audit", .. var flags] when flags.All(f => f == "--fix"): return await AuditMedia(services, flags.Contains("--fix"), output); + case ["backup", "verify", var id]: + return await VerifyBackup(services, id, output); + case ["backup", .. var flags] when BackupFlags(flags, out var kind, out var dbOnly): + return await Backup(services, kind, dbOnly, output); + case ["backups"]: + return ListBackups(services, output); default: Console.Error.WriteLine(Usage); return 2; } } + static readonly string[] CommandKinds = ["manual", "pre-deploy"]; + + static bool BackupFlags(string[] flags, out string kind, out bool dbOnly) + { + kind = "manual"; + dbOnly = false; + for (var i = 0; i < flags.Length; i++) + switch (flags[i]) + { + case "--db-only": + dbOnly = true; + break; + case "--kind" when i + 1 < flags.Length && CommandKinds.Contains(flags[i + 1]): + kind = flags[++i]; + break; + default: + return false; + } + return true; + } + + static async Task Backup(IServiceProvider services, string kind, bool dbOnly, TextWriter output) + { + var (made, error) = await services.GetRequiredService().Create(kind, dbOnly, CancellationToken.None); + if (made == default) + { + Console.Error.WriteLine(error); + return 1; + } + var manifest = made.Manifest; + output.WriteLine($"{made.Id}: {manifest.Collections.Count} collections, {manifest.Collections.Sum(c => c.Count)} documents, {made.Bytes / 1024} KiB"); + output.WriteLine(dbOnly + ? $"{manifest.Media.List.Count} media files listed, not linked" + : $"{manifest.Media.Files} media files, {manifest.Media.Bytes / 1024 / 1024} MiB, {manifest.Media.Missing} missing"); + if (!manifest.Consistent) + output.WriteLine("not read at one instant: Mongo is not a replica set"); + return 0; + } + + static int ListBackups(IServiceProvider services, TextWriter output) + { + foreach (var backup in services.GetRequiredService().List()) + output.WriteLine($"{backup.Id}\t{backup.Kind}\t{backup.CreatedAt:u}\t{backup.Bytes / 1024} KiB\t{backup.Manifest?.Media.Files ?? 0} media{(backup.Manifest?.DbOnly == true ? " (listed)" : string.Empty)}"); + return 0; + } + + static async Task VerifyBackup(IServiceProvider services, string id, TextWriter output) + { + var problems = await services.GetRequiredService().Verify(id, CancellationToken.None); + foreach (var problem in problems) + output.WriteLine(problem); + if (problems.Count > 0) + return 1; + output.WriteLine($"{id}: whole"); + return 0; + } + static async Task AuditMedia(IServiceProvider services, bool fix, TextWriter output) { var report = await Domain.Media.MediaAudit.Run(services.GetRequiredService(), fix, CancellationToken.None); diff --git a/PrivaPub/Infrastructure/Data/EntityMaps.cs b/PrivaPub/Infrastructure/Data/EntityMaps.cs index bbad55c..a75c955 100644 --- a/PrivaPub/Infrastructure/Data/EntityMaps.cs +++ b/PrivaPub/Infrastructure/Data/EntityMaps.cs @@ -4,7 +4,22 @@ namespace PrivaPub.Infrastructure.Data { public static class EntityMaps { + static readonly Lock Gate = new(); + static bool _warm; + + // once per process, one map at a time: test hosts boot side by side public static void Warm() + { + lock (Gate) + { + if (_warm) + return; + Map(); + _warm = true; + } + } + + static void Map() { var collection = typeof(DB).GetMethods() .Single(m => m.Name == nameof(DB.Collection) && m.IsGenericMethodDefinition && m.GetParameters().Length == 0); diff --git a/PrivaPub/Program.cs b/PrivaPub/Program.cs index 0ac26cb..c85c288 100644 --- a/PrivaPub/Program.cs +++ b/PrivaPub/Program.cs @@ -68,6 +68,9 @@ try .AddSingleton() .AddSingleton() .AddHostedService() + .Configure(builder.Configuration.GetSection("Backups")) + .AddSingleton() + .AddHostedService() .PrivaPubMiddlewareConfiguration(); } catch (Exception ex) @@ -86,8 +89,6 @@ try var mongoSettings = builder.Configuration.GetSection(nameof(MongoSettings)).Get(); await DB.InitAsync(mongoSettings.Database, MongoClientSettings.FromConnectionString(mongoSettings.ConnectionString)); EntityMaps.Warm(); - await DB.Default.MigrateAsync(); - await Indexes.Create(); } catch (Exception ex) { @@ -108,6 +109,25 @@ try // Commands get every service but start nothing: no Kestrel, no hosted services, no media directory. The deploy runs // them as its own user, which can read the configuration and reach the private mongod but owns no www-data directory. + // Backups are taken before migrations: the deploy's is of the database as the live build left it. + if (args is ["admin", .. var command] && AdminCommands.BeforeMigrations(command)) + { + using var scope = app.Services.CreateScope(); + Environment.ExitCode = await AdminCommands.Run(command, scope.ServiceProvider); + return; + } + + try + { + await DB.Default.MigrateAsync(); + await Indexes.Create(); + } + catch (Exception ex) + { + Log.ForContext().Fatal(ex, $"{nameof(Program)}.{nameof(Program)}() DB Migrations"); + throw; + } + if (args is ["admin", ..]) { using var scope = app.Services.CreateScope(); diff --git a/PrivaPub/appsettings.Development.json b/PrivaPub/appsettings.Development.json index c0ee1c0..2cbaca3 100644 --- a/PrivaPub/appsettings.Development.json +++ b/PrivaPub/appsettings.Development.json @@ -4,6 +4,9 @@ "LogsDatabase": "logs", "ConnectionString": "mongodb://localhost:27017" }, + "Backups": { + "Nightly": false + }, "AppConfiguration": { "RequiresFirstTimeSetup": null, "Version": "0.0.0", diff --git a/PrivaPub/appsettings.Production.json b/PrivaPub/appsettings.Production.json index e83599c..74c401f 100644 --- a/PrivaPub/appsettings.Production.json +++ b/PrivaPub/appsettings.Production.json @@ -2,6 +2,9 @@ "Media": { "Root": "/var/lib/privapub/media" }, + "Backups": { + "Root": "/var/lib/privapub/backups" + }, "Registrations": { "Mode": "Invitations" }, diff --git a/deploy/max/setup.sh b/deploy/max/setup.sh index 3358f8d..1c66806 100755 --- a/deploy/max/setup.sh +++ b/deploy/max/setup.sh @@ -13,7 +13,9 @@ echo "== directories" install -d -o "$RUNNER" -g www-data -m 755 /var/www/$HOST install -d -o "$RUNNER" -g "$RUNNER" -m 750 /var/backups/$HOST install -d -o www-data -g www-data -m 750 /var/lib/privapub /var/lib/privapub/mongo -# backups: the service writes them, and so does the deploy (as $RUNNER, a member of www-data) before each deploy +# backups: the service writes them, and so does the deploy (as $RUNNER, a member of www-data) before each deploy; a runner +# just added to the group gets it when the runner restarts +id -nG "$RUNNER" | grep -qw www-data || usermod -aG www-data "$RUNNER" install -d -o www-data -g www-data -m 2770 /var/lib/privapub/backups echo "== sudoers" diff --git a/tools/pasture/appsettings.Pasture.json b/tools/pasture/appsettings.Pasture.json index 6a3bee2..61b2395 100644 --- a/tools/pasture/appsettings.Pasture.json +++ b/tools/pasture/appsettings.Pasture.json @@ -26,6 +26,7 @@ "Relays": [ "https://relay.test/actor", "https://aoderelay.test/actor" ] }, "Media": { "Root": "/tmp/privapub-media" }, + "Backups": { "Root": "/tmp/privapub-backups", "Nightly": false }, "RateLimits": { "AccountsPerMinute": 1000, "UploadsBurst": 1000, "UploadsPerMinute": 1000, "ProxyBurst": 100000, "ProxyPerMinute": 100000 }, "Registrations": { "Mode": "Open" }, "Statistics": { "Geo": { "AutoUpdate": false } },