diff --git a/CLAUDE.md b/CLAUDE.md index 3112b29..ad8e074 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -504,6 +504,28 @@ group www-data and reaches the private mongod; `sudo -u www-data` works too. rotation keeps 7 daily and 4 weekly nightly backups, 3 pre-deploy, 3 pre-restore, and manual and uploaded ones until deleted. CLI: `PrivaPub admin backup [--kind manual|pre-deploy] [--db-only]`, `admin backups`, `admin backup verify `; these run before migrations, so the deploy's backup is of the database as the live build left it. +- **Restores** (`ServerRestore`, `ProtectiveMerge`; owner decision 2026-10-07). Asking (`PrivaPub admin restore `, or + the administrator's page) checks the backup (same host, a format and newest migration this build reads, every hash) + and writes `restore.json` in the backups' root; the running service sees it within seconds (`RestoreWatcher`) and + stops, and systemd starts it again. `MaintenanceGate` (in `Program`, right after the build, before migrations, indexes + and hosted services) carries it out: + 1. a pre-restore backup P, taken once (a retry reuses it, never a half-restored database); + 2. every collection the backup holds dropped and imported raw with its indexes, every other one dropped, except what a + backup never holds, which stays as it is; + 3. media files the live directory lacks linked back from the backup (or moved from the trash); a restore deletes no file; + 4. **the protective merge from P: a restore never undoes a protective act.** Followers and follows are P's; blocks, + mutes, domain blocks, being blocked, reserved names, `DeletedObject`, reports, filters and OAuth applications are + the union, P's row winning; roots, personas, groups, posts and remote accounts deleted since are deleted again, with + what their deletion takes away; P's moderation of a persona and a root's password, e-mail, ban and policies stay; + roots, personas and groups made since become tombstones (deleted, names kept), local posts made since answer 410, + media made or trashed since go to the trash; + 5. every session ends (a new `SessionStamp` for each root, persona tokens deleted), circuits close, and a + `RestoreRecord` (never in a backup) tells what happened (`admin restore --status`). + + Each attempt redoes everything. Refused before anything changed, it is abandoned and recorded, and the server boots as + it was; failed midway, the process exits 1 and the next start tries again; after 3 failures it exits 75, which the + unit's `RestartPreventExitStatus` leaves down for someone to look. While `restore.json` exists, commands exit 75 (but + `admin restore --status`) and the deploy refuses to run. ## Code style diff --git a/PrivaPub.Tests/Infrastructure/RestoreTests.cs b/PrivaPub.Tests/Infrastructure/RestoreTests.cs new file mode 100644 index 0000000..45f9c36 --- /dev/null +++ b/PrivaPub.Tests/Infrastructure/RestoreTests.cs @@ -0,0 +1,316 @@ +using Microsoft.Extensions.Logging.Abstractions; + +using MongoDB.Bson; +using MongoDB.Driver; +using MongoDB.Entities; + +using PrivaPub.Infrastructure.Backup; +using PrivaPub.Infrastructure.Cli; +using PrivaPub.Tests.Support; +using PrivaPub.Tests.Support.Host; + +namespace PrivaPub.Tests.Infrastructure +{ + // A backup restored at boot: what was made since is lost, what protects is not (owner decision 2026-10-07). A server + // in a database of its own is backed up, changed, and restored; alone, since the maintenance lock is the server's one. + [Trait("Category", "Integration")] + [Xunit.Collection(nameof(Exclusive))] + public sealed class RestoreTests : IAsyncLifetime + { + const string Host = "https://privapub.test"; + + readonly string _scratch = Path.Combine(Path.GetTempPath(), $"privapub-restore-tests-{Guid.NewGuid():N}"); + IMongoDatabase _database; + + string Backups => Path.Combine(_scratch, "backups"); + string Media => Path.Combine(_scratch, "media"); + string Trash => Path.Combine(_scratch, "media-trash"); + + static CancellationToken Token => TestContext.Current.CancellationToken; + + public async ValueTask InitializeAsync() + { + Assert.SkipUnless(MongoFixture.Enabled, MongoFixture.Skip); + await PrivaPubHost.Shared(); + _database = DB.Default.Database().Client.GetDatabase($"PrivaPubRestore_{Guid.NewGuid():N}"); + Directory.CreateDirectory(Media); + Directory.CreateDirectory(Trash); + } + + public async ValueTask DisposeAsync() + { + if (_database != default) + await _database.Client.DropDatabaseAsync(_database.DatabaseNamespace.DatabaseName); + if (Directory.Exists(_scratch)) + Directory.Delete(_scratch, recursive: true); + } + + BackupContext Context(string host = Host) => new(_database, Backups, Media, Trash, host, new BackupOptions()); + + IMongoCollection C(string name) => _database.GetCollection(name); + + async Task One(string collection, BsonDocument filter) => + await (await C(collection).FindAsync(filter, cancellationToken: Token)).FirstOrDefaultAsync(Token); + + async Task> All(string collection) => + await (await C(collection).FindAsync(FilterDefinition.Empty, cancellationToken: Token)).ToListAsync(Token); + + static BsonDocument Id(ObjectId id) => new("_id", id); + + static Task Set(IMongoCollection collection, ObjectId id, BsonDocument fields) => + collection.UpdateOneAsync(Id(id), new BsonDocument("$set", fields), cancellationToken: Token); + + // the ids of the town this test builds + readonly ObjectId _alice = ObjectId.GenerateNewId(), _alicePersona = ObjectId.GenerateNewId(); + readonly ObjectId _deletedPost = ObjectId.GenerateNewId(), _editedPost = ObjectId.GenerateNewId(), _laterPost = ObjectId.GenerateNewId(); + readonly ObjectId _deletedMedia = ObjectId.GenerateNewId(), _laterMedia = ObjectId.GenerateNewId(); + readonly ObjectId _bob = ObjectId.GenerateNewId(), _bobPersona = ObjectId.GenerateNewId(); + + static string Uri(ObjectId post) => $"{Host}/peasants/alice/posts/{post}"; + + async Task TownBackedUp() + { + await File.WriteAllTextAsync(Path.Combine(Media, "deleted.jpg"), "deleted", Token); + await C("RootUser").InsertOneAsync(new BsonDocument { { "_id", _alice }, { "UserName", "alice" }, { "HashedPassword", "old" }, { "SessionStamp", "s1" }, { "Policies", new BsonArray { "IsUser" } }, { "DeletedAt", BsonNull.Value } }, cancellationToken: Token); + await C("Avatar").InsertOneAsync(new BsonDocument { { "_id", _alicePersona }, { "UserName", "alice" }, { "SuspendedAt", BsonNull.Value }, { "DeletionAt", BsonNull.Value } }, cancellationToken: Token); + await C("RootToAvatar").InsertOneAsync(new BsonDocument { { "RootId", _alice.ToString() }, { "AvatarId", _alicePersona.ToString() } }, cancellationToken: Token); + await C("ReservedName").InsertOneAsync(new BsonDocument("Name", "alice"), cancellationToken: Token); + await C("Post").InsertManyAsync([ + new BsonDocument { { "_id", _deletedPost }, { "GroupUserId", _alicePersona.ToString() }, { "IsFederatedCopy", false }, { "ObjectURI", Uri(_deletedPost) }, { "Text", "regret" }, + { "Media", new BsonArray { new BsonDocument("AttachmentId", _deletedMedia.ToString()) } }, { "DeletedAt", BsonNull.Value } }, + new BsonDocument { { "_id", _editedPost }, { "GroupUserId", _alicePersona.ToString() }, { "IsFederatedCopy", false }, { "ObjectURI", Uri(_editedPost) }, { "Text", "first words" }, { "DeletedAt", BsonNull.Value } } + ], cancellationToken: Token); + await C("TimelineEntry").InsertOneAsync(new BsonDocument { { "PostId", _deletedPost.ToString() }, { "ReblogOfPostId", BsonNull.Value } }, cancellationToken: Token); + await C("MediaAttachment").InsertOneAsync(new BsonDocument { { "_id", _deletedMedia }, { "PostId", _deletedPost.ToString() }, { "FilePath", "deleted.jpg" }, { "TrashedAt", BsonNull.Value } }, cancellationToken: Token); + await C("Follower").InsertOneAsync(new BsonDocument { { "LocalActorId", _alicePersona.ToString() }, { "ActorURI", "https://elsewhere.example/users/left" } }, cancellationToken: Token); + await C("RemoteInstance").InsertOneAsync(new BsonDocument { { "Host", "down.example" }, { "ConsecutiveFailures", 9 }, { "UnavailableUntil", DateTime.UtcNow.AddDays(1) } }, cancellationToken: Token); + await C("_migration_history_").InsertOneAsync(new BsonDocument { { "Number", 16 }, { "Name", "focal points are finite" } }, cancellationToken: Token); + await C("InteractionSalt").InsertOneAsync(new BsonDocument("Salt", "the backup's day"), cancellationToken: Token); + + var (backup, error) = await ServerBackup.Create(Context(), "manual", dbOnly: false, Token); + Assert.Null(error); + return backup; + } + + // what happens after the backup: protective acts, and things merely made or changed + async Task LifeGoesOn() + { + // alice regrets a post: deleted, its media trashed and moved out of what is served + await Set(C("Post"), _deletedPost, new BsonDocument { { "DeletedAt", DateTime.UtcNow }, { "Text", BsonNull.Value }, { "Media", new BsonArray() } }); + await Set(C("MediaAttachment"), _deletedMedia, new BsonDocument { { "TrashedAt", DateTime.UtcNow }, { "TrashReason", "deleted" } }); + File.Move(Path.Combine(Media, "deleted.jpg"), Path.Combine(Trash, "deleted.jpg")); + await C("TimelineEntry").DeleteManyAsync(new BsonDocument("PostId", _deletedPost.ToString()), Token); + // an edit, a new post with a picture: not protective + await Set(C("Post"), _editedPost, new BsonDocument("Text", "second words")); + await File.WriteAllTextAsync(Path.Combine(Media, "later.jpg"), "later", Token); + await C("Post").InsertOneAsync(new BsonDocument { { "_id", _laterPost }, { "GroupUserId", _alicePersona.ToString() }, { "IsFederatedCopy", false }, { "ObjectURI", Uri(_laterPost) }, { "Text", "later" }, { "DeletedAt", BsonNull.Value } }, cancellationToken: Token); + await C("MediaAttachment").InsertOneAsync(new BsonDocument { { "_id", _laterMedia }, { "PostId", _laterPost.ToString() }, { "FilePath", "later.jpg" }, { "TrashedAt", BsonNull.Value } }, cancellationToken: Token); + // a follower leaves, another comes + await C("Follower").DeleteManyAsync(FilterDefinition.Empty, Token); + await C("Follower").InsertOneAsync(new BsonDocument { { "LocalActorId", _alicePersona.ToString() }, { "ActorURI", "https://elsewhere.example/users/came" } }, cancellationToken: Token); + // alice blocks someone, the admin blocks a server, someone is reported, alice's persona is suspended, she changes + // her password + await C("Block").InsertOneAsync(new BsonDocument { { "AvatarId", _alicePersona.ToString() }, { "TargetActorURI", "https://elsewhere.example/users/troll" } }, cancellationToken: Token); + await C("DomainBlock").InsertOneAsync(new BsonDocument { { "Domain", "evil.example" }, { "Severity", "Suspend" } }, cancellationToken: Token); + await C("Report").InsertOneAsync(new BsonDocument("Comment", "harassment"), cancellationToken: Token); + await Set(C("Avatar"), _alicePersona, new BsonDocument("SuspendedAt", DateTime.UtcNow)); + await Set(C("RootUser"), _alice, new BsonDocument("HashedPassword", "new")); + // bob arrives + await C("RootUser").InsertOneAsync(new BsonDocument { { "_id", _bob }, { "UserName", "bob" }, { "HashedPassword", "bob's" }, { "Policies", new BsonArray { "IsUser" } }, { "DeletedAt", BsonNull.Value } }, cancellationToken: Token); + await C("Avatar").InsertOneAsync(new BsonDocument { { "_id", _bobPersona }, { "UserName", "bob" }, { "DeletionAt", BsonNull.Value } }, cancellationToken: Token); + await C("RootToAvatar").InsertOneAsync(new BsonDocument { { "RootId", _bob.ToString() }, { "AvatarId", _bobPersona.ToString() } }, cancellationToken: Token); + await C("ReservedName").InsertOneAsync(new BsonDocument("Name", "bob"), cancellationToken: Token); + // sessions, a list made, today's salt, work queued + await C("openiddict.tokens").InsertOneAsync(new BsonDocument("subject", "alice"), cancellationToken: Token); + await C("PersonaList").InsertOneAsync(new BsonDocument { { "AvatarId", _alicePersona.ToString() }, { "Title", "friends" } }, cancellationToken: Token); + await C("InteractionSalt").DeleteManyAsync(FilterDefinition.Empty, Token); + await C("InteractionSalt").InsertOneAsync(new BsonDocument("Salt", "today's"), cancellationToken: Token); + await C("Job").InsertOneAsync(new BsonDocument("Kind", "Deliver"), cancellationToken: Token); + } + + async Task AssertRestoredAndProtected(BackupInfo backup) + { + // lost: what was merely made or changed since + Assert.Equal("first words", (await One("Post", Id(_editedPost)))["Text"].AsString); + Assert.Null(await One("Post", Id(_laterPost))); + Assert.Empty(await All("PersonaList")); + Assert.Equal(16, (await One("_migration_history_", new BsonDocument()))["Number"].ToInt32()); + + // kept: every protective act + var deleted = await One("Post", Id(_deletedPost)); + Assert.False(deleted["DeletedAt"].IsBsonNull); + Assert.True(deleted["Text"].IsBsonNull); + Assert.Null(await One("TimelineEntry", new BsonDocument("PostId", _deletedPost.ToString()))); + Assert.False((await One("MediaAttachment", Id(_deletedMedia)))["TrashedAt"].IsBsonNull); + Assert.NotNull(await One("DeletedObject", new BsonDocument("ObjectURI", Uri(_laterPost)))); + var later = await One("MediaAttachment", Id(_laterMedia)); + Assert.Equal("restore: made after the backup", later["TrashReason"].AsString); + Assert.Equal(["https://elsewhere.example/users/came"], (await All("Follower")).Select(f => f["ActorURI"].AsString)); + Assert.NotNull(await One("Block", new BsonDocument("TargetActorURI", "https://elsewhere.example/users/troll"))); + Assert.NotNull(await One("DomainBlock", new BsonDocument("Domain", "evil.example"))); + Assert.NotNull(await One("Report", new BsonDocument("Comment", "harassment"))); + Assert.False((await One("Avatar", Id(_alicePersona)))["SuspendedAt"].IsBsonNull); + var alice = await One("RootUser", Id(_alice)); + Assert.Equal("new", alice["HashedPassword"].AsString); + Assert.NotEqual("s1", alice["SessionStamp"].AsString); + + // bob, made since: deleted, his name kept + var bob = await One("RootUser", Id(_bob)); + Assert.Equal($"deleted-{_bob}", bob["UserName"].AsString); + Assert.True(bob["HashedPassword"].IsBsonNull); + Assert.False(bob["DeletedAt"].IsBsonNull); + Assert.False((await One("Avatar", Id(_bobPersona)))["DeletionAt"].IsBsonNull); + Assert.NotNull(await One("ReservedName", new BsonDocument("Name", "bob"))); + + // what a backup never holds is as it was; sessions end; circuits close + Assert.Equal("today's", (await One("InteractionSalt", new BsonDocument()))["Salt"].AsString); + Assert.Single(await All("Job")); + Assert.Empty(await All("openiddict.tokens")); + Assert.Equal(0, (await One("RemoteInstance", new BsonDocument("Host", "down.example")))["ConsecutiveFailures"].ToInt32()); + + // the picture of the deleted post came back from the backup, to be trashed again by the janitor + Assert.True(File.Exists(Path.Combine(Media, "deleted.jpg"))); + Assert.Null(RestoreMarker.Read(Backups)); + var record = await (await _database.GetCollection(nameof(RestoreRecord)).FindAsync(r => r.Backup == backup.Id, cancellationToken: Token)).FirstAsync(Token); + Assert.Equal(["bob"], record.Report.RootsTombstoned); + Assert.Equal(["bob"], record.Report.PersonasTombstoned); + Assert.Equal(1, record.Report.PostsGone); + Assert.Equal(1, record.Report.PostsDeleted); + } + + [Fact] + public async Task A_restore_loses_what_was_made_since_but_never_a_protective_act() + { + var backup = await TownBackedUp(); + await LifeGoesOn(); + + Assert.Null(await ServerRestore.Request(Context(), backup.Id, "admin", Token)); + Assert.Contains("already pending", await ServerRestore.Request(Context(), backup.Id, "admin", Token)); + Assert.Equal(RestoreOutcome.Restored, await ServerRestore.ApplyPending(Context(), NullLogger.Instance, Token)); + + await AssertRestoredAndProtected(backup); + var before = ServerBackup.List(Backups).Single(b => b.Kind == "pre-restore"); + Assert.Equal(before.Id, (await (await _database.GetCollection(nameof(RestoreRecord)).FindAsync(FilterDefinition.Empty, cancellationToken: Token)).FirstAsync(Token)).PreRestore); + } + + // an attempt that died midway: the next one starts again from the backup, with the same pre-restore backup, never + // one of the half-restored database + [Fact] + public async Task A_restore_that_died_midway_converges_on_the_next_boot() + { + var backup = await TownBackedUp(); + await LifeGoesOn(); + Assert.Null(await ServerRestore.Request(Context(), backup.Id, "admin", Token)); + Assert.Equal(RestoreOutcome.Restored, await ServerRestore.ApplyPending(Context(), NullLogger.Instance, Token)); + var before = ServerBackup.List(Backups).Single(b => b.Kind == "pre-restore"); + + // as if the boot died after importing a collection, with the marker saying so + await C("Post").DeleteManyAsync(FilterDefinition.Empty, Token); + await C("Block").DeleteManyAsync(FilterDefinition.Empty, Token); + new RestoreMarker { Backup = backup.Id, PreRestore = before.Id, State = "running", Attempts = 1, RequestedBy = "admin" }.Write(Backups); + + Assert.Equal(RestoreOutcome.Restored, await ServerRestore.ApplyPending(Context(), NullLogger.Instance, Token)); + Assert.Single(ServerBackup.List(Backups), b => b.Kind == "pre-restore"); + await AssertRestoredAndProtected(backup); + } + + [Fact] + public async Task A_backup_from_elsewhere_from_a_newer_build_or_altered_is_refused() + { + var backup = await TownBackedUp(); + + Assert.Contains("not https://other.test", await ServerRestore.Request(Context("https://other.test"), backup.Id, "admin", Token)); + Assert.Contains("no such backup", await ServerRestore.Request(Context(), "../" + backup.Id, "admin", Token)); + + var manifest = ArchiveManifest.Read(Path.Combine(Backups, backup.Id)); + manifest.MigrationNumber = ServerBackup.CodeMigration() + 1; + manifest.Write(Path.Combine(Backups, backup.Id)); + Assert.Contains("newer build", await ServerRestore.Request(Context(), backup.Id, "admin", Token)); + + manifest.MigrationNumber = 16; + manifest.Write(Path.Combine(Backups, backup.Id)); + await File.AppendAllTextAsync(Path.Combine(Backups, backup.Id, "db", "Post.jsonl.gz"), "x", Token); + Assert.Contains("Post: altered", await ServerRestore.Request(Context(), backup.Id, "admin", Token)); + Assert.Null(RestoreMarker.Read(Backups)); + } + + // refused at boot before anything changed (the backup went meanwhile): recorded, and the server boots as it was + [Fact] + public async Task A_restore_that_cannot_start_is_abandoned_and_changes_nothing() + { + var backup = await TownBackedUp(); + await LifeGoesOn(); + Assert.Null(await ServerRestore.Request(Context(), backup.Id, "admin", Token)); + ServerBackup.Delete(Backups, backup.Id); + + Assert.Equal(RestoreOutcome.Abandoned, await ServerRestore.ApplyPending(Context(), NullLogger.Instance, Token)); + + Assert.Equal("second words", (await One("Post", Id(_editedPost)))["Text"].AsString); + Assert.Null(RestoreMarker.Read(Backups)); + Assert.DoesNotContain(ServerBackup.List(Backups), b => b.Kind == "pre-restore"); + var record = await (await _database.GetCollection(nameof(RestoreRecord)).FindAsync(FilterDefinition.Empty, cancellationToken: Token)).FirstAsync(Token); + Assert.True(record.Abandoned); + Assert.Contains("no such backup", record.Error); + } + + [Fact] + public async Task After_three_failures_it_waits_for_someone_to_look() + { + var backup = await TownBackedUp(); + await LifeGoesOn(); + new RestoreMarker { Backup = backup.Id, PreRestore = "gone", State = "running", Attempts = RestoreMarker.MaxAttempts, Error = "disk full" }.Write(Backups); + + Assert.Equal(RestoreOutcome.GaveUp, await ServerRestore.ApplyPending(Context(), NullLogger.Instance, Token)); + Assert.Equal("second words", (await One("Post", Id(_editedPost)))["Text"].AsString); + Assert.NotNull(RestoreMarker.Read(Backups)); + + await File.WriteAllTextAsync(RestoreMarker.PathIn(Backups), "{ not json", Token); + Assert.Equal(RestoreOutcome.GaveUp, await ServerRestore.ApplyPending(Context(), NullLogger.Instance, Token)); + } + + // a command waits while a restore does; asking how it goes doesn't + [Fact] + public async Task Commands_wait_for_a_restore_asked_for() + { + var host = await PrivaPubHost.Shared(); + var backups = host.Get(); + new RestoreMarker { Backup = "20260101-000000-manual", RequestedBy = "cli" }.Write(backups.Root); + try + { + Assert.Equal(MaintenanceGate.TryLater, await MaintenanceGate.Enter(host.Services, ["promote", "someone"], Token)); + Assert.Null(await MaintenanceGate.Enter(host.Services, ["restore", "--status"], Token)); + var output = new StringWriter(); + Assert.Equal(0, await AdminCommands.Run(["restore", "--status"], host.Services, output: output)); + Assert.Contains("waiting: 20260101-000000-manual, pending", output.ToString()); + Assert.Equal(1, await AdminCommands.Run(["restore", "20260101-000000-manual"], host.Services, output: TextWriter.Null)); + } + finally + { + RestoreMarker.Clear(backups.Root); + } + } + + // FEP-8fcf: for two weeks after a restore, no digests, and no follow undone for them + [Fact] + public async Task Followers_digests_rest_for_two_weeks_after_a_restore() + { + RestoreRecord.Forget(); + Assert.False(await RestoreRecord.InFollowersGrace(Token)); + var record = new RestoreRecord { Backup = "test", RestoredAt = DateTime.UtcNow.AddDays(-13) }; + await DB.Default.SaveAsync(record, Token); + try + { + RestoreRecord.Forget(); + Assert.True(await RestoreRecord.InFollowersGrace(Token)); + await DB.Default.Update().MatchID(record.ID).Modify(r => r.RestoredAt, DateTime.UtcNow.AddDays(-15)).ExecuteAsync(Token); + RestoreRecord.Forget(); + Assert.False(await RestoreRecord.InFollowersGrace(Token)); + } + finally + { + await DB.Default.DeleteAsync(record.ID); + RestoreRecord.Forget(); + } + } + } +} diff --git a/PrivaPub.Tests/Support/Host/PrivaPubHost.cs b/PrivaPub.Tests/Support/Host/PrivaPubHost.cs index 9873700..626dc97 100644 --- a/PrivaPub.Tests/Support/Host/PrivaPubHost.cs +++ b/PrivaPub.Tests/Support/Host/PrivaPubHost.cs @@ -24,7 +24,8 @@ namespace PrivaPub.Tests.Support.Host static readonly SemaphoreSlim Boot = new(1, 1); static readonly Type[] Unwanted = { typeof(JobWorker), typeof(MediaJanitor), typeof(OAuthPruner), typeof(StatisticsSchedule), - typeof(PrivaPub.Domain.Social.FollowResender), typeof(PrivaPub.Infrastructure.Backup.BackupScheduler) }; + typeof(PrivaPub.Domain.Social.FollowResender), typeof(PrivaPub.Infrastructure.Backup.BackupScheduler), + typeof(PrivaPub.Infrastructure.Backup.RestoreWatcher) }; static PrivaPubHost _shared; readonly string _mediaRoot = Path.Combine(Path.GetTempPath(), $"privapub-tests-{Guid.NewGuid():N}"); diff --git a/PrivaPub/Infrastructure/Backup/MaintenanceGate.cs b/PrivaPub/Infrastructure/Backup/MaintenanceGate.cs new file mode 100644 index 0000000..e14a66c --- /dev/null +++ b/PrivaPub/Infrastructure/Backup/MaintenanceGate.cs @@ -0,0 +1,59 @@ +namespace PrivaPub.Infrastructure.Backup +{ + // What Program asks before migrations: whether a restore waits, and what this process does about it. The service carries + // it out; a command waits for it (exit 75, EX_TEMPFAIL), except asking how it went. + public static class MaintenanceGate + { + /// The exit code that stops systemd restarting the service (RestartPreventExitStatus=75): a restore gave up. + public const int TryLater = 75; + + /// Null to go on booting; else the code to exit with. + public static async Task Enter(IServiceProvider services, string[] command, CancellationToken token) + { + var backups = services.GetRequiredService(); + var logger = services.GetRequiredService().CreateLogger(nameof(MaintenanceGate)); + if (command != default) + { + var waiting = RestoreMarker.Read(backups.Root); + if (waiting == default || command is ["restore", ..]) + return default; + Console.Error.WriteLine($"the restore of {waiting.Backup} is {waiting.State}{(waiting.Error == default ? string.Empty : $" ({waiting.Error})")}: try again once the service has carried it out (PrivaPub admin restore --status)"); + return TryLater; + } + return await ServerRestore.ApplyPending(backups.Context(), logger, token) switch + { + RestoreOutcome.Failed => 1,//systemd starts it again, and the next attempt redoes everything + RestoreOutcome.GaveUp => TryLater, + _ => (int?)null + }; + } + } + + // A restore asked for while the service runs (from the administrator's page or the CLI): the service stops within + // seconds, systemd starts it again, and the restore runs before anything else. + public class RestoreWatcher(Backups backups, IHostApplicationLifetime lifetime, ILogger logger) : BackgroundService + { + static readonly TimeSpan Interval = TimeSpan.FromSeconds(3); + + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + while (!stoppingToken.IsCancellationRequested) + { + try + { + await Task.Delay(Interval, stoppingToken); + } + catch (OperationCanceledException) + { + return; + } + if (RestoreMarker.Read(backups.Root) is { State: "pending" } marker) + { + logger.LogWarning("A restore of {Backup} was asked for by {RequestedBy}: stopping, to restore it at the next start", marker.Backup, marker.RequestedBy); + lifetime.StopApplication(); + return; + } + } + } + } +} diff --git a/PrivaPub/Infrastructure/Backup/ProtectiveMerge.cs b/PrivaPub/Infrastructure/Backup/ProtectiveMerge.cs new file mode 100644 index 0000000..eabb587 --- /dev/null +++ b/PrivaPub/Infrastructure/Backup/ProtectiveMerge.cs @@ -0,0 +1,264 @@ +using MongoDB.Bson; +using MongoDB.Driver; + +namespace PrivaPub.Infrastructure.Backup +{ + // A restore may lose what was made since the backup, but it never undoes a protective act (owner decision 2026-10-07). + // From the pre-restore backup P, after the backup is imported: + // - who follows whom is P's: a follower that left stays gone, so no followers-only post reaches it, and one gained is + // kept; the persona's follows are P's too; + // - blocks, mutes, domain blocks (the server's and the personas'), being blocked, reserved names, deletion tombstones, + // reports, filters and OAuth applications are the union of both, P's row winning; + // - deletions win: a root, persona, group, post or remote account deleted since is deleted again (P's row, as its + // deletion left it), with what a deletion takes away; P's moderation of a persona (silenced, suspended, banned) stays; + // - a root keeps P's password, e-mail, ban and policies; + // - roots, personas and groups made since become tombstones: deleted, their names kept; local posts made since answer + // 410; media made since go to the trash, as do media trashed since (and they leave the posts restored with them); + // - every session ends, and every server's circuit is closed again. + // Run on the imported database, it reads P's files only, so it can run again after a failed attempt. + public static class ProtectiveMerge + { + static readonly (string Collection, string[] Key)[] Unions = + [ + ("ReservedName", ["Name"]), + ("DomainBlock", ["Domain"]), + ("Block", ["AvatarId", "TargetActorURI"]), + ("Mute", ["AvatarId", "TargetActorURI"]), + ("AccountDomainBlock", ["AvatarId", "Domain"]), + ("BlockedBy", ["AvatarId", "ActorURI"]), + ("DeletedObject", ["ObjectURI"]), + ("Report", ["_id"]), + ("PersonaFilter", ["_id"]), + ("openiddict.applications", ["client_id"]) + ]; + + const string Since = "restore: made after the backup"; + + public static async Task Apply(IMongoDatabase database, string previous, RestoreReport report, CancellationToken token) + { + IEnumerable P(string collection) => ServerRestore.Read(Path.Combine(previous, collection + ".jsonl.gz")); + var now = DateTime.UtcNow; + + // who follows whom: P's + foreach (var collection in new[] { "Follower", "Following" }) + { + await database.DropCollectionAsync(collection, token); + await ServerRestore.Insert(database.GetCollection(collection), P(collection), token); + } + + foreach (var (collection, key) in Unions) + report.ProtectiveKept += await Union(database.GetCollection(collection), P(collection), key, token); + + var roots = database.GetCollection("RootUser"); + foreach (var root in P("RootUser")) + { + var restored = await ById(roots, root["_id"], token); + if (restored == default) + { + // made since: deleted, as RootRemoval leaves a root + report.RootsTombstoned.Add(root.GetValue("UserName", "").ToString()); + root["UserName"] = $"deleted-{root["_id"]}"; + root["Email"] = BsonNull.Value; + root["HashedPassword"] = BsonNull.Value; + root["Policies"] = new BsonArray(); + root["IsBanned"] = false; + root["IsEmailValidated"] = false; + root["DeletedAt"] = Deleted(root, "DeletedAt", now); + await roots.InsertOneAsync(root, cancellationToken: token); + continue; + } + if (IsSet(root, "DeletedAt")) + { + if (!IsSet(restored, "DeletedAt")) + report.RootsDeleted++; + await roots.ReplaceOneAsync(Id(root["_id"]), root, cancellationToken: token); + continue; + } + await roots.UpdateOneAsync(Id(root["_id"]), Copy(root, "HashedPassword", "Email", "IsEmailValidated", "IsBanned", "Policies", + "CredentialsChangedAt", "ResetPasswordToken", "ResetPasswordTokenSentAt"), cancellationToken: token); + } + + var avatars = database.GetCollection("Avatar"); + foreach (var avatar in P("Avatar")) + { + var restored = await ById(avatars, avatar["_id"], token); + if (restored == default) + { + report.PersonasTombstoned.Add(avatar.GetValue("UserName", "").ToString()); + avatar["DeletionAt"] = Deleted(avatar, "DeletionAt", now); + await avatars.InsertOneAsync(avatar, cancellationToken: token); + continue; + } + if (IsSet(avatar, "DeletionAt")) + { + if (!IsSet(restored, "DeletionAt")) + report.PersonasDeleted++; + await avatars.ReplaceOneAsync(Id(avatar["_id"]), avatar, cancellationToken: token); + // what RootRemoval takes with a persona, which the backup still had + foreach (var collection in new[] { "PersonaListMember", "PersonaList", "PersonaFilter", "FollowedTag", "ScheduledStatus" }) + await database.GetCollection(collection).DeleteManyAsync(new BsonDocument("AvatarId", avatar["_id"].ToString()), token); + continue; + } + var moderated = new[] { "SilencedAt", "SuspendedAt", "BannedAt" }.Where(field => IsSet(avatar, field)).ToArray(); + if (moderated.Length > 0) + await avatars.UpdateOneAsync(Id(avatar["_id"]), Copy(avatar, moderated), cancellationToken: token); + } + + var links = database.GetCollection("RootToAvatar"); + var linked = (await (await links.FindAsync(FilterDefinition.Empty, cancellationToken: token)).ToListAsync(token)) + .Select(l => l.GetValue("AvatarId", BsonNull.Value).ToString()).ToHashSet(StringComparer.Ordinal); + await ServerRestore.Insert(links, P("RootToAvatar").Where(l => !linked.Contains(l.GetValue("AvatarId", BsonNull.Value).ToString())), token); + + var groups = database.GetCollection("Group"); + foreach (var group in P("Group")) + { + var restored = await ById(groups, group["_id"], token); + if (restored == default) + { + report.GroupsTombstoned.Add(group.GetValue("UserName", "").ToString()); + group["DeletionAt"] = Deleted(group, "DeletionAt", now); + await groups.InsertOneAsync(group, cancellationToken: token); + continue; + } + if (!IsSet(group, "DeletionAt")) + continue; + if (!IsSet(restored, "DeletionAt")) + report.GroupsDeleted++; + await groups.ReplaceOneAsync(Id(group["_id"]), group, cancellationToken: token); + } + + // remote accounts deleted or gone since: as P has them + var foreign = database.GetCollection("ForeignAvatar"); + foreach (var account in P("ForeignAvatar").Where(a => IsSet(a, "DeletionAt") || IsSet(a, "ForgottenAt"))) + await foreign.ReplaceOneAsync(Id(account["_id"]), account, cancellationToken: token); + + await Posts(database, P("Post"), report, token); + await Media(database, P("MediaAttachment"), report, token); + + // every session ends: a new stamp for each root's tokens, and the personas' OAuth tokens go + foreach (var root in await (await roots.FindAsync(FilterDefinition.Empty, cancellationToken: token)).ToListAsync(token)) + await roots.UpdateOneAsync(Id(root["_id"]), Builders.Update + .Set("SessionStamp", Guid.NewGuid().ToString("N")) + .Set("CredentialsChangedAt", now), cancellationToken: token); + foreach (var collection in new[] { "openiddict.tokens", "openiddict.authorizations" }) + report.SessionsEnded += (int)(await database.GetCollection(collection).DeleteManyAsync(FilterDefinition.Empty, token)).DeletedCount; + + // what the backup remembered of servers being down is long past + await database.GetCollection("RemoteInstance").UpdateManyAsync(FilterDefinition.Empty, + Builders.Update.Set("ConsecutiveFailures", 0).Set("UnavailableUntil", BsonNull.Value), cancellationToken: token); + } + + // posts deleted since: deleted again, out of timelines and pins; local posts made since: 410 + static async Task Posts(IMongoDatabase database, IEnumerable previous, RestoreReport report, CancellationToken token) + { + var posts = database.GetCollection("Post"); + var gone = database.GetCollection("DeletedObject"); + foreach (var batch in previous.Where(p => IsSet(p, "DeletedAt") || IsLocal(p)).Chunk(500)) + { + var ids = new BsonArray(batch.Select(p => p["_id"])); + var restored = (await (await posts.FindAsync(new BsonDocument("_id", new BsonDocument("$in", ids)), cancellationToken: token)).ToListAsync(token)) + .ToDictionary(p => p["_id"]); + foreach (var post in batch) + { + if (!restored.TryGetValue(post["_id"], out var mine)) + { + if (IsLocal(post) && post.GetValue("ObjectURI", BsonNull.Value) is BsonString uri) + { + await gone.UpdateOneAsync(new BsonDocument("ObjectURI", uri), Builders.Update + .SetOnInsert("ObjectURI", uri).SetOnInsert("DeletedAt", DateTime.UtcNow), new UpdateOptions { IsUpsert = true }, token); + report.PostsGone++; + } + continue; + } + if (!IsSet(post, "DeletedAt") || IsSet(mine, "DeletedAt")) + continue; + await posts.ReplaceOneAsync(Id(post["_id"]), post, cancellationToken: token); + var id = post["_id"].ToString(); + await database.GetCollection("TimelineEntry").DeleteManyAsync( + new BsonDocument("$or", new BsonArray { new BsonDocument("PostId", id), new BsonDocument("ReblogOfPostId", id) }), token); + await database.GetCollection("Pin").DeleteManyAsync(new BsonDocument("PostId", id), token); + report.PostsDeleted++; + } + } + } + + // media trashed since, and media made since: in the trash (the janitor deletes their files after its grace), and out + // of the posts restored; media of posts deleted now: in the trash too + static async Task Media(IMongoDatabase database, IEnumerable previous, RestoreReport report, CancellationToken token) + { + var media = database.GetCollection("MediaAttachment"); + var posts = database.GetCollection("Post"); + var now = DateTime.UtcNow; + foreach (var batch in previous.Chunk(1000)) + { + var ids = new BsonArray(batch.Select(m => m["_id"])); + var restored = (await (await media.FindAsync(new BsonDocument("_id", new BsonDocument("$in", ids)), cancellationToken: token)).ToListAsync(token)) + .ToDictionary(m => m["_id"]); + var made = new List(); + foreach (var row in batch) + { + if (!restored.TryGetValue(row["_id"], out var mine)) + { + if (!IsSet(row, "TrashedAt")) + { + row["TrashedAt"] = now; + row["TrashReason"] = Since; + } + made.Add(row); + continue; + } + if (!IsSet(row, "TrashedAt") || IsSet(mine, "TrashedAt")) + continue; + await media.UpdateOneAsync(Id(row["_id"]), Copy(row, "TrashedAt", "TrashReason"), cancellationToken: token); + if (mine.GetValue("PostId", BsonNull.Value) is BsonString postId) + await posts.UpdateOneAsync(Id(ObjectId.TryParse(postId.AsString, out var post) ? post : postId), Builders.Update.PullFilter("Media", + new BsonDocument("AttachmentId", row["_id"].ToString())), cancellationToken: token); + report.MediaTrashed++; + } + await ServerRestore.Insert(media, made, token); + report.MediaTrashed += made.Count(m => m["TrashReason"] == Since); + } + + var deleted = await (await posts.FindAsync(new BsonDocument("DeletedAt", new BsonDocument("$ne", BsonNull.Value)), + new FindOptions { Projection = new BsonDocument("_id", 1) }, token)).ToListAsync(token); + foreach (var batch in deleted.Select(p => (BsonValue)p["_id"].ToString()).Chunk(1000)) + report.MediaTrashed += (int)(await media.UpdateManyAsync( + new BsonDocument { { "PostId", new BsonDocument("$in", new BsonArray(batch)) }, { "TrashedAt", BsonNull.Value } }, + Builders.Update.Set("TrashedAt", now).Set("TrashReason", "restore: its post was deleted"), cancellationToken: token)).ModifiedCount; + } + + // P's rows added, each replacing the restored row with the same key: how many P had + static async Task Union(IMongoCollection collection, IEnumerable previous, string[] key, CancellationToken token) + { + var count = 0; + foreach (var batch in previous.Chunk(500)) + { + var writes = new List>(); + foreach (var row in batch) + { + var match = new BsonDocument(key.Select(field => new BsonElement(field, row.GetValue(field, BsonNull.Value)))); + writes.Add(new DeleteManyModel(match)); + writes.Add(new InsertOneModel(row)); + } + await collection.BulkWriteAsync(writes, new BulkWriteOptions { IsOrdered = true, BypassDocumentValidation = true }, token); + count += batch.Length; + } + return count; + } + + static bool IsLocal(BsonDocument post) => + post.GetValue("GroupUserId", BsonNull.Value) is BsonString && !post.GetValue("IsFederatedCopy", false).ToBoolean(); + + static bool IsSet(BsonDocument row, string field) => row.TryGetValue(field, out var value) && !value.IsBsonNull; + + static BsonValue Deleted(BsonDocument row, string field, DateTime now) => IsSet(row, field) ? row[field] : now; + + static FilterDefinition Id(BsonValue id) => new BsonDocument("_id", id); + + static async Task ById(IMongoCollection collection, BsonValue id, CancellationToken token) => + await (await collection.FindAsync(Id(id), cancellationToken: token)).FirstOrDefaultAsync(token); + + static UpdateDefinition Copy(BsonDocument from, params string[] fields) => + Builders.Update.Combine(fields.Select(field => Builders.Update.Set(field, from.GetValue(field, BsonNull.Value)))); + } +} diff --git a/PrivaPub/Infrastructure/Backup/RestoreMarker.cs b/PrivaPub/Infrastructure/Backup/RestoreMarker.cs index ac9abc7..96dc618 100644 --- a/PrivaPub/Infrastructure/Backup/RestoreMarker.cs +++ b/PrivaPub/Infrastructure/Backup/RestoreMarker.cs @@ -12,6 +12,7 @@ namespace PrivaPub.Infrastructure.Backup public string Backup { get; set; }//the backup restored public string PreRestore { get; set; }//the backup taken just before, what the protective merge reads public string State { get; set; } = "pending";//pending, then running + public string RequestedBy { get; set; }//who asked: a root's name, or "cli" public int Attempts { get; set; } public DateTime RequestedAt { get; set; } = DateTime.UtcNow; public string Error { get; set; } @@ -29,7 +30,8 @@ namespace PrivaPub.Infrastructure.Backup } catch (JsonException) { - return default; + //never written half (Write is atomic): someone's hand; the server won't guess what was meant + return new RestoreMarker { State = "unreadable", Attempts = MaxAttempts, Error = $"{FileName} can't be read" }; } } diff --git a/PrivaPub/Infrastructure/Backup/RestoreRecord.cs b/PrivaPub/Infrastructure/Backup/RestoreRecord.cs new file mode 100644 index 0000000..2b75cbc --- /dev/null +++ b/PrivaPub/Infrastructure/Backup/RestoreRecord.cs @@ -0,0 +1,62 @@ +using MongoDB.Entities; + +namespace PrivaPub.Infrastructure.Backup +{ + // What a restore did (or why it was abandoned), kept for the administrator's page and for the followers' grace: for + // FollowersGrace after a restore the followers' digests (FEP-8fcf) are neither sent nor acted on to undo a follow, since + // what the restore lost of either side is not the other's fault. + public class RestoreRecord : Entity + { + public static readonly TimeSpan FollowersGrace = TimeSpan.FromDays(14); + + public string Backup { get; set; } + public DateTime BackupCreatedAt { get; set; } + public string PreRestore { get; set; } + public string RequestedBy { get; set; } + public DateTime RequestedAt { get; set; } + public DateTime RestoredAt { get; set; } = DateTime.UtcNow; + public int Attempts { get; set; } + public bool Abandoned { get; set; }//nothing was changed: the server booted as it was + public string Error { get; set; } + public RestoreReport Report { get; set; } = new(); + + static (DateTime Until, DateTime Read) _grace; + + /// Whether a restore ended less than FollowersGrace ago (read at most once a minute). + public static async Task InFollowersGrace(CancellationToken token) + { + var now = DateTime.UtcNow; + var cached = _grace; + if (now - cached.Read > TimeSpan.FromMinutes(1)) + { + var last = await DB.Default.Find().Match(r => !r.Abandoned).Sort(r => r.RestoredAt, Order.Descending).ExecuteFirstAsync(token); + cached = (last == default ? DateTime.MinValue : last.RestoredAt + FollowersGrace, now); + _grace = cached; + } + return now < cached.Until; + } + + /// Forgets what was read, so a restore made in this process counts at once. + public static void Forget() => _grace = default; + } + + public class RestoreReport + { + public int Collections { get; set; } + public long Documents { get; set; } + public int CollectionsDropped { get; set; }//live collections the backup had no documents in + public int MediaRestored { get; set; }//files the live directory lacked, brought back + public int MediaMissing { get; set; } + public int RootsDeleted { get; set; }//deleted since the backup, deleted again + public int PersonasDeleted { get; set; } + public int GroupsDeleted { get; set; } + public int PostsDeleted { get; set; } + public List RootsTombstoned { get; set; } = [];//made after the backup: deleted, their names kept + public List PersonasTombstoned { get; set; } = []; + public List GroupsTombstoned { get; set; } = []; + public int PostsGone { get; set; }//local posts made after the backup: 410 from now on + public int MediaTrashed { get; set; } + public int ProtectiveKept { get; set; }//blocks, mutes, domain blocks, reserved names, reports, filters, deletions kept from since + public int SessionsEnded { get; set; } + } +} diff --git a/PrivaPub/Infrastructure/Backup/ServerBackup.cs b/PrivaPub/Infrastructure/Backup/ServerBackup.cs index 5a3f8ec..de1d8b4 100644 --- a/PrivaPub/Infrastructure/Backup/ServerBackup.cs +++ b/PrivaPub/Infrastructure/Backup/ServerBackup.cs @@ -38,7 +38,8 @@ namespace PrivaPub.Infrastructure.Backup ["openiddict.tokens"] = "sessions: a restore ends every one", ["openiddict.authorizations"] = "sessions: a restore ends every one", [nameof(MaintenanceLock)] = "who is backing up or restoring now", - ["AppConfiguration"] = "the configuration's copy holds the SMTP password, and is made again from appsettings at boot" + ["AppConfiguration"] = "the configuration's copy holds the SMTP password, and is made again from appsettings at boot", + [nameof(RestoreRecord)] = "what restores did outlives what they restore" }; static readonly JsonWriterSettings Json = new() { OutputMode = JsonOutputMode.CanonicalExtendedJson, Indent = false }; diff --git a/PrivaPub/Infrastructure/Backup/ServerRestore.cs b/PrivaPub/Infrastructure/Backup/ServerRestore.cs new file mode 100644 index 0000000..d0e4121 --- /dev/null +++ b/PrivaPub/Infrastructure/Backup/ServerRestore.cs @@ -0,0 +1,247 @@ +using MongoDB.Bson; +using MongoDB.Driver; + +using System.IO.Compression; + +namespace PrivaPub.Infrastructure.Backup +{ + public enum RestoreOutcome + { + None,//nothing was asked + Restored, + Abandoned,//refused before anything changed: the server boots as it was + Failed,//failed midway: the next boot tries again from the start + GaveUp//failed MaxAttempts times: the server stays down until someone looks + } + + // A backup restored (owner decision 2026-10-07: from the CLI or the administrator's page). Asking writes restore.json + // and the running service stops at once (RestoreWatcher); the restore itself runs at the next boot, before migrations, + // indexes and every hosted service, with nothing else touching the database: + // 1. a pre-restore backup P is taken, once (a retry reuses it); + // 2. every collection the backup holds is dropped and imported raw, with its indexes; every other one is dropped, + // except what a backup never holds (ServerBackup.Excluded), which stays as it is; + // 3. media files the live directory lacks come back from the backup (or the trash); a restore deletes no file; + // 4. ProtectiveMerge brings back from P every protective act made since the backup; + // 5. every session ends, every server's circuit closes, and a RestoreRecord tells what happened. + // Each attempt redoes everything, so one that dies midway converges on the next. + public static class ServerRestore + { + static readonly TimeSpan LockWait = TimeSpan.FromMinutes(10); + + /// Why a backup can't be restored here: none when it can. + public static async Task> Check(BackupContext context, string id, CancellationToken token) + { + var backup = ServerBackup.Find(context.BackupsRoot, id); + if (backup?.Manifest == default) + return ["no such backup"]; + var manifest = backup.Manifest; + var problems = new List(); + if (manifest.Format != ArchiveManifest.CurrentFormat) + problems.Add($"its format is {manifest.Format}, this build reads {ArchiveManifest.CurrentFormat}"); + if (!string.Equals(manifest.Host?.TrimEnd('/'), context.Host, StringComparison.OrdinalIgnoreCase)) + problems.Add($"it was made by {manifest.Host}, not {context.Host}: every id and address in it names its host"); + var code = ServerBackup.CodeMigration(); + if (manifest.MigrationNumber > code) + problems.Add($"it was made by a newer build (migration {manifest.MigrationNumber}; this one knows {code})"); + problems.AddRange(await ServerBackup.Verify(context.BackupsRoot, id, token)); + return problems; + } + + /// Asks for a backup to be restored at the next boot: why not, or null once asked. + public static async Task Request(BackupContext context, string id, string requestedBy, CancellationToken token) + { + var problems = await Check(context, id, token); + if (problems.Count > 0) + return string.Join("; ", problems); + var waiting = RestoreMarker.Read(context.BackupsRoot); + if (waiting != default && waiting.Attempts < RestoreMarker.MaxAttempts) + return $"the restore of {waiting.Backup} is already {waiting.State}"; + if (await MaintenanceLock.Current(token) is { } held) + return $"a {held.What} is running"; + new RestoreMarker { Backup = id, RequestedBy = requestedBy }.Write(context.BackupsRoot); + return default; + } + + /// The restore asked for, carried out (at boot, before migrations): what came of it. + public static async Task ApplyPending(BackupContext context, ILogger logger, CancellationToken token) + { + var marker = RestoreMarker.Read(context.BackupsRoot); + if (marker == default) + return RestoreOutcome.None; + if (marker.Attempts >= RestoreMarker.MaxAttempts) + { + logger.LogCritical("The restore of {Backup} failed {Attempts} times ({Error}). The database may be half restored; {PreRestore} holds it as it was before. Restore that backup or another (PrivaPub admin restore ), or delete {Marker} to boot as it is", + marker.Backup, marker.Attempts, marker.Error, marker.PreRestore ?? "no backup", RestoreMarker.PathIn(context.BackupsRoot)); + return RestoreOutcome.GaveUp; + } + + await using var held = await TakeLock(token); + if (held == default) + return await Abandon(context, marker, "a backup kept the maintenance lock for ten minutes", logger, token); + + var problems = await Check(context, marker.Backup, token); + if (problems.Count > 0 && marker.PreRestore == default) + return await Abandon(context, marker, string.Join("; ", problems), logger, token); + + if (marker.PreRestore == default) + { + var (taken, error) = await ServerBackup.CreateHeld(context, "pre-restore", dbOnly: false, token); + if (taken == default) + return await Abandon(context, marker, $"the pre-restore backup could not be made: {error}", logger, token); + marker.PreRestore = taken.Id; + } + marker.State = "running"; + marker.Attempts++; + marker.Error = default; + marker.Write(context.BackupsRoot); + logger.LogWarning("Restoring {Backup} (attempt {Attempt}); the server as it was is {PreRestore}", marker.Backup, marker.Attempts, marker.PreRestore); + + try + { + if (problems.Count > 0) + throw new InvalidOperationException(string.Join("; ", problems)); + var backup = ServerBackup.Find(context.BackupsRoot, marker.Backup); + var report = new RestoreReport(); + await Import(context, backup, report, token); + Media(context, backup, report); + await ProtectiveMerge.Apply(context.Database, Path.Combine(context.BackupsRoot, marker.PreRestore, "db"), report, token); + await Record(context, new RestoreRecord + { + Backup = backup.Id, + BackupCreatedAt = backup.CreatedAt, + PreRestore = marker.PreRestore, + RequestedBy = marker.RequestedBy, + RequestedAt = marker.RequestedAt, + Attempts = marker.Attempts, + Report = report + }, token); + RestoreRecord.Forget(); + RestoreMarker.Clear(context.BackupsRoot); + logger.LogWarning("Restored {Backup}: {Documents} documents in {Collections} collections, {Media} media files brought back, {Tombstoned} accounts made since deleted", + backup.Id, report.Documents, report.Collections, report.MediaRestored, report.RootsTombstoned.Count + report.PersonasTombstoned.Count + report.GroupsTombstoned.Count); + return RestoreOutcome.Restored; + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + marker.Error = ex.Message; + marker.Write(context.BackupsRoot); + logger.LogError(ex, "The restore of {Backup} failed (attempt {Attempt} of {Max})", marker.Backup, marker.Attempts, RestoreMarker.MaxAttempts); + return marker.Attempts >= RestoreMarker.MaxAttempts ? RestoreOutcome.GaveUp : RestoreOutcome.Failed; + } + } + + static async Task TakeLock(CancellationToken token) + { + var until = DateTime.UtcNow + LockWait; + while (true) + { + var held = await MaintenanceLock.Take("restore", token); + if (held != default || DateTime.UtcNow > until) + return held; + await Task.Delay(TimeSpan.FromSeconds(5), token); + } + } + + // refused before anything changed: recorded for the administrator's page, and the server boots as it was + static async Task Abandon(BackupContext context, RestoreMarker marker, string why, ILogger logger, CancellationToken token) + { + logger.LogError("The restore of {Backup} was abandoned, nothing changed: {Why}", marker.Backup, why); + await Record(context, new RestoreRecord + { + Backup = marker.Backup, + RequestedBy = marker.RequestedBy, + RequestedAt = marker.RequestedAt, + Attempts = marker.Attempts, + Abandoned = true, + Error = why + }, token); + RestoreMarker.Clear(context.BackupsRoot); + return RestoreOutcome.Abandoned; + } + + // in the database restored (a backup never holds these records: they outlive what they restore) + static async Task Record(BackupContext context, RestoreRecord record, CancellationToken token) + { + record.ID = ObjectId.GenerateNewId().ToString(); + await context.Database.GetCollection(nameof(RestoreRecord)).InsertOneAsync(record, cancellationToken: token); + } + + // each collection dropped and imported as the backup holds it, its indexes made again; the others dropped + static async Task Import(BackupContext context, BackupInfo backup, RestoreReport report, CancellationToken token) + { + var database = context.Database; + var directory = Path.Combine(context.BackupsRoot, backup.Id, "db"); + foreach (var entry in backup.Manifest.Collections) + { + await database.DropCollectionAsync(entry.Name, token); + await database.CreateCollectionAsync(entry.Name, cancellationToken: token); + report.Documents += await Insert(database.GetCollection(entry.Name), Read(Path.Combine(directory, entry.Name + ".jsonl.gz")), token); + var indexes = entry.Indexes.Select(BsonDocument.Parse).Where(i => i.GetValue("name", "").AsString != "_id_").ToList(); + foreach (var index in indexes) + index.Remove("ns"); + if (indexes.Count > 0) + await database.RunCommandAsync(new BsonDocument { { "createIndexes", entry.Name }, { "indexes", new BsonArray(indexes) } }, cancellationToken: token); + report.Collections++; + } + var held = backup.Manifest.Collections.Select(c => c.Name).ToHashSet(StringComparer.Ordinal); + foreach (var name in await (await database.ListCollectionNamesAsync(cancellationToken: token)).ToListAsync(token)) + { + if (held.Contains(name) || ServerBackup.Excluded.ContainsKey(name) || name.StartsWith("system.", StringComparison.Ordinal)) + continue; + await database.DropCollectionAsync(name, token); + report.CollectionsDropped++; + } + } + + /// Documents inserted in batches, unordered and unvalidated, as they were: how many. + public static async Task Insert(IMongoCollection collection, IEnumerable documents, CancellationToken token) + { + var count = 0L; + foreach (var batch in documents.Chunk(1000)) + { + await collection.InsertManyAsync(batch, new InsertManyOptions { IsOrdered = false, BypassDocumentValidation = true }, token); + count += batch.Length; + } + return count; + } + + /// A collection's documents in a backup, one by one; none when it has no file. + public static IEnumerable Read(string file) + { + if (!File.Exists(file)) + yield break; + using var gzip = new GZipStream(File.OpenRead(file), CompressionMode.Decompress); + using var reader = new StreamReader(gzip); + while (reader.ReadLine() is { } line) + if (line.Length > 0) + yield return BsonDocument.Parse(line); + } + + // the backup's media files the live directory lacks: linked from the backup, or moved back from the trash + static void Media(BackupContext context, BackupInfo backup, RestoreReport report) + { + var kept = Path.Combine(context.BackupsRoot, backup.Id, "media"); + foreach (var relative in backup.Manifest.Media.List) + { + var live = ServerBackup.Inside(context.MediaRoot, relative); + if (File.Exists(live)) + continue; + var fromBackup = ServerBackup.Inside(kept, relative); + var fromTrash = ServerBackup.Inside(context.TrashRoot, relative); + if (File.Exists(fromBackup)) + HardLink.LinkOrCopy(fromBackup, live); + else if (File.Exists(fromTrash)) + { + Directory.CreateDirectory(Path.GetDirectoryName(live)!); + File.Move(fromTrash, live); + } + else + { + report.MediaMissing++; + continue; + } + report.MediaRestored++; + } + } + } +} diff --git a/PrivaPub/Infrastructure/Cli/AdminCommands.cs b/PrivaPub/Infrastructure/Cli/AdminCommands.cs index c875532..513433d 100644 --- a/PrivaPub/Infrastructure/Cli/AdminCommands.cs +++ b/PrivaPub/Infrastructure/Cli/AdminCommands.cs @@ -3,6 +3,7 @@ using MongoDB.Entities; using PrivaPub.ClientModels; using PrivaPub.ClientModels.User; using PrivaPub.ClientModels.User.Avatar; +using PrivaPub.Infrastructure.Backup; using PrivaPub.Models.User; using PrivaPub.Services; using PrivaPub.Services.ClientToServer.Private; @@ -29,10 +30,15 @@ namespace PrivaPub.Infrastructure.Cli linking them (the deploy's, which can't link www-data's files) PrivaPub admin backups the backups kept, newest first PrivaPub admin backup verify whether a backup's files are what its manifest says + PrivaPub admin restore restores a backup: the service stops within seconds and + restores it when systemd starts it again; everyone signs + in again, and what was made since is lost, except what + protects (blocks, deletions, ...) + PrivaPub admin restore --status the restore waiting, and the last one done """; /// Whether a command runs before migrations: backups are of the database as it was. - public static bool BeforeMigrations(string[] args) => args is ["backup", ..] or ["backups"]; + public static bool BeforeMigrations(string[] args) => args is ["backup", ..] or ["backups"] or ["restore", ..]; public static async Task Run(string[] args, IServiceProvider services, TextReader input = default, TextWriter output = default) { @@ -51,9 +57,13 @@ namespace PrivaPub.Infrastructure.Cli case ["backup", "verify", var id]: return await VerifyBackup(services, id, output); case ["backup", .. var flags] when BackupFlags(flags, out var kind, out var dbOnly): - return await Backup(services, kind, dbOnly, output); + return await BackUp(services, kind, dbOnly, output); case ["backups"]: return ListBackups(services, output); + case ["restore", "--status"]: + return await RestoreStatus(services, output); + case ["restore", var id] when !id.StartsWith('-'): + return await Restore(services, id, output); default: Console.Error.WriteLine(Usage); return 2; @@ -81,9 +91,9 @@ namespace PrivaPub.Infrastructure.Cli return true; } - static async Task Backup(IServiceProvider services, string kind, bool dbOnly, TextWriter output) + static async Task BackUp(IServiceProvider services, string kind, bool dbOnly, TextWriter output) { - var (made, error) = await services.GetRequiredService().Create(kind, dbOnly, CancellationToken.None); + var (made, error) = await services.GetRequiredService().Create(kind, dbOnly, CancellationToken.None); if (made == default) { Console.Error.WriteLine(error); @@ -101,14 +111,14 @@ namespace PrivaPub.Infrastructure.Cli static int ListBackups(IServiceProvider services, TextWriter output) { - foreach (var backup in services.GetRequiredService().List()) + foreach (var backup in services.GetRequiredService().List()) output.WriteLine($"{backup.Id}\t{backup.Kind}\t{backup.CreatedAt:u}\t{backup.Bytes / 1024} KiB\t{backup.Manifest?.Media.Files ?? 0} media{(backup.Manifest?.DbOnly == true ? " (listed)" : string.Empty)}"); return 0; } static async Task VerifyBackup(IServiceProvider services, string id, TextWriter output) { - var problems = await services.GetRequiredService().Verify(id, CancellationToken.None); + var problems = await services.GetRequiredService().Verify(id, CancellationToken.None); foreach (var problem in problems) output.WriteLine(problem); if (problems.Count > 0) @@ -117,6 +127,48 @@ namespace PrivaPub.Infrastructure.Cli return 0; } + static async Task Restore(IServiceProvider services, string id, TextWriter output) + { + var backups = services.GetRequiredService(); + var refused = await ServerRestore.Request(backups.Context(), id, "cli", CancellationToken.None); + if (refused != default) + { + Console.Error.WriteLine(refused); + return 1; + } + output.WriteLine($"{id} will be restored: the running service stops within seconds and restores it when it starts again"); + output.WriteLine("(systemd starts it again; follow it with journalctl -u privapub, then PrivaPub admin restore --status)"); + return 0; + } + + static async Task RestoreStatus(IServiceProvider services, TextWriter output) + { + var waiting = RestoreMarker.Read(services.GetRequiredService().Root); + if (waiting != default) + output.WriteLine($"waiting: {waiting.Backup}, {waiting.State}, attempt {waiting.Attempts} of {RestoreMarker.MaxAttempts}, asked by {waiting.RequestedBy} at {waiting.RequestedAt:u}{(waiting.Error == default ? string.Empty : $": {waiting.Error}")}"); + var last = await DB.Default.Find().Sort(r => r.RestoredAt, Order.Descending).ExecuteFirstAsync(); + if (last == default) + { + if (waiting == default) + output.WriteLine("no restore asked for, none done"); + return 0; + } + if (last.Abandoned) + { + output.WriteLine($"last: {last.Backup} abandoned at {last.RestoredAt:u}, nothing changed: {last.Error}"); + return 0; + } + var report = last.Report; + output.WriteLine($"last: {last.Backup} (made {last.BackupCreatedAt:u}) restored at {last.RestoredAt:u}, asked by {last.RequestedBy}; before it: {last.PreRestore}"); + output.WriteLine($" {report.Documents} documents in {report.Collections} collections, {report.MediaRestored} media files brought back, {report.MediaMissing} missing"); + output.WriteLine($" deleted again: {report.RootsDeleted} roots, {report.PersonasDeleted} personas, {report.GroupsDeleted} groups, {report.PostsDeleted} posts"); + output.WriteLine($" made since, now deleted: roots {Names(report.RootsTombstoned)}, personas {Names(report.PersonasTombstoned)}, groups {Names(report.GroupsTombstoned)}; {report.PostsGone} posts gone"); + output.WriteLine($" {report.ProtectiveKept} protective rows kept, {report.MediaTrashed} media trashed, {report.SessionsEnded} sessions ended"); + return 0; + } + + static string Names(List names) => names.Count == 0 ? "none" : string.Join(", ", names); + static async Task AuditMedia(IServiceProvider services, bool fix, TextWriter output) { var report = await Domain.Media.MediaAudit.Run(services.GetRequiredService(), fix, CancellationToken.None); diff --git a/PrivaPub/Program.cs b/PrivaPub/Program.cs index c85c288..39d9982 100644 --- a/PrivaPub/Program.cs +++ b/PrivaPub/Program.cs @@ -71,6 +71,7 @@ try .Configure(builder.Configuration.GetSection("Backups")) .AddSingleton() .AddHostedService() + .AddHostedService() .PrivaPubMiddlewareConfiguration(); } catch (Exception ex) @@ -107,9 +108,17 @@ try throw; } + // A restore asked for runs here, before migrations, indexes and every hosted service; a command waits for it. + var gate = await PrivaPub.Infrastructure.Backup.MaintenanceGate.Enter(app.Services, args is ["admin", .. var asked] ? asked : default, CancellationToken.None); + if (gate is { } stop) + { + Environment.ExitCode = stop; + return; + } + // Commands get every service but start nothing: no Kestrel, no hosted services, no media directory. The deploy runs // them as its own user, which can read the configuration and reach the private mongod but owns no www-data directory. - // Backups are taken before migrations: the deploy's is of the database as the live build left it. + // Backups and restores are asked for before migrations: the deploy's backup is of the database as the live build left it. if (args is ["admin", .. var command] && AdminCommands.BeforeMigrations(command)) { using var scope = app.Services.CreateScope(); diff --git a/deploy/systemd/privapub.service b/deploy/systemd/privapub.service index 3dbdbcf..3badc3a 100644 --- a/deploy/systemd/privapub.service +++ b/deploy/systemd/privapub.service @@ -14,6 +14,9 @@ Environment=ASPNETCORE_ENVIRONMENT=Production Environment=DOTNET_CLI_TELEMETRY_OPTOUT=1 Restart=always RestartSec=5 +# a restore asked for stops the service, and the next start carries it out; one that failed three times exits 75 and +# waits for someone to look (journalctl -u privapub, PrivaPub admin restore --status) +RestartPreventExitStatus=75 # its own /tmp: ffmpeg's and the uploads' temporary files are nobody else's PrivateTmp=true SyslogIdentifier=privapub diff --git a/tools/pasture/lib/pasture.sh b/tools/pasture/lib/pasture.sh index 315068a..23e8c3a 100644 --- a/tools/pasture/lib/pasture.sh +++ b/tools/pasture/lib/pasture.sh @@ -29,7 +29,8 @@ pasture_base_up() { caddy_up local extra=() for setting in ${PRIVAPUB_ENV:-}; do extra+=(-e "$setting"); done - podman run -d --replace --name pasture-privapub --network $net -p "127.0.0.1:$port:80" \ + # --restart=always, as systemd's unit: a restore asked for stops PrivaPub, and the next start carries it out + podman run -d --replace --restart=always --name pasture-privapub --network $net -p "127.0.0.1:$port:80" \ --sysctl net.ipv4.ip_unprivileged_port_start=0 -e ASPNETCORE_ENVIRONMENT=Pasture "${extra[@]}" -w /app -v "$publish:/app:Z,ro" \ -v pasture-privapub-media:/tmp/privapub-media \ mcr.microsoft.com/dotnet/runtime-deps:10.0 /app/PrivaPub >/dev/null