using Microsoft.Extensions.Logging.Abstractions; using MongoDB.Bson; using MongoDB.Driver; using MongoDB.Entities; using PrivaPub.Infrastructure.Backup; using PrivaPub.Infrastructure.Cli; using PrivaPub.Tests.Support; using PrivaPub.Tests.Support.Host; namespace PrivaPub.Tests.Infrastructure { // A backup restored at boot: what was made since is lost, what protects is not (owner decision 2026-10-07). A server // in a database of its own is backed up, changed, and restored; alone, since the maintenance lock is the server's one. [Trait("Category", "Integration")] [Xunit.Collection(nameof(Exclusive))] public sealed class RestoreTests : IAsyncLifetime { const string Host = "https://privapub.test"; readonly string _scratch = Path.Combine(Path.GetTempPath(), $"privapub-restore-tests-{Guid.NewGuid():N}"); IMongoDatabase _database; string Backups => Path.Combine(_scratch, "backups"); string Media => Path.Combine(_scratch, "media"); string Trash => Path.Combine(_scratch, "media-trash"); static CancellationToken Token => TestContext.Current.CancellationToken; public async ValueTask InitializeAsync() { Assert.SkipUnless(MongoFixture.Enabled, MongoFixture.Skip); await PrivaPubHost.Shared(); _database = DB.Default.Database().Client.GetDatabase($"PrivaPubRestore_{Guid.NewGuid():N}"); Directory.CreateDirectory(Media); Directory.CreateDirectory(Trash); } public async ValueTask DisposeAsync() { if (_database != default) await _database.Client.DropDatabaseAsync(_database.DatabaseNamespace.DatabaseName); if (Directory.Exists(_scratch)) Directory.Delete(_scratch, recursive: true); } BackupContext Context(string host = Host) => new(_database, Backups, Media, Trash, host, new BackupOptions()); IMongoCollection C(string name) => _database.GetCollection(name); async Task One(string collection, BsonDocument filter) => await (await C(collection).FindAsync(filter, cancellationToken: Token)).FirstOrDefaultAsync(Token); async Task> All(string collection) => await (await C(collection).FindAsync(FilterDefinition.Empty, cancellationToken: Token)).ToListAsync(Token); static BsonDocument Id(ObjectId id) => new("_id", id); static Task Set(IMongoCollection collection, ObjectId id, BsonDocument fields) => collection.UpdateOneAsync(Id(id), new BsonDocument("$set", fields), cancellationToken: Token); // the ids of the town this test builds readonly ObjectId _alice = ObjectId.GenerateNewId(), _alicePersona = ObjectId.GenerateNewId(); readonly ObjectId _deletedPost = ObjectId.GenerateNewId(), _editedPost = ObjectId.GenerateNewId(), _laterPost = ObjectId.GenerateNewId(); readonly ObjectId _deletedMedia = ObjectId.GenerateNewId(), _laterMedia = ObjectId.GenerateNewId(); readonly ObjectId _bob = ObjectId.GenerateNewId(), _bobPersona = ObjectId.GenerateNewId(); static string Uri(ObjectId post) => $"{Host}/peasants/alice/posts/{post}"; async Task TownBackedUp() { await File.WriteAllTextAsync(Path.Combine(Media, "deleted.jpg"), "deleted", Token); await C("RootUser").InsertOneAsync(new BsonDocument { { "_id", _alice }, { "UserName", "alice" }, { "HashedPassword", "old" }, { "SessionStamp", "s1" }, { "Policies", new BsonArray { "IsUser" } }, { "DeletedAt", BsonNull.Value } }, cancellationToken: Token); await C("Avatar").InsertOneAsync(new BsonDocument { { "_id", _alicePersona }, { "UserName", "alice" }, { "SuspendedAt", BsonNull.Value }, { "DeletionAt", BsonNull.Value } }, cancellationToken: Token); await C("RootToAvatar").InsertOneAsync(new BsonDocument { { "RootId", _alice.ToString() }, { "AvatarId", _alicePersona.ToString() } }, cancellationToken: Token); await C("ReservedName").InsertOneAsync(new BsonDocument("Name", "alice"), cancellationToken: Token); await C("Post").InsertManyAsync([ new BsonDocument { { "_id", _deletedPost }, { "GroupUserId", _alicePersona.ToString() }, { "IsFederatedCopy", false }, { "ObjectURI", Uri(_deletedPost) }, { "Text", "regret" }, { "Media", new BsonArray { new BsonDocument("AttachmentId", _deletedMedia.ToString()) } }, { "DeletedAt", BsonNull.Value } }, new BsonDocument { { "_id", _editedPost }, { "GroupUserId", _alicePersona.ToString() }, { "IsFederatedCopy", false }, { "ObjectURI", Uri(_editedPost) }, { "Text", "first words" }, { "DeletedAt", BsonNull.Value } } ], cancellationToken: Token); await C("TimelineEntry").InsertOneAsync(new BsonDocument { { "PostId", _deletedPost.ToString() }, { "ReblogOfPostId", BsonNull.Value } }, cancellationToken: Token); await C("MediaAttachment").InsertOneAsync(new BsonDocument { { "_id", _deletedMedia }, { "PostId", _deletedPost.ToString() }, { "FilePath", "deleted.jpg" }, { "TrashedAt", BsonNull.Value } }, cancellationToken: Token); await C("Follower").InsertOneAsync(new BsonDocument { { "LocalActorId", _alicePersona.ToString() }, { "ActorURI", "https://elsewhere.example/users/left" } }, cancellationToken: Token); await C("RemoteInstance").InsertOneAsync(new BsonDocument { { "Host", "down.example" }, { "ConsecutiveFailures", 9 }, { "UnavailableUntil", DateTime.UtcNow.AddDays(1) } }, cancellationToken: Token); await C("_migration_history_").InsertOneAsync(new BsonDocument { { "Number", 16 }, { "Name", "focal points are finite" } }, cancellationToken: Token); await C("InteractionSalt").InsertOneAsync(new BsonDocument("Salt", "the backup's day"), cancellationToken: Token); var (backup, error) = await ServerBackup.Create(Context(), "manual", dbOnly: false, Token); Assert.Null(error); return backup; } // what happens after the backup: protective acts, and things merely made or changed async Task LifeGoesOn() { // alice regrets a post: deleted, its media trashed and moved out of what is served await Set(C("Post"), _deletedPost, new BsonDocument { { "DeletedAt", DateTime.UtcNow }, { "Text", BsonNull.Value }, { "Media", new BsonArray() } }); await Set(C("MediaAttachment"), _deletedMedia, new BsonDocument { { "TrashedAt", DateTime.UtcNow }, { "TrashReason", "deleted" } }); File.Move(Path.Combine(Media, "deleted.jpg"), Path.Combine(Trash, "deleted.jpg")); await C("TimelineEntry").DeleteManyAsync(new BsonDocument("PostId", _deletedPost.ToString()), Token); // an edit, a new post with a picture: not protective await Set(C("Post"), _editedPost, new BsonDocument("Text", "second words")); await File.WriteAllTextAsync(Path.Combine(Media, "later.jpg"), "later", Token); await C("Post").InsertOneAsync(new BsonDocument { { "_id", _laterPost }, { "GroupUserId", _alicePersona.ToString() }, { "IsFederatedCopy", false }, { "ObjectURI", Uri(_laterPost) }, { "Text", "later" }, { "DeletedAt", BsonNull.Value } }, cancellationToken: Token); await C("MediaAttachment").InsertOneAsync(new BsonDocument { { "_id", _laterMedia }, { "PostId", _laterPost.ToString() }, { "FilePath", "later.jpg" }, { "TrashedAt", BsonNull.Value } }, cancellationToken: Token); // a follower leaves, another comes await C("Follower").DeleteManyAsync(FilterDefinition.Empty, Token); await C("Follower").InsertOneAsync(new BsonDocument { { "LocalActorId", _alicePersona.ToString() }, { "ActorURI", "https://elsewhere.example/users/came" } }, cancellationToken: Token); // alice blocks someone, the admin blocks a server, someone is reported, alice's persona is suspended, she changes // her password await C("Block").InsertOneAsync(new BsonDocument { { "AvatarId", _alicePersona.ToString() }, { "TargetActorURI", "https://elsewhere.example/users/troll" } }, cancellationToken: Token); await C("DomainBlock").InsertOneAsync(new BsonDocument { { "Domain", "evil.example" }, { "Severity", "Suspend" } }, cancellationToken: Token); await C("Report").InsertOneAsync(new BsonDocument("Comment", "harassment"), cancellationToken: Token); await Set(C("Avatar"), _alicePersona, new BsonDocument("SuspendedAt", DateTime.UtcNow)); await Set(C("RootUser"), _alice, new BsonDocument("HashedPassword", "new")); // bob arrives await C("RootUser").InsertOneAsync(new BsonDocument { { "_id", _bob }, { "UserName", "bob" }, { "HashedPassword", "bob's" }, { "Policies", new BsonArray { "IsUser" } }, { "DeletedAt", BsonNull.Value } }, cancellationToken: Token); await C("Avatar").InsertOneAsync(new BsonDocument { { "_id", _bobPersona }, { "UserName", "bob" }, { "DeletionAt", BsonNull.Value } }, cancellationToken: Token); await C("RootToAvatar").InsertOneAsync(new BsonDocument { { "RootId", _bob.ToString() }, { "AvatarId", _bobPersona.ToString() } }, cancellationToken: Token); await C("ReservedName").InsertOneAsync(new BsonDocument("Name", "bob"), cancellationToken: Token); // sessions, a list made, today's salt, work queued await C("openiddict.tokens").InsertOneAsync(new BsonDocument("subject", "alice"), cancellationToken: Token); await C("PersonaList").InsertOneAsync(new BsonDocument { { "AvatarId", _alicePersona.ToString() }, { "Title", "friends" } }, cancellationToken: Token); await C("InteractionSalt").DeleteManyAsync(FilterDefinition.Empty, Token); await C("InteractionSalt").InsertOneAsync(new BsonDocument("Salt", "today's"), cancellationToken: Token); await C("Job").InsertOneAsync(new BsonDocument("Kind", "Deliver"), cancellationToken: Token); } async Task AssertRestoredAndProtected(BackupInfo backup) { // lost: what was merely made or changed since Assert.Equal("first words", (await One("Post", Id(_editedPost)))["Text"].AsString); Assert.Null(await One("Post", Id(_laterPost))); Assert.Empty(await All("PersonaList")); Assert.Equal(16, (await One("_migration_history_", new BsonDocument()))["Number"].ToInt32()); // kept: every protective act var deleted = await One("Post", Id(_deletedPost)); Assert.False(deleted["DeletedAt"].IsBsonNull); Assert.True(deleted["Text"].IsBsonNull); Assert.Null(await One("TimelineEntry", new BsonDocument("PostId", _deletedPost.ToString()))); Assert.False((await One("MediaAttachment", Id(_deletedMedia)))["TrashedAt"].IsBsonNull); Assert.NotNull(await One("DeletedObject", new BsonDocument("ObjectURI", Uri(_laterPost)))); var later = await One("MediaAttachment", Id(_laterMedia)); Assert.Equal("restore: made after the backup", later["TrashReason"].AsString); Assert.Equal(["https://elsewhere.example/users/came"], (await All("Follower")).Select(f => f["ActorURI"].AsString)); Assert.NotNull(await One("Block", new BsonDocument("TargetActorURI", "https://elsewhere.example/users/troll"))); Assert.NotNull(await One("DomainBlock", new BsonDocument("Domain", "evil.example"))); Assert.NotNull(await One("Report", new BsonDocument("Comment", "harassment"))); Assert.False((await One("Avatar", Id(_alicePersona)))["SuspendedAt"].IsBsonNull); var alice = await One("RootUser", Id(_alice)); Assert.Equal("new", alice["HashedPassword"].AsString); Assert.NotEqual("s1", alice["SessionStamp"].AsString); // bob, made since: deleted, his name kept var bob = await One("RootUser", Id(_bob)); Assert.Equal($"deleted-{_bob}", bob["UserName"].AsString); Assert.True(bob["HashedPassword"].IsBsonNull); Assert.False(bob["DeletedAt"].IsBsonNull); Assert.False((await One("Avatar", Id(_bobPersona)))["DeletionAt"].IsBsonNull); Assert.NotNull(await One("ReservedName", new BsonDocument("Name", "bob"))); // what a backup never holds is as it was; sessions end; circuits close Assert.Equal("today's", (await One("InteractionSalt", new BsonDocument()))["Salt"].AsString); Assert.Single(await All("Job")); Assert.Empty(await All("openiddict.tokens")); Assert.Equal(0, (await One("RemoteInstance", new BsonDocument("Host", "down.example")))["ConsecutiveFailures"].ToInt32()); // the picture of the deleted post came back from the backup, to be trashed again by the janitor Assert.True(File.Exists(Path.Combine(Media, "deleted.jpg"))); Assert.Null(RestoreMarker.Read(Backups)); var record = await (await _database.GetCollection(nameof(RestoreRecord)).FindAsync(r => r.Backup == backup.Id, cancellationToken: Token)).FirstAsync(Token); Assert.Equal(["bob"], record.Report.RootsTombstoned); Assert.Equal(["bob"], record.Report.PersonasTombstoned); Assert.Equal(1, record.Report.PostsGone); Assert.Equal(1, record.Report.PostsDeleted); } [Fact] public async Task A_restore_loses_what_was_made_since_but_never_a_protective_act() { var backup = await TownBackedUp(); await LifeGoesOn(); Assert.Null(await ServerRestore.Request(Context(), backup.Id, "admin", Token)); Assert.Contains("already pending", await ServerRestore.Request(Context(), backup.Id, "admin", Token)); Assert.Equal(RestoreOutcome.Restored, await ServerRestore.ApplyPending(Context(), NullLogger.Instance, Token)); await AssertRestoredAndProtected(backup); var before = ServerBackup.List(Backups).Single(b => b.Kind == "pre-restore"); Assert.Equal(before.Id, (await (await _database.GetCollection(nameof(RestoreRecord)).FindAsync(FilterDefinition.Empty, cancellationToken: Token)).FirstAsync(Token)).PreRestore); } // an attempt that died midway: the next one starts again from the backup, with the same pre-restore backup, never // one of the half-restored database [Fact] public async Task A_restore_that_died_midway_converges_on_the_next_boot() { var backup = await TownBackedUp(); await LifeGoesOn(); Assert.Null(await ServerRestore.Request(Context(), backup.Id, "admin", Token)); Assert.Equal(RestoreOutcome.Restored, await ServerRestore.ApplyPending(Context(), NullLogger.Instance, Token)); var before = ServerBackup.List(Backups).Single(b => b.Kind == "pre-restore"); // as if the boot died after importing a collection, with the marker saying so await C("Post").DeleteManyAsync(FilterDefinition.Empty, Token); await C("Block").DeleteManyAsync(FilterDefinition.Empty, Token); new RestoreMarker { Backup = backup.Id, PreRestore = before.Id, State = "running", Attempts = 1, RequestedBy = "admin" }.Write(Backups); Assert.Equal(RestoreOutcome.Restored, await ServerRestore.ApplyPending(Context(), NullLogger.Instance, Token)); Assert.Single(ServerBackup.List(Backups), b => b.Kind == "pre-restore"); await AssertRestoredAndProtected(backup); } [Fact] public async Task A_backup_from_elsewhere_from_a_newer_build_or_altered_is_refused() { var backup = await TownBackedUp(); Assert.Contains("not https://other.test", await ServerRestore.Request(Context("https://other.test"), backup.Id, "admin", Token)); Assert.Contains("no such backup", await ServerRestore.Request(Context(), "../" + backup.Id, "admin", Token)); var manifest = ArchiveManifest.Read(Path.Combine(Backups, backup.Id)); manifest.MigrationNumber = ServerBackup.CodeMigration() + 1; manifest.Write(Path.Combine(Backups, backup.Id)); Assert.Contains("newer build", await ServerRestore.Request(Context(), backup.Id, "admin", Token)); manifest.MigrationNumber = 16; manifest.Write(Path.Combine(Backups, backup.Id)); await File.AppendAllTextAsync(Path.Combine(Backups, backup.Id, "db", "Post.jsonl.gz"), "x", Token); Assert.Contains("Post: altered", await ServerRestore.Request(Context(), backup.Id, "admin", Token)); Assert.Null(RestoreMarker.Read(Backups)); } // refused at boot before anything changed (the backup went meanwhile): recorded, and the server boots as it was [Fact] public async Task A_restore_that_cannot_start_is_abandoned_and_changes_nothing() { var backup = await TownBackedUp(); await LifeGoesOn(); Assert.Null(await ServerRestore.Request(Context(), backup.Id, "admin", Token)); ServerBackup.Delete(Backups, backup.Id); Assert.Equal(RestoreOutcome.Abandoned, await ServerRestore.ApplyPending(Context(), NullLogger.Instance, Token)); Assert.Equal("second words", (await One("Post", Id(_editedPost)))["Text"].AsString); Assert.Null(RestoreMarker.Read(Backups)); Assert.DoesNotContain(ServerBackup.List(Backups), b => b.Kind == "pre-restore"); var record = await (await _database.GetCollection(nameof(RestoreRecord)).FindAsync(FilterDefinition.Empty, cancellationToken: Token)).FirstAsync(Token); Assert.True(record.Abandoned); Assert.Contains("no such backup", record.Error); } [Fact] public async Task After_three_failures_it_waits_for_someone_to_look() { var backup = await TownBackedUp(); await LifeGoesOn(); new RestoreMarker { Backup = backup.Id, PreRestore = "gone", State = "running", Attempts = RestoreMarker.MaxAttempts, Error = "disk full" }.Write(Backups); Assert.Equal(RestoreOutcome.GaveUp, await ServerRestore.ApplyPending(Context(), NullLogger.Instance, Token)); Assert.Equal("second words", (await One("Post", Id(_editedPost)))["Text"].AsString); Assert.NotNull(RestoreMarker.Read(Backups)); await File.WriteAllTextAsync(RestoreMarker.PathIn(Backups), "{ not json", Token); Assert.Equal(RestoreOutcome.GaveUp, await ServerRestore.ApplyPending(Context(), NullLogger.Instance, Token)); } // a command waits while a restore does; asking how it goes doesn't [Fact] public async Task Commands_wait_for_a_restore_asked_for() { var host = await PrivaPubHost.Shared(); var backups = host.Get(); new RestoreMarker { Backup = "20260101-000000-manual", RequestedBy = "cli" }.Write(backups.Root); try { Assert.Equal(MaintenanceGate.TryLater, await MaintenanceGate.Enter(host.Services, ["promote", "someone"], Token)); Assert.Null(await MaintenanceGate.Enter(host.Services, ["restore", "--status"], Token)); var output = new StringWriter(); Assert.Equal(0, await AdminCommands.Run(["restore", "--status"], host.Services, output: output)); Assert.Contains("waiting: 20260101-000000-manual, pending", output.ToString()); Assert.Equal(1, await AdminCommands.Run(["restore", "20260101-000000-manual"], host.Services, output: TextWriter.Null)); } finally { RestoreMarker.Clear(backups.Root); } } // FEP-8fcf: for two weeks after a restore, no digests, and no follow undone for them [Fact] public async Task Followers_digests_rest_for_two_weeks_after_a_restore() { RestoreRecord.Forget(); Assert.False(await RestoreRecord.InFollowersGrace(Token)); var record = new RestoreRecord { Backup = "test", RestoredAt = DateTime.UtcNow.AddDays(-13) }; await DB.Default.SaveAsync(record, Token); try { RestoreRecord.Forget(); Assert.True(await RestoreRecord.InFollowersGrace(Token)); await DB.Default.Update().MatchID(record.ID).Modify(r => r.RestoredAt, DateTime.UtcNow.AddDays(-15)).ExecuteAsync(Token); RestoreRecord.Forget(); Assert.False(await RestoreRecord.InFollowersGrace(Token)); } finally { await DB.Default.DeleteAsync(record.ID); RestoreRecord.Forget(); } } } }