A backup is restored at boot, and never undoes a protective act

PrivaPub admin restore <id> (and soon the administrator's page) checks the backup (same host, a format and newest
migration this build reads, every hash) and writes restore.json; the running service sees it within seconds and stops,
and the next start restores it in MaintenanceGate, before migrations, indexes and hosted services: a pre-restore backup
taken once, every collection dropped and imported raw with its indexes, the media the live directory lacks brought
back, then the protective merge from the pre-restore backup. Followers and follows are the live ones; blocks, mutes,
domain blocks, reserved names, tombstones, reports, filters and OAuth applications are the union; deletions win;
accounts made since become tombstones and local posts made since answer 410; every session ends.

Each attempt redoes everything; one refused before any change is abandoned and recorded, one failed midway exits 1 for
systemd to retry, and after three it exits 75, which the unit no longer restarts. Commands wait (exit 75) while a
restore is pending. RestoreRecord tells what happened (admin restore --status).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01LsXgEaXee4GCU1hwYgPJXw

PrivaPub.Tests/Infrastructure/RestoreTests.cs

@@ -0,0 +1,316 @@
+using Microsoft.Extensions.Logging.Abstractions;
+
+using MongoDB.Bson;
+using MongoDB.Driver;
+using MongoDB.Entities;
+
+using PrivaPub.Infrastructure.Backup;
+using PrivaPub.Infrastructure.Cli;
+using PrivaPub.Tests.Support;
+using PrivaPub.Tests.Support.Host;
+
+namespace PrivaPub.Tests.Infrastructure
+{
+	// A backup restored at boot: what was made since is lost, what protects is not (owner decision 2026-10-07). A server
+	// in a database of its own is backed up, changed, and restored; alone, since the maintenance lock is the server's one.
+	[Trait("Category", "Integration")]
+	[Xunit.Collection(nameof(Exclusive))]
+	public sealed class RestoreTests : IAsyncLifetime
+	{
+		const string Host = "https://privapub.test";
+
+		readonly string _scratch = Path.Combine(Path.GetTempPath(), $"privapub-restore-tests-{Guid.NewGuid():N}");
+		IMongoDatabase _database;
+
+		string Backups => Path.Combine(_scratch, "backups");
+		string Media => Path.Combine(_scratch, "media");
+		string Trash => Path.Combine(_scratch, "media-trash");
+
+		static CancellationToken Token => TestContext.Current.CancellationToken;
+
+		public async ValueTask InitializeAsync()
+		{
+			Assert.SkipUnless(MongoFixture.Enabled, MongoFixture.Skip);
+			await PrivaPubHost.Shared();
+			_database = DB.Default.Database().Client.GetDatabase($"PrivaPubRestore_{Guid.NewGuid():N}");
+			Directory.CreateDirectory(Media);
+			Directory.CreateDirectory(Trash);
+		}
+
+		public async ValueTask DisposeAsync()
+		{
+			if (_database != default)
+				await _database.Client.DropDatabaseAsync(_database.DatabaseNamespace.DatabaseName);
+			if (Directory.Exists(_scratch))
+				Directory.Delete(_scratch, recursive: true);
+		}
+
+		BackupContext Context(string host = Host) => new(_database, Backups, Media, Trash, host, new BackupOptions());
+
+		IMongoCollection<BsonDocument> C(string name) => _database.GetCollection<BsonDocument>(name);
+
+		async Task<BsonDocument> One(string collection, BsonDocument filter) =>
+			await (await C(collection).FindAsync(filter, cancellationToken: Token)).FirstOrDefaultAsync(Token);
+
+		async Task<List<BsonDocument>> All(string collection) =>
+			await (await C(collection).FindAsync(FilterDefinition<BsonDocument>.Empty, cancellationToken: Token)).ToListAsync(Token);
+
+		static BsonDocument Id(ObjectId id) => new("_id", id);
+
+		static Task Set(IMongoCollection<BsonDocument> collection, ObjectId id, BsonDocument fields) =>
+			collection.UpdateOneAsync(Id(id), new BsonDocument("$set", fields), cancellationToken: Token);
+
+		// the ids of the town this test builds
+		readonly ObjectId _alice = ObjectId.GenerateNewId(), _alicePersona = ObjectId.GenerateNewId();
+		readonly ObjectId _deletedPost = ObjectId.GenerateNewId(), _editedPost = ObjectId.GenerateNewId(), _laterPost = ObjectId.GenerateNewId();
+		readonly ObjectId _deletedMedia = ObjectId.GenerateNewId(), _laterMedia = ObjectId.GenerateNewId();
+		readonly ObjectId _bob = ObjectId.GenerateNewId(), _bobPersona = ObjectId.GenerateNewId();
+
+		static string Uri(ObjectId post) => $"{Host}/peasants/alice/posts/{post}";
+
+		async Task<BackupInfo> TownBackedUp()
+		{
+			await File.WriteAllTextAsync(Path.Combine(Media, "deleted.jpg"), "deleted", Token);
+			await C("RootUser").InsertOneAsync(new BsonDocument { { "_id", _alice }, { "UserName", "alice" }, { "HashedPassword", "old" }, { "SessionStamp", "s1" }, { "Policies", new BsonArray { "IsUser" } }, { "DeletedAt", BsonNull.Value } }, cancellationToken: Token);
+			await C("Avatar").InsertOneAsync(new BsonDocument { { "_id", _alicePersona }, { "UserName", "alice" }, { "SuspendedAt", BsonNull.Value }, { "DeletionAt", BsonNull.Value } }, cancellationToken: Token);
+			await C("RootToAvatar").InsertOneAsync(new BsonDocument { { "RootId", _alice.ToString() }, { "AvatarId", _alicePersona.ToString() } }, cancellationToken: Token);
+			await C("ReservedName").InsertOneAsync(new BsonDocument("Name", "alice"), cancellationToken: Token);
+			await C("Post").InsertManyAsync([
+				new BsonDocument { { "_id", _deletedPost }, { "GroupUserId", _alicePersona.ToString() }, { "IsFederatedCopy", false }, { "ObjectURI", Uri(_deletedPost) }, { "Text", "regret" },
+					{ "Media", new BsonArray { new BsonDocument("AttachmentId", _deletedMedia.ToString()) } }, { "DeletedAt", BsonNull.Value } },
+				new BsonDocument { { "_id", _editedPost }, { "GroupUserId", _alicePersona.ToString() }, { "IsFederatedCopy", false }, { "ObjectURI", Uri(_editedPost) }, { "Text", "first words" }, { "DeletedAt", BsonNull.Value } }
+			], cancellationToken: Token);
+			await C("TimelineEntry").InsertOneAsync(new BsonDocument { { "PostId", _deletedPost.ToString() }, { "ReblogOfPostId", BsonNull.Value } }, cancellationToken: Token);
+			await C("MediaAttachment").InsertOneAsync(new BsonDocument { { "_id", _deletedMedia }, { "PostId", _deletedPost.ToString() }, { "FilePath", "deleted.jpg" }, { "TrashedAt", BsonNull.Value } }, cancellationToken: Token);
+			await C("Follower").InsertOneAsync(new BsonDocument { { "LocalActorId", _alicePersona.ToString() }, { "ActorURI", "https://elsewhere.example/users/left" } }, cancellationToken: Token);
+			await C("RemoteInstance").InsertOneAsync(new BsonDocument { { "Host", "down.example" }, { "ConsecutiveFailures", 9 }, { "UnavailableUntil", DateTime.UtcNow.AddDays(1) } }, cancellationToken: Token);
+			await C("_migration_history_").InsertOneAsync(new BsonDocument { { "Number", 16 }, { "Name", "focal points are finite" } }, cancellationToken: Token);
+			await C("InteractionSalt").InsertOneAsync(new BsonDocument("Salt", "the backup's day"), cancellationToken: Token);
+
+			var (backup, error) = await ServerBackup.Create(Context(), "manual", dbOnly: false, Token);
+			Assert.Null(error);
+			return backup;
+		}
+
+		// what happens after the backup: protective acts, and things merely made or changed
+		async Task LifeGoesOn()
+		{
+			// alice regrets a post: deleted, its media trashed and moved out of what is served
+			await Set(C("Post"), _deletedPost, new BsonDocument { { "DeletedAt", DateTime.UtcNow }, { "Text", BsonNull.Value }, { "Media", new BsonArray() } });
+			await Set(C("MediaAttachment"), _deletedMedia, new BsonDocument { { "TrashedAt", DateTime.UtcNow }, { "TrashReason", "deleted" } });
+			File.Move(Path.Combine(Media, "deleted.jpg"), Path.Combine(Trash, "deleted.jpg"));
+			await C("TimelineEntry").DeleteManyAsync(new BsonDocument("PostId", _deletedPost.ToString()), Token);
+			// an edit, a new post with a picture: not protective
+			await Set(C("Post"), _editedPost, new BsonDocument("Text", "second words"));
+			await File.WriteAllTextAsync(Path.Combine(Media, "later.jpg"), "later", Token);
+			await C("Post").InsertOneAsync(new BsonDocument { { "_id", _laterPost }, { "GroupUserId", _alicePersona.ToString() }, { "IsFederatedCopy", false }, { "ObjectURI", Uri(_laterPost) }, { "Text", "later" }, { "DeletedAt", BsonNull.Value } }, cancellationToken: Token);
+			await C("MediaAttachment").InsertOneAsync(new BsonDocument { { "_id", _laterMedia }, { "PostId", _laterPost.ToString() }, { "FilePath", "later.jpg" }, { "TrashedAt", BsonNull.Value } }, cancellationToken: Token);
+			// a follower leaves, another comes
+			await C("Follower").DeleteManyAsync(FilterDefinition<BsonDocument>.Empty, Token);
+			await C("Follower").InsertOneAsync(new BsonDocument { { "LocalActorId", _alicePersona.ToString() }, { "ActorURI", "https://elsewhere.example/users/came" } }, cancellationToken: Token);
+			// alice blocks someone, the admin blocks a server, someone is reported, alice's persona is suspended, she changes
+			// her password
+			await C("Block").InsertOneAsync(new BsonDocument { { "AvatarId", _alicePersona.ToString() }, { "TargetActorURI", "https://elsewhere.example/users/troll" } }, cancellationToken: Token);
+			await C("DomainBlock").InsertOneAsync(new BsonDocument { { "Domain", "evil.example" }, { "Severity", "Suspend" } }, cancellationToken: Token);
+			await C("Report").InsertOneAsync(new BsonDocument("Comment", "harassment"), cancellationToken: Token);
+			await Set(C("Avatar"), _alicePersona, new BsonDocument("SuspendedAt", DateTime.UtcNow));
+			await Set(C("RootUser"), _alice, new BsonDocument("HashedPassword", "new"));
+			// bob arrives
+			await C("RootUser").InsertOneAsync(new BsonDocument { { "_id", _bob }, { "UserName", "bob" }, { "HashedPassword", "bob's" }, { "Policies", new BsonArray { "IsUser" } }, { "DeletedAt", BsonNull.Value } }, cancellationToken: Token);
+			await C("Avatar").InsertOneAsync(new BsonDocument { { "_id", _bobPersona }, { "UserName", "bob" }, { "DeletionAt", BsonNull.Value } }, cancellationToken: Token);
+			await C("RootToAvatar").InsertOneAsync(new BsonDocument { { "RootId", _bob.ToString() }, { "AvatarId", _bobPersona.ToString() } }, cancellationToken: Token);
+			await C("ReservedName").InsertOneAsync(new BsonDocument("Name", "bob"), cancellationToken: Token);
+			// sessions, a list made, today's salt, work queued
+			await C("openiddict.tokens").InsertOneAsync(new BsonDocument("subject", "alice"), cancellationToken: Token);
+			await C("PersonaList").InsertOneAsync(new BsonDocument { { "AvatarId", _alicePersona.ToString() }, { "Title", "friends" } }, cancellationToken: Token);
+			await C("InteractionSalt").DeleteManyAsync(FilterDefinition<BsonDocument>.Empty, Token);
+			await C("InteractionSalt").InsertOneAsync(new BsonDocument("Salt", "today's"), cancellationToken: Token);
+			await C("Job").InsertOneAsync(new BsonDocument("Kind", "Deliver"), cancellationToken: Token);
+		}
+
+		async Task AssertRestoredAndProtected(BackupInfo backup)
+		{
+			// lost: what was merely made or changed since
+			Assert.Equal("first words", (await One("Post", Id(_editedPost)))["Text"].AsString);
+			Assert.Null(await One("Post", Id(_laterPost)));
+			Assert.Empty(await All("PersonaList"));
+			Assert.Equal(16, (await One("_migration_history_", new BsonDocument()))["Number"].ToInt32());
+
+			// kept: every protective act
+			var deleted = await One("Post", Id(_deletedPost));
+			Assert.False(deleted["DeletedAt"].IsBsonNull);
+			Assert.True(deleted["Text"].IsBsonNull);
+			Assert.Null(await One("TimelineEntry", new BsonDocument("PostId", _deletedPost.ToString())));
+			Assert.False((await One("MediaAttachment", Id(_deletedMedia)))["TrashedAt"].IsBsonNull);
+			Assert.NotNull(await One("DeletedObject", new BsonDocument("ObjectURI", Uri(_laterPost))));
+			var later = await One("MediaAttachment", Id(_laterMedia));
+			Assert.Equal("restore: made after the backup", later["TrashReason"].AsString);
+			Assert.Equal(["https://elsewhere.example/users/came"], (await All("Follower")).Select(f => f["ActorURI"].AsString));
+			Assert.NotNull(await One("Block", new BsonDocument("TargetActorURI", "https://elsewhere.example/users/troll")));
+			Assert.NotNull(await One("DomainBlock", new BsonDocument("Domain", "evil.example")));
+			Assert.NotNull(await One("Report", new BsonDocument("Comment", "harassment")));
+			Assert.False((await One("Avatar", Id(_alicePersona)))["SuspendedAt"].IsBsonNull);
+			var alice = await One("RootUser", Id(_alice));
+			Assert.Equal("new", alice["HashedPassword"].AsString);
+			Assert.NotEqual("s1", alice["SessionStamp"].AsString);
+
+			// bob, made since: deleted, his name kept
+			var bob = await One("RootUser", Id(_bob));
+			Assert.Equal($"deleted-{_bob}", bob["UserName"].AsString);
+			Assert.True(bob["HashedPassword"].IsBsonNull);
+			Assert.False(bob["DeletedAt"].IsBsonNull);
+			Assert.False((await One("Avatar", Id(_bobPersona)))["DeletionAt"].IsBsonNull);
+			Assert.NotNull(await One("ReservedName", new BsonDocument("Name", "bob")));
+
+			// what a backup never holds is as it was; sessions end; circuits close
+			Assert.Equal("today's", (await One("InteractionSalt", new BsonDocument()))["Salt"].AsString);
+			Assert.Single(await All("Job"));
+			Assert.Empty(await All("openiddict.tokens"));
+			Assert.Equal(0, (await One("RemoteInstance", new BsonDocument("Host", "down.example")))["ConsecutiveFailures"].ToInt32());
+
+			// the picture of the deleted post came back from the backup, to be trashed again by the janitor
+			Assert.True(File.Exists(Path.Combine(Media, "deleted.jpg")));
+			Assert.Null(RestoreMarker.Read(Backups));
+			var record = await (await _database.GetCollection<RestoreRecord>(nameof(RestoreRecord)).FindAsync(r => r.Backup == backup.Id, cancellationToken: Token)).FirstAsync(Token);
+			Assert.Equal(["bob"], record.Report.RootsTombstoned);
+			Assert.Equal(["bob"], record.Report.PersonasTombstoned);
+			Assert.Equal(1, record.Report.PostsGone);
+			Assert.Equal(1, record.Report.PostsDeleted);
+		}
+
+		[Fact]
+		public async Task A_restore_loses_what_was_made_since_but_never_a_protective_act()
+		{
+			var backup = await TownBackedUp();
+			await LifeGoesOn();
+
+			Assert.Null(await ServerRestore.Request(Context(), backup.Id, "admin", Token));
+			Assert.Contains("already pending", await ServerRestore.Request(Context(), backup.Id, "admin", Token));
+			Assert.Equal(RestoreOutcome.Restored, await ServerRestore.ApplyPending(Context(), NullLogger.Instance, Token));
+
+			await AssertRestoredAndProtected(backup);
+			var before = ServerBackup.List(Backups).Single(b => b.Kind == "pre-restore");
+			Assert.Equal(before.Id, (await (await _database.GetCollection<RestoreRecord>(nameof(RestoreRecord)).FindAsync(FilterDefinition<RestoreRecord>.Empty, cancellationToken: Token)).FirstAsync(Token)).PreRestore);
+		}
+
+		// an attempt that died midway: the next one starts again from the backup, with the same pre-restore backup, never
+		// one of the half-restored database
+		[Fact]
+		public async Task A_restore_that_died_midway_converges_on_the_next_boot()
+		{
+			var backup = await TownBackedUp();
+			await LifeGoesOn();
+			Assert.Null(await ServerRestore.Request(Context(), backup.Id, "admin", Token));
+			Assert.Equal(RestoreOutcome.Restored, await ServerRestore.ApplyPending(Context(), NullLogger.Instance, Token));
+			var before = ServerBackup.List(Backups).Single(b => b.Kind == "pre-restore");
+
+			// as if the boot died after importing a collection, with the marker saying so
+			await C("Post").DeleteManyAsync(FilterDefinition<BsonDocument>.Empty, Token);
+			await C("Block").DeleteManyAsync(FilterDefinition<BsonDocument>.Empty, Token);
+			new RestoreMarker { Backup = backup.Id, PreRestore = before.Id, State = "running", Attempts = 1, RequestedBy = "admin" }.Write(Backups);
+
+			Assert.Equal(RestoreOutcome.Restored, await ServerRestore.ApplyPending(Context(), NullLogger.Instance, Token));
+			Assert.Single(ServerBackup.List(Backups), b => b.Kind == "pre-restore");
+			await AssertRestoredAndProtected(backup);
+		}
+
+		[Fact]
+		public async Task A_backup_from_elsewhere_from_a_newer_build_or_altered_is_refused()
+		{
+			var backup = await TownBackedUp();
+
+			Assert.Contains("not https://other.test", await ServerRestore.Request(Context("https://other.test"), backup.Id, "admin", Token));
+			Assert.Contains("no such backup", await ServerRestore.Request(Context(), "../" + backup.Id, "admin", Token));
+
+			var manifest = ArchiveManifest.Read(Path.Combine(Backups, backup.Id));
+			manifest.MigrationNumber = ServerBackup.CodeMigration() + 1;
+			manifest.Write(Path.Combine(Backups, backup.Id));
+			Assert.Contains("newer build", await ServerRestore.Request(Context(), backup.Id, "admin", Token));
+
+			manifest.MigrationNumber = 16;
+			manifest.Write(Path.Combine(Backups, backup.Id));
+			await File.AppendAllTextAsync(Path.Combine(Backups, backup.Id, "db", "Post.jsonl.gz"), "x", Token);
+			Assert.Contains("Post: altered", await ServerRestore.Request(Context(), backup.Id, "admin", Token));
+			Assert.Null(RestoreMarker.Read(Backups));
+		}
+
+		// refused at boot before anything changed (the backup went meanwhile): recorded, and the server boots as it was
+		[Fact]
+		public async Task A_restore_that_cannot_start_is_abandoned_and_changes_nothing()
+		{
+			var backup = await TownBackedUp();
+			await LifeGoesOn();
+			Assert.Null(await ServerRestore.Request(Context(), backup.Id, "admin", Token));
+			ServerBackup.Delete(Backups, backup.Id);
+
+			Assert.Equal(RestoreOutcome.Abandoned, await ServerRestore.ApplyPending(Context(), NullLogger.Instance, Token));
+
+			Assert.Equal("second words", (await One("Post", Id(_editedPost)))["Text"].AsString);
+			Assert.Null(RestoreMarker.Read(Backups));
+			Assert.DoesNotContain(ServerBackup.List(Backups), b => b.Kind == "pre-restore");
+			var record = await (await _database.GetCollection<RestoreRecord>(nameof(RestoreRecord)).FindAsync(FilterDefinition<RestoreRecord>.Empty, cancellationToken: Token)).FirstAsync(Token);
+			Assert.True(record.Abandoned);
+			Assert.Contains("no such backup", record.Error);
+		}
+
+		[Fact]
+		public async Task After_three_failures_it_waits_for_someone_to_look()
+		{
+			var backup = await TownBackedUp();
+			await LifeGoesOn();
+			new RestoreMarker { Backup = backup.Id, PreRestore = "gone", State = "running", Attempts = RestoreMarker.MaxAttempts, Error = "disk full" }.Write(Backups);
+
+			Assert.Equal(RestoreOutcome.GaveUp, await ServerRestore.ApplyPending(Context(), NullLogger.Instance, Token));
+			Assert.Equal("second words", (await One("Post", Id(_editedPost)))["Text"].AsString);
+			Assert.NotNull(RestoreMarker.Read(Backups));
+
+			await File.WriteAllTextAsync(RestoreMarker.PathIn(Backups), "{ not json", Token);
+			Assert.Equal(RestoreOutcome.GaveUp, await ServerRestore.ApplyPending(Context(), NullLogger.Instance, Token));
+		}
+
+		// a command waits while a restore does; asking how it goes doesn't
+		[Fact]
+		public async Task Commands_wait_for_a_restore_asked_for()
+		{
+			var host = await PrivaPubHost.Shared();
+			var backups = host.Get<Backups>();
+			new RestoreMarker { Backup = "20260101-000000-manual", RequestedBy = "cli" }.Write(backups.Root);
+			try
+			{
+				Assert.Equal(MaintenanceGate.TryLater, await MaintenanceGate.Enter(host.Services, ["promote", "someone"], Token));
+				Assert.Null(await MaintenanceGate.Enter(host.Services, ["restore", "--status"], Token));
+				var output = new StringWriter();
+				Assert.Equal(0, await AdminCommands.Run(["restore", "--status"], host.Services, output: output));
+				Assert.Contains("waiting: 20260101-000000-manual, pending", output.ToString());
+				Assert.Equal(1, await AdminCommands.Run(["restore", "20260101-000000-manual"], host.Services, output: TextWriter.Null));
+			}
+			finally
+			{
+				RestoreMarker.Clear(backups.Root);
+			}
+		}
+
+		// FEP-8fcf: for two weeks after a restore, no digests, and no follow undone for them
+		[Fact]
+		public async Task Followers_digests_rest_for_two_weeks_after_a_restore()
+		{
+			RestoreRecord.Forget();
+			Assert.False(await RestoreRecord.InFollowersGrace(Token));
+			var record = new RestoreRecord { Backup = "test", RestoredAt = DateTime.UtcNow.AddDays(-13) };
+			await DB.Default.SaveAsync(record, Token);
+			try
+			{
+				RestoreRecord.Forget();
+				Assert.True(await RestoreRecord.InFollowersGrace(Token));
+				await DB.Default.Update<RestoreRecord>().MatchID(record.ID).Modify(r => r.RestoredAt, DateTime.UtcNow.AddDays(-15)).ExecuteAsync(Token);
+				RestoreRecord.Forget();
+				Assert.False(await RestoreRecord.InFollowersGrace(Token));
+			}
+			finally
+			{
+				await DB.Default.DeleteAsync<RestoreRecord>(record.ID);
+				RestoreRecord.Forget();
+			}
+		}
+	}
+}

PrivaPub.Tests/Support/Host/PrivaPubHost.cs

@@ -24,7 +24,8 @@ namespace PrivaPub.Tests.Support.Host
 
 		static readonly SemaphoreSlim Boot = new(1, 1);
 		static readonly Type[] Unwanted = { typeof(JobWorker), typeof(MediaJanitor), typeof(OAuthPruner), typeof(StatisticsSchedule),
-			typeof(PrivaPub.Domain.Social.FollowResender), typeof(PrivaPub.Infrastructure.Backup.BackupScheduler) };
+			typeof(PrivaPub.Domain.Social.FollowResender), typeof(PrivaPub.Infrastructure.Backup.BackupScheduler),
+			typeof(PrivaPub.Infrastructure.Backup.RestoreWatcher) };
 		static PrivaPubHost _shared;
 
 		readonly string _mediaRoot = Path.Combine(Path.GetTempPath(), $"privapub-tests-{Guid.NewGuid():N}");