SocialPub/PrivaPub.Tests/Infrastructure/RestoreTests.cs

using Microsoft.Extensions.Logging.Abstractions;

using MongoDB.Bson;
using MongoDB.Driver;
using MongoDB.Entities;

using PrivaPub.Infrastructure.Backup;
using PrivaPub.Infrastructure.Cli;
using PrivaPub.Tests.Support;
using PrivaPub.Tests.Support.Host;

namespace PrivaPub.Tests.Infrastructure
{
	// A backup restored at boot: what was made since is lost, what protects is not (owner decision 2026-10-07). A server
	// in a database of its own is backed up, changed, and restored; alone, since the maintenance lock is the server's one.
	[Trait("Category", "Integration")]
	[Xunit.Collection(nameof(Exclusive))]
	public sealed class RestoreTests : IAsyncLifetime
	{
		const string Host = "https://privapub.test";

		readonly string _scratch = Path.Combine(Path.GetTempPath(), $"privapub-restore-tests-{Guid.NewGuid():N}");
		IMongoDatabase _database;

		string Backups => Path.Combine(_scratch, "backups");
		string Media => Path.Combine(_scratch, "media");
		string Trash => Path.Combine(_scratch, "media-trash");

		static CancellationToken Token => TestContext.Current.CancellationToken;

		public async ValueTask InitializeAsync()
		{
			Assert.SkipUnless(MongoFixture.Enabled, MongoFixture.Skip);
			await PrivaPubHost.Shared();
			_database = DB.Default.Database().Client.GetDatabase($"PrivaPubRestore_{Guid.NewGuid():N}");
			Directory.CreateDirectory(Media);
			Directory.CreateDirectory(Trash);
		}

		public async ValueTask DisposeAsync()
		{
			if (_database != default)
				await _database.Client.DropDatabaseAsync(_database.DatabaseNamespace.DatabaseName);
			if (Directory.Exists(_scratch))
				Directory.Delete(_scratch, recursive: true);
		}

		BackupContext Context(string host = Host) => new(_database, Backups, Media, Trash, host, new BackupOptions());

		IMongoCollection<BsonDocument> C(string name) => _database.GetCollection<BsonDocument>(name);

		async Task<BsonDocument> One(string collection, BsonDocument filter) =>
			await (await C(collection).FindAsync(filter, cancellationToken: Token)).FirstOrDefaultAsync(Token);

		async Task<List<BsonDocument>> All(string collection) =>
			await (await C(collection).FindAsync(FilterDefinition<BsonDocument>.Empty, cancellationToken: Token)).ToListAsync(Token);

		static BsonDocument Id(ObjectId id) => new("_id", id);

		static Task Set(IMongoCollection<BsonDocument> collection, ObjectId id, BsonDocument fields) =>
			collection.UpdateOneAsync(Id(id), new BsonDocument("$set", fields), cancellationToken: Token);

		// the ids of the town this test builds
		readonly ObjectId _alice = ObjectId.GenerateNewId(), _alicePersona = ObjectId.GenerateNewId();
		readonly ObjectId _deletedPost = ObjectId.GenerateNewId(), _editedPost = ObjectId.GenerateNewId(), _laterPost = ObjectId.GenerateNewId();
		readonly ObjectId _deletedMedia = ObjectId.GenerateNewId(), _laterMedia = ObjectId.GenerateNewId();
		readonly ObjectId _bob = ObjectId.GenerateNewId(), _bobPersona = ObjectId.GenerateNewId();

		static string Uri(ObjectId post) => $"{Host}/peasants/alice/posts/{post}";

		async Task<BackupInfo> TownBackedUp()
		{
			await File.WriteAllTextAsync(Path.Combine(Media, "deleted.jpg"), "deleted", Token);
			await C("RootUser").InsertOneAsync(new BsonDocument { { "_id", _alice }, { "UserName", "alice" }, { "HashedPassword", "old" }, { "SessionStamp", "s1" }, { "Policies", new BsonArray { "IsUser" } }, { "DeletedAt", BsonNull.Value } }, cancellationToken: Token);
			await C("Avatar").InsertOneAsync(new BsonDocument { { "_id", _alicePersona }, { "UserName", "alice" }, { "SuspendedAt", BsonNull.Value }, { "DeletionAt", BsonNull.Value } }, cancellationToken: Token);
			await C("RootToAvatar").InsertOneAsync(new BsonDocument { { "RootId", _alice.ToString() }, { "AvatarId", _alicePersona.ToString() } }, cancellationToken: Token);
			await C("ReservedName").InsertOneAsync(new BsonDocument("Name", "alice"), cancellationToken: Token);
			await C("Post").InsertManyAsync([
				new BsonDocument { { "_id", _deletedPost }, { "GroupUserId", _alicePersona.ToString() }, { "IsFederatedCopy", false }, { "ObjectURI", Uri(_deletedPost) }, { "Text", "regret" },
					{ "Media", new BsonArray { new BsonDocument("AttachmentId", _deletedMedia.ToString()) } }, { "DeletedAt", BsonNull.Value } },
				new BsonDocument { { "_id", _editedPost }, { "GroupUserId", _alicePersona.ToString() }, { "IsFederatedCopy", false }, { "ObjectURI", Uri(_editedPost) }, { "Text", "first words" }, { "DeletedAt", BsonNull.Value } }
			], cancellationToken: Token);
			await C("TimelineEntry").InsertOneAsync(new BsonDocument { { "PostId", _deletedPost.ToString() }, { "ReblogOfPostId", BsonNull.Value } }, cancellationToken: Token);
			await C("MediaAttachment").InsertOneAsync(new BsonDocument { { "_id", _deletedMedia }, { "PostId", _deletedPost.ToString() }, { "FilePath", "deleted.jpg" }, { "TrashedAt", BsonNull.Value } }, cancellationToken: Token);
			await C("Follower").InsertOneAsync(new BsonDocument { { "LocalActorId", _alicePersona.ToString() }, { "ActorURI", "https://elsewhere.example/users/left" } }, cancellationToken: Token);
			await C("RemoteInstance").InsertOneAsync(new BsonDocument { { "Host", "down.example" }, { "ConsecutiveFailures", 9 }, { "UnavailableUntil", DateTime.UtcNow.AddDays(1) } }, cancellationToken: Token);
			await C("_migration_history_").InsertOneAsync(new BsonDocument { { "Number", 16 }, { "Name", "focal points are finite" } }, cancellationToken: Token);
			await C("InteractionSalt").InsertOneAsync(new BsonDocument("Salt", "the backup's day"), cancellationToken: Token);

			var (backup, error) = await ServerBackup.Create(Context(), "manual", dbOnly: false, Token);
			Assert.Null(error);
			return backup;
		}

		// what happens after the backup: protective acts, and things merely made or changed
		async Task LifeGoesOn()
		{
			// alice regrets a post: deleted, its media trashed and moved out of what is served
			await Set(C("Post"), _deletedPost, new BsonDocument { { "DeletedAt", DateTime.UtcNow }, { "Text", BsonNull.Value }, { "Media", new BsonArray() } });
			await Set(C("MediaAttachment"), _deletedMedia, new BsonDocument { { "TrashedAt", DateTime.UtcNow }, { "TrashReason", "deleted" } });
			File.Move(Path.Combine(Media, "deleted.jpg"), Path.Combine(Trash, "deleted.jpg"));
			await C("TimelineEntry").DeleteManyAsync(new BsonDocument("PostId", _deletedPost.ToString()), Token);
			// an edit, a new post with a picture: not protective
			await Set(C("Post"), _editedPost, new BsonDocument("Text", "second words"));
			await File.WriteAllTextAsync(Path.Combine(Media, "later.jpg"), "later", Token);
			await C("Post").InsertOneAsync(new BsonDocument { { "_id", _laterPost }, { "GroupUserId", _alicePersona.ToString() }, { "IsFederatedCopy", false }, { "ObjectURI", Uri(_laterPost) }, { "Text", "later" }, { "DeletedAt", BsonNull.Value } }, cancellationToken: Token);
			await C("MediaAttachment").InsertOneAsync(new BsonDocument { { "_id", _laterMedia }, { "PostId", _laterPost.ToString() }, { "FilePath", "later.jpg" }, { "TrashedAt", BsonNull.Value } }, cancellationToken: Token);
			// a follower leaves, another comes
			await C("Follower").DeleteManyAsync(FilterDefinition<BsonDocument>.Empty, Token);
			await C("Follower").InsertOneAsync(new BsonDocument { { "LocalActorId", _alicePersona.ToString() }, { "ActorURI", "https://elsewhere.example/users/came" } }, cancellationToken: Token);
			// alice blocks someone, the admin blocks a server, someone is reported, alice's persona is suspended, she changes
			// her password
			await C("Block").InsertOneAsync(new BsonDocument { { "AvatarId", _alicePersona.ToString() }, { "TargetActorURI", "https://elsewhere.example/users/troll" } }, cancellationToken: Token);
			await C("DomainBlock").InsertOneAsync(new BsonDocument { { "Domain", "evil.example" }, { "Severity", "Suspend" } }, cancellationToken: Token);
			await C("Report").InsertOneAsync(new BsonDocument("Comment", "harassment"), cancellationToken: Token);
			await Set(C("Avatar"), _alicePersona, new BsonDocument("SuspendedAt", DateTime.UtcNow));
			await Set(C("RootUser"), _alice, new BsonDocument("HashedPassword", "new"));
			// bob arrives
			await C("RootUser").InsertOneAsync(new BsonDocument { { "_id", _bob }, { "UserName", "bob" }, { "HashedPassword", "bob's" }, { "Policies", new BsonArray { "IsUser" } }, { "DeletedAt", BsonNull.Value } }, cancellationToken: Token);
			await C("Avatar").InsertOneAsync(new BsonDocument { { "_id", _bobPersona }, { "UserName", "bob" }, { "DeletionAt", BsonNull.Value } }, cancellationToken: Token);
			await C("RootToAvatar").InsertOneAsync(new BsonDocument { { "RootId", _bob.ToString() }, { "AvatarId", _bobPersona.ToString() } }, cancellationToken: Token);
			await C("ReservedName").InsertOneAsync(new BsonDocument("Name", "bob"), cancellationToken: Token);
			// sessions, a list made, today's salt, work queued
			await C("openiddict.tokens").InsertOneAsync(new BsonDocument("subject", "alice"), cancellationToken: Token);
			await C("PersonaList").InsertOneAsync(new BsonDocument { { "AvatarId", _alicePersona.ToString() }, { "Title", "friends" } }, cancellationToken: Token);
			await C("InteractionSalt").DeleteManyAsync(FilterDefinition<BsonDocument>.Empty, Token);
			await C("InteractionSalt").InsertOneAsync(new BsonDocument("Salt", "today's"), cancellationToken: Token);
			await C("Job").InsertOneAsync(new BsonDocument("Kind", "Deliver"), cancellationToken: Token);
		}

		async Task AssertRestoredAndProtected(BackupInfo backup)
		{
			// lost: what was merely made or changed since
			Assert.Equal("first words", (await One("Post", Id(_editedPost)))["Text"].AsString);
			Assert.Null(await One("Post", Id(_laterPost)));
			Assert.Empty(await All("PersonaList"));
			Assert.Equal(16, (await One("_migration_history_", new BsonDocument()))["Number"].ToInt32());

			// kept: every protective act
			var deleted = await One("Post", Id(_deletedPost));
			Assert.False(deleted["DeletedAt"].IsBsonNull);
			Assert.True(deleted["Text"].IsBsonNull);
			Assert.Null(await One("TimelineEntry", new BsonDocument("PostId", _deletedPost.ToString())));
			Assert.False((await One("MediaAttachment", Id(_deletedMedia)))["TrashedAt"].IsBsonNull);
			Assert.NotNull(await One("DeletedObject", new BsonDocument("ObjectURI", Uri(_laterPost))));
			var later = await One("MediaAttachment", Id(_laterMedia));
			Assert.Equal("restore: made after the backup", later["TrashReason"].AsString);
			Assert.Equal(["https://elsewhere.example/users/came"], (await All("Follower")).Select(f => f["ActorURI"].AsString));
			Assert.NotNull(await One("Block", new BsonDocument("TargetActorURI", "https://elsewhere.example/users/troll")));
			Assert.NotNull(await One("DomainBlock", new BsonDocument("Domain", "evil.example")));
			Assert.NotNull(await One("Report", new BsonDocument("Comment", "harassment")));
			Assert.False((await One("Avatar", Id(_alicePersona)))["SuspendedAt"].IsBsonNull);
			var alice = await One("RootUser", Id(_alice));
			Assert.Equal("new", alice["HashedPassword"].AsString);
			Assert.NotEqual("s1", alice["SessionStamp"].AsString);

			// bob, made since: deleted, his name kept
			var bob = await One("RootUser", Id(_bob));
			Assert.Equal($"deleted-{_bob}", bob["UserName"].AsString);
			Assert.True(bob["HashedPassword"].IsBsonNull);
			Assert.False(bob["DeletedAt"].IsBsonNull);
			Assert.False((await One("Avatar", Id(_bobPersona)))["DeletionAt"].IsBsonNull);
			Assert.NotNull(await One("ReservedName", new BsonDocument("Name", "bob")));

			// what a backup never holds is as it was; sessions end; circuits close
			Assert.Equal("today's", (await One("InteractionSalt", new BsonDocument()))["Salt"].AsString);
			Assert.Single(await All("Job"));
			Assert.Empty(await All("openiddict.tokens"));
			Assert.Equal(0, (await One("RemoteInstance", new BsonDocument("Host", "down.example")))["ConsecutiveFailures"].ToInt32());

			// the picture of the deleted post came back from the backup, to be trashed again by the janitor
			Assert.True(File.Exists(Path.Combine(Media, "deleted.jpg")));
			Assert.Null(RestoreMarker.Read(Backups));
			var record = await (await _database.GetCollection<RestoreRecord>(nameof(RestoreRecord)).FindAsync(r => r.Backup == backup.Id, cancellationToken: Token)).FirstAsync(Token);
			Assert.Equal(["bob"], record.Report.RootsTombstoned);
			Assert.Equal(["bob"], record.Report.PersonasTombstoned);
			Assert.Equal(1, record.Report.PostsGone);
			Assert.Equal(1, record.Report.PostsDeleted);
		}

		[Fact]
		public async Task A_restore_loses_what_was_made_since_but_never_a_protective_act()
		{
			var backup = await TownBackedUp();
			await LifeGoesOn();

			Assert.Null(await ServerRestore.Request(Context(), backup.Id, "admin", Token));
			Assert.Contains("already pending", await ServerRestore.Request(Context(), backup.Id, "admin", Token));
			Assert.Equal(RestoreOutcome.Restored, await ServerRestore.ApplyPending(Context(), NullLogger.Instance, Token));

			await AssertRestoredAndProtected(backup);
			var before = ServerBackup.List(Backups).Single(b => b.Kind == "pre-restore");
			Assert.Equal(before.Id, (await (await _database.GetCollection<RestoreRecord>(nameof(RestoreRecord)).FindAsync(FilterDefinition<RestoreRecord>.Empty, cancellationToken: Token)).FirstAsync(Token)).PreRestore);
		}

		// an attempt that died midway: the next one starts again from the backup, with the same pre-restore backup, never
		// one of the half-restored database
		[Fact]
		public async Task A_restore_that_died_midway_converges_on_the_next_boot()
		{
			var backup = await TownBackedUp();
			await LifeGoesOn();
			Assert.Null(await ServerRestore.Request(Context(), backup.Id, "admin", Token));
			Assert.Equal(RestoreOutcome.Restored, await ServerRestore.ApplyPending(Context(), NullLogger.Instance, Token));
			var before = ServerBackup.List(Backups).Single(b => b.Kind == "pre-restore");

			// as if the boot died after importing a collection, with the marker saying so
			await C("Post").DeleteManyAsync(FilterDefinition<BsonDocument>.Empty, Token);
			await C("Block").DeleteManyAsync(FilterDefinition<BsonDocument>.Empty, Token);
			new RestoreMarker { Backup = backup.Id, PreRestore = before.Id, State = "running", Attempts = 1, RequestedBy = "admin" }.Write(Backups);

			Assert.Equal(RestoreOutcome.Restored, await ServerRestore.ApplyPending(Context(), NullLogger.Instance, Token));
			Assert.Single(ServerBackup.List(Backups), b => b.Kind == "pre-restore");
			await AssertRestoredAndProtected(backup);
		}

		[Fact]
		public async Task A_backup_from_elsewhere_from_a_newer_build_or_altered_is_refused()
		{
			var backup = await TownBackedUp();

			Assert.Contains("not https://other.test", await ServerRestore.Request(Context("https://other.test"), backup.Id, "admin", Token));
			Assert.Contains("no such backup", await ServerRestore.Request(Context(), "../" + backup.Id, "admin", Token));

			var manifest = ArchiveManifest.Read(Path.Combine(Backups, backup.Id));
			manifest.MigrationNumber = ServerBackup.CodeMigration() + 1;
			manifest.Write(Path.Combine(Backups, backup.Id));
			Assert.Contains("newer build", await ServerRestore.Request(Context(), backup.Id, "admin", Token));

			manifest.MigrationNumber = 16;
			manifest.Write(Path.Combine(Backups, backup.Id));
			await File.AppendAllTextAsync(Path.Combine(Backups, backup.Id, "db", "Post.jsonl.gz"), "x", Token);
			Assert.Contains("Post: altered", await ServerRestore.Request(Context(), backup.Id, "admin", Token));
			Assert.Null(RestoreMarker.Read(Backups));
		}

		// refused at boot before anything changed (the backup went meanwhile): recorded, and the server boots as it was
		[Fact]
		public async Task A_restore_that_cannot_start_is_abandoned_and_changes_nothing()
		{
			var backup = await TownBackedUp();
			await LifeGoesOn();
			Assert.Null(await ServerRestore.Request(Context(), backup.Id, "admin", Token));
			ServerBackup.Delete(Backups, backup.Id);

			Assert.Equal(RestoreOutcome.Abandoned, await ServerRestore.ApplyPending(Context(), NullLogger.Instance, Token));

			Assert.Equal("second words", (await One("Post", Id(_editedPost)))["Text"].AsString);
			Assert.Null(RestoreMarker.Read(Backups));
			Assert.DoesNotContain(ServerBackup.List(Backups), b => b.Kind == "pre-restore");
			var record = await (await _database.GetCollection<RestoreRecord>(nameof(RestoreRecord)).FindAsync(FilterDefinition<RestoreRecord>.Empty, cancellationToken: Token)).FirstAsync(Token);
			Assert.True(record.Abandoned);
			Assert.Contains("no such backup", record.Error);
		}

		[Fact]
		public async Task After_three_failures_it_waits_for_someone_to_look()
		{
			var backup = await TownBackedUp();
			await LifeGoesOn();
			new RestoreMarker { Backup = backup.Id, PreRestore = "gone", State = "running", Attempts = RestoreMarker.MaxAttempts, Error = "disk full" }.Write(Backups);

			Assert.Equal(RestoreOutcome.GaveUp, await ServerRestore.ApplyPending(Context(), NullLogger.Instance, Token));
			Assert.Equal("second words", (await One("Post", Id(_editedPost)))["Text"].AsString);
			Assert.NotNull(RestoreMarker.Read(Backups));

			await File.WriteAllTextAsync(RestoreMarker.PathIn(Backups), "{ not json", Token);
			Assert.Equal(RestoreOutcome.GaveUp, await ServerRestore.ApplyPending(Context(), NullLogger.Instance, Token));
		}

		// a command waits while a restore does; asking how it goes doesn't
		[Fact]
		public async Task Commands_wait_for_a_restore_asked_for()
		{
			var host = await PrivaPubHost.Shared();
			var backups = host.Get<Backups>();
			new RestoreMarker { Backup = "20260101-000000-manual", RequestedBy = "cli" }.Write(backups.Root);
			try
			{
				Assert.Equal(MaintenanceGate.TryLater, await MaintenanceGate.Enter(host.Services, ["promote", "someone"], Token));
				Assert.Null(await MaintenanceGate.Enter(host.Services, ["restore", "--status"], Token));
				var output = new StringWriter();
				Assert.Equal(0, await AdminCommands.Run(["restore", "--status"], host.Services, output: output));
				Assert.Contains("waiting: 20260101-000000-manual, pending", output.ToString());
				Assert.Equal(1, await AdminCommands.Run(["restore", "20260101-000000-manual"], host.Services, output: TextWriter.Null));
			}
			finally
			{
				RestoreMarker.Clear(backups.Root);
			}
		}

		// FEP-8fcf: for two weeks after a restore, no digests, and no follow undone for them
		[Fact]
		public async Task Followers_digests_rest_for_two_weeks_after_a_restore()
		{
			RestoreRecord.Forget();
			Assert.False(await RestoreRecord.InFollowersGrace(Token));
			var record = new RestoreRecord { Backup = "test", RestoredAt = DateTime.UtcNow.AddDays(-13) };
			await DB.Default.SaveAsync(record, Token);
			try
			{
				RestoreRecord.Forget();
				Assert.True(await RestoreRecord.InFollowersGrace(Token));
				await DB.Default.Update<RestoreRecord>().MatchID(record.ID).Modify(r => r.RestoredAt, DateTime.UtcNow.AddDays(-15)).ExecuteAsync(Token);
				RestoreRecord.Forget();
				Assert.False(await RestoreRecord.InFollowersGrace(Token));
			}
			finally
			{
				await DB.Default.DeleteAsync<RestoreRecord>(record.ID);
				RestoreRecord.Forget();
			}
		}
	}
}