A backup is restored at boot, and never undoes a protective act

PrivaPub admin restore <id> (and soon the administrator's page) checks the backup (same host, a format and newest
migration this build reads, every hash) and writes restore.json; the running service sees it within seconds and stops,
and the next start restores it in MaintenanceGate, before migrations, indexes and hosted services: a pre-restore backup
taken once, every collection dropped and imported raw with its indexes, the media the live directory lacks brought
back, then the protective merge from the pre-restore backup. Followers and follows are the live ones; blocks, mutes,
domain blocks, reserved names, tombstones, reports, filters and OAuth applications are the union; deletions win;
accounts made since become tombstones and local posts made since answer 410; every session ends.

Each attempt redoes everything; one refused before any change is abandoned and recorded, one failed midway exits 1 for
systemd to retry, and after three it exits 75, which the unit no longer restarts. Commands wait (exit 75) while a
restore is pending. RestoreRecord tells what happened (admin restore --status).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01LsXgEaXee4GCU1hwYgPJXw

PrivaPub/Infrastructure/Backup/MaintenanceGate.cs

@@ -0,0 +1,59 @@
+namespace PrivaPub.Infrastructure.Backup
+{
+	// What Program asks before migrations: whether a restore waits, and what this process does about it. The service carries
+	// it out; a command waits for it (exit 75, EX_TEMPFAIL), except asking how it went.
+	public static class MaintenanceGate
+	{
+		/// <summary>The exit code that stops systemd restarting the service (RestartPreventExitStatus=75): a restore gave up.</summary>
+		public const int TryLater = 75;
+
+		/// <summary>Null to go on booting; else the code to exit with.</summary>
+		public static async Task<int?> Enter(IServiceProvider services, string[] command, CancellationToken token)
+		{
+			var backups = services.GetRequiredService<Backups>();
+			var logger = services.GetRequiredService<ILoggerFactory>().CreateLogger(nameof(MaintenanceGate));
+			if (command != default)
+			{
+				var waiting = RestoreMarker.Read(backups.Root);
+				if (waiting == default || command is ["restore", ..])
+					return default;
+				Console.Error.WriteLine($"the restore of {waiting.Backup} is {waiting.State}{(waiting.Error == default ? string.Empty : $" ({waiting.Error})")}: try again once the service has carried it out (PrivaPub admin restore --status)");
+				return TryLater;
+			}
+			return await ServerRestore.ApplyPending(backups.Context(), logger, token) switch
+			{
+				RestoreOutcome.Failed => 1,//systemd starts it again, and the next attempt redoes everything
+				RestoreOutcome.GaveUp => TryLater,
+				_ => (int?)null
+			};
+		}
+	}
+
+	// A restore asked for while the service runs (from the administrator's page or the CLI): the service stops within
+	// seconds, systemd starts it again, and the restore runs before anything else.
+	public class RestoreWatcher(Backups backups, IHostApplicationLifetime lifetime, ILogger<RestoreWatcher> logger) : BackgroundService
+	{
+		static readonly TimeSpan Interval = TimeSpan.FromSeconds(3);
+
+		protected override async Task ExecuteAsync(CancellationToken stoppingToken)
+		{
+			while (!stoppingToken.IsCancellationRequested)
+			{
+				try
+				{
+					await Task.Delay(Interval, stoppingToken);
+				}
+				catch (OperationCanceledException)
+				{
+					return;
+				}
+				if (RestoreMarker.Read(backups.Root) is { State: "pending" } marker)
+				{
+					logger.LogWarning("A restore of {Backup} was asked for by {RequestedBy}: stopping, to restore it at the next start", marker.Backup, marker.RequestedBy);
+					lifetime.StopApplication();
+					return;
+				}
+			}
+		}
+	}
+}

PrivaPub/Infrastructure/Backup/ProtectiveMerge.cs

@@ -0,0 +1,264 @@
+using MongoDB.Bson;
+using MongoDB.Driver;
+
+namespace PrivaPub.Infrastructure.Backup
+{
+	// A restore may lose what was made since the backup, but it never undoes a protective act (owner decision 2026-10-07).
+	// From the pre-restore backup P, after the backup is imported:
+	// - who follows whom is P's: a follower that left stays gone, so no followers-only post reaches it, and one gained is
+	//   kept; the persona's follows are P's too;
+	// - blocks, mutes, domain blocks (the server's and the personas'), being blocked, reserved names, deletion tombstones,
+	//   reports, filters and OAuth applications are the union of both, P's row winning;
+	// - deletions win: a root, persona, group, post or remote account deleted since is deleted again (P's row, as its
+	//   deletion left it), with what a deletion takes away; P's moderation of a persona (silenced, suspended, banned) stays;
+	// - a root keeps P's password, e-mail, ban and policies;
+	// - roots, personas and groups made since become tombstones: deleted, their names kept; local posts made since answer
+	//   410; media made since go to the trash, as do media trashed since (and they leave the posts restored with them);
+	// - every session ends, and every server's circuit is closed again.
+	// Run on the imported database, it reads P's files only, so it can run again after a failed attempt.
+	public static class ProtectiveMerge
+	{
+		static readonly (string Collection, string[] Key)[] Unions =
+		[
+			("ReservedName", ["Name"]),
+			("DomainBlock", ["Domain"]),
+			("Block", ["AvatarId", "TargetActorURI"]),
+			("Mute", ["AvatarId", "TargetActorURI"]),
+			("AccountDomainBlock", ["AvatarId", "Domain"]),
+			("BlockedBy", ["AvatarId", "ActorURI"]),
+			("DeletedObject", ["ObjectURI"]),
+			("Report", ["_id"]),
+			("PersonaFilter", ["_id"]),
+			("openiddict.applications", ["client_id"])
+		];
+
+		const string Since = "restore: made after the backup";
+
+		public static async Task Apply(IMongoDatabase database, string previous, RestoreReport report, CancellationToken token)
+		{
+			IEnumerable<BsonDocument> P(string collection) => ServerRestore.Read(Path.Combine(previous, collection + ".jsonl.gz"));
+			var now = DateTime.UtcNow;
+
+			// who follows whom: P's
+			foreach (var collection in new[] { "Follower", "Following" })
+			{
+				await database.DropCollectionAsync(collection, token);
+				await ServerRestore.Insert(database.GetCollection<BsonDocument>(collection), P(collection), token);
+			}
+
+			foreach (var (collection, key) in Unions)
+				report.ProtectiveKept += await Union(database.GetCollection<BsonDocument>(collection), P(collection), key, token);
+
+			var roots = database.GetCollection<BsonDocument>("RootUser");
+			foreach (var root in P("RootUser"))
+			{
+				var restored = await ById(roots, root["_id"], token);
+				if (restored == default)
+				{
+					// made since: deleted, as RootRemoval leaves a root
+					report.RootsTombstoned.Add(root.GetValue("UserName", "").ToString());
+					root["UserName"] = $"deleted-{root["_id"]}";
+					root["Email"] = BsonNull.Value;
+					root["HashedPassword"] = BsonNull.Value;
+					root["Policies"] = new BsonArray();
+					root["IsBanned"] = false;
+					root["IsEmailValidated"] = false;
+					root["DeletedAt"] = Deleted(root, "DeletedAt", now);
+					await roots.InsertOneAsync(root, cancellationToken: token);
+					continue;
+				}
+				if (IsSet(root, "DeletedAt"))
+				{
+					if (!IsSet(restored, "DeletedAt"))
+						report.RootsDeleted++;
+					await roots.ReplaceOneAsync(Id(root["_id"]), root, cancellationToken: token);
+					continue;
+				}
+				await roots.UpdateOneAsync(Id(root["_id"]), Copy(root, "HashedPassword", "Email", "IsEmailValidated", "IsBanned", "Policies",
+					"CredentialsChangedAt", "ResetPasswordToken", "ResetPasswordTokenSentAt"), cancellationToken: token);
+			}
+
+			var avatars = database.GetCollection<BsonDocument>("Avatar");
+			foreach (var avatar in P("Avatar"))
+			{
+				var restored = await ById(avatars, avatar["_id"], token);
+				if (restored == default)
+				{
+					report.PersonasTombstoned.Add(avatar.GetValue("UserName", "").ToString());
+					avatar["DeletionAt"] = Deleted(avatar, "DeletionAt", now);
+					await avatars.InsertOneAsync(avatar, cancellationToken: token);
+					continue;
+				}
+				if (IsSet(avatar, "DeletionAt"))
+				{
+					if (!IsSet(restored, "DeletionAt"))
+						report.PersonasDeleted++;
+					await avatars.ReplaceOneAsync(Id(avatar["_id"]), avatar, cancellationToken: token);
+					// what RootRemoval takes with a persona, which the backup still had
+					foreach (var collection in new[] { "PersonaListMember", "PersonaList", "PersonaFilter", "FollowedTag", "ScheduledStatus" })
+						await database.GetCollection<BsonDocument>(collection).DeleteManyAsync(new BsonDocument("AvatarId", avatar["_id"].ToString()), token);
+					continue;
+				}
+				var moderated = new[] { "SilencedAt", "SuspendedAt", "BannedAt" }.Where(field => IsSet(avatar, field)).ToArray();
+				if (moderated.Length > 0)
+					await avatars.UpdateOneAsync(Id(avatar["_id"]), Copy(avatar, moderated), cancellationToken: token);
+			}
+
+			var links = database.GetCollection<BsonDocument>("RootToAvatar");
+			var linked = (await (await links.FindAsync(FilterDefinition<BsonDocument>.Empty, cancellationToken: token)).ToListAsync(token))
+				.Select(l => l.GetValue("AvatarId", BsonNull.Value).ToString()).ToHashSet(StringComparer.Ordinal);
+			await ServerRestore.Insert(links, P("RootToAvatar").Where(l => !linked.Contains(l.GetValue("AvatarId", BsonNull.Value).ToString())), token);
+
+			var groups = database.GetCollection<BsonDocument>("Group");
+			foreach (var group in P("Group"))
+			{
+				var restored = await ById(groups, group["_id"], token);
+				if (restored == default)
+				{
+					report.GroupsTombstoned.Add(group.GetValue("UserName", "").ToString());
+					group["DeletionAt"] = Deleted(group, "DeletionAt", now);
+					await groups.InsertOneAsync(group, cancellationToken: token);
+					continue;
+				}
+				if (!IsSet(group, "DeletionAt"))
+					continue;
+				if (!IsSet(restored, "DeletionAt"))
+					report.GroupsDeleted++;
+				await groups.ReplaceOneAsync(Id(group["_id"]), group, cancellationToken: token);
+			}
+
+			// remote accounts deleted or gone since: as P has them
+			var foreign = database.GetCollection<BsonDocument>("ForeignAvatar");
+			foreach (var account in P("ForeignAvatar").Where(a => IsSet(a, "DeletionAt") || IsSet(a, "ForgottenAt")))
+				await foreign.ReplaceOneAsync(Id(account["_id"]), account, cancellationToken: token);
+
+			await Posts(database, P("Post"), report, token);
+			await Media(database, P("MediaAttachment"), report, token);
+
+			// every session ends: a new stamp for each root's tokens, and the personas' OAuth tokens go
+			foreach (var root in await (await roots.FindAsync(FilterDefinition<BsonDocument>.Empty, cancellationToken: token)).ToListAsync(token))
+				await roots.UpdateOneAsync(Id(root["_id"]), Builders<BsonDocument>.Update
+					.Set("SessionStamp", Guid.NewGuid().ToString("N"))
+					.Set("CredentialsChangedAt", now), cancellationToken: token);
+			foreach (var collection in new[] { "openiddict.tokens", "openiddict.authorizations" })
+				report.SessionsEnded += (int)(await database.GetCollection<BsonDocument>(collection).DeleteManyAsync(FilterDefinition<BsonDocument>.Empty, token)).DeletedCount;
+
+			// what the backup remembered of servers being down is long past
+			await database.GetCollection<BsonDocument>("RemoteInstance").UpdateManyAsync(FilterDefinition<BsonDocument>.Empty,
+				Builders<BsonDocument>.Update.Set("ConsecutiveFailures", 0).Set("UnavailableUntil", BsonNull.Value), cancellationToken: token);
+		}
+
+		// posts deleted since: deleted again, out of timelines and pins; local posts made since: 410
+		static async Task Posts(IMongoDatabase database, IEnumerable<BsonDocument> previous, RestoreReport report, CancellationToken token)
+		{
+			var posts = database.GetCollection<BsonDocument>("Post");
+			var gone = database.GetCollection<BsonDocument>("DeletedObject");
+			foreach (var batch in previous.Where(p => IsSet(p, "DeletedAt") || IsLocal(p)).Chunk(500))
+			{
+				var ids = new BsonArray(batch.Select(p => p["_id"]));
+				var restored = (await (await posts.FindAsync(new BsonDocument("_id", new BsonDocument("$in", ids)), cancellationToken: token)).ToListAsync(token))
+					.ToDictionary(p => p["_id"]);
+				foreach (var post in batch)
+				{
+					if (!restored.TryGetValue(post["_id"], out var mine))
+					{
+						if (IsLocal(post) && post.GetValue("ObjectURI", BsonNull.Value) is BsonString uri)
+						{
+							await gone.UpdateOneAsync(new BsonDocument("ObjectURI", uri), Builders<BsonDocument>.Update
+								.SetOnInsert("ObjectURI", uri).SetOnInsert("DeletedAt", DateTime.UtcNow), new UpdateOptions { IsUpsert = true }, token);
+							report.PostsGone++;
+						}
+						continue;
+					}
+					if (!IsSet(post, "DeletedAt") || IsSet(mine, "DeletedAt"))
+						continue;
+					await posts.ReplaceOneAsync(Id(post["_id"]), post, cancellationToken: token);
+					var id = post["_id"].ToString();
+					await database.GetCollection<BsonDocument>("TimelineEntry").DeleteManyAsync(
+						new BsonDocument("$or", new BsonArray { new BsonDocument("PostId", id), new BsonDocument("ReblogOfPostId", id) }), token);
+					await database.GetCollection<BsonDocument>("Pin").DeleteManyAsync(new BsonDocument("PostId", id), token);
+					report.PostsDeleted++;
+				}
+			}
+		}
+
+		// media trashed since, and media made since: in the trash (the janitor deletes their files after its grace), and out
+		// of the posts restored; media of posts deleted now: in the trash too
+		static async Task Media(IMongoDatabase database, IEnumerable<BsonDocument> previous, RestoreReport report, CancellationToken token)
+		{
+			var media = database.GetCollection<BsonDocument>("MediaAttachment");
+			var posts = database.GetCollection<BsonDocument>("Post");
+			var now = DateTime.UtcNow;
+			foreach (var batch in previous.Chunk(1000))
+			{
+				var ids = new BsonArray(batch.Select(m => m["_id"]));
+				var restored = (await (await media.FindAsync(new BsonDocument("_id", new BsonDocument("$in", ids)), cancellationToken: token)).ToListAsync(token))
+					.ToDictionary(m => m["_id"]);
+				var made = new List<BsonDocument>();
+				foreach (var row in batch)
+				{
+					if (!restored.TryGetValue(row["_id"], out var mine))
+					{
+						if (!IsSet(row, "TrashedAt"))
+						{
+							row["TrashedAt"] = now;
+							row["TrashReason"] = Since;
+						}
+						made.Add(row);
+						continue;
+					}
+					if (!IsSet(row, "TrashedAt") || IsSet(mine, "TrashedAt"))
+						continue;
+					await media.UpdateOneAsync(Id(row["_id"]), Copy(row, "TrashedAt", "TrashReason"), cancellationToken: token);
+					if (mine.GetValue("PostId", BsonNull.Value) is BsonString postId)
+						await posts.UpdateOneAsync(Id(ObjectId.TryParse(postId.AsString, out var post) ? post : postId), Builders<BsonDocument>.Update.PullFilter<BsonDocument>("Media",
+							new BsonDocument("AttachmentId", row["_id"].ToString())), cancellationToken: token);
+					report.MediaTrashed++;
+				}
+				await ServerRestore.Insert(media, made, token);
+				report.MediaTrashed += made.Count(m => m["TrashReason"] == Since);
+			}
+
+			var deleted = await (await posts.FindAsync(new BsonDocument("DeletedAt", new BsonDocument("$ne", BsonNull.Value)),
+				new FindOptions<BsonDocument> { Projection = new BsonDocument("_id", 1) }, token)).ToListAsync(token);
+			foreach (var batch in deleted.Select(p => (BsonValue)p["_id"].ToString()).Chunk(1000))
+				report.MediaTrashed += (int)(await media.UpdateManyAsync(
+					new BsonDocument { { "PostId", new BsonDocument("$in", new BsonArray(batch)) }, { "TrashedAt", BsonNull.Value } },
+					Builders<BsonDocument>.Update.Set("TrashedAt", now).Set("TrashReason", "restore: its post was deleted"), cancellationToken: token)).ModifiedCount;
+		}
+
+		// P's rows added, each replacing the restored row with the same key: how many P had
+		static async Task<int> Union(IMongoCollection<BsonDocument> collection, IEnumerable<BsonDocument> previous, string[] key, CancellationToken token)
+		{
+			var count = 0;
+			foreach (var batch in previous.Chunk(500))
+			{
+				var writes = new List<WriteModel<BsonDocument>>();
+				foreach (var row in batch)
+				{
+					var match = new BsonDocument(key.Select(field => new BsonElement(field, row.GetValue(field, BsonNull.Value))));
+					writes.Add(new DeleteManyModel<BsonDocument>(match));
+					writes.Add(new InsertOneModel<BsonDocument>(row));
+				}
+				await collection.BulkWriteAsync(writes, new BulkWriteOptions { IsOrdered = true, BypassDocumentValidation = true }, token);
+				count += batch.Length;
+			}
+			return count;
+		}
+
+		static bool IsLocal(BsonDocument post) =>
+			post.GetValue("GroupUserId", BsonNull.Value) is BsonString && !post.GetValue("IsFederatedCopy", false).ToBoolean();
+
+		static bool IsSet(BsonDocument row, string field) => row.TryGetValue(field, out var value) && !value.IsBsonNull;
+
+		static BsonValue Deleted(BsonDocument row, string field, DateTime now) => IsSet(row, field) ? row[field] : now;
+
+		static FilterDefinition<BsonDocument> Id(BsonValue id) => new BsonDocument("_id", id);
+
+		static async Task<BsonDocument> ById(IMongoCollection<BsonDocument> collection, BsonValue id, CancellationToken token) =>
+			await (await collection.FindAsync(Id(id), cancellationToken: token)).FirstOrDefaultAsync(token);
+
+		static UpdateDefinition<BsonDocument> Copy(BsonDocument from, params string[] fields) =>
+			Builders<BsonDocument>.Update.Combine(fields.Select(field => Builders<BsonDocument>.Update.Set(field, from.GetValue(field, BsonNull.Value))));
+	}
+}

PrivaPub/Infrastructure/Backup/RestoreMarker.cs

@@ -12,6 +12,7 @@ namespace PrivaPub.Infrastructure.Backup
 		public string Backup { get; set; }//the backup restored
 		public string PreRestore { get; set; }//the backup taken just before, what the protective merge reads
 		public string State { get; set; } = "pending";//pending, then running
+		public string RequestedBy { get; set; }//who asked: a root's name, or "cli"
 		public int Attempts { get; set; }
 		public DateTime RequestedAt { get; set; } = DateTime.UtcNow;
 		public string Error { get; set; }
@@ -29,7 +30,8 @@ namespace PrivaPub.Infrastructure.Backup
 			}
 			catch (JsonException)
 			{
-				return default;
+				//never written half (Write is atomic): someone's hand; the server won't guess what was meant
+				return new RestoreMarker { State = "unreadable", Attempts = MaxAttempts, Error = $"{FileName} can't be read" };
 			}
 		}
 

PrivaPub/Infrastructure/Backup/RestoreRecord.cs

@@ -0,0 +1,62 @@
+using MongoDB.Entities;
+
+namespace PrivaPub.Infrastructure.Backup
+{
+	// What a restore did (or why it was abandoned), kept for the administrator's page and for the followers' grace: for
+	// FollowersGrace after a restore the followers' digests (FEP-8fcf) are neither sent nor acted on to undo a follow, since
+	// what the restore lost of either side is not the other's fault.
+	public class RestoreRecord : Entity
+	{
+		public static readonly TimeSpan FollowersGrace = TimeSpan.FromDays(14);
+
+		public string Backup { get; set; }
+		public DateTime BackupCreatedAt { get; set; }
+		public string PreRestore { get; set; }
+		public string RequestedBy { get; set; }
+		public DateTime RequestedAt { get; set; }
+		public DateTime RestoredAt { get; set; } = DateTime.UtcNow;
+		public int Attempts { get; set; }
+		public bool Abandoned { get; set; }//nothing was changed: the server booted as it was
+		public string Error { get; set; }
+		public RestoreReport Report { get; set; } = new();
+
+		static (DateTime Until, DateTime Read) _grace;
+
+		/// <summary>Whether a restore ended less than FollowersGrace ago (read at most once a minute).</summary>
+		public static async Task<bool> InFollowersGrace(CancellationToken token)
+		{
+			var now = DateTime.UtcNow;
+			var cached = _grace;
+			if (now - cached.Read > TimeSpan.FromMinutes(1))
+			{
+				var last = await DB.Default.Find<RestoreRecord>().Match(r => !r.Abandoned).Sort(r => r.RestoredAt, Order.Descending).ExecuteFirstAsync(token);
+				cached = (last == default ? DateTime.MinValue : last.RestoredAt + FollowersGrace, now);
+				_grace = cached;
+			}
+			return now < cached.Until;
+		}
+
+		/// <summary>Forgets what was read, so a restore made in this process counts at once.</summary>
+		public static void Forget() => _grace = default;
+	}
+
+	public class RestoreReport
+	{
+		public int Collections { get; set; }
+		public long Documents { get; set; }
+		public int CollectionsDropped { get; set; }//live collections the backup had no documents in
+		public int MediaRestored { get; set; }//files the live directory lacked, brought back
+		public int MediaMissing { get; set; }
+		public int RootsDeleted { get; set; }//deleted since the backup, deleted again
+		public int PersonasDeleted { get; set; }
+		public int GroupsDeleted { get; set; }
+		public int PostsDeleted { get; set; }
+		public List<string> RootsTombstoned { get; set; } = [];//made after the backup: deleted, their names kept
+		public List<string> PersonasTombstoned { get; set; } = [];
+		public List<string> GroupsTombstoned { get; set; } = [];
+		public int PostsGone { get; set; }//local posts made after the backup: 410 from now on
+		public int MediaTrashed { get; set; }
+		public int ProtectiveKept { get; set; }//blocks, mutes, domain blocks, reserved names, reports, filters, deletions kept from since
+		public int SessionsEnded { get; set; }
+	}
+}

PrivaPub/Infrastructure/Backup/ServerBackup.cs

@@ -38,7 +38,8 @@ namespace PrivaPub.Infrastructure.Backup
 			["openiddict.tokens"] = "sessions: a restore ends every one",
 			["openiddict.authorizations"] = "sessions: a restore ends every one",
 			[nameof(MaintenanceLock)] = "who is backing up or restoring now",
-			["AppConfiguration"] = "the configuration's copy holds the SMTP password, and is made again from appsettings at boot"
+			["AppConfiguration"] = "the configuration's copy holds the SMTP password, and is made again from appsettings at boot",
+			[nameof(RestoreRecord)] = "what restores did outlives what they restore"
 		};
 
 		static readonly JsonWriterSettings Json = new() { OutputMode = JsonOutputMode.CanonicalExtendedJson, Indent = false };

PrivaPub/Infrastructure/Backup/ServerRestore.cs

@@ -0,0 +1,247 @@
+using MongoDB.Bson;
+using MongoDB.Driver;
+
+using System.IO.Compression;
+
+namespace PrivaPub.Infrastructure.Backup
+{
+	public enum RestoreOutcome
+	{
+		None,//nothing was asked
+		Restored,
+		Abandoned,//refused before anything changed: the server boots as it was
+		Failed,//failed midway: the next boot tries again from the start
+		GaveUp//failed MaxAttempts times: the server stays down until someone looks
+	}
+
+	// A backup restored (owner decision 2026-10-07: from the CLI or the administrator's page). Asking writes restore.json
+	// and the running service stops at once (RestoreWatcher); the restore itself runs at the next boot, before migrations,
+	// indexes and every hosted service, with nothing else touching the database:
+	// 1. a pre-restore backup P is taken, once (a retry reuses it);
+	// 2. every collection the backup holds is dropped and imported raw, with its indexes; every other one is dropped,
+	//    except what a backup never holds (ServerBackup.Excluded), which stays as it is;
+	// 3. media files the live directory lacks come back from the backup (or the trash); a restore deletes no file;
+	// 4. ProtectiveMerge brings back from P every protective act made since the backup;
+	// 5. every session ends, every server's circuit closes, and a RestoreRecord tells what happened.
+	// Each attempt redoes everything, so one that dies midway converges on the next.
+	public static class ServerRestore
+	{
+		static readonly TimeSpan LockWait = TimeSpan.FromMinutes(10);
+
+		/// <summary>Why a backup can't be restored here: none when it can.</summary>
+		public static async Task<List<string>> Check(BackupContext context, string id, CancellationToken token)
+		{
+			var backup = ServerBackup.Find(context.BackupsRoot, id);
+			if (backup?.Manifest == default)
+				return ["no such backup"];
+			var manifest = backup.Manifest;
+			var problems = new List<string>();
+			if (manifest.Format != ArchiveManifest.CurrentFormat)
+				problems.Add($"its format is {manifest.Format}, this build reads {ArchiveManifest.CurrentFormat}");
+			if (!string.Equals(manifest.Host?.TrimEnd('/'), context.Host, StringComparison.OrdinalIgnoreCase))
+				problems.Add($"it was made by {manifest.Host}, not {context.Host}: every id and address in it names its host");
+			var code = ServerBackup.CodeMigration();
+			if (manifest.MigrationNumber > code)
+				problems.Add($"it was made by a newer build (migration {manifest.MigrationNumber}; this one knows {code})");
+			problems.AddRange(await ServerBackup.Verify(context.BackupsRoot, id, token));
+			return problems;
+		}
+
+		/// <summary>Asks for a backup to be restored at the next boot: why not, or null once asked.</summary>
+		public static async Task<string> Request(BackupContext context, string id, string requestedBy, CancellationToken token)
+		{
+			var problems = await Check(context, id, token);
+			if (problems.Count > 0)
+				return string.Join("; ", problems);
+			var waiting = RestoreMarker.Read(context.BackupsRoot);
+			if (waiting != default && waiting.Attempts < RestoreMarker.MaxAttempts)
+				return $"the restore of {waiting.Backup} is already {waiting.State}";
+			if (await MaintenanceLock.Current(token) is { } held)
+				return $"a {held.What} is running";
+			new RestoreMarker { Backup = id, RequestedBy = requestedBy }.Write(context.BackupsRoot);
+			return default;
+		}
+
+		/// <summary>The restore asked for, carried out (at boot, before migrations): what came of it.</summary>
+		public static async Task<RestoreOutcome> ApplyPending(BackupContext context, ILogger logger, CancellationToken token)
+		{
+			var marker = RestoreMarker.Read(context.BackupsRoot);
+			if (marker == default)
+				return RestoreOutcome.None;
+			if (marker.Attempts >= RestoreMarker.MaxAttempts)
+			{
+				logger.LogCritical("The restore of {Backup} failed {Attempts} times ({Error}). The database may be half restored; {PreRestore} holds it as it was before. Restore that backup or another (PrivaPub admin restore <id>), or delete {Marker} to boot as it is",
+					marker.Backup, marker.Attempts, marker.Error, marker.PreRestore ?? "no backup", RestoreMarker.PathIn(context.BackupsRoot));
+				return RestoreOutcome.GaveUp;
+			}
+
+			await using var held = await TakeLock(token);
+			if (held == default)
+				return await Abandon(context, marker, "a backup kept the maintenance lock for ten minutes", logger, token);
+
+			var problems = await Check(context, marker.Backup, token);
+			if (problems.Count > 0 && marker.PreRestore == default)
+				return await Abandon(context, marker, string.Join("; ", problems), logger, token);
+
+			if (marker.PreRestore == default)
+			{
+				var (taken, error) = await ServerBackup.CreateHeld(context, "pre-restore", dbOnly: false, token);
+				if (taken == default)
+					return await Abandon(context, marker, $"the pre-restore backup could not be made: {error}", logger, token);
+				marker.PreRestore = taken.Id;
+			}
+			marker.State = "running";
+			marker.Attempts++;
+			marker.Error = default;
+			marker.Write(context.BackupsRoot);
+			logger.LogWarning("Restoring {Backup} (attempt {Attempt}); the server as it was is {PreRestore}", marker.Backup, marker.Attempts, marker.PreRestore);
+
+			try
+			{
+				if (problems.Count > 0)
+					throw new InvalidOperationException(string.Join("; ", problems));
+				var backup = ServerBackup.Find(context.BackupsRoot, marker.Backup);
+				var report = new RestoreReport();
+				await Import(context, backup, report, token);
+				Media(context, backup, report);
+				await ProtectiveMerge.Apply(context.Database, Path.Combine(context.BackupsRoot, marker.PreRestore, "db"), report, token);
+				await Record(context, new RestoreRecord
+				{
+					Backup = backup.Id,
+					BackupCreatedAt = backup.CreatedAt,
+					PreRestore = marker.PreRestore,
+					RequestedBy = marker.RequestedBy,
+					RequestedAt = marker.RequestedAt,
+					Attempts = marker.Attempts,
+					Report = report
+				}, token);
+				RestoreRecord.Forget();
+				RestoreMarker.Clear(context.BackupsRoot);
+				logger.LogWarning("Restored {Backup}: {Documents} documents in {Collections} collections, {Media} media files brought back, {Tombstoned} accounts made since deleted",
+					backup.Id, report.Documents, report.Collections, report.MediaRestored, report.RootsTombstoned.Count + report.PersonasTombstoned.Count + report.GroupsTombstoned.Count);
+				return RestoreOutcome.Restored;
+			}
+			catch (Exception ex) when (ex is not OperationCanceledException)
+			{
+				marker.Error = ex.Message;
+				marker.Write(context.BackupsRoot);
+				logger.LogError(ex, "The restore of {Backup} failed (attempt {Attempt} of {Max})", marker.Backup, marker.Attempts, RestoreMarker.MaxAttempts);
+				return marker.Attempts >= RestoreMarker.MaxAttempts ? RestoreOutcome.GaveUp : RestoreOutcome.Failed;
+			}
+		}
+
+		static async Task<MaintenanceLock.Held> TakeLock(CancellationToken token)
+		{
+			var until = DateTime.UtcNow + LockWait;
+			while (true)
+			{
+				var held = await MaintenanceLock.Take("restore", token);
+				if (held != default || DateTime.UtcNow > until)
+					return held;
+				await Task.Delay(TimeSpan.FromSeconds(5), token);
+			}
+		}
+
+		// refused before anything changed: recorded for the administrator's page, and the server boots as it was
+		static async Task<RestoreOutcome> Abandon(BackupContext context, RestoreMarker marker, string why, ILogger logger, CancellationToken token)
+		{
+			logger.LogError("The restore of {Backup} was abandoned, nothing changed: {Why}", marker.Backup, why);
+			await Record(context, new RestoreRecord
+			{
+				Backup = marker.Backup,
+				RequestedBy = marker.RequestedBy,
+				RequestedAt = marker.RequestedAt,
+				Attempts = marker.Attempts,
+				Abandoned = true,
+				Error = why
+			}, token);
+			RestoreMarker.Clear(context.BackupsRoot);
+			return RestoreOutcome.Abandoned;
+		}
+
+		// in the database restored (a backup never holds these records: they outlive what they restore)
+		static async Task Record(BackupContext context, RestoreRecord record, CancellationToken token)
+		{
+			record.ID = ObjectId.GenerateNewId().ToString();
+			await context.Database.GetCollection<RestoreRecord>(nameof(RestoreRecord)).InsertOneAsync(record, cancellationToken: token);
+		}
+
+		// each collection dropped and imported as the backup holds it, its indexes made again; the others dropped
+		static async Task Import(BackupContext context, BackupInfo backup, RestoreReport report, CancellationToken token)
+		{
+			var database = context.Database;
+			var directory = Path.Combine(context.BackupsRoot, backup.Id, "db");
+			foreach (var entry in backup.Manifest.Collections)
+			{
+				await database.DropCollectionAsync(entry.Name, token);
+				await database.CreateCollectionAsync(entry.Name, cancellationToken: token);
+				report.Documents += await Insert(database.GetCollection<BsonDocument>(entry.Name), Read(Path.Combine(directory, entry.Name + ".jsonl.gz")), token);
+				var indexes = entry.Indexes.Select(BsonDocument.Parse).Where(i => i.GetValue("name", "").AsString != "_id_").ToList();
+				foreach (var index in indexes)
+					index.Remove("ns");
+				if (indexes.Count > 0)
+					await database.RunCommandAsync<BsonDocument>(new BsonDocument { { "createIndexes", entry.Name }, { "indexes", new BsonArray(indexes) } }, cancellationToken: token);
+				report.Collections++;
+			}
+			var held = backup.Manifest.Collections.Select(c => c.Name).ToHashSet(StringComparer.Ordinal);
+			foreach (var name in await (await database.ListCollectionNamesAsync(cancellationToken: token)).ToListAsync(token))
+			{
+				if (held.Contains(name) || ServerBackup.Excluded.ContainsKey(name) || name.StartsWith("system.", StringComparison.Ordinal))
+					continue;
+				await database.DropCollectionAsync(name, token);
+				report.CollectionsDropped++;
+			}
+		}
+
+		/// <summary>Documents inserted in batches, unordered and unvalidated, as they were: how many.</summary>
+		public static async Task<long> Insert(IMongoCollection<BsonDocument> collection, IEnumerable<BsonDocument> documents, CancellationToken token)
+		{
+			var count = 0L;
+			foreach (var batch in documents.Chunk(1000))
+			{
+				await collection.InsertManyAsync(batch, new InsertManyOptions { IsOrdered = false, BypassDocumentValidation = true }, token);
+				count += batch.Length;
+			}
+			return count;
+		}
+
+		/// <summary>A collection's documents in a backup, one by one; none when it has no file.</summary>
+		public static IEnumerable<BsonDocument> Read(string file)
+		{
+			if (!File.Exists(file))
+				yield break;
+			using var gzip = new GZipStream(File.OpenRead(file), CompressionMode.Decompress);
+			using var reader = new StreamReader(gzip);
+			while (reader.ReadLine() is { } line)
+				if (line.Length > 0)
+					yield return BsonDocument.Parse(line);
+		}
+
+		// the backup's media files the live directory lacks: linked from the backup, or moved back from the trash
+		static void Media(BackupContext context, BackupInfo backup, RestoreReport report)
+		{
+			var kept = Path.Combine(context.BackupsRoot, backup.Id, "media");
+			foreach (var relative in backup.Manifest.Media.List)
+			{
+				var live = ServerBackup.Inside(context.MediaRoot, relative);
+				if (File.Exists(live))
+					continue;
+				var fromBackup = ServerBackup.Inside(kept, relative);
+				var fromTrash = ServerBackup.Inside(context.TrashRoot, relative);
+				if (File.Exists(fromBackup))
+					HardLink.LinkOrCopy(fromBackup, live);
+				else if (File.Exists(fromTrash))
+				{
+					Directory.CreateDirectory(Path.GetDirectoryName(live)!);
+					File.Move(fromTrash, live);
+				}
+				else
+				{
+					report.MediaMissing++;
+					continue;
+				}
+				report.MediaRestored++;
+			}
+		}
+	}
+}

PrivaPub/Infrastructure/Cli/AdminCommands.cs

@@ -3,6 +3,7 @@ using MongoDB.Entities;
 using PrivaPub.ClientModels;
 using PrivaPub.ClientModels.User;
 using PrivaPub.ClientModels.User.Avatar;
+using PrivaPub.Infrastructure.Backup;
 using PrivaPub.Models.User;
 using PrivaPub.Services;
 using PrivaPub.Services.ClientToServer.Private;
@@ -29,10 +30,15 @@ namespace PrivaPub.Infrastructure.Cli
 			                                                      linking them (the deploy's, which can't link www-data's files)
 			       PrivaPub admin backups                         the backups kept, newest first
 			       PrivaPub admin backup verify <id>              whether a backup's files are what its manifest says
+			       PrivaPub admin restore <id>                    restores a backup: the service stops within seconds and
+			                                                      restores it when systemd starts it again; everyone signs
+			                                                      in again, and what was made since is lost, except what
+			                                                      protects (blocks, deletions, ...)
+			       PrivaPub admin restore --status                the restore waiting, and the last one done
 			""";
 
 		/// <summary>Whether a command runs before migrations: backups are of the database as it was.</summary>
-		public static bool BeforeMigrations(string[] args) => args is ["backup", ..] or ["backups"];
+		public static bool BeforeMigrations(string[] args) => args is ["backup", ..] or ["backups"] or ["restore", ..];
 
 		public static async Task<int> Run(string[] args, IServiceProvider services, TextReader input = default, TextWriter output = default)
 		{
@@ -51,9 +57,13 @@ namespace PrivaPub.Infrastructure.Cli
 				case ["backup", "verify", var id]:
 					return await VerifyBackup(services, id, output);
 				case ["backup", .. var flags] when BackupFlags(flags, out var kind, out var dbOnly):
-					return await Backup(services, kind, dbOnly, output);
+					return await BackUp(services, kind, dbOnly, output);
 				case ["backups"]:
 					return ListBackups(services, output);
+				case ["restore", "--status"]:
+					return await RestoreStatus(services, output);
+				case ["restore", var id] when !id.StartsWith('-'):
+					return await Restore(services, id, output);
 				default:
 					Console.Error.WriteLine(Usage);
 					return 2;
@@ -81,9 +91,9 @@ namespace PrivaPub.Infrastructure.Cli
 			return true;
 		}
 
-		static async Task<int> Backup(IServiceProvider services, string kind, bool dbOnly, TextWriter output)
+		static async Task<int> BackUp(IServiceProvider services, string kind, bool dbOnly, TextWriter output)
 		{
-			var (made, error) = await services.GetRequiredService<Backup.Backups>().Create(kind, dbOnly, CancellationToken.None);
+			var (made, error) = await services.GetRequiredService<Backups>().Create(kind, dbOnly, CancellationToken.None);
 			if (made == default)
 			{
 				Console.Error.WriteLine(error);
@@ -101,14 +111,14 @@ namespace PrivaPub.Infrastructure.Cli
 
 		static int ListBackups(IServiceProvider services, TextWriter output)
 		{
-			foreach (var backup in services.GetRequiredService<Backup.Backups>().List())
+			foreach (var backup in services.GetRequiredService<Backups>().List())
 				output.WriteLine($"{backup.Id}\t{backup.Kind}\t{backup.CreatedAt:u}\t{backup.Bytes / 1024} KiB\t{backup.Manifest?.Media.Files ?? 0} media{(backup.Manifest?.DbOnly == true ? " (listed)" : string.Empty)}");
 			return 0;
 		}
 
 		static async Task<int> VerifyBackup(IServiceProvider services, string id, TextWriter output)
 		{
-			var problems = await services.GetRequiredService<Backup.Backups>().Verify(id, CancellationToken.None);
+			var problems = await services.GetRequiredService<Backups>().Verify(id, CancellationToken.None);
 			foreach (var problem in problems)
 				output.WriteLine(problem);
 			if (problems.Count > 0)
@@ -117,6 +127,48 @@ namespace PrivaPub.Infrastructure.Cli
 			return 0;
 		}
 
+		static async Task<int> Restore(IServiceProvider services, string id, TextWriter output)
+		{
+			var backups = services.GetRequiredService<Backups>();
+			var refused = await ServerRestore.Request(backups.Context(), id, "cli", CancellationToken.None);
+			if (refused != default)
+			{
+				Console.Error.WriteLine(refused);
+				return 1;
+			}
+			output.WriteLine($"{id} will be restored: the running service stops within seconds and restores it when it starts again");
+			output.WriteLine("(systemd starts it again; follow it with journalctl -u privapub, then PrivaPub admin restore --status)");
+			return 0;
+		}
+
+		static async Task<int> RestoreStatus(IServiceProvider services, TextWriter output)
+		{
+			var waiting = RestoreMarker.Read(services.GetRequiredService<Backups>().Root);
+			if (waiting != default)
+				output.WriteLine($"waiting: {waiting.Backup}, {waiting.State}, attempt {waiting.Attempts} of {RestoreMarker.MaxAttempts}, asked by {waiting.RequestedBy} at {waiting.RequestedAt:u}{(waiting.Error == default ? string.Empty : $": {waiting.Error}")}");
+			var last = await DB.Default.Find<RestoreRecord>().Sort(r => r.RestoredAt, Order.Descending).ExecuteFirstAsync();
+			if (last == default)
+			{
+				if (waiting == default)
+					output.WriteLine("no restore asked for, none done");
+				return 0;
+			}
+			if (last.Abandoned)
+			{
+				output.WriteLine($"last: {last.Backup} abandoned at {last.RestoredAt:u}, nothing changed: {last.Error}");
+				return 0;
+			}
+			var report = last.Report;
+			output.WriteLine($"last: {last.Backup} (made {last.BackupCreatedAt:u}) restored at {last.RestoredAt:u}, asked by {last.RequestedBy}; before it: {last.PreRestore}");
+			output.WriteLine($"  {report.Documents} documents in {report.Collections} collections, {report.MediaRestored} media files brought back, {report.MediaMissing} missing");
+			output.WriteLine($"  deleted again: {report.RootsDeleted} roots, {report.PersonasDeleted} personas, {report.GroupsDeleted} groups, {report.PostsDeleted} posts");
+			output.WriteLine($"  made since, now deleted: roots {Names(report.RootsTombstoned)}, personas {Names(report.PersonasTombstoned)}, groups {Names(report.GroupsTombstoned)}; {report.PostsGone} posts gone");
+			output.WriteLine($"  {report.ProtectiveKept} protective rows kept, {report.MediaTrashed} media trashed, {report.SessionsEnded} sessions ended");
+			return 0;
+		}
+
+		static string Names(List<string> names) => names.Count == 0 ? "none" : string.Join(", ", names);
+
 		static async Task<int> AuditMedia(IServiceProvider services, bool fix, TextWriter output)
 		{
 			var report = await Domain.Media.MediaAudit.Run(services.GetRequiredService<Domain.Media.IMediaService>(), fix, CancellationToken.None);

PrivaPub/Program.cs

@@ -71,6 +71,7 @@ try
 			.Configure<PrivaPub.Infrastructure.Backup.BackupOptions>(builder.Configuration.GetSection("Backups"))
 			.AddSingleton<PrivaPub.Infrastructure.Backup.Backups>()
 			.AddHostedService<PrivaPub.Infrastructure.Backup.BackupScheduler>()
+			.AddHostedService<PrivaPub.Infrastructure.Backup.RestoreWatcher>()
 			.PrivaPubMiddlewareConfiguration();
 	}
 	catch (Exception ex)
@@ -107,9 +108,17 @@ try
 		throw;
 	}
 
+	// A restore asked for runs here, before migrations, indexes and every hosted service; a command waits for it.
+	var gate = await PrivaPub.Infrastructure.Backup.MaintenanceGate.Enter(app.Services, args is ["admin", .. var asked] ? asked : default, CancellationToken.None);
+	if (gate is { } stop)
+	{
+		Environment.ExitCode = stop;
+		return;
+	}
+
 	// Commands get every service but start nothing: no Kestrel, no hosted services, no media directory. The deploy runs
 	// them as its own user, which can read the configuration and reach the private mongod but owns no www-data directory.
-	// Backups are taken before migrations: the deploy's is of the database as the live build left it.
+	// Backups and restores are asked for before migrations: the deploy's backup is of the database as the live build left it.
 	if (args is ["admin", .. var command] && AdminCommands.BeforeMigrations(command))
 	{
 		using var scope = app.Services.CreateScope();