SocialPub/PrivaPub/Infrastructure/Backup/ServerBackup.cs

using MongoDB.Bson;
using MongoDB.Bson.IO;
using MongoDB.Driver;

using PrivaPub.Infrastructure.Data;

using System.Globalization;
using System.IO.Compression;
using System.Security.Cryptography;
using System.Text;

namespace PrivaPub.Infrastructure.Backup
{
	// What a backup needs to know: where things are, and whose host it is.
	public sealed record BackupContext(IMongoDatabase Database, string BackupsRoot, string MediaRoot, string TrashRoot, string Host, BackupOptions Options);

	public sealed record BackupInfo(string Id, string Kind, DateTime CreatedAt, long Bytes, ArchiveManifest Manifest);

	// The whole server, backed up as files (owner decision 2026-10-07: a whole-server backup, plain, protected by file
	// permissions). Each backup is a directory <stamp>-<kind> in the backups' root:
	// - manifest.json: what it holds (ArchiveManifest);
	// - db/<collection>.jsonl.gz: every document of each collection, one per line, in canonical Extended JSON (every BSON
	//   type kept as it was), all read at one instant when Mongo is a replica set (a snapshot session);
	// - media/<path>: the media files rows hold, as hard links to the live ones (no disk spent; a deleted file stays here
	//   until the backup is rotated out), copied where a link can't be made; a db-only backup lists them instead.
	// It is written as <id>.partial and renamed once whole. Left out: what belongs to the moment (Excluded). Everything is
	// readable by the service's user and group only: it holds every persona's private key and private posts.
	public static class ServerBackup
	{
		public const string PartialSuffix = ".partial";

		public static readonly IReadOnlyDictionary<string, string> Excluded = new Dictionary<string, string>
		{
			["InteractionSalt"] = "the day's statistics salt never outlives its day (owner rule)",
			["Job"] = "work in flight: deliveries and inbox processing belong to the moment",
			["Delivery"] = "work in flight, from before jobs",
			["EmailRecovery"] = "recovery codes live an hour",
			["openiddict.tokens"] = "sessions: a restore ends every one",
			["openiddict.authorizations"] = "sessions: a restore ends every one",
			[nameof(MaintenanceLock)] = "who is backing up or restoring now",
			["AppConfiguration"] = "the configuration's copy holds the SMTP password, and is made again from appsettings at boot",
			[nameof(RestoreRecord)] = "what restores did outlives what they restore"
		};

		static readonly JsonWriterSettings Json = new() { OutputMode = JsonOutputMode.CanonicalExtendedJson, Indent = false };

		// 2770: the service (www-data) and the deploy (in www-data's group) each read and rotate what the other wrote, and new
		// files take the directory's group
		const UnixFileMode DirectoryMode = UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute
			| UnixFileMode.GroupRead | UnixFileMode.GroupWrite | UnixFileMode.GroupExecute | UnixFileMode.SetGroup;
		const UnixFileMode FileMode0640 = UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.GroupRead;

		/// <summary>Takes the maintenance lock and backs the server up: the backup, or why not.</summary>
		public static async Task<(BackupInfo Backup, string Error)> Create(BackupContext context, string kind, bool dbOnly, CancellationToken token)
		{
			await using var held = await MaintenanceLock.Take("backup", token);
			if (held == default)
				return (default, "a backup or a restore is already running");
			return await CreateHeld(context, kind, dbOnly, token);
		}

		/// <summary>Backs the server up with the maintenance lock already held (a restore's own backup, taken first).</summary>
		public static async Task<(BackupInfo Backup, string Error)> CreateHeld(BackupContext context, string kind, bool dbOnly, CancellationToken token)
		{
			var database = context.Database;
			var names = (await (await database.ListCollectionNamesAsync(cancellationToken: token)).ToListAsync(token))
				.Where(n => !n.StartsWith("system.", StringComparison.Ordinal))
				.OrderBy(n => n, StringComparer.Ordinal)
				.ToList();
			MakeDirectory(context.BackupsRoot);
			var stats = await database.RunCommandAsync<BsonDocument>(new BsonDocument("dbStats", 1), cancellationToken: token);
			var needed = (long)(stats.GetValue("dataSize", 0).ToDouble() * 1.1);
			if (new DriveInfo(Path.GetFullPath(context.BackupsRoot)).AvailableFreeSpace < needed)
				return (default, $"not enough free disk for a backup: about {needed / 1024 / 1024} MB needed");

			var id = $"{DateTime.UtcNow.ToString("yyyyMMdd-HHmmss", CultureInfo.InvariantCulture)}-{kind}";
			var partial = Path.Combine(context.BackupsRoot, id + PartialSuffix);
			MakeDirectory(partial);
			MakeDirectory(Path.Combine(partial, "db"));
			try
			{
				var manifest = new ArchiveManifest
				{
					Kind = kind,
					Host = context.Host,
					AppCommit = BuildInfo.Commit,
					AppRef = BuildInfo.Ref,
					DbOnly = dbOnly
				};
				var replica = await MongoTopology.IsReplicaSet(database, token);
				var media = new SortedSet<string>(StringComparer.Ordinal);
				using var session = replica ? await database.Client.StartSessionAsync(new ClientSessionOptions { Snapshot = true }, token) : default;
				var window = replica ? await RaiseSnapshotWindow(database, token) : default(int?);
				try
				{
					foreach (var name in names)
					{
						if (Excluded.TryGetValue(name, out var why))
						{
							manifest.Excluded.Add(new ArchiveManifest.ExcludedEntry { Name = name, Why = why });
							continue;
						}
						manifest.Collections.Add(await Dump(database, session, name, partial, name == "MediaAttachment" ? media : default, token));
					}
					(manifest.NewestMigration, manifest.MigrationNumber) = await NewestMigration(database, session, token);
				}
				finally
				{
					if (window is { } previous)
						await SetSnapshotWindow(database, previous, CancellationToken.None);
				}
				manifest.Consistent = replica;

				manifest.Media.List = [.. media];
				if (!dbOnly)
					foreach (var relative in media)
					{
						var source = new[] { context.MediaRoot, context.TrashRoot }.Select(root => Inside(root, relative)).FirstOrDefault(File.Exists);
						if (source == default)
						{
							manifest.Media.Missing++;
							continue;
						}
						HardLink.LinkOrCopy(source, Inside(Path.Combine(partial, "media"), relative));
						manifest.Media.Files++;
						manifest.Media.Bytes += new FileInfo(source).Length;
					}

				manifest.Write(partial);
				Directory.Move(partial, Path.Combine(context.BackupsRoot, id));
				Retain(context.BackupsRoot, context.Options);
				return (Info(context.BackupsRoot, id), default);
			}
			catch
			{
				if (Directory.Exists(partial))
					Directory.Delete(partial, recursive: true);
				throw;
			}
		}

		// a collection read as raw BSON in batches, each document a line of canonical Extended JSON, gzipped; the media rows'
		// files collected on the way
		static async Task<ArchiveManifest.CollectionEntry> Dump(IMongoDatabase database, IClientSessionHandle session, string name, string directory,
			ISet<string> media, CancellationToken token)
		{
			var collection = database.GetCollection<RawBsonDocument>(name);
			var path = Path.Combine(directory, "db", name + ".jsonl.gz");
			var count = 0L;
			await using (var file = NewFile(path))
			await using (var gzip = new GZipStream(file, CompressionLevel.Fastest))
			await using (var writer = new StreamWriter(gzip, new UTF8Encoding(false)))
			{
				var options = new FindOptions<RawBsonDocument> { BatchSize = 1000 };
				using var cursor = session == default
					? await collection.FindAsync(FilterDefinition<RawBsonDocument>.Empty, options, token)
					: await collection.FindAsync(session, FilterDefinition<RawBsonDocument>.Empty, options, token);
				while (await cursor.MoveNextAsync(token))
					foreach (var document in cursor.Current)
						using (document)
						{
							await writer.WriteLineAsync(document.ToJson(Json));
							count++;
							if (media != default)
								Collect(document, media);
						}
			}
			var indexes = await (await collection.Indexes.ListAsync(token)).ToListAsync(token);
			return new ArchiveManifest.CollectionEntry
			{
				Name = name,
				Count = count,
				Bytes = new FileInfo(path).Length,
				Sha256 = await Hash(path, token),
				Indexes = [.. indexes.Select(i => i.ToJson(Json))]
			};
		}

		// a media row's files, unless it is trashed
		static void Collect(RawBsonDocument row, ISet<string> media)
		{
			if (row.TryGetValue("TrashedAt", out var trashed) && !trashed.IsBsonNull)
				return;
			foreach (var field in new[] { "FilePath", "PreviewPath" })
				if (row.TryGetValue(field, out var value) && value.IsString && Safe(value.AsString))
					media.Add(value.AsString);
		}

		// MongoDB.Entities records each migration run with its class's number (_016_... is 16) and its name in words
		static async Task<(string Name, int Number)> NewestMigration(IMongoDatabase database, IClientSessionHandle session, CancellationToken token)
		{
			var history = database.GetCollection<BsonDocument>("_migration_history_");
			var options = new FindOptions<BsonDocument> { Sort = new BsonDocument("Number", -1), Limit = 1 };
			var newest = session == default
				? await (await history.FindAsync(FilterDefinition<BsonDocument>.Empty, options, token)).FirstOrDefaultAsync(token)
				: await (await history.FindAsync(session, FilterDefinition<BsonDocument>.Empty, options, token)).FirstOrDefaultAsync(token);
			return newest == default ? default : (newest.GetValue("Name", BsonNull.Value).ToString(), newest.GetValue("Number", 0).ToInt32());
		}

		/// <summary>The newest migration this build has: a backup made by a newer one can't be restored by it.</summary>
		public static int CodeMigration() => typeof(ServerBackup).Assembly.GetTypes()
			.Where(t => typeof(MongoDB.Entities.IMigration).IsAssignableFrom(t) && !t.IsAbstract)
			.Select(t => int.TryParse(new string(t.Name.TrimStart('_').TakeWhile(char.IsDigit).ToArray()), out var number) ? number : 0)
			.DefaultIfEmpty(0)
			.Max();

		// how long a snapshot stays readable: raised only while a backup reads (a longer window keeps old versions in the
		// small cache all day); the value it had, to set back
		static async Task<int?> RaiseSnapshotWindow(IMongoDatabase database, CancellationToken token)
		{
			var admin = database.Client.GetDatabase("admin");
			try
			{
				var current = await admin.RunCommandAsync<BsonDocument>(new BsonDocument { { "getParameter", 1 }, { "minSnapshotHistoryWindowInSeconds", 1 } }, cancellationToken: token);
				var previous = current.GetValue("minSnapshotHistoryWindowInSeconds", 300).ToInt32();
				await SetSnapshotWindow(database, Math.Max(previous, 3600), token);
				return previous;
			}
			catch (MongoCommandException)
			{
				return default;
			}
		}

		static async Task SetSnapshotWindow(IMongoDatabase database, int seconds, CancellationToken token) =>
			await database.Client.GetDatabase("admin").RunCommandAsync<BsonDocument>(
				new BsonDocument { { "setParameter", 1 }, { "minSnapshotHistoryWindowInSeconds", seconds } }, cancellationToken: token);

		/// <summary>The backups kept, newest first (not one still being written).</summary>
		public static List<BackupInfo> List(string backupsRoot)
		{
			if (!Directory.Exists(backupsRoot))
				return [];
			return Directory.EnumerateDirectories(backupsRoot)
				.Select(Path.GetFileName)
				.Where(name => !name.EndsWith(PartialSuffix, StringComparison.Ordinal) && File.Exists(Path.Combine(backupsRoot, name, ArchiveManifest.FileName)))
				.Select(name => Info(backupsRoot, name))
				.OrderByDescending(b => b.CreatedAt)
				.ToList();
		}

		public static BackupInfo Find(string backupsRoot, string id) =>
			Safe(id) && !id.Contains('/') && Directory.Exists(Path.Combine(backupsRoot, id)) && !id.EndsWith(PartialSuffix, StringComparison.Ordinal)
				? Info(backupsRoot, id)
				: default;

		static BackupInfo Info(string backupsRoot, string id)
		{
			var directory = Path.Combine(backupsRoot, id);
			var manifest = ArchiveManifest.Read(directory);
			var bytes = Directory.EnumerateFiles(Path.Combine(directory, "db")).Sum(f => new FileInfo(f).Length);
			return new BackupInfo(id, manifest?.Kind, manifest?.CreatedAt ?? Directory.GetCreationTimeUtc(directory), bytes, manifest);
		}

		/// <summary>Whether every file of a backup is what its manifest says: the problems found (none when whole).</summary>
		public static async Task<List<string>> Verify(string backupsRoot, string id, CancellationToken token)
		{
			var problems = new List<string>();
			var backup = Find(backupsRoot, id);
			if (backup?.Manifest == default)
				return ["no such backup, or no manifest"];
			var directory = Path.Combine(backupsRoot, id);
			foreach (var collection in backup.Manifest.Collections)
			{
				var path = Path.Combine(directory, "db", collection.Name + ".jsonl.gz");
				if (!File.Exists(path))
					problems.Add($"{collection.Name}: missing");
				else if (await Hash(path, token) != collection.Sha256)
					problems.Add($"{collection.Name}: altered");
			}
			if (!backup.Manifest.DbOnly)
				foreach (var relative in backup.Manifest.Media.List.Where(r => !File.Exists(Inside(Path.Combine(directory, "media"), r))))
					problems.Add($"media {relative}: missing");
			return problems;
		}

		/// <summary>Deletes a backup (its media links go; the live files stay).</summary>
		public static bool Delete(string backupsRoot, string id)
		{
			if (Find(backupsRoot, id) == default)
				return false;
			Directory.Delete(Path.Combine(backupsRoot, id), recursive: true);
			return true;
		}

		// what is kept: the newest nightly ones for KeepDaily days and the newest of each of KeepWeekly weeks before, the
		// newest pre-deploy and pre-restore ones; manual and uploaded ones until deleted; a backup that died writing goes
		public static void Retain(string backupsRoot, BackupOptions options)
		{
			var all = List(backupsRoot);
			var nightly = all.Where(b => b.Kind == "nightly").OrderByDescending(b => b.CreatedAt).ToList();
			var kept = nightly.Take(options.KeepDaily).ToHashSet();
			foreach (var week in nightly.Skip(options.KeepDaily).GroupBy(b => (ISOWeek.GetYear(b.CreatedAt), ISOWeek.GetWeekOfYear(b.CreatedAt))).Take(options.KeepWeekly))
				kept.Add(week.First());
			var doomed = nightly.Where(b => !kept.Contains(b))
				.Concat(all.Where(b => b.Kind == "pre-deploy").OrderByDescending(b => b.CreatedAt).Skip(options.KeepPreDeploy))
				.Concat(all.Where(b => b.Kind == "pre-restore").OrderByDescending(b => b.CreatedAt).Skip(options.KeepPreRestore));
			var stale = Directory.EnumerateDirectories(backupsRoot, "*" + PartialSuffix).Where(p => Directory.GetLastWriteTimeUtc(p) < DateTime.UtcNow.AddDays(-1));
			foreach (var directory in doomed.Select(b => Path.Combine(backupsRoot, b.Id)).Concat(stale).ToList())
				try
				{
					Directory.Delete(directory, recursive: true);
				}
				catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
				{
					//another user's backup this one may not delete: the next rotation tries again
				}
		}

		// a relative path from a database row or a manifest, never outside its root
		static bool Safe(string relative) =>
			!string.IsNullOrEmpty(relative) && !Path.IsPathRooted(relative) && !relative.Split('/', '\\').Any(part => part is ".." or ".");

		public static string Inside(string root, string relative)
		{
			var full = Path.GetFullPath(Path.Combine(root, relative));
			if (!Safe(relative) || !full.StartsWith(Path.GetFullPath(root) + Path.DirectorySeparatorChar, StringComparison.Ordinal))
				throw new InvalidOperationException($"{relative} is not inside {root}");
			return full;
		}

		static async Task<string> Hash(string path, CancellationToken token)
		{
			await using var file = File.OpenRead(path);
			return Convert.ToHexStringLower(await SHA256.HashDataAsync(file, token));
		}

		static FileStream NewFile(string path) => OperatingSystem.IsWindows()
			? new FileStream(path, FileMode.CreateNew, FileAccess.Write)
			: new FileStream(path, new FileStreamOptions { Mode = FileMode.CreateNew, Access = FileAccess.Write, UnixCreateMode = FileMode0640 });

		// set after creating, since the umask would take the group's write away
		public static void MakeDirectory(string path)
		{
			var existed = Directory.Exists(path);
			Directory.CreateDirectory(path);
			if (!existed && !OperatingSystem.IsWindows())
				File.SetUnixFileMode(path, DirectoryMode);
		}
	}
}