using MongoDB.Bson; using MongoDB.Bson.IO; using MongoDB.Driver; using PrivaPub.Infrastructure.Data; using System.Globalization; using System.IO.Compression; using System.Security.Cryptography; using System.Text; namespace PrivaPub.Infrastructure.Backup { // What a backup needs to know: where things are, and whose host it is. public sealed record BackupContext(IMongoDatabase Database, string BackupsRoot, string MediaRoot, string TrashRoot, string Host, BackupOptions Options); public sealed record BackupInfo(string Id, string Kind, DateTime CreatedAt, long Bytes, ArchiveManifest Manifest); // The whole server, backed up as files (owner decision 2026-10-07: a whole-server backup, plain, protected by file // permissions). Each backup is a directory - in the backups' root: // - manifest.json: what it holds (ArchiveManifest); // - db/.jsonl.gz: every document of each collection, one per line, in canonical Extended JSON (every BSON // type kept as it was), all read at one instant when Mongo is a replica set (a snapshot session); // - media/: the media files rows hold, as hard links to the live ones (no disk spent; a deleted file stays here // until the backup is rotated out), copied where a link can't be made; a db-only backup lists them instead. // It is written as .partial and renamed once whole. Left out: what belongs to the moment (Excluded). Everything is // readable by the service's user and group only: it holds every persona's private key and private posts. public static class ServerBackup { public const string PartialSuffix = ".partial"; public static readonly IReadOnlyDictionary Excluded = new Dictionary { ["InteractionSalt"] = "the day's statistics salt never outlives its day (owner rule)", ["Job"] = "work in flight: deliveries and inbox processing belong to the moment", ["Delivery"] = "work in flight, from before jobs", ["EmailRecovery"] = "recovery codes live an hour", ["openiddict.tokens"] = "sessions: a restore ends every one", ["openiddict.authorizations"] = "sessions: a restore ends every one", [nameof(MaintenanceLock)] = "who is backing up or restoring now", ["AppConfiguration"] = "the configuration's copy holds the SMTP password, and is made again from appsettings at boot" }; static readonly JsonWriterSettings Json = new() { OutputMode = JsonOutputMode.CanonicalExtendedJson, Indent = false }; // 2770: the service (www-data) and the deploy (in www-data's group) each read and rotate what the other wrote, and new // files take the directory's group const UnixFileMode DirectoryMode = UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute | UnixFileMode.GroupRead | UnixFileMode.GroupWrite | UnixFileMode.GroupExecute | UnixFileMode.SetGroup; const UnixFileMode FileMode0640 = UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.GroupRead; /// Takes the maintenance lock and backs the server up: the backup, or why not. public static async Task<(BackupInfo Backup, string Error)> Create(BackupContext context, string kind, bool dbOnly, CancellationToken token) { await using var held = await MaintenanceLock.Take("backup", token); if (held == default) return (default, "a backup or a restore is already running"); return await CreateHeld(context, kind, dbOnly, token); } /// Backs the server up with the maintenance lock already held (a restore's own backup, taken first). public static async Task<(BackupInfo Backup, string Error)> CreateHeld(BackupContext context, string kind, bool dbOnly, CancellationToken token) { var database = context.Database; var names = (await (await database.ListCollectionNamesAsync(cancellationToken: token)).ToListAsync(token)) .Where(n => !n.StartsWith("system.", StringComparison.Ordinal)) .OrderBy(n => n, StringComparer.Ordinal) .ToList(); MakeDirectory(context.BackupsRoot); var stats = await database.RunCommandAsync(new BsonDocument("dbStats", 1), cancellationToken: token); var needed = (long)(stats.GetValue("dataSize", 0).ToDouble() * 1.1); if (new DriveInfo(Path.GetFullPath(context.BackupsRoot)).AvailableFreeSpace < needed) return (default, $"not enough free disk for a backup: about {needed / 1024 / 1024} MB needed"); var id = $"{DateTime.UtcNow.ToString("yyyyMMdd-HHmmss", CultureInfo.InvariantCulture)}-{kind}"; var partial = Path.Combine(context.BackupsRoot, id + PartialSuffix); MakeDirectory(partial); MakeDirectory(Path.Combine(partial, "db")); try { var manifest = new ArchiveManifest { Kind = kind, Host = context.Host, AppCommit = BuildInfo.Commit, AppRef = BuildInfo.Ref, DbOnly = dbOnly }; var replica = await MongoTopology.IsReplicaSet(database, token); var media = new SortedSet(StringComparer.Ordinal); using var session = replica ? await database.Client.StartSessionAsync(new ClientSessionOptions { Snapshot = true }, token) : default; var window = replica ? await RaiseSnapshotWindow(database, token) : default(int?); try { foreach (var name in names) { if (Excluded.TryGetValue(name, out var why)) { manifest.Excluded.Add(new ArchiveManifest.ExcludedEntry { Name = name, Why = why }); continue; } manifest.Collections.Add(await Dump(database, session, name, partial, name == "MediaAttachment" ? media : default, token)); } (manifest.NewestMigration, manifest.MigrationNumber) = await NewestMigration(database, session, token); } finally { if (window is { } previous) await SetSnapshotWindow(database, previous, CancellationToken.None); } manifest.Consistent = replica; manifest.Media.List = [.. media]; if (!dbOnly) foreach (var relative in media) { var source = new[] { context.MediaRoot, context.TrashRoot }.Select(root => Inside(root, relative)).FirstOrDefault(File.Exists); if (source == default) { manifest.Media.Missing++; continue; } HardLink.LinkOrCopy(source, Inside(Path.Combine(partial, "media"), relative)); manifest.Media.Files++; manifest.Media.Bytes += new FileInfo(source).Length; } manifest.Write(partial); Directory.Move(partial, Path.Combine(context.BackupsRoot, id)); Retain(context.BackupsRoot, context.Options); return (Info(context.BackupsRoot, id), default); } catch { if (Directory.Exists(partial)) Directory.Delete(partial, recursive: true); throw; } } // a collection read as raw BSON in batches, each document a line of canonical Extended JSON, gzipped; the media rows' // files collected on the way static async Task Dump(IMongoDatabase database, IClientSessionHandle session, string name, string directory, ISet media, CancellationToken token) { var collection = database.GetCollection(name); var path = Path.Combine(directory, "db", name + ".jsonl.gz"); var count = 0L; await using (var file = NewFile(path)) await using (var gzip = new GZipStream(file, CompressionLevel.Fastest)) await using (var writer = new StreamWriter(gzip, new UTF8Encoding(false))) { var options = new FindOptions { BatchSize = 1000 }; using var cursor = session == default ? await collection.FindAsync(FilterDefinition.Empty, options, token) : await collection.FindAsync(session, FilterDefinition.Empty, options, token); while (await cursor.MoveNextAsync(token)) foreach (var document in cursor.Current) using (document) { await writer.WriteLineAsync(document.ToJson(Json)); count++; if (media != default) Collect(document, media); } } var indexes = await (await collection.Indexes.ListAsync(token)).ToListAsync(token); return new ArchiveManifest.CollectionEntry { Name = name, Count = count, Bytes = new FileInfo(path).Length, Sha256 = await Hash(path, token), Indexes = [.. indexes.Select(i => i.ToJson(Json))] }; } // a media row's files, unless it is trashed static void Collect(RawBsonDocument row, ISet media) { if (row.TryGetValue("TrashedAt", out var trashed) && !trashed.IsBsonNull) return; foreach (var field in new[] { "FilePath", "PreviewPath" }) if (row.TryGetValue(field, out var value) && value.IsString && Safe(value.AsString)) media.Add(value.AsString); } // MongoDB.Entities records each migration run with its class's number (_016_... is 16) and its name in words static async Task<(string Name, int Number)> NewestMigration(IMongoDatabase database, IClientSessionHandle session, CancellationToken token) { var history = database.GetCollection("_migration_history_"); var options = new FindOptions { Sort = new BsonDocument("Number", -1), Limit = 1 }; var newest = session == default ? await (await history.FindAsync(FilterDefinition.Empty, options, token)).FirstOrDefaultAsync(token) : await (await history.FindAsync(session, FilterDefinition.Empty, options, token)).FirstOrDefaultAsync(token); return newest == default ? default : (newest.GetValue("Name", BsonNull.Value).ToString(), newest.GetValue("Number", 0).ToInt32()); } /// The newest migration this build has: a backup made by a newer one can't be restored by it. public static int CodeMigration() => typeof(ServerBackup).Assembly.GetTypes() .Where(t => typeof(MongoDB.Entities.IMigration).IsAssignableFrom(t) && !t.IsAbstract) .Select(t => int.TryParse(new string(t.Name.TrimStart('_').TakeWhile(char.IsDigit).ToArray()), out var number) ? number : 0) .DefaultIfEmpty(0) .Max(); // how long a snapshot stays readable: raised only while a backup reads (a longer window keeps old versions in the // small cache all day); the value it had, to set back static async Task RaiseSnapshotWindow(IMongoDatabase database, CancellationToken token) { var admin = database.Client.GetDatabase("admin"); try { var current = await admin.RunCommandAsync(new BsonDocument { { "getParameter", 1 }, { "minSnapshotHistoryWindowInSeconds", 1 } }, cancellationToken: token); var previous = current.GetValue("minSnapshotHistoryWindowInSeconds", 300).ToInt32(); await SetSnapshotWindow(database, Math.Max(previous, 3600), token); return previous; } catch (MongoCommandException) { return default; } } static async Task SetSnapshotWindow(IMongoDatabase database, int seconds, CancellationToken token) => await database.Client.GetDatabase("admin").RunCommandAsync( new BsonDocument { { "setParameter", 1 }, { "minSnapshotHistoryWindowInSeconds", seconds } }, cancellationToken: token); /// The backups kept, newest first (not one still being written). public static List List(string backupsRoot) { if (!Directory.Exists(backupsRoot)) return []; return Directory.EnumerateDirectories(backupsRoot) .Select(Path.GetFileName) .Where(name => !name.EndsWith(PartialSuffix, StringComparison.Ordinal) && File.Exists(Path.Combine(backupsRoot, name, ArchiveManifest.FileName))) .Select(name => Info(backupsRoot, name)) .OrderByDescending(b => b.CreatedAt) .ToList(); } public static BackupInfo Find(string backupsRoot, string id) => Safe(id) && !id.Contains('/') && Directory.Exists(Path.Combine(backupsRoot, id)) && !id.EndsWith(PartialSuffix, StringComparison.Ordinal) ? Info(backupsRoot, id) : default; static BackupInfo Info(string backupsRoot, string id) { var directory = Path.Combine(backupsRoot, id); var manifest = ArchiveManifest.Read(directory); var bytes = Directory.EnumerateFiles(Path.Combine(directory, "db")).Sum(f => new FileInfo(f).Length); return new BackupInfo(id, manifest?.Kind, manifest?.CreatedAt ?? Directory.GetCreationTimeUtc(directory), bytes, manifest); } /// Whether every file of a backup is what its manifest says: the problems found (none when whole). public static async Task> Verify(string backupsRoot, string id, CancellationToken token) { var problems = new List(); var backup = Find(backupsRoot, id); if (backup?.Manifest == default) return ["no such backup, or no manifest"]; var directory = Path.Combine(backupsRoot, id); foreach (var collection in backup.Manifest.Collections) { var path = Path.Combine(directory, "db", collection.Name + ".jsonl.gz"); if (!File.Exists(path)) problems.Add($"{collection.Name}: missing"); else if (await Hash(path, token) != collection.Sha256) problems.Add($"{collection.Name}: altered"); } if (!backup.Manifest.DbOnly) foreach (var relative in backup.Manifest.Media.List.Where(r => !File.Exists(Inside(Path.Combine(directory, "media"), r)))) problems.Add($"media {relative}: missing"); return problems; } /// Deletes a backup (its media links go; the live files stay). public static bool Delete(string backupsRoot, string id) { if (Find(backupsRoot, id) == default) return false; Directory.Delete(Path.Combine(backupsRoot, id), recursive: true); return true; } // what is kept: the newest nightly ones for KeepDaily days and the newest of each of KeepWeekly weeks before, the // newest pre-deploy and pre-restore ones; manual and uploaded ones until deleted; a backup that died writing goes public static void Retain(string backupsRoot, BackupOptions options) { var all = List(backupsRoot); var nightly = all.Where(b => b.Kind == "nightly").OrderByDescending(b => b.CreatedAt).ToList(); var kept = nightly.Take(options.KeepDaily).ToHashSet(); foreach (var week in nightly.Skip(options.KeepDaily).GroupBy(b => (ISOWeek.GetYear(b.CreatedAt), ISOWeek.GetWeekOfYear(b.CreatedAt))).Take(options.KeepWeekly)) kept.Add(week.First()); var doomed = nightly.Where(b => !kept.Contains(b)) .Concat(all.Where(b => b.Kind == "pre-deploy").OrderByDescending(b => b.CreatedAt).Skip(options.KeepPreDeploy)) .Concat(all.Where(b => b.Kind == "pre-restore").OrderByDescending(b => b.CreatedAt).Skip(options.KeepPreRestore)); var stale = Directory.EnumerateDirectories(backupsRoot, "*" + PartialSuffix).Where(p => Directory.GetLastWriteTimeUtc(p) < DateTime.UtcNow.AddDays(-1)); foreach (var directory in doomed.Select(b => Path.Combine(backupsRoot, b.Id)).Concat(stale).ToList()) try { Directory.Delete(directory, recursive: true); } catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) { //another user's backup this one may not delete: the next rotation tries again } } // a relative path from a database row or a manifest, never outside its root static bool Safe(string relative) => !string.IsNullOrEmpty(relative) && !Path.IsPathRooted(relative) && !relative.Split('/', '\\').Any(part => part is ".." or "."); public static string Inside(string root, string relative) { var full = Path.GetFullPath(Path.Combine(root, relative)); if (!Safe(relative) || !full.StartsWith(Path.GetFullPath(root) + Path.DirectorySeparatorChar, StringComparison.Ordinal)) throw new InvalidOperationException($"{relative} is not inside {root}"); return full; } static async Task Hash(string path, CancellationToken token) { await using var file = File.OpenRead(path); return Convert.ToHexStringLower(await SHA256.HashDataAsync(file, token)); } static FileStream NewFile(string path) => OperatingSystem.IsWindows() ? new FileStream(path, FileMode.CreateNew, FileAccess.Write) : new FileStream(path, new FileStreamOptions { Mode = FileMode.CreateNew, Access = FileAccess.Write, UnixCreateMode = FileMode0640 }); // set after creating, since the umask would take the group's write away public static void MakeDirectory(string path) { var existed = Directory.Exists(path); Directory.CreateDirectory(path); if (!existed && !OperatingSystem.IsWindows()) File.SetUnixFileMode(path, DirectoryMode); } } }