using MongoDB.Bson; using MongoDB.Driver; using System.IO.Compression; namespace PrivaPub.Infrastructure.Backup { public enum RestoreOutcome { None,//nothing was asked Restored, Abandoned,//refused before anything changed: the server boots as it was Failed,//failed midway: the next boot tries again from the start GaveUp//failed MaxAttempts times: the server stays down until someone looks } // A backup restored (owner decision 2026-10-07: from the CLI or the administrator's page). Asking writes restore.json // and the running service stops at once (RestoreWatcher); the restore itself runs at the next boot, before migrations, // indexes and every hosted service, with nothing else touching the database: // 1. a pre-restore backup P is taken, once (a retry reuses it); // 2. every collection the backup holds is dropped and imported raw, with its indexes; every other one is dropped, // except what a backup never holds (ServerBackup.Excluded), which stays as it is; // 3. media files the live directory lacks come back from the backup (or the trash); a restore deletes no file; // 4. ProtectiveMerge brings back from P every protective act made since the backup; // 5. every session ends, every server's circuit closes, and a RestoreRecord tells what happened. // Each attempt redoes everything, so one that dies midway converges on the next. public static class ServerRestore { static readonly TimeSpan LockWait = TimeSpan.FromMinutes(10); /// Why a backup can't be restored here: none when it can. public static async Task> Check(BackupContext context, string id, CancellationToken token) { var backup = ServerBackup.Find(context.BackupsRoot, id); if (backup?.Manifest == default) return ["no such backup"]; var manifest = backup.Manifest; var problems = new List(); if (manifest.Format != ArchiveManifest.CurrentFormat) problems.Add($"its format is {manifest.Format}, this build reads {ArchiveManifest.CurrentFormat}"); if (!string.Equals(manifest.Host?.TrimEnd('/'), context.Host, StringComparison.OrdinalIgnoreCase)) problems.Add($"it was made by {manifest.Host}, not {context.Host}: every id and address in it names its host"); var code = ServerBackup.CodeMigration(); if (manifest.MigrationNumber > code) problems.Add($"it was made by a newer build (migration {manifest.MigrationNumber}; this one knows {code})"); problems.AddRange(await ServerBackup.Verify(context.BackupsRoot, id, token)); return problems; } /// Asks for a backup to be restored at the next boot: why not, or null once asked. public static async Task Request(BackupContext context, string id, string requestedBy, CancellationToken token) { var problems = await Check(context, id, token); if (problems.Count > 0) return string.Join("; ", problems); var waiting = RestoreMarker.Read(context.BackupsRoot); if (waiting != default && waiting.Attempts < RestoreMarker.MaxAttempts) return $"the restore of {waiting.Backup} is already {waiting.State}"; if (await MaintenanceLock.Current(token) is { } held) return $"a {held.What} is running"; new RestoreMarker { Backup = id, RequestedBy = requestedBy }.Write(context.BackupsRoot); return default; } /// The restore asked for, carried out (at boot, before migrations): what came of it. public static async Task ApplyPending(BackupContext context, ILogger logger, CancellationToken token) { var marker = RestoreMarker.Read(context.BackupsRoot); if (marker == default) return RestoreOutcome.None; if (marker.Attempts >= RestoreMarker.MaxAttempts) { logger.LogCritical("The restore of {Backup} failed {Attempts} times ({Error}). The database may be half restored; {PreRestore} holds it as it was before. Restore that backup or another (PrivaPub admin restore ), or delete {Marker} to boot as it is", marker.Backup, marker.Attempts, marker.Error, marker.PreRestore ?? "no backup", RestoreMarker.PathIn(context.BackupsRoot)); return RestoreOutcome.GaveUp; } await using var held = await TakeLock(token); if (held == default) return await Abandon(context, marker, "a backup kept the maintenance lock for ten minutes", logger, token); var problems = await Check(context, marker.Backup, token); if (problems.Count > 0 && marker.PreRestore == default) return await Abandon(context, marker, string.Join("; ", problems), logger, token); if (marker.PreRestore == default) { var (taken, error) = await ServerBackup.CreateHeld(context, "pre-restore", dbOnly: false, token); if (taken == default) return await Abandon(context, marker, $"the pre-restore backup could not be made: {error}", logger, token); marker.PreRestore = taken.Id; } marker.State = "running"; marker.Attempts++; marker.Error = default; marker.Write(context.BackupsRoot); logger.LogWarning("Restoring {Backup} (attempt {Attempt}); the server as it was is {PreRestore}", marker.Backup, marker.Attempts, marker.PreRestore); try { if (problems.Count > 0) throw new InvalidOperationException(string.Join("; ", problems)); var backup = ServerBackup.Find(context.BackupsRoot, marker.Backup); var report = new RestoreReport(); await Import(context, backup, report, token); Media(context, backup, report); await ProtectiveMerge.Apply(context.Database, Path.Combine(context.BackupsRoot, marker.PreRestore, "db"), report, token); await Record(context, new RestoreRecord { Backup = backup.Id, BackupCreatedAt = backup.CreatedAt, PreRestore = marker.PreRestore, RequestedBy = marker.RequestedBy, RequestedAt = marker.RequestedAt, Attempts = marker.Attempts, Report = report }, token); RestoreRecord.Forget(); RestoreMarker.Clear(context.BackupsRoot); logger.LogWarning("Restored {Backup}: {Documents} documents in {Collections} collections, {Media} media files brought back, {Tombstoned} accounts made since deleted", backup.Id, report.Documents, report.Collections, report.MediaRestored, report.RootsTombstoned.Count + report.PersonasTombstoned.Count + report.GroupsTombstoned.Count); return RestoreOutcome.Restored; } catch (Exception ex) when (ex is not OperationCanceledException) { marker.Error = ex.Message; marker.Write(context.BackupsRoot); logger.LogError(ex, "The restore of {Backup} failed (attempt {Attempt} of {Max})", marker.Backup, marker.Attempts, RestoreMarker.MaxAttempts); return marker.Attempts >= RestoreMarker.MaxAttempts ? RestoreOutcome.GaveUp : RestoreOutcome.Failed; } } static async Task TakeLock(CancellationToken token) { var until = DateTime.UtcNow + LockWait; while (true) { var held = await MaintenanceLock.Take("restore", token); if (held != default || DateTime.UtcNow > until) return held; await Task.Delay(TimeSpan.FromSeconds(5), token); } } // refused before anything changed: recorded for the administrator's page, and the server boots as it was static async Task Abandon(BackupContext context, RestoreMarker marker, string why, ILogger logger, CancellationToken token) { logger.LogError("The restore of {Backup} was abandoned, nothing changed: {Why}", marker.Backup, why); await Record(context, new RestoreRecord { Backup = marker.Backup, RequestedBy = marker.RequestedBy, RequestedAt = marker.RequestedAt, Attempts = marker.Attempts, Abandoned = true, Error = why }, token); RestoreMarker.Clear(context.BackupsRoot); return RestoreOutcome.Abandoned; } // in the database restored (a backup never holds these records: they outlive what they restore) static async Task Record(BackupContext context, RestoreRecord record, CancellationToken token) { record.ID = ObjectId.GenerateNewId().ToString(); await context.Database.GetCollection(nameof(RestoreRecord)).InsertOneAsync(record, cancellationToken: token); } // each collection dropped and imported as the backup holds it, its indexes made again; the others dropped static async Task Import(BackupContext context, BackupInfo backup, RestoreReport report, CancellationToken token) { var database = context.Database; var directory = Path.Combine(context.BackupsRoot, backup.Id, "db"); foreach (var entry in backup.Manifest.Collections) { await database.DropCollectionAsync(entry.Name, token); await database.CreateCollectionAsync(entry.Name, cancellationToken: token); report.Documents += await Insert(database.GetCollection(entry.Name), Read(Path.Combine(directory, entry.Name + ".jsonl.gz")), token); var indexes = entry.Indexes.Select(BsonDocument.Parse).Where(i => i.GetValue("name", "").AsString != "_id_").ToList(); foreach (var index in indexes) index.Remove("ns"); if (indexes.Count > 0) await database.RunCommandAsync(new BsonDocument { { "createIndexes", entry.Name }, { "indexes", new BsonArray(indexes) } }, cancellationToken: token); report.Collections++; } var held = backup.Manifest.Collections.Select(c => c.Name).ToHashSet(StringComparer.Ordinal); foreach (var name in await (await database.ListCollectionNamesAsync(cancellationToken: token)).ToListAsync(token)) { if (held.Contains(name) || ServerBackup.Excluded.ContainsKey(name) || name.StartsWith("system.", StringComparison.Ordinal)) continue; await database.DropCollectionAsync(name, token); report.CollectionsDropped++; } } /// Documents inserted in batches, unordered and unvalidated, as they were: how many. public static async Task Insert(IMongoCollection collection, IEnumerable documents, CancellationToken token) { var count = 0L; foreach (var batch in documents.Chunk(1000)) { await collection.InsertManyAsync(batch, new InsertManyOptions { IsOrdered = false, BypassDocumentValidation = true }, token); count += batch.Length; } return count; } /// A collection's documents in a backup, one by one; none when it has no file. public static IEnumerable Read(string file) { if (!File.Exists(file)) yield break; using var gzip = new GZipStream(File.OpenRead(file), CompressionMode.Decompress); using var reader = new StreamReader(gzip); while (reader.ReadLine() is { } line) if (line.Length > 0) yield return BsonDocument.Parse(line); } // the backup's media files the live directory lacks: linked from the backup, or moved back from the trash static void Media(BackupContext context, BackupInfo backup, RestoreReport report) { var kept = Path.Combine(context.BackupsRoot, backup.Id, "media"); foreach (var relative in backup.Manifest.Media.List) { var live = ServerBackup.Inside(context.MediaRoot, relative); if (File.Exists(live)) continue; var fromBackup = ServerBackup.Inside(kept, relative); var fromTrash = ServerBackup.Inside(context.TrashRoot, relative); if (File.Exists(fromBackup)) HardLink.LinkOrCopy(fromBackup, live); else if (File.Exists(fromTrash)) { Directory.CreateDirectory(Path.GetDirectoryName(live)!); File.Move(fromTrash, live); } else { report.MediaMissing++; continue; } report.MediaRestored++; } } } }