using MongoDB.Bson;
using MongoDB.Driver;
using System.IO.Compression;
namespace PrivaPub.Infrastructure.Backup
{
public enum RestoreOutcome
{
None,//nothing was asked
Restored,
Abandoned,//refused before anything changed: the server boots as it was
Failed,//failed midway: the next boot tries again from the start
GaveUp//failed MaxAttempts times: the server stays down until someone looks
}
// A backup restored (owner decision 2026-10-07: from the CLI or the administrator's page). Asking writes restore.json
// and the running service stops at once (RestoreWatcher); the restore itself runs at the next boot, before migrations,
// indexes and every hosted service, with nothing else touching the database:
// 1. a pre-restore backup P is taken, once (a retry reuses it);
// 2. every collection the backup holds is dropped and imported raw, with its indexes; every other one is dropped,
// except what a backup never holds (ServerBackup.Excluded), which stays as it is;
// 3. media files the live directory lacks come back from the backup (or the trash); a restore deletes no file;
// 4. ProtectiveMerge brings back from P every protective act made since the backup;
// 5. every session ends, every server's circuit closes, and a RestoreRecord tells what happened.
// Each attempt redoes everything, so one that dies midway converges on the next.
public static class ServerRestore
{
static readonly TimeSpan LockWait = TimeSpan.FromMinutes(10);
/// Why a backup can't be restored here: none when it can.
public static async Task> Check(BackupContext context, string id, CancellationToken token)
{
var backup = ServerBackup.Find(context.BackupsRoot, id);
if (backup?.Manifest == default)
return ["no such backup"];
var manifest = backup.Manifest;
var problems = new List();
if (manifest.Format != ArchiveManifest.CurrentFormat)
problems.Add($"its format is {manifest.Format}, this build reads {ArchiveManifest.CurrentFormat}");
if (!string.Equals(manifest.Host?.TrimEnd('/'), context.Host, StringComparison.OrdinalIgnoreCase))
problems.Add($"it was made by {manifest.Host}, not {context.Host}: every id and address in it names its host");
var code = ServerBackup.CodeMigration();
if (manifest.MigrationNumber > code)
problems.Add($"it was made by a newer build (migration {manifest.MigrationNumber}; this one knows {code})");
problems.AddRange(await ServerBackup.Verify(context.BackupsRoot, id, token));
return problems;
}
/// Asks for a backup to be restored at the next boot: why not, or null once asked.
public static async Task Request(BackupContext context, string id, string requestedBy, CancellationToken token)
{
var problems = await Check(context, id, token);
if (problems.Count > 0)
return string.Join("; ", problems.Take(5)) + (problems.Count > 5 ? $"; and {problems.Count - 5} more" : string.Empty);
var waiting = RestoreMarker.Read(context.BackupsRoot);
if (waiting != default && waiting.Attempts < RestoreMarker.MaxAttempts)
return $"the restore of {waiting.Backup} is already {waiting.State}";
if (await MaintenanceLock.Current(token) is { } held)
return $"a {held.What} is running";
new RestoreMarker { Backup = id, RequestedBy = requestedBy }.Write(context.BackupsRoot);
return default;
}
/// The restore asked for, carried out (at boot, before migrations): what came of it.
public static async Task ApplyPending(BackupContext context, ILogger logger, CancellationToken token)
{
var marker = RestoreMarker.Read(context.BackupsRoot);
if (marker == default)
return RestoreOutcome.None;
if (marker.Attempts >= RestoreMarker.MaxAttempts)
{
logger.LogCritical("The restore of {Backup} failed {Attempts} times ({Error}). The database may be half restored; {PreRestore} holds it as it was before. Restore that backup or another (PrivaPub admin restore ), or delete {Marker} to boot as it is",
marker.Backup, marker.Attempts, marker.Error, marker.PreRestore ?? "no backup", RestoreMarker.PathIn(context.BackupsRoot));
return RestoreOutcome.GaveUp;
}
await using var held = await TakeLock(token);
if (held == default)
return await Abandon(context, marker, "a backup kept the maintenance lock for ten minutes", logger, token);
var problems = await Check(context, marker.Backup, token);
if (problems.Count > 0 && marker.PreRestore == default)
return await Abandon(context, marker, string.Join("; ", problems), logger, token);
if (marker.PreRestore == default)
{
var (taken, error) = await ServerBackup.CreateHeld(context, "pre-restore", dbOnly: false, token);
if (taken == default)
return await Abandon(context, marker, $"the pre-restore backup could not be made: {error}", logger, token);
marker.PreRestore = taken.Id;
}
marker.State = "running";
marker.Attempts++;
marker.Error = default;
marker.Write(context.BackupsRoot);
logger.LogWarning("Restoring {Backup} (attempt {Attempt}); the server as it was is {PreRestore}", marker.Backup, marker.Attempts, marker.PreRestore);
try
{
if (problems.Count > 0)
throw new InvalidOperationException(string.Join("; ", problems));
var backup = ServerBackup.Find(context.BackupsRoot, marker.Backup);
var report = new RestoreReport();
await Import(context, backup, report, token);
Media(context, backup, report);
await ProtectiveMerge.Apply(context.Database, Path.Combine(context.BackupsRoot, marker.PreRestore, "db"), report, token);
await Record(context, new RestoreRecord
{
Backup = backup.Id,
BackupCreatedAt = backup.CreatedAt,
PreRestore = marker.PreRestore,
RequestedBy = marker.RequestedBy,
RequestedAt = marker.RequestedAt,
Attempts = marker.Attempts,
Report = report
}, token);
RestoreRecord.Forget();
RestoreMarker.Clear(context.BackupsRoot);
logger.LogWarning("Restored {Backup}: {Documents} documents in {Collections} collections, {Media} media files brought back, {Tombstoned} accounts made since deleted",
backup.Id, report.Documents, report.Collections, report.MediaRestored, report.RootsTombstoned.Count + report.PersonasTombstoned.Count + report.GroupsTombstoned.Count);
return RestoreOutcome.Restored;
}
catch (Exception ex) when (ex is not OperationCanceledException)
{
marker.Error = ex.Message;
marker.Write(context.BackupsRoot);
logger.LogError(ex, "The restore of {Backup} failed (attempt {Attempt} of {Max})", marker.Backup, marker.Attempts, RestoreMarker.MaxAttempts);
return marker.Attempts >= RestoreMarker.MaxAttempts ? RestoreOutcome.GaveUp : RestoreOutcome.Failed;
}
}
static async Task TakeLock(CancellationToken token)
{
var until = DateTime.UtcNow + LockWait;
while (true)
{
var held = await MaintenanceLock.Take("restore", token);
if (held != default || DateTime.UtcNow > until)
return held;
await Task.Delay(TimeSpan.FromSeconds(5), token);
}
}
// refused before anything changed: recorded for the administrator's page, and the server boots as it was
static async Task Abandon(BackupContext context, RestoreMarker marker, string why, ILogger logger, CancellationToken token)
{
logger.LogError("The restore of {Backup} was abandoned, nothing changed: {Why}", marker.Backup, why);
await Record(context, new RestoreRecord
{
Backup = marker.Backup,
RequestedBy = marker.RequestedBy,
RequestedAt = marker.RequestedAt,
Attempts = marker.Attempts,
Abandoned = true,
Error = why
}, token);
RestoreMarker.Clear(context.BackupsRoot);
return RestoreOutcome.Abandoned;
}
// in the database restored (a backup never holds these records: they outlive what they restore)
static async Task Record(BackupContext context, RestoreRecord record, CancellationToken token)
{
record.ID = ObjectId.GenerateNewId().ToString();
await context.Database.GetCollection(nameof(RestoreRecord)).InsertOneAsync(record, cancellationToken: token);
}
// each collection dropped and imported as the backup holds it, its indexes made again; the others dropped
static async Task Import(BackupContext context, BackupInfo backup, RestoreReport report, CancellationToken token)
{
var database = context.Database;
var directory = Path.Combine(context.BackupsRoot, backup.Id, "db");
foreach (var entry in backup.Manifest.Collections)
{
await database.DropCollectionAsync(entry.Name, token);
await database.CreateCollectionAsync(entry.Name, cancellationToken: token);
report.Documents += await Insert(database.GetCollection(entry.Name), Read(Path.Combine(directory, entry.Name + ".jsonl.gz")), token);
var indexes = entry.Indexes.Select(BsonDocument.Parse).Where(i => i.GetValue("name", "").AsString != "_id_").ToList();
foreach (var index in indexes)
index.Remove("ns");
if (indexes.Count > 0)
await database.RunCommandAsync(new BsonDocument { { "createIndexes", entry.Name }, { "indexes", new BsonArray(indexes) } }, cancellationToken: token);
report.Collections++;
}
var held = backup.Manifest.Collections.Select(c => c.Name).ToHashSet(StringComparer.Ordinal);
foreach (var name in await (await database.ListCollectionNamesAsync(cancellationToken: token)).ToListAsync(token))
{
if (held.Contains(name) || ServerBackup.Excluded.ContainsKey(name) || name.StartsWith("system.", StringComparison.Ordinal))
continue;
await database.DropCollectionAsync(name, token);
report.CollectionsDropped++;
}
}
/// Documents inserted in batches, unordered and unvalidated, as they were: how many.
public static async Task Insert(IMongoCollection collection, IEnumerable documents, CancellationToken token)
{
var count = 0L;
foreach (var batch in documents.Chunk(1000))
{
await collection.InsertManyAsync(batch, new InsertManyOptions { IsOrdered = false, BypassDocumentValidation = true }, token);
count += batch.Length;
}
return count;
}
/// A collection's documents in a backup, one by one; none when it has no file.
public static IEnumerable Read(string file)
{
if (!File.Exists(file))
yield break;
using var gzip = new GZipStream(File.OpenRead(file), CompressionMode.Decompress);
using var reader = new StreamReader(gzip);
while (reader.ReadLine() is { } line)
if (line.Length > 0)
yield return BsonDocument.Parse(line);
}
// the backup's media files the live directory lacks: linked from the backup, or moved back from the trash
static void Media(BackupContext context, BackupInfo backup, RestoreReport report)
{
var kept = Path.Combine(context.BackupsRoot, backup.Id, "media");
foreach (var relative in backup.Manifest.Media.List)
{
var live = ServerBackup.Inside(context.MediaRoot, relative);
if (File.Exists(live))
continue;
var fromBackup = ServerBackup.Inside(kept, relative);
var fromTrash = ServerBackup.Inside(context.TrashRoot, relative);
if (File.Exists(fromBackup))
HardLink.LinkOrCopy(fromBackup, live);
else if (File.Exists(fromTrash))
{
Directory.CreateDirectory(Path.GetDirectoryName(live)!);
File.Move(fromTrash, live);
}
else
{
report.MediaMissing++;
continue;
}
report.MediaRestored++;
}
}
}
}