A backup is restored at boot, and never undoes a protective act
PrivaPub admin restore <id> (and soon the administrator's page) checks the backup (same host, a format and newest migration this build reads, every hash) and writes restore.json; the running service sees it within seconds and stops, and the next start restores it in MaintenanceGate, before migrations, indexes and hosted services: a pre-restore backup taken once, every collection dropped and imported raw with its indexes, the media the live directory lacks brought back, then the protective merge from the pre-restore backup. Followers and follows are the live ones; blocks, mutes, domain blocks, reserved names, tombstones, reports, filters and OAuth applications are the union; deletions win; accounts made since become tombstones and local posts made since answer 410; every session ends. Each attempt redoes everything; one refused before any change is abandoned and recorded, one failed midway exits 1 for systemd to retry, and after three it exits 75, which the unit no longer restarts. Commands wait (exit 75) while a restore is pending. RestoreRecord tells what happened (admin restore --status). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01LsXgEaXee4GCU1hwYgPJXw
This commit is contained in:
thepraandClaude Opus 5.5 committed
1 parent
12bb75809f
commit
fba57318fa
13 files changed
+1050
-11
No files matched your search
@@ -0,0 +1,59 @@
|
||||
namespace PrivaPub.Infrastructure.Backup
|
||||
{
|
||||
// What Program asks before migrations: whether a restore waits, and what this process does about it. The service carries
|
||||
// it out; a command waits for it (exit 75, EX_TEMPFAIL), except asking how it went.
|
||||
public static class MaintenanceGate
|
||||
{
|
||||
/// <summary>The exit code that stops systemd restarting the service (RestartPreventExitStatus=75): a restore gave up.</summary>
|
||||
public const int TryLater = 75;
|
||||
|
||||
/// <summary>Null to go on booting; else the code to exit with.</summary>
|
||||
public static async Task<int?> Enter(IServiceProvider services, string[] command, CancellationToken token)
|
||||
{
|
||||
var backups = services.GetRequiredService<Backups>();
|
||||
var logger = services.GetRequiredService<ILoggerFactory>().CreateLogger(nameof(MaintenanceGate));
|
||||
if (command != default)
|
||||
{
|
||||
var waiting = RestoreMarker.Read(backups.Root);
|
||||
if (waiting == default || command is ["restore", ..])
|
||||
return default;
|
||||
Console.Error.WriteLine($"the restore of {waiting.Backup} is {waiting.State}{(waiting.Error == default ? string.Empty : $" ({waiting.Error})")}: try again once the service has carried it out (PrivaPub admin restore --status)");
|
||||
return TryLater;
|
||||
}
|
||||
return await ServerRestore.ApplyPending(backups.Context(), logger, token) switch
|
||||
{
|
||||
RestoreOutcome.Failed => 1,//systemd starts it again, and the next attempt redoes everything
|
||||
RestoreOutcome.GaveUp => TryLater,
|
||||
_ => (int?)null
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// A restore asked for while the service runs (from the administrator's page or the CLI): the service stops within
|
||||
// seconds, systemd starts it again, and the restore runs before anything else.
|
||||
public class RestoreWatcher(Backups backups, IHostApplicationLifetime lifetime, ILogger<RestoreWatcher> logger) : BackgroundService
|
||||
{
|
||||
static readonly TimeSpan Interval = TimeSpan.FromSeconds(3);
|
||||
|
||||
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
||||
{
|
||||
while (!stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
try
|
||||
{
|
||||
await Task.Delay(Interval, stoppingToken);
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
return;
|
||||
}
|
||||
if (RestoreMarker.Read(backups.Root) is { State: "pending" } marker)
|
||||
{
|
||||
logger.LogWarning("A restore of {Backup} was asked for by {RequestedBy}: stopping, to restore it at the next start", marker.Backup, marker.RequestedBy);
|
||||
lifetime.StopApplication();
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,264 @@
|
||||
using MongoDB.Bson;
|
||||
using MongoDB.Driver;
|
||||
|
||||
namespace PrivaPub.Infrastructure.Backup
|
||||
{
|
||||
// A restore may lose what was made since the backup, but it never undoes a protective act (owner decision 2026-10-07).
|
||||
// From the pre-restore backup P, after the backup is imported:
|
||||
// - who follows whom is P's: a follower that left stays gone, so no followers-only post reaches it, and one gained is
|
||||
// kept; the persona's follows are P's too;
|
||||
// - blocks, mutes, domain blocks (the server's and the personas'), being blocked, reserved names, deletion tombstones,
|
||||
// reports, filters and OAuth applications are the union of both, P's row winning;
|
||||
// - deletions win: a root, persona, group, post or remote account deleted since is deleted again (P's row, as its
|
||||
// deletion left it), with what a deletion takes away; P's moderation of a persona (silenced, suspended, banned) stays;
|
||||
// - a root keeps P's password, e-mail, ban and policies;
|
||||
// - roots, personas and groups made since become tombstones: deleted, their names kept; local posts made since answer
|
||||
// 410; media made since go to the trash, as do media trashed since (and they leave the posts restored with them);
|
||||
// - every session ends, and every server's circuit is closed again.
|
||||
// Run on the imported database, it reads P's files only, so it can run again after a failed attempt.
|
||||
public static class ProtectiveMerge
|
||||
{
|
||||
static readonly (string Collection, string[] Key)[] Unions =
|
||||
[
|
||||
("ReservedName", ["Name"]),
|
||||
("DomainBlock", ["Domain"]),
|
||||
("Block", ["AvatarId", "TargetActorURI"]),
|
||||
("Mute", ["AvatarId", "TargetActorURI"]),
|
||||
("AccountDomainBlock", ["AvatarId", "Domain"]),
|
||||
("BlockedBy", ["AvatarId", "ActorURI"]),
|
||||
("DeletedObject", ["ObjectURI"]),
|
||||
("Report", ["_id"]),
|
||||
("PersonaFilter", ["_id"]),
|
||||
("openiddict.applications", ["client_id"])
|
||||
];
|
||||
|
||||
const string Since = "restore: made after the backup";
|
||||
|
||||
public static async Task Apply(IMongoDatabase database, string previous, RestoreReport report, CancellationToken token)
|
||||
{
|
||||
IEnumerable<BsonDocument> P(string collection) => ServerRestore.Read(Path.Combine(previous, collection + ".jsonl.gz"));
|
||||
var now = DateTime.UtcNow;
|
||||
|
||||
// who follows whom: P's
|
||||
foreach (var collection in new[] { "Follower", "Following" })
|
||||
{
|
||||
await database.DropCollectionAsync(collection, token);
|
||||
await ServerRestore.Insert(database.GetCollection<BsonDocument>(collection), P(collection), token);
|
||||
}
|
||||
|
||||
foreach (var (collection, key) in Unions)
|
||||
report.ProtectiveKept += await Union(database.GetCollection<BsonDocument>(collection), P(collection), key, token);
|
||||
|
||||
var roots = database.GetCollection<BsonDocument>("RootUser");
|
||||
foreach (var root in P("RootUser"))
|
||||
{
|
||||
var restored = await ById(roots, root["_id"], token);
|
||||
if (restored == default)
|
||||
{
|
||||
// made since: deleted, as RootRemoval leaves a root
|
||||
report.RootsTombstoned.Add(root.GetValue("UserName", "").ToString());
|
||||
root["UserName"] = $"deleted-{root["_id"]}";
|
||||
root["Email"] = BsonNull.Value;
|
||||
root["HashedPassword"] = BsonNull.Value;
|
||||
root["Policies"] = new BsonArray();
|
||||
root["IsBanned"] = false;
|
||||
root["IsEmailValidated"] = false;
|
||||
root["DeletedAt"] = Deleted(root, "DeletedAt", now);
|
||||
await roots.InsertOneAsync(root, cancellationToken: token);
|
||||
continue;
|
||||
}
|
||||
if (IsSet(root, "DeletedAt"))
|
||||
{
|
||||
if (!IsSet(restored, "DeletedAt"))
|
||||
report.RootsDeleted++;
|
||||
await roots.ReplaceOneAsync(Id(root["_id"]), root, cancellationToken: token);
|
||||
continue;
|
||||
}
|
||||
await roots.UpdateOneAsync(Id(root["_id"]), Copy(root, "HashedPassword", "Email", "IsEmailValidated", "IsBanned", "Policies",
|
||||
"CredentialsChangedAt", "ResetPasswordToken", "ResetPasswordTokenSentAt"), cancellationToken: token);
|
||||
}
|
||||
|
||||
var avatars = database.GetCollection<BsonDocument>("Avatar");
|
||||
foreach (var avatar in P("Avatar"))
|
||||
{
|
||||
var restored = await ById(avatars, avatar["_id"], token);
|
||||
if (restored == default)
|
||||
{
|
||||
report.PersonasTombstoned.Add(avatar.GetValue("UserName", "").ToString());
|
||||
avatar["DeletionAt"] = Deleted(avatar, "DeletionAt", now);
|
||||
await avatars.InsertOneAsync(avatar, cancellationToken: token);
|
||||
continue;
|
||||
}
|
||||
if (IsSet(avatar, "DeletionAt"))
|
||||
{
|
||||
if (!IsSet(restored, "DeletionAt"))
|
||||
report.PersonasDeleted++;
|
||||
await avatars.ReplaceOneAsync(Id(avatar["_id"]), avatar, cancellationToken: token);
|
||||
// what RootRemoval takes with a persona, which the backup still had
|
||||
foreach (var collection in new[] { "PersonaListMember", "PersonaList", "PersonaFilter", "FollowedTag", "ScheduledStatus" })
|
||||
await database.GetCollection<BsonDocument>(collection).DeleteManyAsync(new BsonDocument("AvatarId", avatar["_id"].ToString()), token);
|
||||
continue;
|
||||
}
|
||||
var moderated = new[] { "SilencedAt", "SuspendedAt", "BannedAt" }.Where(field => IsSet(avatar, field)).ToArray();
|
||||
if (moderated.Length > 0)
|
||||
await avatars.UpdateOneAsync(Id(avatar["_id"]), Copy(avatar, moderated), cancellationToken: token);
|
||||
}
|
||||
|
||||
var links = database.GetCollection<BsonDocument>("RootToAvatar");
|
||||
var linked = (await (await links.FindAsync(FilterDefinition<BsonDocument>.Empty, cancellationToken: token)).ToListAsync(token))
|
||||
.Select(l => l.GetValue("AvatarId", BsonNull.Value).ToString()).ToHashSet(StringComparer.Ordinal);
|
||||
await ServerRestore.Insert(links, P("RootToAvatar").Where(l => !linked.Contains(l.GetValue("AvatarId", BsonNull.Value).ToString())), token);
|
||||
|
||||
var groups = database.GetCollection<BsonDocument>("Group");
|
||||
foreach (var group in P("Group"))
|
||||
{
|
||||
var restored = await ById(groups, group["_id"], token);
|
||||
if (restored == default)
|
||||
{
|
||||
report.GroupsTombstoned.Add(group.GetValue("UserName", "").ToString());
|
||||
group["DeletionAt"] = Deleted(group, "DeletionAt", now);
|
||||
await groups.InsertOneAsync(group, cancellationToken: token);
|
||||
continue;
|
||||
}
|
||||
if (!IsSet(group, "DeletionAt"))
|
||||
continue;
|
||||
if (!IsSet(restored, "DeletionAt"))
|
||||
report.GroupsDeleted++;
|
||||
await groups.ReplaceOneAsync(Id(group["_id"]), group, cancellationToken: token);
|
||||
}
|
||||
|
||||
// remote accounts deleted or gone since: as P has them
|
||||
var foreign = database.GetCollection<BsonDocument>("ForeignAvatar");
|
||||
foreach (var account in P("ForeignAvatar").Where(a => IsSet(a, "DeletionAt") || IsSet(a, "ForgottenAt")))
|
||||
await foreign.ReplaceOneAsync(Id(account["_id"]), account, cancellationToken: token);
|
||||
|
||||
await Posts(database, P("Post"), report, token);
|
||||
await Media(database, P("MediaAttachment"), report, token);
|
||||
|
||||
// every session ends: a new stamp for each root's tokens, and the personas' OAuth tokens go
|
||||
foreach (var root in await (await roots.FindAsync(FilterDefinition<BsonDocument>.Empty, cancellationToken: token)).ToListAsync(token))
|
||||
await roots.UpdateOneAsync(Id(root["_id"]), Builders<BsonDocument>.Update
|
||||
.Set("SessionStamp", Guid.NewGuid().ToString("N"))
|
||||
.Set("CredentialsChangedAt", now), cancellationToken: token);
|
||||
foreach (var collection in new[] { "openiddict.tokens", "openiddict.authorizations" })
|
||||
report.SessionsEnded += (int)(await database.GetCollection<BsonDocument>(collection).DeleteManyAsync(FilterDefinition<BsonDocument>.Empty, token)).DeletedCount;
|
||||
|
||||
// what the backup remembered of servers being down is long past
|
||||
await database.GetCollection<BsonDocument>("RemoteInstance").UpdateManyAsync(FilterDefinition<BsonDocument>.Empty,
|
||||
Builders<BsonDocument>.Update.Set("ConsecutiveFailures", 0).Set("UnavailableUntil", BsonNull.Value), cancellationToken: token);
|
||||
}
|
||||
|
||||
// posts deleted since: deleted again, out of timelines and pins; local posts made since: 410
|
||||
static async Task Posts(IMongoDatabase database, IEnumerable<BsonDocument> previous, RestoreReport report, CancellationToken token)
|
||||
{
|
||||
var posts = database.GetCollection<BsonDocument>("Post");
|
||||
var gone = database.GetCollection<BsonDocument>("DeletedObject");
|
||||
foreach (var batch in previous.Where(p => IsSet(p, "DeletedAt") || IsLocal(p)).Chunk(500))
|
||||
{
|
||||
var ids = new BsonArray(batch.Select(p => p["_id"]));
|
||||
var restored = (await (await posts.FindAsync(new BsonDocument("_id", new BsonDocument("$in", ids)), cancellationToken: token)).ToListAsync(token))
|
||||
.ToDictionary(p => p["_id"]);
|
||||
foreach (var post in batch)
|
||||
{
|
||||
if (!restored.TryGetValue(post["_id"], out var mine))
|
||||
{
|
||||
if (IsLocal(post) && post.GetValue("ObjectURI", BsonNull.Value) is BsonString uri)
|
||||
{
|
||||
await gone.UpdateOneAsync(new BsonDocument("ObjectURI", uri), Builders<BsonDocument>.Update
|
||||
.SetOnInsert("ObjectURI", uri).SetOnInsert("DeletedAt", DateTime.UtcNow), new UpdateOptions { IsUpsert = true }, token);
|
||||
report.PostsGone++;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if (!IsSet(post, "DeletedAt") || IsSet(mine, "DeletedAt"))
|
||||
continue;
|
||||
await posts.ReplaceOneAsync(Id(post["_id"]), post, cancellationToken: token);
|
||||
var id = post["_id"].ToString();
|
||||
await database.GetCollection<BsonDocument>("TimelineEntry").DeleteManyAsync(
|
||||
new BsonDocument("$or", new BsonArray { new BsonDocument("PostId", id), new BsonDocument("ReblogOfPostId", id) }), token);
|
||||
await database.GetCollection<BsonDocument>("Pin").DeleteManyAsync(new BsonDocument("PostId", id), token);
|
||||
report.PostsDeleted++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// media trashed since, and media made since: in the trash (the janitor deletes their files after its grace), and out
|
||||
// of the posts restored; media of posts deleted now: in the trash too
|
||||
static async Task Media(IMongoDatabase database, IEnumerable<BsonDocument> previous, RestoreReport report, CancellationToken token)
|
||||
{
|
||||
var media = database.GetCollection<BsonDocument>("MediaAttachment");
|
||||
var posts = database.GetCollection<BsonDocument>("Post");
|
||||
var now = DateTime.UtcNow;
|
||||
foreach (var batch in previous.Chunk(1000))
|
||||
{
|
||||
var ids = new BsonArray(batch.Select(m => m["_id"]));
|
||||
var restored = (await (await media.FindAsync(new BsonDocument("_id", new BsonDocument("$in", ids)), cancellationToken: token)).ToListAsync(token))
|
||||
.ToDictionary(m => m["_id"]);
|
||||
var made = new List<BsonDocument>();
|
||||
foreach (var row in batch)
|
||||
{
|
||||
if (!restored.TryGetValue(row["_id"], out var mine))
|
||||
{
|
||||
if (!IsSet(row, "TrashedAt"))
|
||||
{
|
||||
row["TrashedAt"] = now;
|
||||
row["TrashReason"] = Since;
|
||||
}
|
||||
made.Add(row);
|
||||
continue;
|
||||
}
|
||||
if (!IsSet(row, "TrashedAt") || IsSet(mine, "TrashedAt"))
|
||||
continue;
|
||||
await media.UpdateOneAsync(Id(row["_id"]), Copy(row, "TrashedAt", "TrashReason"), cancellationToken: token);
|
||||
if (mine.GetValue("PostId", BsonNull.Value) is BsonString postId)
|
||||
await posts.UpdateOneAsync(Id(ObjectId.TryParse(postId.AsString, out var post) ? post : postId), Builders<BsonDocument>.Update.PullFilter<BsonDocument>("Media",
|
||||
new BsonDocument("AttachmentId", row["_id"].ToString())), cancellationToken: token);
|
||||
report.MediaTrashed++;
|
||||
}
|
||||
await ServerRestore.Insert(media, made, token);
|
||||
report.MediaTrashed += made.Count(m => m["TrashReason"] == Since);
|
||||
}
|
||||
|
||||
var deleted = await (await posts.FindAsync(new BsonDocument("DeletedAt", new BsonDocument("$ne", BsonNull.Value)),
|
||||
new FindOptions<BsonDocument> { Projection = new BsonDocument("_id", 1) }, token)).ToListAsync(token);
|
||||
foreach (var batch in deleted.Select(p => (BsonValue)p["_id"].ToString()).Chunk(1000))
|
||||
report.MediaTrashed += (int)(await media.UpdateManyAsync(
|
||||
new BsonDocument { { "PostId", new BsonDocument("$in", new BsonArray(batch)) }, { "TrashedAt", BsonNull.Value } },
|
||||
Builders<BsonDocument>.Update.Set("TrashedAt", now).Set("TrashReason", "restore: its post was deleted"), cancellationToken: token)).ModifiedCount;
|
||||
}
|
||||
|
||||
// P's rows added, each replacing the restored row with the same key: how many P had
|
||||
static async Task<int> Union(IMongoCollection<BsonDocument> collection, IEnumerable<BsonDocument> previous, string[] key, CancellationToken token)
|
||||
{
|
||||
var count = 0;
|
||||
foreach (var batch in previous.Chunk(500))
|
||||
{
|
||||
var writes = new List<WriteModel<BsonDocument>>();
|
||||
foreach (var row in batch)
|
||||
{
|
||||
var match = new BsonDocument(key.Select(field => new BsonElement(field, row.GetValue(field, BsonNull.Value))));
|
||||
writes.Add(new DeleteManyModel<BsonDocument>(match));
|
||||
writes.Add(new InsertOneModel<BsonDocument>(row));
|
||||
}
|
||||
await collection.BulkWriteAsync(writes, new BulkWriteOptions { IsOrdered = true, BypassDocumentValidation = true }, token);
|
||||
count += batch.Length;
|
||||
}
|
||||
return count;
|
||||
}
|
||||
|
||||
static bool IsLocal(BsonDocument post) =>
|
||||
post.GetValue("GroupUserId", BsonNull.Value) is BsonString && !post.GetValue("IsFederatedCopy", false).ToBoolean();
|
||||
|
||||
static bool IsSet(BsonDocument row, string field) => row.TryGetValue(field, out var value) && !value.IsBsonNull;
|
||||
|
||||
static BsonValue Deleted(BsonDocument row, string field, DateTime now) => IsSet(row, field) ? row[field] : now;
|
||||
|
||||
static FilterDefinition<BsonDocument> Id(BsonValue id) => new BsonDocument("_id", id);
|
||||
|
||||
static async Task<BsonDocument> ById(IMongoCollection<BsonDocument> collection, BsonValue id, CancellationToken token) =>
|
||||
await (await collection.FindAsync(Id(id), cancellationToken: token)).FirstOrDefaultAsync(token);
|
||||
|
||||
static UpdateDefinition<BsonDocument> Copy(BsonDocument from, params string[] fields) =>
|
||||
Builders<BsonDocument>.Update.Combine(fields.Select(field => Builders<BsonDocument>.Update.Set(field, from.GetValue(field, BsonNull.Value))));
|
||||
}
|
||||
}
|
||||
@@ -12,6 +12,7 @@ namespace PrivaPub.Infrastructure.Backup
|
||||
public string Backup { get; set; }//the backup restored
|
||||
public string PreRestore { get; set; }//the backup taken just before, what the protective merge reads
|
||||
public string State { get; set; } = "pending";//pending, then running
|
||||
public string RequestedBy { get; set; }//who asked: a root's name, or "cli"
|
||||
public int Attempts { get; set; }
|
||||
public DateTime RequestedAt { get; set; } = DateTime.UtcNow;
|
||||
public string Error { get; set; }
|
||||
@@ -29,7 +30,8 @@ namespace PrivaPub.Infrastructure.Backup
|
||||
}
|
||||
catch (JsonException)
|
||||
{
|
||||
return default;
|
||||
//never written half (Write is atomic): someone's hand; the server won't guess what was meant
|
||||
return new RestoreMarker { State = "unreadable", Attempts = MaxAttempts, Error = $"{FileName} can't be read" };
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,62 @@
|
||||
using MongoDB.Entities;
|
||||
|
||||
namespace PrivaPub.Infrastructure.Backup
|
||||
{
|
||||
// What a restore did (or why it was abandoned), kept for the administrator's page and for the followers' grace: for
|
||||
// FollowersGrace after a restore the followers' digests (FEP-8fcf) are neither sent nor acted on to undo a follow, since
|
||||
// what the restore lost of either side is not the other's fault.
|
||||
public class RestoreRecord : Entity
|
||||
{
|
||||
public static readonly TimeSpan FollowersGrace = TimeSpan.FromDays(14);
|
||||
|
||||
public string Backup { get; set; }
|
||||
public DateTime BackupCreatedAt { get; set; }
|
||||
public string PreRestore { get; set; }
|
||||
public string RequestedBy { get; set; }
|
||||
public DateTime RequestedAt { get; set; }
|
||||
public DateTime RestoredAt { get; set; } = DateTime.UtcNow;
|
||||
public int Attempts { get; set; }
|
||||
public bool Abandoned { get; set; }//nothing was changed: the server booted as it was
|
||||
public string Error { get; set; }
|
||||
public RestoreReport Report { get; set; } = new();
|
||||
|
||||
static (DateTime Until, DateTime Read) _grace;
|
||||
|
||||
/// <summary>Whether a restore ended less than FollowersGrace ago (read at most once a minute).</summary>
|
||||
public static async Task<bool> InFollowersGrace(CancellationToken token)
|
||||
{
|
||||
var now = DateTime.UtcNow;
|
||||
var cached = _grace;
|
||||
if (now - cached.Read > TimeSpan.FromMinutes(1))
|
||||
{
|
||||
var last = await DB.Default.Find<RestoreRecord>().Match(r => !r.Abandoned).Sort(r => r.RestoredAt, Order.Descending).ExecuteFirstAsync(token);
|
||||
cached = (last == default ? DateTime.MinValue : last.RestoredAt + FollowersGrace, now);
|
||||
_grace = cached;
|
||||
}
|
||||
return now < cached.Until;
|
||||
}
|
||||
|
||||
/// <summary>Forgets what was read, so a restore made in this process counts at once.</summary>
|
||||
public static void Forget() => _grace = default;
|
||||
}
|
||||
|
||||
public class RestoreReport
|
||||
{
|
||||
public int Collections { get; set; }
|
||||
public long Documents { get; set; }
|
||||
public int CollectionsDropped { get; set; }//live collections the backup had no documents in
|
||||
public int MediaRestored { get; set; }//files the live directory lacked, brought back
|
||||
public int MediaMissing { get; set; }
|
||||
public int RootsDeleted { get; set; }//deleted since the backup, deleted again
|
||||
public int PersonasDeleted { get; set; }
|
||||
public int GroupsDeleted { get; set; }
|
||||
public int PostsDeleted { get; set; }
|
||||
public List<string> RootsTombstoned { get; set; } = [];//made after the backup: deleted, their names kept
|
||||
public List<string> PersonasTombstoned { get; set; } = [];
|
||||
public List<string> GroupsTombstoned { get; set; } = [];
|
||||
public int PostsGone { get; set; }//local posts made after the backup: 410 from now on
|
||||
public int MediaTrashed { get; set; }
|
||||
public int ProtectiveKept { get; set; }//blocks, mutes, domain blocks, reserved names, reports, filters, deletions kept from since
|
||||
public int SessionsEnded { get; set; }
|
||||
}
|
||||
}
|
||||
@@ -38,7 +38,8 @@ namespace PrivaPub.Infrastructure.Backup
|
||||
["openiddict.tokens"] = "sessions: a restore ends every one",
|
||||
["openiddict.authorizations"] = "sessions: a restore ends every one",
|
||||
[nameof(MaintenanceLock)] = "who is backing up or restoring now",
|
||||
["AppConfiguration"] = "the configuration's copy holds the SMTP password, and is made again from appsettings at boot"
|
||||
["AppConfiguration"] = "the configuration's copy holds the SMTP password, and is made again from appsettings at boot",
|
||||
[nameof(RestoreRecord)] = "what restores did outlives what they restore"
|
||||
};
|
||||
|
||||
static readonly JsonWriterSettings Json = new() { OutputMode = JsonOutputMode.CanonicalExtendedJson, Indent = false };
|
||||
|
||||
@@ -0,0 +1,247 @@
|
||||
using MongoDB.Bson;
|
||||
using MongoDB.Driver;
|
||||
|
||||
using System.IO.Compression;
|
||||
|
||||
namespace PrivaPub.Infrastructure.Backup
|
||||
{
|
||||
public enum RestoreOutcome
|
||||
{
|
||||
None,//nothing was asked
|
||||
Restored,
|
||||
Abandoned,//refused before anything changed: the server boots as it was
|
||||
Failed,//failed midway: the next boot tries again from the start
|
||||
GaveUp//failed MaxAttempts times: the server stays down until someone looks
|
||||
}
|
||||
|
||||
// A backup restored (owner decision 2026-10-07: from the CLI or the administrator's page). Asking writes restore.json
|
||||
// and the running service stops at once (RestoreWatcher); the restore itself runs at the next boot, before migrations,
|
||||
// indexes and every hosted service, with nothing else touching the database:
|
||||
// 1. a pre-restore backup P is taken, once (a retry reuses it);
|
||||
// 2. every collection the backup holds is dropped and imported raw, with its indexes; every other one is dropped,
|
||||
// except what a backup never holds (ServerBackup.Excluded), which stays as it is;
|
||||
// 3. media files the live directory lacks come back from the backup (or the trash); a restore deletes no file;
|
||||
// 4. ProtectiveMerge brings back from P every protective act made since the backup;
|
||||
// 5. every session ends, every server's circuit closes, and a RestoreRecord tells what happened.
|
||||
// Each attempt redoes everything, so one that dies midway converges on the next.
|
||||
public static class ServerRestore
|
||||
{
|
||||
static readonly TimeSpan LockWait = TimeSpan.FromMinutes(10);
|
||||
|
||||
/// <summary>Why a backup can't be restored here: none when it can.</summary>
|
||||
public static async Task<List<string>> Check(BackupContext context, string id, CancellationToken token)
|
||||
{
|
||||
var backup = ServerBackup.Find(context.BackupsRoot, id);
|
||||
if (backup?.Manifest == default)
|
||||
return ["no such backup"];
|
||||
var manifest = backup.Manifest;
|
||||
var problems = new List<string>();
|
||||
if (manifest.Format != ArchiveManifest.CurrentFormat)
|
||||
problems.Add($"its format is {manifest.Format}, this build reads {ArchiveManifest.CurrentFormat}");
|
||||
if (!string.Equals(manifest.Host?.TrimEnd('/'), context.Host, StringComparison.OrdinalIgnoreCase))
|
||||
problems.Add($"it was made by {manifest.Host}, not {context.Host}: every id and address in it names its host");
|
||||
var code = ServerBackup.CodeMigration();
|
||||
if (manifest.MigrationNumber > code)
|
||||
problems.Add($"it was made by a newer build (migration {manifest.MigrationNumber}; this one knows {code})");
|
||||
problems.AddRange(await ServerBackup.Verify(context.BackupsRoot, id, token));
|
||||
return problems;
|
||||
}
|
||||
|
||||
/// <summary>Asks for a backup to be restored at the next boot: why not, or null once asked.</summary>
|
||||
public static async Task<string> Request(BackupContext context, string id, string requestedBy, CancellationToken token)
|
||||
{
|
||||
var problems = await Check(context, id, token);
|
||||
if (problems.Count > 0)
|
||||
return string.Join("; ", problems);
|
||||
var waiting = RestoreMarker.Read(context.BackupsRoot);
|
||||
if (waiting != default && waiting.Attempts < RestoreMarker.MaxAttempts)
|
||||
return $"the restore of {waiting.Backup} is already {waiting.State}";
|
||||
if (await MaintenanceLock.Current(token) is { } held)
|
||||
return $"a {held.What} is running";
|
||||
new RestoreMarker { Backup = id, RequestedBy = requestedBy }.Write(context.BackupsRoot);
|
||||
return default;
|
||||
}
|
||||
|
||||
/// <summary>The restore asked for, carried out (at boot, before migrations): what came of it.</summary>
|
||||
public static async Task<RestoreOutcome> ApplyPending(BackupContext context, ILogger logger, CancellationToken token)
|
||||
{
|
||||
var marker = RestoreMarker.Read(context.BackupsRoot);
|
||||
if (marker == default)
|
||||
return RestoreOutcome.None;
|
||||
if (marker.Attempts >= RestoreMarker.MaxAttempts)
|
||||
{
|
||||
logger.LogCritical("The restore of {Backup} failed {Attempts} times ({Error}). The database may be half restored; {PreRestore} holds it as it was before. Restore that backup or another (PrivaPub admin restore <id>), or delete {Marker} to boot as it is",
|
||||
marker.Backup, marker.Attempts, marker.Error, marker.PreRestore ?? "no backup", RestoreMarker.PathIn(context.BackupsRoot));
|
||||
return RestoreOutcome.GaveUp;
|
||||
}
|
||||
|
||||
await using var held = await TakeLock(token);
|
||||
if (held == default)
|
||||
return await Abandon(context, marker, "a backup kept the maintenance lock for ten minutes", logger, token);
|
||||
|
||||
var problems = await Check(context, marker.Backup, token);
|
||||
if (problems.Count > 0 && marker.PreRestore == default)
|
||||
return await Abandon(context, marker, string.Join("; ", problems), logger, token);
|
||||
|
||||
if (marker.PreRestore == default)
|
||||
{
|
||||
var (taken, error) = await ServerBackup.CreateHeld(context, "pre-restore", dbOnly: false, token);
|
||||
if (taken == default)
|
||||
return await Abandon(context, marker, $"the pre-restore backup could not be made: {error}", logger, token);
|
||||
marker.PreRestore = taken.Id;
|
||||
}
|
||||
marker.State = "running";
|
||||
marker.Attempts++;
|
||||
marker.Error = default;
|
||||
marker.Write(context.BackupsRoot);
|
||||
logger.LogWarning("Restoring {Backup} (attempt {Attempt}); the server as it was is {PreRestore}", marker.Backup, marker.Attempts, marker.PreRestore);
|
||||
|
||||
try
|
||||
{
|
||||
if (problems.Count > 0)
|
||||
throw new InvalidOperationException(string.Join("; ", problems));
|
||||
var backup = ServerBackup.Find(context.BackupsRoot, marker.Backup);
|
||||
var report = new RestoreReport();
|
||||
await Import(context, backup, report, token);
|
||||
Media(context, backup, report);
|
||||
await ProtectiveMerge.Apply(context.Database, Path.Combine(context.BackupsRoot, marker.PreRestore, "db"), report, token);
|
||||
await Record(context, new RestoreRecord
|
||||
{
|
||||
Backup = backup.Id,
|
||||
BackupCreatedAt = backup.CreatedAt,
|
||||
PreRestore = marker.PreRestore,
|
||||
RequestedBy = marker.RequestedBy,
|
||||
RequestedAt = marker.RequestedAt,
|
||||
Attempts = marker.Attempts,
|
||||
Report = report
|
||||
}, token);
|
||||
RestoreRecord.Forget();
|
||||
RestoreMarker.Clear(context.BackupsRoot);
|
||||
logger.LogWarning("Restored {Backup}: {Documents} documents in {Collections} collections, {Media} media files brought back, {Tombstoned} accounts made since deleted",
|
||||
backup.Id, report.Documents, report.Collections, report.MediaRestored, report.RootsTombstoned.Count + report.PersonasTombstoned.Count + report.GroupsTombstoned.Count);
|
||||
return RestoreOutcome.Restored;
|
||||
}
|
||||
catch (Exception ex) when (ex is not OperationCanceledException)
|
||||
{
|
||||
marker.Error = ex.Message;
|
||||
marker.Write(context.BackupsRoot);
|
||||
logger.LogError(ex, "The restore of {Backup} failed (attempt {Attempt} of {Max})", marker.Backup, marker.Attempts, RestoreMarker.MaxAttempts);
|
||||
return marker.Attempts >= RestoreMarker.MaxAttempts ? RestoreOutcome.GaveUp : RestoreOutcome.Failed;
|
||||
}
|
||||
}
|
||||
|
||||
static async Task<MaintenanceLock.Held> TakeLock(CancellationToken token)
|
||||
{
|
||||
var until = DateTime.UtcNow + LockWait;
|
||||
while (true)
|
||||
{
|
||||
var held = await MaintenanceLock.Take("restore", token);
|
||||
if (held != default || DateTime.UtcNow > until)
|
||||
return held;
|
||||
await Task.Delay(TimeSpan.FromSeconds(5), token);
|
||||
}
|
||||
}
|
||||
|
||||
// refused before anything changed: recorded for the administrator's page, and the server boots as it was
|
||||
static async Task<RestoreOutcome> Abandon(BackupContext context, RestoreMarker marker, string why, ILogger logger, CancellationToken token)
|
||||
{
|
||||
logger.LogError("The restore of {Backup} was abandoned, nothing changed: {Why}", marker.Backup, why);
|
||||
await Record(context, new RestoreRecord
|
||||
{
|
||||
Backup = marker.Backup,
|
||||
RequestedBy = marker.RequestedBy,
|
||||
RequestedAt = marker.RequestedAt,
|
||||
Attempts = marker.Attempts,
|
||||
Abandoned = true,
|
||||
Error = why
|
||||
}, token);
|
||||
RestoreMarker.Clear(context.BackupsRoot);
|
||||
return RestoreOutcome.Abandoned;
|
||||
}
|
||||
|
||||
// in the database restored (a backup never holds these records: they outlive what they restore)
|
||||
static async Task Record(BackupContext context, RestoreRecord record, CancellationToken token)
|
||||
{
|
||||
record.ID = ObjectId.GenerateNewId().ToString();
|
||||
await context.Database.GetCollection<RestoreRecord>(nameof(RestoreRecord)).InsertOneAsync(record, cancellationToken: token);
|
||||
}
|
||||
|
||||
// each collection dropped and imported as the backup holds it, its indexes made again; the others dropped
|
||||
static async Task Import(BackupContext context, BackupInfo backup, RestoreReport report, CancellationToken token)
|
||||
{
|
||||
var database = context.Database;
|
||||
var directory = Path.Combine(context.BackupsRoot, backup.Id, "db");
|
||||
foreach (var entry in backup.Manifest.Collections)
|
||||
{
|
||||
await database.DropCollectionAsync(entry.Name, token);
|
||||
await database.CreateCollectionAsync(entry.Name, cancellationToken: token);
|
||||
report.Documents += await Insert(database.GetCollection<BsonDocument>(entry.Name), Read(Path.Combine(directory, entry.Name + ".jsonl.gz")), token);
|
||||
var indexes = entry.Indexes.Select(BsonDocument.Parse).Where(i => i.GetValue("name", "").AsString != "_id_").ToList();
|
||||
foreach (var index in indexes)
|
||||
index.Remove("ns");
|
||||
if (indexes.Count > 0)
|
||||
await database.RunCommandAsync<BsonDocument>(new BsonDocument { { "createIndexes", entry.Name }, { "indexes", new BsonArray(indexes) } }, cancellationToken: token);
|
||||
report.Collections++;
|
||||
}
|
||||
var held = backup.Manifest.Collections.Select(c => c.Name).ToHashSet(StringComparer.Ordinal);
|
||||
foreach (var name in await (await database.ListCollectionNamesAsync(cancellationToken: token)).ToListAsync(token))
|
||||
{
|
||||
if (held.Contains(name) || ServerBackup.Excluded.ContainsKey(name) || name.StartsWith("system.", StringComparison.Ordinal))
|
||||
continue;
|
||||
await database.DropCollectionAsync(name, token);
|
||||
report.CollectionsDropped++;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Documents inserted in batches, unordered and unvalidated, as they were: how many.</summary>
|
||||
public static async Task<long> Insert(IMongoCollection<BsonDocument> collection, IEnumerable<BsonDocument> documents, CancellationToken token)
|
||||
{
|
||||
var count = 0L;
|
||||
foreach (var batch in documents.Chunk(1000))
|
||||
{
|
||||
await collection.InsertManyAsync(batch, new InsertManyOptions { IsOrdered = false, BypassDocumentValidation = true }, token);
|
||||
count += batch.Length;
|
||||
}
|
||||
return count;
|
||||
}
|
||||
|
||||
/// <summary>A collection's documents in a backup, one by one; none when it has no file.</summary>
|
||||
public static IEnumerable<BsonDocument> Read(string file)
|
||||
{
|
||||
if (!File.Exists(file))
|
||||
yield break;
|
||||
using var gzip = new GZipStream(File.OpenRead(file), CompressionMode.Decompress);
|
||||
using var reader = new StreamReader(gzip);
|
||||
while (reader.ReadLine() is { } line)
|
||||
if (line.Length > 0)
|
||||
yield return BsonDocument.Parse(line);
|
||||
}
|
||||
|
||||
// the backup's media files the live directory lacks: linked from the backup, or moved back from the trash
|
||||
static void Media(BackupContext context, BackupInfo backup, RestoreReport report)
|
||||
{
|
||||
var kept = Path.Combine(context.BackupsRoot, backup.Id, "media");
|
||||
foreach (var relative in backup.Manifest.Media.List)
|
||||
{
|
||||
var live = ServerBackup.Inside(context.MediaRoot, relative);
|
||||
if (File.Exists(live))
|
||||
continue;
|
||||
var fromBackup = ServerBackup.Inside(kept, relative);
|
||||
var fromTrash = ServerBackup.Inside(context.TrashRoot, relative);
|
||||
if (File.Exists(fromBackup))
|
||||
HardLink.LinkOrCopy(fromBackup, live);
|
||||
else if (File.Exists(fromTrash))
|
||||
{
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(live)!);
|
||||
File.Move(fromTrash, live);
|
||||
}
|
||||
else
|
||||
{
|
||||
report.MediaMissing++;
|
||||
continue;
|
||||
}
|
||||
report.MediaRestored++;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user