thepra/SocialPub
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

The server backs itself up: every collection byte for byte, the media linked

Browse Source 
A backup is a directory <stamp>-<kind> under Backups:Root (/var/lib/privapub/backups, 2770, files 0640), written as
.partial and renamed once whole: a manifest (host, build, newest migration, each collection's count, size, sha256 and
indexes, what was left out and why, the media list), each collection as gzipped canonical Extended JSON read raw, and
hard links to the files of untrashed media rows (copies where a link can't be made). On a replica set every collection
is read in one snapshot session. Never in a backup: the statistics salt, jobs, recovery codes, sessions, the
maintenance lock and the configuration's copy with its SMTP password.

One backup or restore at a time (MaintenanceLock, a heartbeat document), and the janitor purges nothing meanwhile.
BackupScheduler backs up nightly at 03:30 UTC (or at once after missing a night); rotation keeps 7 daily, 4 weekly,
3 pre-deploy and 3 pre-restore backups. CLI: admin backup [--kind] [--db-only], admin backups, admin backup verify;
these run before migrations, so the deploy's own pre-deploy backup, which replaces mongodump, is of the database as the
live build left it. EntityMaps.Warm runs once under a lock, since test hosts now boot side by side.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01LsXgEaXee4GCU1hwYgPJXw
This commit is contained in:
thepraandClaude Opus 5.5 committed 2026-10-07 11:40:06 +02:00
1 parent 37b12c5fee
commit 12bb75809f
20 files changed
+1132 -22
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

PrivaPub/Infrastructure/Backup/ServerBackup.cs
+340
View File
@@ -0,0 +1,340 @@
using MongoDB.Bson;
using MongoDB.Bson.IO;
using MongoDB.Driver;
using PrivaPub.Infrastructure.Data;
using System.Globalization;
using System.IO.Compression;
using System.Security.Cryptography;
using System.Text;
namespace PrivaPub.Infrastructure.Backup
{
// What a backup needs to know: where things are, and whose host it is.
public sealed record BackupContext(IMongoDatabase Database, string BackupsRoot, string MediaRoot, string TrashRoot, string Host, BackupOptions Options);
public sealed record BackupInfo(string Id, string Kind, DateTime CreatedAt, long Bytes, ArchiveManifest Manifest);
// The whole server, backed up as files (owner decision 2026-10-07: a whole-server backup, plain, protected by file
// permissions). Each backup is a directory <stamp>-<kind> in the backups' root:
// - manifest.json: what it holds (ArchiveManifest);
// - db/<collection>.jsonl.gz: every document of each collection, one per line, in canonical Extended JSON (every BSON
// type kept as it was), all read at one instant when Mongo is a replica set (a snapshot session);
// - media/<path>: the media files rows hold, as hard links to the live ones (no disk spent; a deleted file stays here
// until the backup is rotated out), copied where a link can't be made; a db-only backup lists them instead.
// It is written as <id>.partial and renamed once whole. Left out: what belongs to the moment (Excluded). Everything is
// readable by the service's user and group only: it holds every persona's private key and private posts.
public static class ServerBackup
{
public const string PartialSuffix = ".partial";
public static readonly IReadOnlyDictionary<string, string> Excluded = new Dictionary<string, string>
{
["InteractionSalt"] = "the day's statistics salt never outlives its day (owner rule)",
["Job"] = "work in flight: deliveries and inbox processing belong to the moment",
["Delivery"] = "work in flight, from before jobs",
["EmailRecovery"] = "recovery codes live an hour",
["openiddict.tokens"] = "sessions: a restore ends every one",
["openiddict.authorizations"] = "sessions: a restore ends every one",
[nameof(MaintenanceLock)] = "who is backing up or restoring now",
["AppConfiguration"] = "the configuration's copy holds the SMTP password, and is made again from appsettings at boot"
};
static readonly JsonWriterSettings Json = new() { OutputMode = JsonOutputMode.CanonicalExtendedJson, Indent = false };
// 2770: the service (www-data) and the deploy (in www-data's group) each read and rotate what the other wrote, and new
// files take the directory's group
const UnixFileMode DirectoryMode = UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute
| UnixFileMode.GroupRead | UnixFileMode.GroupWrite | UnixFileMode.GroupExecute | UnixFileMode.SetGroup;
const UnixFileMode FileMode0640 = UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.GroupRead;
/// <summary>Takes the maintenance lock and backs the server up: the backup, or why not.</summary>
public static async Task<(BackupInfo Backup, string Error)> Create(BackupContext context, string kind, bool dbOnly, CancellationToken token)
{
await using var held = await MaintenanceLock.Take("backup", token);
if (held == default)
return (default, "a backup or a restore is already running");
return await CreateHeld(context, kind, dbOnly, token);
}
/// <summary>Backs the server up with the maintenance lock already held (a restore's own backup, taken first).</summary>
public static async Task<(BackupInfo Backup, string Error)> CreateHeld(BackupContext context, string kind, bool dbOnly, CancellationToken token)
{
var database = context.Database;
var names = (await (await database.ListCollectionNamesAsync(cancellationToken: token)).ToListAsync(token))
.Where(n => !n.StartsWith("system.", StringComparison.Ordinal))
.OrderBy(n => n, StringComparer.Ordinal)
.ToList();
MakeDirectory(context.BackupsRoot);
var stats = await database.RunCommandAsync<BsonDocument>(new BsonDocument("dbStats", 1), cancellationToken: token);
var needed = (long)(stats.GetValue("dataSize", 0).ToDouble() * 1.1);
if (new DriveInfo(Path.GetFullPath(context.BackupsRoot)).AvailableFreeSpace < needed)
return (default, $"not enough free disk for a backup: about {needed / 1024 / 1024} MB needed");
var id = $"{DateTime.UtcNow.ToString("yyyyMMdd-HHmmss", CultureInfo.InvariantCulture)}-{kind}";
var partial = Path.Combine(context.BackupsRoot, id + PartialSuffix);
MakeDirectory(partial);
MakeDirectory(Path.Combine(partial, "db"));
try
{
var manifest = new ArchiveManifest
{
Kind = kind,
Host = context.Host,
AppCommit = BuildInfo.Commit,
AppRef = BuildInfo.Ref,
DbOnly = dbOnly
};
var replica = await MongoTopology.IsReplicaSet(database, token);
var media = new SortedSet<string>(StringComparer.Ordinal);
using var session = replica ? await database.Client.StartSessionAsync(new ClientSessionOptions { Snapshot = true }, token) : default;
var window = replica ? await RaiseSnapshotWindow(database, token) : default(int?);
try
{
foreach (var name in names)
{
if (Excluded.TryGetValue(name, out var why))
{
manifest.Excluded.Add(new ArchiveManifest.ExcludedEntry { Name = name, Why = why });
continue;
}
manifest.Collections.Add(await Dump(database, session, name, partial, name == "MediaAttachment" ? media : default, token));
}
(manifest.NewestMigration, manifest.MigrationNumber) = await NewestMigration(database, session, token);
}
finally
{
if (window is { } previous)
await SetSnapshotWindow(database, previous, CancellationToken.None);
}
manifest.Consistent = replica;
manifest.Media.List = [.. media];
if (!dbOnly)
foreach (var relative in media)
{
var source = new[] { context.MediaRoot, context.TrashRoot }.Select(root => Inside(root, relative)).FirstOrDefault(File.Exists);
if (source == default)
{
manifest.Media.Missing++;
continue;
}
HardLink.LinkOrCopy(source, Inside(Path.Combine(partial, "media"), relative));
manifest.Media.Files++;
manifest.Media.Bytes += new FileInfo(source).Length;
}
manifest.Write(partial);
Directory.Move(partial, Path.Combine(context.BackupsRoot, id));
Retain(context.BackupsRoot, context.Options);
return (Info(context.BackupsRoot, id), default);
}
catch
{
if (Directory.Exists(partial))
Directory.Delete(partial, recursive: true);
throw;
}
}
// a collection read as raw BSON in batches, each document a line of canonical Extended JSON, gzipped; the media rows'
// files collected on the way
static async Task<ArchiveManifest.CollectionEntry> Dump(IMongoDatabase database, IClientSessionHandle session, string name, string directory,
ISet<string> media, CancellationToken token)
{
var collection = database.GetCollection<RawBsonDocument>(name);
var path = Path.Combine(directory, "db", name + ".jsonl.gz");
var count = 0L;
await using (var file = NewFile(path))
await using (var gzip = new GZipStream(file, CompressionLevel.Fastest))
await using (var writer = new StreamWriter(gzip, new UTF8Encoding(false)))
{
var options = new FindOptions<RawBsonDocument> { BatchSize = 1000 };
using var cursor = session == default
? await collection.FindAsync(FilterDefinition<RawBsonDocument>.Empty, options, token)
: await collection.FindAsync(session, FilterDefinition<RawBsonDocument>.Empty, options, token);
while (await cursor.MoveNextAsync(token))
foreach (var document in cursor.Current)
using (document)
{
await writer.WriteLineAsync(document.ToJson(Json));
count++;
if (media != default)
Collect(document, media);
}
}
var indexes = await (await collection.Indexes.ListAsync(token)).ToListAsync(token);
return new ArchiveManifest.CollectionEntry
{
Name = name,
Count = count,
Bytes = new FileInfo(path).Length,
Sha256 = await Hash(path, token),
Indexes = [.. indexes.Select(i => i.ToJson(Json))]
};
}
// a media row's files, unless it is trashed
static void Collect(RawBsonDocument row, ISet<string> media)
{
if (row.TryGetValue("TrashedAt", out var trashed) && !trashed.IsBsonNull)
return;
foreach (var field in new[] { "FilePath", "PreviewPath" })
if (row.TryGetValue(field, out var value) && value.IsString && Safe(value.AsString))
media.Add(value.AsString);
}
// MongoDB.Entities records each migration run with its class's number (_016_... is 16) and its name in words
static async Task<(string Name, int Number)> NewestMigration(IMongoDatabase database, IClientSessionHandle session, CancellationToken token)
{
var history = database.GetCollection<BsonDocument>("_migration_history_");
var options = new FindOptions<BsonDocument> { Sort = new BsonDocument("Number", -1), Limit = 1 };
var newest = session == default
? await (await history.FindAsync(FilterDefinition<BsonDocument>.Empty, options, token)).FirstOrDefaultAsync(token)
: await (await history.FindAsync(session, FilterDefinition<BsonDocument>.Empty, options, token)).FirstOrDefaultAsync(token);
return newest == default ? default : (newest.GetValue("Name", BsonNull.Value).ToString(), newest.GetValue("Number", 0).ToInt32());
}
/// <summary>The newest migration this build has: a backup made by a newer one can't be restored by it.</summary>
public static int CodeMigration() => typeof(ServerBackup).Assembly.GetTypes()
.Where(t => typeof(MongoDB.Entities.IMigration).IsAssignableFrom(t) && !t.IsAbstract)
.Select(t => int.TryParse(new string(t.Name.TrimStart('_').TakeWhile(char.IsDigit).ToArray()), out var number) ? number : 0)
.DefaultIfEmpty(0)
.Max();
// how long a snapshot stays readable: raised only while a backup reads (a longer window keeps old versions in the
// small cache all day); the value it had, to set back
static async Task<int?> RaiseSnapshotWindow(IMongoDatabase database, CancellationToken token)
{
var admin = database.Client.GetDatabase("admin");
try
{
var current = await admin.RunCommandAsync<BsonDocument>(new BsonDocument { { "getParameter", 1 }, { "minSnapshotHistoryWindowInSeconds", 1 } }, cancellationToken: token);
var previous = current.GetValue("minSnapshotHistoryWindowInSeconds", 300).ToInt32();
await SetSnapshotWindow(database, Math.Max(previous, 3600), token);
return previous;
}
catch (MongoCommandException)
{
return default;
}
}
static async Task SetSnapshotWindow(IMongoDatabase database, int seconds, CancellationToken token) =>
await database.Client.GetDatabase("admin").RunCommandAsync<BsonDocument>(
new BsonDocument { { "setParameter", 1 }, { "minSnapshotHistoryWindowInSeconds", seconds } }, cancellationToken: token);
/// <summary>The backups kept, newest first (not one still being written).</summary>
public static List<BackupInfo> List(string backupsRoot)
{
if (!Directory.Exists(backupsRoot))
return [];
return Directory.EnumerateDirectories(backupsRoot)
.Select(Path.GetFileName)
.Where(name => !name.EndsWith(PartialSuffix, StringComparison.Ordinal) && File.Exists(Path.Combine(backupsRoot, name, ArchiveManifest.FileName)))
.Select(name => Info(backupsRoot, name))
.OrderByDescending(b => b.CreatedAt)
.ToList();
}
public static BackupInfo Find(string backupsRoot, string id) =>
Safe(id) && !id.Contains('/') && Directory.Exists(Path.Combine(backupsRoot, id)) && !id.EndsWith(PartialSuffix, StringComparison.Ordinal)
? Info(backupsRoot, id)
: default;
static BackupInfo Info(string backupsRoot, string id)
{
var directory = Path.Combine(backupsRoot, id);
var manifest = ArchiveManifest.Read(directory);
var bytes = Directory.EnumerateFiles(Path.Combine(directory, "db")).Sum(f => new FileInfo(f).Length);
return new BackupInfo(id, manifest?.Kind, manifest?.CreatedAt ?? Directory.GetCreationTimeUtc(directory), bytes, manifest);
}
/// <summary>Whether every file of a backup is what its manifest says: the problems found (none when whole).</summary>
public static async Task<List<string>> Verify(string backupsRoot, string id, CancellationToken token)
{
var problems = new List<string>();
var backup = Find(backupsRoot, id);
if (backup?.Manifest == default)
return ["no such backup, or no manifest"];
var directory = Path.Combine(backupsRoot, id);
foreach (var collection in backup.Manifest.Collections)
{
var path = Path.Combine(directory, "db", collection.Name + ".jsonl.gz");
if (!File.Exists(path))
problems.Add($"{collection.Name}: missing");
else if (await Hash(path, token) != collection.Sha256)
problems.Add($"{collection.Name}: altered");
}
if (!backup.Manifest.DbOnly)
foreach (var relative in backup.Manifest.Media.List.Where(r => !File.Exists(Inside(Path.Combine(directory, "media"), r))))
problems.Add($"media {relative}: missing");
return problems;
}
/// <summary>Deletes a backup (its media links go; the live files stay).</summary>
public static bool Delete(string backupsRoot, string id)
{
if (Find(backupsRoot, id) == default)
return false;
Directory.Delete(Path.Combine(backupsRoot, id), recursive: true);
return true;
}
// what is kept: the newest nightly ones for KeepDaily days and the newest of each of KeepWeekly weeks before, the
// newest pre-deploy and pre-restore ones; manual and uploaded ones until deleted; a backup that died writing goes
public static void Retain(string backupsRoot, BackupOptions options)
{
var all = List(backupsRoot);
var nightly = all.Where(b => b.Kind == "nightly").OrderByDescending(b => b.CreatedAt).ToList();
var kept = nightly.Take(options.KeepDaily).ToHashSet();
foreach (var week in nightly.Skip(options.KeepDaily).GroupBy(b => (ISOWeek.GetYear(b.CreatedAt), ISOWeek.GetWeekOfYear(b.CreatedAt))).Take(options.KeepWeekly))
kept.Add(week.First());
var doomed = nightly.Where(b => !kept.Contains(b))
.Concat(all.Where(b => b.Kind == "pre-deploy").OrderByDescending(b => b.CreatedAt).Skip(options.KeepPreDeploy))
.Concat(all.Where(b => b.Kind == "pre-restore").OrderByDescending(b => b.CreatedAt).Skip(options.KeepPreRestore));
var stale = Directory.EnumerateDirectories(backupsRoot, "*" + PartialSuffix).Where(p => Directory.GetLastWriteTimeUtc(p) < DateTime.UtcNow.AddDays(-1));
foreach (var directory in doomed.Select(b => Path.Combine(backupsRoot, b.Id)).Concat(stale).ToList())
try
{
Directory.Delete(directory, recursive: true);
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
//another user's backup this one may not delete: the next rotation tries again
}
}
// a relative path from a database row or a manifest, never outside its root
static bool Safe(string relative) =>
!string.IsNullOrEmpty(relative) && !Path.IsPathRooted(relative) && !relative.Split('/', '\\').Any(part => part is ".." or ".");
public static string Inside(string root, string relative)
{
var full = Path.GetFullPath(Path.Combine(root, relative));
if (!Safe(relative) || !full.StartsWith(Path.GetFullPath(root) + Path.DirectorySeparatorChar, StringComparison.Ordinal))
throw new InvalidOperationException($"{relative} is not inside {root}");
return full;
}
static async Task<string> Hash(string path, CancellationToken token)
{
await using var file = File.OpenRead(path);
return Convert.ToHexStringLower(await SHA256.HashDataAsync(file, token));
}
static FileStream NewFile(string path) => OperatingSystem.IsWindows()
? new FileStream(path, FileMode.CreateNew, FileAccess.Write)
: new FileStream(path, new FileStreamOptions { Mode = FileMode.CreateNew, Access = FileAccess.Write, UnixCreateMode = FileMode0640 });
// set after creating, since the umask would take the group's write away
public static void MakeDirectory(string path)
{
var existed = Directory.Exists(path);
Directory.CreateDirectory(path);
if (!existed && !OperatingSystem.IsWindows())
File.SetUnixFileMode(path, DirectoryMode);
}
}
}