The server backs itself up: every collection byte for byte, the media linked
A backup is a directory <stamp>-<kind> under Backups:Root (/var/lib/privapub/backups, 2770, files 0640), written as .partial and renamed once whole: a manifest (host, build, newest migration, each collection's count, size, sha256 and indexes, what was left out and why, the media list), each collection as gzipped canonical Extended JSON read raw, and hard links to the files of untrashed media rows (copies where a link can't be made). On a replica set every collection is read in one snapshot session. Never in a backup: the statistics salt, jobs, recovery codes, sessions, the maintenance lock and the configuration's copy with its SMTP password. One backup or restore at a time (MaintenanceLock, a heartbeat document), and the janitor purges nothing meanwhile. BackupScheduler backs up nightly at 03:30 UTC (or at once after missing a night); rotation keeps 7 daily, 4 weekly, 3 pre-deploy and 3 pre-restore backups. CLI: admin backup [--kind] [--db-only], admin backups, admin backup verify; these run before migrations, so the deploy's own pre-deploy backup, which replaces mongodump, is of the database as the live build left it. EntityMaps.Warm runs once under a lock, since test hosts now boot side by side. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01LsXgEaXee4GCU1hwYgPJXw
This commit is contained in:
thepraandClaude Opus 5.5 committed
1 parent
37b12c5fee
commit
12bb75809f
20 files changed
+1132
-22
No files matched your search
@@ -0,0 +1,286 @@
|
||||
using MongoDB.Bson;
|
||||
using MongoDB.Bson.IO;
|
||||
using MongoDB.Driver;
|
||||
using MongoDB.Entities;
|
||||
|
||||
using PrivaPub.Infrastructure.Backup;
|
||||
using PrivaPub.Infrastructure.Cli;
|
||||
using PrivaPub.Tests.Support;
|
||||
using PrivaPub.Tests.Support.Host;
|
||||
|
||||
using System.IO.Compression;
|
||||
|
||||
namespace PrivaPub.Tests.Infrastructure
|
||||
{
|
||||
// The server's backup: every collection as it was, byte for byte, read at one instant on a replica set, without what
|
||||
// belongs to the moment (the statistics salt above all), with the media rows' files linked, checkable and rotated.
|
||||
// Each test backs up a database of its own; alone, since the maintenance lock is the server's one.
|
||||
[Trait("Category", "Integration")]
|
||||
[Xunit.Collection(nameof(Exclusive))]
|
||||
public sealed class BackupTests : IAsyncLifetime
|
||||
{
|
||||
readonly string _scratch = Path.Combine(Path.GetTempPath(), $"privapub-backup-tests-{Guid.NewGuid():N}");
|
||||
IMongoDatabase _database;
|
||||
|
||||
string Backups => Path.Combine(_scratch, "backups");
|
||||
string Media => Path.Combine(_scratch, "media");
|
||||
string Trash => Path.Combine(_scratch, "media-trash");
|
||||
|
||||
public async ValueTask InitializeAsync()
|
||||
{
|
||||
Assert.SkipUnless(MongoFixture.Enabled, MongoFixture.Skip);
|
||||
await PrivaPubHost.Shared();
|
||||
_database = DB.Default.Database().Client.GetDatabase($"PrivaPubBackup_{Guid.NewGuid():N}");
|
||||
Directory.CreateDirectory(Media);
|
||||
Directory.CreateDirectory(Trash);
|
||||
}
|
||||
|
||||
public async ValueTask DisposeAsync()
|
||||
{
|
||||
if (_database != default)
|
||||
await _database.Client.DropDatabaseAsync(_database.DatabaseNamespace.DatabaseName);
|
||||
if (Directory.Exists(_scratch))
|
||||
Directory.Delete(_scratch, recursive: true);
|
||||
}
|
||||
|
||||
BackupContext Context(BackupOptions options = default) => new(_database, Backups, Media, Trash, "https://privapub.test", options ?? new BackupOptions());
|
||||
|
||||
static CancellationToken Token => TestContext.Current.CancellationToken;
|
||||
|
||||
static List<BsonDocument> Read(string file)
|
||||
{
|
||||
using var gzip = new GZipStream(File.OpenRead(file), CompressionMode.Decompress);
|
||||
using var reader = new StreamReader(gzip);
|
||||
var documents = new List<BsonDocument>();
|
||||
while (reader.ReadLine() is { } line)
|
||||
documents.Add(BsonDocument.Parse(line));
|
||||
return documents;
|
||||
}
|
||||
|
||||
async Task<List<byte[]>> Raw(string collection) =>
|
||||
(await (await _database.GetCollection<RawBsonDocument>(collection).FindAsync(FilterDefinition<RawBsonDocument>.Empty, cancellationToken: Token)).ToListAsync(Token))
|
||||
.Select(d =>
|
||||
{
|
||||
var bytes = new byte[d.Slice.Length];
|
||||
d.Slice.GetBytes(0, bytes, 0, bytes.Length);
|
||||
return bytes;
|
||||
})
|
||||
.ToList();
|
||||
|
||||
[Fact]
|
||||
public async Task Every_document_comes_back_byte_for_byte_and_the_salt_never_leaves()
|
||||
{
|
||||
var odd = new BsonDocument
|
||||
{
|
||||
{ "_id", ObjectId.GenerateNewId() },
|
||||
{ "Guid", new BsonBinaryData(Guid.NewGuid(), GuidRepresentation.Standard) },
|
||||
{ "OldGuid", new BsonBinaryData(new byte[16], BsonBinarySubType.UuidLegacy) },
|
||||
{ "Money", new BsonDecimal128(Decimal128.Parse("12345.678900")) },
|
||||
{ "Long", new BsonInt64(long.MaxValue) },
|
||||
{ "Int", 7 },
|
||||
{ "Double", -0.0 },
|
||||
{ "NaN", double.NaN },
|
||||
{ "At", new BsonDateTime(new DateTime(2026, 10, 7, 3, 30, 0, 123, DateTimeKind.Utc)) },
|
||||
{ "Stamp", new BsonTimestamp(1, 2) },
|
||||
{ "Null", BsonNull.Value },
|
||||
{ "Regex", new BsonRegularExpression("^a.b$", "i") },
|
||||
{ "Nested", new BsonDocument { { "z", 1 }, { "a", new BsonArray { 1, "two", new BsonDocument("three", 3) } } } },
|
||||
{ "Unicode", "città 🌍 \u0000 end" }
|
||||
};
|
||||
// an ObjectRecord as big as a remote's long post gets
|
||||
var big = new BsonDocument { { "_id", ObjectId.GenerateNewId() }, { "Json", new string('x', 10 * 1024 * 1024) } };
|
||||
await _database.GetCollection<BsonDocument>("Odd").InsertOneAsync(odd, cancellationToken: Token);
|
||||
await _database.GetCollection<BsonDocument>("ObjectRecord").InsertOneAsync(big, cancellationToken: Token);
|
||||
await _database.GetCollection<BsonDocument>("Odd").Indexes.CreateOneAsync(
|
||||
new CreateIndexModel<BsonDocument>(Builders<BsonDocument>.IndexKeys.Ascending("At"), new CreateIndexOptions { Name = "at", Unique = true }), cancellationToken: Token);
|
||||
await _database.GetCollection<BsonDocument>("InteractionSalt").InsertOneAsync(new BsonDocument("Salt", "never in a backup"), cancellationToken: Token);
|
||||
await _database.GetCollection<BsonDocument>("Job").InsertOneAsync(new BsonDocument("Kind", "Deliver"), cancellationToken: Token);
|
||||
await _database.GetCollection<BsonDocument>("_migration_history_").InsertManyAsync([
|
||||
new BsonDocument { { "Number", 15 }, { "Name", "older" } }, new BsonDocument { { "Number", 16 }, { "Name", "focal points are finite" } }], cancellationToken: Token);
|
||||
|
||||
var (backup, error) = await ServerBackup.Create(Context(), "manual", dbOnly: false, Token);
|
||||
|
||||
Assert.Null(error);
|
||||
var directory = Path.Combine(Backups, backup.Id);
|
||||
Assert.False(Directory.Exists(directory + ServerBackup.PartialSuffix));
|
||||
Assert.Equal(await Raw("Odd"), Read(Path.Combine(directory, "db", "Odd.jsonl.gz")).Select(d => d.ToBson()).ToList());
|
||||
Assert.Equal(await Raw("ObjectRecord"), Read(Path.Combine(directory, "db", "ObjectRecord.jsonl.gz")).Select(d => d.ToBson()).ToList());
|
||||
|
||||
Assert.False(File.Exists(Path.Combine(directory, "db", "InteractionSalt.jsonl.gz")));
|
||||
Assert.False(File.Exists(Path.Combine(directory, "db", "Job.jsonl.gz")));
|
||||
foreach (var file in Directory.EnumerateFiles(directory, "*", SearchOption.AllDirectories))
|
||||
Assert.DoesNotContain("never in a backup", await ReadAll(file));
|
||||
Assert.Contains(backup.Manifest.Excluded, e => e.Name == "InteractionSalt");
|
||||
|
||||
var odds = backup.Manifest.Collections.Single(c => c.Name == "Odd");
|
||||
Assert.Equal(1, odds.Count);
|
||||
Assert.Contains(odds.Indexes, i => BsonDocument.Parse(i)["name"] == "at" && BsonDocument.Parse(i)["unique"].ToBoolean());
|
||||
Assert.Equal((16, "focal points are finite"), (backup.Manifest.MigrationNumber, backup.Manifest.NewestMigration));
|
||||
Assert.Equal("https://privapub.test", backup.Manifest.Host);
|
||||
Assert.Equal(MongoFixture.Connection.Contains("directConnection=true"), backup.Manifest.Consistent);
|
||||
Assert.Empty(await ServerBackup.Verify(Backups, backup.Id, Token));
|
||||
if (!OperatingSystem.IsWindows())
|
||||
{
|
||||
Assert.Equal(UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.GroupRead, File.GetUnixFileMode(Path.Combine(directory, "db", "Odd.jsonl.gz")));
|
||||
Assert.False(File.GetUnixFileMode(directory).HasFlag(UnixFileMode.OtherRead));
|
||||
}
|
||||
}
|
||||
|
||||
static async Task<string> ReadAll(string file)
|
||||
{
|
||||
if (!file.EndsWith(".gz", StringComparison.Ordinal))
|
||||
return await File.ReadAllTextAsync(file, Token);
|
||||
await using var gzip = new GZipStream(File.OpenRead(file), CompressionMode.Decompress);
|
||||
using var reader = new StreamReader(gzip);
|
||||
return await reader.ReadToEndAsync(Token);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Media_rows_files_are_linked_from_the_live_directory_or_the_trash()
|
||||
{
|
||||
Directory.CreateDirectory(Path.Combine(Media, "2026", "10"));
|
||||
await File.WriteAllTextAsync(Path.Combine(Media, "2026", "10", "live.jpg"), "live", Token);
|
||||
await File.WriteAllTextAsync(Path.Combine(Media, "2026", "10", "live-preview.jpg"), "preview", Token);
|
||||
Directory.CreateDirectory(Path.Combine(Trash, "2026", "10"));
|
||||
await File.WriteAllTextAsync(Path.Combine(Trash, "2026", "10", "moving.jpg"), "moving", Token);
|
||||
await File.WriteAllTextAsync(Path.Combine(Media, "2026", "10", "trashed.jpg"), "trashed", Token);
|
||||
await _database.GetCollection<BsonDocument>("MediaAttachment").InsertManyAsync([
|
||||
new BsonDocument { { "FilePath", "2026/10/live.jpg" }, { "PreviewPath", "2026/10/live-preview.jpg" }, { "TrashedAt", BsonNull.Value } },
|
||||
new BsonDocument { { "FilePath", "2026/10/moving.jpg" } },//a file the janitor moved while its row was being trashed
|
||||
new BsonDocument { { "FilePath", "2026/10/gone.jpg" } },
|
||||
new BsonDocument { { "FilePath", "2026/10/trashed.jpg" }, { "TrashedAt", DateTime.UtcNow } },
|
||||
new BsonDocument { { "FilePath", "../../etc/passwd" } }], cancellationToken: Token);
|
||||
|
||||
var (backup, _) = await ServerBackup.Create(Context(), "manual", dbOnly: false, Token);
|
||||
|
||||
var media = Path.Combine(Backups, backup.Id, "media");
|
||||
Assert.Equal("live", await File.ReadAllTextAsync(Path.Combine(media, "2026", "10", "live.jpg"), Token));
|
||||
Assert.Equal("preview", await File.ReadAllTextAsync(Path.Combine(media, "2026", "10", "live-preview.jpg"), Token));
|
||||
Assert.Equal("moving", await File.ReadAllTextAsync(Path.Combine(media, "2026", "10", "moving.jpg"), Token));
|
||||
Assert.False(File.Exists(Path.Combine(media, "2026", "10", "trashed.jpg")));
|
||||
Assert.Equal(["2026/10/gone.jpg", "2026/10/live-preview.jpg", "2026/10/live.jpg", "2026/10/moving.jpg"], backup.Manifest.Media.List);
|
||||
Assert.Equal((3, 1), (backup.Manifest.Media.Files, backup.Manifest.Media.Missing));
|
||||
|
||||
// a link, not a copy: the live file deleted, the backup's stays
|
||||
File.Delete(Path.Combine(Media, "2026", "10", "live.jpg"));
|
||||
Assert.Equal("live", await File.ReadAllTextAsync(Path.Combine(media, "2026", "10", "live.jpg"), Token));
|
||||
|
||||
// db-only lists them and links none
|
||||
var (listed, _) = await ServerBackup.Create(Context(), "pre-deploy", dbOnly: true, Token);
|
||||
Assert.False(Directory.Exists(Path.Combine(Backups, listed.Id, "media")));
|
||||
Assert.Equal(backup.Manifest.Media.List, listed.Manifest.Media.List);
|
||||
Assert.Empty(await ServerBackup.Verify(Backups, listed.Id, Token));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Verify_finds_an_altered_or_missing_file()
|
||||
{
|
||||
await _database.GetCollection<BsonDocument>("Post").InsertOneAsync(new BsonDocument("Content", "hello"), cancellationToken: Token);
|
||||
await _database.GetCollection<BsonDocument>("Avatar").InsertOneAsync(new BsonDocument("UserName", "someone"), cancellationToken: Token);
|
||||
var (backup, _) = await ServerBackup.Create(Context(), "manual", dbOnly: false, Token);
|
||||
var db = Path.Combine(Backups, backup.Id, "db");
|
||||
|
||||
await File.AppendAllTextAsync(Path.Combine(db, "Post.jsonl.gz"), "tampered", Token);
|
||||
File.Delete(Path.Combine(db, "Avatar.jsonl.gz"));
|
||||
|
||||
Assert.Equal(["Avatar: missing", "Post: altered"], (await ServerBackup.Verify(Backups, backup.Id, Token)).Order());
|
||||
Assert.Equal(["no such backup, or no manifest"], await ServerBackup.Verify(Backups, "../" + backup.Id, Token));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task One_backup_at_a_time()
|
||||
{
|
||||
await using (var held = await MaintenanceLock.Take("restore", Token))
|
||||
{
|
||||
Assert.NotNull(held);
|
||||
var (backup, error) = await ServerBackup.Create(Context(), "manual", dbOnly: true, Token);
|
||||
Assert.Null(backup);
|
||||
Assert.Contains("already running", error);
|
||||
Assert.Equal("restore", (await MaintenanceLock.Current(Token)).What);
|
||||
}
|
||||
Assert.Null(await MaintenanceLock.Current(Token));
|
||||
Assert.NotNull((await ServerBackup.Create(Context(), "manual", dbOnly: true, Token)).Backup);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_holder_that_died_is_taken_over()
|
||||
{
|
||||
await using var dead = await MaintenanceLock.Take("backup", Token);
|
||||
await DB.Default.Update<MaintenanceLock>().Match(l => l.ID == MaintenanceLock.Name)
|
||||
.Modify(l => l.Heartbeat, DateTime.UtcNow.AddMinutes(-3)).ExecuteAsync(Token);
|
||||
|
||||
Assert.Null(await MaintenanceLock.Current(Token));
|
||||
await using var alive = await MaintenanceLock.Take("restore", Token);
|
||||
Assert.NotNull(alive);
|
||||
Assert.Equal("restore", (await MaintenanceLock.Current(Token)).What);
|
||||
}
|
||||
|
||||
// the newest 7 nightly, the newest of each of the 4 weeks before, the newest 3 pre-deploy; manual ones kept
|
||||
[Fact]
|
||||
public void Rotation_keeps_days_weeks_and_the_last_deploys()
|
||||
{
|
||||
var today = new DateTime(2026, 10, 7, 3, 30, 0, DateTimeKind.Utc);
|
||||
for (var day = 0; day < 60; day++)
|
||||
Fake("nightly", today.AddDays(-day));
|
||||
for (var deploy = 0; deploy < 5; deploy++)
|
||||
Fake("pre-deploy", today.AddDays(-deploy).AddHours(5));
|
||||
Fake("manual", today.AddYears(-1));
|
||||
Directory.CreateDirectory(Path.Combine(Backups, "20260901-000000-nightly" + ServerBackup.PartialSuffix));
|
||||
Directory.SetLastWriteTimeUtc(Path.Combine(Backups, "20260901-000000-nightly" + ServerBackup.PartialSuffix), today.AddDays(-30));
|
||||
|
||||
ServerBackup.Retain(Backups, new BackupOptions());
|
||||
|
||||
var kept = ServerBackup.List(Backups);
|
||||
var nightly = kept.Where(b => b.Kind == "nightly").Select(b => b.CreatedAt).ToList();
|
||||
Assert.Equal(11, nightly.Count);
|
||||
Assert.Equal(Enumerable.Range(0, 7).Select(d => today.AddDays(-d)), nightly.Take(7));
|
||||
Assert.Equal(4, nightly.Skip(7).Select(d => System.Globalization.ISOWeek.GetWeekOfYear(d)).Distinct().Count());
|
||||
Assert.Equal(3, kept.Count(b => b.Kind == "pre-deploy"));
|
||||
Assert.Single(kept, b => b.Kind == "manual");
|
||||
Assert.Empty(Directory.EnumerateDirectories(Backups, "*" + ServerBackup.PartialSuffix));
|
||||
}
|
||||
|
||||
void Fake(string kind, DateTime at)
|
||||
{
|
||||
var directory = Path.Combine(Backups, $"{at:yyyyMMdd-HHmmss}-{kind}");
|
||||
Directory.CreateDirectory(Path.Combine(directory, "db"));
|
||||
new ArchiveManifest { Kind = kind, CreatedAt = at }.Write(directory);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData("2026-10-07T03:29:00", "2026-10-06T03:31:00", false)]//yesterday's is the last one due
|
||||
[InlineData("2026-10-07T03:31:00", "2026-10-06T03:31:00", true)]
|
||||
[InlineData("2026-10-07T03:31:00", "2026-10-07T03:30:30", false)]
|
||||
[InlineData("2026-10-07T01:00:00", "2026-10-05T03:31:00", true)]//missed last night: at once
|
||||
public void A_nightly_backup_is_due_once_a_night(string now, string last, bool due) =>
|
||||
Assert.Equal(due, BackupScheduler.IsDue(DateTime.Parse(now, null, System.Globalization.DateTimeStyles.AdjustToUniversal),
|
||||
new TimeOnly(3, 30), [DateTime.Parse(last, null, System.Globalization.DateTimeStyles.AdjustToUniversal)]));
|
||||
|
||||
// the deploy's: the server's own database, through the configuration, without media links
|
||||
[Fact]
|
||||
public async Task The_deploy_backs_up_from_the_command_line()
|
||||
{
|
||||
var host = await PrivaPubHost.Shared();
|
||||
var output = new StringWriter();
|
||||
|
||||
Assert.Equal(0, await AdminCommands.Run(["backup", "--kind", "pre-deploy", "--db-only"], host.Services, output: output));
|
||||
var id = output.ToString().Split(':')[0];
|
||||
Assert.EndsWith("-pre-deploy", id);
|
||||
Assert.Equal(2, await AdminCommands.Run(["backup", "--kind", "nightly"], host.Services, output: TextWriter.Null));
|
||||
|
||||
output = new StringWriter();
|
||||
Assert.Equal(0, await AdminCommands.Run(["backups"], host.Services, output: output));
|
||||
Assert.StartsWith(id + "\tpre-deploy", output.ToString());
|
||||
output = new StringWriter();
|
||||
Assert.Equal(0, await AdminCommands.Run(["backup", "verify", id], host.Services, output: output));
|
||||
Assert.Contains("whole", output.ToString());
|
||||
|
||||
var backups = host.Get<Backups>();
|
||||
var manifest = backups.Find(id).Manifest;
|
||||
Assert.Contains(manifest.Collections, c => c.Name == "Avatar");
|
||||
Assert.DoesNotContain(manifest.Collections, c => c.Name is "InteractionSalt" or "Job" or "MaintenanceLock" or "openiddict.tokens" or "AppConfiguration");
|
||||
Assert.Equal(ServerBackup.CodeMigration(), manifest.MigrationNumber);
|
||||
Assert.True(backups.Delete(id));
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user