SocialPub/PrivaPub.Tests/Infrastructure/BackupTests.cs

using MongoDB.Bson;
using MongoDB.Bson.IO;
using MongoDB.Driver;
using MongoDB.Entities;

using PrivaPub.Infrastructure.Backup;
using PrivaPub.Infrastructure.Cli;
using PrivaPub.Tests.Support;
using PrivaPub.Tests.Support.Host;

using System.IO.Compression;

namespace PrivaPub.Tests.Infrastructure
{
	// The server's backup: every collection as it was, byte for byte, read at one instant on a replica set, without what
	// belongs to the moment (the statistics salt above all), with the media rows' files linked, checkable and rotated.
	// Each test backs up a database of its own; alone, since the maintenance lock is the server's one.
	[Trait("Category", "Integration")]
	[Xunit.Collection(nameof(Exclusive))]
	public sealed class BackupTests : IAsyncLifetime
	{
		readonly string _scratch = Path.Combine(Path.GetTempPath(), $"privapub-backup-tests-{Guid.NewGuid():N}");
		IMongoDatabase _database;

		string Backups => Path.Combine(_scratch, "backups");
		string Media => Path.Combine(_scratch, "media");
		string Trash => Path.Combine(_scratch, "media-trash");

		public async ValueTask InitializeAsync()
		{
			Assert.SkipUnless(MongoFixture.Enabled, MongoFixture.Skip);
			await PrivaPubHost.Shared();
			_database = DB.Default.Database().Client.GetDatabase($"PrivaPubBackup_{Guid.NewGuid():N}");
			Directory.CreateDirectory(Media);
			Directory.CreateDirectory(Trash);
		}

		public async ValueTask DisposeAsync()
		{
			if (_database != default)
				await _database.Client.DropDatabaseAsync(_database.DatabaseNamespace.DatabaseName);
			if (Directory.Exists(_scratch))
				Directory.Delete(_scratch, recursive: true);
		}

		BackupContext Context(BackupOptions options = default) => new(_database, Backups, Media, Trash, "https://privapub.test", options ?? new BackupOptions());

		static CancellationToken Token => TestContext.Current.CancellationToken;

		static List<BsonDocument> Read(string file)
		{
			using var gzip = new GZipStream(File.OpenRead(file), CompressionMode.Decompress);
			using var reader = new StreamReader(gzip);
			var documents = new List<BsonDocument>();
			while (reader.ReadLine() is { } line)
				documents.Add(BsonDocument.Parse(line));
			return documents;
		}

		async Task<List<byte[]>> Raw(string collection) =>
			(await (await _database.GetCollection<RawBsonDocument>(collection).FindAsync(FilterDefinition<RawBsonDocument>.Empty, cancellationToken: Token)).ToListAsync(Token))
				.Select(d =>
				{
					var bytes = new byte[d.Slice.Length];
					d.Slice.GetBytes(0, bytes, 0, bytes.Length);
					return bytes;
				})
				.ToList();

		[Fact]
		public async Task Every_document_comes_back_byte_for_byte_and_the_salt_never_leaves()
		{
			var odd = new BsonDocument
			{
				{ "_id", ObjectId.GenerateNewId() },
				{ "Guid", new BsonBinaryData(Guid.NewGuid(), GuidRepresentation.Standard) },
				{ "OldGuid", new BsonBinaryData(new byte[16], BsonBinarySubType.UuidLegacy) },
				{ "Money", new BsonDecimal128(Decimal128.Parse("12345.678900")) },
				{ "Long", new BsonInt64(long.MaxValue) },
				{ "Int", 7 },
				{ "Double", -0.0 },
				{ "NaN", double.NaN },
				{ "At", new BsonDateTime(new DateTime(2026, 10, 7, 3, 30, 0, 123, DateTimeKind.Utc)) },
				{ "Stamp", new BsonTimestamp(1, 2) },
				{ "Null", BsonNull.Value },
				{ "Regex", new BsonRegularExpression("^a.b$", "i") },
				{ "Nested", new BsonDocument { { "z", 1 }, { "a", new BsonArray { 1, "two", new BsonDocument("three", 3) } } } },
				{ "Unicode", "città 🌍 \u0000 end" }
			};
			// an ObjectRecord as big as a remote's long post gets
			var big = new BsonDocument { { "_id", ObjectId.GenerateNewId() }, { "Json", new string('x', 10 * 1024 * 1024) } };
			await _database.GetCollection<BsonDocument>("Odd").InsertOneAsync(odd, cancellationToken: Token);
			await _database.GetCollection<BsonDocument>("ObjectRecord").InsertOneAsync(big, cancellationToken: Token);
			await _database.GetCollection<BsonDocument>("Odd").Indexes.CreateOneAsync(
				new CreateIndexModel<BsonDocument>(Builders<BsonDocument>.IndexKeys.Ascending("At"), new CreateIndexOptions { Name = "at", Unique = true }), cancellationToken: Token);
			await _database.GetCollection<BsonDocument>("InteractionSalt").InsertOneAsync(new BsonDocument("Salt", "never in a backup"), cancellationToken: Token);
			await _database.GetCollection<BsonDocument>("Job").InsertOneAsync(new BsonDocument("Kind", "Deliver"), cancellationToken: Token);
			await _database.GetCollection<BsonDocument>("_migration_history_").InsertManyAsync([
				new BsonDocument { { "Number", 15 }, { "Name", "older" } }, new BsonDocument { { "Number", 16 }, { "Name", "focal points are finite" } }], cancellationToken: Token);

			var (backup, error) = await ServerBackup.Create(Context(), "manual", dbOnly: false, Token);

			Assert.Null(error);
			var directory = Path.Combine(Backups, backup.Id);
			Assert.False(Directory.Exists(directory + ServerBackup.PartialSuffix));
			Assert.Equal(await Raw("Odd"), Read(Path.Combine(directory, "db", "Odd.jsonl.gz")).Select(d => d.ToBson()).ToList());
			Assert.Equal(await Raw("ObjectRecord"), Read(Path.Combine(directory, "db", "ObjectRecord.jsonl.gz")).Select(d => d.ToBson()).ToList());

			Assert.False(File.Exists(Path.Combine(directory, "db", "InteractionSalt.jsonl.gz")));
			Assert.False(File.Exists(Path.Combine(directory, "db", "Job.jsonl.gz")));
			foreach (var file in Directory.EnumerateFiles(directory, "*", SearchOption.AllDirectories))
				Assert.DoesNotContain("never in a backup", await ReadAll(file));
			Assert.Contains(backup.Manifest.Excluded, e => e.Name == "InteractionSalt");

			var odds = backup.Manifest.Collections.Single(c => c.Name == "Odd");
			Assert.Equal(1, odds.Count);
			Assert.Contains(odds.Indexes, i => BsonDocument.Parse(i)["name"] == "at" && BsonDocument.Parse(i)["unique"].ToBoolean());
			Assert.Equal((16, "focal points are finite"), (backup.Manifest.MigrationNumber, backup.Manifest.NewestMigration));
			Assert.Equal("https://privapub.test", backup.Manifest.Host);
			Assert.Equal(MongoFixture.Connection.Contains("directConnection=true"), backup.Manifest.Consistent);
			Assert.Empty(await ServerBackup.Verify(Backups, backup.Id, Token));
			if (!OperatingSystem.IsWindows())
			{
				Assert.Equal(UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.GroupRead, File.GetUnixFileMode(Path.Combine(directory, "db", "Odd.jsonl.gz")));
				Assert.False(File.GetUnixFileMode(directory).HasFlag(UnixFileMode.OtherRead));
			}
		}

		static async Task<string> ReadAll(string file)
		{
			if (!file.EndsWith(".gz", StringComparison.Ordinal))
				return await File.ReadAllTextAsync(file, Token);
			await using var gzip = new GZipStream(File.OpenRead(file), CompressionMode.Decompress);
			using var reader = new StreamReader(gzip);
			return await reader.ReadToEndAsync(Token);
		}

		[Fact]
		public async Task Media_rows_files_are_linked_from_the_live_directory_or_the_trash()
		{
			Directory.CreateDirectory(Path.Combine(Media, "2026", "10"));
			await File.WriteAllTextAsync(Path.Combine(Media, "2026", "10", "live.jpg"), "live", Token);
			await File.WriteAllTextAsync(Path.Combine(Media, "2026", "10", "live-preview.jpg"), "preview", Token);
			Directory.CreateDirectory(Path.Combine(Trash, "2026", "10"));
			await File.WriteAllTextAsync(Path.Combine(Trash, "2026", "10", "moving.jpg"), "moving", Token);
			await File.WriteAllTextAsync(Path.Combine(Media, "2026", "10", "trashed.jpg"), "trashed", Token);
			await _database.GetCollection<BsonDocument>("MediaAttachment").InsertManyAsync([
				new BsonDocument { { "FilePath", "2026/10/live.jpg" }, { "PreviewPath", "2026/10/live-preview.jpg" }, { "TrashedAt", BsonNull.Value } },
				new BsonDocument { { "FilePath", "2026/10/moving.jpg" } },//a file the janitor moved while its row was being trashed
				new BsonDocument { { "FilePath", "2026/10/gone.jpg" } },
				new BsonDocument { { "FilePath", "2026/10/trashed.jpg" }, { "TrashedAt", DateTime.UtcNow } },
				new BsonDocument { { "FilePath", "../../etc/passwd" } }], cancellationToken: Token);

			var (backup, _) = await ServerBackup.Create(Context(), "manual", dbOnly: false, Token);

			var media = Path.Combine(Backups, backup.Id, "media");
			Assert.Equal("live", await File.ReadAllTextAsync(Path.Combine(media, "2026", "10", "live.jpg"), Token));
			Assert.Equal("preview", await File.ReadAllTextAsync(Path.Combine(media, "2026", "10", "live-preview.jpg"), Token));
			Assert.Equal("moving", await File.ReadAllTextAsync(Path.Combine(media, "2026", "10", "moving.jpg"), Token));
			Assert.False(File.Exists(Path.Combine(media, "2026", "10", "trashed.jpg")));
			Assert.Equal(["2026/10/live-preview.jpg", "2026/10/live.jpg", "2026/10/moving.jpg"], backup.Manifest.Media.List);
			Assert.Equal((3, 1), (backup.Manifest.Media.Files, backup.Manifest.Media.Missing));

			// a link, not a copy: the live file deleted, the backup's stays
			File.Delete(Path.Combine(Media, "2026", "10", "live.jpg"));
			Assert.Equal("live", await File.ReadAllTextAsync(Path.Combine(media, "2026", "10", "live.jpg"), Token));

			// db-only lists them and links none
			var (listed, _) = await ServerBackup.Create(Context(), "pre-deploy", dbOnly: true, Token);
			Assert.False(Directory.Exists(Path.Combine(Backups, listed.Id, "media")));
			Assert.Equal(["2026/10/live-preview.jpg", "2026/10/moving.jpg"], listed.Manifest.Media.List);
			Assert.Empty(await ServerBackup.Verify(Backups, listed.Id, Token));
			Assert.Empty(await ServerBackup.Verify(Backups, backup.Id, Token));
		}

		[Fact]
		public async Task Verify_finds_an_altered_or_missing_file()
		{
			await _database.GetCollection<BsonDocument>("Post").InsertOneAsync(new BsonDocument("Content", "hello"), cancellationToken: Token);
			await _database.GetCollection<BsonDocument>("Avatar").InsertOneAsync(new BsonDocument("UserName", "someone"), cancellationToken: Token);
			var (backup, _) = await ServerBackup.Create(Context(), "manual", dbOnly: false, Token);
			var db = Path.Combine(Backups, backup.Id, "db");

			await File.AppendAllTextAsync(Path.Combine(db, "Post.jsonl.gz"), "tampered", Token);
			File.Delete(Path.Combine(db, "Avatar.jsonl.gz"));

			Assert.Equal(["Avatar: missing", "Post: altered"], (await ServerBackup.Verify(Backups, backup.Id, Token)).Order());
			Assert.Equal(["no such backup, or no manifest"], await ServerBackup.Verify(Backups, "../" + backup.Id, Token));
		}

		[Fact]
		public async Task One_backup_at_a_time()
		{
			await using (var held = await MaintenanceLock.Take("restore", Token))
			{
				Assert.NotNull(held);
				var (backup, error) = await ServerBackup.Create(Context(), "manual", dbOnly: true, Token);
				Assert.Null(backup);
				Assert.Contains("already running", error);
				Assert.Equal("restore", (await MaintenanceLock.Current(Token)).What);
			}
			Assert.Null(await MaintenanceLock.Current(Token));
			Assert.NotNull((await ServerBackup.Create(Context(), "manual", dbOnly: true, Token)).Backup);
		}

		[Fact]
		public async Task A_holder_that_died_is_taken_over()
		{
			await using var dead = await MaintenanceLock.Take("backup", Token);
			await DB.Default.Update<MaintenanceLock>().Match(l => l.ID == MaintenanceLock.Name)
				.Modify(l => l.Heartbeat, DateTime.UtcNow.AddMinutes(-3)).ExecuteAsync(Token);

			Assert.Null(await MaintenanceLock.Current(Token));
			await using var alive = await MaintenanceLock.Take("restore", Token);
			Assert.NotNull(alive);
			Assert.Equal("restore", (await MaintenanceLock.Current(Token)).What);
		}

		// the newest 7 nightly, the newest of each of the 4 weeks before, the newest 3 pre-deploy; manual ones kept
		[Fact]
		public void Rotation_keeps_days_weeks_and_the_last_deploys()
		{
			var today = new DateTime(2026, 10, 7, 3, 30, 0, DateTimeKind.Utc);
			for (var day = 0; day < 60; day++)
				Fake("nightly", today.AddDays(-day));
			for (var deploy = 0; deploy < 5; deploy++)
				Fake("pre-deploy", today.AddDays(-deploy).AddHours(5));
			Fake("manual", today.AddYears(-1));
			Directory.CreateDirectory(Path.Combine(Backups, "20260901-000000-nightly" + ServerBackup.PartialSuffix));
			Directory.SetLastWriteTimeUtc(Path.Combine(Backups, "20260901-000000-nightly" + ServerBackup.PartialSuffix), today.AddDays(-30));

			ServerBackup.Retain(Backups, new BackupOptions());

			var kept = ServerBackup.List(Backups);
			var nightly = kept.Where(b => b.Kind == "nightly").Select(b => b.CreatedAt).ToList();
			Assert.Equal(11, nightly.Count);
			Assert.Equal(Enumerable.Range(0, 7).Select(d => today.AddDays(-d)), nightly.Take(7));
			Assert.Equal(4, nightly.Skip(7).Select(d => System.Globalization.ISOWeek.GetWeekOfYear(d)).Distinct().Count());
			Assert.Equal(3, kept.Count(b => b.Kind == "pre-deploy"));
			Assert.Single(kept, b => b.Kind == "manual");
			Assert.Empty(Directory.EnumerateDirectories(Backups, "*" + ServerBackup.PartialSuffix));
		}

		void Fake(string kind, DateTime at)
		{
			var directory = Path.Combine(Backups, $"{at:yyyyMMdd-HHmmss}-{kind}");
			Directory.CreateDirectory(Path.Combine(directory, "db"));
			new ArchiveManifest { Kind = kind, CreatedAt = at }.Write(directory);
		}

		[Theory]
		[InlineData("2026-10-07T03:29:00", "2026-10-06T03:31:00", false)]//yesterday's is the last one due
		[InlineData("2026-10-07T03:31:00", "2026-10-06T03:31:00", true)]
		[InlineData("2026-10-07T03:31:00", "2026-10-07T03:30:30", false)]
		[InlineData("2026-10-07T01:00:00", "2026-10-05T03:31:00", true)]//missed last night: at once
		public void A_nightly_backup_is_due_once_a_night(string now, string last, bool due) =>
			Assert.Equal(due, BackupScheduler.IsDue(DateTime.Parse(now, null, System.Globalization.DateTimeStyles.AdjustToUniversal),
				new TimeOnly(3, 30), [DateTime.Parse(last, null, System.Globalization.DateTimeStyles.AdjustToUniversal)]));

		// the deploy's: the server's own database, through the configuration, without media links
		[Fact]
		public async Task The_deploy_backs_up_from_the_command_line()
		{
			var host = await PrivaPubHost.Shared();
			var output = new StringWriter();

			Assert.Equal(0, await AdminCommands.Run(["backup", "--kind", "pre-deploy", "--db-only"], host.Services, output: output));
			var id = output.ToString().Split(':')[0];
			Assert.EndsWith("-pre-deploy", id);
			Assert.Equal(2, await AdminCommands.Run(["backup", "--kind", "nightly"], host.Services, output: TextWriter.Null));

			output = new StringWriter();
			Assert.Equal(0, await AdminCommands.Run(["backups"], host.Services, output: output));
			Assert.StartsWith(id + "\tpre-deploy", output.ToString());
			output = new StringWriter();
			Assert.Equal(0, await AdminCommands.Run(["backup", "verify", id], host.Services, output: output));
			Assert.Contains("whole", output.ToString());

			var backups = host.Get<Backups>();
			var manifest = backups.Find(id).Manifest;
			Assert.Contains(manifest.Collections, c => c.Name == "Avatar");
			Assert.DoesNotContain(manifest.Collections, c => c.Name is "InteractionSalt" or "Job" or "MaintenanceLock" or "openiddict.tokens" or "AppConfiguration");
			Assert.Equal(ServerBackup.CodeMigration(), manifest.MigrationNumber);
			Assert.True(backups.Delete(id));
		}
	}
}