diff --git a/CLAUDE.md b/CLAUDE.md index ad8e074..a7f4c33 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -526,6 +526,19 @@ group www-data and reaches the private mongod; `sudo -u www-data` works too. it was; failed midway, the process exits 1 and the next start tries again; after 3 failures it exits 75, which the unit's `RestartPreventExitStatus` leaves down for someone to look. While `restore.json` exists, commands exit 75 (but `admin restore --status`) and the deploy refuses to run. +- **The administrator's page** (`BackupController`, `/clientapi/admin/backups`; owner decision 2026-10-07, which approved + these endpoints in production): the list (with what runs, the restore waiting, the last restore's report, logins + deleted since each backup), back up now (202; the page polls), delete, and: + - **download** for the password: a ticket good for ten minutes in the link's path (`/download/{ticket}`, anonymous, so a + browser saves it to disk), the backup as one tar (`BackupTar`: its exact length known first, written from the + files, `Range` honoured); + - **upload** in pieces of at most 32 MB (`TransferStore`): each `PUT ?offset=` must start where the upload stands, else + 409 with what was received, so a broken upload resumes; `finish` reads it into a backup (`kind` uploaded), refusing + anything but plain files under one backup's folder, a path leaving it, or more than the disk holds; + - **restore** for the password and the server's host typed out. + + Every call checks the administrator against the database, not only the token. nginx streams downloads for an hour and + takes upload pieces unbuffered (`deploy/nginx`, applied by `setup.sh`). ## Code style diff --git a/PrivaPub.ClientModels/Admin/ViewBackups.cs b/PrivaPub.ClientModels/Admin/ViewBackups.cs new file mode 100644 index 0000000..c898a4f --- /dev/null +++ b/PrivaPub.ClientModels/Admin/ViewBackups.cs @@ -0,0 +1,103 @@ +using PrivaPub.ClientModels.Resources; + +using System.ComponentModel.DataAnnotations; + +namespace PrivaPub.ClientModels.Admin +{ + // the server's backups as the administrator's page shows them, with what is running and the last restore + public class ViewBackups + { + public List Backups { get; set; } = []; + public string Running { get; set; }//"backup" or "restore" while one runs + public ViewRestoreWaiting Waiting { get; set; } + public ViewRestore LastRestore { get; set; } + public string Host { get; set; }//what a restore asks to be typed + public long FreeBytes { get; set; } + } + + public class ViewBackup + { + public string Id { get; set; } + public string Kind { get; set; }//nightly, manual, pre-deploy, pre-restore, uploaded + public DateTime CreatedAt { get; set; } + public long Bytes { get; set; }//the database's files + public long MediaBytes { get; set; } + public int MediaFiles { get; set; } + public int MediaMissing { get; set; } + public bool DbOnly { get; set; } + public bool Consistent { get; set; } + public int Collections { get; set; } + public long Documents { get; set; } + public string AppCommit { get; set; } + public int MigrationNumber { get; set; } + public int RootsDeletedSince { get; set; }//logins deleted after it, which a restore deletes again + public string NotRestorable { get; set; }//why not, from its manifest; hashes are checked when asked + } + + public class ViewRestoreWaiting + { + public string Backup { get; set; } + public string State { get; set; } + public int Attempts { get; set; } + public string RequestedBy { get; set; } + public DateTime RequestedAt { get; set; } + public string Error { get; set; } + } + + public class ViewRestore + { + public string Backup { get; set; } + public DateTime BackupCreatedAt { get; set; } + public string PreRestore { get; set; } + public string RequestedBy { get; set; } + public DateTime RestoredAt { get; set; } + public bool Abandoned { get; set; } + public string Error { get; set; } + public long Documents { get; set; } + public int Collections { get; set; } + public int MediaRestored { get; set; } + public int MediaMissing { get; set; } + public int RootsDeleted { get; set; } + public int PersonasDeleted { get; set; } + public int GroupsDeleted { get; set; } + public int PostsDeleted { get; set; } + public List RootsTombstoned { get; set; } = []; + public List PersonasTombstoned { get; set; } = []; + public List GroupsTombstoned { get; set; } = []; + public int PostsGone { get; set; } + public int MediaTrashed { get; set; } + public int ProtectiveKept { get; set; } + public int SessionsEnded { get; set; } + } + + // a backup downloaded or restored: the administrator's password again, and for a restore the server's host typed out + public class BackupPasswordForm + { + [Required(ErrorMessageResourceName = "Required", ErrorMessageResourceType = typeof(ErrorsResource)), + StringLength(200, ErrorMessageResourceName = "MaxLengthString", ErrorMessageResourceType = typeof(ErrorsResource))] + public string Password { get; set; } + } + + public class RestoreBackupForm : BackupPasswordForm + { + [Required(ErrorMessageResourceName = "Required", ErrorMessageResourceType = typeof(ErrorsResource)), + StringLength(253, ErrorMessageResourceName = "MaxLengthString", ErrorMessageResourceType = typeof(ErrorsResource))] + public string Host { get; set; } + } + + // a download link good for a while, for one backup + public class ViewBackupTicket + { + public string Url { get; set; } + public DateTime ExpiresAt { get; set; } + public long Bytes { get; set; } + } + + // a backup being uploaded in pieces: how much has arrived + public class ViewBackupUpload + { + public string Id { get; set; } + public long Received { get; set; } + public long ChunkBytes { get; set; }//the largest piece accepted + } +} diff --git a/PrivaPub.Tests/Http/BackupEndpointTests.cs b/PrivaPub.Tests/Http/BackupEndpointTests.cs new file mode 100644 index 0000000..c0ff2bf --- /dev/null +++ b/PrivaPub.Tests/Http/BackupEndpointTests.cs @@ -0,0 +1,200 @@ +using MongoDB.Entities; + +using PrivaPub.ClientModels; +using PrivaPub.Infrastructure.Backup; +using PrivaPub.Models.User; +using PrivaPub.Tests.Support; +using PrivaPub.Tests.Support.Host; + +using System.Formats.Tar; +using System.Net; +using System.Net.Http.Headers; +using System.Net.Http.Json; +using System.Text.Json.Nodes; + +namespace PrivaPub.Tests.Http +{ + // The administrator's page backs up, downloads, uploads and restores (owner decision 2026-10-07): only an + // administrator the database still says is one; a download and a restore ask for the password again, a restore for + // the host typed out; an upload resumes where it stopped, and one that would write outside its backup is refused. + [Trait("Category", "Integration")] + [Xunit.Collection(nameof(Exclusive))] + public sealed class BackupEndpointTests : IAsyncLifetime + { + const string Base = "/clientapi/admin/backups"; + PrivaPubHost _host; + + static CancellationToken Token => TestContext.Current.CancellationToken; + + public async ValueTask InitializeAsync() + { + Assert.SkipUnless(MongoFixture.Enabled, MongoFixture.Skip); + _host = await PrivaPubHost.Shared(); + } + + public ValueTask DisposeAsync() + { + RestoreMarker.Clear(_host?.Get().Root ?? Path.GetTempPath()); + return ValueTask.CompletedTask; + } + + async Task BackedUp(HttpClient admin) + { + var before = _host.Get().List().Select(b => b.Id).ToHashSet(); + Assert.Equal(HttpStatusCode.Accepted, (await admin.PostAsync(Base, null, Token)).StatusCode); + var made = default(string); + Assert.True(await Wait(async () => + { + var list = (await admin.GetFromJsonAsync(Base, Token))!; + made = list["backups"]!.AsArray().Select(b => b!["id"]!.GetValue()).FirstOrDefault(id => !before.Contains(id)); + return made != default && list["running"] is null; + }), "the backup was never made"); + return made; + } + + static async Task Wait(Func> done) + { + for (var i = 0; i < 120; i++) + { + if (await done()) + return true; + await Task.Delay(250, Token); + } + return false; + } + + [Fact] + public async Task Only_an_administrator_the_database_still_knows() + { + var root = await _host.SignUp("plain"); + using (var plain = _host.As(root.Jwt)) + Assert.Equal(HttpStatusCode.Forbidden, (await plain.GetAsync(Base, Token)).StatusCode); + + var admin = await _host.Admin(); + using var client = _host.As(admin.Jwt); + Assert.Equal(HttpStatusCode.OK, (await client.GetAsync(Base, Token)).StatusCode); + await DB.Default.Update().MatchID(admin.Id).Modify(u => u.Policies, [Policies.IsUser]).ExecuteAsync(Token); + Assert.Equal(HttpStatusCode.Forbidden, (await client.GetAsync(Base, Token)).StatusCode); + Assert.Equal(HttpStatusCode.Forbidden, (await client.PostAsync(Base, null, Token)).StatusCode); + } + + [Fact] + public async Task A_backup_is_made_downloaded_whole_or_in_part_and_uploaded_again() + { + var admin = await _host.Admin(); + using var client = _host.As(admin.Jwt); + var id = await BackedUp(client); + var listed = (await client.GetFromJsonAsync(Base, Token))!; + Assert.Equal("privapub.test", listed["host"]!.GetValue()); + Assert.Equal("manual", listed["backups"]!.AsArray().Single(b => b!["id"]!.GetValue() == id)!["kind"]!.GetValue()); + + Assert.Equal(HttpStatusCode.UnprocessableEntity, (await client.PostAsJsonAsync($"{Base}/{id}/ticket", new { password = "wrong" }, Token)).StatusCode); + var ticket = (await (await client.PostAsJsonAsync($"{Base}/{id}/ticket", new { password = admin.Password }, Token)).Content.ReadFromJsonAsync(Token))!; + var url = ticket["url"]!.GetValue(); + + // a plain link: no token needed, the ticket is the credential + using var anonymous = _host.Client(); + var whole = await anonymous.GetAsync(url, Token); + Assert.Equal(HttpStatusCode.OK, whole.StatusCode); + var bytes = await whole.Content.ReadAsByteArrayAsync(Token); + Assert.Equal(ticket["bytes"]!.GetValue(), bytes.Length); + Assert.Equal(bytes.Length, whole.Content.Headers.ContentLength); + var names = new List(); + await using (var reader = new TarReader(new MemoryStream(bytes))) + while (await reader.GetNextEntryAsync(cancellationToken: Token) is { } entry) + names.Add(entry.Name); + Assert.Contains($"{id}/manifest.json", names); + Assert.Contains($"{id}/db/Avatar.jsonl.gz", names); + + var part = new HttpRequestMessage(HttpMethod.Get, url); + part.Headers.Range = new RangeHeaderValue(700, 2747); + var partial = await anonymous.SendAsync(part, Token); + Assert.Equal(HttpStatusCode.PartialContent, partial.StatusCode); + Assert.Equal($"bytes 700-2747/{bytes.Length}", partial.Content.Headers.ContentRange!.ToString()); + Assert.Equal(bytes.AsSpan(700, 2048).ToArray(), await partial.Content.ReadAsByteArrayAsync(Token)); + Assert.Equal(HttpStatusCode.NotFound, (await anonymous.GetAsync($"{Base}/download/{new string('0', 64)}", Token)).StatusCode); + + // gone, then uploaded again in three pieces, one sent at the wrong place and refused with where to go on + Assert.Equal(HttpStatusCode.OK, (await client.DeleteAsync($"{Base}/{id}", Token)).StatusCode); + Assert.Equal(HttpStatusCode.NotFound, (await client.DeleteAsync($"{Base}/{id}", Token)).StatusCode); + var upload = (await (await client.PostAsync($"{Base}/uploads", null, Token)).Content.ReadFromJsonAsync(Token))!["id"]!.GetValue(); + var third = bytes.Length / 3; + Assert.Equal(HttpStatusCode.OK, (await Piece(client, upload, 0, bytes[..third])).StatusCode); + var gap = await Piece(client, upload, third + 10, bytes[(third + 10)..]); + Assert.Equal(HttpStatusCode.Conflict, gap.StatusCode); + Assert.Equal(third, (await gap.Content.ReadFromJsonAsync(Token))!["received"]!.GetValue()); + Assert.Equal(HttpStatusCode.OK, (await Piece(client, upload, third, bytes[third..(2 * third)])).StatusCode); + Assert.Equal(HttpStatusCode.OK, (await Piece(client, upload, 2 * third, bytes[(2 * third)..])).StatusCode); + var finished = await client.PostAsync($"{Base}/uploads/{upload}/finish", null, Token); + Assert.Equal(HttpStatusCode.OK, finished.StatusCode); + Assert.Equal("uploaded", (await finished.Content.ReadFromJsonAsync(Token))!["kind"]!.GetValue()); + Assert.Empty(await _host.Get().Verify(id, Token)); + Assert.Equal(HttpStatusCode.UnprocessableEntity, (await client.PostAsync($"{Base}/uploads/{upload}/finish", null, Token)).StatusCode); + _host.Get().Delete(id); + } + + Task Piece(HttpClient client, string upload, long offset, byte[] bytes) => + client.PutAsync($"{Base}/uploads/{upload}?offset={offset}", new ByteArrayContent(bytes), Token); + + [Theory] + [InlineData("escape")] + [InlineData("link")] + [InlineData("elsewhere")] + [InlineData("junk")] + public async Task An_upload_that_is_not_a_plain_backup_is_refused_and_leaves_nothing(string how) + { + var admin = await _host.Admin(); + using var client = _host.As(admin.Jwt); + const string id = "20260101-000000-manual"; + var tar = new MemoryStream(); + if (how == "junk") + tar.Write("not a tar at all, not even close"u8); + else + await using (var writer = new TarWriter(tar, TarEntryFormat.Pax, leaveOpen: true)) + { + await writer.WriteEntryAsync(new PaxTarEntry(TarEntryType.RegularFile, $"{id}/manifest.json") { DataStream = new MemoryStream("{}"u8.ToArray()) }, Token); + await writer.WriteEntryAsync(how switch + { + "escape" => new PaxTarEntry(TarEntryType.RegularFile, $"{id}/../../evil") { DataStream = new MemoryStream("x"u8.ToArray()) }, + "link" => new PaxTarEntry(TarEntryType.SymbolicLink, $"{id}/media/x.jpg") { LinkName = "/etc/passwd" }, + _ => new PaxTarEntry(TarEntryType.RegularFile, "20260101-000000-nightly/db/Post.jsonl.gz") { DataStream = new MemoryStream("x"u8.ToArray()) } + }, Token); + } + var upload = (await (await client.PostAsync($"{Base}/uploads", null, Token)).Content.ReadFromJsonAsync(Token))!["id"]!.GetValue(); + Assert.Equal(HttpStatusCode.OK, (await Piece(client, upload, 0, tar.ToArray())).StatusCode); + + Assert.Equal(HttpStatusCode.UnprocessableEntity, (await client.PostAsync($"{Base}/uploads/{upload}/finish", null, Token)).StatusCode); + var root = _host.Get().Root; + Assert.False(Directory.Exists(Path.Combine(root, id))); + Assert.False(Directory.Exists(Path.Combine(root, id + ServerBackup.PartialSuffix))); + Assert.False(File.Exists(Path.Combine(root, "..", "evil"))); + } + + // asked for: written for the next start (the test host has no watcher to stop it, and the marker is taken away) + [Fact] + public async Task A_restore_asks_for_the_password_and_the_host_typed_out() + { + var admin = await _host.Admin(); + using var client = _host.As(admin.Jwt); + var id = await BackedUp(client); + var root = _host.Get().Root; + try + { + Assert.Equal(HttpStatusCode.UnprocessableEntity, (await client.PostAsJsonAsync($"{Base}/{id}/restore", new { password = "wrong", host = "privapub.test" }, Token)).StatusCode); + Assert.Equal(HttpStatusCode.UnprocessableEntity, (await client.PostAsJsonAsync($"{Base}/{id}/restore", new { password = admin.Password, host = "elsewhere.test" }, Token)).StatusCode); + Assert.Null(RestoreMarker.Read(root)); + + Assert.Equal(HttpStatusCode.Accepted, (await client.PostAsJsonAsync($"{Base}/{id}/restore", new { password = admin.Password, host = "PrivaPub.test" }, Token)).StatusCode); + var marker = RestoreMarker.Read(root); + Assert.Equal((id, "pending", admin.UserName), (marker.Backup, marker.State, marker.RequestedBy)); + Assert.Equal(id, (await client.GetFromJsonAsync(Base, Token))!["waiting"]!["backup"]!.GetValue()); + Assert.Equal(HttpStatusCode.Conflict, (await client.DeleteAsync($"{Base}/{id}", Token)).StatusCode); + } + finally + { + RestoreMarker.Clear(root); + _host.Get().Delete(id); + } + } + } +} diff --git a/PrivaPub/Controllers/ClientToServer/BackupController.cs b/PrivaPub/Controllers/ClientToServer/BackupController.cs new file mode 100644 index 0000000..7ebe633 --- /dev/null +++ b/PrivaPub/Controllers/ClientToServer/BackupController.cs @@ -0,0 +1,280 @@ +using Microsoft.AspNetCore.Authorization; +using Microsoft.AspNetCore.Http.Features; +using Microsoft.AspNetCore.Mvc; +using Microsoft.Extensions.Localization; +using Microsoft.Net.Http.Headers; + +using MongoDB.Entities; + +using PrivaPub.ClientModels; +using PrivaPub.ClientModels.Admin; +using PrivaPub.Extensions; +using PrivaPub.Infrastructure.Backup; +using PrivaPub.Models.User; +using PrivaPub.Resources; +using PrivaPub.StaticServices; + +namespace PrivaPub.Controllers.ClientToServer +{ + // The server's backups from the administrator's page (owner decision 2026-10-07: it backs up and restores too). Only + // an administrator, checked again against the database; a download and a restore ask for the password again, and a + // restore for the server's host typed out. A restore asked for stops the service within seconds, and the next start + // carries it out (ServerRestore). + [ApiController, + Route("clientapi/admin/backups"), + Authorize(Policy = Policies.IsAdmin)] + public class BackupController(Backups backups, TransferStore transfers, IPasswordHasher hasher, IStringLocalizer localizer, + ILogger logger) : ControllerBase + { + [HttpGet, Route("")] + public async Task List(CancellationToken token) + { + if (await Administrator(token) == default) + return Forbid(); + var deleted = await DB.Default.Find().Match(u => u.DeletedAt != null).Project(u => u.Include(x => x.DeletedAt)).ExecuteAsync(token); + var code = ServerBackup.CodeMigration(); + var waiting = RestoreMarker.Read(backups.Root); + var last = await DB.Default.Find().Sort(r => r.RestoredAt, Order.Descending).ExecuteFirstAsync(token); + ServerBackup.MakeDirectory(backups.Root); + return Ok(new ViewBackups + { + Backups = [.. backups.List().Select(b => View(b, deleted.Count(u => u.DeletedAt > b.CreatedAt), code))], + Running = (await MaintenanceLock.Current(token))?.What, + Waiting = waiting == default ? default : new ViewRestoreWaiting + { + Backup = waiting.Backup, + State = waiting.State, + Attempts = waiting.Attempts, + RequestedBy = waiting.RequestedBy, + RequestedAt = waiting.RequestedAt, + Error = waiting.Error + }, + LastRestore = last == default ? default : View(last), + Host = HostName, + FreeBytes = new DriveInfo(backups.Root).AvailableFreeSpace + }); + } + + // a backup made now, while the page polls the list + [HttpPost, Route("")] + public async Task Create(CancellationToken token) + { + var admin = await Administrator(token); + if (admin == default) + return Forbid(); + if (await MaintenanceLock.Current(token) is { } held) + return Conflict(new WebResult().Invalidate(localizer["A {0} is running.", held.What])); + _ = Task.Run(async () => + { + try + { + var (made, error) = await backups.Create("manual", dbOnly: false, CancellationToken.None); + if (made == default) + logger.LogWarning("The backup {Admin} asked for was not made: {Error}", admin.UserName, error); + else + logger.LogInformation("Backup {Id} made for {Admin}", made.Id, admin.UserName); + } + catch (Exception ex) + { + logger.LogError(ex, "The backup {Admin} asked for failed", admin.UserName); + } + }, CancellationToken.None); + return Accepted(); + } + + [HttpDelete, Route("{id}")] + public async Task Delete(string id, CancellationToken token) + { + if (await Administrator(token) == default) + return Forbid(); + if (RestoreMarker.Read(backups.Root) is { } waiting && (waiting.Backup == id || waiting.PreRestore == id)) + return Conflict(new WebResult().Invalidate(localizer["A restore waits for this backup."])); + return backups.Delete(id) ? Ok() : NotFound(); + } + + // a link to download it with, for the password + [HttpPost, Route("{id}/ticket")] + public async Task Ticket(string id, BackupPasswordForm form, CancellationToken token) + { + var admin = await Administrator(token); + if (admin == default) + return Forbid(); + if (!PasswordHolds(admin, form.Password)) + return UnprocessableEntity(new WebResult().Invalidate(localizer["Wrong password."])); + if (backups.Find(id) == default) + return NotFound(); + var (ticket, expires) = transfers.Ticket(id); + return Ok(new ViewBackupTicket { Url = $"/clientapi/admin/backups/download/{ticket}", ExpiresAt = expires, Bytes = BackupTar.Of(backups.Root, id).Length }); + } + + // the backup as one tar, for a ticket: a plain link, so the browser downloads it to disk and resumes it + [HttpGet, Route("download/{ticket}"), AllowAnonymous] + public async Task Download(string ticket, CancellationToken token) + { + var id = transfers.Redeem(ticket); + if (id == default || backups.Find(id) == default) + { + Response.StatusCode = StatusCodes.Status404NotFound; + return; + } + HttpContext.Features.Get()?.DisableBuffering(); + var tar = BackupTar.Of(backups.Root, id); + var (from, to) = (0L, tar.Length - 1); + Response.Headers.AcceptRanges = "bytes"; + Response.Headers.CacheControl = "no-store"; + Response.Headers.ContentDisposition = new ContentDispositionHeaderValue("attachment") { FileName = $"privapub-{id}.tar" }.ToString(); + var range = Request.GetTypedHeaders().Range; + if (range is { Unit.Value: "bytes", Ranges.Count: 1 }) + { + var asked = range.Ranges.First(); + var start = asked.From ?? tar.Length - asked.To.GetValueOrDefault(); + var end = asked.From.HasValue ? Math.Min(asked.To ?? tar.Length - 1, tar.Length - 1) : tar.Length - 1; + if (start < 0 || start > end) + { + Response.StatusCode = StatusCodes.Status416RangeNotSatisfiable; + Response.Headers.ContentRange = $"bytes */{tar.Length}"; + return; + } + (from, to) = (start, end); + Response.StatusCode = StatusCodes.Status206PartialContent; + Response.Headers.ContentRange = $"bytes {from}-{to}/{tar.Length}"; + } + Response.ContentType = "application/x-tar"; + Response.ContentLength = to - from + 1; + await tar.Write(Response.Body, from, to, token); + } + + [HttpPost, Route("uploads")] + public async Task StartUpload(CancellationToken token) + { + var admin = await Administrator(token); + if (admin == default) + return Forbid(); + return Ok(new ViewBackupUpload { Id = transfers.Start(admin.UserName), ChunkBytes = TransferStore.ChunkBytes }); + } + + [HttpGet, Route("uploads/{id}")] + public async Task Upload(string id, CancellationToken token) + { + if (await Administrator(token) == default) + return Forbid(); + var received = transfers.Received(id); + return received < 0 ? NotFound() : Ok(new ViewBackupUpload { Id = id, Received = received, ChunkBytes = TransferStore.ChunkBytes }); + } + + // a piece, at the offset already received: 409 with what was received when it isn't + [HttpPut, Route("uploads/{id}"), RequestSizeLimit(TransferStore.ChunkBytes + 1024 * 1024)] + public async Task Append(string id, [FromQuery] long offset, CancellationToken token) + { + if (await Administrator(token) == default) + return Forbid(); + var (received, taken) = await transfers.Append(id, offset, Request.Body, token); + if (received < 0) + return NotFound(); + var view = new ViewBackupUpload { Id = id, Received = received, ChunkBytes = TransferStore.ChunkBytes }; + return taken ? Ok(view) : Conflict(view); + } + + [HttpPost, Route("uploads/{id}/finish")] + public async Task FinishUpload(string id, CancellationToken token) + { + if (await Administrator(token) == default) + return Forbid(); + if (await MaintenanceLock.Current(token) is { What: "restore" }) + return Conflict(new WebResult().Invalidate(localizer["A {0} is running.", "restore"])); + var (backup, error) = await transfers.Finish(id, token); + if (backup == default) + return UnprocessableEntity(new WebResult().Invalidate(error)); + return Ok(View(backup, 0, ServerBackup.CodeMigration())); + } + + [HttpDelete, Route("uploads/{id}")] + public async Task CancelUpload(string id, CancellationToken token) + { + if (await Administrator(token) == default) + return Forbid(); + return transfers.Cancel(id) ? Ok() : NotFound(); + } + + // asked for with the password and the host typed out: the service stops within seconds and restores it as it starts + [HttpPost, Route("{id}/restore")] + public async Task Restore(string id, RestoreBackupForm form, CancellationToken token) + { + var admin = await Administrator(token); + if (admin == default) + return Forbid(); + if (!PasswordHolds(admin, form.Password)) + return UnprocessableEntity(new WebResult().Invalidate(localizer["Wrong password."])); + if (!string.Equals(form.Host?.Trim(), HostName, StringComparison.OrdinalIgnoreCase)) + return UnprocessableEntity(new WebResult().Invalidate(localizer["Type this server's host to restore it."])); + var refused = await ServerRestore.Request(backups.Context(), id, admin.UserName, token); + if (refused != default) + return UnprocessableEntity(new WebResult().Invalidate(refused)); + logger.LogWarning("{Admin} asked to restore {Backup}", admin.UserName, id); + return Accepted(); + } + + string HostName => Uri.TryCreate(backups.Host, UriKind.Absolute, out var host) ? host.Authority : backups.Host; + + // the token says administrator; the database has the last word + async Task Administrator(CancellationToken token) + { + var root = await DB.Default.Find().MatchID(User.GetUserId()).ExecuteFirstAsync(token); + return root is { IsBanned: false, DeletedAt: null } && root.Policies.Contains(Policies.IsAdmin) ? root : default; + } + + bool PasswordHolds(RootUser root, string password) => + !string.IsNullOrEmpty(password) && !string.IsNullOrEmpty(root.HashedPassword) && hasher.Check(root.HashedPassword, password).verified; + + static ViewBackup View(BackupInfo backup, int rootsDeletedSince, int code) + { + var manifest = backup.Manifest; + return new ViewBackup + { + Id = backup.Id, + Kind = backup.Kind, + CreatedAt = backup.CreatedAt, + Bytes = backup.Bytes, + MediaBytes = manifest.Media.Bytes, + MediaFiles = manifest.DbOnly ? manifest.Media.List.Count : manifest.Media.Files, + MediaMissing = manifest.Media.Missing, + DbOnly = manifest.DbOnly, + Consistent = manifest.Consistent, + Collections = manifest.Collections.Count, + Documents = manifest.Collections.Sum(c => c.Count), + AppCommit = manifest.AppCommit, + MigrationNumber = manifest.MigrationNumber, + RootsDeletedSince = rootsDeletedSince, + NotRestorable = manifest.Format != ArchiveManifest.CurrentFormat ? "format" + : manifest.MigrationNumber > code ? "newer" + : default + }; + } + + static ViewRestore View(RestoreRecord record) => new() + { + Backup = record.Backup, + BackupCreatedAt = record.BackupCreatedAt, + PreRestore = record.PreRestore, + RequestedBy = record.RequestedBy, + RestoredAt = record.RestoredAt, + Abandoned = record.Abandoned, + Error = record.Error, + Documents = record.Report.Documents, + Collections = record.Report.Collections, + MediaRestored = record.Report.MediaRestored, + MediaMissing = record.Report.MediaMissing, + RootsDeleted = record.Report.RootsDeleted, + PersonasDeleted = record.Report.PersonasDeleted, + GroupsDeleted = record.Report.GroupsDeleted, + PostsDeleted = record.Report.PostsDeleted, + RootsTombstoned = record.Report.RootsTombstoned, + PersonasTombstoned = record.Report.PersonasTombstoned, + GroupsTombstoned = record.Report.GroupsTombstoned, + PostsGone = record.Report.PostsGone, + MediaTrashed = record.Report.MediaTrashed, + ProtectiveKept = record.Report.ProtectiveKept, + SessionsEnded = record.Report.SessionsEnded + }; + } +} diff --git a/PrivaPub/Infrastructure/Backup/BackupTar.cs b/PrivaPub/Infrastructure/Backup/BackupTar.cs new file mode 100644 index 0000000..a38e77e --- /dev/null +++ b/PrivaPub/Infrastructure/Backup/BackupTar.cs @@ -0,0 +1,224 @@ +using System.Formats.Tar; +using System.Text; +using System.Text.RegularExpressions; + +namespace PrivaPub.Infrastructure.Backup +{ + // A backup as one tar file, for the administrator to download and to upload again: every file under /, its length + // known before the first byte (so a download has a Content-Length and resumes with Range), written straight from the + // backup's files, never held in memory. + public sealed partial class BackupTar + { + const int Block = 512; + + readonly List _entries = []; + + public long Length { get; private set; } + + sealed record Entry(string Name, string Path, long Size, DateTime ModifiedAt, long Offset); + + public static BackupTar Of(string backupsRoot, string id) + { + var directory = Path.Combine(backupsRoot, id); + var tar = new BackupTar(); + var offset = 0L; + foreach (var file in Directory.EnumerateFiles(directory, "*", SearchOption.AllDirectories).OrderBy(f => f, StringComparer.Ordinal)) + { + var info = new FileInfo(file); + var name = id + "/" + Path.GetRelativePath(directory, file).Replace('\\', '/'); + tar._entries.Add(new Entry(name, file, info.Length, info.LastWriteTimeUtc, offset)); + offset += Block + Padded(info.Length); + } + tar.Length = offset + 2 * Block; + return tar; + } + + static long Padded(long size) => (size + Block - 1) / Block * Block; + + /// Writes bytes from to to (inclusive) of the tar. + public async Task Write(Stream output, long from, long to, CancellationToken token) + { + var buffer = new byte[81920]; + foreach (var entry in _entries) + { + var headerEnd = entry.Offset + Block; + var dataEnd = headerEnd + entry.Size; + var entryEnd = headerEnd + Padded(entry.Size); + if (entryEnd <= from) + continue; + if (entry.Offset > to) + return; + if (from < headerEnd) + { + var header = Header(entry); + var start = (int)Math.Max(0, from - entry.Offset); + var end = (int)Math.Min(Block, to - entry.Offset + 1); + await output.WriteAsync(header.AsMemory(start, end - start), token); + } + if (from < dataEnd && to >= headerEnd && entry.Size > 0) + { + var start = Math.Max(0, from - headerEnd); + var end = Math.Min(entry.Size, to - headerEnd + 1); + await using var file = new FileStream(entry.Path, FileMode.Open, FileAccess.Read, FileShare.Read, 1, FileOptions.SequentialScan); + file.Seek(start, SeekOrigin.Begin); + var left = end - start; + while (left > 0) + { + var read = await file.ReadAsync(buffer.AsMemory(0, (int)Math.Min(buffer.Length, left)), token); + if (read == 0) + throw new IOException($"{entry.Name} shrank while it was sent"); + await output.WriteAsync(buffer.AsMemory(0, read), token); + left -= read; + } + } + if (to >= dataEnd && from < entryEnd) + { + var start = Math.Max(from, dataEnd); + var end = Math.Min(entryEnd - 1, to); + if (end >= start) + await output.WriteAsync(new byte[end - start + 1], token); + } + } + var trailer = Length - 2 * Block; + if (to >= trailer) + await output.WriteAsync(new byte[to - Math.Max(from, trailer) + 1], token); + } + + // a ustar header (name split into prefix and name past 100 bytes; sizes past 8 GiB in base 256) + static byte[] Header(Entry entry) + { + var header = new byte[Block]; + var name = entry.Name; + var prefix = string.Empty; + if (Encoding.UTF8.GetByteCount(name) > 100) + { + var cut = name.LastIndexOf('/', Math.Min(name.Length - 1, 155)); + while (cut > 0 && Encoding.UTF8.GetByteCount(name[(cut + 1)..]) > 100) + cut = name.IndexOf('/', cut + 1); + if (cut <= 0 || Encoding.UTF8.GetByteCount(name[..cut]) > 155) + throw new InvalidOperationException($"{name} is too long for a tar header"); + prefix = name[..cut]; + name = name[(cut + 1)..]; + } + Text(header, 0, 100, name); + Octal(header, 100, 8, Convert.ToInt32("640", 8)); + Octal(header, 108, 8, 0); + Octal(header, 116, 8, 0); + if (entry.Size < 1L << 33) + Octal(header, 124, 12, entry.Size); + else + { + header[124] = 0x80; + for (var i = 0; i < 8; i++) + header[135 - i] = (byte)(entry.Size >> (8 * i)); + } + Octal(header, 136, 12, new DateTimeOffset(entry.ModifiedAt).ToUnixTimeSeconds()); + header[156] = (byte)'0'; + Text(header, 257, 6, "ustar"); + Text(header, 263, 2, "00"); + Text(header, 265, 32, "privapub"); + Text(header, 297, 32, "privapub"); + Text(header, 345, 155, prefix); + for (var i = 148; i < 156; i++) + header[i] = (byte)' '; + var sum = header.Sum(b => (long)b); + Text(header, 148, 7, Convert.ToString(sum, 8).PadLeft(6, '0')); + header[155] = (byte)' '; + return header; + } + + static void Text(byte[] header, int at, int length, string value) + { + var bytes = Encoding.UTF8.GetBytes(value); + Array.Copy(bytes, 0, header, at, Math.Min(bytes.Length, length)); + } + + static void Octal(byte[] header, int at, int length, long value) => + Text(header, at, length - 1, Convert.ToString(value, 8).PadLeft(length - 1, '0')); + + [GeneratedRegex(@"^\d{8}-\d{6}-[a-z-]{1,20}$")] + public static partial Regex BackupId(); + + /// + /// A backup read from an uploaded tar into the backups' root: refused when it holds anything but plain files and + /// folders under one backup's folder, a path leaving it, or more than the disk can take. The backup, or why not. + /// + public static async Task<(BackupInfo Backup, string Error)> Import(Stream tar, string backupsRoot, CancellationToken token) + { + const int MaxEntries = 2_000_000; + var id = default(string); + var partial = default(string); + var entries = 0; + var free = new DriveInfo(Path.GetFullPath(backupsRoot)).AvailableFreeSpace; + var written = 0L; + try + { + await using var reader = new TarReader(tar, leaveOpen: true); + while (await reader.GetNextEntryAsync(copyData: false, token) is { } entry) + { + if (++entries > MaxEntries) + return (default, "too many files"); + var name = entry.Name.TrimEnd('/'); + var slash = name.IndexOf('/'); + var top = slash < 0 ? name : name[..slash]; + if (id == default) + { + if (!BackupId().IsMatch(top)) + return (default, $"{top} is not a backup's folder"); + id = top; + if (Directory.Exists(Path.Combine(backupsRoot, id))) + return (default, $"{id} is already here"); + partial = Path.Combine(backupsRoot, id + ServerBackup.PartialSuffix); + if (Directory.Exists(partial)) + Directory.Delete(partial, recursive: true); + ServerBackup.MakeDirectory(partial); + } + if (top != id) + return (default, $"{name} is outside {id}"); + if (entry.EntryType is TarEntryType.Directory) + continue; + if (entry.EntryType is not (TarEntryType.RegularFile or TarEntryType.V7RegularFile)) + return (default, $"{name} is not a plain file"); + if (slash < 0) + return (default, $"{name} is not inside a folder"); + written += entry.Length; + if (written > free - 512L * 1024 * 1024) + return (default, "not enough free disk"); + var path = ServerBackup.Inside(partial, name[(slash + 1)..]); + Directory.CreateDirectory(Path.GetDirectoryName(path)!); + await using var file = OperatingSystem.IsWindows() + ? new FileStream(path, FileMode.CreateNew, FileAccess.Write) + : new FileStream(path, new FileStreamOptions { Mode = FileMode.CreateNew, Access = FileAccess.Write, + UnixCreateMode = UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.GroupRead }); + if (entry.DataStream != default) + await entry.DataStream.CopyToAsync(file, token); + } + if (id == default) + return (default, "the file holds nothing"); + var manifest = ArchiveManifest.Read(partial); + if (manifest == default) + return (default, "it has no manifest"); + manifest.Kind = "uploaded"; + manifest.Write(partial); + Directory.Move(partial, Path.Combine(backupsRoot, id)); + partial = default; + var problems = await ServerBackup.Verify(backupsRoot, id, token); + if (problems.Count > 0) + { + ServerBackup.Delete(backupsRoot, id); + return (default, string.Join("; ", problems.Take(5))); + } + return (ServerBackup.Find(backupsRoot, id), default); + } + catch (Exception ex) when (ex is FormatException or InvalidDataException or InvalidOperationException or EndOfStreamException or System.Text.Json.JsonException) + { + return (default, $"not a backup: {ex.Message}"); + } + finally + { + if (partial != default && Directory.Exists(partial)) + Directory.Delete(partial, recursive: true); + } + } + } +} diff --git a/PrivaPub/Infrastructure/Backup/TransferStore.cs b/PrivaPub/Infrastructure/Backup/TransferStore.cs new file mode 100644 index 0000000..d4e262d --- /dev/null +++ b/PrivaPub/Infrastructure/Backup/TransferStore.cs @@ -0,0 +1,136 @@ +using System.Collections.Concurrent; +using System.Security.Cryptography; +using System.Text.Json; + +namespace PrivaPub.Infrastructure.Backup +{ + // Backups leaving and arriving through the administrator's page, never through the client's memory: + // - a download is a link with a ticket in its path, good for ten minutes for one backup (a browser downloads it, and + // resumes it with Range), given for the administrator's password; + // - an upload arrives in pieces appended at the offset already received (a piece that would leave a gap or overlap is + // refused with what was received, so a broken upload resumes), into /.uploads, and is read into a backup + // once whole. One left for a day is deleted. + public class TransferStore(Backups backups) + { + public static readonly TimeSpan TicketLifetime = TimeSpan.FromMinutes(10); + public const long ChunkBytes = 32L * 1024 * 1024; + const string Uploads = ".uploads"; + + readonly ConcurrentDictionary _tickets = new(); + readonly ConcurrentDictionary _appending = new(); + + public (string Ticket, DateTime Expires) Ticket(string backup) + { + foreach (var (ticket, stale) in _tickets.Where(t => t.Value.Expires < DateTime.UtcNow).ToList()) + _tickets.TryRemove(ticket, out _); + var expires = DateTime.UtcNow + TicketLifetime; + var token = Convert.ToHexStringLower(RandomNumberGenerator.GetBytes(32)); + _tickets[token] = (backup, expires); + return (token, expires); + } + + /// The backup a ticket is for, while it is good. + public string Redeem(string ticket) => + ticket != default && _tickets.TryGetValue(ticket, out var held) && held.Expires > DateTime.UtcNow ? held.Backup : default; + + string Folder => Path.Combine(backups.Root, Uploads); + + string File(string id) => Path.Combine(Folder, id + ".tar"); + + static bool IsId(string id) => id is { Length: 32 } && id.All(char.IsAsciiHexDigitLower); + + public string Start(string by) + { + ServerBackup.MakeDirectory(backups.Root); + ServerBackup.MakeDirectory(Folder); + Forget(DateTime.UtcNow.AddDays(-1)); + var id = Convert.ToHexStringLower(RandomNumberGenerator.GetBytes(16)); + using (OperatingSystem.IsWindows() + ? new FileStream(File(id), FileMode.CreateNew, FileAccess.Write) + : new FileStream(File(id), new FileStreamOptions { Mode = FileMode.CreateNew, Access = FileAccess.Write, + UnixCreateMode = UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.GroupRead })) + { + } + System.IO.File.WriteAllText(Path.Combine(Folder, id + ".json"), JsonSerializer.Serialize(new { by, at = DateTime.UtcNow })); + return id; + } + + /// How much of an upload has arrived; -1 when there is no such upload. + public long Received(string id) => IsId(id) && System.IO.File.Exists(File(id)) ? new FileInfo(File(id)).Length : -1; + + /// A piece appended where the upload stands: what has arrived since, and whether it was taken. + public async Task<(long Received, bool Taken)> Append(string id, long offset, Stream piece, CancellationToken token) + { + if (!IsId(id) || !System.IO.File.Exists(File(id))) + return (-1, false); + var gate = _appending.GetOrAdd(id, _ => new SemaphoreSlim(1, 1)); + await gate.WaitAsync(token); + try + { + await using var file = new FileStream(File(id), FileMode.Open, FileAccess.Write); + if (file.Length != offset) + return (file.Length, false); + file.Seek(offset, SeekOrigin.Begin); + var buffer = new byte[81920]; + var total = 0L; + int read; + while ((read = await piece.ReadAsync(buffer, token)) > 0) + { + total += read; + if (total > ChunkBytes) + { + file.SetLength(offset); + return (offset, false); + } + await file.WriteAsync(buffer.AsMemory(0, read), token); + } + await file.FlushAsync(token); + return (file.Length, true); + } + catch (IOException) + { + //the piece broke off: what arrived of it is dropped, so the next one starts where this one did + await using var file = new FileStream(File(id), FileMode.Open, FileAccess.Write); + file.SetLength(offset); + return (offset, false); + } + finally + { + gate.Release(); + } + } + + /// The whole upload read into a backup: the backup, or why not. The upload goes either way. + public async Task<(BackupInfo Backup, string Error)> Finish(string id, CancellationToken token) + { + if (!IsId(id) || !System.IO.File.Exists(File(id))) + return (default, "no such upload"); + try + { + await using var tar = new FileStream(File(id), FileMode.Open, FileAccess.Read, FileShare.Read, 81920, FileOptions.SequentialScan); + return await BackupTar.Import(tar, backups.Root, token); + } + finally + { + Cancel(id); + } + } + + public bool Cancel(string id) + { + if (!IsId(id) || !System.IO.File.Exists(File(id))) + return false; + System.IO.File.Delete(File(id)); + System.IO.File.Delete(Path.Combine(Folder, id + ".json")); + _appending.TryRemove(id, out _); + return true; + } + + // uploads left behind + void Forget(DateTime before) + { + foreach (var file in Directory.EnumerateFiles(Folder).Where(f => System.IO.File.GetLastWriteTimeUtc(f) < before)) + System.IO.File.Delete(file); + } + } +} diff --git a/PrivaPub/Program.cs b/PrivaPub/Program.cs index 39d9982..7d3fe81 100644 --- a/PrivaPub/Program.cs +++ b/PrivaPub/Program.cs @@ -70,6 +70,7 @@ try .AddHostedService() .Configure(builder.Configuration.GetSection("Backups")) .AddSingleton() + .AddSingleton() .AddHostedService() .AddHostedService() .PrivaPubMiddlewareConfiguration(); diff --git a/deploy/nginx/privapub.thepra.dev.conf b/deploy/nginx/privapub.thepra.dev.conf index 1971f43..f885bfc 100644 --- a/deploy/nginx/privapub.thepra.dev.conf +++ b/deploy/nginx/privapub.thepra.dev.conf @@ -1,3 +1,6 @@ +# backup uploads: a few pieces a second at most, per address +limit_req_zone $binary_remote_addr zone=privapub_backup_uploads:1m rate=120r/m; + server { listen 80; listen [::]:80; @@ -48,6 +51,33 @@ server { proxy_read_timeout 300s; } + # the administrator's backups (owner decision 2026-10-07): a download streams a whole backup for as long as it takes; + # an upload arrives in pieces of at most 32 MB, unbuffered + location ^~ /clientapi/admin/backups/download/ { + proxy_buffering off; + proxy_pass http://127.0.0.1:6970; + proxy_http_version 1.1; + proxy_set_header Connection ""; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_read_timeout 3600s; + proxy_send_timeout 3600s; + } + location ^~ /clientapi/admin/backups/uploads { + client_max_body_size 40m; + proxy_request_buffering off; + limit_req zone=privapub_backup_uploads burst=30 nodelay; + proxy_pass http://127.0.0.1:6970; + proxy_http_version 1.1; + proxy_set_header Connection ""; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_read_timeout 300s; + } location ^~ /media/proxy/ { proxy_buffering off; proxy_pass http://127.0.0.1:6970;