From bb680e8cb8d7aca3bb62432569481e10e29c096a Mon Sep 17 00:00:00 2001 From: thepra Date: Wed, 7 Oct 2026 10:31:14 +0200 Subject: [PATCH] A file lives exactly as long as something holds it Deleting a post, editing media out, replacing an avatar or a header, and removing a whole root deleted no file: every one stayed on disk and publicly served from /media/files with a year-long immutable cache, its row orphaned. Now every upload is a row, profile pictures too (Kind avatar or header, ProfileOfAvatarId), and each of those acts trashes what it held, as does an upload never posted for a day and a dropped scheduled post. A trashed row is marked in one conditional update (an upload attached meanwhile is left alone), its files move into media-trash, beside the media root and outside what /media/files serves, and the janitor deletes them a day later. Nothing is deleted for looking unused. Along the way: a profile picture that isn't an image, or can't be read, answers 422 instead of being silently ignored with a 200; a removed root's scheduled posts are dropped, so nothing of it publishes later; media rows get indexes (they had none), and the janitor's first pass comes five minutes after boot instead of an hour. `PrivaPub admin media audit [--fix]` compares the disk with the database. With --fix (as www-data) it gives the pictures personas show today a row, and trashes media of deleted posts or personas, rows whose files are missing, and files nothing holds: the leftovers of every deletion until now. MediaLifecycleTests covers each act, that the trash is never served, and the audit. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01LsXgEaXee4GCU1hwYgPJXw --- CLAUDE.md | 21 ++- PrivaPub.Tests/Federation/JobHandlerTests.cs | 10 +- PrivaPub.Tests/Http/MediaLifecycleTests.cs | 153 ++++++++++++++++++ .../Controllers/AccountsController.cs | 25 ++- .../Mastodon/Controllers/MediaController.cs | 4 +- .../ScheduledStatusesController.cs | 3 +- .../Controllers/StatusesController.cs | 3 +- PrivaPub/Domain/Media/MediaAudit.cs | 101 ++++++++++++ PrivaPub/Domain/Media/MediaProxy.cs | 34 +++- PrivaPub/Domain/Media/MediaService.cs | 97 +++++++++-- PrivaPub/Domain/Statuses/StatusService.cs | 7 +- PrivaPub/Infrastructure/Cli/AdminCommands.cs | 19 +++ PrivaPub/Infrastructure/Data/Indexes.cs | 7 + PrivaPub/Models/Media/MediaAttachment.cs | 5 + PrivaPub/Services/RootRemoval.cs | 9 +- 15 files changed, 462 insertions(+), 36 deletions(-) create mode 100644 PrivaPub.Tests/Http/MediaLifecycleTests.cs create mode 100644 PrivaPub/Domain/Media/MediaAudit.cs diff --git a/CLAUDE.md b/CLAUDE.md index 3e5fb40..be5a6f4 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -297,19 +297,30 @@ group www-data and reaches the private mongod; `sudo -u www-data` works too. 1. **No upload keeps its metadata.** Images are re-encoded by libvips with `keep=none`; audio and video are remuxed with `-map_metadata -1`. `MediaProcessingTests` checks EXIF and XMP are gone. 2. Files live under `Media:Root` (`/var/lib/privapub/media`), never in the published directory; the proxy cache is the - sibling `media-proxy`, which `/media/files` does not serve. -3. **A client never contacts a remote server for media:** every remote URL the API returns goes through + sibling `media-proxy` and the trash the sibling `media-trash`, neither of which `/media/files` serves. +3. **A file lives exactly as long as something holds it.** Every upload is a `MediaAttachment` row, profile pictures + too (`Kind` avatar or header, `ProfileOfAvatarId`). Deleting a post, an edit leaving media out, a replaced picture, + a dropped scheduled post, a removed root, and an upload never posted for a day each trash theirs + (`IMediaService.Trash`): + - the row gets `TrashedAt` in one conditional update, so a row attached meanwhile is left alone; + - its files move into `media-trash` at once, so `/media/files` stops serving them; + - `MediaJanitor` deletes them a day later (`TrashGrace`). + + Nothing is deleted because it looks unused. `PrivaPub admin media audit [--fix]` compares disk and database: with + `--fix` (as www-data) it gives pictures shown from before their rows a row, and trashes media of deleted posts or + personas, rows whose files are missing, and files nothing holds. +4. **A client never contacts a remote server for media:** every remote URL the API returns goes through `IMediaProxy.Wrap`, an HMAC-signed `/media/proxy/` URL fetched by `IFederationHttp.GetMedia`. -4. **The proxy serves three ways:** +5. **The proxy serves three ways:** - **Cached:** a file already cached is served from disk, ranges included. - **Downloaded:** a request without a `Range` is downloaded whole, up to `Media:MaxProxiedBytes`, then cached. - **Streamed:** a ranged request, or anything too big to cache, is streamed from the origin with the range passed on, and never cached. That is how remote video plays. nginx has a `/media/proxy/` location with `proxy_buffering off` and a 600 s read timeout for those streams. -5. **A focal point is two finite numbers** within -1..1 (`FocalPoint.Parse`); anything else is ignored. A stored NaN made +6. **A focal point is two finite numbers** within -1..1 (`FocalPoint.Parse`); anything else is ignored. A stored NaN made every status, timeline and Note holding its post fail to serialise; migration `_016` removed the ones stored before. -6. **Remote video and audio become one playable attachment** in the Mastodon API (`MastodonMapper.Playable`): the best MP4 +7. **Remote video and audio become one playable attachment** in the Mastodon API (`MastodonMapper.Playable`): the best MP4 up to 720p that carries both sound and picture, including PeerTube's fragmented files inside an HLS entry. HLS playlists themselves are not rewritten. diff --git a/PrivaPub.Tests/Federation/JobHandlerTests.cs b/PrivaPub.Tests/Federation/JobHandlerTests.cs index 529146c..dbb2486 100644 --- a/PrivaPub.Tests/Federation/JobHandlerTests.cs +++ b/PrivaPub.Tests/Federation/JobHandlerTests.cs @@ -489,9 +489,17 @@ namespace PrivaPub.Tests.Federation await new MediaJanitor(media, options, NullLogger.Instance).Sweep(token); - Assert.False(await DB.Default.Find().Match(m => m.ID == stale.ID).ExecuteAnyAsync(token)); + // never posted for a day: trashed, its files out of what /media/files serves at once + Assert.NotNull((await DB.Default.Find().OneAsync(stale.ID, token)).TrashedAt); Assert.False(File.Exists(Path.Combine(root, stale.FilePath))); Assert.False(File.Exists(Path.Combine(root, stale.PreviewPath))); + Assert.True(File.Exists(Path.Combine(media.TrashRoot, stale.FilePath))); + // and once its grace has passed, deleted with its row + await DB.Default.Update().MatchID(stale.ID).Modify(m => m.TrashedAt, DateTime.UtcNow - MediaJanitor.TrashGrace - TimeSpan.FromMinutes(1)).ExecuteAsync(token); + await new MediaJanitor(media, options, NullLogger.Instance).Sweep(token); + Assert.False(await DB.Default.Find().Match(m => m.ID == stale.ID).ExecuteAnyAsync(token)); + Assert.False(File.Exists(Path.Combine(media.TrashRoot, stale.FilePath))); + Assert.False(File.Exists(Path.Combine(media.TrashRoot, stale.PreviewPath))); Assert.True(await DB.Default.Find().Match(m => m.ID == fresh.ID).ExecuteAnyAsync(token)); Assert.True(await DB.Default.Find().Match(m => m.ID == attached.ID).ExecuteAnyAsync(token)); Assert.True(File.Exists(Path.Combine(root, fresh.FilePath))); diff --git a/PrivaPub.Tests/Http/MediaLifecycleTests.cs b/PrivaPub.Tests/Http/MediaLifecycleTests.cs new file mode 100644 index 0000000..72c6f17 --- /dev/null +++ b/PrivaPub.Tests/Http/MediaLifecycleTests.cs @@ -0,0 +1,153 @@ +using MongoDB.Entities; + +using PrivaPub.Domain.Media; +using PrivaPub.Models.Media; +using PrivaPub.Services; +using PrivaPub.Tests.Support; +using PrivaPub.Tests.Support.Host; + +using System.Net; + +namespace PrivaPub.Tests.Http +{ + // A file lives exactly as long as something holds it: a deleted post, an edit leaving media out, a replaced picture and + // a removed root each trash theirs, which stops /media/files serving them at once; the janitor deletes them later. + [Trait("Category", "Integration")] + public sealed class MediaLifecycleTests : IAsyncLifetime + { + PrivaPubHost _host; + + public async ValueTask InitializeAsync() + { + Assert.SkipUnless(MongoFixture.Enabled, MongoFixture.Skip); + _host = await PrivaPubHost.Shared(); + } + + public ValueTask DisposeAsync() => ValueTask.CompletedTask; + + static CancellationToken Token => TestContext.Current.CancellationToken; + + async Task Upload(Mastodon account, string name) + { + var form = MastodonHelpers.Multipart(("file", MastodonHelpers.JpegWithMetadata(64, 48), "image/jpeg", name)); + return (await account.Client.Exchange(new HttpRequestMessage(HttpMethod.Post, "/api/v2/media") { Content = form })).Ok().Body.Text("id"); + } + + async Task Served(string url) => + (await _host.Client().GetAsync(new Uri(url).PathAndQuery, Token)).StatusCode; + + [Fact] + public async Task A_deleted_posts_media_stop_being_served() + { + var alice = await _host.Mastodon("alice"); + var id = await Upload(alice, "a.jpg"); + var status = (await alice.Client.Post("/api/v1/statuses", ("status", "with a picture"), ("media_ids[]", id))).Ok(); + var url = status.Body["media_attachments"]![0]!["url"]!.GetValue(); + var preview = status.Body["media_attachments"]![0]!["preview_url"]!.GetValue(); + Assert.Equal(HttpStatusCode.OK, await Served(url)); + + (await alice.Client.Delete($"/api/v1/statuses/{status.Body.Text("id")}")).Ok(); + + Assert.Equal(HttpStatusCode.NotFound, await Served(url)); + Assert.Equal(HttpStatusCode.NotFound, await Served(preview)); + Assert.Equal("post deleted", (await DB.Default.Find().OneAsync(id, Token)).TrashReason); + // the file waits in the trash, beside what /media/files serves + var relative = url[(url.IndexOf("/media/files/", StringComparison.Ordinal) + "/media/files/".Length)..]; + Assert.True(File.Exists(Path.Combine(_host.Get().TrashRoot, relative))); + Assert.Equal(HttpStatusCode.NotFound, await Served(url.Replace("/media/files/", "/media/files/.trash/"))); + } + + [Fact] + public async Task An_edit_that_leaves_media_out_trashes_them() + { + var alice = await _host.Mastodon("alice"); + var kept = await Upload(alice, "kept.jpg"); + var dropped = await Upload(alice, "dropped.jpg"); + var status = (await alice.Client.Post("/api/v1/statuses", ("status", "two pictures"), ("media_ids[]", kept), ("media_ids[]", dropped))).Ok(); + var urls = status.Body["media_attachments"]!.AsArray().ToDictionary(m => m!["id"]!.GetValue(), m => m!["url"]!.GetValue()); + + (await alice.Client.Put($"/api/v1/statuses/{status.Body.Text("id")}", ("status", "one picture"), ("media_ids[]", kept))).Ok(); + + Assert.Equal(HttpStatusCode.OK, await Served(urls[kept])); + Assert.Equal(HttpStatusCode.NotFound, await Served(urls[dropped])); + Assert.Null((await DB.Default.Find().OneAsync(kept, Token)).TrashedAt); + Assert.Equal("edited out", (await DB.Default.Find().OneAsync(dropped, Token)).TrashReason); + } + + [Fact] + public async Task A_replaced_avatar_is_trashed_and_a_picture_that_is_not_one_is_refused() + { + var alice = await _host.Mastodon("alice"); + async Task Picture(byte[] bytes, string type, string name) => + await alice.Client.Exchange(new HttpRequestMessage(HttpMethod.Patch, "/api/v1/accounts/update_credentials") + { + Content = MastodonHelpers.Multipart(("avatar", bytes, type, name)) + }); + var first = (await Picture(MastodonHelpers.JpegWithMetadata(300, 300), "image/jpeg", "one.jpg")).Ok().Body.Text("avatar"); + var second = (await Picture(MastodonHelpers.JpegWithMetadata(320, 300), "image/jpeg", "two.jpg")).Ok().Body.Text("avatar"); + + Assert.NotEqual(first, second); + Assert.Equal(HttpStatusCode.NotFound, await Served(first)); + Assert.Equal(HttpStatusCode.OK, await Served(second)); + var row = await DB.Default.Find().Match(m => m.ProfileOfAvatarId == alice.Persona.Id && m.Kind == "avatar" && m.TrashedAt == null).ExecuteSingleAsync(Token); + Assert.EndsWith(row.FilePath, second); + + // not an image: 422, and the avatar stays as it was + var refused = await Picture(""u8.ToArray(), "image/svg+xml", "x.svg"); + Assert.Equal(HttpStatusCode.UnprocessableEntity, refused.Status); + Assert.Equal(second, (await alice.Client.Get("/api/v1/accounts/verify_credentials")).Ok().Body.Text("avatar")); + } + + [Fact] + public async Task A_removed_roots_media_and_pictures_are_trashed_and_its_scheduled_posts_dropped() + { + var gone = await _host.Mastodon($"gone{Guid.NewGuid():N}"[..12]); + var posted = await Upload(gone, "posted.jpg"); + var url = (await gone.Client.Post("/api/v1/statuses", ("status", "soon gone"), ("media_ids[]", posted))).Ok() + .Body["media_attachments"]![0]!["url"]!.GetValue(); + var held = await Upload(gone, "held.jpg"); + (await gone.Client.Post("/api/v1/statuses", ("status", "later"), ("media_ids[]", held), + ("scheduled_at", DateTime.UtcNow.AddHours(2).ToString("O")))).Ok(); + var avatar = (await gone.Client.Exchange(new HttpRequestMessage(HttpMethod.Patch, "/api/v1/accounts/update_credentials") + { + Content = MastodonHelpers.Multipart(("avatar", MastodonHelpers.JpegWithMetadata(200, 200), "image/jpeg", "me.jpg")) + })).Ok().Body.Text("avatar"); + + Assert.True(await _host.Get().Remove(gone.Persona.Root.Id, Token)); + + Assert.Equal(HttpStatusCode.NotFound, await Served(url)); + Assert.Equal(HttpStatusCode.NotFound, await Served(avatar)); + Assert.False(await DB.Default.Find().Match(m => m.OwnerAvatarId == gone.Persona.Id && m.TrashedAt == null).ExecuteAnyAsync(Token)); + Assert.False(await DB.Default.Find().Match(s => s.AvatarId == gone.Persona.Id).ExecuteAnyAsync(Token)); + } + + [Fact] + public async Task The_audit_adopts_todays_pictures_and_trashes_what_nothing_holds() + { + var media = _host.Get(); + var bob = await _host.Mastodon($"bob{Guid.NewGuid():N}"[..12]); + // a picture shown from before pictures had rows, and a leftover nothing holds + var picture = $"2020/01/{Guid.NewGuid():N}.jpg"; + var leftover = $"2020/01/{Guid.NewGuid():N}.jpg"; + foreach (var relative in new[] { picture, leftover }) + { + Directory.CreateDirectory(Path.GetDirectoryName(Path.Combine(media.Root, relative))!); + await File.WriteAllBytesAsync(Path.Combine(media.Root, relative), new byte[] { 1, 2, 3 }, Token); + } + await DB.Default.Update().MatchID(bob.Persona.Id).Modify(a => a.PictureURL, media.Url(picture)).ExecuteAsync(Token); + + var looked = await MediaAudit.Run(media, fix: false, Token); + Assert.True(looked.Adopted >= 1); + Assert.True(looked.Unheld >= 1); + Assert.True(File.Exists(Path.Combine(media.Root, leftover))); + + await MediaAudit.Run(media, fix: true, Token); + + Assert.True(await DB.Default.Find().Match(m => m.ProfileOfAvatarId == bob.Persona.Id && m.FilePath == picture && m.TrashedAt == null).ExecuteAnyAsync(Token)); + Assert.Equal(HttpStatusCode.OK, await Served(media.Url(picture))); + Assert.Equal(HttpStatusCode.NotFound, await Served(media.Url(leftover))); + Assert.True(File.Exists(Path.Combine(media.TrashRoot, leftover))); + Assert.True(await DB.Default.Find().Match(m => m.FilePath == leftover && m.TrashedAt != null).ExecuteAnyAsync(Token)); + } + } +} diff --git a/PrivaPub/Api/Mastodon/Controllers/AccountsController.cs b/PrivaPub/Api/Mastodon/Controllers/AccountsController.cs index 3347e42..3f6aa78 100644 --- a/PrivaPub/Api/Mastodon/Controllers/AccountsController.cs +++ b/PrivaPub/Api/Mastodon/Controllers/AccountsController.cs @@ -51,13 +51,28 @@ namespace PrivaPub.Api.Mastodon.Controllers public async Task UpdateCredentials([FromServices] IMediaService media, CancellationToken token) { var avatar = await _dbEntities.Avatars.MatchID(Me.Id).ExecuteFirstAsync(token); + // a new picture is a row of its own; the one it replaces goes to the trash once the profile is saved + var replaced = new List(); if (Request.HasFormContentType) { var form = await Request.ReadFormAsync(token); - if (form.Files["avatar"] is { } picture && await media.ProfileImage(picture, 400, 400, token) is { } pictureUrl) - avatar.PictureURL = pictureUrl; - if (form.Files["header"] is { } header && await media.ProfileImage(header, 1500, 500, token) is { } headerUrl) - avatar.ThumbnailURL = headerUrl; + foreach (var (field, kind, width, height) in new[] { ("avatar", "avatar", 400, 400), ("header", "header", 1500, 500) }) + { + if (form.Files[field] is not { } file) + continue; + var outcome = await media.ProfileImage(avatar.ID, kind, file, width, height, token); + if (!outcome.Ok) + { + foreach (var made in replaced) + await media.Trash(m => m.ID == made.ID, "profile not saved", token); + return Error(outcome.Status, outcome.Error); + } + replaced.Add(outcome.Attachment); + if (kind == "avatar") + avatar.PictureURL = media.Url(outcome.Attachment.FilePath); + else + avatar.ThumbnailURL = media.Url(outcome.Attachment.FilePath); + } } if (Params.Has("display_name")) avatar.Name = Params.Get("display_name")?.Trim(); @@ -98,6 +113,8 @@ namespace PrivaPub.Api.Mastodon.Controllers avatar.Fields = fields; avatar.UpdatedAt = DateTime.UtcNow; await DB.Default.SaveAsync(avatar, token); + foreach (var made in replaced) + await media.Trash(m => m.ProfileOfAvatarId == avatar.ID && m.Kind == made.Kind && m.ID != made.ID, "replaced", token); var actor = _localActors.FromAvatar(avatar); await _outbox.PublishProfile(actor, token); diff --git a/PrivaPub/Api/Mastodon/Controllers/MediaController.cs b/PrivaPub/Api/Mastodon/Controllers/MediaController.cs index 0eec267..2aee672 100644 --- a/PrivaPub/Api/Mastodon/Controllers/MediaController.cs +++ b/PrivaPub/Api/Mastodon/Controllers/MediaController.cs @@ -42,14 +42,14 @@ namespace PrivaPub.Api.Mastodon.Controllers [HttpGet("/api/v1/media/{id}"), Scope("write:media")] public async Task Get(string id, CancellationToken token) { - var attachment = await DB.Default.Find().Match(m => m.ID == id && m.OwnerAvatarId == MyId).ExecuteFirstAsync(token); + var attachment = await DB.Default.Find().Match(m => m.ID == id && m.OwnerAvatarId == MyId && m.TrashedAt == null && m.ProfileOfAvatarId == null).ExecuteFirstAsync(token); return attachment == default ? NotFoundError() : Json(View(attachment)); } [HttpPut("/api/v1/media/{id}"), Scope("write:media")] public async Task Update(string id, CancellationToken token) { - var attachment = await DB.Default.Find().Match(m => m.ID == id && m.OwnerAvatarId == MyId).ExecuteFirstAsync(token); + var attachment = await DB.Default.Find().Match(m => m.ID == id && m.OwnerAvatarId == MyId && m.TrashedAt == null && m.ProfileOfAvatarId == null).ExecuteFirstAsync(token); if (attachment == default) return NotFoundError(); if (Params.Has("description")) diff --git a/PrivaPub/Api/Mastodon/Controllers/ScheduledStatusesController.cs b/PrivaPub/Api/Mastodon/Controllers/ScheduledStatusesController.cs index d8c6f47..47a0932 100644 --- a/PrivaPub/Api/Mastodon/Controllers/ScheduledStatusesController.cs +++ b/PrivaPub/Api/Mastodon/Controllers/ScheduledStatusesController.cs @@ -96,7 +96,8 @@ namespace PrivaPub.Api.Mastodon.Controllers if (await Mine(id, token) is not { } scheduled) return NotFoundError(); await DB.Default.DeleteAsync(scheduled.ID); - await ScheduledStatuses.Release(scheduled.ID, token); + // its media were the scheduled post's alone, never posted + await _media.Trash(m => m.ScheduledStatusId == scheduled.ID, "scheduled post dropped", token); return Json(new { }); } } diff --git a/PrivaPub/Api/Mastodon/Controllers/StatusesController.cs b/PrivaPub/Api/Mastodon/Controllers/StatusesController.cs index 8960f45..8611564 100644 --- a/PrivaPub/Api/Mastodon/Controllers/StatusesController.cs +++ b/PrivaPub/Api/Mastodon/Controllers/StatusesController.cs @@ -111,7 +111,8 @@ namespace PrivaPub.Api.Mastodon.Controllers return Error(StatusCodes.Status422UnprocessableEntity, "Validation failed: Text can't be blank"); var mediaIds = asked.MediaIds.Distinct().ToList(); if (mediaIds.Count > 4 || mediaIds.Count > 0 && await DB.Default.CountAsync( - m => mediaIds.Contains(m.ID) && m.OwnerAvatarId == MyId && m.PostId == null && m.ScheduledStatusId == null, token) != mediaIds.Count) + m => mediaIds.Contains(m.ID) && m.OwnerAvatarId == MyId && m.PostId == null && m.ScheduledStatusId == null + && m.TrashedAt == null && m.ProfileOfAvatarId == null, token) != mediaIds.Count) return Error(StatusCodes.Status422UnprocessableEntity, "Validation failed: Media attachments are not yours, already posted or too many"); var scheduled = new ScheduledStatus { AvatarId = MyId, ScheduledAt = at, Params = asked }; diff --git a/PrivaPub/Domain/Media/MediaAudit.cs b/PrivaPub/Domain/Media/MediaAudit.cs new file mode 100644 index 0000000..63456f3 --- /dev/null +++ b/PrivaPub/Domain/Media/MediaAudit.cs @@ -0,0 +1,101 @@ +using MongoDB.Entities; + +using PrivaPub.Models.Media; +using PrivaPub.Models.User; + +using PostEntity = PrivaPub.Models.Post.Post; + +namespace PrivaPub.Domain.Media +{ + // What the media directory holds against what the database says holds it, and, with fix, the two made to agree: + // - a persona's current avatar or header with no row (they had none before) becomes a row of its own; + // - media of a deleted post or of a deleted persona go to the trash; + // - rows whose files are gone go to the trash; + // - files nothing holds (deleted posts' media and replaced pictures, from before media were trashed) get a trashed row + // each. + // Trashed files leave what /media/files serves at once, and the janitor deletes them after its grace. Run it without + // fix first; with fix it must run as the user that owns the media (www-data). + public static class MediaAudit + { + public sealed record Report(int Files, int Held, int Adopted, int OfDeleted, int MissingFiles, int Unheld, IReadOnlyList Examples, bool Fixed); + + public static async Task Run(IMediaService media, bool fix, CancellationToken token) + { + var files = Served(media.Root); + var rows = await DB.Default.Find().Match(m => m.TrashedAt == null).ExecuteAsync(token); + var held = rows.SelectMany(r => new[] { r.FilePath, r.PreviewPath }).Where(p => !string.IsNullOrEmpty(p)).ToHashSet(); + var examples = new List(); + + // pictures personas show today, kept as rows from now on + var prefix = media.Url(string.Empty); + var adopted = 0; + foreach (var avatar in await DB.Default.Find().Match(a => !a.DeletionAt.HasValue && (a.PictureURL != null || a.ThumbnailURL != null)).ExecuteAsync(token)) + foreach (var (url, kind) in new[] { (avatar.PictureURL, "avatar"), (avatar.ThumbnailURL, "header") }) + { + if (url?.StartsWith(prefix, StringComparison.Ordinal) != true) + continue; + var relative = url[prefix.Length..]; + if (held.Contains(relative) || !files.Contains(relative)) + continue; + adopted++; + held.Add(relative); + if (fix) + await DB.Default.SaveAsync(new MediaAttachment + { + OwnerAvatarId = avatar.ID, + ProfileOfAvatarId = avatar.ID, + Kind = kind, + ContentType = "image/jpeg", + FilePath = relative, + Size = new FileInfo(Path.Combine(media.Root, relative)).Length, + AttachedAt = DateTime.UtcNow + }, token); + } + + // held by something that is gone: a deleted post, a deleted persona + var postIds = rows.Where(r => r.PostId != null).Select(r => r.PostId).Distinct().ToList(); + var livePosts = (await DB.Default.Find().Match(p => postIds.Contains(p.ID) && !p.DeletedAt.HasValue).Project(p => p.Include(x => x.ID)).ExecuteAsync(token)) + .Select(p => p.ID).ToHashSet(); + var ownerIds = rows.Select(r => r.OwnerAvatarId).Where(id => id != null).Distinct().ToList(); + var liveOwners = (await DB.Default.Find().Match(a => ownerIds.Contains(a.ID) && !a.DeletionAt.HasValue).Project(a => a.Include(x => x.ID)).ExecuteAsync(token)) + .Select(a => a.ID).ToHashSet(); + var ofDeleted = rows.Where(r => (r.PostId != null && !livePosts.Contains(r.PostId)) || (r.OwnerAvatarId != null && !liveOwners.Contains(r.OwnerAvatarId))).ToList(); + var missing = rows.Except(ofDeleted).Where(r => !string.IsNullOrEmpty(r.FilePath) && !files.Contains(r.FilePath)).ToList(); + if (fix) + foreach (var row in ofDeleted.Concat(missing)) + await media.Trash(m => m.ID == row.ID, ofDeleted.Contains(row) ? "audit: its holder is deleted" : "audit: its file is missing", token); + foreach (var row in ofDeleted) + held.Remove(row.FilePath); + + // files nothing holds + var unheld = files.Where(f => !held.Contains(f)).OrderBy(f => f, StringComparer.Ordinal).ToList(); + examples.AddRange(unheld.Take(10)); + if (fix) + foreach (var relative in unheld) + { + var orphan = new MediaAttachment + { + Kind = "orphan", + FilePath = relative, + Size = new FileInfo(Path.Combine(media.Root, relative)).Length, + TrashedAt = DateTime.UtcNow, + TrashReason = "audit: nothing holds it" + }; + await DB.Default.SaveAsync(orphan, token); + media.Hide(orphan); + } + + return new Report(files.Count, rows.Count, adopted, ofDeleted.Count, missing.Count, unheld.Count, examples, fix); + } + + // every file /media/files serves: everything under the root + static HashSet Served(string root) + { + if (!Directory.Exists(root)) + return new HashSet(); + return Directory.EnumerateFiles(root, "*", SearchOption.AllDirectories) + .Select(f => Path.GetRelativePath(root, f).Replace('\\', '/')) + .ToHashSet(); + } + } +} diff --git a/PrivaPub/Domain/Media/MediaProxy.cs b/PrivaPub/Domain/Media/MediaProxy.cs index 1cc5ea3..920afc1 100644 --- a/PrivaPub/Domain/Media/MediaProxy.cs +++ b/PrivaPub/Domain/Media/MediaProxy.cs @@ -123,8 +123,11 @@ namespace PrivaPub.Domain.Media public class MediaJanitor : BackgroundService { + static readonly TimeSpan FirstPass = TimeSpan.FromMinutes(5); static readonly TimeSpan Interval = TimeSpan.FromHours(1); static readonly TimeSpan UnattachedLifetime = TimeSpan.FromDays(1); + // a trashed file waits this long before it is deleted, already out of what /media/files serves + public static readonly TimeSpan TrashGrace = TimeSpan.FromDays(1); readonly IMediaService _media; readonly IOptionsMonitor _options; @@ -139,11 +142,13 @@ namespace PrivaPub.Domain.Media protected override async Task ExecuteAsync(CancellationToken stoppingToken) { + var wait = FirstPass; while (!stoppingToken.IsCancellationRequested) { try { - await Task.Delay(Interval, stoppingToken); + await Task.Delay(wait, stoppingToken); + wait = Interval; await Sweep(stoppingToken); } catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) @@ -157,12 +162,33 @@ namespace PrivaPub.Domain.Media } } - //one pass: uploads left unattached for a day go (unless a scheduled post waits for them), then the proxy cache is trimmed to its size, oldest first + // one pass: + // - uploads never posted for a day (and not waiting for a scheduled post, nor a profile picture) go to the trash; + // - trashed files still served (a crash between the mark and the move) are moved out; + // - trashed files past their grace are deleted, with their rows; + // - the proxy cache is trimmed to its size, oldest first public async Task Sweep(CancellationToken token) { var cutoff = DateTime.UtcNow - UnattachedLifetime; - foreach (var stale in await DB.Default.Find().Match(m => m.PostId == null && m.ScheduledStatusId == null && m.CreatedAt < cutoff).Limit(500).ExecuteAsync(token)) - await _media.Delete(stale); + var unattached = await _media.Trash(m => m.PostId == null && m.ScheduledStatusId == null && m.ProfileOfAvatarId == null && m.CreatedAt < cutoff, + "never posted", token); + + var trashed = await DB.Default.Find().Match(m => m.TrashedAt != null).Limit(2000).ExecuteAsync(token); + var purgeBefore = DateTime.UtcNow - TrashGrace; + var purged = 0; + foreach (var row in trashed) + { + if (row.TrashedAt < purgeBefore) + { + await _media.Purge(row, token); + purged++; + } + else + _media.Hide(row); + } + if (unattached > 0 || purged > 0) + _logger.LogInformation("{Service}: {Unattached} uploads never posted trashed, {Purged} trashed files deleted", + nameof(MediaJanitor), unattached, purged); TrimProxyCache(); } diff --git a/PrivaPub/Domain/Media/MediaService.cs b/PrivaPub/Domain/Media/MediaService.cs index 5542de3..4ad63ce 100644 --- a/PrivaPub/Domain/Media/MediaService.cs +++ b/PrivaPub/Domain/Media/MediaService.cs @@ -27,9 +27,20 @@ namespace PrivaPub.Domain.Media string Root { get; } string ProxyRoot { get; } string Url(string relativePath); + /// Where trashed files wait for the janitor: the sibling of Root, on the same disk (a move is a rename) and + /// outside what /media/files serves. (Not a dot-prefixed folder inside Root: the file provider only hides a file whose + /// own name starts with a dot.) + string TrashRoot { get; } Task Upload(LocalActor owner, IFormFile file, string description, string focus, CancellationToken token); - Task ProfileImage(IFormFile file, int width, int height, CancellationToken token); - Task Delete(MediaAttachment attachment); + /// A persona's new avatar or header ("avatar" or "header"), cropped to its size, as a row of its own. + Task ProfileImage(string avatarId, string kind, IFormFile file, int width, int height, CancellationToken token); + /// Trashes the media matches (and that isn't trashed yet): each row is marked in one + /// conditional update, so a row attached meanwhile is left alone, and its files stop being served at once. + Task Trash(System.Linq.Expressions.Expression> which, string reason, CancellationToken token); + /// Moves a trashed row's files out of what is served, if they are still there (a crash between the mark and the move). + void Hide(MediaAttachment trashed); + /// Deletes a trashed row's files and the row. + Task Purge(MediaAttachment trashed, CancellationToken token); } public class MediaService : IMediaService @@ -62,6 +73,8 @@ namespace PrivaPub.Domain.Media public string ProxyRoot => Root.TrimEnd(Path.DirectorySeparatorChar) + "-proxy"; + public string TrashRoot => Root.TrimEnd(Path.DirectorySeparatorChar) + "-trash"; + public string Url(string relativePath) => relativePath == default ? default : $"{_localActors.BaseAddress}/media/files/{relativePath.Replace('\\', '/')}"; public async Task Upload(LocalActor owner, IFormFile file, string description, string focus, CancellationToken token) @@ -118,34 +131,86 @@ namespace PrivaPub.Domain.Media return new MediaOutcome(attachment); } - public async Task ProfileImage(IFormFile file, int width, int height, CancellationToken token) + public async Task ProfileImage(string avatarId, string kind, IFormFile file, int width, int height, CancellationToken token) { - if (file == default || file.Length == 0 || file.Length > _options.CurrentValue.MaxImageBytes) - return default; + var contentType = file?.ContentType?.Split(';')[0].Trim().ToLowerInvariant(); + if (file == default || file.Length == 0) + return MediaOutcome.Fail(StatusCodes.Status422UnprocessableEntity, "Validation failed: File can't be blank"); + if (!ImageTypes.Contains(contentType)) + return MediaOutcome.Fail(StatusCodes.Status422UnprocessableEntity, "Validation failed: File type is not supported"); + if (file.Length > _options.CurrentValue.MaxImageBytes) + return MediaOutcome.Fail(StatusCodes.Status422UnprocessableEntity, "Validation failed: File is too big"); await using var stream = file.OpenReadStream(); using var buffer = new MemoryStream(); await stream.CopyToAsync(buffer, token); + byte[] bytes; + int outWidth, outHeight; try { using var image = Image.ThumbnailBuffer(buffer.ToArray(), width, height: height, crop: Enums.Interesting.Centre, size: Enums.Size.Down); using var flat = Flatten(image); - return Url(await Save(flat.WriteToBuffer(".jpg[Q=85,keep=none]"), "jpg", token)); + bytes = flat.WriteToBuffer(".jpg[Q=85,keep=none]"); + (outWidth, outHeight) = (flat.Width, flat.Height); } - catch (VipsException) + catch (VipsException ex) { - return default; + _logger.LogInformation("Refused a profile picture: {Error}", ex.Message); + return MediaOutcome.Fail(StatusCodes.Status422UnprocessableEntity, "Validation failed: The file is not a readable image"); + } + var attachment = new MediaAttachment + { + OwnerAvatarId = avatarId, + ProfileOfAvatarId = avatarId, + Kind = kind, + ContentType = "image/jpeg", + FilePath = await Save(bytes, "jpg", token), + Size = bytes.Length, + Width = outWidth, + Height = outHeight, + AttachedAt = DateTime.UtcNow + }; + await DB.Default.SaveAsync(attachment, token); + return new MediaOutcome(attachment); + } + + public async Task Trash(System.Linq.Expressions.Expression> which, string reason, CancellationToken token) + { + var trashed = 0L; + foreach (var candidate in await DB.Default.Find().Match(which).Match(m => m.TrashedAt == null).ExecuteAsync(token)) + { + var marked = await DB.Default.Update() + .Match(which).Match(m => m.ID == candidate.ID && m.TrashedAt == null) + .Modify(m => m.TrashedAt, DateTime.UtcNow) + .Modify(m => m.TrashReason, reason) + .ExecuteAsync(token); + if (marked.ModifiedCount == 0) + continue; + Hide(candidate); + trashed++; + } + return trashed; + } + + public void Hide(MediaAttachment trashed) + { + foreach (var relative in new[] { trashed.FilePath, trashed.PreviewPath }.Where(p => !string.IsNullOrEmpty(p))) + { + var served = Path.Combine(Root, relative); + if (!File.Exists(served)) + continue; + var hidden = Path.Combine(TrashRoot, relative); + Directory.CreateDirectory(Path.GetDirectoryName(hidden)!); + File.Move(served, hidden, overwrite: true); } } - public Task Delete(MediaAttachment attachment) + public async Task Purge(MediaAttachment trashed, CancellationToken token) { - foreach (var path in new[] { attachment.FilePath, attachment.PreviewPath }.Where(p => p != default)) - { - var full = Path.Combine(Root, path); - if (File.Exists(full)) - File.Delete(full); - } - return DB.Default.DeleteAsync(attachment.ID); + foreach (var relative in new[] { trashed.FilePath, trashed.PreviewPath }.Where(p => !string.IsNullOrEmpty(p))) + foreach (var full in new[] { Path.Combine(TrashRoot, relative), Path.Combine(Root, relative) }) + if (File.Exists(full)) + File.Delete(full); + await DB.Default.DeleteAsync(m => m.ID == trashed.ID && m.TrashedAt != null); } public static ProcessedImage ProcessImage(byte[] input, int maxSide, int previewSide, bool animated) diff --git a/PrivaPub/Domain/Statuses/StatusService.cs b/PrivaPub/Domain/Statuses/StatusService.cs index fefaf45..a58b434 100644 --- a/PrivaPub/Domain/Statuses/StatusService.cs +++ b/PrivaPub/Domain/Statuses/StatusService.cs @@ -320,6 +320,9 @@ namespace PrivaPub.Domain.Statuses { post.Media = media.Select(ToPostMedia).ToList(); await Attach(media, post.ID, token); + // what the edit left out is no longer held by anything + var kept = media.Select(m => m.ID).ToList(); + await _media.Trash(m => m.PostId == post.ID && !kept.Contains(m.ID), "edited out", token); } post.Title = draft.Title == default ? post.Title : Clean(draft.Title); post.SpoilerText = Clean(draft.SpoilerText); @@ -380,6 +383,7 @@ namespace PrivaPub.Domain.Statuses .Modify(p => p.Revisions, new List()) .ExecuteAsync(token); await Fanout.Deleting(post, token); + await _media.Trash(m => m.PostId == post.ID, "post deleted", token); await DB.Default.DeleteAsync(e => e.PostId == post.ID || e.ReblogOfPostId == post.ID); // boosts of it end with it, as on Mastodon, so no count keeps them await DB.Default.Update().Match(p => p.ReblogOfPostId == post.ID && !p.DeletedAt.HasValue) @@ -692,7 +696,8 @@ namespace PrivaPub.Domain.Statuses return default; var wanted = ids.Distinct().ToList(); var found = await DB.Default.Find() - .Match(m => wanted.Contains(m.ID) && m.OwnerAvatarId == author.Id && (m.PostId == null || m.PostId == postId)) + .Match(m => wanted.Contains(m.ID) && m.OwnerAvatarId == author.Id && (m.PostId == null || m.PostId == postId) + && m.TrashedAt == null && m.ProfileOfAvatarId == null) .ExecuteAsync(token); return found.Count == wanted.Count ? wanted.Select(id => found.First(m => m.ID == id)).ToList() : default; } diff --git a/PrivaPub/Infrastructure/Cli/AdminCommands.cs b/PrivaPub/Infrastructure/Cli/AdminCommands.cs index a24f3e8..2827a97 100644 --- a/PrivaPub/Infrastructure/Cli/AdminCommands.cs +++ b/PrivaPub/Infrastructure/Cli/AdminCommands.cs @@ -22,6 +22,8 @@ namespace PrivaPub.Infrastructure.Cli usage: PrivaPub admin promote|demote PrivaPub admin create-root [--admin] the password is read from standard input PrivaPub admin smoke prints " " for the deploy's signed-in check + PrivaPub admin media audit [--fix] media files against what holds them; --fix (as www-data) trashes + what nothing holds and gives today's pictures their rows """; public static async Task Run(string[] args, IServiceProvider services, TextReader input = default, TextWriter output = default) @@ -36,12 +38,29 @@ namespace PrivaPub.Infrastructure.Cli return await CreateRoot(services, login, input.ReadLine(), flags.Contains("--admin"), output); case ["smoke", var persona]: return await Smoke(services, persona, output); + case ["media", "audit", .. var flags] when flags.All(f => f == "--fix"): + return await AuditMedia(services, flags.Contains("--fix"), output); default: Console.Error.WriteLine(Usage); return 2; } } + static async Task AuditMedia(IServiceProvider services, bool fix, TextWriter output) + { + var report = await Domain.Media.MediaAudit.Run(services.GetRequiredService(), fix, CancellationToken.None); + output.WriteLine($"{report.Files} files served, {report.Held} rows holding media"); + output.WriteLine($"{report.Adopted} pictures personas show without a row{(fix ? ": given one" : string.Empty)}"); + output.WriteLine($"{report.OfDeleted} media of deleted posts or personas{(fix ? ": trashed" : string.Empty)}"); + output.WriteLine($"{report.MissingFiles} rows whose files are missing{(fix ? ": trashed" : string.Empty)}"); + output.WriteLine($"{report.Unheld} files nothing holds{(fix ? ": trashed" : string.Empty)}"); + foreach (var example in report.Examples) + output.WriteLine($" {example}"); + if (!fix && report.Adopted + report.OfDeleted + report.MissingFiles + report.Unheld > 0) + output.WriteLine("run again with --fix, as www-data, to apply this; trashed files are deleted after a day"); + return 0; + } + static async Task Promote(bool promote, string userName, TextWriter output) { userName = userName.ToLowerInvariant(); diff --git a/PrivaPub/Infrastructure/Data/Indexes.cs b/PrivaPub/Infrastructure/Data/Indexes.cs index 1ad3218..6840e5c 100644 --- a/PrivaPub/Infrastructure/Data/Indexes.cs +++ b/PrivaPub/Infrastructure/Data/Indexes.cs @@ -32,6 +32,13 @@ namespace PrivaPub.Infrastructure.Data await Plain(token, p => p.AnsweringToPostId); await DB.Default.Index().Key(p => p.Geo, KeyType.Geo2DSphere).CreateAsync(token); + // media: a post's, a persona's, what a scheduled post holds, the janitor's never-posted uploads and its trash + await Plain(token, m => m.PostId, m => m.ScheduledStatusId, m => m.CreatedAt); + await Plain(token, m => m.OwnerAvatarId); + await Plain(token, m => m.ScheduledStatusId); + await Plain(token, m => m.ProfileOfAvatarId); + await Plain(token, m => m.TrashedAt); + await Unique(p => p.ObjectURI, Builders.Filter.Gt(p => p.ObjectURI, ""), token); await Plain(token, p => p.GroupId, p => p.ID); diff --git a/PrivaPub/Models/Media/MediaAttachment.cs b/PrivaPub/Models/Media/MediaAttachment.cs index e60527f..d9318b0 100644 --- a/PrivaPub/Models/Media/MediaAttachment.cs +++ b/PrivaPub/Models/Media/MediaAttachment.cs @@ -19,6 +19,11 @@ namespace PrivaPub.Models.Media public DateTime CreatedAt { get; set; } = DateTime.UtcNow; public DateTime? AttachedAt { get; set; } public string ScheduledStatusId { get; set; }//kept for a scheduled post until it is published or dropped + public string ProfileOfAvatarId { get; set; }//a persona's avatar or header (Kind "avatar" or "header"), never a post's + // set once nothing holds it any more (its post deleted, edited out, replaced, its root removed, never posted): its + // files move out of what /media/files serves at once, and the janitor deletes them after a grace + public DateTime? TrashedAt { get; set; } + public string TrashReason { get; set; } } public class MediaSecret : Entity diff --git a/PrivaPub/Services/RootRemoval.cs b/PrivaPub/Services/RootRemoval.cs index ac6f876..e35657f 100644 --- a/PrivaPub/Services/RootRemoval.cs +++ b/PrivaPub/Services/RootRemoval.cs @@ -7,6 +7,7 @@ using PrivaPub.Models.Federation; using PrivaPub.Models.Social; using PrivaPub.Models.User; using PrivaPub.StaticServices; +using PrivaPub.Domain.Media; using System.Text.Json.Nodes; @@ -30,9 +31,11 @@ namespace PrivaPub.Services readonly ILocalActorService _localActors; readonly IDeliveryService _delivery; readonly IRootSessions _sessions; + readonly IMediaService _media; - public RootRemoval(DbEntities dbEntities, ILocalActorService localActors, IDeliveryService delivery, IRootSessions sessions) + public RootRemoval(DbEntities dbEntities, ILocalActorService localActors, IDeliveryService delivery, IRootSessions sessions, IMediaService media) { + _media = media; _dbEntities = dbEntities; _localActors = localActors; _delivery = delivery; @@ -66,6 +69,10 @@ namespace PrivaPub.Services await DB.Default.DeleteAsync(l => l.AvatarId == avatar.ID); await DB.Default.DeleteAsync(f => f.AvatarId == avatar.ID); await DB.Default.DeleteAsync(t => t.AvatarId == avatar.ID); + // nothing of it publishes later, and none of its files stays served: its posts' media, its pictures and + // what its scheduled posts held + await DB.Default.DeleteAsync(s => s.AvatarId == avatar.ID); + await _media.Trash(m => m.OwnerAvatarId == avatar.ID, "root removed", token); } await DB.Default.Update().MatchID(root.ID)