A file lives exactly as long as something holds it

Deleting a post, editing media out, replacing an avatar or a header, and removing a whole root deleted no file: every
one stayed on disk and publicly served from /media/files with a year-long immutable cache, its row orphaned. Now every
upload is a row, profile pictures too (Kind avatar or header, ProfileOfAvatarId), and each of those acts trashes what
it held, as does an upload never posted for a day and a dropped scheduled post. A trashed row is marked in one
conditional update (an upload attached meanwhile is left alone), its files move into media-trash, beside the media
root and outside what /media/files serves, and the janitor deletes them a day later. Nothing is deleted for looking
unused.

Along the way: a profile picture that isn't an image, or can't be read, answers 422 instead of being silently ignored
with a 200; a removed root's scheduled posts are dropped, so nothing of it publishes later; media rows get indexes
(they had none), and the janitor's first pass comes five minutes after boot instead of an hour.

`PrivaPub admin media audit [--fix]` compares the disk with the database. With --fix (as www-data) it gives the
pictures personas show today a row, and trashes media of deleted posts or personas, rows whose files are missing, and
files nothing holds: the leftovers of every deletion until now. MediaLifecycleTests covers each act, that the trash is
never served, and the audit.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01LsXgEaXee4GCU1hwYgPJXw

PrivaPub.Tests/Federation/JobHandlerTests.cs

@@ -489,9 +489,17 @@ namespace PrivaPub.Tests.Federation
 
 				await new MediaJanitor(media, options, NullLogger<MediaJanitor>.Instance).Sweep(token);
 
-				Assert.False(await DB.Default.Find<MediaAttachment>().Match(m => m.ID == stale.ID).ExecuteAnyAsync(token));
+				// never posted for a day: trashed, its files out of what /media/files serves at once
+				Assert.NotNull((await DB.Default.Find<MediaAttachment>().OneAsync(stale.ID, token)).TrashedAt);
 				Assert.False(File.Exists(Path.Combine(root, stale.FilePath)));
 				Assert.False(File.Exists(Path.Combine(root, stale.PreviewPath)));
+				Assert.True(File.Exists(Path.Combine(media.TrashRoot, stale.FilePath)));
+				// and once its grace has passed, deleted with its row
+				await DB.Default.Update<MediaAttachment>().MatchID(stale.ID).Modify(m => m.TrashedAt, DateTime.UtcNow - MediaJanitor.TrashGrace - TimeSpan.FromMinutes(1)).ExecuteAsync(token);
+				await new MediaJanitor(media, options, NullLogger<MediaJanitor>.Instance).Sweep(token);
+				Assert.False(await DB.Default.Find<MediaAttachment>().Match(m => m.ID == stale.ID).ExecuteAnyAsync(token));
+				Assert.False(File.Exists(Path.Combine(media.TrashRoot, stale.FilePath)));
+				Assert.False(File.Exists(Path.Combine(media.TrashRoot, stale.PreviewPath)));
 				Assert.True(await DB.Default.Find<MediaAttachment>().Match(m => m.ID == fresh.ID).ExecuteAnyAsync(token));
 				Assert.True(await DB.Default.Find<MediaAttachment>().Match(m => m.ID == attached.ID).ExecuteAnyAsync(token));
 				Assert.True(File.Exists(Path.Combine(root, fresh.FilePath)));

PrivaPub.Tests/Http/MediaLifecycleTests.cs

@@ -0,0 +1,153 @@
+using MongoDB.Entities;
+
+using PrivaPub.Domain.Media;
+using PrivaPub.Models.Media;
+using PrivaPub.Services;
+using PrivaPub.Tests.Support;
+using PrivaPub.Tests.Support.Host;
+
+using System.Net;
+
+namespace PrivaPub.Tests.Http
+{
+	// A file lives exactly as long as something holds it: a deleted post, an edit leaving media out, a replaced picture and
+	// a removed root each trash theirs, which stops /media/files serving them at once; the janitor deletes them later.
+	[Trait("Category", "Integration")]
+	public sealed class MediaLifecycleTests : IAsyncLifetime
+	{
+		PrivaPubHost _host;
+
+		public async ValueTask InitializeAsync()
+		{
+			Assert.SkipUnless(MongoFixture.Enabled, MongoFixture.Skip);
+			_host = await PrivaPubHost.Shared();
+		}
+
+		public ValueTask DisposeAsync() => ValueTask.CompletedTask;
+
+		static CancellationToken Token => TestContext.Current.CancellationToken;
+
+		async Task<string> Upload(Mastodon account, string name)
+		{
+			var form = MastodonHelpers.Multipart(("file", MastodonHelpers.JpegWithMetadata(64, 48), "image/jpeg", name));
+			return (await account.Client.Exchange(new HttpRequestMessage(HttpMethod.Post, "/api/v2/media") { Content = form })).Ok().Body.Text("id");
+		}
+
+		async Task<HttpStatusCode> Served(string url) =>
+			(await _host.Client().GetAsync(new Uri(url).PathAndQuery, Token)).StatusCode;
+
+		[Fact]
+		public async Task A_deleted_posts_media_stop_being_served()
+		{
+			var alice = await _host.Mastodon("alice");
+			var id = await Upload(alice, "a.jpg");
+			var status = (await alice.Client.Post("/api/v1/statuses", ("status", "with a picture"), ("media_ids[]", id))).Ok();
+			var url = status.Body["media_attachments"]![0]!["url"]!.GetValue<string>();
+			var preview = status.Body["media_attachments"]![0]!["preview_url"]!.GetValue<string>();
+			Assert.Equal(HttpStatusCode.OK, await Served(url));
+
+			(await alice.Client.Delete($"/api/v1/statuses/{status.Body.Text("id")}")).Ok();
+
+			Assert.Equal(HttpStatusCode.NotFound, await Served(url));
+			Assert.Equal(HttpStatusCode.NotFound, await Served(preview));
+			Assert.Equal("post deleted", (await DB.Default.Find<MediaAttachment>().OneAsync(id, Token)).TrashReason);
+			// the file waits in the trash, beside what /media/files serves
+			var relative = url[(url.IndexOf("/media/files/", StringComparison.Ordinal) + "/media/files/".Length)..];
+			Assert.True(File.Exists(Path.Combine(_host.Get<IMediaService>().TrashRoot, relative)));
+			Assert.Equal(HttpStatusCode.NotFound, await Served(url.Replace("/media/files/", "/media/files/.trash/")));
+		}
+
+		[Fact]
+		public async Task An_edit_that_leaves_media_out_trashes_them()
+		{
+			var alice = await _host.Mastodon("alice");
+			var kept = await Upload(alice, "kept.jpg");
+			var dropped = await Upload(alice, "dropped.jpg");
+			var status = (await alice.Client.Post("/api/v1/statuses", ("status", "two pictures"), ("media_ids[]", kept), ("media_ids[]", dropped))).Ok();
+			var urls = status.Body["media_attachments"]!.AsArray().ToDictionary(m => m!["id"]!.GetValue<string>(), m => m!["url"]!.GetValue<string>());
+
+			(await alice.Client.Put($"/api/v1/statuses/{status.Body.Text("id")}", ("status", "one picture"), ("media_ids[]", kept))).Ok();
+
+			Assert.Equal(HttpStatusCode.OK, await Served(urls[kept]));
+			Assert.Equal(HttpStatusCode.NotFound, await Served(urls[dropped]));
+			Assert.Null((await DB.Default.Find<MediaAttachment>().OneAsync(kept, Token)).TrashedAt);
+			Assert.Equal("edited out", (await DB.Default.Find<MediaAttachment>().OneAsync(dropped, Token)).TrashReason);
+		}
+
+		[Fact]
+		public async Task A_replaced_avatar_is_trashed_and_a_picture_that_is_not_one_is_refused()
+		{
+			var alice = await _host.Mastodon("alice");
+			async Task<ApiAnswer> Picture(byte[] bytes, string type, string name) =>
+				await alice.Client.Exchange(new HttpRequestMessage(HttpMethod.Patch, "/api/v1/accounts/update_credentials")
+				{
+					Content = MastodonHelpers.Multipart(("avatar", bytes, type, name))
+				});
+			var first = (await Picture(MastodonHelpers.JpegWithMetadata(300, 300), "image/jpeg", "one.jpg")).Ok().Body.Text("avatar");
+			var second = (await Picture(MastodonHelpers.JpegWithMetadata(320, 300), "image/jpeg", "two.jpg")).Ok().Body.Text("avatar");
+
+			Assert.NotEqual(first, second);
+			Assert.Equal(HttpStatusCode.NotFound, await Served(first));
+			Assert.Equal(HttpStatusCode.OK, await Served(second));
+			var row = await DB.Default.Find<MediaAttachment>().Match(m => m.ProfileOfAvatarId == alice.Persona.Id && m.Kind == "avatar" && m.TrashedAt == null).ExecuteSingleAsync(Token);
+			Assert.EndsWith(row.FilePath, second);
+
+			// not an image: 422, and the avatar stays as it was
+			var refused = await Picture("<svg xmlns='http://www.w3.org/2000/svg'/>"u8.ToArray(), "image/svg+xml", "x.svg");
+			Assert.Equal(HttpStatusCode.UnprocessableEntity, refused.Status);
+			Assert.Equal(second, (await alice.Client.Get("/api/v1/accounts/verify_credentials")).Ok().Body.Text("avatar"));
+		}
+
+		[Fact]
+		public async Task A_removed_roots_media_and_pictures_are_trashed_and_its_scheduled_posts_dropped()
+		{
+			var gone = await _host.Mastodon($"gone{Guid.NewGuid():N}"[..12]);
+			var posted = await Upload(gone, "posted.jpg");
+			var url = (await gone.Client.Post("/api/v1/statuses", ("status", "soon gone"), ("media_ids[]", posted))).Ok()
+				.Body["media_attachments"]![0]!["url"]!.GetValue<string>();
+			var held = await Upload(gone, "held.jpg");
+			(await gone.Client.Post("/api/v1/statuses", ("status", "later"), ("media_ids[]", held),
+				("scheduled_at", DateTime.UtcNow.AddHours(2).ToString("O")))).Ok();
+			var avatar = (await gone.Client.Exchange(new HttpRequestMessage(HttpMethod.Patch, "/api/v1/accounts/update_credentials")
+			{
+				Content = MastodonHelpers.Multipart(("avatar", MastodonHelpers.JpegWithMetadata(200, 200), "image/jpeg", "me.jpg"))
+			})).Ok().Body.Text("avatar");
+
+			Assert.True(await _host.Get<IRootRemoval>().Remove(gone.Persona.Root.Id, Token));
+
+			Assert.Equal(HttpStatusCode.NotFound, await Served(url));
+			Assert.Equal(HttpStatusCode.NotFound, await Served(avatar));
+			Assert.False(await DB.Default.Find<MediaAttachment>().Match(m => m.OwnerAvatarId == gone.Persona.Id && m.TrashedAt == null).ExecuteAnyAsync(Token));
+			Assert.False(await DB.Default.Find<Models.Social.ScheduledStatus>().Match(s => s.AvatarId == gone.Persona.Id).ExecuteAnyAsync(Token));
+		}
+
+		[Fact]
+		public async Task The_audit_adopts_todays_pictures_and_trashes_what_nothing_holds()
+		{
+			var media = _host.Get<IMediaService>();
+			var bob = await _host.Mastodon($"bob{Guid.NewGuid():N}"[..12]);
+			// a picture shown from before pictures had rows, and a leftover nothing holds
+			var picture = $"2020/01/{Guid.NewGuid():N}.jpg";
+			var leftover = $"2020/01/{Guid.NewGuid():N}.jpg";
+			foreach (var relative in new[] { picture, leftover })
+			{
+				Directory.CreateDirectory(Path.GetDirectoryName(Path.Combine(media.Root, relative))!);
+				await File.WriteAllBytesAsync(Path.Combine(media.Root, relative), new byte[] { 1, 2, 3 }, Token);
+			}
+			await DB.Default.Update<Models.User.Avatar>().MatchID(bob.Persona.Id).Modify(a => a.PictureURL, media.Url(picture)).ExecuteAsync(Token);
+
+			var looked = await MediaAudit.Run(media, fix: false, Token);
+			Assert.True(looked.Adopted >= 1);
+			Assert.True(looked.Unheld >= 1);
+			Assert.True(File.Exists(Path.Combine(media.Root, leftover)));
+
+			await MediaAudit.Run(media, fix: true, Token);
+
+			Assert.True(await DB.Default.Find<MediaAttachment>().Match(m => m.ProfileOfAvatarId == bob.Persona.Id && m.FilePath == picture && m.TrashedAt == null).ExecuteAnyAsync(Token));
+			Assert.Equal(HttpStatusCode.OK, await Served(media.Url(picture)));
+			Assert.Equal(HttpStatusCode.NotFound, await Served(media.Url(leftover)));
+			Assert.True(File.Exists(Path.Combine(media.TrashRoot, leftover)));
+			Assert.True(await DB.Default.Find<MediaAttachment>().Match(m => m.FilePath == leftover && m.TrashedAt != null).ExecuteAnyAsync(Token));
+		}
+	}
+}