A file lives exactly as long as something holds it

Deleting a post, editing media out, replacing an avatar or a header, and removing a whole root deleted no file: every
one stayed on disk and publicly served from /media/files with a year-long immutable cache, its row orphaned. Now every
upload is a row, profile pictures too (Kind avatar or header, ProfileOfAvatarId), and each of those acts trashes what
it held, as does an upload never posted for a day and a dropped scheduled post. A trashed row is marked in one
conditional update (an upload attached meanwhile is left alone), its files move into media-trash, beside the media
root and outside what /media/files serves, and the janitor deletes them a day later. Nothing is deleted for looking
unused.

Along the way: a profile picture that isn't an image, or can't be read, answers 422 instead of being silently ignored
with a 200; a removed root's scheduled posts are dropped, so nothing of it publishes later; media rows get indexes
(they had none), and the janitor's first pass comes five minutes after boot instead of an hour.

`PrivaPub admin media audit [--fix]` compares the disk with the database. With --fix (as www-data) it gives the
pictures personas show today a row, and trashes media of deleted posts or personas, rows whose files are missing, and
files nothing holds: the leftovers of every deletion until now. MediaLifecycleTests covers each act, that the trash is
never served, and the audit.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01LsXgEaXee4GCU1hwYgPJXw

PrivaPub/Domain/Media/MediaAudit.cs

@@ -0,0 +1,101 @@
+using MongoDB.Entities;
+
+using PrivaPub.Models.Media;
+using PrivaPub.Models.User;
+
+using PostEntity = PrivaPub.Models.Post.Post;
+
+namespace PrivaPub.Domain.Media
+{
+	// What the media directory holds against what the database says holds it, and, with fix, the two made to agree:
+	// - a persona's current avatar or header with no row (they had none before) becomes a row of its own;
+	// - media of a deleted post or of a deleted persona go to the trash;
+	// - rows whose files are gone go to the trash;
+	// - files nothing holds (deleted posts' media and replaced pictures, from before media were trashed) get a trashed row
+	//   each.
+	// Trashed files leave what /media/files serves at once, and the janitor deletes them after its grace. Run it without
+	// fix first; with fix it must run as the user that owns the media (www-data).
+	public static class MediaAudit
+	{
+		public sealed record Report(int Files, int Held, int Adopted, int OfDeleted, int MissingFiles, int Unheld, IReadOnlyList<string> Examples, bool Fixed);
+
+		public static async Task<Report> Run(IMediaService media, bool fix, CancellationToken token)
+		{
+			var files = Served(media.Root);
+			var rows = await DB.Default.Find<MediaAttachment>().Match(m => m.TrashedAt == null).ExecuteAsync(token);
+			var held = rows.SelectMany(r => new[] { r.FilePath, r.PreviewPath }).Where(p => !string.IsNullOrEmpty(p)).ToHashSet();
+			var examples = new List<string>();
+
+			// pictures personas show today, kept as rows from now on
+			var prefix = media.Url(string.Empty);
+			var adopted = 0;
+			foreach (var avatar in await DB.Default.Find<Avatar>().Match(a => !a.DeletionAt.HasValue && (a.PictureURL != null || a.ThumbnailURL != null)).ExecuteAsync(token))
+				foreach (var (url, kind) in new[] { (avatar.PictureURL, "avatar"), (avatar.ThumbnailURL, "header") })
+				{
+					if (url?.StartsWith(prefix, StringComparison.Ordinal) != true)
+						continue;
+					var relative = url[prefix.Length..];
+					if (held.Contains(relative) || !files.Contains(relative))
+						continue;
+					adopted++;
+					held.Add(relative);
+					if (fix)
+						await DB.Default.SaveAsync(new MediaAttachment
+						{
+							OwnerAvatarId = avatar.ID,
+							ProfileOfAvatarId = avatar.ID,
+							Kind = kind,
+							ContentType = "image/jpeg",
+							FilePath = relative,
+							Size = new FileInfo(Path.Combine(media.Root, relative)).Length,
+							AttachedAt = DateTime.UtcNow
+						}, token);
+				}
+
+			// held by something that is gone: a deleted post, a deleted persona
+			var postIds = rows.Where(r => r.PostId != null).Select(r => r.PostId).Distinct().ToList();
+			var livePosts = (await DB.Default.Find<PostEntity>().Match(p => postIds.Contains(p.ID) && !p.DeletedAt.HasValue).Project(p => p.Include(x => x.ID)).ExecuteAsync(token))
+				.Select(p => p.ID).ToHashSet();
+			var ownerIds = rows.Select(r => r.OwnerAvatarId).Where(id => id != null).Distinct().ToList();
+			var liveOwners = (await DB.Default.Find<Avatar>().Match(a => ownerIds.Contains(a.ID) && !a.DeletionAt.HasValue).Project(a => a.Include(x => x.ID)).ExecuteAsync(token))
+				.Select(a => a.ID).ToHashSet();
+			var ofDeleted = rows.Where(r => (r.PostId != null && !livePosts.Contains(r.PostId)) || (r.OwnerAvatarId != null && !liveOwners.Contains(r.OwnerAvatarId))).ToList();
+			var missing = rows.Except(ofDeleted).Where(r => !string.IsNullOrEmpty(r.FilePath) && !files.Contains(r.FilePath)).ToList();
+			if (fix)
+				foreach (var row in ofDeleted.Concat(missing))
+					await media.Trash(m => m.ID == row.ID, ofDeleted.Contains(row) ? "audit: its holder is deleted" : "audit: its file is missing", token);
+			foreach (var row in ofDeleted)
+				held.Remove(row.FilePath);
+
+			// files nothing holds
+			var unheld = files.Where(f => !held.Contains(f)).OrderBy(f => f, StringComparer.Ordinal).ToList();
+			examples.AddRange(unheld.Take(10));
+			if (fix)
+				foreach (var relative in unheld)
+				{
+					var orphan = new MediaAttachment
+					{
+						Kind = "orphan",
+						FilePath = relative,
+						Size = new FileInfo(Path.Combine(media.Root, relative)).Length,
+						TrashedAt = DateTime.UtcNow,
+						TrashReason = "audit: nothing holds it"
+					};
+					await DB.Default.SaveAsync(orphan, token);
+					media.Hide(orphan);
+				}
+
+			return new Report(files.Count, rows.Count, adopted, ofDeleted.Count, missing.Count, unheld.Count, examples, fix);
+		}
+
+		// every file /media/files serves: everything under the root
+		static HashSet<string> Served(string root)
+		{
+			if (!Directory.Exists(root))
+				return new HashSet<string>();
+			return Directory.EnumerateFiles(root, "*", SearchOption.AllDirectories)
+				.Select(f => Path.GetRelativePath(root, f).Replace('\\', '/'))
+				.ToHashSet();
+		}
+	}
+}

PrivaPub/Domain/Media/MediaProxy.cs

@@ -123,8 +123,11 @@ namespace PrivaPub.Domain.Media
 
 	public class MediaJanitor : BackgroundService
 	{
+		static readonly TimeSpan FirstPass = TimeSpan.FromMinutes(5);
 		static readonly TimeSpan Interval = TimeSpan.FromHours(1);
 		static readonly TimeSpan UnattachedLifetime = TimeSpan.FromDays(1);
+		// a trashed file waits this long before it is deleted, already out of what /media/files serves
+		public static readonly TimeSpan TrashGrace = TimeSpan.FromDays(1);
 
 		readonly IMediaService _media;
 		readonly IOptionsMonitor<MediaOptions> _options;
@@ -139,11 +142,13 @@ namespace PrivaPub.Domain.Media
 
 		protected override async Task ExecuteAsync(CancellationToken stoppingToken)
 		{
+			var wait = FirstPass;
 			while (!stoppingToken.IsCancellationRequested)
 			{
 				try
 				{
-					await Task.Delay(Interval, stoppingToken);
+					await Task.Delay(wait, stoppingToken);
+					wait = Interval;
 					await Sweep(stoppingToken);
 				}
 				catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
@@ -157,12 +162,33 @@ namespace PrivaPub.Domain.Media
 			}
 		}
 
-		//one pass: uploads left unattached for a day go (unless a scheduled post waits for them), then the proxy cache is trimmed to its size, oldest first
+		// one pass:
+		// - uploads never posted for a day (and not waiting for a scheduled post, nor a profile picture) go to the trash;
+		// - trashed files still served (a crash between the mark and the move) are moved out;
+		// - trashed files past their grace are deleted, with their rows;
+		// - the proxy cache is trimmed to its size, oldest first
 		public async Task Sweep(CancellationToken token)
 		{
 			var cutoff = DateTime.UtcNow - UnattachedLifetime;
-			foreach (var stale in await DB.Default.Find<MediaAttachment>().Match(m => m.PostId == null && m.ScheduledStatusId == null && m.CreatedAt < cutoff).Limit(500).ExecuteAsync(token))
-				await _media.Delete(stale);
+			var unattached = await _media.Trash(m => m.PostId == null && m.ScheduledStatusId == null && m.ProfileOfAvatarId == null && m.CreatedAt < cutoff,
+				"never posted", token);
+
+			var trashed = await DB.Default.Find<MediaAttachment>().Match(m => m.TrashedAt != null).Limit(2000).ExecuteAsync(token);
+			var purgeBefore = DateTime.UtcNow - TrashGrace;
+			var purged = 0;
+			foreach (var row in trashed)
+			{
+				if (row.TrashedAt < purgeBefore)
+				{
+					await _media.Purge(row, token);
+					purged++;
+				}
+				else
+					_media.Hide(row);
+			}
+			if (unattached > 0 || purged > 0)
+				_logger.LogInformation("{Service}: {Unattached} uploads never posted trashed, {Purged} trashed files deleted",
+					nameof(MediaJanitor), unattached, purged);
 			TrimProxyCache();
 		}
 

PrivaPub/Domain/Media/MediaService.cs

@@ -27,9 +27,20 @@ namespace PrivaPub.Domain.Media
 		string Root { get; }
 		string ProxyRoot { get; }
 		string Url(string relativePath);
+		/// <summary>Where trashed files wait for the janitor: the sibling of Root, on the same disk (a move is a rename) and
+		/// outside what /media/files serves. (Not a dot-prefixed folder inside Root: the file provider only hides a file whose
+		/// own name starts with a dot.)</summary>
+		string TrashRoot { get; }
 		Task<MediaOutcome> Upload(LocalActor owner, IFormFile file, string description, string focus, CancellationToken token);
-		Task<string> ProfileImage(IFormFile file, int width, int height, CancellationToken token);
-		Task Delete(MediaAttachment attachment);
+		/// <summary>A persona's new avatar or header ("avatar" or "header"), cropped to its size, as a row of its own.</summary>
+		Task<MediaOutcome> ProfileImage(string avatarId, string kind, IFormFile file, int width, int height, CancellationToken token);
+		/// <summary>Trashes the media <paramref name="which"/> matches (and that isn't trashed yet): each row is marked in one
+		/// conditional update, so a row attached meanwhile is left alone, and its files stop being served at once.</summary>
+		Task<long> Trash(System.Linq.Expressions.Expression<Func<MediaAttachment, bool>> which, string reason, CancellationToken token);
+		/// <summary>Moves a trashed row's files out of what is served, if they are still there (a crash between the mark and the move).</summary>
+		void Hide(MediaAttachment trashed);
+		/// <summary>Deletes a trashed row's files and the row.</summary>
+		Task Purge(MediaAttachment trashed, CancellationToken token);
 	}
 
 	public class MediaService : IMediaService
@@ -62,6 +73,8 @@ namespace PrivaPub.Domain.Media
 
 		public string ProxyRoot => Root.TrimEnd(Path.DirectorySeparatorChar) + "-proxy";
 
+		public string TrashRoot => Root.TrimEnd(Path.DirectorySeparatorChar) + "-trash";
+
 		public string Url(string relativePath) => relativePath == default ? default : $"{_localActors.BaseAddress}/media/files/{relativePath.Replace('\\', '/')}";
 
 		public async Task<MediaOutcome> Upload(LocalActor owner, IFormFile file, string description, string focus, CancellationToken token)
@@ -118,34 +131,86 @@ namespace PrivaPub.Domain.Media
 			return new MediaOutcome(attachment);
 		}
 
-		public async Task<string> ProfileImage(IFormFile file, int width, int height, CancellationToken token)
+		public async Task<MediaOutcome> ProfileImage(string avatarId, string kind, IFormFile file, int width, int height, CancellationToken token)
 		{
-			if (file == default || file.Length == 0 || file.Length > _options.CurrentValue.MaxImageBytes)
-				return default;
+			var contentType = file?.ContentType?.Split(';')[0].Trim().ToLowerInvariant();
+			if (file == default || file.Length == 0)
+				return MediaOutcome.Fail(StatusCodes.Status422UnprocessableEntity, "Validation failed: File can't be blank");
+			if (!ImageTypes.Contains(contentType))
+				return MediaOutcome.Fail(StatusCodes.Status422UnprocessableEntity, "Validation failed: File type is not supported");
+			if (file.Length > _options.CurrentValue.MaxImageBytes)
+				return MediaOutcome.Fail(StatusCodes.Status422UnprocessableEntity, "Validation failed: File is too big");
 			await using var stream = file.OpenReadStream();
 			using var buffer = new MemoryStream();
 			await stream.CopyToAsync(buffer, token);
+			byte[] bytes;
+			int outWidth, outHeight;
 			try
 			{
 				using var image = Image.ThumbnailBuffer(buffer.ToArray(), width, height: height, crop: Enums.Interesting.Centre, size: Enums.Size.Down);
 				using var flat = Flatten(image);
-				return Url(await Save(flat.WriteToBuffer(".jpg[Q=85,keep=none]"), "jpg", token));
+				bytes = flat.WriteToBuffer(".jpg[Q=85,keep=none]");
+				(outWidth, outHeight) = (flat.Width, flat.Height);
 			}
-			catch (VipsException)
+			catch (VipsException ex)
 			{
-				return default;
+				_logger.LogInformation("Refused a profile picture: {Error}", ex.Message);
+				return MediaOutcome.Fail(StatusCodes.Status422UnprocessableEntity, "Validation failed: The file is not a readable image");
+			}
+			var attachment = new MediaAttachment
+			{
+				OwnerAvatarId = avatarId,
+				ProfileOfAvatarId = avatarId,
+				Kind = kind,
+				ContentType = "image/jpeg",
+				FilePath = await Save(bytes, "jpg", token),
+				Size = bytes.Length,
+				Width = outWidth,
+				Height = outHeight,
+				AttachedAt = DateTime.UtcNow
+			};
+			await DB.Default.SaveAsync(attachment, token);
+			return new MediaOutcome(attachment);
+		}
+
+		public async Task<long> Trash(System.Linq.Expressions.Expression<Func<MediaAttachment, bool>> which, string reason, CancellationToken token)
+		{
+			var trashed = 0L;
+			foreach (var candidate in await DB.Default.Find<MediaAttachment>().Match(which).Match(m => m.TrashedAt == null).ExecuteAsync(token))
+			{
+				var marked = await DB.Default.Update<MediaAttachment>()
+					.Match(which).Match(m => m.ID == candidate.ID && m.TrashedAt == null)
+					.Modify(m => m.TrashedAt, DateTime.UtcNow)
+					.Modify(m => m.TrashReason, reason)
+					.ExecuteAsync(token);
+				if (marked.ModifiedCount == 0)
+					continue;
+				Hide(candidate);
+				trashed++;
+			}
+			return trashed;
+		}
+
+		public void Hide(MediaAttachment trashed)
+		{
+			foreach (var relative in new[] { trashed.FilePath, trashed.PreviewPath }.Where(p => !string.IsNullOrEmpty(p)))
+			{
+				var served = Path.Combine(Root, relative);
+				if (!File.Exists(served))
+					continue;
+				var hidden = Path.Combine(TrashRoot, relative);
+				Directory.CreateDirectory(Path.GetDirectoryName(hidden)!);
+				File.Move(served, hidden, overwrite: true);
 			}
 		}
 
-		public Task Delete(MediaAttachment attachment)
+		public async Task Purge(MediaAttachment trashed, CancellationToken token)
 		{
-			foreach (var path in new[] { attachment.FilePath, attachment.PreviewPath }.Where(p => p != default))
-			{
-				var full = Path.Combine(Root, path);
-				if (File.Exists(full))
-					File.Delete(full);
-			}
-			return DB.Default.DeleteAsync<MediaAttachment>(attachment.ID);
+			foreach (var relative in new[] { trashed.FilePath, trashed.PreviewPath }.Where(p => !string.IsNullOrEmpty(p)))
+				foreach (var full in new[] { Path.Combine(TrashRoot, relative), Path.Combine(Root, relative) })
+					if (File.Exists(full))
+						File.Delete(full);
+			await DB.Default.DeleteAsync<MediaAttachment>(m => m.ID == trashed.ID && m.TrashedAt != null);
 		}
 
 		public static ProcessedImage ProcessImage(byte[] input, int maxSide, int previewSide, bool animated)