A file lives exactly as long as something holds it
Deleting a post, editing media out, replacing an avatar or a header, and removing a whole root deleted no file: every one stayed on disk and publicly served from /media/files with a year-long immutable cache, its row orphaned. Now every upload is a row, profile pictures too (Kind avatar or header, ProfileOfAvatarId), and each of those acts trashes what it held, as does an upload never posted for a day and a dropped scheduled post. A trashed row is marked in one conditional update (an upload attached meanwhile is left alone), its files move into media-trash, beside the media root and outside what /media/files serves, and the janitor deletes them a day later. Nothing is deleted for looking unused. Along the way: a profile picture that isn't an image, or can't be read, answers 422 instead of being silently ignored with a 200; a removed root's scheduled posts are dropped, so nothing of it publishes later; media rows get indexes (they had none), and the janitor's first pass comes five minutes after boot instead of an hour. `PrivaPub admin media audit [--fix]` compares the disk with the database. With --fix (as www-data) it gives the pictures personas show today a row, and trashes media of deleted posts or personas, rows whose files are missing, and files nothing holds: the leftovers of every deletion until now. MediaLifecycleTests covers each act, that the trash is never served, and the audit. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01LsXgEaXee4GCU1hwYgPJXw
This commit is contained in:
1 parent
52d201eee9
commit
bb680e8cb8
15 files changed
+462
-36
No files matched your search
@@ -123,8 +123,11 @@ namespace PrivaPub.Domain.Media
|
||||
|
||||
public class MediaJanitor : BackgroundService
|
||||
{
|
||||
static readonly TimeSpan FirstPass = TimeSpan.FromMinutes(5);
|
||||
static readonly TimeSpan Interval = TimeSpan.FromHours(1);
|
||||
static readonly TimeSpan UnattachedLifetime = TimeSpan.FromDays(1);
|
||||
// a trashed file waits this long before it is deleted, already out of what /media/files serves
|
||||
public static readonly TimeSpan TrashGrace = TimeSpan.FromDays(1);
|
||||
|
||||
readonly IMediaService _media;
|
||||
readonly IOptionsMonitor<MediaOptions> _options;
|
||||
@@ -139,11 +142,13 @@ namespace PrivaPub.Domain.Media
|
||||
|
||||
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
||||
{
|
||||
var wait = FirstPass;
|
||||
while (!stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
try
|
||||
{
|
||||
await Task.Delay(Interval, stoppingToken);
|
||||
await Task.Delay(wait, stoppingToken);
|
||||
wait = Interval;
|
||||
await Sweep(stoppingToken);
|
||||
}
|
||||
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
|
||||
@@ -157,12 +162,33 @@ namespace PrivaPub.Domain.Media
|
||||
}
|
||||
}
|
||||
|
||||
//one pass: uploads left unattached for a day go (unless a scheduled post waits for them), then the proxy cache is trimmed to its size, oldest first
|
||||
// one pass:
|
||||
// - uploads never posted for a day (and not waiting for a scheduled post, nor a profile picture) go to the trash;
|
||||
// - trashed files still served (a crash between the mark and the move) are moved out;
|
||||
// - trashed files past their grace are deleted, with their rows;
|
||||
// - the proxy cache is trimmed to its size, oldest first
|
||||
public async Task Sweep(CancellationToken token)
|
||||
{
|
||||
var cutoff = DateTime.UtcNow - UnattachedLifetime;
|
||||
foreach (var stale in await DB.Default.Find<MediaAttachment>().Match(m => m.PostId == null && m.ScheduledStatusId == null && m.CreatedAt < cutoff).Limit(500).ExecuteAsync(token))
|
||||
await _media.Delete(stale);
|
||||
var unattached = await _media.Trash(m => m.PostId == null && m.ScheduledStatusId == null && m.ProfileOfAvatarId == null && m.CreatedAt < cutoff,
|
||||
"never posted", token);
|
||||
|
||||
var trashed = await DB.Default.Find<MediaAttachment>().Match(m => m.TrashedAt != null).Limit(2000).ExecuteAsync(token);
|
||||
var purgeBefore = DateTime.UtcNow - TrashGrace;
|
||||
var purged = 0;
|
||||
foreach (var row in trashed)
|
||||
{
|
||||
if (row.TrashedAt < purgeBefore)
|
||||
{
|
||||
await _media.Purge(row, token);
|
||||
purged++;
|
||||
}
|
||||
else
|
||||
_media.Hide(row);
|
||||
}
|
||||
if (unattached > 0 || purged > 0)
|
||||
_logger.LogInformation("{Service}: {Unattached} uploads never posted trashed, {Purged} trashed files deleted",
|
||||
nameof(MediaJanitor), unattached, purged);
|
||||
TrimProxyCache();
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user