using MongoDB.Entities; using PrivaPub.Models.Media; using PrivaPub.Models.User; using PostEntity = PrivaPub.Models.Post.Post; namespace PrivaPub.Domain.Media { // What the media directory holds against what the database says holds it, and, with fix, the two made to agree: // - a persona's current avatar or header with no row (they had none before) becomes a row of its own; // - media of a deleted post or of a deleted persona go to the trash; // - rows whose files are gone go to the trash; // - files nothing holds (deleted posts' media and replaced pictures, from before media were trashed) get a trashed row // each. // Trashed files leave what /media/files serves at once, and the janitor deletes them after its grace. Run it without // fix first; with fix it must run as the user that owns the media (www-data). public static class MediaAudit { public sealed record Report(int Files, int Held, int Adopted, int OfDeleted, int MissingFiles, int Unheld, IReadOnlyList Examples, bool Fixed); public static async Task Run(IMediaService media, bool fix, CancellationToken token) { var files = Served(media.Root); var rows = await DB.Default.Find().Match(m => m.TrashedAt == null).ExecuteAsync(token); var held = rows.SelectMany(r => new[] { r.FilePath, r.PreviewPath }).Where(p => !string.IsNullOrEmpty(p)).ToHashSet(); var examples = new List(); // pictures personas show today, kept as rows from now on var prefix = media.Url(string.Empty); var adopted = 0; foreach (var avatar in await DB.Default.Find().Match(a => !a.DeletionAt.HasValue && (a.PictureURL != null || a.ThumbnailURL != null)).ExecuteAsync(token)) foreach (var (url, kind) in new[] { (avatar.PictureURL, "avatar"), (avatar.ThumbnailURL, "header") }) { if (url?.StartsWith(prefix, StringComparison.Ordinal) != true) continue; var relative = url[prefix.Length..]; if (held.Contains(relative) || !files.Contains(relative)) continue; adopted++; held.Add(relative); if (fix) await DB.Default.SaveAsync(new MediaAttachment { OwnerAvatarId = avatar.ID, ProfileOfAvatarId = avatar.ID, Kind = kind, ContentType = "image/jpeg", FilePath = relative, Size = new FileInfo(Path.Combine(media.Root, relative)).Length, AttachedAt = DateTime.UtcNow }, token); } // held by something that is gone: a deleted post, a deleted persona var postIds = rows.Where(r => r.PostId != null).Select(r => r.PostId).Distinct().ToList(); var livePosts = (await DB.Default.Find().Match(p => postIds.Contains(p.ID) && !p.DeletedAt.HasValue).Project(p => p.Include(x => x.ID)).ExecuteAsync(token)) .Select(p => p.ID).ToHashSet(); var ownerIds = rows.Select(r => r.OwnerAvatarId).Where(id => id != null).Distinct().ToList(); var liveOwners = (await DB.Default.Find().Match(a => ownerIds.Contains(a.ID) && !a.DeletionAt.HasValue).Project(a => a.Include(x => x.ID)).ExecuteAsync(token)) .Select(a => a.ID).ToHashSet(); var ofDeleted = rows.Where(r => (r.PostId != null && !livePosts.Contains(r.PostId)) || (r.OwnerAvatarId != null && !liveOwners.Contains(r.OwnerAvatarId))).ToList(); var missing = rows.Except(ofDeleted).Where(r => !string.IsNullOrEmpty(r.FilePath) && !files.Contains(r.FilePath)).ToList(); if (fix) foreach (var row in ofDeleted.Concat(missing)) await media.Trash(m => m.ID == row.ID, ofDeleted.Contains(row) ? "audit: its holder is deleted" : "audit: its file is missing", token); foreach (var row in ofDeleted) held.Remove(row.FilePath); // files nothing holds var unheld = files.Where(f => !held.Contains(f)).OrderBy(f => f, StringComparer.Ordinal).ToList(); examples.AddRange(unheld.Take(10)); if (fix) foreach (var relative in unheld) { var orphan = new MediaAttachment { Kind = "orphan", FilePath = relative, Size = new FileInfo(Path.Combine(media.Root, relative)).Length, TrashedAt = DateTime.UtcNow, TrashReason = "audit: nothing holds it" }; await DB.Default.SaveAsync(orphan, token); media.Hide(orphan); } return new Report(files.Count, rows.Count, adopted, ofDeleted.Count, missing.Count, unheld.Count, examples, fix); } // every file /media/files serves: everything under the root static HashSet Served(string root) { if (!Directory.Exists(root)) return new HashSet(); return Directory.EnumerateFiles(root, "*", SearchOption.AllDirectories) .Select(f => Path.GetRelativePath(root, f).Replace('\\', '/')) .ToHashSet(); } } }