using FFMpegCore; using Microsoft.Extensions.Options; using MongoDB.Entities; using NetVips; using PrivaPub.Federation.Actors; using PrivaPub.Models.Media; using System.Globalization; using System.Security.Cryptography; namespace PrivaPub.Domain.Media { public sealed record MediaOutcome(MediaAttachment Attachment, int Status = StatusCodes.Status200OK, string Error = default) { public bool Ok => Error == default; public static MediaOutcome Fail(int status, string error) => new(default, status, error); } public sealed record ProcessedImage(byte[] Bytes, string Extension, string ContentType, int Width, int Height, byte[] Preview, string Blurhash); // what libvips makes of an upload's header alone, before anything is decoded public sealed record ImageHeader(string Loader, int Width, int PageHeight, int Pages) { public bool Animated => Pages > 1 && Loader?.StartsWith("gifload", StringComparison.Ordinal) == true; public long Pixels => (long)Width * PageHeight * (Animated ? Pages : 1); } public interface IMediaService { string Root { get; } string ProxyRoot { get; } string Url(string relativePath); /// Where trashed files wait for the janitor: the sibling of Root, on the same disk (a move is a rename) and /// outside what /media/files serves. (Not a dot-prefixed folder inside Root: the file provider only hides a file whose /// own name starts with a dot.) string TrashRoot { get; } /// Audio and video waiting for their processing: beside the media root, never served. string IncomingRoot { get; } /// An upload; with , audio and video are only stored, "pending" for ProcessMedia. Task Upload(LocalActor owner, IFormFile file, string description, string focus, bool later, CancellationToken token); /// Processes a pending upload's audio or video; false when it can't be (the row then says why). Task ProcessPending(MediaAttachment pending, CancellationToken token); /// A persona's new avatar or header ("avatar" or "header"), cropped to its size, as a row of its own. Task ProfileImage(string avatarId, string kind, IFormFile file, int width, int height, CancellationToken token); /// Trashes the media matches (and that isn't trashed yet): each row is marked in one /// conditional update, so a row attached meanwhile is left alone, and its files stop being served at once. Task Trash(System.Linq.Expressions.Expression> which, string reason, CancellationToken token); /// Moves a trashed row's files out of what is served, if they are still there (a crash between the mark and the move). void Hide(MediaAttachment trashed); /// Deletes a trashed row's files and the row. Task Purge(MediaAttachment trashed, CancellationToken token); } public class MediaService : IMediaService { // HEIC and HEIF are not among them: the libvips bundled here decodes AVIF but has no HEVC decoder static readonly string[] ImageTypes = { "image/jpeg", "image/png", "image/gif", "image/webp", "image/avif" }; // the only loaders an upload ever reaches: everything else libvips could read (SVG, PDF, TIFF, ImageMagick, ...) is // blocked, whatever type the upload claims static readonly string[] Loaders = { "VipsForeignLoadJpeg", "VipsForeignLoadPng", "VipsForeignLoadNsgif", "VipsForeignLoadWebp", "VipsForeignLoadHeif" }; static readonly Dictionary AvTypes = new() { ["video/mp4"] = "mp4", ["video/quicktime"] = "mp4", ["video/webm"] = "webm", ["audio/mpeg"] = "mp3", ["audio/mp3"] = "mp3", ["audio/ogg"] = "ogg", ["audio/wav"] = "wav", ["audio/x-wav"] = "wav", ["audio/mp4"] = "m4a", ["audio/x-m4a"] = "m4a", ["audio/flac"] = "flac", ["audio/webm"] = "webm" }; // what /media/files says each file is: ASP.NET's map, which has no entry for FLAC public static Microsoft.AspNetCore.StaticFiles.FileExtensionContentTypeProvider ContentTypes { get; } = new() { Mappings = { [".flac"] = "audio/flac" } }; // what an upload may be, as the instance API advertises it public static IReadOnlyList SupportedTypes => ImageTypes.Concat(AvTypes.Keys).ToList(); static MediaService() { NetVips.NetVips.BlockUntrusted = true; Operation.Block("VipsForeignLoad", true); foreach (var loader in Loaders) Operation.Block(loader, false); Cache.Max = 0;//an upload is decoded once: nothing to keep between operations } readonly IOptionsMonitor _options; readonly ILocalActorService _localActors; readonly SemaphoreSlim _processing; readonly IWebHostEnvironment _environment; readonly ILogger _logger; public MediaService(IOptionsMonitor options, ILocalActorService localActors, IWebHostEnvironment environment, ILogger logger) { _options = options; _localActors = localActors; _processing = new SemaphoreSlim(Math.Max(1, options.CurrentValue.Concurrency)); _environment = environment; _logger = logger; } public string Root => Path.GetFullPath(_options.CurrentValue.Root ?? Path.Combine(_environment.ContentRootPath, "media-store")); public string ProxyRoot => Root.TrimEnd(Path.DirectorySeparatorChar) + "-proxy"; public string TrashRoot => Root.TrimEnd(Path.DirectorySeparatorChar) + "-trash"; public string IncomingRoot => Root.TrimEnd(Path.DirectorySeparatorChar) + "-incoming"; string Incoming(string id) => Path.Combine(IncomingRoot, id + ".in"); public string Url(string relativePath) => relativePath == default ? default : $"{_localActors.BaseAddress}/media/files/{relativePath.Replace('\\', '/')}"; public async Task Upload(LocalActor owner, IFormFile file, string description, string focus, bool later, CancellationToken token) { if (file == default || file.Length == 0) return MediaOutcome.Fail(StatusCodes.Status422UnprocessableEntity, "Validation failed: File can't be blank"); var options = _options.CurrentValue; var contentType = file.ContentType?.Split(';')[0].Trim().ToLowerInvariant(); if (await OverQuota(owner.Id, file.Length, token) is { } full) return full; var attachment = new MediaAttachment { OwnerAvatarId = owner.Id, Description = Clean(description, 1500), Focus = FocalPoint.Parse(focus) }; if (ImageTypes.Contains(contentType)) { if (file.Length > options.MaxImageBytes) return MediaOutcome.Fail(StatusCodes.Status422UnprocessableEntity, "Validation failed: File is too big"); var bytes = await Read(file, token); if (Refusal(bytes, options) is { } refused) return refused; await _processing.WaitAsync(token); try { var header = Inspect(bytes); if (header.Animated) { var animated = await ProcessAnimation(bytes, header, attachment, token); if (!animated.Ok) return animated; } else { var processed = ProcessImage(bytes, options.MaxImageSide, options.PreviewSide); attachment.Kind = "image"; attachment.ContentType = processed.ContentType; attachment.FilePath = await Save(processed.Bytes, processed.Extension, token); attachment.PreviewPath = await Save(processed.Preview, "jpg", token); attachment.Width = processed.Width; attachment.Height = processed.Height; attachment.Blurhash = processed.Blurhash; attachment.Size = processed.Bytes.Length; } } catch (VipsException ex) { _logger.LogInformation("Refused an image upload: {Error}", ex.Message); return MediaOutcome.Fail(StatusCodes.Status422UnprocessableEntity, "Validation failed: The file is not a readable image"); } finally { _processing.Release(); } } else if (contentType != default && AvTypes.ContainsKey(contentType)) { if (file.Length > options.MaxVideoBytes) return MediaOutcome.Fail(StatusCodes.Status422UnprocessableEntity, "Validation failed: File is too big"); // stored as it came, outside what is served, then processed: by a job when asked later (v2), here otherwise (v1) attachment.ID = (string)attachment.GenerateNewID(); attachment.Kind = contentType.StartsWith("video/", StringComparison.Ordinal) ? "video" : "audio"; attachment.ContentType = contentType; Directory.CreateDirectory(IncomingRoot); await using (var stored = File.Create(Incoming(attachment.ID))) await file.CopyToAsync(stored, token); if (later) attachment.ProcessingState = "pending"; else { var outcome = await ProcessIncoming(attachment, token); if (!outcome.Ok) return outcome; } } else return MediaOutcome.Fail(StatusCodes.Status422UnprocessableEntity, "Validation failed: File type is not supported"); await DB.Default.SaveAsync(attachment, token); return new MediaOutcome(attachment); } public async Task ProfileImage(string avatarId, string kind, IFormFile file, int width, int height, CancellationToken token) { var contentType = file?.ContentType?.Split(';')[0].Trim().ToLowerInvariant(); if (file == default || file.Length == 0) return MediaOutcome.Fail(StatusCodes.Status422UnprocessableEntity, "Validation failed: File can't be blank"); if (!ImageTypes.Contains(contentType)) return MediaOutcome.Fail(StatusCodes.Status422UnprocessableEntity, "Validation failed: File type is not supported"); if (file.Length > _options.CurrentValue.MaxImageBytes) return MediaOutcome.Fail(StatusCodes.Status422UnprocessableEntity, "Validation failed: File is too big"); if (await OverQuota(avatarId, file.Length, token) is { } full) return full; var input = await Read(file, token); if (Refusal(input, _options.CurrentValue) is { } refused) return refused; byte[] bytes; int outWidth, outHeight; await _processing.WaitAsync(token); try { using var image = Image.ThumbnailBuffer(input, width, height: height, crop: Enums.Interesting.Centre, size: Enums.Size.Down); using var flat = Flatten(image); bytes = flat.WriteToBuffer(".jpg[Q=85,keep=none]"); (outWidth, outHeight) = (flat.Width, flat.Height); } catch (VipsException ex) { _logger.LogInformation("Refused a profile picture: {Error}", ex.Message); return MediaOutcome.Fail(StatusCodes.Status422UnprocessableEntity, "Validation failed: The file is not a readable image"); } finally { _processing.Release(); } var attachment = new MediaAttachment { OwnerAvatarId = avatarId, ProfileOfAvatarId = avatarId, Kind = kind, ContentType = "image/jpeg", FilePath = await Save(bytes, "jpg", token), Size = bytes.Length, Width = outWidth, Height = outHeight, AttachedAt = DateTime.UtcNow }; await DB.Default.SaveAsync(attachment, token); return new MediaOutcome(attachment); } // a login over its quota (Media:QuotaBytesPerRoot; none when 0) uploads nothing more async Task OverQuota(string avatarId, long incoming, CancellationToken token) { var quota = _options.CurrentValue.QuotaBytesPerRoot; if (quota <= 0) return default; var rootId = (await DB.Default.Find().Match(r => r.AvatarId == avatarId).ExecuteFirstAsync(token))?.RootId; if (rootId == default) return default; var (bytes, _) = await Privacy.Counted.MediaOfRoot(rootId, token); return bytes + incoming > quota ? MediaOutcome.Fail(StatusCodes.Status422UnprocessableEntity, "Validation failed: Your storage is full") : default; } public async Task Trash(System.Linq.Expressions.Expression> which, string reason, CancellationToken token) { var trashed = 0L; foreach (var candidate in await DB.Default.Find().Match(which).Match(m => m.TrashedAt == null).ExecuteAsync(token)) { var marked = await DB.Default.Update() .Match(which).Match(m => m.ID == candidate.ID && m.TrashedAt == null) .Modify(m => m.TrashedAt, DateTime.UtcNow) .Modify(m => m.TrashReason, reason) .ExecuteAsync(token); if (marked.ModifiedCount == 0) continue; Hide(candidate); trashed++; } return trashed; } public void Hide(MediaAttachment trashed) { foreach (var relative in new[] { trashed.FilePath, trashed.PreviewPath }.Where(p => !string.IsNullOrEmpty(p))) { var served = Path.Combine(Root, relative); if (!File.Exists(served)) continue; var hidden = Path.Combine(TrashRoot, relative); Directory.CreateDirectory(Path.GetDirectoryName(hidden)!); File.Move(served, hidden, overwrite: true); } } public async Task Purge(MediaAttachment trashed, CancellationToken token) { foreach (var relative in new[] { trashed.FilePath, trashed.PreviewPath }.Where(p => !string.IsNullOrEmpty(p))) foreach (var full in new[] { Path.Combine(TrashRoot, relative), Path.Combine(Root, relative) }) if (File.Exists(full)) File.Delete(full); if (File.Exists(Incoming(trashed.ID))) File.Delete(Incoming(trashed.ID)); await DB.Default.DeleteAsync(m => m.ID == trashed.ID && m.TrashedAt != null); } // reads only the header: which loader takes the bytes and how big they would decode (throws VipsException when none) public static ImageHeader Inspect(byte[] input) { using var image = Image.NewFromBuffer(input, access: Enums.Access.Sequential); var pages = image.Contains("n-pages") ? Math.Max(1, (int)image.Get("n-pages")) : 1; var pageHeight = image.Contains("page-height") ? (int)image.Get("page-height") : image.Height; return new ImageHeader(image.Contains("vips-loader") ? (string)image.Get("vips-loader") : default, image.Width, pageHeight, pages); } // an upload refused before anything is decoded: no loader takes it, or it would decode to too many pixels or frames static MediaOutcome Refusal(byte[] input, MediaOptions options) { ImageHeader header; try { header = Inspect(input); } catch (VipsException) { return MediaOutcome.Fail(StatusCodes.Status422UnprocessableEntity, "Validation failed: The file is not a readable image"); } if (header.Pixels > options.MaxPixels || header.Pages > options.MaxFrames) return MediaOutcome.Fail(StatusCodes.Status422UnprocessableEntity, "Validation failed: The image is too large"); return default; } // a still image: shrunk on load to the largest side allowed, turned by its orientation, its colours brought into sRGB // (thumbnail does all three), then written without its metadata, with a preview and a blurhash public static ProcessedImage ProcessImage(byte[] input, int maxSide, int previewSide) { // rendered into memory once (it is read three times below, and a shrunk-on-load image can be read only in order) using var shrunk = Image.ThumbnailBuffer(input, maxSide, height: maxSide, size: Enums.Size.Down); using var resized = shrunk.CopyMemory(); var keepsAlpha = resized.HasAlpha(); var bytes = keepsAlpha ? resized.WriteToBuffer(".png[keep=none]") : resized.WriteToBuffer(".jpg[Q=88,keep=none]"); var (extension, contentType) = keepsAlpha ? ("png", "image/png") : ("jpg", "image/jpeg"); using var preview = resized.ThumbnailImage(previewSide, height: previewSide, size: Enums.Size.Down); using var previewFlat = Flatten(preview); var previewBytes = previewFlat.WriteToBuffer(".jpg[Q=80,keep=none]"); using var tiny = resized.ThumbnailImage(32, height: 32); using var tinyFlat = Flatten(tiny); var pixels = tinyFlat.WriteToMemory(); return new ProcessedImage(bytes, extension, contentType, resized.Width, resized.Height, previewBytes, Blurhash.Encode(pixels, tinyFlat.Width, tinyFlat.Height)); } // an animated GIF becomes what Mastodon makes of one: a silent, looping H.264 mp4 (a gifv) at most MaxGifSide wide, // its first frame the poster async Task ProcessAnimation(byte[] input, ImageHeader header, MediaAttachment attachment, CancellationToken token) { var options = _options.CurrentValue; var temp = Path.Combine(Path.GetTempPath(), $"privapub-{Guid.NewGuid():N}"); var gif = temp + ".gif"; var mp4 = temp + ".mp4"; try { await File.WriteAllBytesAsync(gif, input, token); var width = Math.Min(header.Width, options.MaxGifSide) / 2 * 2; var height = Math.Max(2, (int)Math.Round(header.PageHeight * (double)width / header.Width / 2) * 2); await FFMpegArguments.FromFileInput(gif, true, o => o.ForceFormat("gif").WithCustomArgument("-protocol_whitelist file")) .OutputToFile(mp4, true, o => o.WithCustomArgument( $"-an -map_metadata -1 -map_chapters -1 -c:v libx264 -preset veryfast -crf 23 -pix_fmt yuv420p -movflags +faststart -vf scale={width}:{height}")) .CancellableThrough(token) .ProcessAsynchronously(); var poster = ProcessImage(input, options.PreviewSide, options.PreviewSide); attachment.Kind = "gifv"; attachment.ContentType = "video/mp4"; attachment.Width = width; attachment.Height = height; attachment.Size = new FileInfo(mp4).Length; attachment.Blurhash = poster.Blurhash; attachment.PreviewPath = await Save(poster.Preview, "jpg", token); attachment.FilePath = SaveFile(mp4, "mp4"); return new MediaOutcome(attachment); } catch (Exception ex) when (ex is not OperationCanceledException) { _logger.LogInformation(ex, "Refused a GIF upload"); return MediaOutcome.Fail(StatusCodes.Status422UnprocessableEntity, "Validation failed: The file could not be processed"); } finally { foreach (var path in new[] { gif, mp4 }) if (File.Exists(path)) File.Delete(path); } } static async Task Read(IFormFile file, CancellationToken token) { await using var stream = file.OpenReadStream(); using var buffer = new MemoryStream((int)Math.Min(file.Length, int.MaxValue)); await stream.CopyToAsync(buffer, token); return buffer.ToArray(); } static Image Flatten(Image image) { var srgb = image.Interpretation == Enums.Interpretation.Srgb ? image.Copy() : image.Colourspace(Enums.Interpretation.Srgb); var flat = srgb.HasAlpha() ? srgb.Flatten(background: new double[] { 255, 255, 255 }) : srgb.Copy(); srgb.Dispose(); var bands = flat.Bands > 3 ? flat.ExtractBand(0, n: 3) : flat.Bands == 1 ? flat.Bandjoin(flat, flat) : flat.Copy(); flat.Dispose(); using var cast = bands; return bands.Format == Enums.BandFormat.Uchar ? bands.Copy() : bands.Cast(Enums.BandFormat.Uchar); } public async Task ProcessPending(MediaAttachment pending, CancellationToken token) { var outcome = await ProcessIncoming(pending, token); if (outcome.Ok) { await DB.Default.Update().Match(m => m.ID == pending.ID && m.TrashedAt == null) .Modify(m => m.Kind, pending.Kind) .Modify(m => m.ContentType, pending.ContentType) .Modify(m => m.FilePath, pending.FilePath) .Modify(m => m.PreviewPath, pending.PreviewPath) .Modify(m => m.Size, pending.Size) .Modify(m => m.Width, pending.Width) .Modify(m => m.Height, pending.Height) .Modify(m => m.Blurhash, pending.Blurhash) .Modify(m => m.DurationSeconds, pending.DurationSeconds) .Modify(m => m.ProcessingState, null) .ExecuteAsync(token); return true; } await DB.Default.Update().MatchID(pending.ID) .Modify(m => m.ProcessingState, "failed").Modify(m => m.ProcessingError, outcome.Error).ExecuteAsync(token); return false; } // the stored upload, made playable: probed, then remuxed without its metadata when browsers play it as it is (H.264, // VP8, VP9 or AV1 within MaxVideoPixels and MaxFrameRate), or else transcoded to H.264 that fits; a frame becomes the // poster. ffmpeg only ever reads the local file (no protocol but file, its format forced from the probe). Nothing is // saved until everything succeeded, and every temporary file goes, the stored upload too. async Task ProcessIncoming(MediaAttachment attachment, CancellationToken token) { var options = _options.CurrentValue; var input = Incoming(attachment.ID); var temp = Path.Combine(Path.GetTempPath(), $"privapub-{Guid.NewGuid():N}"); var frame = temp + ".png"; var output = default(string); await _processing.WaitAsync(token); try { var probe = await FFProbe.AnalyseAsync(input, default, token, "-protocol_whitelist file"); if (probe.Duration > TimeSpan.FromSeconds(options.MaxSeconds)) return MediaOutcome.Fail(StatusCodes.Status422UnprocessableEntity, "Validation failed: The file is too long"); var video = attachment.ContentType.StartsWith("video/", StringComparison.Ordinal) ? probe.PrimaryVideoStream : default; var audio = probe.PrimaryAudioStream; if (video == default && audio == default) return MediaOutcome.Fail(StatusCodes.Status422UnprocessableEntity, "Validation failed: The file could not be processed"); var extension = AvTypes[attachment.ContentType]; var format = probe.Format.FormatName.Split(',')[0]; string arguments; if (video == default) arguments = "-map 0:a:0 -vn -dn -sn -map_metadata -1 -map_chapters -1 -c copy" + (extension == "m4a" ? " -movflags +faststart" : string.Empty); else if (video.CodecName is "h264" or "vp8" or "vp9" or "av1" && (long)video.Width * video.Height <= options.MaxVideoPixels && video.FrameRate <= options.MaxFrameRate + 0.5) arguments = "-map 0:v:0 -map 0:a:0? -dn -sn -map_metadata -1 -map_chapters -1 -c copy" + (extension == "mp4" ? " -movflags +faststart" : string.Empty); else { var scale = Math.Min(1, Math.Sqrt(options.MaxVideoPixels / (double)((long)video.Width * video.Height))); var width = Math.Max(2, (int)(video.Width * scale) / 2 * 2); var height = Math.Max(2, (int)(video.Height * scale) / 2 * 2); extension = "mp4"; arguments = $"-map 0:v:0 -map 0:a:0? -dn -sn -map_metadata -1 -map_chapters -1 -c:v libx264 -preset veryfast -crf 23 -pix_fmt yuv420p " + $"-vf scale={width}:{height} -fpsmax {options.MaxFrameRate.ToString(CultureInfo.InvariantCulture)} -c:a aac -b:a 128k -movflags +faststart"; } output = temp + "." + extension; await FFMpegArguments.FromFileInput(input, true, o => o.ForceFormat(format).WithCustomArgument("-protocol_whitelist file")) .OutputToFile(output, true, o => o.WithCustomArgument(arguments)) .CancellableThrough(token) .ProcessAsynchronously(); var made = await FFProbe.AnalyseAsync(output, default, token, "-protocol_whitelist file"); byte[] poster = default; if (video != default) { attachment.Width = made.PrimaryVideoStream?.Width; attachment.Height = made.PrimaryVideoStream?.Height; await FFMpeg.SnapshotAsync(output, frame, captureTime: TimeSpan.FromSeconds(Math.Min(1, made.Duration.TotalSeconds / 2))); var still = ProcessImage(await File.ReadAllBytesAsync(frame, token), 640, options.PreviewSide); poster = still.Preview; attachment.Blurhash = still.Blurhash; } attachment.Kind = video != default ? "video" : "audio"; attachment.ContentType = extension switch { "mp4" => "video/mp4", "webm" => video != default ? "video/webm" : "audio/webm", "m4a" => "audio/mp4", "mp3" => "audio/mpeg", "ogg" => "audio/ogg", "wav" => "audio/wav", "flac" => "audio/flac", _ => attachment.ContentType }; attachment.DurationSeconds = Math.Round(made.Duration.TotalSeconds, 2); attachment.Size = new FileInfo(output).Length; attachment.FilePath = SaveFile(output, extension); if (poster != default) attachment.PreviewPath = await Save(poster, "jpg", token); return new MediaOutcome(attachment); } catch (Exception ex) when (ex is not OperationCanceledException) { _logger.LogInformation(ex, "Refused an audio or video upload"); return MediaOutcome.Fail(StatusCodes.Status422UnprocessableEntity, "Validation failed: The file could not be processed"); } finally { _processing.Release(); foreach (var path in new[] { input, output, frame }.Where(p => p != default)) if (File.Exists(path)) File.Delete(path); } } string SaveFile(string source, string extension) { var (relative, full) = NewPath(extension); File.Move(source, full); return relative; } (string Relative, string Full) NewPath(string extension) { var now = DateTime.UtcNow; var relative = Path.Combine(now.ToString("yyyy", CultureInfo.InvariantCulture), now.ToString("MM", CultureInfo.InvariantCulture), $"{Convert.ToHexStringLower(RandomNumberGenerator.GetBytes(16))}.{extension}"); var full = Path.Combine(Root, relative); Directory.CreateDirectory(Path.GetDirectoryName(full)!); return (relative.Replace('\\', '/'), full); } async Task Save(byte[] bytes, string extension, CancellationToken token) { var (relative, full) = NewPath(extension); await File.WriteAllBytesAsync(full, bytes, token); return relative; } static string Clean(string value, int max) => string.IsNullOrWhiteSpace(value) ? default : value.Trim().Length <= max ? value.Trim() : value.Trim()[..max]; } }