using FFMpegCore; using Microsoft.Extensions.Options; using MongoDB.Entities; using NetVips; using PrivaPub.Federation.Actors; using PrivaPub.Models.Media; using System.Globalization; using System.Security.Cryptography; namespace PrivaPub.Domain.Media { public sealed record MediaOutcome(MediaAttachment Attachment, int Status = StatusCodes.Status200OK, string Error = default) { public bool Ok => Error == default; public static MediaOutcome Fail(int status, string error) => new(default, status, error); } public sealed record ProcessedImage(byte[] Bytes, string Extension, string ContentType, int Width, int Height, byte[] Preview, string Blurhash); public interface IMediaService { string Root { get; } string ProxyRoot { get; } string Url(string relativePath); Task Upload(LocalActor owner, IFormFile file, string description, string focus, CancellationToken token); Task ProfileImage(IFormFile file, int width, int height, CancellationToken token); Task Delete(MediaAttachment attachment); } public class MediaService : IMediaService { static readonly string[] ImageTypes = { "image/jpeg", "image/png", "image/gif", "image/webp", "image/heic", "image/heif", "image/avif" }; static readonly Dictionary AvTypes = new() { ["video/mp4"] = "mp4", ["video/quicktime"] = "mp4", ["video/webm"] = "webm", ["audio/mpeg"] = "mp3", ["audio/mp3"] = "mp3", ["audio/ogg"] = "ogg", ["audio/wav"] = "wav", ["audio/x-wav"] = "wav", ["audio/mp4"] = "m4a", ["audio/x-m4a"] = "m4a", ["audio/flac"] = "flac", ["audio/webm"] = "webm" }; // what an upload may be, as the instance API advertises it public static IReadOnlyList SupportedTypes => ImageTypes.Concat(AvTypes.Keys).ToList(); readonly IOptionsMonitor _options; readonly ILocalActorService _localActors; readonly IWebHostEnvironment _environment; readonly ILogger _logger; public MediaService(IOptionsMonitor options, ILocalActorService localActors, IWebHostEnvironment environment, ILogger logger) { _options = options; _localActors = localActors; _environment = environment; _logger = logger; } public string Root => Path.GetFullPath(_options.CurrentValue.Root ?? Path.Combine(_environment.ContentRootPath, "media-store")); public string ProxyRoot => Root.TrimEnd(Path.DirectorySeparatorChar) + "-proxy"; public string Url(string relativePath) => relativePath == default ? default : $"{_localActors.BaseAddress}/media/files/{relativePath.Replace('\\', '/')}"; public async Task Upload(LocalActor owner, IFormFile file, string description, string focus, CancellationToken token) { if (file == default || file.Length == 0) return MediaOutcome.Fail(StatusCodes.Status422UnprocessableEntity, "Validation failed: File can't be blank"); var options = _options.CurrentValue; var contentType = file.ContentType?.Split(';')[0].Trim().ToLowerInvariant(); var attachment = new MediaAttachment { OwnerAvatarId = owner.Id, Description = Clean(description, 1500), Focus = Focus(focus) }; if (ImageTypes.Contains(contentType)) { if (file.Length > options.MaxImageBytes) return MediaOutcome.Fail(StatusCodes.Status422UnprocessableEntity, "Validation failed: File is too big"); await using var stream = file.OpenReadStream(); using var buffer = new MemoryStream(); await stream.CopyToAsync(buffer, token); ProcessedImage processed; try { processed = ProcessImage(buffer.ToArray(), options.MaxImageSide, options.PreviewSide, contentType == "image/gif"); } catch (VipsException ex) { _logger.LogInformation("Refused an image upload: {Error}", ex.Message); return MediaOutcome.Fail(StatusCodes.Status422UnprocessableEntity, "Validation failed: The file is not a readable image"); } attachment.Kind = contentType == "image/gif" ? "gifv" : "image"; attachment.ContentType = processed.ContentType; attachment.FilePath = await Save(processed.Bytes, processed.Extension, token); attachment.PreviewPath = await Save(processed.Preview, "jpg", token); attachment.Width = processed.Width; attachment.Height = processed.Height; attachment.Blurhash = processed.Blurhash; attachment.Size = processed.Bytes.Length; } else if (contentType != default && AvTypes.TryGetValue(contentType, out var extension)) { if (file.Length > options.MaxVideoBytes) return MediaOutcome.Fail(StatusCodes.Status422UnprocessableEntity, "Validation failed: File is too big"); var outcome = await ProcessAv(file, extension, contentType, attachment, token); if (!outcome.Ok) return outcome; } else return MediaOutcome.Fail(StatusCodes.Status422UnprocessableEntity, "Validation failed: File type is not supported"); await DB.Default.SaveAsync(attachment, token); return new MediaOutcome(attachment); } public async Task ProfileImage(IFormFile file, int width, int height, CancellationToken token) { if (file == default || file.Length == 0 || file.Length > _options.CurrentValue.MaxImageBytes) return default; await using var stream = file.OpenReadStream(); using var buffer = new MemoryStream(); await stream.CopyToAsync(buffer, token); try { using var image = Image.ThumbnailBuffer(buffer.ToArray(), width, height: height, crop: Enums.Interesting.Centre, size: Enums.Size.Down); using var flat = Flatten(image); return Url(await Save(flat.WriteToBuffer(".jpg[Q=85,keep=none]"), "jpg", token)); } catch (VipsException) { return default; } } public Task Delete(MediaAttachment attachment) { foreach (var path in new[] { attachment.FilePath, attachment.PreviewPath }.Where(p => p != default)) { var full = Path.Combine(Root, path); if (File.Exists(full)) File.Delete(full); } return DB.Default.DeleteAsync(attachment.ID); } public static ProcessedImage ProcessImage(byte[] input, int maxSide, int previewSide, bool animated) { using var loaded = animated ? Image.NewFromBuffer(input, kwargs: new VOption { { "n", -1 } }) : Image.NewFromBuffer(input); using var rotated = animated ? loaded.Copy() : loaded.Autorot(); var pageHeight = animated && rotated.Contains("page-height") ? (int)rotated.Get("page-height") : rotated.Height; var longest = Math.Max(rotated.Width, pageHeight); byte[] bytes; string extension, contentType; int width, height; if (animated) { bytes = rotated.WriteToBuffer(".gif"); (extension, contentType, width, height) = ("gif", "image/gif", rotated.Width, pageHeight); } else { using var resized = longest > maxSide ? rotated.ThumbnailImage(maxSide, height: maxSide, size: Enums.Size.Down) : rotated.Copy(); var keepsAlpha = resized.HasAlpha(); bytes = keepsAlpha ? resized.WriteToBuffer(".png[keep=none]") : resized.WriteToBuffer(".jpg[Q=88,keep=none]"); (extension, contentType, width, height) = keepsAlpha ? ("png", "image/png", resized.Width, resized.Height) : ("jpg", "image/jpeg", resized.Width, resized.Height); } using var firstFrame = animated ? rotated.Crop(0, 0, rotated.Width, pageHeight) : rotated.Copy(); using var preview = firstFrame.ThumbnailImage(previewSide, height: previewSide, size: Enums.Size.Down); using var previewFlat = Flatten(preview); var previewBytes = previewFlat.WriteToBuffer(".jpg[Q=80,keep=none]"); using var tiny = firstFrame.ThumbnailImage(32, height: 32); using var tinyFlat = Flatten(tiny); var pixels = tinyFlat.WriteToMemory(); return new ProcessedImage(bytes, extension, contentType, width, height, previewBytes, Blurhash.Encode(pixels, tinyFlat.Width, tinyFlat.Height)); } static Image Flatten(Image image) { var srgb = image.Interpretation == Enums.Interpretation.Srgb ? image.Copy() : image.Colourspace(Enums.Interpretation.Srgb); var flat = srgb.HasAlpha() ? srgb.Flatten(background: new double[] { 255, 255, 255 }) : srgb.Copy(); srgb.Dispose(); var bands = flat.Bands > 3 ? flat.ExtractBand(0, n: 3) : flat.Bands == 1 ? flat.Bandjoin(flat, flat) : flat.Copy(); flat.Dispose(); using var cast = bands; return bands.Format == Enums.BandFormat.Uchar ? bands.Copy() : bands.Cast(Enums.BandFormat.Uchar); } async Task ProcessAv(IFormFile file, string extension, string contentType, MediaAttachment attachment, CancellationToken token) { var temp = Path.Combine(Path.GetTempPath(), $"privapub-{Guid.NewGuid():N}"); var input = temp + ".in"; var output = temp + "." + extension; try { await using (var target = File.Create(input)) await file.CopyToAsync(target, token); var probe = await FFProbe.AnalyseAsync(input, cancellationToken: token); var isVideo = contentType.StartsWith("video/") && probe.PrimaryVideoStream != default; await FFMpegArguments.FromFileInput(input) .OutputToFile(output, true, o => o.WithCustomArgument("-map 0 -map_metadata -1 -map_chapters -1 -c copy" + (extension is "mp4" or "m4a" ? " -movflags +faststart" : string.Empty))) .CancellableThrough(token) .ProcessAsynchronously(); attachment.Kind = isVideo ? "video" : "audio"; attachment.ContentType = extension switch { "mp4" => "video/mp4", "webm" => isVideo ? "video/webm" : "audio/webm", "m4a" => "audio/mp4", "mp3" => "audio/mpeg", _ => contentType }; attachment.FilePath = await Save(await File.ReadAllBytesAsync(output, token), extension, token); attachment.Size = new FileInfo(output).Length; if (isVideo) { attachment.Width = probe.PrimaryVideoStream.Width; attachment.Height = probe.PrimaryVideoStream.Height; var frame = temp + ".png"; await FFMpeg.SnapshotAsync(input, frame, captureTime: TimeSpan.FromSeconds(Math.Min(1, probe.Duration.TotalSeconds / 2))); var still = ProcessImage(await File.ReadAllBytesAsync(frame, token), 640, _options.CurrentValue.PreviewSide, false); attachment.PreviewPath = await Save(still.Preview, "jpg", token); attachment.Blurhash = still.Blurhash; File.Delete(frame); } return new MediaOutcome(attachment); } catch (Exception ex) when (ex is not OperationCanceledException) { _logger.LogInformation(ex, "Refused an audio or video upload"); return MediaOutcome.Fail(StatusCodes.Status422UnprocessableEntity, "Validation failed: The file could not be processed"); } finally { foreach (var path in new[] { input, output }) if (File.Exists(path)) File.Delete(path); } } async Task Save(byte[] bytes, string extension, CancellationToken token) { var now = DateTime.UtcNow; var relative = Path.Combine(now.ToString("yyyy", CultureInfo.InvariantCulture), now.ToString("MM", CultureInfo.InvariantCulture), $"{Convert.ToHexStringLower(RandomNumberGenerator.GetBytes(16))}.{extension}"); var full = Path.Combine(Root, relative); Directory.CreateDirectory(Path.GetDirectoryName(full)!); await File.WriteAllBytesAsync(full, bytes, token); return relative.Replace('\\', '/'); } static string Clean(string value, int max) => string.IsNullOrWhiteSpace(value) ? default : value.Trim().Length <= max ? value.Trim() : value.Trim()[..max]; static float[] Focus(string focus) { var parts = focus?.Split(','); if (parts is not { Length: 2 } || !float.TryParse(parts[0], NumberStyles.Float, CultureInfo.InvariantCulture, out var x) || !float.TryParse(parts[1], NumberStyles.Float, CultureInfo.InvariantCulture, out var y)) return default; return new[] { Math.Clamp(x, -1, 1), Math.Clamp(y, -1, 1) }; } } }