From 8e591458262ecba1aff71db04e702235da433ea1 Mon Sep 17 00:00:00 2001 From: thepra Date: Wed, 7 Oct 2026 10:48:56 +0200 Subject: [PATCH] Audio and video are processed off the request Every audio and video upload was probed and remuxed inside the request, whatever its length; ffmpeg would read any protocol and probe any format; `-map 0` kept the data tracks iPhones add, which mp4 refuses; nothing was ever transcoded, so HEVC or MPEG-4 Part 2 reached browsers that can't play them, and the advertised video_matrix_limit and frame rate limit were never applied; the output was read whole into memory, the video was saved before its poster could fail, and the poster's frame leaked in /tmp. FLAC uploads were served as 404. Now an upload sent to /api/v2/media is stored as sent in media-incoming (beside the media root, never served) and answered with 202 and no url, while a ProcessMedia job, one at a time, does the work; GET /api/v1/media/:id answers 206 until it is ready, or 422 with why, and media still processing can't be posted. v1 processes before answering. ffmpeg reads only that file (protocol whitelist, format forced from the probe) and drops data and subtitle tracks. A video browsers play as it is (H.264, VP8, VP9, AV1 within Media:MaxVideoPixels and MaxFrameRate) is remuxed, anything else transcoded to H.264 that fits, as Mastodon does; longer than Media:MaxSeconds is refused. Outputs move into place only once everything succeeded, every temporary file goes, durations are kept, FLAC is served as audio/flac, and the unit gets PrivateTmp. The instance API advertises the limits that are now applied. No pasture scenario uploads audio or video through PrivaPub, so the sweep could not see this. MastodonMediaTests: v2 answers 202 then the job makes it playable (and an unreadable file 422 once processed), media still processing can't be posted, MPEG-4 Part 2 becomes H.264, a video over the limit is made smaller, FLAC is served. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01LsXgEaXee4GCU1hwYgPJXw --- CLAUDE.md | 24 ++- PrivaPub.Tests/Domain/MediaFlowTests.cs | 4 +- PrivaPub.Tests/Http/MastodonMediaTests.cs | 57 ++++++- .../Controllers/InstanceController.cs | 6 +- .../Mastodon/Controllers/MediaController.cs | 24 ++- PrivaPub/Domain/Media/MediaOptions.cs | 3 + PrivaPub/Domain/Media/MediaService.cs | 151 ++++++++++++++---- PrivaPub/Domain/Media/ProcessMediaJob.cs | 45 ++++++ PrivaPub/Domain/Statuses/StatusService.cs | 4 +- .../Middleware/SocialPubConfigurations.cs | 1 + PrivaPub/Models/Jobs/Job.cs | 3 +- PrivaPub/Models/Media/MediaAttachment.cs | 4 + PrivaPub/Program.cs | 1 + deploy/systemd/privapub.service | 2 + 14 files changed, 281 insertions(+), 48 deletions(-) create mode 100644 PrivaPub/Domain/Media/ProcessMediaJob.cs diff --git a/CLAUDE.md b/CLAUDE.md index 5236c11..715132b 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -308,9 +308,21 @@ group www-data and reaches the private mongod; `sudo -u www-data` works too. - **Types:** HEIC and HEIF are not accepted, since the bundled libvips has no HEVC decoder. - **Load:** processing runs `Media:Concurrency` at a time, and uploads have their own rate limit (`uploads`, per credential). -3. Files live under `Media:Root` (`/var/lib/privapub/media`), never in the published directory; the proxy cache is the +3. **Audio and video are processed off the request when the client allows it.** + - **v2:** `POST /api/v2/media` stores them as sent in `media-incoming` (beside the root, never served) and answers + 202 with no url, while a `ProcessMedia` job (one at a time, its lease renewed) does the work. Until then + `GET /api/v1/media/:id` answers 206, and 422 with the reason if it failed. Media still processing can't be posted. + - **v1:** processes before answering. + - **ffmpeg** reads only the stored file: `-protocol_whitelist file`, and the input format is forced from the probe. + Data and subtitle tracks are dropped (`-dn -sn`, which iPhone MOVs need). + - **Remux or transcode:** a video browsers play as it is (H.264, VP8, VP9 or AV1, within `Media:MaxVideoPixels` and + `MaxFrameRate`) is remuxed; anything else is transcoded to H.264 that fits, as Mastodon does. + - **Limits:** longer than `Media:MaxSeconds` is refused. + - **Saving:** nothing is saved until everything succeeded; outputs move into place rather than being read into + memory, and every temporary file goes. FLAC is served as `audio/flac`, and the unit runs with `PrivateTmp`. +4. Files live under `Media:Root` (`/var/lib/privapub/media`), never in the published directory; the proxy cache is the sibling `media-proxy` and the trash the sibling `media-trash`, neither of which `/media/files` serves. -4. **A file lives exactly as long as something holds it.** Every upload is a `MediaAttachment` row, profile pictures +5. **A file lives exactly as long as something holds it.** Every upload is a `MediaAttachment` row, profile pictures too (`Kind` avatar or header, `ProfileOfAvatarId`). Deleting a post, an edit leaving media out, a replaced picture, a dropped scheduled post, a removed root, and an upload never posted for a day each trash theirs (`IMediaService.Trash`): @@ -321,18 +333,18 @@ group www-data and reaches the private mongod; `sudo -u www-data` works too. Nothing is deleted because it looks unused. `PrivaPub admin media audit [--fix]` compares disk and database: with `--fix` (as www-data) it gives pictures shown from before their rows a row, and trashes media of deleted posts or personas, rows whose files are missing, and files nothing holds. -5. **A client never contacts a remote server for media:** every remote URL the API returns goes through +6. **A client never contacts a remote server for media:** every remote URL the API returns goes through `IMediaProxy.Wrap`, an HMAC-signed `/media/proxy/` URL fetched by `IFederationHttp.GetMedia`. -6. **The proxy serves three ways:** +7. **The proxy serves three ways:** - **Cached:** a file already cached is served from disk, ranges included. - **Downloaded:** a request without a `Range` is downloaded whole, up to `Media:MaxProxiedBytes`, then cached. - **Streamed:** a ranged request, or anything too big to cache, is streamed from the origin with the range passed on, and never cached. That is how remote video plays. nginx has a `/media/proxy/` location with `proxy_buffering off` and a 600 s read timeout for those streams. -7. **A focal point is two finite numbers** within -1..1 (`FocalPoint.Parse`); anything else is ignored. A stored NaN made +8. **A focal point is two finite numbers** within -1..1 (`FocalPoint.Parse`); anything else is ignored. A stored NaN made every status, timeline and Note holding its post fail to serialise; migration `_016` removed the ones stored before. -8. **Remote video and audio become one playable attachment** in the Mastodon API (`MastodonMapper.Playable`): the best MP4 +9. **Remote video and audio become one playable attachment** in the Mastodon API (`MastodonMapper.Playable`): the best MP4 up to 720p that carries both sound and picture, including PeerTube's fragmented files inside an HLS entry. HLS playlists themselves are not rewritten. diff --git a/PrivaPub.Tests/Domain/MediaFlowTests.cs b/PrivaPub.Tests/Domain/MediaFlowTests.cs index 6a5aed8..29c0a0e 100644 --- a/PrivaPub.Tests/Domain/MediaFlowTests.cs +++ b/PrivaPub.Tests/Domain/MediaFlowTests.cs @@ -46,7 +46,7 @@ namespace PrivaPub.Tests.Domain var (_, alice) = await _harness.Persona("alice"); var (_, mallory) = await _harness.Persona("mallory"); - var upload = await _harness.Media.Upload(alice, Upload(Png(300, 200), "image/png"), "a blue square", "0.25,-0.5", token); + var upload = await _harness.Media.Upload(alice, Upload(Png(300, 200), "image/png"), "a blue square", "0.25,-0.5", false, token); Assert.True(upload.Ok); Assert.True(File.Exists(Path.Combine(_harness.Media.Root, upload.Attachment.FilePath))); @@ -73,7 +73,7 @@ namespace PrivaPub.Tests.Domain { var (_, alice) = await _harness.Persona("alice"); - var outcome = await _harness.Media.Upload(alice, Upload(System.Text.Encoding.UTF8.GetBytes(""), "image/svg+xml"), default, default, TestContext.Current.CancellationToken); + var outcome = await _harness.Media.Upload(alice, Upload(System.Text.Encoding.UTF8.GetBytes(""), "image/svg+xml"), default, default, false, TestContext.Current.CancellationToken); Assert.False(outcome.Ok); Assert.Equal(422, outcome.Status); diff --git a/PrivaPub.Tests/Http/MastodonMediaTests.cs b/PrivaPub.Tests/Http/MastodonMediaTests.cs index 2e87c0e..a5d2075 100644 --- a/PrivaPub.Tests/Http/MastodonMediaTests.cs +++ b/PrivaPub.Tests/Http/MastodonMediaTests.cs @@ -1,3 +1,4 @@ +using PrivaPub.Models.Jobs; using PrivaPub.Domain.Media; using PrivaPub.Tests.Support; using PrivaPub.Tests.Support.Host; @@ -31,6 +32,14 @@ namespace PrivaPub.Tests.Http static CancellationToken Token => TestContext.Current.CancellationToken; + // what GET /api/v1/media/:id answers once the job processing an upload sent to v2 has run + async Task Processed(Mastodon account, ApiAnswer accepted) + { + var id = accepted.Body.Text("id"); + await _host.Run(j => j.Kind == JobKind.ProcessMedia && j.Payload == id, Token); + return await account.Client.Get($"/api/v1/media/{id}"); + } + static Task Upload(Mastodon account, string path, byte[] bytes, string contentType, string fileName, params (string Key, string Value)[] fields) { var form = MastodonHelpers.Multipart(("file", bytes, contentType, fileName)); @@ -164,7 +173,11 @@ namespace PrivaPub.Tests.Http Assert.Equal(HttpStatusCode.UnprocessableEntity, text.Status); Assert.Contains("not supported", text.Body.Text("error")); Assert.Equal(HttpStatusCode.UnprocessableEntity, (await Upload(alice, "/api/v2/media", Encoding.UTF8.GetBytes("not a jpeg"), "image/jpeg", "a.jpg")).Status); - Assert.Equal(HttpStatusCode.UnprocessableEntity, (await Upload(alice, "/api/v2/media", Encoding.UTF8.GetBytes("not a video"), "video/mp4", "a.mp4")).Status); + Assert.Equal(HttpStatusCode.UnprocessableEntity, (await Upload(alice, "/api/v1/media", Encoding.UTF8.GetBytes("not a video"), "video/mp4", "a.mp4")).Status); + // sent to v2, it is taken, then refused once processed + var later = await Upload(alice, "/api/v2/media", Encoding.UTF8.GetBytes("not a video"), "video/mp4", "a.mp4"); + Assert.Equal(HttpStatusCode.Accepted, later.Status); + Assert.Equal(HttpStatusCode.UnprocessableEntity, (await Processed(alice, later)).Status); var empty = new MultipartFormDataContent { { new StringContent("no file"), "description" } }; Assert.Equal(HttpStatusCode.UnprocessableEntity, (await alice.Client.Exchange(new HttpRequestMessage(HttpMethod.Post, "/api/v2/media") { Content = empty })).Status); Assert.Equal(HttpStatusCode.UnprocessableEntity, (await alice.Client.Post("/api/v1/media")).Status); @@ -213,7 +226,13 @@ namespace PrivaPub.Tests.Http "-c:v", "mpeg4", "-c:a", "aac", "-shortest"); Assert.Contains("secret title", Encoding.Latin1.GetString(video)); - var uploaded = (await Upload(alice, "/api/v2/media", video, "video/mp4", "clip.mp4")).Ok(); + // v2 answers before it is processed: 202, no url yet, and it can't be posted until it is + var accepted = await Upload(alice, "/api/v2/media", video, "video/mp4", "clip.mp4"); + Assert.Equal(HttpStatusCode.Accepted, accepted.Status); + Assert.Null(accepted.Body["url"]); + Assert.Equal(HttpStatusCode.PartialContent, (await alice.Client.Get($"/api/v1/media/{accepted.Body.Text("id")}")).Status); + Assert.Equal(HttpStatusCode.UnprocessableEntity, (await alice.Client.Post("/api/v1/statuses", ("status", "too soon"), ("media_ids[]", accepted.Body.Text("id")))).Status); + var uploaded = (await Processed(alice, accepted)).Ok(); Assert.Equal("video", uploaded.Body.Text("type")); Assert.Equal(320, uploaded.Body["meta"]!["original"].Number("width")); @@ -221,6 +240,8 @@ namespace PrivaPub.Tests.Http Assert.EndsWith(".jpg", uploaded.Body.Text("preview_url")); var probe = await Probe(uploaded.Body.Text("url")); Assert.Contains("\"codec_type\": \"video\"", probe); + // MPEG-4 Part 2, which browsers don't play, made H.264 + Assert.Contains("\"codec_name\": \"h264\"", probe); Assert.DoesNotContain("secret title", probe); Assert.DoesNotContain("filmed at home", probe); Assert.DoesNotContain("hidden handler", probe); @@ -233,7 +254,8 @@ namespace PrivaPub.Tests.Http var audio = await Made("m4a", "-f", "lavfi", "-i", "sine=frequency=330:duration=1", "-metadata", "title=secret song", "-metadata", "artist=Alice Smith", "-c:a", "aac"); - var uploaded = (await Upload(alice, "/api/v2/media", audio, "audio/mp4", "song.m4a")).Ok(); + // v1 waits for it + var uploaded = (await Upload(alice, "/api/v1/media", audio, "audio/mp4", "song.m4a")).Ok(); Assert.Equal("audio", uploaded.Body.Text("type")); var probe = await Probe(uploaded.Body.Text("url")); @@ -242,6 +264,35 @@ namespace PrivaPub.Tests.Http Assert.DoesNotContain("Alice Smith", probe); } + // FLAC is served as what it is (ASP.NET's map has no entry for it, so it was a 404) + [Fact] + public async Task Flac_is_served() + { + var alice = await _host.Mastodon("alice"); + var flac = await Made("flac", "-f", "lavfi", "-i", "sine=frequency=500:duration=1", "-c:a", "flac"); + + var uploaded = (await Upload(alice, "/api/v1/media", flac, "audio/flac", "song.flac")).Ok(); + + var served = await _host.Client().GetAsync(new Uri(uploaded.Body.Text("url")).PathAndQuery, Token); + Assert.Equal(HttpStatusCode.OK, served.StatusCode); + Assert.Equal("audio/flac", served.Content.Headers.ContentType?.MediaType); + } + + // a video larger than video_matrix_limit is made smaller, its shape kept + [Fact] + public async Task A_video_larger_than_the_limit_is_made_smaller() + { + var alice = await _host.Mastodon("alice"); + var video = await Made("mp4", "-f", "lavfi", "-i", "testsrc=duration=1:size=2000x1500:rate=5", "-c:v", "libx264", "-preset", "ultrafast", "-pix_fmt", "yuv420p"); + + var uploaded = (await Upload(alice, "/api/v1/media", video, "video/mp4", "big.mp4")).Ok(); + + var width = uploaded.Body["meta"]!["original"].Number("width"); + var height = uploaded.Body["meta"]!["original"].Number("height"); + Assert.True((long)width * height <= 2_304_000, $"{width}x{height}"); + Assert.InRange((double)width / height, 1.3, 1.37); + } + static byte[] Bytes(int length) { var bytes = new byte[length]; diff --git a/PrivaPub/Api/Mastodon/Controllers/InstanceController.cs b/PrivaPub/Api/Mastodon/Controllers/InstanceController.cs index 3ecb7df..135e796 100644 --- a/PrivaPub/Api/Mastodon/Controllers/InstanceController.cs +++ b/PrivaPub/Api/Mastodon/Controllers/InstanceController.cs @@ -56,10 +56,10 @@ namespace PrivaPub.Api.Mastodon.Controllers { supported_mime_types = PrivaPub.Domain.Media.MediaService.SupportedTypes, image_size_limit = media.MaxImageBytes, - image_matrix_limit = media.MaxImageSide * media.MaxImageSide, + image_matrix_limit = media.MaxPixels, video_size_limit = media.MaxVideoBytes, - video_frame_rate_limit = 60, - video_matrix_limit = 2_304_000 + video_frame_rate_limit = (int)media.MaxFrameRate, + video_matrix_limit = media.MaxVideoPixels }; } } diff --git a/PrivaPub/Api/Mastodon/Controllers/MediaController.cs b/PrivaPub/Api/Mastodon/Controllers/MediaController.cs index 4b9cd7f..1f19579 100644 --- a/PrivaPub/Api/Mastodon/Controllers/MediaController.cs +++ b/PrivaPub/Api/Mastodon/Controllers/MediaController.cs @@ -9,6 +9,7 @@ using PrivaPub.Api.Mastodon.Infrastructure; using PrivaPub.Domain.Media; using PrivaPub.Models.Media; using PrivaPub.Infrastructure; +using PrivaPub.Infrastructure.Jobs; using System.Globalization; @@ -20,13 +21,15 @@ namespace PrivaPub.Api.Mastodon.Controllers readonly IMediaService _media; readonly IMediaProxy _proxy; + readonly IJobQueue _jobs; readonly IInteractionLedger _ledger; - public MediaController(IMediaService media, IMediaProxy proxy, IInteractionLedger ledger = default) + public MediaController(IMediaService media, IMediaProxy proxy, IJobQueue jobs, IInteractionLedger ledger = default) { _media = media; _proxy = proxy; + _jobs = jobs; _ledger = ledger; } @@ -37,15 +40,28 @@ namespace PrivaPub.Api.Mastodon.Controllers if (!Request.HasFormContentType) return Error(StatusCodes.Status422UnprocessableEntity, "Validation failed: File can't be blank"); var form = await Request.ReadFormAsync(token); - var outcome = await _media.Upload(Me, form.Files["file"], form["description"], form["focus"], token); - return outcome.Ok ? Json(View(outcome.Attachment)) : Error(outcome.Status, outcome.Error); + // v2 may answer before audio or video is processed (202, its url null until then), as Mastodon does; v1 waits + var later = Request.Path.StartsWithSegments("/api/v2/media"); + var outcome = await _media.Upload(Me, form.Files["file"], form["description"], form["focus"], later, token); + if (!outcome.Ok) + return Error(outcome.Status, outcome.Error); + if (outcome.Attachment.ProcessingState != "pending") + return Json(View(outcome.Attachment)); + await _jobs.EnqueueMany(new[] { ProcessMediaJob.JobFor(outcome.Attachment, new Uri(Me.BaseAddress).Host) }, token); + return new JsonResult(View(outcome.Attachment)) { StatusCode = StatusCodes.Status202Accepted }; } [HttpGet("/api/v1/media/{id}"), Scope("write:media")] public async Task Get(string id, CancellationToken token) { var attachment = await DB.Default.Find().Match(m => m.ID == id && m.OwnerAvatarId == MyId && m.TrashedAt == null && m.ProfileOfAvatarId == null).ExecuteFirstAsync(token); - return attachment == default ? NotFoundError() : Json(View(attachment)); + return attachment?.ProcessingState switch + { + null when attachment == default => NotFoundError(), + "pending" => new JsonResult(View(attachment)) { StatusCode = StatusCodes.Status206PartialContent }, + "failed" => Error(StatusCodes.Status422UnprocessableEntity, attachment.ProcessingError ?? "Validation failed: The file could not be processed"), + _ => Json(View(attachment)) + }; } [HttpPut("/api/v1/media/{id}"), Scope("write:media")] diff --git a/PrivaPub/Domain/Media/MediaOptions.cs b/PrivaPub/Domain/Media/MediaOptions.cs index a739883..07a65f6 100644 --- a/PrivaPub/Domain/Media/MediaOptions.cs +++ b/PrivaPub/Domain/Media/MediaOptions.cs @@ -13,5 +13,8 @@ namespace PrivaPub.Domain.Media public int MaxFrames { get; set; } = 500; public int MaxGifSide { get; set; } = 1280;//a GIF becomes an mp4 at most this wide public int Concurrency { get; set; } = 2;//uploads processed at once + public long MaxVideoPixels { get; set; } = 2_304_000;//a larger video is made smaller (as Mastodon's video_matrix_limit) + public double MaxFrameRate { get; set; } = 60; + public int MaxSeconds { get; set; } = 3600;//audio or video may not last longer } } diff --git a/PrivaPub/Domain/Media/MediaService.cs b/PrivaPub/Domain/Media/MediaService.cs index 14b2f61..2262282 100644 --- a/PrivaPub/Domain/Media/MediaService.cs +++ b/PrivaPub/Domain/Media/MediaService.cs @@ -38,7 +38,12 @@ namespace PrivaPub.Domain.Media /// outside what /media/files serves. (Not a dot-prefixed folder inside Root: the file provider only hides a file whose /// own name starts with a dot.) string TrashRoot { get; } - Task Upload(LocalActor owner, IFormFile file, string description, string focus, CancellationToken token); + /// Audio and video waiting for their processing: beside the media root, never served. + string IncomingRoot { get; } + /// An upload; with , audio and video are only stored, "pending" for ProcessMedia. + Task Upload(LocalActor owner, IFormFile file, string description, string focus, bool later, CancellationToken token); + /// Processes a pending upload's audio or video; false when it can't be (the row then says why). + Task ProcessPending(MediaAttachment pending, CancellationToken token); /// A persona's new avatar or header ("avatar" or "header"), cropped to its size, as a row of its own. Task ProfileImage(string avatarId, string kind, IFormFile file, int width, int height, CancellationToken token); /// Trashes the media matches (and that isn't trashed yet): each row is marked in one @@ -64,6 +69,12 @@ namespace PrivaPub.Domain.Media ["audio/mp4"] = "m4a", ["audio/x-m4a"] = "m4a", ["audio/flac"] = "flac", ["audio/webm"] = "webm" }; + // what /media/files says each file is: ASP.NET's map, which has no entry for FLAC + public static Microsoft.AspNetCore.StaticFiles.FileExtensionContentTypeProvider ContentTypes { get; } = new() + { + Mappings = { [".flac"] = "audio/flac" } + }; + // what an upload may be, as the instance API advertises it public static IReadOnlyList SupportedTypes => ImageTypes.Concat(AvTypes.Keys).ToList(); @@ -97,9 +108,13 @@ namespace PrivaPub.Domain.Media public string TrashRoot => Root.TrimEnd(Path.DirectorySeparatorChar) + "-trash"; + public string IncomingRoot => Root.TrimEnd(Path.DirectorySeparatorChar) + "-incoming"; + + string Incoming(string id) => Path.Combine(IncomingRoot, id + ".in"); + public string Url(string relativePath) => relativePath == default ? default : $"{_localActors.BaseAddress}/media/files/{relativePath.Replace('\\', '/')}"; - public async Task Upload(LocalActor owner, IFormFile file, string description, string focus, CancellationToken token) + public async Task Upload(LocalActor owner, IFormFile file, string description, string focus, bool later, CancellationToken token) { if (file == default || file.Length == 0) return MediaOutcome.Fail(StatusCodes.Status422UnprocessableEntity, "Validation failed: File can't be blank"); @@ -152,13 +167,25 @@ namespace PrivaPub.Domain.Media _processing.Release(); } } - else if (contentType != default && AvTypes.TryGetValue(contentType, out var extension)) + else if (contentType != default && AvTypes.ContainsKey(contentType)) { if (file.Length > options.MaxVideoBytes) return MediaOutcome.Fail(StatusCodes.Status422UnprocessableEntity, "Validation failed: File is too big"); - var outcome = await ProcessAv(file, extension, contentType, attachment, token); - if (!outcome.Ok) - return outcome; + // stored as it came, outside what is served, then processed: by a job when asked later (v2), here otherwise (v1) + attachment.ID = (string)attachment.GenerateNewID(); + attachment.Kind = contentType.StartsWith("video/", StringComparison.Ordinal) ? "video" : "audio"; + attachment.ContentType = contentType; + Directory.CreateDirectory(IncomingRoot); + await using (var stored = File.Create(Incoming(attachment.ID))) + await file.CopyToAsync(stored, token); + if (later) + attachment.ProcessingState = "pending"; + else + { + var outcome = await ProcessIncoming(attachment, token); + if (!outcome.Ok) + return outcome; + } } else return MediaOutcome.Fail(StatusCodes.Status422UnprocessableEntity, "Validation failed: File type is not supported"); @@ -251,6 +278,8 @@ namespace PrivaPub.Domain.Media foreach (var full in new[] { Path.Combine(TrashRoot, relative), Path.Combine(Root, relative) }) if (File.Exists(full)) File.Delete(full); + if (File.Exists(Incoming(trashed.ID))) + File.Delete(Incoming(trashed.ID)); await DB.Default.DeleteAsync(m => m.ID == trashed.ID && m.TrashedAt != null); } @@ -362,37 +391,102 @@ namespace PrivaPub.Domain.Media return bands.Format == Enums.BandFormat.Uchar ? bands.Copy() : bands.Cast(Enums.BandFormat.Uchar); } - async Task ProcessAv(IFormFile file, string extension, string contentType, MediaAttachment attachment, CancellationToken token) + public async Task ProcessPending(MediaAttachment pending, CancellationToken token) { + var outcome = await ProcessIncoming(pending, token); + if (outcome.Ok) + { + await DB.Default.Update().Match(m => m.ID == pending.ID && m.TrashedAt == null) + .Modify(m => m.Kind, pending.Kind) + .Modify(m => m.ContentType, pending.ContentType) + .Modify(m => m.FilePath, pending.FilePath) + .Modify(m => m.PreviewPath, pending.PreviewPath) + .Modify(m => m.Size, pending.Size) + .Modify(m => m.Width, pending.Width) + .Modify(m => m.Height, pending.Height) + .Modify(m => m.Blurhash, pending.Blurhash) + .Modify(m => m.DurationSeconds, pending.DurationSeconds) + .Modify(m => m.ProcessingState, null) + .ExecuteAsync(token); + return true; + } + await DB.Default.Update().MatchID(pending.ID) + .Modify(m => m.ProcessingState, "failed").Modify(m => m.ProcessingError, outcome.Error).ExecuteAsync(token); + return false; + } + + // the stored upload, made playable: probed, then remuxed without its metadata when browsers play it as it is (H.264, + // VP8, VP9 or AV1 within MaxVideoPixels and MaxFrameRate), or else transcoded to H.264 that fits; a frame becomes the + // poster. ffmpeg only ever reads the local file (no protocol but file, its format forced from the probe). Nothing is + // saved until everything succeeded, and every temporary file goes, the stored upload too. + async Task ProcessIncoming(MediaAttachment attachment, CancellationToken token) + { + var options = _options.CurrentValue; + var input = Incoming(attachment.ID); var temp = Path.Combine(Path.GetTempPath(), $"privapub-{Guid.NewGuid():N}"); - var input = temp + ".in"; - var output = temp + "." + extension; + var frame = temp + ".png"; + var output = default(string); + await _processing.WaitAsync(token); try { - await using (var target = File.Create(input)) - await file.CopyToAsync(target, token); - var probe = await FFProbe.AnalyseAsync(input, cancellationToken: token); - var isVideo = contentType.StartsWith("video/") && probe.PrimaryVideoStream != default; - await FFMpegArguments.FromFileInput(input) - .OutputToFile(output, true, o => o.WithCustomArgument("-map 0 -map_metadata -1 -map_chapters -1 -c copy" + (extension is "mp4" or "m4a" ? " -movflags +faststart" : string.Empty))) + var probe = await FFProbe.AnalyseAsync(input, default, token, "-protocol_whitelist file"); + if (probe.Duration > TimeSpan.FromSeconds(options.MaxSeconds)) + return MediaOutcome.Fail(StatusCodes.Status422UnprocessableEntity, "Validation failed: The file is too long"); + var video = attachment.ContentType.StartsWith("video/", StringComparison.Ordinal) ? probe.PrimaryVideoStream : default; + var audio = probe.PrimaryAudioStream; + if (video == default && audio == default) + return MediaOutcome.Fail(StatusCodes.Status422UnprocessableEntity, "Validation failed: The file could not be processed"); + var extension = AvTypes[attachment.ContentType]; + var format = probe.Format.FormatName.Split(',')[0]; + string arguments; + if (video == default) + arguments = "-map 0:a:0 -vn -dn -sn -map_metadata -1 -map_chapters -1 -c copy" + (extension == "m4a" ? " -movflags +faststart" : string.Empty); + else if (video.CodecName is "h264" or "vp8" or "vp9" or "av1" && (long)video.Width * video.Height <= options.MaxVideoPixels + && video.FrameRate <= options.MaxFrameRate + 0.5) + arguments = "-map 0:v:0 -map 0:a:0? -dn -sn -map_metadata -1 -map_chapters -1 -c copy" + (extension == "mp4" ? " -movflags +faststart" : string.Empty); + else + { + var scale = Math.Min(1, Math.Sqrt(options.MaxVideoPixels / (double)((long)video.Width * video.Height))); + var width = Math.Max(2, (int)(video.Width * scale) / 2 * 2); + var height = Math.Max(2, (int)(video.Height * scale) / 2 * 2); + extension = "mp4"; + arguments = $"-map 0:v:0 -map 0:a:0? -dn -sn -map_metadata -1 -map_chapters -1 -c:v libx264 -preset veryfast -crf 23 -pix_fmt yuv420p " + + $"-vf scale={width}:{height} -fpsmax {options.MaxFrameRate.ToString(CultureInfo.InvariantCulture)} -c:a aac -b:a 128k -movflags +faststart"; + } + output = temp + "." + extension; + await FFMpegArguments.FromFileInput(input, true, o => o.ForceFormat(format).WithCustomArgument("-protocol_whitelist file")) + .OutputToFile(output, true, o => o.WithCustomArgument(arguments)) .CancellableThrough(token) .ProcessAsynchronously(); - attachment.Kind = isVideo ? "video" : "audio"; - attachment.ContentType = extension switch { "mp4" => "video/mp4", "webm" => isVideo ? "video/webm" : "audio/webm", "m4a" => "audio/mp4", "mp3" => "audio/mpeg", _ => contentType }; - attachment.FilePath = await Save(await File.ReadAllBytesAsync(output, token), extension, token); - attachment.Size = new FileInfo(output).Length; - if (isVideo) + var made = await FFProbe.AnalyseAsync(output, default, token, "-protocol_whitelist file"); + byte[] poster = default; + if (video != default) { - attachment.Width = probe.PrimaryVideoStream.Width; - attachment.Height = probe.PrimaryVideoStream.Height; - var frame = temp + ".png"; - await FFMpeg.SnapshotAsync(input, frame, captureTime: TimeSpan.FromSeconds(Math.Min(1, probe.Duration.TotalSeconds / 2))); - var still = ProcessImage(await File.ReadAllBytesAsync(frame, token), 640, _options.CurrentValue.PreviewSide); - attachment.PreviewPath = await Save(still.Preview, "jpg", token); + attachment.Width = made.PrimaryVideoStream?.Width; + attachment.Height = made.PrimaryVideoStream?.Height; + await FFMpeg.SnapshotAsync(output, frame, captureTime: TimeSpan.FromSeconds(Math.Min(1, made.Duration.TotalSeconds / 2))); + var still = ProcessImage(await File.ReadAllBytesAsync(frame, token), 640, options.PreviewSide); + poster = still.Preview; attachment.Blurhash = still.Blurhash; - File.Delete(frame); } + attachment.Kind = video != default ? "video" : "audio"; + attachment.ContentType = extension switch + { + "mp4" => "video/mp4", + "webm" => video != default ? "video/webm" : "audio/webm", + "m4a" => "audio/mp4", + "mp3" => "audio/mpeg", + "ogg" => "audio/ogg", + "wav" => "audio/wav", + "flac" => "audio/flac", + _ => attachment.ContentType + }; + attachment.DurationSeconds = Math.Round(made.Duration.TotalSeconds, 2); + attachment.Size = new FileInfo(output).Length; + attachment.FilePath = SaveFile(output, extension); + if (poster != default) + attachment.PreviewPath = await Save(poster, "jpg", token); return new MediaOutcome(attachment); } catch (Exception ex) when (ex is not OperationCanceledException) @@ -402,7 +496,8 @@ namespace PrivaPub.Domain.Media } finally { - foreach (var path in new[] { input, output }) + _processing.Release(); + foreach (var path in new[] { input, output, frame }.Where(p => p != default)) if (File.Exists(path)) File.Delete(path); } diff --git a/PrivaPub/Domain/Media/ProcessMediaJob.cs b/PrivaPub/Domain/Media/ProcessMediaJob.cs new file mode 100644 index 0000000..a1c266f --- /dev/null +++ b/PrivaPub/Domain/Media/ProcessMediaJob.cs @@ -0,0 +1,45 @@ +using MongoDB.Entities; + +using PrivaPub.Infrastructure.Jobs; +using PrivaPub.Models.Jobs; +using PrivaPub.Models.Media; + +namespace PrivaPub.Domain.Media +{ + // Audio and video sent to /api/v2/media, processed off the request (its lease renewed while ffmpeg runs): playable, or + // marked failed with why. One at a time, since a transcode takes the machine's cores. + public class ProcessMediaJob : IJobHandler + { + readonly IMediaService _media; + readonly ILogger _logger; + + public ProcessMediaJob(IMediaService media, ILogger logger) + { + _media = media; + _logger = logger; + } + + public JobKind Kind => JobKind.ProcessMedia; + public int Concurrency => 1; + public int MaxAttempts => 3; + public int PerHostLimit => 1; + + public static Job JobFor(MediaAttachment pending, string host) => new() + { + Kind = JobKind.ProcessMedia, + Payload = pending.ID, + Host = host, + DedupeKey = $"media|{pending.ID}" + }; + + public async Task Handle(Job job, CancellationToken token) + { + var pending = await DB.Default.Find().Match(m => m.ID == job.Payload && m.ProcessingState == "pending" && m.TrashedAt == null).ExecuteFirstAsync(token); + if (pending == default) + return JobOutcome.Done;//processed already, or trashed meanwhile + if (!await _media.ProcessPending(pending, token)) + _logger.LogInformation("Upload {Id} could not be processed", pending.ID); + return JobOutcome.Done; + } + } +} diff --git a/PrivaPub/Domain/Statuses/StatusService.cs b/PrivaPub/Domain/Statuses/StatusService.cs index d810b02..09a06d9 100644 --- a/PrivaPub/Domain/Statuses/StatusService.cs +++ b/PrivaPub/Domain/Statuses/StatusService.cs @@ -703,7 +703,8 @@ namespace PrivaPub.Domain.Statuses var wanted = ids.Distinct().ToList(); var found = await DB.Default.Find() .Match(m => wanted.Contains(m.ID) && m.OwnerAvatarId == author.Id && (m.PostId == null || m.PostId == postId) - && (m.ScheduledStatusId == null || m.ScheduledStatusId == scheduledId) && m.TrashedAt == null && m.ProfileOfAvatarId == null) + && (m.ScheduledStatusId == null || m.ScheduledStatusId == scheduledId) && m.TrashedAt == null && m.ProfileOfAvatarId == null + && m.ProcessingState == null) .ExecuteAsync(token); return found.Count == wanted.Count ? wanted.Select(id => found.First(m => m.ID == id)).ToList() : default; } @@ -732,6 +733,7 @@ namespace PrivaPub.Domain.Statuses AttachmentId = attachment.ID, ContentType = attachment.ContentType, Kind = attachment.Kind, + DurationSeconds = attachment.DurationSeconds, URL = _media.Url(attachment.FilePath), PreviewURL = _media.Url(attachment.PreviewPath ?? attachment.FilePath), Description = attachment.Description, diff --git a/PrivaPub/Middleware/SocialPubConfigurations.cs b/PrivaPub/Middleware/SocialPubConfigurations.cs index fed3a5a..38fb3c3 100644 --- a/PrivaPub/Middleware/SocialPubConfigurations.cs +++ b/PrivaPub/Middleware/SocialPubConfigurations.cs @@ -121,6 +121,7 @@ namespace PrivaPub.Middleware .AddSingleton() .AddSingleton() .AddSingleton() + .AddSingleton() .AddSingleton() .AddSingleton() .AddSingleton(services => services.GetRequiredService()) diff --git a/PrivaPub/Models/Jobs/Job.cs b/PrivaPub/Models/Jobs/Job.cs index c22cef5..dd38d69 100644 --- a/PrivaPub/Models/Jobs/Job.cs +++ b/PrivaPub/Models/Jobs/Job.cs @@ -35,7 +35,8 @@ namespace PrivaPub.Models.Jobs PublishScheduled, FetchReplies, BackfillOutbox, - SynchronizeFollowing + SynchronizeFollowing, + ProcessMedia } public enum JobState diff --git a/PrivaPub/Models/Media/MediaAttachment.cs b/PrivaPub/Models/Media/MediaAttachment.cs index d9318b0..b5e9298 100644 --- a/PrivaPub/Models/Media/MediaAttachment.cs +++ b/PrivaPub/Models/Media/MediaAttachment.cs @@ -16,6 +16,10 @@ namespace PrivaPub.Models.Media public int? Width { get; set; } public int? Height { get; set; } public float[] Focus { get; set; } + public double? DurationSeconds { get; set; } + // audio and video sent to /api/v2/media are processed by a job: "pending" until then, "failed" (with why) if it can't be + public string ProcessingState { get; set; } + public string ProcessingError { get; set; } public DateTime CreatedAt { get; set; } = DateTime.UtcNow; public DateTime? AttachedAt { get; set; } public string ScheduledStatusId { get; set; }//kept for a scheduled post until it is published or dropped diff --git a/PrivaPub/Program.cs b/PrivaPub/Program.cs index e63f31e..0ac26cb 100644 --- a/PrivaPub/Program.cs +++ b/PrivaPub/Program.cs @@ -144,6 +144,7 @@ try { FileProvider = new Microsoft.Extensions.FileProviders.PhysicalFileProvider(mediaRoot), RequestPath = "/media/files", + ContentTypeProvider = PrivaPub.Domain.Media.MediaService.ContentTypes, OnPrepareResponse = context => { context.Context.Response.Headers["X-Content-Type-Options"] = "nosniff"; diff --git a/deploy/systemd/privapub.service b/deploy/systemd/privapub.service index ba999a9..3dbdbcf 100644 --- a/deploy/systemd/privapub.service +++ b/deploy/systemd/privapub.service @@ -14,6 +14,8 @@ Environment=ASPNETCORE_ENVIRONMENT=Production Environment=DOTNET_CLI_TELEMETRY_OPTOUT=1 Restart=always RestartSec=5 +# its own /tmp: ffmpeg's and the uploads' temporary files are nobody else's +PrivateTmp=true SyslogIdentifier=privapub [Install]