diff --git a/CLAUDE.md b/CLAUDE.md index 5236c11..715132b 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -308,9 +308,21 @@ group www-data and reaches the private mongod; `sudo -u www-data` works too. - **Types:** HEIC and HEIF are not accepted, since the bundled libvips has no HEVC decoder. - **Load:** processing runs `Media:Concurrency` at a time, and uploads have their own rate limit (`uploads`, per credential). -3. Files live under `Media:Root` (`/var/lib/privapub/media`), never in the published directory; the proxy cache is the +3. **Audio and video are processed off the request when the client allows it.** + - **v2:** `POST /api/v2/media` stores them as sent in `media-incoming` (beside the root, never served) and answers + 202 with no url, while a `ProcessMedia` job (one at a time, its lease renewed) does the work. Until then + `GET /api/v1/media/:id` answers 206, and 422 with the reason if it failed. Media still processing can't be posted. + - **v1:** processes before answering. + - **ffmpeg** reads only the stored file: `-protocol_whitelist file`, and the input format is forced from the probe. + Data and subtitle tracks are dropped (`-dn -sn`, which iPhone MOVs need). + - **Remux or transcode:** a video browsers play as it is (H.264, VP8, VP9 or AV1, within `Media:MaxVideoPixels` and + `MaxFrameRate`) is remuxed; anything else is transcoded to H.264 that fits, as Mastodon does. + - **Limits:** longer than `Media:MaxSeconds` is refused. + - **Saving:** nothing is saved until everything succeeded; outputs move into place rather than being read into + memory, and every temporary file goes. FLAC is served as `audio/flac`, and the unit runs with `PrivateTmp`. +4. Files live under `Media:Root` (`/var/lib/privapub/media`), never in the published directory; the proxy cache is the sibling `media-proxy` and the trash the sibling `media-trash`, neither of which `/media/files` serves. -4. **A file lives exactly as long as something holds it.** Every upload is a `MediaAttachment` row, profile pictures +5. **A file lives exactly as long as something holds it.** Every upload is a `MediaAttachment` row, profile pictures too (`Kind` avatar or header, `ProfileOfAvatarId`). Deleting a post, an edit leaving media out, a replaced picture, a dropped scheduled post, a removed root, and an upload never posted for a day each trash theirs (`IMediaService.Trash`): @@ -321,18 +333,18 @@ group www-data and reaches the private mongod; `sudo -u www-data` works too. Nothing is deleted because it looks unused. `PrivaPub admin media audit [--fix]` compares disk and database: with `--fix` (as www-data) it gives pictures shown from before their rows a row, and trashes media of deleted posts or personas, rows whose files are missing, and files nothing holds. -5. **A client never contacts a remote server for media:** every remote URL the API returns goes through +6. **A client never contacts a remote server for media:** every remote URL the API returns goes through `IMediaProxy.Wrap`, an HMAC-signed `/media/proxy/` URL fetched by `IFederationHttp.GetMedia`. -6. **The proxy serves three ways:** +7. **The proxy serves three ways:** - **Cached:** a file already cached is served from disk, ranges included. - **Downloaded:** a request without a `Range` is downloaded whole, up to `Media:MaxProxiedBytes`, then cached. - **Streamed:** a ranged request, or anything too big to cache, is streamed from the origin with the range passed on, and never cached. That is how remote video plays. nginx has a `/media/proxy/` location with `proxy_buffering off` and a 600 s read timeout for those streams. -7. **A focal point is two finite numbers** within -1..1 (`FocalPoint.Parse`); anything else is ignored. A stored NaN made +8. **A focal point is two finite numbers** within -1..1 (`FocalPoint.Parse`); anything else is ignored. A stored NaN made every status, timeline and Note holding its post fail to serialise; migration `_016` removed the ones stored before. -8. **Remote video and audio become one playable attachment** in the Mastodon API (`MastodonMapper.Playable`): the best MP4 +9. **Remote video and audio become one playable attachment** in the Mastodon API (`MastodonMapper.Playable`): the best MP4 up to 720p that carries both sound and picture, including PeerTube's fragmented files inside an HLS entry. HLS playlists themselves are not rewritten. diff --git a/PrivaPub.Tests/Domain/MediaFlowTests.cs b/PrivaPub.Tests/Domain/MediaFlowTests.cs index 6a5aed8..29c0a0e 100644 --- a/PrivaPub.Tests/Domain/MediaFlowTests.cs +++ b/PrivaPub.Tests/Domain/MediaFlowTests.cs @@ -46,7 +46,7 @@ namespace PrivaPub.Tests.Domain var (_, alice) = await _harness.Persona("alice"); var (_, mallory) = await _harness.Persona("mallory"); - var upload = await _harness.Media.Upload(alice, Upload(Png(300, 200), "image/png"), "a blue square", "0.25,-0.5", token); + var upload = await _harness.Media.Upload(alice, Upload(Png(300, 200), "image/png"), "a blue square", "0.25,-0.5", false, token); Assert.True(upload.Ok); Assert.True(File.Exists(Path.Combine(_harness.Media.Root, upload.Attachment.FilePath))); @@ -73,7 +73,7 @@ namespace PrivaPub.Tests.Domain { var (_, alice) = await _harness.Persona("alice"); - var outcome = await _harness.Media.Upload(alice, Upload(System.Text.Encoding.UTF8.GetBytes(""), "image/svg+xml"), default, default, TestContext.Current.CancellationToken); + var outcome = await _harness.Media.Upload(alice, Upload(System.Text.Encoding.UTF8.GetBytes(""), "image/svg+xml"), default, default, false, TestContext.Current.CancellationToken); Assert.False(outcome.Ok); Assert.Equal(422, outcome.Status); diff --git a/PrivaPub.Tests/Http/MastodonMediaTests.cs b/PrivaPub.Tests/Http/MastodonMediaTests.cs index 2e87c0e..a5d2075 100644 --- a/PrivaPub.Tests/Http/MastodonMediaTests.cs +++ b/PrivaPub.Tests/Http/MastodonMediaTests.cs @@ -1,3 +1,4 @@ +using PrivaPub.Models.Jobs; using PrivaPub.Domain.Media; using PrivaPub.Tests.Support; using PrivaPub.Tests.Support.Host; @@ -31,6 +32,14 @@ namespace PrivaPub.Tests.Http static CancellationToken Token => TestContext.Current.CancellationToken; + // what GET /api/v1/media/:id answers once the job processing an upload sent to v2 has run + async Task Processed(Mastodon account, ApiAnswer accepted) + { + var id = accepted.Body.Text("id"); + await _host.Run(j => j.Kind == JobKind.ProcessMedia && j.Payload == id, Token); + return await account.Client.Get($"/api/v1/media/{id}"); + } + static Task Upload(Mastodon account, string path, byte[] bytes, string contentType, string fileName, params (string Key, string Value)[] fields) { var form = MastodonHelpers.Multipart(("file", bytes, contentType, fileName)); @@ -164,7 +173,11 @@ namespace PrivaPub.Tests.Http Assert.Equal(HttpStatusCode.UnprocessableEntity, text.Status); Assert.Contains("not supported", text.Body.Text("error")); Assert.Equal(HttpStatusCode.UnprocessableEntity, (await Upload(alice, "/api/v2/media", Encoding.UTF8.GetBytes("not a jpeg"), "image/jpeg", "a.jpg")).Status); - Assert.Equal(HttpStatusCode.UnprocessableEntity, (await Upload(alice, "/api/v2/media", Encoding.UTF8.GetBytes("not a video"), "video/mp4", "a.mp4")).Status); + Assert.Equal(HttpStatusCode.UnprocessableEntity, (await Upload(alice, "/api/v1/media", Encoding.UTF8.GetBytes("not a video"), "video/mp4", "a.mp4")).Status); + // sent to v2, it is taken, then refused once processed + var later = await Upload(alice, "/api/v2/media", Encoding.UTF8.GetBytes("not a video"), "video/mp4", "a.mp4"); + Assert.Equal(HttpStatusCode.Accepted, later.Status); + Assert.Equal(HttpStatusCode.UnprocessableEntity, (await Processed(alice, later)).Status); var empty = new MultipartFormDataContent { { new StringContent("no file"), "description" } }; Assert.Equal(HttpStatusCode.UnprocessableEntity, (await alice.Client.Exchange(new HttpRequestMessage(HttpMethod.Post, "/api/v2/media") { Content = empty })).Status); Assert.Equal(HttpStatusCode.UnprocessableEntity, (await alice.Client.Post("/api/v1/media")).Status); @@ -213,7 +226,13 @@ namespace PrivaPub.Tests.Http "-c:v", "mpeg4", "-c:a", "aac", "-shortest"); Assert.Contains("secret title", Encoding.Latin1.GetString(video)); - var uploaded = (await Upload(alice, "/api/v2/media", video, "video/mp4", "clip.mp4")).Ok(); + // v2 answers before it is processed: 202, no url yet, and it can't be posted until it is + var accepted = await Upload(alice, "/api/v2/media", video, "video/mp4", "clip.mp4"); + Assert.Equal(HttpStatusCode.Accepted, accepted.Status); + Assert.Null(accepted.Body["url"]); + Assert.Equal(HttpStatusCode.PartialContent, (await alice.Client.Get($"/api/v1/media/{accepted.Body.Text("id")}")).Status); + Assert.Equal(HttpStatusCode.UnprocessableEntity, (await alice.Client.Post("/api/v1/statuses", ("status", "too soon"), ("media_ids[]", accepted.Body.Text("id")))).Status); + var uploaded = (await Processed(alice, accepted)).Ok(); Assert.Equal("video", uploaded.Body.Text("type")); Assert.Equal(320, uploaded.Body["meta"]!["original"].Number("width")); @@ -221,6 +240,8 @@ namespace PrivaPub.Tests.Http Assert.EndsWith(".jpg", uploaded.Body.Text("preview_url")); var probe = await Probe(uploaded.Body.Text("url")); Assert.Contains("\"codec_type\": \"video\"", probe); + // MPEG-4 Part 2, which browsers don't play, made H.264 + Assert.Contains("\"codec_name\": \"h264\"", probe); Assert.DoesNotContain("secret title", probe); Assert.DoesNotContain("filmed at home", probe); Assert.DoesNotContain("hidden handler", probe); @@ -233,7 +254,8 @@ namespace PrivaPub.Tests.Http var audio = await Made("m4a", "-f", "lavfi", "-i", "sine=frequency=330:duration=1", "-metadata", "title=secret song", "-metadata", "artist=Alice Smith", "-c:a", "aac"); - var uploaded = (await Upload(alice, "/api/v2/media", audio, "audio/mp4", "song.m4a")).Ok(); + // v1 waits for it + var uploaded = (await Upload(alice, "/api/v1/media", audio, "audio/mp4", "song.m4a")).Ok(); Assert.Equal("audio", uploaded.Body.Text("type")); var probe = await Probe(uploaded.Body.Text("url")); @@ -242,6 +264,35 @@ namespace PrivaPub.Tests.Http Assert.DoesNotContain("Alice Smith", probe); } + // FLAC is served as what it is (ASP.NET's map has no entry for it, so it was a 404) + [Fact] + public async Task Flac_is_served() + { + var alice = await _host.Mastodon("alice"); + var flac = await Made("flac", "-f", "lavfi", "-i", "sine=frequency=500:duration=1", "-c:a", "flac"); + + var uploaded = (await Upload(alice, "/api/v1/media", flac, "audio/flac", "song.flac")).Ok(); + + var served = await _host.Client().GetAsync(new Uri(uploaded.Body.Text("url")).PathAndQuery, Token); + Assert.Equal(HttpStatusCode.OK, served.StatusCode); + Assert.Equal("audio/flac", served.Content.Headers.ContentType?.MediaType); + } + + // a video larger than video_matrix_limit is made smaller, its shape kept + [Fact] + public async Task A_video_larger_than_the_limit_is_made_smaller() + { + var alice = await _host.Mastodon("alice"); + var video = await Made("mp4", "-f", "lavfi", "-i", "testsrc=duration=1:size=2000x1500:rate=5", "-c:v", "libx264", "-preset", "ultrafast", "-pix_fmt", "yuv420p"); + + var uploaded = (await Upload(alice, "/api/v1/media", video, "video/mp4", "big.mp4")).Ok(); + + var width = uploaded.Body["meta"]!["original"].Number("width"); + var height = uploaded.Body["meta"]!["original"].Number("height"); + Assert.True((long)width * height <= 2_304_000, $"{width}x{height}"); + Assert.InRange((double)width / height, 1.3, 1.37); + } + static byte[] Bytes(int length) { var bytes = new byte[length]; diff --git a/PrivaPub/Api/Mastodon/Controllers/InstanceController.cs b/PrivaPub/Api/Mastodon/Controllers/InstanceController.cs index 3ecb7df..135e796 100644 --- a/PrivaPub/Api/Mastodon/Controllers/InstanceController.cs +++ b/PrivaPub/Api/Mastodon/Controllers/InstanceController.cs @@ -56,10 +56,10 @@ namespace PrivaPub.Api.Mastodon.Controllers { supported_mime_types = PrivaPub.Domain.Media.MediaService.SupportedTypes, image_size_limit = media.MaxImageBytes, - image_matrix_limit = media.MaxImageSide * media.MaxImageSide, + image_matrix_limit = media.MaxPixels, video_size_limit = media.MaxVideoBytes, - video_frame_rate_limit = 60, - video_matrix_limit = 2_304_000 + video_frame_rate_limit = (int)media.MaxFrameRate, + video_matrix_limit = media.MaxVideoPixels }; } } diff --git a/PrivaPub/Api/Mastodon/Controllers/MediaController.cs b/PrivaPub/Api/Mastodon/Controllers/MediaController.cs index 4b9cd7f..1f19579 100644 --- a/PrivaPub/Api/Mastodon/Controllers/MediaController.cs +++ b/PrivaPub/Api/Mastodon/Controllers/MediaController.cs @@ -9,6 +9,7 @@ using PrivaPub.Api.Mastodon.Infrastructure; using PrivaPub.Domain.Media; using PrivaPub.Models.Media; using PrivaPub.Infrastructure; +using PrivaPub.Infrastructure.Jobs; using System.Globalization; @@ -20,13 +21,15 @@ namespace PrivaPub.Api.Mastodon.Controllers readonly IMediaService _media; readonly IMediaProxy _proxy; + readonly IJobQueue _jobs; readonly IInteractionLedger _ledger; - public MediaController(IMediaService media, IMediaProxy proxy, IInteractionLedger ledger = default) + public MediaController(IMediaService media, IMediaProxy proxy, IJobQueue jobs, IInteractionLedger ledger = default) { _media = media; _proxy = proxy; + _jobs = jobs; _ledger = ledger; } @@ -37,15 +40,28 @@ namespace PrivaPub.Api.Mastodon.Controllers if (!Request.HasFormContentType) return Error(StatusCodes.Status422UnprocessableEntity, "Validation failed: File can't be blank"); var form = await Request.ReadFormAsync(token); - var outcome = await _media.Upload(Me, form.Files["file"], form["description"], form["focus"], token); - return outcome.Ok ? Json(View(outcome.Attachment)) : Error(outcome.Status, outcome.Error); + // v2 may answer before audio or video is processed (202, its url null until then), as Mastodon does; v1 waits + var later = Request.Path.StartsWithSegments("/api/v2/media"); + var outcome = await _media.Upload(Me, form.Files["file"], form["description"], form["focus"], later, token); + if (!outcome.Ok) + return Error(outcome.Status, outcome.Error); + if (outcome.Attachment.ProcessingState != "pending") + return Json(View(outcome.Attachment)); + await _jobs.EnqueueMany(new[] { ProcessMediaJob.JobFor(outcome.Attachment, new Uri(Me.BaseAddress).Host) }, token); + return new JsonResult(View(outcome.Attachment)) { StatusCode = StatusCodes.Status202Accepted }; } [HttpGet("/api/v1/media/{id}"), Scope("write:media")] public async Task Get(string id, CancellationToken token) { var attachment = await DB.Default.Find().Match(m => m.ID == id && m.OwnerAvatarId == MyId && m.TrashedAt == null && m.ProfileOfAvatarId == null).ExecuteFirstAsync(token); - return attachment == default ? NotFoundError() : Json(View(attachment)); + return attachment?.ProcessingState switch + { + null when attachment == default => NotFoundError(), + "pending" => new JsonResult(View(attachment)) { StatusCode = StatusCodes.Status206PartialContent }, + "failed" => Error(StatusCodes.Status422UnprocessableEntity, attachment.ProcessingError ?? "Validation failed: The file could not be processed"), + _ => Json(View(attachment)) + }; } [HttpPut("/api/v1/media/{id}"), Scope("write:media")] diff --git a/PrivaPub/Domain/Media/MediaOptions.cs b/PrivaPub/Domain/Media/MediaOptions.cs index a739883..07a65f6 100644 --- a/PrivaPub/Domain/Media/MediaOptions.cs +++ b/PrivaPub/Domain/Media/MediaOptions.cs @@ -13,5 +13,8 @@ namespace PrivaPub.Domain.Media public int MaxFrames { get; set; } = 500; public int MaxGifSide { get; set; } = 1280;//a GIF becomes an mp4 at most this wide public int Concurrency { get; set; } = 2;//uploads processed at once + public long MaxVideoPixels { get; set; } = 2_304_000;//a larger video is made smaller (as Mastodon's video_matrix_limit) + public double MaxFrameRate { get; set; } = 60; + public int MaxSeconds { get; set; } = 3600;//audio or video may not last longer } } diff --git a/PrivaPub/Domain/Media/MediaService.cs b/PrivaPub/Domain/Media/MediaService.cs index 14b2f61..2262282 100644 --- a/PrivaPub/Domain/Media/MediaService.cs +++ b/PrivaPub/Domain/Media/MediaService.cs @@ -38,7 +38,12 @@ namespace PrivaPub.Domain.Media /// outside what /media/files serves. (Not a dot-prefixed folder inside Root: the file provider only hides a file whose /// own name starts with a dot.) string TrashRoot { get; } - Task Upload(LocalActor owner, IFormFile file, string description, string focus, CancellationToken token); + /// Audio and video waiting for their processing: beside the media root, never served. + string IncomingRoot { get; } + /// An upload; with , audio and video are only stored, "pending" for ProcessMedia. + Task Upload(LocalActor owner, IFormFile file, string description, string focus, bool later, CancellationToken token); + /// Processes a pending upload's audio or video; false when it can't be (the row then says why). + Task ProcessPending(MediaAttachment pending, CancellationToken token); /// A persona's new avatar or header ("avatar" or "header"), cropped to its size, as a row of its own. Task ProfileImage(string avatarId, string kind, IFormFile file, int width, int height, CancellationToken token); /// Trashes the media matches (and that isn't trashed yet): each row is marked in one @@ -64,6 +69,12 @@ namespace PrivaPub.Domain.Media ["audio/mp4"] = "m4a", ["audio/x-m4a"] = "m4a", ["audio/flac"] = "flac", ["audio/webm"] = "webm" }; + // what /media/files says each file is: ASP.NET's map, which has no entry for FLAC + public static Microsoft.AspNetCore.StaticFiles.FileExtensionContentTypeProvider ContentTypes { get; } = new() + { + Mappings = { [".flac"] = "audio/flac" } + }; + // what an upload may be, as the instance API advertises it public static IReadOnlyList SupportedTypes => ImageTypes.Concat(AvTypes.Keys).ToList(); @@ -97,9 +108,13 @@ namespace PrivaPub.Domain.Media public string TrashRoot => Root.TrimEnd(Path.DirectorySeparatorChar) + "-trash"; + public string IncomingRoot => Root.TrimEnd(Path.DirectorySeparatorChar) + "-incoming"; + + string Incoming(string id) => Path.Combine(IncomingRoot, id + ".in"); + public string Url(string relativePath) => relativePath == default ? default : $"{_localActors.BaseAddress}/media/files/{relativePath.Replace('\\', '/')}"; - public async Task Upload(LocalActor owner, IFormFile file, string description, string focus, CancellationToken token) + public async Task Upload(LocalActor owner, IFormFile file, string description, string focus, bool later, CancellationToken token) { if (file == default || file.Length == 0) return MediaOutcome.Fail(StatusCodes.Status422UnprocessableEntity, "Validation failed: File can't be blank"); @@ -152,13 +167,25 @@ namespace PrivaPub.Domain.Media _processing.Release(); } } - else if (contentType != default && AvTypes.TryGetValue(contentType, out var extension)) + else if (contentType != default && AvTypes.ContainsKey(contentType)) { if (file.Length > options.MaxVideoBytes) return MediaOutcome.Fail(StatusCodes.Status422UnprocessableEntity, "Validation failed: File is too big"); - var outcome = await ProcessAv(file, extension, contentType, attachment, token); - if (!outcome.Ok) - return outcome; + // stored as it came, outside what is served, then processed: by a job when asked later (v2), here otherwise (v1) + attachment.ID = (string)attachment.GenerateNewID(); + attachment.Kind = contentType.StartsWith("video/", StringComparison.Ordinal) ? "video" : "audio"; + attachment.ContentType = contentType; + Directory.CreateDirectory(IncomingRoot); + await using (var stored = File.Create(Incoming(attachment.ID))) + await file.CopyToAsync(stored, token); + if (later) + attachment.ProcessingState = "pending"; + else + { + var outcome = await ProcessIncoming(attachment, token); + if (!outcome.Ok) + return outcome; + } } else return MediaOutcome.Fail(StatusCodes.Status422UnprocessableEntity, "Validation failed: File type is not supported"); @@ -251,6 +278,8 @@ namespace PrivaPub.Domain.Media foreach (var full in new[] { Path.Combine(TrashRoot, relative), Path.Combine(Root, relative) }) if (File.Exists(full)) File.Delete(full); + if (File.Exists(Incoming(trashed.ID))) + File.Delete(Incoming(trashed.ID)); await DB.Default.DeleteAsync(m => m.ID == trashed.ID && m.TrashedAt != null); } @@ -362,37 +391,102 @@ namespace PrivaPub.Domain.Media return bands.Format == Enums.BandFormat.Uchar ? bands.Copy() : bands.Cast(Enums.BandFormat.Uchar); } - async Task ProcessAv(IFormFile file, string extension, string contentType, MediaAttachment attachment, CancellationToken token) + public async Task ProcessPending(MediaAttachment pending, CancellationToken token) { + var outcome = await ProcessIncoming(pending, token); + if (outcome.Ok) + { + await DB.Default.Update().Match(m => m.ID == pending.ID && m.TrashedAt == null) + .Modify(m => m.Kind, pending.Kind) + .Modify(m => m.ContentType, pending.ContentType) + .Modify(m => m.FilePath, pending.FilePath) + .Modify(m => m.PreviewPath, pending.PreviewPath) + .Modify(m => m.Size, pending.Size) + .Modify(m => m.Width, pending.Width) + .Modify(m => m.Height, pending.Height) + .Modify(m => m.Blurhash, pending.Blurhash) + .Modify(m => m.DurationSeconds, pending.DurationSeconds) + .Modify(m => m.ProcessingState, null) + .ExecuteAsync(token); + return true; + } + await DB.Default.Update().MatchID(pending.ID) + .Modify(m => m.ProcessingState, "failed").Modify(m => m.ProcessingError, outcome.Error).ExecuteAsync(token); + return false; + } + + // the stored upload, made playable: probed, then remuxed without its metadata when browsers play it as it is (H.264, + // VP8, VP9 or AV1 within MaxVideoPixels and MaxFrameRate), or else transcoded to H.264 that fits; a frame becomes the + // poster. ffmpeg only ever reads the local file (no protocol but file, its format forced from the probe). Nothing is + // saved until everything succeeded, and every temporary file goes, the stored upload too. + async Task ProcessIncoming(MediaAttachment attachment, CancellationToken token) + { + var options = _options.CurrentValue; + var input = Incoming(attachment.ID); var temp = Path.Combine(Path.GetTempPath(), $"privapub-{Guid.NewGuid():N}"); - var input = temp + ".in"; - var output = temp + "." + extension; + var frame = temp + ".png"; + var output = default(string); + await _processing.WaitAsync(token); try { - await using (var target = File.Create(input)) - await file.CopyToAsync(target, token); - var probe = await FFProbe.AnalyseAsync(input, cancellationToken: token); - var isVideo = contentType.StartsWith("video/") && probe.PrimaryVideoStream != default; - await FFMpegArguments.FromFileInput(input) - .OutputToFile(output, true, o => o.WithCustomArgument("-map 0 -map_metadata -1 -map_chapters -1 -c copy" + (extension is "mp4" or "m4a" ? " -movflags +faststart" : string.Empty))) + var probe = await FFProbe.AnalyseAsync(input, default, token, "-protocol_whitelist file"); + if (probe.Duration > TimeSpan.FromSeconds(options.MaxSeconds)) + return MediaOutcome.Fail(StatusCodes.Status422UnprocessableEntity, "Validation failed: The file is too long"); + var video = attachment.ContentType.StartsWith("video/", StringComparison.Ordinal) ? probe.PrimaryVideoStream : default; + var audio = probe.PrimaryAudioStream; + if (video == default && audio == default) + return MediaOutcome.Fail(StatusCodes.Status422UnprocessableEntity, "Validation failed: The file could not be processed"); + var extension = AvTypes[attachment.ContentType]; + var format = probe.Format.FormatName.Split(',')[0]; + string arguments; + if (video == default) + arguments = "-map 0:a:0 -vn -dn -sn -map_metadata -1 -map_chapters -1 -c copy" + (extension == "m4a" ? " -movflags +faststart" : string.Empty); + else if (video.CodecName is "h264" or "vp8" or "vp9" or "av1" && (long)video.Width * video.Height <= options.MaxVideoPixels + && video.FrameRate <= options.MaxFrameRate + 0.5) + arguments = "-map 0:v:0 -map 0:a:0? -dn -sn -map_metadata -1 -map_chapters -1 -c copy" + (extension == "mp4" ? " -movflags +faststart" : string.Empty); + else + { + var scale = Math.Min(1, Math.Sqrt(options.MaxVideoPixels / (double)((long)video.Width * video.Height))); + var width = Math.Max(2, (int)(video.Width * scale) / 2 * 2); + var height = Math.Max(2, (int)(video.Height * scale) / 2 * 2); + extension = "mp4"; + arguments = $"-map 0:v:0 -map 0:a:0? -dn -sn -map_metadata -1 -map_chapters -1 -c:v libx264 -preset veryfast -crf 23 -pix_fmt yuv420p " + + $"-vf scale={width}:{height} -fpsmax {options.MaxFrameRate.ToString(CultureInfo.InvariantCulture)} -c:a aac -b:a 128k -movflags +faststart"; + } + output = temp + "." + extension; + await FFMpegArguments.FromFileInput(input, true, o => o.ForceFormat(format).WithCustomArgument("-protocol_whitelist file")) + .OutputToFile(output, true, o => o.WithCustomArgument(arguments)) .CancellableThrough(token) .ProcessAsynchronously(); - attachment.Kind = isVideo ? "video" : "audio"; - attachment.ContentType = extension switch { "mp4" => "video/mp4", "webm" => isVideo ? "video/webm" : "audio/webm", "m4a" => "audio/mp4", "mp3" => "audio/mpeg", _ => contentType }; - attachment.FilePath = await Save(await File.ReadAllBytesAsync(output, token), extension, token); - attachment.Size = new FileInfo(output).Length; - if (isVideo) + var made = await FFProbe.AnalyseAsync(output, default, token, "-protocol_whitelist file"); + byte[] poster = default; + if (video != default) { - attachment.Width = probe.PrimaryVideoStream.Width; - attachment.Height = probe.PrimaryVideoStream.Height; - var frame = temp + ".png"; - await FFMpeg.SnapshotAsync(input, frame, captureTime: TimeSpan.FromSeconds(Math.Min(1, probe.Duration.TotalSeconds / 2))); - var still = ProcessImage(await File.ReadAllBytesAsync(frame, token), 640, _options.CurrentValue.PreviewSide); - attachment.PreviewPath = await Save(still.Preview, "jpg", token); + attachment.Width = made.PrimaryVideoStream?.Width; + attachment.Height = made.PrimaryVideoStream?.Height; + await FFMpeg.SnapshotAsync(output, frame, captureTime: TimeSpan.FromSeconds(Math.Min(1, made.Duration.TotalSeconds / 2))); + var still = ProcessImage(await File.ReadAllBytesAsync(frame, token), 640, options.PreviewSide); + poster = still.Preview; attachment.Blurhash = still.Blurhash; - File.Delete(frame); } + attachment.Kind = video != default ? "video" : "audio"; + attachment.ContentType = extension switch + { + "mp4" => "video/mp4", + "webm" => video != default ? "video/webm" : "audio/webm", + "m4a" => "audio/mp4", + "mp3" => "audio/mpeg", + "ogg" => "audio/ogg", + "wav" => "audio/wav", + "flac" => "audio/flac", + _ => attachment.ContentType + }; + attachment.DurationSeconds = Math.Round(made.Duration.TotalSeconds, 2); + attachment.Size = new FileInfo(output).Length; + attachment.FilePath = SaveFile(output, extension); + if (poster != default) + attachment.PreviewPath = await Save(poster, "jpg", token); return new MediaOutcome(attachment); } catch (Exception ex) when (ex is not OperationCanceledException) @@ -402,7 +496,8 @@ namespace PrivaPub.Domain.Media } finally { - foreach (var path in new[] { input, output }) + _processing.Release(); + foreach (var path in new[] { input, output, frame }.Where(p => p != default)) if (File.Exists(path)) File.Delete(path); } diff --git a/PrivaPub/Domain/Media/ProcessMediaJob.cs b/PrivaPub/Domain/Media/ProcessMediaJob.cs new file mode 100644 index 0000000..a1c266f --- /dev/null +++ b/PrivaPub/Domain/Media/ProcessMediaJob.cs @@ -0,0 +1,45 @@ +using MongoDB.Entities; + +using PrivaPub.Infrastructure.Jobs; +using PrivaPub.Models.Jobs; +using PrivaPub.Models.Media; + +namespace PrivaPub.Domain.Media +{ + // Audio and video sent to /api/v2/media, processed off the request (its lease renewed while ffmpeg runs): playable, or + // marked failed with why. One at a time, since a transcode takes the machine's cores. + public class ProcessMediaJob : IJobHandler + { + readonly IMediaService _media; + readonly ILogger _logger; + + public ProcessMediaJob(IMediaService media, ILogger logger) + { + _media = media; + _logger = logger; + } + + public JobKind Kind => JobKind.ProcessMedia; + public int Concurrency => 1; + public int MaxAttempts => 3; + public int PerHostLimit => 1; + + public static Job JobFor(MediaAttachment pending, string host) => new() + { + Kind = JobKind.ProcessMedia, + Payload = pending.ID, + Host = host, + DedupeKey = $"media|{pending.ID}" + }; + + public async Task Handle(Job job, CancellationToken token) + { + var pending = await DB.Default.Find().Match(m => m.ID == job.Payload && m.ProcessingState == "pending" && m.TrashedAt == null).ExecuteFirstAsync(token); + if (pending == default) + return JobOutcome.Done;//processed already, or trashed meanwhile + if (!await _media.ProcessPending(pending, token)) + _logger.LogInformation("Upload {Id} could not be processed", pending.ID); + return JobOutcome.Done; + } + } +} diff --git a/PrivaPub/Domain/Statuses/StatusService.cs b/PrivaPub/Domain/Statuses/StatusService.cs index d810b02..09a06d9 100644 --- a/PrivaPub/Domain/Statuses/StatusService.cs +++ b/PrivaPub/Domain/Statuses/StatusService.cs @@ -703,7 +703,8 @@ namespace PrivaPub.Domain.Statuses var wanted = ids.Distinct().ToList(); var found = await DB.Default.Find() .Match(m => wanted.Contains(m.ID) && m.OwnerAvatarId == author.Id && (m.PostId == null || m.PostId == postId) - && (m.ScheduledStatusId == null || m.ScheduledStatusId == scheduledId) && m.TrashedAt == null && m.ProfileOfAvatarId == null) + && (m.ScheduledStatusId == null || m.ScheduledStatusId == scheduledId) && m.TrashedAt == null && m.ProfileOfAvatarId == null + && m.ProcessingState == null) .ExecuteAsync(token); return found.Count == wanted.Count ? wanted.Select(id => found.First(m => m.ID == id)).ToList() : default; } @@ -732,6 +733,7 @@ namespace PrivaPub.Domain.Statuses AttachmentId = attachment.ID, ContentType = attachment.ContentType, Kind = attachment.Kind, + DurationSeconds = attachment.DurationSeconds, URL = _media.Url(attachment.FilePath), PreviewURL = _media.Url(attachment.PreviewPath ?? attachment.FilePath), Description = attachment.Description, diff --git a/PrivaPub/Middleware/SocialPubConfigurations.cs b/PrivaPub/Middleware/SocialPubConfigurations.cs index fed3a5a..38fb3c3 100644 --- a/PrivaPub/Middleware/SocialPubConfigurations.cs +++ b/PrivaPub/Middleware/SocialPubConfigurations.cs @@ -121,6 +121,7 @@ namespace PrivaPub.Middleware .AddSingleton() .AddSingleton() .AddSingleton() + .AddSingleton() .AddSingleton() .AddSingleton() .AddSingleton(services => services.GetRequiredService()) diff --git a/PrivaPub/Models/Jobs/Job.cs b/PrivaPub/Models/Jobs/Job.cs index c22cef5..dd38d69 100644 --- a/PrivaPub/Models/Jobs/Job.cs +++ b/PrivaPub/Models/Jobs/Job.cs @@ -35,7 +35,8 @@ namespace PrivaPub.Models.Jobs PublishScheduled, FetchReplies, BackfillOutbox, - SynchronizeFollowing + SynchronizeFollowing, + ProcessMedia } public enum JobState diff --git a/PrivaPub/Models/Media/MediaAttachment.cs b/PrivaPub/Models/Media/MediaAttachment.cs index d9318b0..b5e9298 100644 --- a/PrivaPub/Models/Media/MediaAttachment.cs +++ b/PrivaPub/Models/Media/MediaAttachment.cs @@ -16,6 +16,10 @@ namespace PrivaPub.Models.Media public int? Width { get; set; } public int? Height { get; set; } public float[] Focus { get; set; } + public double? DurationSeconds { get; set; } + // audio and video sent to /api/v2/media are processed by a job: "pending" until then, "failed" (with why) if it can't be + public string ProcessingState { get; set; } + public string ProcessingError { get; set; } public DateTime CreatedAt { get; set; } = DateTime.UtcNow; public DateTime? AttachedAt { get; set; } public string ScheduledStatusId { get; set; }//kept for a scheduled post until it is published or dropped diff --git a/PrivaPub/Program.cs b/PrivaPub/Program.cs index e63f31e..0ac26cb 100644 --- a/PrivaPub/Program.cs +++ b/PrivaPub/Program.cs @@ -144,6 +144,7 @@ try { FileProvider = new Microsoft.Extensions.FileProviders.PhysicalFileProvider(mediaRoot), RequestPath = "/media/files", + ContentTypeProvider = PrivaPub.Domain.Media.MediaService.ContentTypes, OnPrepareResponse = context => { context.Context.Response.Headers["X-Content-Type-Options"] = "nosniff"; diff --git a/deploy/systemd/privapub.service b/deploy/systemd/privapub.service index ba999a9..3dbdbcf 100644 --- a/deploy/systemd/privapub.service +++ b/deploy/systemd/privapub.service @@ -14,6 +14,8 @@ Environment=ASPNETCORE_ENVIRONMENT=Production Environment=DOTNET_CLI_TELEMETRY_OPTOUT=1 Restart=always RestartSec=5 +# its own /tmp: ffmpeg's and the uploads' temporary files are nobody else's +PrivateTmp=true SyslogIdentifier=privapub [Install]