From f9658a8e7a0455fc47b544175f1d94c09c7b70df Mon Sep 17 00:00:00 2001 From: thepra Date: Thu, 1 Oct 2026 18:59:14 +0200 Subject: [PATCH] P6: remote video and audio play through the proxy - The media proxy streams ranged requests from the origin (passing the range on, never caching), downloads and caches whole files otherwise, and serves cached files with range support. A PeerTube video is never fetched whole for one viewer, and clients still never contact the remote host. - PeerTube's fragmented MP4 files inside an HLS entry are read as variants, so HLS-only instances play too. - A remote Video or Audio post becomes one playable Mastodon attachment: the best MP4 up to 720p that carries both sound and picture, with its poster and duration; the card is kept only when nothing is playable. - nginx: /media/proxy/ with proxy_buffering off and a 600 s read timeout (applied on the box, with a backup). Checked live: a GoToSocial image through the proxy answers 206 with exactly the asked range when streamed, 200 when cached, and 206 with the right Content-Range from the cache. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012CzABvBkbcFqoHdmi8b9WB --- CLAUDE.md | 10 +++++ FEDERATION.md | 3 +- .../Federation/ObjectShapesTests.cs | 24 ++++++++++ .../Mastodon/Controllers/MediaController.cs | 31 +++++++++++-- .../Api/Mastodon/Mappers/MastodonMapper.cs | 35 ++++++++++++++- PrivaPub/Domain/Media/MediaProxy.cs | 44 ++++++++++++++----- PrivaPub/Federation/Objects/ObjectShapes.cs | 5 ++- .../Infrastructure/Http/FederationHttp.cs | 41 +++++++++++++++++ deploy/nginx/privapub.thepra.dev.conf | 12 +++++ docs/ROADMAP.md | 10 +++-- 10 files changed, 193 insertions(+), 22 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 6d4ffc8..e47a26b 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -212,6 +212,16 @@ cd /var/www/privapub.thepra.dev && sudo -u www-data ASPNETCORE_ENVIRONMENT=Produ sibling `media-proxy`, which `/media/files` does not serve. 3. **A client never contacts a remote server for media:** every remote URL the API returns goes through `IMediaProxy.Wrap`, an HMAC-signed `/media/proxy/` URL fetched by `IFederationHttp.GetMedia`. +4. **The proxy serves three ways:** + - **Cached:** a file already cached is served from disk, ranges included. + - **Downloaded:** a request without a `Range` is downloaded whole, up to `Media:MaxProxiedBytes`, then cached. + - **Streamed:** a ranged request, or anything too big to cache, is streamed from the origin with the range passed on, + and never cached. That is how remote video plays. + + nginx has a `/media/proxy/` location with `proxy_buffering off` and a 600 s read timeout for those streams. +5. **Remote video and audio become one playable attachment** in the Mastodon API (`MastodonMapper.Playable`): the best MP4 + up to 720p that carries both sound and picture, including PeerTube's fragmented files inside an HLS entry. HLS + playlists themselves are not rewritten. ## Privacy invariants diff --git a/FEDERATION.md b/FEDERATION.md index 9ceaabe..9d7f2eb 100644 --- a/FEDERATION.md +++ b/FEDERATION.md @@ -181,6 +181,7 @@ Posts with a location (shown to nearby users of this server) never leave the ser ## Known limitations - Our own posts carry no custom emoji: emoji are received and shown, not offered. -- Remote media is fetched through this server's proxy when a local client displays it. +- Remote media is fetched through this server's proxy when a local client displays it. Video and audio are streamed + through it with byte ranges passed on, so a PeerTube file is never fetched whole for one viewer. - Collections expose counts, not members. - Only `rsa-sha256`-style keys are verified. RFC 9421 signatures are planned. diff --git a/PrivaPub.Tests/Federation/ObjectShapesTests.cs b/PrivaPub.Tests/Federation/ObjectShapesTests.cs index c66b911..c81b227 100644 --- a/PrivaPub.Tests/Federation/ObjectShapesTests.cs +++ b/PrivaPub.Tests/Federation/ObjectShapesTests.cs @@ -214,6 +214,30 @@ namespace PrivaPub.Tests.Federation Assert.Equal(new DateTime(2026, 9, 30, 12, 0, 0, DateTimeKind.Utc), closed.Poll.ClosedAt); } + [Fact] + public void Finds_playable_files_inside_an_hls_only_peertube_video_and_picks_one_with_sound() + { + var note = Parse(""" + { + "id": "https://tube.example/videos/watch/2", "type": "Video", "name": "HLS only", "attributedTo": "https://tube.example/accounts/ann", + "url": [ + { "type": "Link", "mediaType": "text/html", "href": "https://tube.example/w/def" }, + { "type": "Link", "mediaType": "application/x-mpegURL", "href": "https://tube.example/master.m3u8", "tag": [ + { "type": "Link", "mediaType": "video/mp4", "href": "https://tube.example/1080-fragmented.mp4", "height": 1080, + "attachment": [{ "type": "PropertyValue", "name": "ffprobe_codec_type", "value": "video" }, { "type": "PropertyValue", "name": "ffprobe_codec_type", "value": "audio" }] }, + { "type": "Link", "mediaType": "video/mp4", "href": "https://tube.example/720-fragmented.mp4", "height": 720, + "attachment": [{ "type": "PropertyValue", "name": "ffprobe_codec_type", "value": "video" }] }, + { "type": "Link", "mediaType": "video/mp4", "href": "https://tube.example/480-fragmented.mp4", "height": 480, + "attachment": [{ "type": "PropertyValue", "name": "ffprobe_codec_type", "value": "video" }, { "type": "PropertyValue", "name": "ffprobe_codec_type", "value": "audio" }] } + ] } + ] + } + """); + + Assert.Equal(3, note.Video.Variants.Count); + Assert.Equal("https://tube.example/480-fragmented.mp4", MastodonMapper.BestVideo(note.Video.Variants).Url); + } + [Fact] public void Normalises_hashtags_like_mastodon() => Assert.Equal("ｆｅｄｉ".Normalize(System.Text.NormalizationForm.FormKC).ToLowerInvariant(), Assert.Single(Parse(""" diff --git a/PrivaPub/Api/Mastodon/Controllers/MediaController.cs b/PrivaPub/Api/Mastodon/Controllers/MediaController.cs index 5244771..b445b3b 100644 --- a/PrivaPub/Api/Mastodon/Controllers/MediaController.cs +++ b/PrivaPub/Api/Mastodon/Controllers/MediaController.cs @@ -60,13 +60,38 @@ namespace PrivaPub.Api.Mastodon.Controllers [HttpGet("/media/proxy/{signature}/{encoded}"), AllowAnonymous, ApiExplorerSettings(IgnoreApi = true)] public async Task Proxy(string signature, string encoded, CancellationToken token) { - var (path, contentType) = await _proxy.Fetch(signature, encoded, token); - if (path == default) + var url = _proxy.Verified(signature, encoded); + if (url == default) return NotFound(); Response.Headers["X-Content-Type-Options"] = "nosniff"; Response.Headers["Content-Security-Policy"] = "default-src 'none'; sandbox"; Response.Headers["Cache-Control"] = "public, max-age=604800"; - return PhysicalFile(path, contentType); + if (_proxy.Cached(url) is { Path: not null } cached) + return PhysicalFile(cached.Path, cached.ContentType, enableRangeProcessing: true); + if (Request.Headers.Range.Count == 0) + { + var (path, contentType) = await _proxy.Fetch(signature, encoded, token); + if (path != default) + return PhysicalFile(path, contentType, enableRangeProcessing: true); + } + return await Stream(url, token); + } + + async Task Stream(string url, CancellationToken token) + { + var range = System.Net.Http.Headers.RangeHeaderValue.TryParse(Request.Headers.Range.ToString(), out var asked) ? asked : default; + using var upstream = await _proxy.Open(url, range, token); + if (upstream == default) + return NotFound(); + Response.StatusCode = (int)upstream.StatusCode; + Response.ContentType = upstream.Content.Headers.ContentType?.ToString() ?? "application/octet-stream"; + if (upstream.Content.Headers.ContentLength is { } length) + Response.ContentLength = length; + if (upstream.Content.Headers.ContentRange is { } contentRange) + Response.Headers.ContentRange = contentRange.ToString(); + Response.Headers.AcceptRanges = "bytes"; + await upstream.Content.CopyToAsync(Response.Body, token); + return new EmptyResult(); } object View(MediaAttachment attachment) => new diff --git a/PrivaPub/Api/Mastodon/Mappers/MastodonMapper.cs b/PrivaPub/Api/Mastodon/Mappers/MastodonMapper.cs index 435d2c3..06a5357 100644 --- a/PrivaPub/Api/Mastodon/Mappers/MastodonMapper.cs +++ b/PrivaPub/Api/Mastodon/Mappers/MastodonMapper.cs @@ -225,7 +225,7 @@ namespace PrivaPub.Api.Mastodon.Mappers SpoilerText = post.SpoilerText ?? (post.HasContentWarning ? post.Title ?? ActivityPubRenderer.ContentWarning : string.Empty), Visibility = Visibility(post.Visibility), Language = post.Language, - MediaAttachments = post.Media.Select(Media).ToList(), + MediaAttachments = post.Media.Count > 0 ? post.Media.Select(Media).ToList() : Playable(post), Card = Card(post), Poll = Poll(post, viewerId, ownVotes.GetValueOrDefault(post.ID)), Emojis = Emojis(post.Emojis), @@ -445,11 +445,42 @@ namespace PrivaPub.Api.Mastodon.Mappers Streams = variant.Streams }; + List Playable(PostEntity post) + { + var (variant, kind, duration) = post.Video is { } video ? (BestVideo(video.Variants), "video", video.DurationSeconds) + : post.Audio is { } audio ? (audio.Variants.FirstOrDefault(), "audio", audio.DurationSeconds) + : default; + if (variant == default) + return new List(); + return new List + { + Media(new PostMedia + { + Id = new Guid(System.Security.Cryptography.MD5.HashData(System.Text.Encoding.UTF8.GetBytes(variant.Url))), + RemoteURL = variant.Url, + ContentType = variant.MediaType ?? kind + "/*", + RemotePreviewURL = post.CoverURL, + Width = variant.Width, + Height = variant.Height, + DurationSeconds = duration, + Description = post.Title + }) + }; + } + + public static MediaVariant BestVideo(IReadOnlyCollection variants) + { + var playable = variants.Where(v => v.MediaType?.StartsWith("video/") == true && (v.Streams.Count == 0 || v.Streams.Contains("audio") && v.Streams.Contains("video"))).ToList(); + return playable.Where(v => v.Height <= 720).OrderByDescending(v => v.Height).FirstOrDefault() + ?? playable.OrderBy(v => v.Height ?? int.MaxValue).FirstOrDefault(); + } + PreviewCard Card(PostEntity post) { if (post.Link is { } link) return Card(link.Href, link.Title ?? post.Title, link.Description ?? (post.Media.Count == 0 ? post.Excerpt : default), link.ImageURL); - if (post.IsFederatedCopy && post.ObjectType is "Article" or "Event" or "Video" or "Audio" && post.Url != default && post.Media.Count == 0) + if (post.IsFederatedCopy && post.ObjectType is "Article" or "Event" && post.Url != default && post.Media.Count == 0 + || post.IsFederatedCopy && post.ObjectType is "Video" or "Audio" && post.Url != default && post.Media.Count == 0 && Playable(post).Count == 0) return Card(post.Url, post.Title, post.Excerpt, post.CoverURL); return default; } diff --git a/PrivaPub/Domain/Media/MediaProxy.cs b/PrivaPub/Domain/Media/MediaProxy.cs index c5c3a22..d3c8d96 100644 --- a/PrivaPub/Domain/Media/MediaProxy.cs +++ b/PrivaPub/Domain/Media/MediaProxy.cs @@ -15,6 +15,9 @@ namespace PrivaPub.Domain.Media { string Wrap(string remoteUrl); Task<(string Path, string ContentType)> Fetch(string signature, string encodedUrl, CancellationToken token); + string Verified(string signature, string encodedUrl); + (string Path, string ContentType) Cached(string url); + Task Open(string url, System.Net.Http.Headers.RangeHeaderValue range, CancellationToken token); } public class MediaProxy : IMediaProxy @@ -43,7 +46,7 @@ namespace PrivaPub.Domain.Media return $"{_localActors.BaseAddress}/media/proxy/{Sign(remoteUrl)}/{encoded}"; } - public async Task<(string Path, string ContentType)> Fetch(string signature, string encodedUrl, CancellationToken token) + public string Verified(string signature, string encodedUrl) { string url; try @@ -54,18 +57,37 @@ namespace PrivaPub.Domain.Media { return default; } - if (!CryptographicOperations.FixedTimeEquals(Encoding.ASCII.GetBytes(signature ?? string.Empty), Encoding.ASCII.GetBytes(Sign(url)))) - return default; + return CryptographicOperations.FixedTimeEquals(Encoding.ASCII.GetBytes(signature ?? string.Empty), Encoding.ASCII.GetBytes(Sign(url))) ? url : default; + } + public (string Path, string ContentType) Cached(string url) + { + var (path, typePath) = CachePaths(url); + if (!File.Exists(path) || !File.Exists(typePath)) + return default; + File.SetLastWriteTimeUtc(path, DateTime.UtcNow); + return (path, File.ReadAllText(typePath)); + } + + public Task Open(string url, System.Net.Http.Headers.RangeHeaderValue range, CancellationToken token) => + _http.OpenMedia(url, range, token); + + (string Path, string TypePath) CachePaths(string url) + { var name = Convert.ToHexStringLower(SHA256.HashData(Encoding.UTF8.GetBytes(url))); - var directory = System.IO.Path.Combine(_media.ProxyRoot, name[..2]); - var path = System.IO.Path.Combine(directory, name); - var typePath = path + ".type"; - if (File.Exists(path) && File.Exists(typePath)) - { - File.SetLastWriteTimeUtc(path, DateTime.UtcNow); - return (path, await File.ReadAllTextAsync(typePath, token)); - } + var path = System.IO.Path.Combine(_media.ProxyRoot, name[..2], name); + return (path, path + ".type"); + } + + public async Task<(string Path, string ContentType)> Fetch(string signature, string encodedUrl, CancellationToken token) + { + var url = Verified(signature, encodedUrl); + if (url == default) + return default; + if (Cached(url) is { Path: not null } cached) + return cached; + var (path, typePath) = CachePaths(url); + var directory = System.IO.Path.GetDirectoryName(path); var (bytes, contentType) = await _http.GetMedia(url, _options.CurrentValue.MaxProxiedBytes, token); if (bytes == default) diff --git a/PrivaPub/Federation/Objects/ObjectShapes.cs b/PrivaPub/Federation/Objects/ObjectShapes.cs index feec14c..506a613 100644 --- a/PrivaPub/Federation/Objects/ObjectShapes.cs +++ b/PrivaPub/Federation/Objects/ObjectShapes.cs @@ -217,10 +217,13 @@ namespace PrivaPub.Federation.Objects if (Value(note, "type") != "Video") return default; var links = Objects(note["url"]).ToList(); + var nested = links.Where(l => Value(l, "mediaType")?.Equals("application/x-mpegURL", StringComparison.OrdinalIgnoreCase) == true) + .SelectMany(l => Objects(l["tag"])) + .ToList(); return new VideoDetails { DurationSeconds = Seconds(note["duration"]), - Variants = Variants(links, "video/", "audio/"), + Variants = Variants(links.Concat(nested).ToList(), "video/", "audio/"), PlaylistURL = links.Where(l => Value(l, "mediaType")?.Equals("application/x-mpegURL", StringComparison.OrdinalIgnoreCase) == true) .Select(l => Value(l, "href")).FirstOrDefault(IsWeb), IsLive = Bool(note, "isLiveBroadcast") == true, diff --git a/PrivaPub/Infrastructure/Http/FederationHttp.cs b/PrivaPub/Infrastructure/Http/FederationHttp.cs index 1c42039..3d0baae 100644 --- a/PrivaPub/Infrastructure/Http/FederationHttp.cs +++ b/PrivaPub/Infrastructure/Http/FederationHttp.cs @@ -23,6 +23,7 @@ namespace PrivaPub.Infrastructure.Http Task GetJson(string url, string accept, Action sign, CancellationToken token); bool FailedTemporarily(string url); Task<(Uri FinalUri, string Html)> GetPage(string url, CancellationToken token); + Task OpenMedia(string url, System.Net.Http.Headers.RangeHeaderValue range, CancellationToken token); Task Send(HttpRequestMessage request, CancellationToken token); Task<(byte[] Bytes, string ContentType)> GetMedia(string url, long maxBytes, CancellationToken token); } @@ -229,6 +230,46 @@ namespace PrivaPub.Infrastructure.Http } } + public async Task OpenMedia(string url, System.Net.Http.Headers.RangeHeaderValue range, CancellationToken token) + { + if (!Uri.TryCreate(url, UriKind.Absolute, out var target) || !IsAllowed(target)) + return default; + try + { + for (var hop = 0; hop <= MaxRedirects; hop++) + { + using var request = new HttpRequestMessage(HttpMethod.Get, target); + request.Headers.Accept.ParseAdd("video/*, audio/*, image/*"); + request.Headers.Range = range; + var response = await _httpClientFactory.CreateClient(ClientName).SendAsync(request, HttpCompletionOption.ResponseHeadersRead, token); + if (IsRedirect(response.StatusCode)) + { + var location = response.Headers.Location; + response.Dispose(); + var next = location == default ? default : location.IsAbsoluteUri ? location : new Uri(target, location); + if (!IsAllowed(next)) + return default; + target = next; + continue; + } + var mediaType = response.Content.Headers.ContentType?.MediaType?.ToLowerInvariant(); + if (!response.IsSuccessStatusCode || mediaType == default || mediaType.Contains("svg") + || !(mediaType.StartsWith("video/") || mediaType.StartsWith("audio/") || mediaType.StartsWith("image/") || mediaType == "application/octet-stream")) + { + response.Dispose(); + return default; + } + return response; + } + return default; + } + catch (Exception ex) when (ex is HttpRequestException or BlockedDestinationException or OperationCanceledException && !token.IsCancellationRequested) + { + _logger.LogInformation("Media stream {Url} refused: {Reason}", url, ex.Message); + return default; + } + } + const int MaxPageBytes = 512 * 1024; static async Task ReadPrefix(HttpContent content, int limit, CancellationToken token) diff --git a/deploy/nginx/privapub.thepra.dev.conf b/deploy/nginx/privapub.thepra.dev.conf index 064c4d3..1971f43 100644 --- a/deploy/nginx/privapub.thepra.dev.conf +++ b/deploy/nginx/privapub.thepra.dev.conf @@ -48,6 +48,18 @@ server { proxy_read_timeout 300s; } + location ^~ /media/proxy/ { + proxy_buffering off; + proxy_pass http://127.0.0.1:6970; + proxy_http_version 1.1; + proxy_set_header Connection ""; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_read_timeout 600s; + } + location / { proxy_pass http://127.0.0.1:6970; proxy_http_version 1.1; diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index e91384d..3049812 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -465,11 +465,13 @@ it, raw where it doesn't. - otherwise the server reads the page (owner decision 1): public posts only, 0–60 s after arrival, cached per address for 7 days across the whole server, `Federation:FetchLinkPreviews` to switch it off. Checked live against a GoToSocial profile page. -- **Media:** - - video playback through the proxy (Range requests, HLS playlist rewriting, the poster), with a `video` card; +- **Media** (done in v1.14.0 except where noted): + - video playback through the proxy: range requests streamed from the origin, the poster, a playable attachment picked + from MP4 or fragmented MP4 files (HLS playlists are not rewritten; not needed while PeerTube publishes the files); - audio attachments; - - an article reader view; - - JPEG/PNG renditions kept for Pixelfed. + - JPEG/PNG for Pixelfed: uploads were already re-encoded to JPEG, PNG or GIF; + - an article reader view: the data is in `Status.privapub` (title, excerpt, cover, full HTML in `content`); the view + itself belongs to the client. #### P7 Threads, communities and the social graph - **Thread backfill:**