From fc111d8c4604b8e8410d8b37f41439a5cb332f2c Mon Sep 17 00:00:00 2001 From: thepra Date: Thu, 1 Oct 2026 17:53:59 +0200 Subject: [PATCH] P5: remote objects are read in every shape, and videos, events, audio and links keep their own details MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - ObjectShapes reads `url`, `icon` and `image` as a value, a Link or an array; Markdown `content` (PeerTube) is rendered; a missing `mediaType` is inferred from the extension or the attachment type (Bridgy); alt text comes from `name`, `summary` or their language maps; thumbnails come from attachment `icon`, object `icon[]`, `image` or a Loops `preview`; durations are ISO 8601 or seconds; per-attachment `sensitive` is kept; the language falls back to `@context` `@language` (Pleroma); titles and excerpts are plain text; hashtags normalise with NFKC like Mastodon. - Typed details on Post: Link (Lemmy link posts, Mbin `source` URLs, Mastodon 4.7 Link attachments with an FEP-8967 publisher preview), Video (PeerTube files with their streams, HLS playlist, poster, captions, chapters, licence, channel, live state, comment policy), Audio (Funkwhale), Event (Mobilizon, Gancio, Hubzilla: times, zone, floating time, places and addresses, online link, capacity, status), the cover image, and the author's own source text. - Mastodon API: a card built from those details without fetching anything, an event's "when · where" line, attachment thumbnails and durations, and no media file offered as a preview image. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012CzABvBkbcFqoHdmi8b9WB --- CLAUDE.md | 4 + .../Federation/ObjectShapesTests.cs | 179 +++++++++++ PrivaPub/Api/Mastodon/Entities/Entities.cs | 24 +- .../Api/Mastodon/Mappers/MastodonMapper.cs | 95 +++++- .../Inbox/Handlers/UpdateHandler.cs | 6 + PrivaPub/Federation/Inbox/RemotePosts.cs | 6 + PrivaPub/Federation/Objects/NoteParser.cs | 73 +++-- PrivaPub/Federation/Objects/ObjectShapes.cs | 293 ++++++++++++++++++ PrivaPub/Models/Post/Post.cs | 12 + PrivaPub/Models/Post/PostDetails.cs | 86 +++++ PrivaPub/Models/Post/PostMedia.cs | 3 + docs/ROADMAP.md | 8 +- 12 files changed, 751 insertions(+), 38 deletions(-) create mode 100644 PrivaPub.Tests/Federation/ObjectShapesTests.cs create mode 100644 PrivaPub/Federation/Objects/ObjectShapes.cs create mode 100644 PrivaPub/Models/Post/PostDetails.cs diff --git a/CLAUDE.md b/CLAUDE.md index 8f7a4f3..8ed8268 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -244,6 +244,10 @@ cd /var/www/privapub.thepra.dev && sudo -u www-data ASPNETCORE_ENVIRONMENT=Produ revision. Delivery details reach them through `Arrival.Current`, which `InboxProcessor` sets for the handler's duration. The raw form lives outside `Post` so timelines never load it. Read through `/api/privapub/v1/statuses/:id/provenance` and `/api/privapub/v1/instances/:host`. +- **Remote objects are parsed for every shape in `ObjectShapes`:** `url`/`icon`/`image` as a value, an object or an + array; Markdown `content`; missing `mediaType`s inferred; thumbnails from any of their five places; and the typed + `Link`, `Video`, `Audio` and `Event` details. A Mastodon API card is built from those, never by fetching the linked page + (that fetch is the owner's decision 1, for P6). - **A server is described on arrival, never on read.** The first record from a host enqueues `DescribeInstance` (its NodeInfo, at most once a week, into `RemoteInstance`), so opening the details view tells nobody anything. - **Post ids are the timeline order, so they follow arrival, not `published`.** `PrivacyIds.Arrived` gives a remote post diff --git a/PrivaPub.Tests/Federation/ObjectShapesTests.cs b/PrivaPub.Tests/Federation/ObjectShapesTests.cs new file mode 100644 index 0000000..a124365 --- /dev/null +++ b/PrivaPub.Tests/Federation/ObjectShapesTests.cs @@ -0,0 +1,179 @@ +using PrivaPub.Api.Mastodon.Mappers; +using PrivaPub.Federation.Objects; +using PrivaPub.Models.Post; + +using System.Text.Json.Nodes; + +namespace PrivaPub.Tests.Federation +{ + public class ObjectShapesTests + { + static NoteDocument Parse(string json) => NoteParser.Parse(JsonNode.Parse(json)); + + [Fact] + public void Keeps_a_peertube_video_with_its_files_poster_captions_and_markdown_description() + { + var note = Parse(""" + { + "id": "https://tube.example/videos/watch/1", "type": "Video", "name": "A *talk*", "duration": "PT6299S", + "mediaType": "text/markdown", "content": "**Bold** words", "sensitive": false, "views": 14771, "state": 1, + "isLiveBroadcast": false, "commentsPolicy": 2, "downloadEnabled": true, + "licence": { "identifier": "5", "name": "Attribution - Share Alike" }, "category": { "identifier": "2", "name": "Films" }, + "attributedTo": [{ "type": "Person", "id": "https://tube.example/accounts/ann" }, { "type": "Group", "id": "https://tube.example/video-channels/films" }], + "icon": [ + { "type": "Image", "url": "https://tube.example/thumb-280.jpg", "mediaType": "image/jpeg", "width": 280, "height": 157 }, + { "type": "Image", "url": "https://tube.example/thumb-1920.jpg", "mediaType": "image/jpeg", "width": 1920, "height": 1080 }, + { "type": "Image", "url": "https://tube.example/thumb-850.jpg", "mediaType": "image/jpeg", "width": 850, "height": 480 } + ], + "subtitleLanguage": [{ "identifier": "en", "url": [{ "mediaType": "text/vtt", "href": "https://tube.example/en.vtt" }] }], + "url": [ + { "type": "Link", "mediaType": "text/html", "href": "https://tube.example/w/abc" }, + { "type": "Link", "mediaType": "video/mp4", "href": "https://tube.example/720.mp4", "height": 720, "width": 1280, "fps": 24, "size": 942586378, + "attachment": [{ "type": "PropertyValue", "name": "ffprobe_codec_type", "value": "audio" }, { "type": "PropertyValue", "name": "ffprobe_codec_type", "value": "video" }] }, + { "type": "Link", "mediaType": "application/x-mpegURL", "href": "https://tube.example/master.m3u8" }, + { "type": "Link", "mediaType": "application/x-bittorrent;x-scheme-handler/magnet", "href": "magnet:?xt=urn:btih:abc" } + ] + } + """); + + Assert.Equal("https://tube.example/w/abc", note.Url); + Assert.Equal("A *talk*", note.Title); + Assert.Contains("Bold", note.ContentHtml); + Assert.Equal("https://tube.example/thumb-850.jpg", note.CoverURL); + Assert.Equal(6299, note.Video.DurationSeconds); + var file = Assert.Single(note.Video.Variants); + Assert.Equal(new[] { "audio", "video" }, file.Streams); + Assert.Equal(942586378, file.Size); + Assert.Equal("https://tube.example/master.m3u8", note.Video.PlaylistURL); + Assert.Equal(2, note.Video.CommentsPolicy); + Assert.Equal("Attribution - Share Alike", note.Video.Licence); + Assert.Equal("https://tube.example/video-channels/films", note.Video.ChannelURI); + Assert.Equal("https://tube.example/en.vtt", Assert.Single(note.Video.Captions).Url); + Assert.Equal("https://tube.example/accounts/ann", note.AttributedTo); + } + + [Fact] + public void Keeps_a_mobilizon_event_with_its_time_place_and_online_link() + { + var note = Parse(""" + { + "id": "https://events.example/events/9", "type": "Event", "name": "Repair café", + "summary": "

Saturday at the library

", "content": "

Bring broken things.

", + "startTime": "2026-10-03T10:00:00+02:00", "endTime": "2026-10-03T13:00:00+02:00", "timezone": "Europe/Rome", + "joinMode": "free", "participantCount": 12, "maximumAttendeeCapacity": 40, "ical:status": "CONFIRMED", + "location": { "type": "Place", "name": "Biblioteca", "latitude": 45.07, "longitude": 7.68, + "address": { "type": "PostalAddress", "streetAddress": "Via Roma 1", "addressLocality": "Torino", "addressCountry": "IT" } }, + "attachment": [{ "type": "Link", "name": "Website", "href": "https://meet.example/repair" }], + "attributedTo": "https://events.example/@club" + } + """); + + Assert.Null(note.SpoilerText); + Assert.False(note.Sensitive); + Assert.Equal("Saturday at the library", note.Excerpt); + Assert.Equal(new DateTime(2026, 10, 3, 8, 0, 0, DateTimeKind.Utc), note.Event.Start); + Assert.Equal("Via Roma 1, Torino, IT", Assert.Single(note.Event.Places).Address); + Assert.Equal("https://meet.example/repair", note.Event.OnlineURL); + Assert.Equal(40, note.Event.Capacity); + Assert.Equal("CONFIRMED", note.Event.Status); + Assert.Equal("2026-10-03 10:00 – 13:00 (Europe/Rome) · Biblioteca", MastodonMapper.EventLine(note.Event)); + } + + [Fact] + public void Keeps_a_lemmy_link_post_as_a_link_with_its_thumbnail() + { + var note = Parse(""" + { + "id": "https://lemmy.example/post/5", "type": "Page", "name": "A good read", "attributedTo": "https://lemmy.example/u/bo", + "attachment": [{ "type": "Link", "href": "https://news.example/story", "mediaType": "text/html" }], + "image": { "type": "Image", "url": "https://lemmy.example/pictrs/thumb.jpg" }, + "language": { "identifier": "en", "name": "English" } + } + """); + + Assert.Equal("https://news.example/story", note.Link.Href); + Assert.Equal("https://lemmy.example/pictrs/thumb.jpg", note.Link.ImageURL); + Assert.False(note.Link.FromPublisher); + Assert.Empty(note.Attachments); + } + + [Fact] + public void Takes_a_publisher_supplied_preview_from_a_link_attachment() + { + var note = Parse(""" + { + "id": "https://m.example/statuses/1", "type": "Note", "attributedTo": "https://m.example/users/a", "content": "

look

", + "attachment": [{ "type": "Link", "href": "https://news.example/a", + "preview": { "type": "Article", "name": "Headline & more", "summary": "

What happened

", "image": { "type": "Image", "url": "https://news.example/a.jpg" } } }] + } + """); + + Assert.Equal("Headline & more", note.Link.Title); + Assert.Equal("What happened", note.Link.Description); + Assert.Equal("https://news.example/a.jpg", note.Link.ImageURL); + Assert.True(note.Link.FromPublisher); + } + + [Fact] + public void Reads_media_in_every_shape_and_infers_what_is_missing() + { + var note = Parse(""" + { + "id": "https://bsky.brid.gy/convert/ap/at://did:plc:x/post/1", "type": "Note", "attributedTo": "https://bsky.brid.gy/ap/did:plc:x", + "content": "

pics

", + "@context": ["https://www.w3.org/ns/activitystreams", { "@language": "de" }], + "attachment": [ + { "type": "Image", "url": "https://cdn.example/a.jpeg", "summary": "alt from summary" }, + { "type": "Document", "url": [{ "type": "Link", "href": "https://cdn.example/clip", "mediaType": "video/mp4" }], + "icon": { "type": "Image", "url": "https://cdn.example/clip.jpg" }, "duration": "PT12.4S", "sensitive": true }, + { "type": "Image", "url": "https://cdn.example/nameless" } + ] + } + """); + + Assert.Equal("de", note.Language); + Assert.Equal("image/jpeg", note.Attachments[0].ContentType); + Assert.Equal("alt from summary", note.Attachments[0].Description); + Assert.Equal("https://cdn.example/clip", note.Attachments[1].RemoteURL); + Assert.Equal("video/mp4", note.Attachments[1].ContentType); + Assert.Equal("https://cdn.example/clip.jpg", note.Attachments[1].RemotePreviewURL); + Assert.Equal(12.4, note.Attachments[1].DurationSeconds); + Assert.True(note.Attachments[1].Sensitive); + Assert.Equal("image/*", note.Attachments[2].ContentType); + } + + [Fact] + public void Keeps_a_funkwhale_track_and_a_loops_poster() + { + var track = Parse(""" + { + "id": "https://music.example/federation/music/uploads/1", "type": "Audio", "name": "Song", "attributedTo": "https://music.example/@band", + "summary": "#rock #live", "duration": 241, "position": 3, "album": "https://music.example/albums/2", "license": "https://creativecommons.org/licenses/by/4.0/", + "url": [{ "type": "Link", "mediaType": "text/html", "href": "https://music.example/library/tracks/1" }, + { "type": "Link", "mediaType": "audio/mpeg", "href": "https://music.example/listen/1.mp3", "bitrate": 320000, "size": 9640000 }] + } + """); + var loop = Parse(""" + { + "id": "https://loops.example/v/1", "type": "Note", "attributedTo": "https://loops.example/@dancer", "content": "", + "preview": { "type": "Image", "url": { "href": "https://loops.example/poster.jpg", "mediaType": "image/jpeg" } }, + "attachment": [{ "type": "Document", "mediaType": "video/mp4", "url": "https://loops.example/v/1.mp4", "width": 720, "height": 1280 }] + } + """); + + Assert.Equal("#rock #live", track.Excerpt); + Assert.Null(track.SpoilerText); + Assert.Equal(241, track.Audio.DurationSeconds); + Assert.Equal(3, track.Audio.Position); + Assert.Equal("https://music.example/listen/1.mp3", Assert.Single(track.Audio.Variants).Url); + Assert.Equal("https://loops.example/poster.jpg", loop.CoverURL); + } + + [Fact] + public void Normalises_hashtags_like_mastodon() => + Assert.Equal("ｆｅｄｉ".Normalize(System.Text.NormalizationForm.FormKC).ToLowerInvariant(), Assert.Single(Parse(""" + { "id": "https://x.example/1", "type": "Note", "attributedTo": "https://x.example/u", "content": "", + "tag": [{ "type": "Hashtag", "name": "#ＦＥＤＩ" }] } + """).Tags)); + } +} diff --git a/PrivaPub/Api/Mastodon/Entities/Entities.cs b/PrivaPub/Api/Mastodon/Entities/Entities.cs index 4449160..cd055a3 100644 --- a/PrivaPub/Api/Mastodon/Entities/Entities.cs +++ b/PrivaPub/Api/Mastodon/Entities/Entities.cs @@ -49,6 +49,28 @@ namespace PrivaPub.Api.Mastodon.Entities public string AttributionDomains { get; set; } } + public class PreviewCard + { + public string Url { get; set; } + public string Title { get; set; } = string.Empty; + public string Description { get; set; } = string.Empty; + public string Type { get; set; } = "link"; + public string AuthorName { get; set; } = string.Empty; + public string AuthorUrl { get; set; } = string.Empty; + public string ProviderName { get; set; } = string.Empty; + public string ProviderUrl { get; set; } = string.Empty; + public string Html { get; set; } = string.Empty; + public int Width { get; set; } + public int Height { get; set; } + public string Image { get; set; } + public string ImageDescription { get; set; } = string.Empty; + public string EmbedUrl { get; set; } = string.Empty; + public string Blurhash { get; set; } + public string Language { get; set; } + public string PublishedAt { get; set; } + public List Authors { get; set; } = new(); + } + public class Status { public string Id { get; set; } @@ -76,7 +98,7 @@ namespace PrivaPub.Api.Mastodon.Entities public List MediaAttachments { get; set; } = new(); public List Mentions { get; set; } = new(); public List Tags { get; set; } = new(); - public object Card { get; set; } + public PreviewCard Card { get; set; } public object Poll { get; set; } public Application Application { get; set; } public string Language { get; set; } diff --git a/PrivaPub/Api/Mastodon/Mappers/MastodonMapper.cs b/PrivaPub/Api/Mastodon/Mappers/MastodonMapper.cs index 4076b80..e8cc05d 100644 --- a/PrivaPub/Api/Mastodon/Mappers/MastodonMapper.cs +++ b/PrivaPub/Api/Mastodon/Mappers/MastodonMapper.cs @@ -12,6 +12,7 @@ using PrivaPub.Models.Social; using PrivaPub.Models.User; using PrivaPub.StaticServices; +using System.Globalization; using System.Net; using GroupEntity = PrivaPub.Models.Group.Group; @@ -192,6 +193,7 @@ namespace PrivaPub.Api.Mastodon.Mappers Visibility = Visibility(post.Visibility), Language = post.Language, MediaAttachments = post.Media.Select(Media).ToList(), + Card = Card(post), Mentions = post.Mentions.Select(m => Mention(m, mentionAccounts)).Where(m => m != default).ToList(), Tags = post.Tags.Select(t => new StatusTag { Name = t, Url = ActivityPubRenderer.TagUrl(_localActors.BaseAddress, t) }).ToList() }; @@ -227,12 +229,72 @@ namespace PrivaPub.Api.Mastodon.Mappers var content = post.ContentHtml ?? ActivityPubRenderer.Html(post.Text); if (post.IsFederatedCopy && string.IsNullOrWhiteSpace(content) && !string.IsNullOrEmpty(post.Excerpt)) content = $"

{WebUtility.HtmlEncode(post.Excerpt)}

"; + if (post.Event is { } happening && EventLine(happening) is { } line) + content = $"

{WebUtility.HtmlEncode(line)}

{content}"; var titled = !string.IsNullOrEmpty(post.Title) && (!post.IsFederatedCopy || post.ObjectType is not (null or "Note" or "Question")); return titled ? $"

{WebUtility.HtmlEncode(post.Title)}

{content}" : content ?? string.Empty; } + public static string EventLine(EventDetails happening) + { + if (happening.Start is not { } start) + return default; + var zone = Zone(happening.FloatingTime ? default : happening.TimeZone); + var local = zone == default ? start : TimeZoneInfo.ConvertTimeFromUtc(start, zone); + var when = local.ToString("yyyy-MM-dd HH:mm", CultureInfo.InvariantCulture); + if (happening.End is { } end && end > start) + { + var localEnd = zone == default ? end : TimeZoneInfo.ConvertTimeFromUtc(end, zone); + when += " – " + localEnd.ToString(localEnd.Date == local.Date ? "HH:mm" : "yyyy-MM-dd HH:mm", CultureInfo.InvariantCulture); + } + when += happening.FloatingTime ? " (local time)" : zone == default ? " UTC" : $" ({happening.TimeZone})"; + var where = happening.Places.Select(p => p.Name ?? p.Address).FirstOrDefault(p => p != default) + ?? (happening.OnlineURL != default ? "online" : default); + return where == default ? when : $"{when} · {where}"; + } + + static TimeZoneInfo Zone(string id) + { + if (string.IsNullOrWhiteSpace(id)) + return default; + try + { + return TimeZoneInfo.FindSystemTimeZoneById(id); + } + catch (Exception ex) when (ex is TimeZoneNotFoundException or InvalidTimeZoneException) + { + return default; + } + } + + PreviewCard Card(PostEntity post) + { + if (!post.IsFederatedCopy) + return default; + if (post.Link is { } link) + return Card(link.Href, link.Title ?? post.Title, link.Description ?? (post.Media.Count == 0 ? post.Excerpt : default), link.ImageURL); + if (post.ObjectType is "Article" or "Event" or "Video" or "Audio" && post.Url != default && post.Media.Count == 0) + return Card(post.Url, post.Title, post.Excerpt, post.CoverURL); + return default; + } + + PreviewCard Card(string url, string title, string description, string image) + { + if (!Uri.TryCreate(url, UriKind.Absolute, out var uri)) + return default; + return new PreviewCard + { + Url = url, + Title = title ?? uri.Host, + Description = description ?? string.Empty, + ProviderName = uri.Host, + ProviderUrl = uri.GetLeftPart(UriPartial.Authority), + Image = Proxied(image) + }; + } + public static string Visibility(PostVisibility visibility) => visibility switch { PostVisibility.Unlisted => "unlisted", @@ -253,6 +315,28 @@ namespace PrivaPub.Api.Mastodon.Mappers return new StatusMention { Id = account.ID, Username = account.UserName, Acct = $"{account.UserName}@{account.Domain}", Url = account.Url ?? account.ActorURI }; } + static Dictionary Meta(PostMedia media) + { + var original = new Dictionary(); + if (media.Width > 0 && media.Height > 0) + { + original["width"] = media.Width; + original["height"] = media.Height; + original["size"] = $"{media.Width}x{media.Height}"; + original["aspect"] = (double)media.Width.Value / media.Height.Value; + } + if (media.DurationSeconds is { } seconds) + original["duration"] = seconds; + var meta = new Dictionary(); + if (original.Count > 0) + meta["original"] = original; + if (media.DurationSeconds is { } length) + meta["length"] = TimeSpan.FromSeconds(length).ToString(@"h\:mm\:ss\.ff", CultureInfo.InvariantCulture); + if (media.Focus is { Length: 2 }) + meta["focus"] = new { x = media.Focus[0], y = media.Focus[1] }; + return meta; + } + MediaAttachment Media(PostMedia media) => new() { Id = media.AttachmentId ?? media.Id.ToString("N"), @@ -265,17 +349,12 @@ namespace PrivaPub.Api.Mastodon.Mappers _ => "unknown" }, Url = media.URL ?? Proxied(media.RemoteURL), - PreviewUrl = media.PreviewURL ?? media.URL ?? Proxied(media.RemoteURL), + PreviewUrl = media.PreviewURL ?? Proxied(media.RemotePreviewURL) + ?? (media.ContentType?.StartsWith("image/") == true || media.ContentType == default ? media.URL ?? Proxied(media.RemoteURL) : default), RemoteUrl = media.RemoteURL, Description = media.Description, Blurhash = media.Blurhash, - Meta = new - { - original = media.Width.HasValue && media.Height > 0 - ? new { width = media.Width, height = media.Height, size = $"{media.Width}x{media.Height}", aspect = (double)media.Width / media.Height.Value } - : default, - focus = media.Focus is { Length: 2 } ? new { x = media.Focus[0], y = media.Focus[1] } : default - } + Meta = Meta(media) }; } } diff --git a/PrivaPub/Federation/Inbox/Handlers/UpdateHandler.cs b/PrivaPub/Federation/Inbox/Handlers/UpdateHandler.cs index d82cfb6..5b90cc8 100644 --- a/PrivaPub/Federation/Inbox/Handlers/UpdateHandler.cs +++ b/PrivaPub/Federation/Inbox/Handlers/UpdateHandler.cs @@ -76,6 +76,12 @@ namespace PrivaPub.Federation.Inbox.Handlers post.Title = note.Title; post.SpoilerText = note.SpoilerText; post.Excerpt = note.Excerpt; + post.Source = note.Source; + post.CoverURL = note.CoverURL; + post.Link = note.Link; + post.Video = note.Video; + post.Audio = note.Audio; + post.Event = note.Event; post.HasContentWarning = note.Sensitive; post.Text = note.ContentHtml; post.ContentHtml = note.ContentHtml; diff --git a/PrivaPub/Federation/Inbox/RemotePosts.cs b/PrivaPub/Federation/Inbox/RemotePosts.cs index cf9cb84..ceb6914 100644 --- a/PrivaPub/Federation/Inbox/RemotePosts.cs +++ b/PrivaPub/Federation/Inbox/RemotePosts.cs @@ -77,6 +77,12 @@ namespace PrivaPub.Federation.Inbox Title = note.Title, SpoilerText = note.SpoilerText, Excerpt = note.Excerpt, + Source = note.Source, + CoverURL = note.CoverURL, + Link = note.Link, + Video = note.Video, + Audio = note.Audio, + Event = note.Event, HasContentWarning = note.Sensitive, Text = note.ContentHtml, ContentHtml = note.ContentHtml, diff --git a/PrivaPub/Federation/Objects/NoteParser.cs b/PrivaPub/Federation/Objects/NoteParser.cs index a3c83bb..6f3f97c 100644 --- a/PrivaPub/Federation/Objects/NoteParser.cs +++ b/PrivaPub/Federation/Objects/NoteParser.cs @@ -1,6 +1,7 @@ using PrivaPub.Models.Post; using System.Globalization; +using System.Text; using System.Text.Json.Nodes; using static PrivaPub.Federation.Objects.ActivityJson; @@ -31,6 +32,12 @@ namespace PrivaPub.Federation.Objects public IReadOnlyList Mentions { get; init; } = Array.Empty(); public IReadOnlyList Tags { get; init; } = Array.Empty(); public IReadOnlyList Attachments { get; init; } = Array.Empty(); + public PostSource Source { get; init; } + public string CoverURL { get; init; } + public PostLink Link { get; init; } + public VideoDetails Video { get; init; } + public AudioDetails Audio { get; init; } + public EventDetails Event { get; init; } } public static class NoteParser @@ -50,6 +57,8 @@ namespace PrivaPub.Federation.Objects var (contentHtml, language) = Content(note); var summary = Value(note, "summary"); + var excerpt = ObjectShapes.Text(summary, 1000) + ?? (note["preview"] is JsonObject preview && Value(preview, "type") == "Note" ? ObjectShapes.Text(Value(preview, "content"), 1000) : default); var sensitive = note["sensitive"] is JsonValue flag && flag.TryGetValue(out var marked) && marked; var summaryWarns = type is "Note" or "Question" || sensitive; return new NoteDocument @@ -59,9 +68,9 @@ namespace PrivaPub.Federation.Objects Raw = note, AttributedTo = Attribution(note["attributedTo"]), ContentHtml = ContentSanitizer.Html(contentHtml), - Title = Plain(Value(note, "name"), 500), + Title = ObjectShapes.Text(Value(note, "name"), 500), SpoilerText = summaryWarns ? Plain(summary, 500) : default, - Excerpt = summaryWarns ? default : Plain(summary, 1000), + Excerpt = summaryWarns ? default : excerpt, Sensitive = sensitive || summaryWarns && !string.IsNullOrWhiteSpace(summary), Language = language, Url = UrlOf(note["url"]) ?? id, @@ -78,12 +87,18 @@ namespace PrivaPub.Federation.Objects .DistinctBy(m => m.ActorURI) .Take(MaxTags) .ToList(), - Tags = Tags(note, "Hashtag").Select(t => Value(t, "name")?.TrimStart('#').ToLowerInvariant()) + Tags = Tags(note, "Hashtag").Select(t => Value(t, "name")?.TrimStart('#').Normalize(NormalizationForm.FormKC).ToLowerInvariant()) .Where(t => !string.IsNullOrEmpty(t)) .Distinct() .Take(MaxTags) .ToList(), - Attachments = Attachments(note) + Attachments = Attachments(note), + Source = ObjectShapes.Source(note), + CoverURL = ObjectShapes.Cover(note), + Link = ObjectShapes.Link(note), + Video = ObjectShapes.Video(note), + Audio = ObjectShapes.Audio(note), + Event = ObjectShapes.Event(note) }; } @@ -100,7 +115,9 @@ namespace PrivaPub.Federation.Objects } if (string.IsNullOrEmpty(content) && Value(note, "_misskey_content") is { } misskey) content = System.Net.WebUtility.HtmlEncode(misskey).Replace("\n", "
"); - return (content, language); + if (!string.IsNullOrEmpty(content) && Value(note, "mediaType")?.StartsWith("text/markdown", StringComparison.OrdinalIgnoreCase) == true) + content = Rendering.ActivityPubRenderer.Html(content); + return (content, language ?? ObjectShapes.Language(note)); } static string Attribution(JsonNode node) => node switch @@ -120,33 +137,37 @@ namespace PrivaPub.Federation.Objects _ => Enumerable.Empty() }).Where(t => Value(t, "type") == type); - static List Attachments(JsonObject note) - { - var items = note["attachment"] switch - { - JsonArray array => array.OfType(), - JsonObject single => new[] { single }, - _ => Enumerable.Empty() - }; - return items + static List Attachments(JsonObject note) => + ObjectShapes.Objects(note["attachment"]) .Where(a => Value(a, "type") is "Document" or "Image" or "Video" or "Audio") - .Select(a => new PostMedia - { - RemoteURL = UrlOf(a["url"]), - ContentType = Value(a, "mediaType"), - Description = Plain(Value(a, "name") ?? Value(a, "summary"), 1500), - Blurhash = Value(a, "blurhash"), - Width = Int(a["width"]), - Height = Int(a["height"]), - Focus = a["focalPoint"] is JsonArray focus && focus.Count == 2 - ? focus.Select(f => f is JsonValue v && v.TryGetValue(out var d) ? (float)Math.Clamp(d, -1, 1) : 0f).ToArray() - : default - }) + .Select(Attachment) .Where(m => Origin.Of(m.RemoteURL) != default) .Take(MaxAttachments) .ToList(); + + static PostMedia Attachment(JsonObject attachment) + { + var (url, linkMediaType) = ObjectShapes.MediaLink(attachment["url"] ?? attachment["href"]); + return new PostMedia + { + RemoteURL = url, + ContentType = ObjectShapes.InferMediaType(Value(attachment, "mediaType") ?? linkMediaType, url, Value(attachment, "type")), + RemotePreviewURL = ObjectShapes.Image(attachment["icon"]), + DurationSeconds = ObjectShapes.Seconds(attachment["duration"]), + Sensitive = attachment["sensitive"] is JsonValue flag && flag.TryGetValue(out var marked) ? marked : default(bool?), + Description = Plain(Value(attachment, "name") ?? Value(attachment, "summary") ?? FirstOf(attachment["nameMap"]) ?? FirstOf(attachment["summaryMap"]), 1500), + Blurhash = Value(attachment, "blurhash"), + Width = Int(attachment["width"]), + Height = Int(attachment["height"]), + Focus = attachment["focalPoint"] is JsonArray focus && focus.Count == 2 + ? focus.Select(f => f is JsonValue v && v.TryGetValue(out var d) ? (float)Math.Clamp(d, -1, 1) : 0f).ToArray() + : default + }; } + static string FirstOf(JsonNode map) => + map is JsonObject languages ? languages.Select(l => l.Value is JsonValue v && v.TryGetValue(out var text) ? text : default).FirstOrDefault(t => t != default) : default; + static string UrlOf(JsonNode node) => node switch { JsonValue value when value.TryGetValue(out var text) => text, diff --git a/PrivaPub/Federation/Objects/ObjectShapes.cs b/PrivaPub/Federation/Objects/ObjectShapes.cs new file mode 100644 index 0000000..a99c9a4 --- /dev/null +++ b/PrivaPub/Federation/Objects/ObjectShapes.cs @@ -0,0 +1,293 @@ +using PrivaPub.Models.Post; + +using System.Globalization; +using System.Net; +using System.Text.Json.Nodes; +using System.Text.RegularExpressions; +using System.Xml; + +using static PrivaPub.Federation.Objects.ActivityJson; + +namespace PrivaPub.Federation.Objects +{ + public static partial class ObjectShapes + { + const int MaxVariants = 20; + const int MaxCaptions = 20; + const int MaxPlaces = 4; + const int CoverMaxWidth = 1280; + + static readonly Dictionary MediaTypesByExtension = new(StringComparer.OrdinalIgnoreCase) + { + [".jpg"] = "image/jpeg", [".jpeg"] = "image/jpeg", [".png"] = "image/png", [".gif"] = "image/gif", [".webp"] = "image/webp", + [".avif"] = "image/avif", [".heic"] = "image/heic", [".svg"] = "image/svg+xml", + [".mp4"] = "video/mp4", [".m4v"] = "video/mp4", [".webm"] = "video/webm", [".mov"] = "video/quicktime", [".mkv"] = "video/x-matroska", + [".mp3"] = "audio/mpeg", [".ogg"] = "audio/ogg", [".oga"] = "audio/ogg", [".opus"] = "audio/opus", [".m4a"] = "audio/mp4", + [".flac"] = "audio/flac", [".wav"] = "audio/wav", [".m3u8"] = "application/x-mpegURL" + }; + + public static IEnumerable Objects(JsonNode node) => node switch + { + JsonArray array => array.OfType(), + JsonObject single => new[] { single }, + _ => Enumerable.Empty() + }; + + public static (string Url, string MediaType) MediaLink(JsonNode node) + { + switch (node) + { + case JsonValue value when value.TryGetValue(out var text): + return (text, default); + case JsonObject link when Value(link, "href") is { } href: + return (href, Value(link, "mediaType")); + case JsonObject wrapper when wrapper["url"] is { } inner: + var (url, mediaType) = MediaLink(inner); + return (url, mediaType ?? Value(wrapper, "mediaType")); + case JsonArray array: + var links = array.Select(MediaLink).Where(l => l.Url != default).ToList(); + return links.FirstOrDefault(l => l.MediaType?.StartsWith("text/html", StringComparison.OrdinalIgnoreCase) != true); + default: + return default; + } + } + + public static string InferMediaType(string declared, string url, string objectType) + { + if (!string.IsNullOrWhiteSpace(declared) && declared.Contains('/')) + return declared.Trim(); + if (Uri.TryCreate(url, UriKind.Absolute, out var uri) && MediaTypesByExtension.TryGetValue(Path.GetExtension(uri.AbsolutePath), out var byExtension)) + return byExtension; + return objectType switch + { + "Image" => "image/*", + "Video" => "video/*", + "Audio" => "audio/*", + _ => default + }; + } + + public static string Image(JsonNode node) + { + var candidates = node switch + { + JsonValue value when value.TryGetValue(out var text) => new[] { (Url: text, Width: (int?)default) }, + JsonArray array => array.Select(item => (Url: MediaLink(item).Url, Width: Int(item as JsonObject, "width"))).ToArray(), + JsonObject single => new[] { (Url: MediaLink(single).Url, Width: Int(single, "width")) }, + _ => Array.Empty<(string Url, int? Width)>() + }; + var usable = candidates.Where(c => Origin.Of(c.Url) != default).ToList(); + return usable.Where(c => c.Width <= CoverMaxWidth).OrderByDescending(c => c.Width).Select(c => c.Url).FirstOrDefault() + ?? usable.Select(c => c.Url).FirstOrDefault(); + } + + public static string Cover(JsonObject note) => + Image(note["image"]) ?? Image(note["icon"]) ?? (note["preview"] is JsonObject preview && Value(preview, "type") == "Image" ? Image(preview) : default); + + public static double? Seconds(JsonNode node) + { + switch (node) + { + case JsonValue value when value.TryGetValue(out var number): + return number >= 0 && number < 1e7 ? number : default; + case JsonValue value when value.TryGetValue(out var text) && !string.IsNullOrWhiteSpace(text): + try + { + var span = XmlConvert.ToTimeSpan(text.Trim()); + return span >= TimeSpan.Zero ? span.TotalSeconds : default; + } + catch (FormatException) + { + return double.TryParse(text, NumberStyles.Float, CultureInfo.InvariantCulture, out var seconds) && seconds >= 0 ? seconds : default; + } + default: + return default; + } + } + + public static string Text(string html, int max) + { + if (string.IsNullOrWhiteSpace(html)) + return default; + var text = WebUtility.HtmlDecode(Tags().Replace(html, " ")); + text = Spaces().Replace(text, " ").Trim(); + if (text.Length == 0) + return default; + return text.Length <= max ? text : text[..max]; + } + + public static string Language(JsonObject note) => + note["@context"] is JsonArray context + ? context.OfType().Select(entry => Value(entry, "@language")).FirstOrDefault(language => !string.IsNullOrWhiteSpace(language)) + : default; + + public static PostSource Source(JsonObject note) => note["source"] is JsonObject source && Value(source, "content") is { } content + ? new PostSource { Content = content.Length <= 64 * 1024 ? content : content[..(64 * 1024)], MediaType = Value(source, "mediaType") } + : default; + + public static PostLink Link(JsonObject note) + { + var attached = Objects(note["attachment"]).FirstOrDefault(a => Value(a, "type") == "Link" && IsWeb(Value(a, "href"))); + if (attached != default) + { + var preview = attached["preview"] as JsonObject; + return new PostLink + { + Href = Value(attached, "href"), + MediaType = Value(attached, "mediaType"), + Title = Text(Value(preview, "name") ?? Value(attached, "name"), 300), + Description = Text(Value(preview, "summary") ?? Value(preview, "content"), 1000), + ImageURL = preview == default ? Image(note["image"]) : Image(preview["image"]) ?? Image(preview["icon"]), + FromPublisher = preview != default + }; + } + if (note["source"] is JsonValue source && source.TryGetValue(out var sourceUrl) && IsWeb(sourceUrl)) + return new PostLink { Href = sourceUrl, ImageURL = Image(note["image"]) }; + return default; + } + + public static VideoDetails Video(JsonObject note) + { + if (Value(note, "type") != "Video") + return default; + var links = Objects(note["url"]).ToList(); + return new VideoDetails + { + DurationSeconds = Seconds(note["duration"]), + Variants = Variants(links, "video/", "audio/"), + PlaylistURL = links.Where(l => Value(l, "mediaType")?.Equals("application/x-mpegURL", StringComparison.OrdinalIgnoreCase) == true) + .Select(l => Value(l, "href")).FirstOrDefault(IsWeb), + IsLive = Bool(note, "isLiveBroadcast") == true, + State = Int(note, "state"), + Views = Long(note, "views"), + CommentsPolicy = Int(note, "commentsPolicy"), + DownloadEnabled = Bool(note, "downloadEnabled"), + Licence = Named(note["licence"] ?? note["license"]), + Category = Named(note["category"]), + ChannelURI = Objects(note["attributedTo"]).Where(a => Value(a, "type") == "Group").Select(a => Id(a)).FirstOrDefault(), + EmbedURL = Value(note, "embedUrl") is { } embed && IsWeb(embed) ? embed : default, + ChaptersURL = Value(note, "hasParts") is { } chapters && IsWeb(chapters) ? chapters : default, + Captions = Objects(note["subtitleLanguage"]) + .Select(c => new Caption + { + Language = Value(c, "identifier"), + Url = Objects(c["url"]).Where(u => Value(u, "mediaType") == "text/vtt").Select(u => Value(u, "href")).FirstOrDefault(IsWeb) + ?? (Value(c, "url") is { } plain && IsWeb(plain) ? plain : default) + }) + .Where(c => c.Url != default) + .Take(MaxCaptions) + .ToList() + }; + } + + public static AudioDetails Audio(JsonObject note) + { + if (Value(note, "type") != "Audio") + return default; + return new AudioDetails + { + DurationSeconds = Seconds(note["duration"]), + Variants = Variants(Objects(note["url"]).ToList(), "audio/"), + Album = Named(note["album"]), + Position = Int(note, "position"), + Licence = Named(note["license"] ?? note["licence"]) + }; + } + + public static EventDetails Event(JsonObject note) + { + if (Value(note, "type") != "Event") + return default; + var start = Value(note, "startTime"); + var places = Objects(note["location"]).Select(Place).Where(p => p != default).Take(MaxPlaces).ToList(); + return new EventDetails + { + Start = Moment(start), + End = Moment(Value(note, "endTime")), + TimeZone = Value(note, "timezone"), + FloatingTime = start?.EndsWith("-00:00", StringComparison.Ordinal) == true, + Places = places, + OnlineURL = places.Select(p => p.Url).FirstOrDefault(u => u != default) + ?? Objects(note["attachment"]).Where(a => Value(a, "type") == "Link" && Value(a, "name") == "Website").Select(a => Value(a, "href")).FirstOrDefault(IsWeb), + JoinMode = Value(note, "joinMode"), + ParticipantCount = Int(note, "participantCount"), + Capacity = Int(note, "maximumAttendeeCapacity"), + Status = Value(note, "ical:status") ?? Value(note, "eventStatus") ?? Value(note, "status"), + Category = Named(note["category"]) + }; + } + + static EventPlace Place(JsonObject location) + { + var type = Value(location, "type"); + if (type == "VirtualLocation") + return Value(location, "url") is { } url && IsWeb(url) ? new EventPlace { Name = Text(Value(location, "name"), 200), Url = url } : default; + var address = location["address"] switch + { + JsonValue value when value.TryGetValue(out var text) => text, + JsonObject postal => string.Join(", ", new[] { "streetAddress", "postalCode", "addressLocality", "addressRegion", "addressCountry" } + .Select(field => Value(postal, field)).Where(part => !string.IsNullOrWhiteSpace(part))), + _ => default + }; + var place = new EventPlace + { + Name = Text(Value(location, "name") ?? Value(location, "content"), 200), + Address = Text(address, 300), + Latitude = Double(location, "latitude"), + Longitude = Double(location, "longitude") + }; + return place.Name == default && place.Address == default && place.Latitude == default ? default : place; + } + + static List Variants(List links, params string[] prefixes) => + links.Where(l => Value(l, "mediaType") is { } type && prefixes.Any(p => type.StartsWith(p, StringComparison.OrdinalIgnoreCase)) && IsWeb(Value(l, "href"))) + .Select(l => new MediaVariant + { + Url = Value(l, "href"), + MediaType = Value(l, "mediaType"), + Width = Int(l, "width"), + Height = Int(l, "height"), + Fps = Double(l, "fps"), + Size = Long(l, "size"), + Streams = Objects(l["attachment"]) + .Where(p => Value(p, "name") == "ffprobe_codec_type") + .Select(p => Value(p, "value")) + .Where(v => v is "audio" or "video") + .Distinct() + .ToList() + }) + .Take(MaxVariants) + .ToList(); + + static string Named(JsonNode node) => node switch + { + JsonValue value when value.TryGetValue(out var text) => Text(text, 200), + JsonObject named => Text(Value(named, "name") ?? Value(named, "identifier"), 200), + _ => default + }; + + static DateTime? Moment(string value) => + DateTimeOffset.TryParse(value, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal, out var parsed) ? parsed.UtcDateTime : (DateTime?)null; + + static bool IsWeb(string url) => + Uri.TryCreate(url, UriKind.Absolute, out var uri) && (uri.Scheme == Uri.UriSchemeHttps || uri.Scheme == Uri.UriSchemeHttp); + + static int? Int(JsonObject node, string property) => + node?[property] is JsonValue value && value.TryGetValue(out var number) && number is >= 0 and < int.MaxValue ? (int)number : (int?)null; + + static long? Long(JsonObject node, string property) => + node?[property] is JsonValue value && value.TryGetValue(out var number) && number is >= 0 and < long.MaxValue ? (long)number : (long?)null; + + static double? Double(JsonObject node, string property) => + node?[property] is JsonValue value && value.TryGetValue(out var number) && double.IsFinite(number) ? number : (double?)null; + + static bool? Bool(JsonObject node, string property) => + node?[property] is JsonValue value && value.TryGetValue(out var flag) ? flag : (bool?)null; + + [GeneratedRegex("<[^>]*>")] + private static partial Regex Tags(); + + [GeneratedRegex(@"\s+")] + private static partial Regex Spaces(); + } +} diff --git a/PrivaPub/Models/Post/Post.cs b/PrivaPub/Models/Post/Post.cs index 973ef45..8165472 100644 --- a/PrivaPub/Models/Post/Post.cs +++ b/PrivaPub/Models/Post/Post.cs @@ -25,6 +25,18 @@ namespace PrivaPub.Models.Post public List Mentions { get; set; } = new(); public List Tags { get; set; } = new(); public List Media { get; set; } = new(); + [BsonIgnoreIfNull] + public PostSource Source { get; set; }//the remote author's own text (Markdown, MFM, BBCode) + [BsonIgnoreIfNull] + public string CoverURL { get; set; }//a remote object's own image: article cover, video poster, event banner + [BsonIgnoreIfNull] + public PostLink Link { get; set; } + [BsonIgnoreIfNull] + public VideoDetails Video { get; set; } + [BsonIgnoreIfNull] + public AudioDetails Audio { get; set; } + [BsonIgnoreIfNull] + public EventDetails Event { get; set; } public List Location { get; set; } = new(); public float RangeKm { get; set; } = 5.0f; [BsonIgnoreIfNull] diff --git a/PrivaPub/Models/Post/PostDetails.cs b/PrivaPub/Models/Post/PostDetails.cs new file mode 100644 index 0000000..2ae8e0d --- /dev/null +++ b/PrivaPub/Models/Post/PostDetails.cs @@ -0,0 +1,86 @@ +namespace PrivaPub.Models.Post +{ + public class PostSource + { + public string Content { get; set; } + public string MediaType { get; set; }//text/markdown, text/x.misskeymarkdown (MFM), text/bbcode… + } + + public class PostLink + { + public string Href { get; set; } + public string MediaType { get; set; } + public string Title { get; set; } + public string Description { get; set; } + public string ImageURL { get; set; } + public bool FromPublisher { get; set; }//title, description and image came with the post (FEP-8967 preview), nothing was fetched + } + + public class MediaVariant + { + public string Url { get; set; } + public string MediaType { get; set; } + public int? Width { get; set; } + public int? Height { get; set; } + public double? Fps { get; set; } + public long? Size { get; set; } + public List Streams { get; set; } = new();//"audio", "video", from PeerTube's ffprobe_codec_type + } + + public class VideoDetails + { + public double? DurationSeconds { get; set; } + public List Variants { get; set; } = new(); + public string PlaylistURL { get; set; }//HLS + public bool IsLive { get; set; } + public int? State { get; set; }//PeerTube: 1 published, 4 waiting for live, 5 live ended + public long? Views { get; set; } + public int? CommentsPolicy { get; set; }//PeerTube: 1 open, 2 closed, 3 needs approval + public bool? DownloadEnabled { get; set; } + public string Licence { get; set; } + public string Category { get; set; } + public string ChannelURI { get; set; } + public string EmbedURL { get; set; } + public string ChaptersURL { get; set; } + public List Captions { get; set; } = new(); + } + + public class Caption + { + public string Language { get; set; } + public string Url { get; set; } + } + + public class AudioDetails + { + public double? DurationSeconds { get; set; } + public List Variants { get; set; } = new(); + public string Album { get; set; } + public int? Position { get; set; } + public string Licence { get; set; } + } + + public class EventDetails + { + public DateTime? Start { get; set; } + public DateTime? End { get; set; } + public string TimeZone { get; set; } + public bool FloatingTime { get; set; }//"-00:00": a local time with no zone (Hubzilla) + public List Places { get; set; } = new(); + public string OnlineURL { get; set; } + public string JoinMode { get; set; } + public int? ParticipantCount { get; set; } + public int? Capacity { get; set; } + public string Status { get; set; } + public string Category { get; set; } + } + + public class EventPlace + { + public string Name { get; set; } + public string Address { get; set; } + public double? Latitude { get; set; } + public double? Longitude { get; set; } + public string Url { get; set; }//a VirtualLocation + } +} diff --git a/PrivaPub/Models/Post/PostMedia.cs b/PrivaPub/Models/Post/PostMedia.cs index a6b74cc..1c42a65 100644 --- a/PrivaPub/Models/Post/PostMedia.cs +++ b/PrivaPub/Models/Post/PostMedia.cs @@ -16,5 +16,8 @@ namespace PrivaPub.Models.Post public int? Width { get; set; } public int? Height { get; set; } public float[] Focus { get; set; } + public string RemotePreviewURL { get; set; }//the remote thumbnail (attachment icon), served through the proxy + public double? DurationSeconds { get; set; } + public bool? Sensitive { get; set; }//Misskey marks each attachment } } diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index 666cbbb..bf5488b 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -417,15 +417,17 @@ it, raw where it doesn't. - Still open: every activity id dereferences. - **Owner decisions that were small** (done in v1.7.0): blocks federate (`Block`, `Undo{Block}`); a persona's and a group's `published` is a random day up to two weeks before its creation (migration `_007`). -- **Parsing every shape:** +- **Parsing every shape** (done in v1.8.0, `Federation/Objects/ObjectShapes.cs`): - `url`, `icon`, `image`, `attachment` and `attributedTo` as a value, an object or an array (W1). - A Markdown `content` and `source` (W3). - Inferring a missing `mediaType` (W4). - Every thumbnail location (W5). - Language from `@context` (W11); alt text from `summary` (W12). - A null `id`, a 200 `Tombstone`, a `Delete` before its `Create` (W9, W13). -- **`Post.Kind`** plus typed payloads for article, video, audio, event, link, review and thread (INTEROP §4.1). The - Mastodon API view of each kind: content, a card made from the object without fetching, attachments. +- **Typed payloads** (done in v1.8.0 for link, video, audio, event and the cover image; review and thread are still + open): `Post.ObjectType` plus `Link`, `Video`, `Audio`, `Event`, `CoverURL` and `Source` (INTEROP §4.1). The Mastodon + API view: the title above the body, an event's "when · where" line, a card made from the object without fetching, + attachments with their thumbnail and duration. Video and audio playback through the proxy is P6. - **Raw capture for the details view** (done in v1.8.0): the object as received, how it arrived, the signature scheme and key, received versus `published`, the extensions detected, and the origin's software (§4.3). Exposed at `/api/privapub/v1/statuses/:id/provenance` and `/api/privapub/v1/instances/:host`. Checked live against GoToSocial.