From 001fdac3be5b9451d6a0112a462b7c0c6e957c4d Mon Sep 17 00:00:00 2001 From: thepra Date: Thu, 1 Oct 2026 18:43:39 +0200 Subject: [PATCH] P6: link previews read by the server, as the owner decided - A public post that links to a page gets a preview card: from the post's own data first (FEP-8967 Link preview, the object's image, title and summary); otherwise the server reads the page once, 0-60 s after the post arrives, never when someone reads it. OpenGraph and Twitter tags give title, description and image; one LinkPreview per address is cached for 7 days and shared by the whole server, so a fetch never points at a persona. - Lemmy link posts, which carry no title or description, get them filled in. - The page fetch uses the guarded client (public addresses, three redirects, HTML only, first 512 KB). - Federation:FetchLinkPreviews switches page fetching off. - Local public posts get cards too. Checked live: a link to a GoToSocial profile page becomes a card with its title, description and proxied image. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012CzABvBkbcFqoHdmi8b9WB --- CLAUDE.md | 3 + FEDERATION.md | 12 ++ PrivaPub.Tests/Domain/LinkPreviewTests.cs | 43 +++++ .../Federation/InboxScenarioTests.cs | 2 +- PrivaPub.Tests/Support/Harness.cs | 11 +- .../Api/Mastodon/Mappers/MastodonMapper.cs | 4 +- PrivaPub/Domain/Content/LinkPreviews.cs | 155 ++++++++++++++++++ PrivaPub/Domain/Statuses/StatusService.cs | 6 +- .../Inbox/Handlers/CreateHandler.cs | 6 +- PrivaPub/Federation/Inbox/RemotePosts.cs | 6 +- PrivaPub/Infrastructure/Data/Indexes.cs | 1 + .../Infrastructure/Http/FederationHttp.cs | 61 +++++++ .../Infrastructure/Http/FederationOptions.cs | 1 + .../Middleware/SocialPubConfigurations.cs | 3 + PrivaPub/Models/Jobs/Job.cs | 3 +- docs/ROADMAP.md | 6 +- tools/pasture/interop.sh | 4 + 17 files changed, 314 insertions(+), 13 deletions(-) create mode 100644 PrivaPub.Tests/Domain/LinkPreviewTests.cs create mode 100644 PrivaPub/Domain/Content/LinkPreviews.cs diff --git a/CLAUDE.md b/CLAUDE.md index 4ec20df..1c24720 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -258,6 +258,9 @@ cd /var/www/privapub.thepra.dev && sudo -u www-data ASPNETCORE_ENVIRONMENT=Produ - **A poll vote is a `Note` with a `name`, an `inReplyTo` that is a poll we hold, and no content.** `CreateHandler` hands it to `PollService.Receive` before anything else, so it never becomes a reply. Our votes on other servers' polls go only to the poll's author, without `published` (PieFed counts a vote only then). +- **Link previews follow owner decision 1** (`Domain/Content/LinkPreviews.cs`): only public posts, queued on arrival with + 0–60 s of jitter, one cached `LinkPreview` per address for the whole server, never fetched when someone reads. + `LinkPreviews.Wanted` is called where posts are saved (`CreateHandler`, `StoreContext`, `StatusService.Publish`). - **A server is described on arrival, never on read.** The first record from a host enqueues `DescribeInstance` (its NodeInfo, at most once a week, into `RemoteInstance`), so opening the details view tells nobody anything. - **Post ids are the timeline order, so they follow arrival, not `published`.** `PrivacyIds.Arrived` gives a remote post diff --git a/FEDERATION.md b/FEDERATION.md index 100cfcf..ef66ac3 100644 --- a/FEDERATION.md +++ b/FEDERATION.md @@ -128,6 +128,18 @@ Received `summary` is a content warning only on a `Note` or `Question`, or when Visibility is expressed in `to`/`cc` the way Mastodon does it: public, unlisted, followers-only and direct. Inbound followers-only posts are recognised by the author's own `followers` collection. +## Link previews + +For a public post that links to a page, PrivaPub builds a preview card. It uses, in order: +1. a Link attachment's own `preview` (FEP-8967) or the post's own image, title and summary; +2. otherwise, the linked page, read once by the server between 0 and 60 seconds after the post arrives (never when + someone reads it), with an `Accept: text/html` request that reads at most 512 KB, through the same address checks as + every other fetch. + +The page's OpenGraph and Twitter tags give the title, description and image; the result is cached per address for 7 +days and shared by every account on the server. Images are served to clients only through PrivaPub's media proxy. +`Federation:FetchLinkPreviews=false` turns page fetching off. + ## Local-only posts Posts with a location (shown to nearby users of this server) never leave the server, in any form. diff --git a/PrivaPub.Tests/Domain/LinkPreviewTests.cs b/PrivaPub.Tests/Domain/LinkPreviewTests.cs new file mode 100644 index 0000000..1eeb798 --- /dev/null +++ b/PrivaPub.Tests/Domain/LinkPreviewTests.cs @@ -0,0 +1,43 @@ +using PrivaPub.Domain.Content; +using PrivaPub.Models.Post; + +namespace PrivaPub.Tests.Domain +{ + public class LinkPreviewTests + { + [Fact] + public void Reads_open_graph_then_twitter_then_the_page_title() + { + var preview = LinkPreviews.Read("https://news.example/a?utm_source=x", new Uri("https://news.example/a"), """ + Fallback title + + + + + + """); + + Assert.Equal("Rain & sun", preview.Title); + Assert.Equal("A long story", preview.Description); + Assert.Equal("https://news.example/img/cover.jpg", preview.ImageURL); + Assert.Equal("News", preview.SiteName); + Assert.False(preview.Failed); + Assert.True(LinkPreviews.Read("https://x.example", new Uri("https://x.example"), "nothing").Failed); + } + + [Fact] + public void Picks_the_first_real_link_and_skips_mentions_tags_quotes_and_ourselves() => + Assert.Equal("https://blog.example/post", LinkPreviews.FirstLink(new Post + { + Url = "https://m.example/@a/1", + ContentHtml = """ +

@bob + + RE: link + me + self + read this

+ """ + }, "https://privapub.test")); + } +} diff --git a/PrivaPub.Tests/Federation/InboxScenarioTests.cs b/PrivaPub.Tests/Federation/InboxScenarioTests.cs index ad94d5a..5cd6c12 100644 --- a/PrivaPub.Tests/Federation/InboxScenarioTests.cs +++ b/PrivaPub.Tests/Federation/InboxScenarioTests.cs @@ -56,7 +56,7 @@ namespace PrivaPub.Tests.Federation { new FollowHandler(db, _local, remote, delivery), new UndoHandler(db, _local, new Reactions(db, delivery)), - new CreateHandler(db, _local, remote, delivery, _blocks, new Fanout(db), new RemotePosts(db, _local, remote, _blocks, queue, new ObjectRecords(queue)), new GroupDistributor(delivery), new ObjectRecords(queue), new PollService(db, _local, delivery, queue)), + new CreateHandler(db, _local, remote, delivery, _blocks, new Fanout(db), new RemotePosts(db, _local, remote, _blocks, queue, new ObjectRecords(queue), new NoPreviews()), new GroupDistributor(delivery), new ObjectRecords(queue), new PollService(db, _local, delivery, queue), new NoPreviews()), new DeleteHandler(db, _local, remote, delivery, new GroupDistributor(delivery)), new UpdateHandler(db, _local, remote, new GroupDistributor(delivery), new ObjectRecords(queue)) }, NullLogger.Instance); diff --git a/PrivaPub.Tests/Support/Harness.cs b/PrivaPub.Tests/Support/Harness.cs index 7aa32f9..7a5c280 100644 --- a/PrivaPub.Tests/Support/Harness.cs +++ b/PrivaPub.Tests/Support/Harness.cs @@ -46,7 +46,7 @@ namespace PrivaPub.Tests.Support Records = new ObjectRecords(Queue); Polls = new PollService(Db, Local, Delivery, Queue); Reactions = new Reactions(Db, Delivery); - RemotePosts = new RemotePosts(Db, Local, Remote, new NoBlocks(), Queue, Records); + RemotePosts = new RemotePosts(Db, Local, Remote, new NoBlocks(), Queue, Records, new NoPreviews()); Receiver = new InboxReceiver(Local, Remote, Queue, new NoBlocks(), NullLogger.Instance); Processor = new InboxProcessor(Remote, new IActivityHandler[] { @@ -59,7 +59,7 @@ namespace PrivaPub.Tests.Support new DislikeHandler(Db), new JoinHandler(Db, Local, Delivery), new AnnounceHandler(Db, Local, RemotePosts, Fanout, Remote), - new CreateHandler(Db, Local, Remote, Delivery, new NoBlocks(), Fanout, RemotePosts, Groups, Records, Polls), + new CreateHandler(Db, Local, Remote, Delivery, new NoBlocks(), Fanout, RemotePosts, Groups, Records, Polls, new NoPreviews()), new DeleteHandler(Db, Local, Remote, Delivery, Groups), new UpdateHandler(Db, Local, Remote, Groups, Records), new FlagHandler(Db, Local) @@ -69,7 +69,7 @@ namespace PrivaPub.Tests.Support Outbox = new OutboxPublisher(Db, Local, Delivery); Media = new MediaService(new StaticOptions(new MediaOptions { Root = Path.Combine(Path.GetTempPath(), $"privapub-media-{Guid.NewGuid():N}") }), Local, default, NullLogger.Instance); - Statuses = new StatusService(Db, Local, Remote, Delivery, Content, Outbox, Fanout, Media, Groups, Polls); + Statuses = new StatusService(Db, Local, Remote, Delivery, Content, Outbox, Fanout, Media, Groups, Polls, new NoPreviews()); Posts = new PostsService(Db, Local, Statuses, new KeyLocalizer(), NullLogger.Instance); Timelines = new TimelineService(Db, new KeyLocalizer()); Relationships = new RelationshipService(Db, Follows, Delivery); @@ -148,4 +148,9 @@ namespace PrivaPub.Tests.Support public LocalizedString this[string name, params object[] arguments] => new(name, string.Format(name, arguments)); public IEnumerable GetAllStrings(bool includeParentCultures) => Enumerable.Empty(); } + + public sealed class NoPreviews : ILinkPreviews + { + public Task Wanted(PrivaPub.Models.Post.Post post, CancellationToken token) => Task.CompletedTask; + } } diff --git a/PrivaPub/Api/Mastodon/Mappers/MastodonMapper.cs b/PrivaPub/Api/Mastodon/Mappers/MastodonMapper.cs index 64c6132..a7d9728 100644 --- a/PrivaPub/Api/Mastodon/Mappers/MastodonMapper.cs +++ b/PrivaPub/Api/Mastodon/Mappers/MastodonMapper.cs @@ -389,11 +389,9 @@ namespace PrivaPub.Api.Mastodon.Mappers PreviewCard Card(PostEntity post) { - if (!post.IsFederatedCopy) - return default; if (post.Link is { } link) return Card(link.Href, link.Title ?? post.Title, link.Description ?? (post.Media.Count == 0 ? post.Excerpt : default), link.ImageURL); - if (post.ObjectType is "Article" or "Event" or "Video" or "Audio" && post.Url != default && post.Media.Count == 0) + if (post.IsFederatedCopy && post.ObjectType is "Article" or "Event" or "Video" or "Audio" && post.Url != default && post.Media.Count == 0) return Card(post.Url, post.Title, post.Excerpt, post.CoverURL); return default; } diff --git a/PrivaPub/Domain/Content/LinkPreviews.cs b/PrivaPub/Domain/Content/LinkPreviews.cs new file mode 100644 index 0000000..1bfb5aa --- /dev/null +++ b/PrivaPub/Domain/Content/LinkPreviews.cs @@ -0,0 +1,155 @@ +using AngleSharp.Html.Parser; + +using Microsoft.Extensions.Options; + +using MongoDB.Entities; + +using PrivaPub.Federation.Actors; +using PrivaPub.Federation.Objects; +using PrivaPub.Infrastructure.Http; +using PrivaPub.Infrastructure.Jobs; +using PrivaPub.Models.Jobs; +using PrivaPub.Models.Post; +using PrivaPub.StaticServices; + +using System.Security.Cryptography; + +using PostEntity = PrivaPub.Models.Post.Post; + +namespace PrivaPub.Domain.Content +{ + public class LinkPreview : Entity + { + public string Url { get; set; } + public string Title { get; set; } + public string Description { get; set; } + public string ImageURL { get; set; } + public string SiteName { get; set; } + public bool Failed { get; set; } + public DateTime FetchedAt { get; set; } = DateTime.UtcNow; + } + + public interface ILinkPreviews + { + Task Wanted(PostEntity post, CancellationToken token); + } + + public class LinkPreviews : ILinkPreviews, IJobHandler + { + const int MaxJitterSeconds = 60; + static readonly TimeSpan Freshness = TimeSpan.FromDays(7); + static readonly HtmlParser Parser = new(); + + readonly DbEntities _dbEntities; + readonly ILocalActorService _localActors; + readonly IFederationHttp _http; + readonly IJobQueue _queue; + readonly IOptionsMonitor _options; + + public LinkPreviews(DbEntities dbEntities, ILocalActorService localActors, IFederationHttp http, IJobQueue queue, + IOptionsMonitor options) + { + _dbEntities = dbEntities; + _localActors = localActors; + _http = http; + _queue = queue; + _options = options; + } + + public JobKind Kind => JobKind.FetchPreview; + public int Concurrency => 2; + public int MaxAttempts => 1; + public int PerHostLimit => 1; + + public async Task Wanted(PostEntity post, CancellationToken token) + { + if (!_options.CurrentValue.FetchLinkPreviews || post.Visibility is not (PostVisibility.Public or PostVisibility.Unlisted) || post.IsLocalOnly + || post.ReblogOfPostId != default || post.Link?.Title != default || post.Link == default && post.Media.Count > 0) + return; + var url = post.Link?.Href ?? FirstLink(post, _localActors.BaseAddress); + if (url == default) + return; + await _queue.EnqueueMany(new[] + { + new Job + { + Kind = JobKind.FetchPreview, Payload = post.ID, Host = new Uri(url).Host.ToLowerInvariant(), DedupeKey = $"preview|{post.ID}", + RunAt = DateTime.UtcNow.AddSeconds(RandomNumberGenerator.GetInt32(0, MaxJitterSeconds + 1)) + } + }, token); + } + + public async Task Handle(Job job, CancellationToken token) + { + var post = await _dbEntities.Posts.MatchID(job.Payload).ExecuteFirstAsync(token); + if (post == default || post.DeletedAt.HasValue || post.Link?.Title != default) + return JobOutcome.Done; + var url = post.Link?.Href ?? FirstLink(post, _localActors.BaseAddress); + if (url == default) + return JobOutcome.Done; + + var preview = await DB.Default.Find().Match(p => p.Url == url).ExecuteFirstAsync(token); + if (preview == default || preview.FetchedAt < DateTime.UtcNow - Freshness) + preview = await Fetch(url, token); + if (preview == default || preview.Failed) + return JobOutcome.Done; + + var link = post.Link ?? new PostLink { Href = url }; + link.Title ??= preview.Title; + link.Description ??= preview.Description; + link.ImageURL ??= preview.ImageURL; + await DB.Default.Update().MatchID(post.ID).Modify(p => p.Link, link).ExecuteAsync(token); + return JobOutcome.Done; + } + + async Task Fetch(string url, CancellationToken token) + { + var (finalUri, html) = await _http.GetPage(url, token); + var preview = html == default ? new LinkPreview { Url = url, Failed = true } : Read(url, finalUri, html); + await DB.Default.Update() + .Match(p => p.Url == url) + .Modify(p => p.Title, preview.Title) + .Modify(p => p.Description, preview.Description) + .Modify(p => p.ImageURL, preview.ImageURL) + .Modify(p => p.SiteName, preview.SiteName) + .Modify(p => p.Failed, preview.Failed) + .Modify(p => p.FetchedAt, DateTime.UtcNow) + .Option(o => o.IsUpsert = true) + .ExecuteAsync(token); + return preview; + } + + public static LinkPreview Read(string url, Uri finalUri, string html) + { + var document = Parser.ParseDocument(html); + string Meta(params string[] names) => names + .Select(name => document.QuerySelector($"meta[property='{name}'], meta[name='{name}']")?.GetAttribute("content")) + .FirstOrDefault(value => !string.IsNullOrWhiteSpace(value)); + var image = Meta("og:image:secure_url", "og:image", "og:image:url", "twitter:image", "twitter:image:src"); + var imageUri = image != default && Uri.TryCreate(finalUri, image, out var resolved) && resolved.Scheme is "https" or "http" ? resolved.AbsoluteUri : default; + var preview = new LinkPreview + { + Url = url, + Title = ObjectShapes.Text(Meta("og:title", "twitter:title") ?? document.Title, 300), + Description = ObjectShapes.Text(Meta("og:description", "twitter:description", "description"), 1000), + SiteName = ObjectShapes.Text(Meta("og:site_name"), 200), + ImageURL = imageUri + }; + preview.Failed = preview.Title == default && preview.Description == default && preview.ImageURL == default; + return preview; + } + + public static string FirstLink(PostEntity post, string ownBase) + { + if (string.IsNullOrEmpty(post.ContentHtml)) + return default; + var document = Parser.ParseDocument(post.ContentHtml); + return document.QuerySelectorAll("a[href]") + .Where(a => !(a.ClassList.Contains("mention") || a.ClassList.Contains("hashtag") || a.GetAttribute("rel")?.Contains("tag") == true + || a.Closest(".quote-inline") != default)) + .Select(a => a.GetAttribute("href")) + .FirstOrDefault(href => Uri.TryCreate(href, UriKind.Absolute, out var uri) && uri.Scheme is "https" or "http" + && href != post.Url && href != post.ObjectURI && !href.StartsWith(ownBase + "/", StringComparison.OrdinalIgnoreCase)); + } + } +} diff --git a/PrivaPub/Domain/Statuses/StatusService.cs b/PrivaPub/Domain/Statuses/StatusService.cs index 817f028..c251712 100644 --- a/PrivaPub/Domain/Statuses/StatusService.cs +++ b/PrivaPub/Domain/Statuses/StatusService.cs @@ -71,10 +71,13 @@ namespace PrivaPub.Domain.Statuses readonly IMediaService _media; readonly IGroupDistributor _groups; readonly IPollService _polls; + readonly ILinkPreviews _previews; public StatusService(DbEntities dbEntities, ILocalActorService localActors, IRemoteActorService remoteActors, IDeliveryService delivery, - IContentRenderer content, IOutboxPublisher outbox, IFanout fanout, IMediaService media, IGroupDistributor groups, IPollService polls) + IContentRenderer content, IOutboxPublisher outbox, IFanout fanout, IMediaService media, IGroupDistributor groups, IPollService polls, + ILinkPreviews previews) { + _previews = previews; _polls = polls; _media = media; _groups = groups; @@ -207,6 +210,7 @@ namespace PrivaPub.Domain.Statuses await DB.Default.Update().MatchID(parent.ID).Modify(b => b.Inc(p => p.RepliesCount, 1)).ExecuteAsync(token); await _fanout.Distribute(post, token); await _polls.Scheduled(post, token); + await _previews.Wanted(post, token); if (create != default) await _outbox.Publish(author, post, create, token); if (create != default && group is { IsCircle: false } && visibility is PostVisibility.Public or PostVisibility.Unlisted) diff --git a/PrivaPub/Federation/Inbox/Handlers/CreateHandler.cs b/PrivaPub/Federation/Inbox/Handlers/CreateHandler.cs index da57076..d03fc29 100644 --- a/PrivaPub/Federation/Inbox/Handlers/CreateHandler.cs +++ b/PrivaPub/Federation/Inbox/Handlers/CreateHandler.cs @@ -4,6 +4,7 @@ using MongoDB.Entities; using PrivaPub.Domain.Timelines; using PrivaPub.Federation.Actors; using PrivaPub.Federation.Moderation; +using PrivaPub.Domain.Content; using PrivaPub.Domain.Statuses; using PrivaPub.Federation.Objects; using PrivaPub.Federation.Outbox; @@ -37,10 +38,12 @@ namespace PrivaPub.Federation.Inbox.Handlers readonly IGroupDistributor _groups; readonly IObjectRecords _records; readonly IPollService _polls; + readonly ILinkPreviews _previews; public CreateHandler(DbEntities dbEntities, ILocalActorService localActors, IRemoteActorService remoteActors, IDeliveryService delivery, - IDomainBlocks domainBlocks, IFanout fanout, IRemotePosts remotePosts, IGroupDistributor groups, IObjectRecords records, IPollService polls) + IDomainBlocks domainBlocks, IFanout fanout, IRemotePosts remotePosts, IGroupDistributor groups, IObjectRecords records, IPollService polls, ILinkPreviews previews) { + _previews = previews; _records = records; _polls = polls; _groups = groups; @@ -148,6 +151,7 @@ namespace PrivaPub.Federation.Inbox.Handlers return; } await _records.Record(note, post, ObjectPath.Delivered, refetched, token); + await _previews.Wanted(post, token); if (parent != default) await DB.Default.Update().MatchID(parent.ID).Modify(b => b.Inc(p => p.RepliesCount, 1)).ExecuteAsync(token); diff --git a/PrivaPub/Federation/Inbox/RemotePosts.cs b/PrivaPub/Federation/Inbox/RemotePosts.cs index c578a46..ff2c184 100644 --- a/PrivaPub/Federation/Inbox/RemotePosts.cs +++ b/PrivaPub/Federation/Inbox/RemotePosts.cs @@ -1,6 +1,7 @@ using MongoDB.Driver; using MongoDB.Entities; +using PrivaPub.Domain.Content; using PrivaPub.Federation.Actors; using PrivaPub.Federation.Moderation; using PrivaPub.Federation.Objects; @@ -39,10 +40,12 @@ namespace PrivaPub.Federation.Inbox readonly IDomainBlocks _domainBlocks; readonly IJobQueue _queue; readonly IObjectRecords _records; + readonly ILinkPreviews _previews; public RemotePosts(DbEntities dbEntities, ILocalActorService localActors, IRemoteActorService remoteActors, IDomainBlocks domainBlocks, - IJobQueue queue, IObjectRecords records) + IJobQueue queue, IObjectRecords records, ILinkPreviews previews) { + _previews = previews; _records = records; _dbEntities = dbEntities; _localActors = localActors; @@ -143,6 +146,7 @@ namespace PrivaPub.Federation.Inbox return await _dbEntities.Posts.Match(p => p.ObjectURI == objectUri).ExecuteFirstAsync(token); } await _records.Record(note, post, ObjectPath.Fetched, refetched: false, token); + await _previews.Wanted(post, token); if (grandparent == default && !string.IsNullOrEmpty(note.InReplyTo) && depth < MaxDepth) await _queue.Enqueue(JobKind.FetchAncestors, JsonSerializer.Serialize(new AncestorsPayload(post.ID, depth + 1)), new Uri(note.InReplyTo).Host, diff --git a/PrivaPub/Infrastructure/Data/Indexes.cs b/PrivaPub/Infrastructure/Data/Indexes.cs index 9eb97f7..3182aff 100644 --- a/PrivaPub/Infrastructure/Data/Indexes.cs +++ b/PrivaPub/Infrastructure/Data/Indexes.cs @@ -116,6 +116,7 @@ namespace PrivaPub.Infrastructure.Data await DB.Default.Index().Key(r => r.PostId, KeyType.Ascending).Key(r => r.ActorURI, KeyType.Ascending).Key(r => r.Emoji, KeyType.Ascending) .Option(o => o.Unique = true).CreateAsync(token); await Plain(token, r => r.ActivityURI); + await Unique(p => p.Url, Builders.Filter.Type(p => p.Url, BsonType.String), token); } static async Task Unique(System.Linq.Expressions.Expression> key, FilterDefinition partial, diff --git a/PrivaPub/Infrastructure/Http/FederationHttp.cs b/PrivaPub/Infrastructure/Http/FederationHttp.cs index 76d23a6..1c42039 100644 --- a/PrivaPub/Infrastructure/Http/FederationHttp.cs +++ b/PrivaPub/Infrastructure/Http/FederationHttp.cs @@ -22,6 +22,7 @@ namespace PrivaPub.Infrastructure.Http bool IsAllowed(Uri target); Task GetJson(string url, string accept, Action sign, CancellationToken token); bool FailedTemporarily(string url); + Task<(Uri FinalUri, string Html)> GetPage(string url, CancellationToken token); Task Send(HttpRequestMessage request, CancellationToken token); Task<(byte[] Bytes, string ContentType)> GetMedia(string url, long maxBytes, CancellationToken token); } @@ -181,6 +182,66 @@ namespace PrivaPub.Infrastructure.Http } } + public async Task<(Uri FinalUri, string Html)> GetPage(string url, CancellationToken token) + { + if (!Uri.TryCreate(url, UriKind.Absolute, out var target) || !IsAllowed(target)) + return default; + using var timeout = CancellationTokenSource.CreateLinkedTokenSource(token); + timeout.CancelAfter(RequestTimeout); + try + { + for (var hop = 0; hop <= MaxRedirects; hop++) + { + using var request = new HttpRequestMessage(HttpMethod.Get, target); + request.Headers.Accept.ParseAdd("text/html, application/xhtml+xml"); + using var response = await _httpClientFactory.CreateClient(ClientName).SendAsync(request, HttpCompletionOption.ResponseHeadersRead, timeout.Token); + if (IsRedirect(response.StatusCode)) + { + var location = response.Headers.Location; + var next = location == default ? default : location.IsAbsoluteUri ? location : new Uri(target, location); + if (!IsAllowed(next)) + return default; + target = next; + continue; + } + var mediaType = response.Content.Headers.ContentType?.MediaType?.ToLowerInvariant(); + if (!response.IsSuccessStatusCode || mediaType is not ("text/html" or "application/xhtml+xml")) + return default; + var bytes = await ReadPrefix(response.Content, MaxPageBytes, timeout.Token); + var charset = response.Content.Headers.ContentType?.CharSet?.Trim('"'); + var encoding = System.Text.Encoding.UTF8; + try + { + if (!string.IsNullOrEmpty(charset)) + encoding = System.Text.Encoding.GetEncoding(charset); + } + catch (ArgumentException) + { + } + return (target, encoding.GetString(bytes)); + } + return default; + } + catch (Exception ex) when (ex is HttpRequestException or BlockedDestinationException or OperationCanceledException && !token.IsCancellationRequested) + { + _logger.LogInformation("Page {Url} refused: {Reason}", url, ex.Message); + return default; + } + } + + const int MaxPageBytes = 512 * 1024; + + static async Task ReadPrefix(HttpContent content, int limit, CancellationToken token) + { + await using var stream = await content.ReadAsStreamAsync(token); + var buffer = new byte[limit]; + var total = 0; + int read; + while (total < limit && (read = await stream.ReadAsync(buffer.AsMemory(total, limit - total), token)) > 0) + total += read; + return buffer[..total]; + } + public async Task Send(HttpRequestMessage request, CancellationToken token) { if (!IsAllowed(request.RequestUri)) diff --git a/PrivaPub/Infrastructure/Http/FederationOptions.cs b/PrivaPub/Infrastructure/Http/FederationOptions.cs index 96996d3..59ab336 100644 --- a/PrivaPub/Infrastructure/Http/FederationOptions.cs +++ b/PrivaPub/Infrastructure/Http/FederationOptions.cs @@ -6,5 +6,6 @@ namespace PrivaPub.Infrastructure.Http public bool AllowPlainHttp { get; set; } public bool SecureMode { get; set; } public bool AcceptAnyCertificate { get; set; } + public bool FetchLinkPreviews { get; set; } = true;//owner decision 1: the server reads linked pages of public posts } } diff --git a/PrivaPub/Middleware/SocialPubConfigurations.cs b/PrivaPub/Middleware/SocialPubConfigurations.cs index 9e32e93..0d81599 100644 --- a/PrivaPub/Middleware/SocialPubConfigurations.cs +++ b/PrivaPub/Middleware/SocialPubConfigurations.cs @@ -79,6 +79,9 @@ namespace PrivaPub.Middleware .AddSingleton() .AddSingleton() .AddSingleton() + .AddSingleton() + .AddSingleton(services => services.GetRequiredService()) + .AddSingleton(services => services.GetRequiredService()) .AddSingleton() .AddSingleton() .AddSingleton() diff --git a/PrivaPub/Models/Jobs/Job.cs b/PrivaPub/Models/Jobs/Job.cs index 078e236..f1fcebd 100644 --- a/PrivaPub/Models/Jobs/Job.cs +++ b/PrivaPub/Models/Jobs/Job.cs @@ -25,7 +25,8 @@ namespace PrivaPub.Models.Jobs FetchAncestors, DescribeInstance, PollRefresh, - PollClose + PollClose, + FetchPreview } public enum JobState diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index 87b70f5..2637e19 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -458,9 +458,11 @@ it, raw where it doesn't. - then advertise `api_versions.mastodon ≥ 7` (4.5.0). - **Emoji reactions** in all three inbound forms, plus outbound `EmojiReact`, exposed as `emoji_reactions` (done in v1.11.0; Pleroma's `/api/v1/pleroma/statuses/:id/reactions` endpoints and `pleroma:emoji_reaction` notifications). -- **Link cards:** +- **Link cards** (done in v1.12.0): - from the object, or from FEP-8967 `preview`, without fetching; - - fetching the page itself is the owner's decision (INTEROP §6.1). + - otherwise the server reads the page (owner decision 1): public posts only, 0–60 s after arrival, cached per address + for 7 days across the whole server, `Federation:FetchLinkPreviews` to switch it off. Checked live against a + GoToSocial profile page. - **Media:** - video playback through the proxy (Range requests, HLS playlist rewriting, the poster), with a `video` card; - audio attachments; diff --git a/tools/pasture/interop.sh b/tools/pasture/interop.sh index 43864cc..af9e5d1 100755 --- a/tools/pasture/interop.sh +++ b/tools/pasture/interop.sh @@ -114,6 +114,10 @@ gts_poll_on_pp=$(curl -s -H "$PH" "$P/api/v1/timelines/home" | j "print(next(s[' curl -s -o /dev/null -X POST -H "$PH" "$P/api/v1/polls/$gts_poll_on_pp/votes" -d 'choices[]=0' until_true 20 '[ "$(gcurl -s -H "$GH" "$G/api/v1/statuses/$gts_poll" | j "print(d[\"poll\"][\"options\"][0][\"votes_count\"])")" = "1" ]' && ok "alice's vote counts on GoToSocial" || ko "vote not counted on GoToSocial" +echo "link previews" +linked=$(curl -s -X POST -H "$PH" $P/api/v1/statuses --data-urlencode 'status=have a look https://gts.test/@gtsuser' -d 'visibility=public' | j "print(d['id'])") +until_true 45 '[ -n "$(curl -s -H "$PH" "$P/api/v1/statuses/$linked" | j "print((d[\"card\"] or {}).get(\"title\") or \"\")")" ]' && ok "the server builds a card for a linked page" || ko "no card for the linked page" + echo "edits and deletes" curl -s -o /dev/null -X PUT -H "$PH" $P/api/v1/statuses/$pp_post -d 'status=Hello from PrivaPub, edited' until_true 20 'gcurl -s -H "$GH" "$G/api/v1/statuses/$pp_on_gts" | grep -q "edited"' && ok "alice's edit reaches GoToSocial" || ko "edit not applied"