P6: link previews read by the server, as the owner decided

- A public post that links to a page gets a preview card: from the post's own data first (FEP-8967 Link preview, the
  object's image, title and summary); otherwise the server reads the page once, 0-60 s after the post arrives, never
  when someone reads it. OpenGraph and Twitter tags give title, description and image; one LinkPreview per address
  is cached for 7 days and shared by the whole server, so a fetch never points at a persona.
- Lemmy link posts, which carry no title or description, get them filled in.
- The page fetch uses the guarded client (public addresses, three redirects, HTML only, first 512 KB).
- Federation:FetchLinkPreviews switches page fetching off.
- Local public posts get cards too.

Checked live: a link to a GoToSocial profile page becomes a card with its title, description and proxied image.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012CzABvBkbcFqoHdmi8b9WB

PrivaPub/Domain/Content/LinkPreviews.cs

@@ -0,0 +1,155 @@
+using AngleSharp.Html.Parser;
+
+using Microsoft.Extensions.Options;
+
+using MongoDB.Entities;
+
+using PrivaPub.Federation.Actors;
+using PrivaPub.Federation.Objects;
+using PrivaPub.Infrastructure.Http;
+using PrivaPub.Infrastructure.Jobs;
+using PrivaPub.Models.Jobs;
+using PrivaPub.Models.Post;
+using PrivaPub.StaticServices;
+
+using System.Security.Cryptography;
+
+using PostEntity = PrivaPub.Models.Post.Post;
+
+namespace PrivaPub.Domain.Content
+{
+	public class LinkPreview : Entity
+	{
+		public string Url { get; set; }
+		public string Title { get; set; }
+		public string Description { get; set; }
+		public string ImageURL { get; set; }
+		public string SiteName { get; set; }
+		public bool Failed { get; set; }
+		public DateTime FetchedAt { get; set; } = DateTime.UtcNow;
+	}
+
+	public interface ILinkPreviews
+	{
+		Task Wanted(PostEntity post, CancellationToken token);
+	}
+
+	public class LinkPreviews : ILinkPreviews, IJobHandler
+	{
+		const int MaxJitterSeconds = 60;
+		static readonly TimeSpan Freshness = TimeSpan.FromDays(7);
+		static readonly HtmlParser Parser = new();
+
+		readonly DbEntities _dbEntities;
+		readonly ILocalActorService _localActors;
+		readonly IFederationHttp _http;
+		readonly IJobQueue _queue;
+		readonly IOptionsMonitor<FederationOptions> _options;
+
+		public LinkPreviews(DbEntities dbEntities, ILocalActorService localActors, IFederationHttp http, IJobQueue queue,
+			IOptionsMonitor<FederationOptions> options)
+		{
+			_dbEntities = dbEntities;
+			_localActors = localActors;
+			_http = http;
+			_queue = queue;
+			_options = options;
+		}
+
+		public JobKind Kind => JobKind.FetchPreview;
+		public int Concurrency => 2;
+		public int MaxAttempts => 1;
+		public int PerHostLimit => 1;
+
+		public async Task Wanted(PostEntity post, CancellationToken token)
+		{
+			if (!_options.CurrentValue.FetchLinkPreviews || post.Visibility is not (PostVisibility.Public or PostVisibility.Unlisted) || post.IsLocalOnly
+				|| post.ReblogOfPostId != default || post.Link?.Title != default || post.Link == default && post.Media.Count > 0)
+				return;
+			var url = post.Link?.Href ?? FirstLink(post, _localActors.BaseAddress);
+			if (url == default)
+				return;
+			await _queue.EnqueueMany(new[]
+			{
+				new Job
+				{
+					Kind = JobKind.FetchPreview, Payload = post.ID, Host = new Uri(url).Host.ToLowerInvariant(), DedupeKey = $"preview|{post.ID}",
+					RunAt = DateTime.UtcNow.AddSeconds(RandomNumberGenerator.GetInt32(0, MaxJitterSeconds + 1))
+				}
+			}, token);
+		}
+
+		public async Task<JobOutcome> Handle(Job job, CancellationToken token)
+		{
+			var post = await _dbEntities.Posts.MatchID(job.Payload).ExecuteFirstAsync(token);
+			if (post == default || post.DeletedAt.HasValue || post.Link?.Title != default)
+				return JobOutcome.Done;
+			var url = post.Link?.Href ?? FirstLink(post, _localActors.BaseAddress);
+			if (url == default)
+				return JobOutcome.Done;
+
+			var preview = await DB.Default.Find<LinkPreview>().Match(p => p.Url == url).ExecuteFirstAsync(token);
+			if (preview == default || preview.FetchedAt < DateTime.UtcNow - Freshness)
+				preview = await Fetch(url, token);
+			if (preview == default || preview.Failed)
+				return JobOutcome.Done;
+
+			var link = post.Link ?? new PostLink { Href = url };
+			link.Title ??= preview.Title;
+			link.Description ??= preview.Description;
+			link.ImageURL ??= preview.ImageURL;
+			await DB.Default.Update<PostEntity>().MatchID(post.ID).Modify(p => p.Link, link).ExecuteAsync(token);
+			return JobOutcome.Done;
+		}
+
+		async Task<LinkPreview> Fetch(string url, CancellationToken token)
+		{
+			var (finalUri, html) = await _http.GetPage(url, token);
+			var preview = html == default ? new LinkPreview { Url = url, Failed = true } : Read(url, finalUri, html);
+			await DB.Default.Update<LinkPreview>()
+				.Match(p => p.Url == url)
+				.Modify(p => p.Title, preview.Title)
+				.Modify(p => p.Description, preview.Description)
+				.Modify(p => p.ImageURL, preview.ImageURL)
+				.Modify(p => p.SiteName, preview.SiteName)
+				.Modify(p => p.Failed, preview.Failed)
+				.Modify(p => p.FetchedAt, DateTime.UtcNow)
+				.Option(o => o.IsUpsert = true)
+				.ExecuteAsync(token);
+			return preview;
+		}
+
+		public static LinkPreview Read(string url, Uri finalUri, string html)
+		{
+			var document = Parser.ParseDocument(html);
+			string Meta(params string[] names) => names
+				.Select(name => document.QuerySelector($"meta[property='{name}'], meta[name='{name}']")?.GetAttribute("content"))
+				.FirstOrDefault(value => !string.IsNullOrWhiteSpace(value));
+			var image = Meta("og:image:secure_url", "og:image", "og:image:url", "twitter:image", "twitter:image:src");
+			var imageUri = image != default && Uri.TryCreate(finalUri, image, out var resolved) && resolved.Scheme is "https" or "http" ? resolved.AbsoluteUri : default;
+			var preview = new LinkPreview
+			{
+				Url = url,
+				Title = ObjectShapes.Text(Meta("og:title", "twitter:title") ?? document.Title, 300),
+				Description = ObjectShapes.Text(Meta("og:description", "twitter:description", "description"), 1000),
+				SiteName = ObjectShapes.Text(Meta("og:site_name"), 200),
+				ImageURL = imageUri
+			};
+			preview.Failed = preview.Title == default && preview.Description == default && preview.ImageURL == default;
+			return preview;
+		}
+
+		public static string FirstLink(PostEntity post, string ownBase)
+		{
+			if (string.IsNullOrEmpty(post.ContentHtml))
+				return default;
+			var document = Parser.ParseDocument(post.ContentHtml);
+			return document.QuerySelectorAll("a[href]")
+				.Where(a => !(a.ClassList.Contains("mention") || a.ClassList.Contains("hashtag") || a.GetAttribute("rel")?.Contains("tag") == true
+					|| a.Closest(".quote-inline") != default))
+				.Select(a => a.GetAttribute("href"))
+				.FirstOrDefault(href => Uri.TryCreate(href, UriKind.Absolute, out var uri) && uri.Scheme is "https" or "http"
+					&& href != post.Url && href != post.ObjectURI && !href.StartsWith(ownBase + "/", StringComparison.OrdinalIgnoreCase));
+		}
+	}
+}

PrivaPub/Domain/Statuses/StatusService.cs

@@ -71,10 +71,13 @@ namespace PrivaPub.Domain.Statuses
 		readonly IMediaService _media;
 		readonly IGroupDistributor _groups;
 		readonly IPollService _polls;
+		readonly ILinkPreviews _previews;
 
 		public StatusService(DbEntities dbEntities, ILocalActorService localActors, IRemoteActorService remoteActors, IDeliveryService delivery,
-			IContentRenderer content, IOutboxPublisher outbox, IFanout fanout, IMediaService media, IGroupDistributor groups, IPollService polls)
+			IContentRenderer content, IOutboxPublisher outbox, IFanout fanout, IMediaService media, IGroupDistributor groups, IPollService polls,
+			ILinkPreviews previews)
 		{
+			_previews = previews;
 			_polls = polls;
 			_media = media;
 			_groups = groups;
@@ -207,6 +210,7 @@ namespace PrivaPub.Domain.Statuses
 				await DB.Default.Update<PostEntity>().MatchID(parent.ID).Modify(b => b.Inc(p => p.RepliesCount, 1)).ExecuteAsync(token);
 			await _fanout.Distribute(post, token);
 			await _polls.Scheduled(post, token);
+			await _previews.Wanted(post, token);
 			if (create != default)
 				await _outbox.Publish(author, post, create, token);
 			if (create != default && group is { IsCircle: false } && visibility is PostVisibility.Public or PostVisibility.Unlisted)