P6: link previews read by the server, as the owner decided

- A public post that links to a page gets a preview card: from the post's own data first (FEP-8967 Link preview, the
  object's image, title and summary); otherwise the server reads the page once, 0-60 s after the post arrives, never
  when someone reads it. OpenGraph and Twitter tags give title, description and image; one LinkPreview per address
  is cached for 7 days and shared by the whole server, so a fetch never points at a persona.
- Lemmy link posts, which carry no title or description, get them filled in.
- The page fetch uses the guarded client (public addresses, three redirects, HTML only, first 512 KB).
- Federation:FetchLinkPreviews switches page fetching off.
- Local public posts get cards too.

Checked live: a link to a GoToSocial profile page becomes a card with its title, description and proxied image.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012CzABvBkbcFqoHdmi8b9WB

PrivaPub/Api/Mastodon/Mappers/MastodonMapper.cs

@@ -389,11 +389,9 @@ namespace PrivaPub.Api.Mastodon.Mappers
 
 		PreviewCard Card(PostEntity post)
 		{
-			if (!post.IsFederatedCopy)
-				return default;
 			if (post.Link is { } link)
 				return Card(link.Href, link.Title ?? post.Title, link.Description ?? (post.Media.Count == 0 ? post.Excerpt : default), link.ImageURL);
-			if (post.ObjectType is "Article" or "Event" or "Video" or "Audio" && post.Url != default && post.Media.Count == 0)
+			if (post.IsFederatedCopy && post.ObjectType is "Article" or "Event" or "Video" or "Audio" && post.Url != default && post.Media.Count == 0)
 				return Card(post.Url, post.Title, post.Excerpt, post.CoverURL);
 			return default;
 		}

PrivaPub/Domain/Content/LinkPreviews.cs

@@ -0,0 +1,155 @@
+using AngleSharp.Html.Parser;
+
+using Microsoft.Extensions.Options;
+
+using MongoDB.Entities;
+
+using PrivaPub.Federation.Actors;
+using PrivaPub.Federation.Objects;
+using PrivaPub.Infrastructure.Http;
+using PrivaPub.Infrastructure.Jobs;
+using PrivaPub.Models.Jobs;
+using PrivaPub.Models.Post;
+using PrivaPub.StaticServices;
+
+using System.Security.Cryptography;
+
+using PostEntity = PrivaPub.Models.Post.Post;
+
+namespace PrivaPub.Domain.Content
+{
+	public class LinkPreview : Entity
+	{
+		public string Url { get; set; }
+		public string Title { get; set; }
+		public string Description { get; set; }
+		public string ImageURL { get; set; }
+		public string SiteName { get; set; }
+		public bool Failed { get; set; }
+		public DateTime FetchedAt { get; set; } = DateTime.UtcNow;
+	}
+
+	public interface ILinkPreviews
+	{
+		Task Wanted(PostEntity post, CancellationToken token);
+	}
+
+	public class LinkPreviews : ILinkPreviews, IJobHandler
+	{
+		const int MaxJitterSeconds = 60;
+		static readonly TimeSpan Freshness = TimeSpan.FromDays(7);
+		static readonly HtmlParser Parser = new();
+
+		readonly DbEntities _dbEntities;
+		readonly ILocalActorService _localActors;
+		readonly IFederationHttp _http;
+		readonly IJobQueue _queue;
+		readonly IOptionsMonitor<FederationOptions> _options;
+
+		public LinkPreviews(DbEntities dbEntities, ILocalActorService localActors, IFederationHttp http, IJobQueue queue,
+			IOptionsMonitor<FederationOptions> options)
+		{
+			_dbEntities = dbEntities;
+			_localActors = localActors;
+			_http = http;
+			_queue = queue;
+			_options = options;
+		}
+
+		public JobKind Kind => JobKind.FetchPreview;
+		public int Concurrency => 2;
+		public int MaxAttempts => 1;
+		public int PerHostLimit => 1;
+
+		public async Task Wanted(PostEntity post, CancellationToken token)
+		{
+			if (!_options.CurrentValue.FetchLinkPreviews || post.Visibility is not (PostVisibility.Public or PostVisibility.Unlisted) || post.IsLocalOnly
+				|| post.ReblogOfPostId != default || post.Link?.Title != default || post.Link == default && post.Media.Count > 0)
+				return;
+			var url = post.Link?.Href ?? FirstLink(post, _localActors.BaseAddress);
+			if (url == default)
+				return;
+			await _queue.EnqueueMany(new[]
+			{
+				new Job
+				{
+					Kind = JobKind.FetchPreview, Payload = post.ID, Host = new Uri(url).Host.ToLowerInvariant(), DedupeKey = $"preview|{post.ID}",
+					RunAt = DateTime.UtcNow.AddSeconds(RandomNumberGenerator.GetInt32(0, MaxJitterSeconds + 1))
+				}
+			}, token);
+		}
+
+		public async Task<JobOutcome> Handle(Job job, CancellationToken token)
+		{
+			var post = await _dbEntities.Posts.MatchID(job.Payload).ExecuteFirstAsync(token);
+			if (post == default || post.DeletedAt.HasValue || post.Link?.Title != default)
+				return JobOutcome.Done;
+			var url = post.Link?.Href ?? FirstLink(post, _localActors.BaseAddress);
+			if (url == default)
+				return JobOutcome.Done;
+
+			var preview = await DB.Default.Find<LinkPreview>().Match(p => p.Url == url).ExecuteFirstAsync(token);
+			if (preview == default || preview.FetchedAt < DateTime.UtcNow - Freshness)
+				preview = await Fetch(url, token);
+			if (preview == default || preview.Failed)
+				return JobOutcome.Done;
+
+			var link = post.Link ?? new PostLink { Href = url };
+			link.Title ??= preview.Title;
+			link.Description ??= preview.Description;
+			link.ImageURL ??= preview.ImageURL;
+			await DB.Default.Update<PostEntity>().MatchID(post.ID).Modify(p => p.Link, link).ExecuteAsync(token);
+			return JobOutcome.Done;
+		}
+
+		async Task<LinkPreview> Fetch(string url, CancellationToken token)
+		{
+			var (finalUri, html) = await _http.GetPage(url, token);
+			var preview = html == default ? new LinkPreview { Url = url, Failed = true } : Read(url, finalUri, html);
+			await DB.Default.Update<LinkPreview>()
+				.Match(p => p.Url == url)
+				.Modify(p => p.Title, preview.Title)
+				.Modify(p => p.Description, preview.Description)
+				.Modify(p => p.ImageURL, preview.ImageURL)
+				.Modify(p => p.SiteName, preview.SiteName)
+				.Modify(p => p.Failed, preview.Failed)
+				.Modify(p => p.FetchedAt, DateTime.UtcNow)
+				.Option(o => o.IsUpsert = true)
+				.ExecuteAsync(token);
+			return preview;
+		}
+
+		public static LinkPreview Read(string url, Uri finalUri, string html)
+		{
+			var document = Parser.ParseDocument(html);
+			string Meta(params string[] names) => names
+				.Select(name => document.QuerySelector($"meta[property='{name}'], meta[name='{name}']")?.GetAttribute("content"))
+				.FirstOrDefault(value => !string.IsNullOrWhiteSpace(value));
+			var image = Meta("og:image:secure_url", "og:image", "og:image:url", "twitter:image", "twitter:image:src");
+			var imageUri = image != default && Uri.TryCreate(finalUri, image, out var resolved) && resolved.Scheme is "https" or "http" ? resolved.AbsoluteUri : default;
+			var preview = new LinkPreview
+			{
+				Url = url,
+				Title = ObjectShapes.Text(Meta("og:title", "twitter:title") ?? document.Title, 300),
+				Description = ObjectShapes.Text(Meta("og:description", "twitter:description", "description"), 1000),
+				SiteName = ObjectShapes.Text(Meta("og:site_name"), 200),
+				ImageURL = imageUri
+			};
+			preview.Failed = preview.Title == default && preview.Description == default && preview.ImageURL == default;
+			return preview;
+		}
+
+		public static string FirstLink(PostEntity post, string ownBase)
+		{
+			if (string.IsNullOrEmpty(post.ContentHtml))
+				return default;
+			var document = Parser.ParseDocument(post.ContentHtml);
+			return document.QuerySelectorAll("a[href]")
+				.Where(a => !(a.ClassList.Contains("mention") || a.ClassList.Contains("hashtag") || a.GetAttribute("rel")?.Contains("tag") == true
+					|| a.Closest(".quote-inline") != default))
+				.Select(a => a.GetAttribute("href"))
+				.FirstOrDefault(href => Uri.TryCreate(href, UriKind.Absolute, out var uri) && uri.Scheme is "https" or "http"
+					&& href != post.Url && href != post.ObjectURI && !href.StartsWith(ownBase + "/", StringComparison.OrdinalIgnoreCase));
+		}
+	}
+}

PrivaPub/Domain/Statuses/StatusService.cs

@@ -71,10 +71,13 @@ namespace PrivaPub.Domain.Statuses
 		readonly IMediaService _media;
 		readonly IGroupDistributor _groups;
 		readonly IPollService _polls;
+		readonly ILinkPreviews _previews;
 
 		public StatusService(DbEntities dbEntities, ILocalActorService localActors, IRemoteActorService remoteActors, IDeliveryService delivery,
-			IContentRenderer content, IOutboxPublisher outbox, IFanout fanout, IMediaService media, IGroupDistributor groups, IPollService polls)
+			IContentRenderer content, IOutboxPublisher outbox, IFanout fanout, IMediaService media, IGroupDistributor groups, IPollService polls,
+			ILinkPreviews previews)
 		{
+			_previews = previews;
 			_polls = polls;
 			_media = media;
 			_groups = groups;
@@ -207,6 +210,7 @@ namespace PrivaPub.Domain.Statuses
 				await DB.Default.Update<PostEntity>().MatchID(parent.ID).Modify(b => b.Inc(p => p.RepliesCount, 1)).ExecuteAsync(token);
 			await _fanout.Distribute(post, token);
 			await _polls.Scheduled(post, token);
+			await _previews.Wanted(post, token);
 			if (create != default)
 				await _outbox.Publish(author, post, create, token);
 			if (create != default && group is { IsCircle: false } && visibility is PostVisibility.Public or PostVisibility.Unlisted)

PrivaPub/Federation/Inbox/Handlers/CreateHandler.cs

@@ -4,6 +4,7 @@ using MongoDB.Entities;
 using PrivaPub.Domain.Timelines;
 using PrivaPub.Federation.Actors;
 using PrivaPub.Federation.Moderation;
+using PrivaPub.Domain.Content;
 using PrivaPub.Domain.Statuses;
 using PrivaPub.Federation.Objects;
 using PrivaPub.Federation.Outbox;
@@ -37,10 +38,12 @@ namespace PrivaPub.Federation.Inbox.Handlers
 		readonly IGroupDistributor _groups;
 		readonly IObjectRecords _records;
 		readonly IPollService _polls;
+		readonly ILinkPreviews _previews;
 
 		public CreateHandler(DbEntities dbEntities, ILocalActorService localActors, IRemoteActorService remoteActors, IDeliveryService delivery,
-			IDomainBlocks domainBlocks, IFanout fanout, IRemotePosts remotePosts, IGroupDistributor groups, IObjectRecords records, IPollService polls)
+			IDomainBlocks domainBlocks, IFanout fanout, IRemotePosts remotePosts, IGroupDistributor groups, IObjectRecords records, IPollService polls, ILinkPreviews previews)
 		{
+			_previews = previews;
 			_records = records;
 			_polls = polls;
 			_groups = groups;
@@ -148,6 +151,7 @@ namespace PrivaPub.Federation.Inbox.Handlers
 				return;
 			}
 			await _records.Record(note, post, ObjectPath.Delivered, refetched, token);
+			await _previews.Wanted(post, token);
 
 			if (parent != default)
 				await DB.Default.Update<PostEntity>().MatchID(parent.ID).Modify(b => b.Inc(p => p.RepliesCount, 1)).ExecuteAsync(token);

PrivaPub/Federation/Inbox/RemotePosts.cs

@@ -1,6 +1,7 @@
 using MongoDB.Driver;
 using MongoDB.Entities;
 
+using PrivaPub.Domain.Content;
 using PrivaPub.Federation.Actors;
 using PrivaPub.Federation.Moderation;
 using PrivaPub.Federation.Objects;
@@ -39,10 +40,12 @@ namespace PrivaPub.Federation.Inbox
 		readonly IDomainBlocks _domainBlocks;
 		readonly IJobQueue _queue;
 		readonly IObjectRecords _records;
+		readonly ILinkPreviews _previews;
 
 		public RemotePosts(DbEntities dbEntities, ILocalActorService localActors, IRemoteActorService remoteActors, IDomainBlocks domainBlocks,
-			IJobQueue queue, IObjectRecords records)
+			IJobQueue queue, IObjectRecords records, ILinkPreviews previews)
 		{
+			_previews = previews;
 			_records = records;
 			_dbEntities = dbEntities;
 			_localActors = localActors;
@@ -143,6 +146,7 @@ namespace PrivaPub.Federation.Inbox
 				return await _dbEntities.Posts.Match(p => p.ObjectURI == objectUri).ExecuteFirstAsync(token);
 			}
 			await _records.Record(note, post, ObjectPath.Fetched, refetched: false, token);
+			await _previews.Wanted(post, token);
 
 			if (grandparent == default && !string.IsNullOrEmpty(note.InReplyTo) && depth < MaxDepth)
 				await _queue.Enqueue(JobKind.FetchAncestors, JsonSerializer.Serialize(new AncestorsPayload(post.ID, depth + 1)), new Uri(note.InReplyTo).Host,

PrivaPub/Infrastructure/Data/Indexes.cs

@@ -116,6 +116,7 @@ namespace PrivaPub.Infrastructure.Data
 			await DB.Default.Index<Reaction>().Key(r => r.PostId, KeyType.Ascending).Key(r => r.ActorURI, KeyType.Ascending).Key(r => r.Emoji, KeyType.Ascending)
 				.Option(o => o.Unique = true).CreateAsync(token);
 			await Plain<Reaction>(token, r => r.ActivityURI);
+			await Unique<Domain.Content.LinkPreview>(p => p.Url, Builders<Domain.Content.LinkPreview>.Filter.Type(p => p.Url, BsonType.String), token);
 		}
 
 		static async Task Unique<T>(System.Linq.Expressions.Expression<Func<T, object>> key, FilterDefinition<T> partial,

PrivaPub/Infrastructure/Http/FederationHttp.cs

@@ -22,6 +22,7 @@ namespace PrivaPub.Infrastructure.Http
 		bool IsAllowed(Uri target);
 		Task<FetchedJson> GetJson(string url, string accept, Action<HttpRequestMessage> sign, CancellationToken token);
 		bool FailedTemporarily(string url);
+		Task<(Uri FinalUri, string Html)> GetPage(string url, CancellationToken token);
 		Task<HttpResponseMessage> Send(HttpRequestMessage request, CancellationToken token);
 		Task<(byte[] Bytes, string ContentType)> GetMedia(string url, long maxBytes, CancellationToken token);
 	}
@@ -181,6 +182,66 @@ namespace PrivaPub.Infrastructure.Http
 			}
 		}
 
+		public async Task<(Uri FinalUri, string Html)> GetPage(string url, CancellationToken token)
+		{
+			if (!Uri.TryCreate(url, UriKind.Absolute, out var target) || !IsAllowed(target))
+				return default;
+			using var timeout = CancellationTokenSource.CreateLinkedTokenSource(token);
+			timeout.CancelAfter(RequestTimeout);
+			try
+			{
+				for (var hop = 0; hop <= MaxRedirects; hop++)
+				{
+					using var request = new HttpRequestMessage(HttpMethod.Get, target);
+					request.Headers.Accept.ParseAdd("text/html, application/xhtml+xml");
+					using var response = await _httpClientFactory.CreateClient(ClientName).SendAsync(request, HttpCompletionOption.ResponseHeadersRead, timeout.Token);
+					if (IsRedirect(response.StatusCode))
+					{
+						var location = response.Headers.Location;
+						var next = location == default ? default : location.IsAbsoluteUri ? location : new Uri(target, location);
+						if (!IsAllowed(next))
+							return default;
+						target = next;
+						continue;
+					}
+					var mediaType = response.Content.Headers.ContentType?.MediaType?.ToLowerInvariant();
+					if (!response.IsSuccessStatusCode || mediaType is not ("text/html" or "application/xhtml+xml"))
+						return default;
+					var bytes = await ReadPrefix(response.Content, MaxPageBytes, timeout.Token);
+					var charset = response.Content.Headers.ContentType?.CharSet?.Trim('"');
+					var encoding = System.Text.Encoding.UTF8;
+					try
+					{
+						if (!string.IsNullOrEmpty(charset))
+							encoding = System.Text.Encoding.GetEncoding(charset);
+					}
+					catch (ArgumentException)
+					{
+					}
+					return (target, encoding.GetString(bytes));
+				}
+				return default;
+			}
+			catch (Exception ex) when (ex is HttpRequestException or BlockedDestinationException or OperationCanceledException && !token.IsCancellationRequested)
+			{
+				_logger.LogInformation("Page {Url} refused: {Reason}", url, ex.Message);
+				return default;
+			}
+		}
+
+		const int MaxPageBytes = 512 * 1024;
+
+		static async Task<byte[]> ReadPrefix(HttpContent content, int limit, CancellationToken token)
+		{
+			await using var stream = await content.ReadAsStreamAsync(token);
+			var buffer = new byte[limit];
+			var total = 0;
+			int read;
+			while (total < limit && (read = await stream.ReadAsync(buffer.AsMemory(total, limit - total), token)) > 0)
+				total += read;
+			return buffer[..total];
+		}
+
 		public async Task<HttpResponseMessage> Send(HttpRequestMessage request, CancellationToken token)
 		{
 			if (!IsAllowed(request.RequestUri))

PrivaPub/Infrastructure/Http/FederationOptions.cs

@@ -6,5 +6,6 @@ namespace PrivaPub.Infrastructure.Http
 		public bool AllowPlainHttp { get; set; }
 		public bool SecureMode { get; set; }
 		public bool AcceptAnyCertificate { get; set; }
+		public bool FetchLinkPreviews { get; set; } = true;//owner decision 1: the server reads linked pages of public posts
 	}
 }

PrivaPub/Middleware/SocialPubConfigurations.cs

@@ -79,6 +79,9 @@ namespace PrivaPub.Middleware
 				.AddSingleton<IActivityHandler, DislikeHandler>()
 				.AddSingleton<IActivityHandler, EmojiReactHandler>()
 				.AddSingleton<IReactions, Reactions>()
+				.AddSingleton<LinkPreviews>()
+				.AddSingleton<ILinkPreviews>(services => services.GetRequiredService<LinkPreviews>())
+				.AddSingleton<IJobHandler>(services => services.GetRequiredService<LinkPreviews>())
 				.AddSingleton<IActivityHandler, JoinHandler>()
 				.AddSingleton<IActivityHandler, AnnounceHandler>()
 				.AddSingleton<IActivityHandler, FlagHandler>()

PrivaPub/Models/Jobs/Job.cs

@@ -25,7 +25,8 @@ namespace PrivaPub.Models.Jobs
 		FetchAncestors,
 		DescribeInstance,
 		PollRefresh,
-		PollClose
+		PollClose,
+		FetchPreview
 	}
 
 	public enum JobState