using AngleSharp.Html.Parser; using Microsoft.Extensions.Options; using MongoDB.Entities; using PrivaPub.Infrastructure.Statistics; using PrivaPub.Models.Statistics; using PrivaPub.Federation.Actors; using PrivaPub.Federation.Objects; using PrivaPub.Infrastructure.Http; using PrivaPub.Infrastructure.Jobs; using PrivaPub.Models.Jobs; using PrivaPub.Models.Post; using PrivaPub.StaticServices; using System.Security.Cryptography; using PostEntity = PrivaPub.Models.Post.Post; namespace PrivaPub.Domain.Content { public class LinkPreview : Entity { public string Url { get; set; } public string Title { get; set; } public string Description { get; set; } public string ImageURL { get; set; } public string SiteName { get; set; } public bool Failed { get; set; } public DateTime FetchedAt { get; set; } = DateTime.UtcNow; } public interface ILinkPreviews { Task Wanted(PostEntity post, CancellationToken token); } public class LinkPreviews : ILinkPreviews, IJobHandler { const int MaxJitterSeconds = 60; static readonly TimeSpan Freshness = TimeSpan.FromDays(7); static readonly HtmlParser Parser = new(); readonly DbEntities _dbEntities; readonly ILocalActorService _localActors; readonly IFederationHttp _http; readonly IJobQueue _queue; readonly IOptionsMonitor _options; readonly IInteractionLedger _ledger; public LinkPreviews(DbEntities dbEntities, ILocalActorService localActors, IFederationHttp http, IJobQueue queue, IOptionsMonitor options, IInteractionLedger ledger = default) { _dbEntities = dbEntities; _localActors = localActors; _http = http; _queue = queue; _options = options; _ledger = ledger; } public JobKind Kind => JobKind.FetchPreview; public int Concurrency => 2; public int MaxAttempts => 1; public int PerHostLimit => 1; public async Task Wanted(PostEntity post, CancellationToken token) { if (!_options.CurrentValue.FetchLinkPreviews || post.Visibility is not (PostVisibility.Public or PostVisibility.Unlisted) || post.IsLocalOnly || post.ReblogOfPostId != default || post.Link?.Title != default || post.Link == default && post.Media.Count > 0) return; var url = post.Link?.Href ?? FirstLink(post, _localActors.BaseAddress); if (url == default) return; await _queue.EnqueueMany(new[] { new Job { Kind = JobKind.FetchPreview, Payload = post.ID, Host = new Uri(url).Host.ToLowerInvariant(), DedupeKey = $"preview|{post.ID}", RunAt = DateTime.UtcNow.AddSeconds(RandomNumberGenerator.GetInt32(0, MaxJitterSeconds + 1)) } }, token); } public async Task Handle(Job job, CancellationToken token) { var post = await _dbEntities.Posts.MatchID(job.Payload).ExecuteFirstAsync(token); if (post == default || post.DeletedAt.HasValue || post.Link?.Title != default) return JobOutcome.Done; var url = post.Link?.Href ?? FirstLink(post, _localActors.BaseAddress); if (url == default) return JobOutcome.Done; var preview = await DB.Default.Find().Match(p => p.Url == url).ExecuteFirstAsync(token); if (preview == default || preview.FetchedAt < DateTime.UtcNow - Freshness) preview = await Fetch(url, token); if (preview == default || preview.Failed) return JobOutcome.Done; var link = post.Link ?? new PostLink { Href = url }; link.Title ??= preview.Title; link.Description ??= preview.Description; link.ImageURL ??= preview.ImageURL; await DB.Default.Update().MatchID(post.ID).Modify(p => p.Link, link).ExecuteAsync(token); return JobOutcome.Done; } async Task Fetch(string url, CancellationToken token) { var started = System.Diagnostics.Stopwatch.GetTimestamp(); var (finalUri, html) = await _http.GetPage(url, token); var preview = html == default ? new LinkPreview { Url = url, Failed = true } : Read(url, finalUri, html); _ledger?.Record(new InteractionEvent { Channel = Interactions.Preview, Host = Interactions.HostOf(url), Outcome = html == default ? Interactions.Failed : Interactions.Ok, Reason = html == default ? default : preview.Title == default && preview.ImageURL == default ? "no-card" : "card", LatencyMs = (int)System.Diagnostics.Stopwatch.GetElapsedTime(started).TotalMilliseconds, Bytes = html == default ? default : System.Text.Encoding.UTF8.GetByteCount(html), Redirects = finalUri == default || finalUri.AbsoluteUri == url ? 0 : 1 }); await DB.Default.Update() .Match(p => p.Url == url) .Modify(p => p.Title, preview.Title) .Modify(p => p.Description, preview.Description) .Modify(p => p.ImageURL, preview.ImageURL) .Modify(p => p.SiteName, preview.SiteName) .Modify(p => p.Failed, preview.Failed) .Modify(p => p.FetchedAt, DateTime.UtcNow) .Option(o => o.IsUpsert = true) .ExecuteAsync(token); return preview; } public static LinkPreview Read(string url, Uri finalUri, string html) { var document = Parser.ParseDocument(html); string Meta(params string[] names) => names .Select(name => document.QuerySelector($"meta[property='{name}'], meta[name='{name}']")?.GetAttribute("content")) .FirstOrDefault(value => !string.IsNullOrWhiteSpace(value)); var image = Meta("og:image:secure_url", "og:image", "og:image:url", "twitter:image", "twitter:image:src"); var imageUri = image != default && Uri.TryCreate(finalUri, image, out var resolved) && resolved.Scheme is "https" or "http" ? resolved.AbsoluteUri : default; var preview = new LinkPreview { Url = url, Title = ObjectShapes.Text(Meta("og:title", "twitter:title") ?? document.Title, 300), Description = ObjectShapes.Text(Meta("og:description", "twitter:description", "description"), 1000), SiteName = ObjectShapes.Text(Meta("og:site_name"), 200), ImageURL = imageUri }; preview.Failed = preview.Title == default && preview.Description == default && preview.ImageURL == default; return preview; } public static string FirstLink(PostEntity post, string ownBase) { if (string.IsNullOrEmpty(post.ContentHtml)) return default; var document = Parser.ParseDocument(post.ContentHtml); return document.QuerySelectorAll("a[href]") .Where(a => !(a.ClassList.Contains("mention") || a.ClassList.Contains("hashtag") || a.GetAttribute("rel")?.Contains("tag") == true || a.Closest(".quote-inline") != default)) .Select(a => a.GetAttribute("href")) .FirstOrDefault(href => Uri.TryCreate(href, UriKind.Absolute, out var uri) && uri.Scheme is "https" or "http" && href != post.Url && href != post.ObjectURI && !href.StartsWith(ownBase + "/", StringComparison.OrdinalIgnoreCase)); } } }