SocialPub/PrivaPub/Domain/Content/LinkPreviews.cs

using AngleSharp.Html.Parser;

using Microsoft.Extensions.Options;

using MongoDB.Entities;

using PrivaPub.Infrastructure.Statistics;
using PrivaPub.Models.Statistics;
using PrivaPub.Domain.Privacy;
using PrivaPub.Federation.Actors;
using PrivaPub.Federation.Objects;
using PrivaPub.Infrastructure.Http;
using PrivaPub.Infrastructure.Jobs;
using PrivaPub.Models.Jobs;
using PrivaPub.Models.Post;
using PrivaPub.StaticServices;

using System.Security.Cryptography;

using PostEntity = PrivaPub.Models.Post.Post;

namespace PrivaPub.Domain.Content
{
	public class LinkPreview : Entity
	{
		public string Url { get; set; }
		public string Title { get; set; }
		public string Description { get; set; }
		public string ImageURL { get; set; }
		public string SiteName { get; set; }
		public bool Failed { get; set; }
		public DateTime FetchedAt { get; set; } = DateTime.UtcNow;
	}

	public interface ILinkPreviews
	{
		Task Wanted(PostEntity post, CancellationToken token);
	}

	public class LinkPreviews : ILinkPreviews, IJobHandler
	{
		const int MaxJitterSeconds = 60;
		static readonly TimeSpan Freshness = TimeSpan.FromDays(7);
		static readonly HtmlParser Parser = new();

		readonly DbEntities _dbEntities;
		readonly ILocalActorService _localActors;
		readonly IFederationHttp _http;
		readonly IJobQueue _queue;
		readonly IOptionsMonitor<FederationOptions> _options;

		readonly IInteractionLedger _ledger;

		public LinkPreviews(DbEntities dbEntities, ILocalActorService localActors, IFederationHttp http, IJobQueue queue,
			IOptionsMonitor<FederationOptions> options, IInteractionLedger ledger = default)
		{
			_dbEntities = dbEntities;
			_localActors = localActors;
			_http = http;
			_queue = queue;
			_options = options;
			_ledger = ledger;
		}

		public JobKind Kind => JobKind.FetchPreview;
		public int Concurrency => 2;
		public int MaxAttempts => 1;
		public int PerHostLimit => 1;

		public async Task Wanted(PostEntity post, CancellationToken token)
		{
			if (!_options.CurrentValue.FetchLinkPreviews || !Previewable(post) || post.Link == default && post.Media.Count > 0)
				return;
			var url = post.Link?.Href ?? FirstLink(post, _localActors.BaseAddress);
			if (url == default)
				return;
			await _queue.EnqueueMany(new[]
			{
				new Job
				{
					Kind = JobKind.FetchPreview, Payload = post.ID, Host = new Uri(url).Host.ToLowerInvariant(), DedupeKey = $"preview|{post.ID}",
					RunAt = DateTime.UtcNow.AddSeconds(RandomNumberGenerator.GetInt32(0, MaxJitterSeconds + 1))
				}
			}, token);
		}

		public async Task<JobOutcome> Handle(Job job, CancellationToken token)
		{
			var post = await _dbEntities.Posts.MatchID(job.Payload).ExecuteFirstAsync(token);
			if (!VisibilityPolicy.Shown(post) || !Previewable(post))
				return JobOutcome.Done;
			var url = post.Link?.Href ?? FirstLink(post, _localActors.BaseAddress);
			if (url == default)
				return JobOutcome.Done;

			var preview = await DB.Default.Find<LinkPreview>().Match(p => p.Url == url).ExecuteFirstAsync(token);
			if (preview == default || preview.FetchedAt < DateTime.UtcNow - Freshness)
				preview = await Fetch(url, token);
			if (preview == default || preview.Failed)
				return JobOutcome.Done;

			var link = post.Link ?? new PostLink { Href = url };
			link.Title ??= preview.Title;
			link.Description ??= preview.Description;
			link.ImageURL ??= preview.ImageURL;
			await DB.Default.Update<PostEntity>().MatchID(post.ID).Modify(p => p.Link, link).ExecuteAsync(token);
			return JobOutcome.Done;
		}

		async Task<LinkPreview> Fetch(string url, CancellationToken token)
		{
			var started = System.Diagnostics.Stopwatch.GetTimestamp();
			var (finalUri, html) = await _http.GetPage(url, token);
			var preview = html == default ? new LinkPreview { Url = url, Failed = true } : Read(url, finalUri, html);
			_ledger?.Record(new InteractionEvent
			{
				Channel = Interactions.Preview,
				Host = Interactions.HostOf(url),
				Outcome = html == default ? Interactions.Failed : Interactions.Ok,
				Reason = html == default ? default : preview.Title == default && preview.ImageURL == default ? "no-card" : "card",
				LatencyMs = (int)System.Diagnostics.Stopwatch.GetElapsedTime(started).TotalMilliseconds,
				Bytes = html == default ? default : System.Text.Encoding.UTF8.GetByteCount(html),
				Redirects = finalUri == default || finalUri.AbsoluteUri == url ? 0 : 1
			});
			await DB.Default.Update<LinkPreview>()
				.Match(p => p.Url == url)
				.Modify(p => p.Title, preview.Title)
				.Modify(p => p.Description, preview.Description)
				.Modify(p => p.ImageURL, preview.ImageURL)
				.Modify(p => p.SiteName, preview.SiteName)
				.Modify(p => p.Failed, preview.Failed)
				.Modify(p => p.FetchedAt, DateTime.UtcNow)
				.Option(o => o.IsUpsert = true)
				.ExecuteAsync(token);
			return preview;
		}

		//only public posts (owner decision 1), checked again when the job runs; never a boost or a post whose card is complete
		static bool Previewable(PostEntity post) => post.Visibility is (PostVisibility.Public or PostVisibility.Unlisted) && !post.IsLocalOnly
			&& post.ReblogOfPostId == default && post.Link?.Title == default;

		public static LinkPreview Read(string url, Uri finalUri, string html)
		{
			var document = Parser.ParseDocument(html);
			string Meta(params string[] names) => names
				.Select(name => document.QuerySelector($"meta[property='{name}'], meta[name='{name}']")?.GetAttribute("content"))
				.FirstOrDefault(value => !string.IsNullOrWhiteSpace(value));
			var image = Meta("og:image:secure_url", "og:image", "og:image:url", "twitter:image", "twitter:image:src");
			var imageUri = image != default && Uri.TryCreate(finalUri, image, out var resolved) && resolved.Scheme is "https" or "http" ? resolved.AbsoluteUri : default;
			var preview = new LinkPreview
			{
				Url = url,
				Title = ObjectShapes.Text(Meta("og:title", "twitter:title") ?? document.Title, 300),
				Description = ObjectShapes.Text(Meta("og:description", "twitter:description", "description"), 1000),
				SiteName = ObjectShapes.Text(Meta("og:site_name"), 200),
				ImageURL = imageUri
			};
			preview.Failed = preview.Title == default && preview.Description == default && preview.ImageURL == default;
			return preview;
		}

		public static string FirstLink(PostEntity post, string ownBase)
		{
			if (string.IsNullOrEmpty(post.ContentHtml))
				return default;
			var document = Parser.ParseDocument(post.ContentHtml);
			return document.QuerySelectorAll("a[href]")
				.Where(a => !(a.ClassList.Contains("mention") || a.ClassList.Contains("hashtag") || a.GetAttribute("rel")?.Contains("tag") == true
					|| a.Closest(".quote-inline") != default))
				.Select(a => a.GetAttribute("href"))
				.FirstOrDefault(href => Uri.TryCreate(href, UriKind.Absolute, out var uri) && uri.Scheme is "https" or "http"
					&& href != post.Url && href != post.ObjectURI && !href.StartsWith(ownBase + "/", StringComparison.OrdinalIgnoreCase));
		}
	}
}