SocialPub/PrivaPub/Federation/Inbox/RemotePosts.cs

using MongoDB.Driver;
using MongoDB.Entities;

using PrivaPub.Infrastructure.Http;
using PrivaPub.Domain.Content;
using PrivaPub.Federation.Actors;
using PrivaPub.Federation.Moderation;
using PrivaPub.Federation.Objects;
using PrivaPub.Infrastructure.Ids;
using PrivaPub.Infrastructure.Jobs;
using PrivaPub.Models.Federation;
using PrivaPub.Models.Jobs;
using PrivaPub.Models.Post;
using PrivaPub.Models.User;
using PrivaPub.StaticServices;

using System.Text.Json;
using System.Text.Json.Nodes;

using PostEntity = PrivaPub.Models.Post.Post;

namespace PrivaPub.Federation.Inbox
{
	public sealed record AncestorsPayload(string PostId, int Depth);

	public interface IRemotePosts
	{
		Task<PostEntity> Build(NoteDocument note, ForeignAvatar author, PostVisibility visibility, IReadOnlyList<string> to, IReadOnlyList<string> cc,
			PostEntity parent, CancellationToken token);
		Task<PostEntity> Parent(string inReplyTo, int depth, CancellationToken token);
		Task<PostEntity> StoreContext(string objectUri, int depth, CancellationToken token);
	}

	public class RemotePosts : IRemotePosts
	{
		public const int MaxDepth = 10;

		readonly DbEntities _dbEntities;
		readonly ILocalActorService _localActors;
		readonly IRemoteActorService _remoteActors;
		readonly IDomainBlocks _domainBlocks;
		readonly IJobQueue _queue;
		readonly IObjectRecords _records;
		readonly ILinkPreviews _previews;

		public RemotePosts(DbEntities dbEntities, ILocalActorService localActors, IRemoteActorService remoteActors, IDomainBlocks domainBlocks,
			IJobQueue queue, IObjectRecords records, ILinkPreviews previews)
		{
			_previews = previews;
			_records = records;
			_dbEntities = dbEntities;
			_localActors = localActors;
			_remoteActors = remoteActors;
			_domainBlocks = domainBlocks;
			_queue = queue;
		}

		// The accounts a remote post names (its Mention tags), plus our personas it is addressed to without naming them: Mastodon
		// keeps those as silent mentions, and Akkoma's to[], or an edit that dropped the @name, address someone that way. A
		// silent mention lets the persona see the post and puts it in its home, and is never shown as a mention.
		public static async Task<List<PostMention>> Mentions(NoteDocument note, IEnumerable<string> addressed, ILocalActorService localActors,
			CancellationToken token)
		{
			var mentions = new List<PostMention>();
			foreach (var mention in note.Mentions)
			{
				var local = await localActors.FindByUri(mention.ActorURI, token);
				mentions.Add(new PostMention { ActorURI = mention.ActorURI, Handle = mention.Handle, IsLocal = local != default, AccountId = local?.Id });
			}
			foreach (var uri in addressed.Where(a => !string.IsNullOrEmpty(a) && !Addressing.IsPublic(a)).Distinct(StringComparer.Ordinal))
			{
				if (mentions.Any(m => m.ActorURI == uri))
					continue;
				var local = await localActors.FindByUri(uri, token);
				if (local is { Kind: LocalActorKind.Person })
					mentions.Add(new PostMention { ActorURI = uri, Handle = "@" + local.Handle, IsLocal = true, AccountId = local.Id, Silent = true });
			}
			return mentions;
		}

		public async Task<PostEntity> Build(NoteDocument note, ForeignAvatar author, PostVisibility visibility, IReadOnlyList<string> to,
			IReadOnlyList<string> cc, PostEntity parent, CancellationToken token)
		{
			var mentions = await Mentions(note, to.Concat(cc), _localActors, token);

			var post = new PostEntity
			{
				ID = PrivacyIds.Arrived(note.Published),
				ObjectURI = note.Id,
				ActorURI = author.ActorURI,
				AuthorAccountId = author.ID,
				Visibility = visibility,
				To = to.ToList(),
				Cc = cc.ToList(),
				Url = note.Url,
				ContextURI = note.Context,
				// the community it lives in (FEP-1b12), as the post itself names it: a post fetched for a thread knows its
				// community as well as one its community announced
				AudienceURI = note.Audience != default && note.Audience != author.ActorURI && !note.Audience.EndsWith("/followers", StringComparison.Ordinal)
					? note.Audience
					: default,
				QuoteURI = note.QuoteUri,
				ObjectType = note.Type,
				Title = note.Title,
				SpoilerText = note.SpoilerText,
				Excerpt = note.Excerpt,
				Source = note.Source,
				Poll = note.Poll,
				SearchableBy = note.SearchableBy,
				QuotePolicy = note.QuotePolicy,
				ReplyPolicy = note.ReplyPolicy,
				LikePolicy = note.LikePolicy,
				AnnouncePolicy = note.AnnouncePolicy,
				ReplyApprovals = note.ReplyApprovals,
				Emojis = note.Emojis.ToList(),
				CoverURL = note.CoverURL,
				Link = note.Link,
				Video = note.Video,
				Audio = note.Audio,
				Event = note.Event,
				LockedAt = note.CommentsEnabled == false ? DateTime.UtcNow : null,
				Place = note.Place,
				HasContentWarning = note.Sensitive,
				Text = note.ContentHtml,
				ContentHtml = note.ContentHtml,
				ContentFormat = ContentFormat.Html,
				Language = note.Language,
				Mentions = mentions,
				Tags = note.Tags.ToList(),
				Media = _domainBlocks.Find(new Uri(author.ActorURI).Host)?.RejectMedia == true ? new() : note.Attachments.ToList(),
				InReplyToURI = note.InReplyTo,
				AnsweringToPostId = parent?.ID,
				InReplyToAccountId = parent?.AuthorAccountId ?? parent?.GroupUserId,
				IsFederatedCopy = true,
				AuthorGone = author.AccountState == AvatarAccountState.Deleted,
				CreationDate = note.Published,
				UpdateDate = note.Updated ?? note.Published,
				EditedAt = note.Updated,
				Revisions = note.FormerRepresentations.ToList()
			};
			post.Flairs = await Flairs(note, post.AudienceURI, token);
			await Episode(post, note, author, token);
			return post;
		}

		// a post's flairs, filled in from its community's list
		public static async Task<List<PostFlair>> Flairs(NoteDocument note, string communityUri, CancellationToken token)
		{
			if (note.Flairs.Count == 0)
				return default;
			var community = communityUri == default ? default : await DB.Default.Find<ForeignAvatar>().Match(f => f.ActorURI == communityUri).ExecuteFirstAsync(token);
			return ObjectShapes.Described(note.Flairs, community?.Flairs);
		}

		static readonly System.Text.RegularExpressions.Regex Href = new(@"href=""([^""]+)""", System.Text.RegularExpressions.RegexOptions.Compiled);

		// a post that is only a link to a podcast episode of its own author (Castopod announces each episode so): the
		// episode's sound, title and cover, read from it
		async Task Episode(PostEntity post, NoteDocument note, ForeignAvatar author, CancellationToken token)
		{
			if (note.Type != "Note" || post.Media.Count > 0 || string.IsNullOrEmpty(note.ContentHtml))
				return;
			var links = Href.Matches(note.ContentHtml).Select(m => System.Net.WebUtility.HtmlDecode(m.Groups[1].Value)).Distinct().ToList();
			if (links.Count != 1 || links[0] == note.Id || !Origin.Same(links[0], author.ActorURI))
				return;
			using var scope = HttpScope.For("object");
			using var fetched = await _remoteActors.FetchObject(links[0], token);
			var episode = fetched == default ? default : NoteParser.Parse(JsonNode.Parse(fetched.Root.GetRawText()));
			if (episode is not { Type: "PodcastEpisode" } || episode.AttributedTo != author.ActorURI)
				return;
			post.Media = episode.Attachments.ToList();
			post.Audio = episode.Audio;
			post.Title ??= episode.Title;
			post.CoverURL ??= episode.CoverURL;
			post.Excerpt ??= ObjectShapes.Text(episode.ContentHtml, 1000);
		}

		public async Task<PostEntity> Parent(string inReplyTo, int depth, CancellationToken token)
		{
			if (string.IsNullOrEmpty(inReplyTo))
				return default;
			var known = await _dbEntities.Posts.Match(p => p.ObjectURI == inReplyTo && !p.DeletedAt.HasValue).ExecuteFirstAsync(token);
			if (known != default || inReplyTo.StartsWith(_localActors.BaseAddress + "/", StringComparison.OrdinalIgnoreCase))
				return known;
			return await StoreContext(inReplyTo, depth, token);
		}

		public async Task<PostEntity> StoreContext(string objectUri, int depth, CancellationToken token)
		{
			var existing = await _dbEntities.Posts.Match(p => p.ObjectURI == objectUri).ExecuteFirstAsync(token);
			if (existing != default || depth > MaxDepth || await DB.Default.Find<DeletedObject>().Match(d => d.ObjectURI == objectUri).ExecuteAnyAsync(token))
				return existing;

			using var scope = HttpScope.For("context");
			using var fetched = await _remoteActors.FetchObject(objectUri, token);
			var note = fetched == default ? default : NoteParser.Parse(JsonNode.Parse(fetched.Root.GetRawText()));
			if (note == default || !Origin.Same(note.Id, note.AttributedTo))
				return default;
			var visibility = Addressing.Classify(note.To, note.Cc, default);
			if (visibility is not (PostVisibility.Public or PostVisibility.Unlisted))
				return default;
			var author = await _remoteActors.GetActor(note.AttributedTo, refresh: false, token);
			if (author == default)
				return default;

			var grandparent = string.IsNullOrEmpty(note.InReplyTo)
				? default
				: await _dbEntities.Posts.Match(p => p.ObjectURI == note.InReplyTo && !p.DeletedAt.HasValue).ExecuteFirstAsync(token);
			var post = await Build(note, author, visibility, note.To, note.Cc, grandparent, token);
			try
			{
				await DB.Default.SaveAsync(post, token);
			}
			catch (MongoWriteException ex) when (ex.WriteError?.Category == ServerErrorCategory.DuplicateKey)
			{
				return await _dbEntities.Posts.Match(p => p.ObjectURI == objectUri).ExecuteFirstAsync(token);
			}
			await _records.Record(note, post, ObjectPath.Fetched, refetched: false, token);
			await _previews.Wanted(post, token);

			if (grandparent == default && !string.IsNullOrEmpty(note.InReplyTo) && depth < MaxDepth)
				await _queue.Enqueue(JobKind.FetchAncestors, JsonSerializer.Serialize(new AncestorsPayload(post.ID, depth + 1)), new Uri(note.InReplyTo).Host,
					"ancestors|" + post.ID, token);
			return post;
		}
	}

	public class AncestorsJobHandler : IJobHandler
	{
		readonly DbEntities _dbEntities;
		readonly IRemotePosts _remotePosts;

		public AncestorsJobHandler(DbEntities dbEntities, IRemotePosts remotePosts)
		{
			_dbEntities = dbEntities;
			_remotePosts = remotePosts;
		}

		public JobKind Kind => JobKind.FetchAncestors;
		public int Concurrency => 1;
		public int MaxAttempts => 3;
		public int PerHostLimit => 1;

		public async Task<JobOutcome> Handle(Job job, CancellationToken token)
		{
			var payload = JsonSerializer.Deserialize<AncestorsPayload>(job.Payload);
			var child = await _dbEntities.Posts.MatchID(payload.PostId).ExecuteFirstAsync(token);
			if (child == default || string.IsNullOrEmpty(child.InReplyToURI) || !string.IsNullOrEmpty(child.AnsweringToPostId))
				return JobOutcome.Done;

			var parent = await _remotePosts.Parent(child.InReplyToURI, payload.Depth, token);
			if (parent == default)
				return JobOutcome.Done;
			await DB.Default.Update<PostEntity>().MatchID(child.ID)
				.Modify(p => p.AnsweringToPostId, parent.ID)
				.Modify(p => p.InReplyToAccountId, parent.AuthorAccountId ?? parent.GroupUserId)
				.ExecuteAsync(token);
			return JobOutcome.Done;
		}
	}
}