SocialPub/PrivaPub/Federation/Objects/NoteParser.cs

using PrivaPub.Models.Post;

using System.Globalization;
using System.Text;
using System.Text.Json.Nodes;

using static PrivaPub.Federation.Objects.ActivityJson;

namespace PrivaPub.Federation.Objects
{
	public sealed class NoteDocument
	{
		public string Id { get; init; }
		public string Type { get; init; }
		public JsonObject Raw { get; init; }
		public string AttributedTo { get; init; }
		public string ContentHtml { get; init; }
		public string Title { get; init; }
		public string SpoilerText { get; init; }
		public string Excerpt { get; init; }
		public bool Sensitive { get; init; }
		public string Language { get; init; }
		public string Url { get; init; }
		public string InReplyTo { get; init; }
		public string Context { get; init; }
		public string Audience { get; init; }
		public string Wall { get; init; }//FEP-400e: the collection it says it was written in (`target`), a wall
		public string WallOwner { get; init; }//that collection's attributedTo
		public string QuoteUri { get; init; }
		public string ReplyAuthorization { get; init; }//the parent author's ReplyAuthorization (approvedBy before GoToSocial 0.21)
		public string QuoteAuthorization { get; init; }
		public bool QuotesByConsent { get; init; }//FEP-044f `quote`, as opposed to the legacy keys that ask nobody
		public bool QuoteDeleted { get; init; }
		public DateTime Published { get; init; }
		public DateTime? Updated { get; init; }
		public IReadOnlyList<string> To { get; init; } = Array.Empty<string>();
		public IReadOnlyList<string> Cc { get; init; } = Array.Empty<string>();
		public IReadOnlyList<PostMention> Mentions { get; init; } = Array.Empty<PostMention>();
		public IReadOnlyList<string> Tags { get; init; } = Array.Empty<string>();
		public IReadOnlyList<PostMedia> Attachments { get; init; } = Array.Empty<PostMedia>();
		public PostSource Source { get; init; }
		public PostPoll Poll { get; init; }
		public InteractionRule QuotePolicy { get; init; }
		public InteractionRule ReplyPolicy { get; init; }
		public InteractionRule LikePolicy { get; init; }
		public InteractionRule AnnouncePolicy { get; init; }
		public InteractionRule ReplyApprovals { get; init; }//FEP-5624's canReply: who may reply, each reply then approved or refused
		public IReadOnlyList<CustomEmoji> Emojis { get; init; } = Array.Empty<CustomEmoji>();
		public string CoverURL { get; init; }
		public PostLink Link { get; init; }
		public VideoDetails Video { get; init; }
		public AudioDetails Audio { get; init; }
		public EventDetails Event { get; init; }
		public PostPlace Place { get; init; }
		public bool? CommentsEnabled { get; init; }//Lemmy, PieFed and Mbin: false once its moderators lock it; null when not said
		public IReadOnlyList<PostRevision> FormerRepresentations { get; init; } = Array.Empty<PostRevision>();//its earlier versions, oldest first
	}

	public static class NoteParser
	{
		static readonly string[] NoteTypes = { "Note", "Article", "Page", "Question", "Event", "Video", "Audio", "Image", "ChatMessage", "PodcastEpisode" };
		const int MaxAttachments = 16;
		const int MaxTags = 64;
		public const int MaxRevisions = 20;

		public static NoteDocument Parse(JsonNode node)
		{
			if (node is not JsonObject note)
				return default;
			var type = Value(note, "type");
			var id = Id(note);
			if (!NoteTypes.Contains(type) || Origin.Of(id) == default)
				return default;
			if (type == "PodcastEpisode")
				note = Episode(note);

			var (contentHtml, language) = Content(note);
			var summary = Value(note, "summary");
			var excerpt = ObjectShapes.Text(summary, 1000)
				?? (note["preview"] is JsonObject preview && Value(preview, "type") == "Note" ? ObjectShapes.Text(Value(preview, "content"), 1000) : default);
			var (sensitive, summaryWarns) = Warns(note, type, summary);
			return new NoteDocument
			{
				Id = id,
				Type = type,
				Raw = note,
				AttributedTo = Attribution(note["attributedTo"]),
				ContentHtml = ContentSanitizer.Html(contentHtml),
				Title = ObjectShapes.Text(Value(note, "name"), 500),
				SpoilerText = summaryWarns ? Plain(summary, 500) : default,
				Excerpt = summaryWarns ? default : excerpt,
				Sensitive = sensitive,
				Language = language,
				Url = UrlOf(note["url"]) ?? id,
				InReplyTo = Id(note["inReplyTo"]),
				Context = Id(note["context"]) ?? Value(note, "conversation"),
				Audience = Id(note["audience"]),
				Wall = note["target"] is JsonObject target && Value(target, "type") is "Collection" or "OrderedCollection" ? Id(target) : default,
				WallOwner = note["target"] is JsonObject wallTarget ? Id(wallTarget["attributedTo"]) : default,
				QuoteUri = Id(note["quote"]) ?? Value(note, "quoteUrl") ?? Value(note, "quoteUri") ?? Value(note, "_misskey_quote") ?? QuoteLink(note),
				ReplyAuthorization = Id(note["replyAuthorization"]) ?? Id(note["approvedBy"]),
				QuoteAuthorization = Id(note["quoteAuthorization"]),
				QuotesByConsent = note.ContainsKey("quote"),
				QuoteDeleted = note["quote"] is JsonObject quoted && Value(quoted, "type") == "Tombstone",
				Published = Time(Value(note, "published")) ?? DateTime.UtcNow,
				Updated = Time(Value(note, "updated")),
				To = List(note["to"]),
				Cc = List(note["cc"]),
				Mentions = Tags(note, "Mention").Select(t => new PostMention { ActorURI = Value(t, "href"), Handle = Value(t, "name") })
					.Where(m => Origin.Of(m.ActorURI) != default)
					.DistinctBy(m => m.ActorURI)
					.Take(MaxTags)
					.ToList(),
				Tags = Tags(note, "Hashtag").Select(t => Value(t, "name")?.TrimStart('#').Normalize(NormalizationForm.FormKC).ToLowerInvariant())
					.Where(t => !string.IsNullOrEmpty(t))
					.Distinct()
					.Take(MaxTags)
					.ToList(),
				Attachments = Attachments(note),
				Source = ObjectShapes.Source(note),
				Poll = ObjectShapes.Poll(note),
				QuotePolicy = ObjectShapes.QuotePolicy(note),
				ReplyPolicy = ObjectShapes.Policy(note, "canReply"),
				LikePolicy = ObjectShapes.Policy(note, "canLike"),
				AnnouncePolicy = ObjectShapes.Policy(note, "canAnnounce"),
				ReplyApprovals = ObjectShapes.ReplyApprovals(note),
				Emojis = ObjectShapes.Emojis(note["tag"]),
				CoverURL = ObjectShapes.Cover(note),
				Link = ObjectShapes.Link(note),
				Video = ObjectShapes.Video(note),
				Audio = ObjectShapes.Audio(type == "PodcastEpisode" ? note["audio"] as JsonObject : note),
				Event = ObjectShapes.Event(note),
				Place = ObjectShapes.NotePlace(note),
				CommentsEnabled = note["commentsEnabled"] is JsonValue comments && comments.TryGetValue<bool>(out var enabled) ? enabled : null,
				FormerRepresentations = Former(note, type)
			};
		}

		// a summary is a content warning only on a Note or a Question, or a post marked sensitive; elsewhere it is an excerpt
		static (bool Sensitive, bool SummaryWarns) Warns(JsonObject note, string type, string summary)
		{
			var marked = note["sensitive"] is JsonValue flag && flag.TryGetValue<bool>(out var value) && value;
			var warns = type is "Note" or "Question" || marked;
			return (marked || warns && !string.IsNullOrWhiteSpace(summary), warns);
		}

		// Pleroma's and Akkoma's earlier versions of an edited post (`formerRepresentations`, newest first there), each read as
		// the post itself is: the newest twenty, oldest first, each dated when it was written
		static List<PostRevision> Former(JsonObject note, string type)
		{
			if (note["formerRepresentations"] is not JsonObject former || former["orderedItems"] is not JsonArray items)
				return [];
			return items.OfType<JsonObject>().Take(MaxRevisions * 2)
				.Select(version =>
				{
					var summary = Value(version, "summary");
					var (sensitive, warns) = Warns(version, type, summary);
					return new PostRevision
					{
						Title = ObjectShapes.Text(Value(version, "name"), 500),
						SpoilerText = warns ? Plain(summary, 500) : default,
						ContentHtml = ContentSanitizer.Html(Content(version).Html),
						HasContentWarning = sensitive,
						EditedAt = Time(Value(version, "updated")) ?? Time(Value(version, "published")) ?? default
					};
				})
				.Reverse()
				.OrderBy(r => r.EditedAt)
				.TakeLast(MaxRevisions)
				.ToList();
		}

		// Castopod's PodcastEpisode: its words in `description` (a Note of its own), its sound in `audio`, its title the
		// sound's name; read as a post with those
		static JsonObject Episode(JsonObject episode)
		{
			var note = episode.DeepClone().AsObject();
			if (note["description"] is JsonObject description)
			{
				note["content"] = description["content"]?.DeepClone();
				note["contentMap"] = description["contentMap"]?.DeepClone();
				note["mediaType"] = description["mediaType"]?.DeepClone();
			}
			if (note["audio"] is JsonObject audio)
			{
				note["name"] ??= audio["name"]?.DeepClone();
				note["attachment"] = new JsonArray(audio.DeepClone());
			}
			return note;
		}

		static (string Html, string Language) Content(JsonObject note)
		{
			var content = Value(note, "content");
			string language = default;
			if (note["contentMap"] is JsonObject map && map.Count > 0)
			{
				var first = map.First();
				language = first.Key;
				if (string.IsNullOrEmpty(content) && first.Value is JsonValue value && value.TryGetValue<string>(out var mapped))
					content = mapped;
			}
			if (string.IsNullOrEmpty(content) && Value(note, "_misskey_content") is { } misskey)
				content = System.Net.WebUtility.HtmlEncode(misskey).Replace("\n", "<br>");
			if (!string.IsNullOrEmpty(content) && Value(note, "mediaType")?.StartsWith("text/markdown", StringComparison.OrdinalIgnoreCase) == true)
				content = Rendering.ActivityPubRenderer.Html(content);
			return (content, language ?? ObjectShapes.Language(note));
		}

		static string Attribution(JsonNode node) => node switch
		{
			JsonArray array => array.OfType<JsonObject>().Where(o => Value(o, "type") is "Person" or "Service" or "Application" or "Organization").Select(Id)
				.Concat(array.OfType<JsonValue>().Select(Id))
				.Concat(array.OfType<JsonObject>().Where(o => Value(o, "type") == "Group").Select(Id))
				.FirstOrDefault(id => id != default),
			_ => Id(node)
		};

		static IEnumerable<JsonObject> Tags(JsonObject note, string type) =>
			(note["tag"] switch
			{
				JsonArray array => array.OfType<JsonObject>(),
				JsonObject single => new[] { single },
				_ => Enumerable.Empty<JsonObject>()
			}).Where(t => Value(t, "type") == type);

		static List<PostMedia> Attachments(JsonObject note) =>
			ObjectShapes.Objects(note["attachment"])
				.Where(a => Value(a, "type") is "Document" or "Image" or "Video" or "Audio")
				.Select(Attachment)
				.Where(m => Origin.Of(m.RemoteURL) != default)
				.Take(MaxAttachments)
				.ToList();

		static PostMedia Attachment(JsonObject attachment)
		{
			var (url, linkMediaType) = ObjectShapes.MediaLink(attachment["url"] ?? attachment["href"]);
			return new PostMedia
			{
				RemoteURL = url,
				ContentType = ObjectShapes.InferMediaType(Value(attachment, "mediaType") ?? linkMediaType, url, Value(attachment, "type")),
				RemotePreviewURL = ObjectShapes.Image(attachment["icon"]),
				DurationSeconds = ObjectShapes.Seconds(attachment["duration"]),
				Sensitive = attachment["sensitive"] is JsonValue flag && flag.TryGetValue<bool>(out var marked) ? marked : default(bool?),
				Description = Plain(Value(attachment, "name") ?? Value(attachment, "summary") ?? FirstOf(attachment["nameMap"]) ?? FirstOf(attachment["summaryMap"]), 1500),
				Blurhash = Value(attachment, "blurhash"),
				Width = Int(attachment["width"]),
				Height = Int(attachment["height"]),
				Focus = attachment["focalPoint"] is JsonArray focus && focus.Count == 2
					? focus.Select(f => f is JsonValue v && v.TryGetValue<double>(out var d) ? (float)Math.Clamp(d, -1, 1) : 0f).ToArray()
					: default
			};
		}

		static string QuoteLink(JsonObject note) =>
			Tags(note, "Link")
				.Where(t => Value(t, "mediaType") is { } type && (type.StartsWith("application/activity+json") || type.StartsWith("application/ld+json")))
				.Select(t => Value(t, "href"))
				.FirstOrDefault(href => Origin.Of(href) != default);

		static string FirstOf(JsonNode map) =>
			map is JsonObject languages ? languages.Select(l => l.Value is JsonValue v && v.TryGetValue<string>(out var text) ? text : default).FirstOrDefault(t => t != default) : default;

		static string UrlOf(JsonNode node) => node switch
		{
			JsonValue value when value.TryGetValue<string>(out var text) => text,
			JsonObject obj => Value(obj, "href") ?? Value(obj, "id"),
			JsonArray array => array.Select(item => item is JsonObject link && Value(link, "mediaType") is { } mediaType && !mediaType.StartsWith("text/html")
					? default
					: UrlOf(item))
				.FirstOrDefault(url => url != default) ?? array.Select(UrlOf).FirstOrDefault(url => url != default),
			_ => default
		};

		static IReadOnlyList<string> List(JsonNode node) => node switch
		{
			JsonArray array => array.Select(Id).Where(id => id != default).Distinct(StringComparer.Ordinal).ToList(),
			JsonNode single when Id(single) is { } id => new[] { id },
			_ => Array.Empty<string>()
		};

		static DateTime? Time(string value) =>
			DateTimeOffset.TryParse(value, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal, out var parsed)
				? parsed.UtcDateTime
				: (DateTime?)null;

		static int? Int(JsonNode node) =>
			node is JsonValue value && value.TryGetValue<int>(out var number) && number is > 0 and < 100_000 ? number : (int?)null;

		static string Plain(string value, int max)
		{
			if (string.IsNullOrWhiteSpace(value))
				return default;
			value = value.Trim();
			return value.Length <= max ? value : value[..max];
		}
	}
}