SocialPub/PrivaPub/Api/Mastodon/Controllers/ProvenanceController.cs

using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Options;

using MongoDB.Entities;

using PrivaPub.Api.Mastodon.Infrastructure;
using PrivaPub.Domain.Privacy;
using PrivaPub.Domain.Statistics;
using PrivaPub.Federation.Actors;
using PrivaPub.Federation.Objects;
using PrivaPub.Infrastructure;
using PrivaPub.Infrastructure.Geo;
using PrivaPub.Models.Federation;
using PrivaPub.Models.Jobs;
using PrivaPub.Models.Post;
using PrivaPub.StaticServices;

using System.Text.Json.Nodes;

using PostEntity = PrivaPub.Models.Post.Post;

namespace PrivaPub.Api.Mastodon.Controllers
{
	public class ProvenanceController : MastodonController
	{
		const int MaxHosts = 40;

		readonly DbEntities _dbEntities;
		const int MaxWeeks = 260;

		readonly ISelfLocation _self;
		readonly IServerPlaces _places;
		readonly ILocalActorService _localActors;
		readonly IOptionsMonitor<RegistrationOptions> _registrations;

		public ProvenanceController(DbEntities dbEntities, ISelfLocation self, IServerPlaces places, ILocalActorService localActors,
			IOptionsMonitor<RegistrationOptions> registrations)
		{
			_dbEntities = dbEntities;
			_self = self;
			_places = places;
			_localActors = localActors;
			_registrations = registrations;
		}

		[HttpGet("/api/privapub/v1/statuses/{id}/provenance"), Scope("read:statuses", requiresUser: false), AllowAnonymous]
		public async Task<IActionResult> Status(string id, CancellationToken token)
		{
			var post = await _dbEntities.Posts.Match(p => p.ID == id).Match(VisibilityPolicy.IsShown).ExecuteFirstAsync(token);
			if (post?.ReblogOfPostId != default)
				post = await _dbEntities.Posts.Match(p => p.ID == post.ReblogOfPostId).Match(VisibilityPolicy.IsShown).ExecuteFirstAsync(token);
			if (post == default || post.Visibility == PostVisibility.LocalGeo || !await VisibilityPolicy.CanSee(post, MyId, token))
				return NotFoundError();
			if (!post.IsFederatedCopy)
				return Json(new Provenance { ObjectUri = post.ObjectURI, Url = post.Url, Path = "local" });

			var record = await DB.Default.Find<ObjectRecord>().Match(r => r.PostId == post.ID).ExecuteFirstAsync(token)
				?? await DB.Default.Find<ObjectRecord>().Match(r => r.ObjectURI == post.ObjectURI).ExecuteFirstAsync(token);
			var host = record?.Host ?? (Uri.TryCreate(post.ObjectURI, UriKind.Absolute, out var uri) ? uri.Host.ToLowerInvariant() : default);
			var instance = host == default ? default : await DB.Default.Find<RemoteInstance>().Match(i => i.Host == host).ExecuteFirstAsync(token);
			var beforeCdn = instance?.Geo?.Cdn == default ? default : (await _places.BeforeCdn(new[] { instance.Host }, token)).GetValueOrDefault(instance.Host);
			return Json(Describe(post, record, instance, beforeCdn));
		}

		[HttpGet("/api/privapub/v1/instances/{host}"), Scope("read", requiresUser: false), AllowAnonymous]
		public async Task<IActionResult> Instance(string host, CancellationToken token)
		{
			var described = await Describe(new[] { host }, token);
			return described.Count == 0 ? NotFoundError() : Json(described[0]);
		}

		//up to MaxHosts at once (host[]=...), in the order asked; hosts this server does not know are left out
		[HttpGet("/api/privapub/v1/instances"), Scope("read", requiresUser: false), AllowAnonymous]
		public async Task<IActionResult> Instances(CancellationToken token) => Json(await Describe(Params.List("host"), token));

		//the server's weekly snapshots, newest first (weeks=, at most MaxWeeks): its place as shown publicly, its CDN, its size
		[HttpGet("/api/privapub/v1/instances/{host}/history"), Scope("read", requiresUser: false), AllowAnonymous]
		public async Task<IActionResult> History(string host, CancellationToken token)
		{
			host = host?.Trim().ToLowerInvariant();
			if (string.IsNullOrEmpty(host) || host.Length > 253 || !await DB.Default.Find<RemoteInstance>().Match(i => i.Host == host).ExecuteAnyAsync(token))
				return NotFoundError();
			var weeks = Math.Clamp(Params.Int("weeks") ?? 52, 1, MaxWeeks);
			return Json((await _places.History(host, weeks, token)).Select(s => new InstanceWeek
			{
				Week = s.Week,
				TakenAt = MastodonJson.Time(s.TakenAt),
				Reachable = s.Reachable,
				Software = s.Software,
				Version = s.SoftwareVersion,
				UsersTotal = s.UsersTotal,
				UsersActiveMonth = s.UsersActiveMonth,
				LocalPosts = s.LocalPosts,
				OpenRegistrations = s.OpenRegistrations,
				Geo = Located(PublicGeo.Project(new InstanceGeo
				{
					Country = s.Country, City = s.City, Latitude = s.Latitude, Longitude = s.Longitude, Asn = s.Asn,
					Cdn = s.Cdn, CdnDomain = s.CdnDomain, CdnSource = s.CdnSource, Ipv6 = s.Ipv6
				}), new InstanceGeo { Cdn = s.Cdn, CdnDomain = s.CdnDomain ?? CdnCatalog.ByKey(s.Cdn)?.Key, CdnSource = s.CdnSource })
			}).ToList());
		}

		async Task<List<InstanceDescription>> Describe(IEnumerable<string> hosts, CancellationToken token)
		{
			var wanted = hosts.Select(h => h?.Trim().ToLowerInvariant()).Where(h => !string.IsNullOrEmpty(h) && h.Length <= 253)
				.Distinct().Take(MaxHosts).ToList();
			var selfHost = Uri.TryCreate(_localActors.BaseAddress, UriKind.Absolute, out var self) ? self.Host.ToLowerInvariant() : default;
			var remote = await DB.Default.Find<RemoteInstance>().Match(i => wanted.Contains(i.Host)).ExecuteAsync(token);
			var beforeCdn = await _places.BeforeCdn(remote.Where(i => i.Geo?.Cdn != default).Select(i => i.Host), token);
			var described = new List<InstanceDescription>();
			foreach (var host in wanted)
			{
				if (host == selfHost)
					described.Add(await DescribeSelf(host, token));
				else if (remote.FirstOrDefault(i => i.Host == host) is { } instance)
					described.Add(Describe(instance, beforeCdn.GetValueOrDefault(host)));
			}
			return described;
		}

		async Task<InstanceDescription> DescribeSelf(string host, CancellationToken token)
		{
			var geo = await _self.Get(token);
			return new InstanceDescription
			{
				Host = host,
				Software = "privapub",
				Version = BuildInfo.Ref,
				NodeName = "PrivaPub",
				Protocols = new() { "activitypub" },
				OpenRegistrations = _registrations.CurrentValue.IsOpen,
				DescribedAt = MastodonJson.Time(DateTime.UtcNow),
				Geo = Located(PublicGeo.Project(geo), geo)
			};
		}

		//the public projection of a server's place (PublicGeo); for a CDN-fronted one, the CDN's domain, how it was found and
		//where the server was before it
		static InstanceLocation Located(PublicLocation location, InstanceGeo geo, PlaceBeforeCdn beforeCdn = default) => location == default ? default : new InstanceLocation
		{
			Precision = location.Cdn != default ? "cdn" : location.Latitude != default ? "city" : "country",
			Country = location.Country,
			City = location.City,
			Latitude = location.Latitude,
			Longitude = location.Longitude,
			Network = location.Network,
			Cdn = location.Cdn,
			CdnDomain = location.Cdn != default ? ServerPlaces.KeyOf(geo) : default,
			CdnSource = location.Cdn != default ? geo?.CdnSource : default,
			BeforeCdn = location.Cdn == default || beforeCdn == default ? default : new InstancePlace
			{
				Week = beforeCdn.Week, Country = beforeCdn.Country, City = beforeCdn.City, Latitude = beforeCdn.Latitude, Longitude = beforeCdn.Longitude
			},
			Source = geo?.Source,
			LocatedAt = geo?.LocatedAt is { } at ? MastodonJson.Time(at) : default
		};

		Provenance Describe(PostEntity post, ObjectRecord record, RemoteInstance instance, PlaceBeforeCdn beforeCdn)
		{
			var raw = record?.Raw == default ? default : JsonNode.Parse(record.Raw) as JsonObject;
			return new Provenance
			{
				ObjectUri = post.ObjectURI,
				ObjectType = record?.ObjectType ?? post.ObjectType,
				Url = post.Url,
				Path = record == default ? "unrecorded" : record.Path == ObjectPath.Delivered ? "delivered" : "fetched",
				Refetched = record?.Refetched == true,
				Activity = record?.ActivityId == default ? default : new ProvenanceActivity
				{
					Id = record.ActivityId, Type = record.ActivityType, Actor = record.ActivityActorURI
				},
				Inbox = record?.Inbox,
				FetchedBy = record?.Path == ObjectPath.Fetched ? "instance-actor" : default,
				Signature = record?.KeyId == default || record.Path == ObjectPath.Fetched ? default : new ProvenanceSignature
				{
					Scheme = record.SignatureScheme, KeyId = record.KeyId, Algorithm = record.Algorithm, Headers = record.SignedHeaders
				},
				ReceivedAt = record == default ? default : MastodonJson.Time(record.ReceivedAt),
				Published = record?.Published is { } published ? MastodonJson.Time(published) : default,
				Updated = record?.Updated is { } updated ? MastodonJson.Time(updated) : default,
				Extensions = record?.Extensions ?? ObjectFeatures.Detect(raw),
				ContextNamespaces = record?.ContextNamespaces ?? ObjectFeatures.Namespaces(raw, record?.ActivityContext),
				Raw = raw,
				RawBytes = record?.RawBytes ?? 0,
				RawHash = record?.RawHash,
				RawTruncated = record?.RawTruncated == true,
				Revisions = record?.Revisions.Select(r => new ProvenanceRevision
				{
					ActivityId = r.ActivityId,
					Updated = r.Updated is { } at ? MastodonJson.Time(at) : default,
					ReceivedAt = MastodonJson.Time(r.ReceivedAt),
					RawHash = r.RawHash,
					Raw = r.Raw == default ? default : JsonNode.Parse(r.Raw)
				}).ToList() ?? new(),
				Instance = instance == default ? default : Describe(instance, beforeCdn)
			};
		}

		static InstanceDescription Describe(RemoteInstance instance, PlaceBeforeCdn beforeCdn) => new()
		{
			Host = instance.Host,
			Software = instance.Software,
			Version = instance.SoftwareVersion,
			NodeName = instance.NodeName,
			Protocols = instance.Protocols ?? new(),
			OpenRegistrations = instance.OpenRegistrations,
			DescribedAt = instance.DescribedAt is { } described ? MastodonJson.Time(described) : default,
			DescriptionError = instance.DescriptionError,
			NodeInfo = instance.NodeInfo == default ? default : JsonNode.Parse(instance.NodeInfo),
			Delivery = new InstanceDelivery
			{
				ConsecutiveFailures = instance.ConsecutiveFailures,
				UnavailableUntil = instance.UnavailableUntil is { } until ? MastodonJson.Time(until) : default,
				LastSuccessAt = instance.LastSuccessAt is { } success ? MastodonJson.Time(success) : default,
				LastFailureAt = instance.LastFailureAt is { } failure ? MastodonJson.Time(failure) : default
			},
			Geo = Located(PublicGeo.Project(instance.Geo), instance.Geo, beforeCdn)
		};

		public class Provenance
		{
			public string ObjectUri { get; set; }
			public string ObjectType { get; set; }
			public string Url { get; set; }
			public string Path { get; set; }
			public bool Refetched { get; set; }
			public ProvenanceActivity Activity { get; set; }
			public string Inbox { get; set; }
			public string FetchedBy { get; set; }
			public ProvenanceSignature Signature { get; set; }
			public string ReceivedAt { get; set; }
			public string Published { get; set; }
			public string Updated { get; set; }
			public List<string> Extensions { get; set; } = new();
			public List<string> ContextNamespaces { get; set; } = new();
			public JsonNode Raw { get; set; }
			public int RawBytes { get; set; }
			public string RawHash { get; set; }
			public bool RawTruncated { get; set; }
			public List<ProvenanceRevision> Revisions { get; set; } = new();
			public InstanceDescription Instance { get; set; }
		}

		public class ProvenanceActivity
		{
			public string Id { get; set; }
			public string Type { get; set; }
			public string Actor { get; set; }
		}

		public class ProvenanceSignature
		{
			public string Scheme { get; set; }
			public string KeyId { get; set; }
			public string Algorithm { get; set; }
			public List<string> Headers { get; set; } = new();
		}

		public class ProvenanceRevision
		{
			public string ActivityId { get; set; }
			public string Updated { get; set; }
			public string ReceivedAt { get; set; }
			public string RawHash { get; set; }
			public JsonNode Raw { get; set; }
		}

		public class InstanceDescription
		{
			public string Host { get; set; }
			public string Software { get; set; }
			public string Version { get; set; }
			public string NodeName { get; set; }
			public List<string> Protocols { get; set; } = new();
			public bool? OpenRegistrations { get; set; }
			public string DescribedAt { get; set; }
			public string DescriptionError { get; set; }
			public JsonNode NodeInfo { get; set; }
			public InstanceDelivery Delivery { get; set; }
			public InstanceLocation Geo { get; set; }
		}

		//precision: "city" (coordinates to 0.1°, city and network), "country" (located only to a country) or "cdn" (the
		//CDN's name only)
		public class InstanceLocation
		{
			public string Precision { get; set; }
			public string Country { get; set; }
			public string City { get; set; }
			public double? Latitude { get; set; }
			public double? Longitude { get; set; }
			public string Network { get; set; }
			public string Cdn { get; set; }
			public string CdnDomain { get; set; }
			public string CdnSource { get; set; }//"ranges", "headers" or "asn"
			public InstancePlace BeforeCdn { get; set; }//where a CDN-fronted server was located before the CDN, by its snapshots
			public string Source { get; set; }
			public string LocatedAt { get; set; }
		}

		public class InstancePlace
		{
			public string Week { get; set; }
			public string Country { get; set; }
			public string City { get; set; }
			public double? Latitude { get; set; }
			public double? Longitude { get; set; }
		}

		public class InstanceWeek
		{
			public string Week { get; set; }
			public string TakenAt { get; set; }
			public bool Reachable { get; set; }
			public string Software { get; set; }
			public string Version { get; set; }
			public long? UsersTotal { get; set; }
			public long? UsersActiveMonth { get; set; }
			public long? LocalPosts { get; set; }
			public bool? OpenRegistrations { get; set; }
			public InstanceLocation Geo { get; set; }
		}

		public class InstanceDelivery
		{
			public int ConsecutiveFailures { get; set; }
			public string UnavailableUntil { get; set; }
			public string LastSuccessAt { get; set; }
			public string LastFailureAt { get; set; }
		}
	}
}