SocialPub/PrivaPub/Federation/Objects/InstanceDescriber.cs

using MongoDB.Entities;

using PrivaPub.Infrastructure.Geo;
using PrivaPub.Infrastructure.Http;
using PrivaPub.Infrastructure.Jobs;
using PrivaPub.Models.Jobs;

using System.Globalization;
using System.Net.Sockets;
using System.Text.Json;

namespace PrivaPub.Federation.Objects
{
	public class InstanceDescriber : IJobHandler
	{
		const int MaxRawChars = 64 * 1024;

		readonly IFederationHttp _http;
		readonly IGeoLocator _geo;
		readonly IConnectedAddresses _addresses;
		readonly ICdnRanges _cdnRanges;
		readonly IEdgeHints _edges;

		public InstanceDescriber(IFederationHttp http, IGeoLocator geo = default, IConnectedAddresses addresses = default, ICdnRanges cdnRanges = default,
			IEdgeHints edges = default)
		{
			_http = http;
			_geo = geo;
			_addresses = addresses;
			_cdnRanges = cdnRanges;
			_edges = edges;
		}

		public JobKind Kind => JobKind.DescribeInstance;
		public int Concurrency => 1;
		public int MaxAttempts => 2;
		public int PerHostLimit => 1;

		public static string Week(DateTime at) => $"{ISOWeek.GetYear(at)}-W{ISOWeek.GetWeekOfYear(at):00}";

		public static string DedupeKey(string host, DateTime at) => $"describe|{host}|{ISOWeek.GetYear(at)}-{ISOWeek.GetWeekOfYear(at)}";

		public async Task<JobOutcome> Handle(Job job, CancellationToken token)
		{
			await Describe(job.Payload, crawled: false, default, token);
			return JobOutcome.Done;
		}

		//NodeInfo, the Mastodon instance API where the software has one, and where the address we reached sits; unsigned,
		//because none of these are ActivityPub documents. `allowed` is the crawler's robots.txt.
		public async Task<bool> Describe(string host, bool crawled, Func<string, bool> allowed, CancellationToken token)
		{
			allowed ??= _ => true;
			NodeInfoFacts facts = default;
			string rawNodeInfo = default, error;
			using (HttpScope.For("nodeinfo"))
			{
				error = allowed("/.well-known/nodeinfo") ? "no NodeInfo" : "robots.txt disallows NodeInfo";
				using var links = allowed("/.well-known/nodeinfo")
					? await _http.GetJson(Address($"https://{host}/.well-known/nodeinfo"), "application/json", sign: default, token)
					: default;
				var (href, version) = links == default ? default : InstanceDocuments.NodeInfoLink(links.Root);
				if (href != default && allowed(new Uri(href).AbsolutePath))
				{
					using var nodeInfo = await _http.GetJson(Address(href), "application/json", sign: default, token);
					if (nodeInfo != default)
					{
						facts = InstanceDocuments.NodeInfo(nodeInfo.Root, version);
						rawNodeInfo = nodeInfo.Root.GetRawText();
						error = default;
					}
					else
						error = "NodeInfo unreadable";
				}
			}

			InstanceApiFacts api = default;
			string apiKind = default, rawApi = default;
			if (InstanceDocuments.SpeaksMastodonApi(facts?.Software))
				using (HttpScope.For("instance-api"))
					foreach (var (path, kind) in new[] { ("/api/v2/instance", "mastodon-v2"), ("/api/v1/instance", "mastodon-v1") })
					{
						if (!allowed(path))
							continue;
						using var document = await _http.GetJson(Address($"https://{host}{path}"), "application/json", sign: default, token);
						if (document == default || document.Root.ValueKind != JsonValueKind.Object)
							continue;
						api = InstanceDocuments.InstanceApi(document.Root);
						apiKind = kind;
						rawApi = document.Root.GetRawText();
						break;
					}

			var geo = Locate(host);
			var now = DateTime.UtcNow;
			var update = DB.Default.Update<RemoteInstance>()
				.Match(i => i.Host == host)
				.Modify(i => i.DescribedAt, now)
				.Modify(i => i.DescriptionError, error)
				.Option(o => o.IsUpsert = true);
			if (crawled)
				update = update
					.Modify(b => b.SetOnInsert(i => i.Seen, "crawled"))
					.Modify(b => b.SetOnInsert(i => i.FirstSeenAt, now));
			if (facts != default)
				update = update
					.Modify(i => i.Software, facts.Software)
					.Modify(i => i.SoftwareVersion, facts.SoftwareVersion)
					.Modify(i => i.NodeName, facts.NodeName)
					.Modify(i => i.NodeDescription, facts.NodeDescription)
					.Modify(i => i.NodeInfoVersion, facts.Version)
					.Modify(i => i.Protocols, facts.Protocols)
					.Modify(i => i.OpenRegistrations, facts.OpenRegistrations)
					.Modify(i => i.UsersTotal, facts.UsersTotal)
					.Modify(i => i.UsersActiveMonth, facts.UsersActiveMonth)
					.Modify(i => i.UsersActiveHalfyear, facts.UsersActiveHalfyear)
					.Modify(i => i.LocalPosts, facts.LocalPosts)
					.Modify(i => i.LocalComments, facts.LocalComments)
					.Modify(i => i.NodeInfo, rawNodeInfo.Length <= MaxRawChars ? rawNodeInfo : default);
			if (api != default)
				update = update
					.Modify(i => i.InstanceApi, apiKind)
					.Modify(i => i.InstanceApiRaw, rawApi.Length <= MaxRawChars ? rawApi : default)
					.Modify(i => i.Title, api.Title)
					.Modify(i => i.Languages, api.Languages ?? facts?.Languages)
					.Modify(i => i.RegistrationMode, api.RegistrationMode)
					.Modify(i => i.MaxPostChars, api.MaxPostChars)
					.Modify(i => i.MastodonApiVersion, api.ApiVersion)
					.Modify(i => i.SourceUrl, api.SourceUrl);
			else if (facts?.Languages != default)
				update = update.Modify(i => i.Languages, facts.Languages);
			if (geo != default)
				update = update.Modify(i => i.Geo, geo);
			await update.ExecuteAsync(token);
			var instance = await DB.Default.Find<RemoteInstance>().Match(i => i.Host == host).ExecuteFirstAsync(token);

			var week = Week(now);
			await DB.Default.Update<RemoteInstanceSnapshot>()
				.Match(s => s.Host == host && s.Week == week)
				.Modify(s => s.TakenAt, now)
				.Modify(s => s.Seen, instance?.Seen)
				.Modify(s => s.Reachable, facts != default || api != default)
				.Modify(s => s.Software, instance?.Software)
				.Modify(s => s.SoftwareVersion, instance?.SoftwareVersion)
				.Modify(s => s.UsersTotal, instance?.UsersTotal)
				.Modify(s => s.UsersActiveMonth, instance?.UsersActiveMonth)
				.Modify(s => s.UsersActiveHalfyear, instance?.UsersActiveHalfyear)
				.Modify(s => s.LocalPosts, instance?.LocalPosts)
				.Modify(s => s.OpenRegistrations, instance?.OpenRegistrations)
				.Modify(s => s.RegistrationMode, instance?.RegistrationMode)
				.Modify(s => s.Languages, instance?.Languages)
				.Modify(s => s.Country, instance?.Geo?.Country)
				.Modify(s => s.City, instance?.Geo?.City)
				.Modify(s => s.Latitude, instance?.Geo?.Latitude)
				.Modify(s => s.Longitude, instance?.Geo?.Longitude)
				.Modify(s => s.Asn, instance?.Geo?.Asn)
				.Modify(s => s.Cdn, instance?.Geo?.Cdn)
				.Modify(s => s.CdnDomain, instance?.Geo?.CdnDomain)
				.Modify(s => s.CdnSource, instance?.Geo?.CdnSource)
				.Modify(s => s.Ipv6, instance?.Geo?.Ipv6)
				.Option(o => o.IsUpsert = true)
				.ExecuteAsync(token);
			return facts != default;
		}

		// Where the address we reached sits, and the CDN in front of it, found by its published ranges, then by the
		// fingerprint in the responses we just got, then by a network that carries only a CDN.
		InstanceGeo Locate(string host)
		{
			var reachedHost = new Uri(Address($"https://{host}/")).Host;
			var reached = _addresses?.Of(reachedHost);
			var fix = reached == default ? default : _geo?.Locate(reached);
			var (cdn, cdnSource) = Cdn(reached, reachedHost, fix?.Asn, _cdnRanges, _edges);
			if (fix == default && cdn == default)
				return default;
			return new InstanceGeo
			{
				Country = fix?.Country,
				City = fix?.City,
				Latitude = fix?.Latitude,
				Longitude = fix?.Longitude,
				Asn = fix?.Asn,
				AsnOrg = fix?.AsnOrg,
				Cdn = cdn?.Name,
				CdnDomain = cdn?.Domain,
				CdnSource = cdnSource,
				Ipv6 = reached?.AddressFamily == AddressFamily.InterNetworkV6,
				Source = _geo?.Source,
				LocatedAt = DateTime.UtcNow
			};
		}

		public static (CdnProvider Cdn, string Source) Cdn(System.Net.IPAddress address, string host, int? asn, ICdnRanges ranges, IEdgeHints edges)
		{
			if (ranges?.Of(address) is { } listed)
				return (listed, "ranges");
			if (edges?.Of(host) is { } hinted)
				return (hinted, "headers");
			if (CdnCatalog.OfAsn(asn) is { } network)
				return (network, "asn");
			return default;
		}

		//where an https address is fetched from: itself, except in tests, which point it at a plain-http peer
		protected virtual string Address(string url) => url;
	}
}