using MongoDB.Entities; using PrivaPub.Infrastructure.Geo; using PrivaPub.Infrastructure.Http; using PrivaPub.Infrastructure.Jobs; using PrivaPub.Models.Jobs; using System.Globalization; using System.Net.Sockets; using System.Text.Json; namespace PrivaPub.Federation.Objects { public class InstanceDescriber : IJobHandler { const int MaxRawChars = 64 * 1024; readonly IFederationHttp _http; readonly IGeoLocator _geo; readonly IConnectedAddresses _addresses; public InstanceDescriber(IFederationHttp http, IGeoLocator geo = default, IConnectedAddresses addresses = default) { _http = http; _geo = geo; _addresses = addresses; } public JobKind Kind => JobKind.DescribeInstance; public int Concurrency => 1; public int MaxAttempts => 2; public int PerHostLimit => 1; public static string Week(DateTime at) => $"{ISOWeek.GetYear(at)}-W{ISOWeek.GetWeekOfYear(at):00}"; public static string DedupeKey(string host, DateTime at) => $"describe|{host}|{ISOWeek.GetYear(at)}-{ISOWeek.GetWeekOfYear(at)}"; public async Task Handle(Job job, CancellationToken token) { await Describe(job.Payload, crawled: false, default, token); return JobOutcome.Done; } //NodeInfo, the Mastodon instance API where the software has one, and where the address we reached sits; unsigned, //because none of these are ActivityPub documents. `allowed` is the crawler's robots.txt. public async Task Describe(string host, bool crawled, Func allowed, CancellationToken token) { allowed ??= _ => true; NodeInfoFacts facts = default; string rawNodeInfo = default, error; using (HttpScope.For("nodeinfo")) { error = allowed("/.well-known/nodeinfo") ? "no NodeInfo" : "robots.txt disallows NodeInfo"; using var links = allowed("/.well-known/nodeinfo") ? await _http.GetJson(Address($"https://{host}/.well-known/nodeinfo"), "application/json", sign: default, token) : default; var (href, version) = links == default ? default : InstanceDocuments.NodeInfoLink(links.Root); if (href != default && allowed(new Uri(href).AbsolutePath)) { using var nodeInfo = await _http.GetJson(Address(href), "application/json", sign: default, token); if (nodeInfo != default) { facts = InstanceDocuments.NodeInfo(nodeInfo.Root, version); rawNodeInfo = nodeInfo.Root.GetRawText(); error = default; } else error = "NodeInfo unreadable"; } } InstanceApiFacts api = default; string apiKind = default, rawApi = default; if (InstanceDocuments.SpeaksMastodonApi(facts?.Software)) using (HttpScope.For("instance-api")) foreach (var (path, kind) in new[] { ("/api/v2/instance", "mastodon-v2"), ("/api/v1/instance", "mastodon-v1") }) { if (!allowed(path)) continue; using var document = await _http.GetJson(Address($"https://{host}{path}"), "application/json", sign: default, token); if (document == default || document.Root.ValueKind != JsonValueKind.Object) continue; api = InstanceDocuments.InstanceApi(document.Root); apiKind = kind; rawApi = document.Root.GetRawText(); break; } var geo = Locate(host); var now = DateTime.UtcNow; var update = DB.Default.Update() .Match(i => i.Host == host) .Modify(i => i.DescribedAt, now) .Modify(i => i.DescriptionError, error) .Option(o => o.IsUpsert = true); if (crawled) update = update .Modify(b => b.SetOnInsert(i => i.Seen, "crawled")) .Modify(b => b.SetOnInsert(i => i.FirstSeenAt, now)); if (facts != default) update = update .Modify(i => i.Software, facts.Software) .Modify(i => i.SoftwareVersion, facts.SoftwareVersion) .Modify(i => i.NodeName, facts.NodeName) .Modify(i => i.NodeDescription, facts.NodeDescription) .Modify(i => i.NodeInfoVersion, facts.Version) .Modify(i => i.Protocols, facts.Protocols) .Modify(i => i.OpenRegistrations, facts.OpenRegistrations) .Modify(i => i.UsersTotal, facts.UsersTotal) .Modify(i => i.UsersActiveMonth, facts.UsersActiveMonth) .Modify(i => i.UsersActiveHalfyear, facts.UsersActiveHalfyear) .Modify(i => i.LocalPosts, facts.LocalPosts) .Modify(i => i.LocalComments, facts.LocalComments) .Modify(i => i.NodeInfo, rawNodeInfo.Length <= MaxRawChars ? rawNodeInfo : default); if (api != default) update = update .Modify(i => i.InstanceApi, apiKind) .Modify(i => i.InstanceApiRaw, rawApi.Length <= MaxRawChars ? rawApi : default) .Modify(i => i.Title, api.Title) .Modify(i => i.Languages, api.Languages ?? facts?.Languages) .Modify(i => i.RegistrationMode, api.RegistrationMode) .Modify(i => i.MaxPostChars, api.MaxPostChars) .Modify(i => i.MastodonApiVersion, api.ApiVersion) .Modify(i => i.SourceUrl, api.SourceUrl); else if (facts?.Languages != default) update = update.Modify(i => i.Languages, facts.Languages); if (geo != default) update = update.Modify(i => i.Geo, geo); await update.ExecuteAsync(token); var instance = await DB.Default.Find().Match(i => i.Host == host).ExecuteFirstAsync(token); var week = Week(now); await DB.Default.Update() .Match(s => s.Host == host && s.Week == week) .Modify(s => s.TakenAt, now) .Modify(s => s.Seen, instance?.Seen) .Modify(s => s.Reachable, facts != default || api != default) .Modify(s => s.Software, instance?.Software) .Modify(s => s.SoftwareVersion, instance?.SoftwareVersion) .Modify(s => s.UsersTotal, instance?.UsersTotal) .Modify(s => s.UsersActiveMonth, instance?.UsersActiveMonth) .Modify(s => s.UsersActiveHalfyear, instance?.UsersActiveHalfyear) .Modify(s => s.LocalPosts, instance?.LocalPosts) .Modify(s => s.OpenRegistrations, instance?.OpenRegistrations) .Modify(s => s.RegistrationMode, instance?.RegistrationMode) .Modify(s => s.Languages, instance?.Languages) .Modify(s => s.Country, instance?.Geo?.Country) .Modify(s => s.Asn, instance?.Geo?.Asn) .Modify(s => s.Cdn, instance?.Geo?.Cdn) .Modify(s => s.Ipv6, instance?.Geo?.Ipv6) .Option(o => o.IsUpsert = true) .ExecuteAsync(token); return facts != default; } InstanceGeo Locate(string host) { var reached = _addresses?.Of(new Uri(Address($"https://{host}/")).Host); var fix = reached == default ? default : _geo?.Locate(reached); if (fix == default) return default; return new InstanceGeo { Country = fix.Country, City = fix.City, Latitude = fix.Latitude, Longitude = fix.Longitude, Asn = fix.Asn, AsnOrg = fix.AsnOrg, Cdn = CdnNetworks.Of(fix.Asn), Ipv6 = reached.AddressFamily == AddressFamily.InterNetworkV6, Source = _geo.Source, LocatedAt = DateTime.UtcNow }; } //where an https address is fetched from: itself, except in tests, which point it at a plain-http peer protected virtual string Address(string url) => url; } }