using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Mvc; using Microsoft.Extensions.Options; using MongoDB.Entities; using PrivaPub.Api.Mastodon.Infrastructure; using PrivaPub.Domain.Privacy; using PrivaPub.Domain.Statistics; using PrivaPub.Federation.Actors; using PrivaPub.Federation.Objects; using PrivaPub.Infrastructure; using PrivaPub.Infrastructure.Geo; using PrivaPub.Models.Federation; using PrivaPub.Models.Jobs; using PrivaPub.Models.Post; using PrivaPub.StaticServices; using System.Text.Json.Nodes; using PostEntity = PrivaPub.Models.Post.Post; namespace PrivaPub.Api.Mastodon.Controllers { public class ProvenanceController : MastodonController { const int MaxHosts = 40; readonly DbEntities _dbEntities; const int MaxWeeks = 260; readonly ISelfLocation _self; readonly IServerPlaces _places; readonly ILocalActorService _localActors; readonly IOptionsMonitor _registrations; public ProvenanceController(DbEntities dbEntities, ISelfLocation self, IServerPlaces places, ILocalActorService localActors, IOptionsMonitor registrations) { _dbEntities = dbEntities; _self = self; _places = places; _localActors = localActors; _registrations = registrations; } [HttpGet("/api/privapub/v1/statuses/{id}/provenance"), Scope("read:statuses", requiresUser: false), AllowAnonymous] public async Task Status(string id, CancellationToken token) { var post = await _dbEntities.Posts.Match(p => p.ID == id).Match(VisibilityPolicy.IsShown).ExecuteFirstAsync(token); if (post?.ReblogOfPostId != default) post = await _dbEntities.Posts.Match(p => p.ID == post.ReblogOfPostId).Match(VisibilityPolicy.IsShown).ExecuteFirstAsync(token); if (post == default || post.Visibility == PostVisibility.LocalGeo || !await VisibilityPolicy.CanSee(post, MyId, token)) return NotFoundError(); if (!post.IsFederatedCopy) return Json(new Provenance { ObjectUri = post.ObjectURI, Url = post.Url, Path = "local" }); var record = await DB.Default.Find().Match(r => r.PostId == post.ID).ExecuteFirstAsync(token) ?? await DB.Default.Find().Match(r => r.ObjectURI == post.ObjectURI).ExecuteFirstAsync(token); var host = record?.Host ?? (Uri.TryCreate(post.ObjectURI, UriKind.Absolute, out var uri) ? uri.Host.ToLowerInvariant() : default); var instance = host == default ? default : await DB.Default.Find().Match(i => i.Host == host).ExecuteFirstAsync(token); var beforeCdn = instance?.Geo?.Cdn == default ? default : (await _places.BeforeCdn(new[] { instance.Host }, token)).GetValueOrDefault(instance.Host); return Json(Describe(post, record, instance, beforeCdn)); } [HttpGet("/api/privapub/v1/instances/{host}"), Scope("read", requiresUser: false), AllowAnonymous] public async Task Instance(string host, CancellationToken token) { var described = await Describe(new[] { host }, token); return described.Count == 0 ? NotFoundError() : Json(described[0]); } //up to MaxHosts at once (host[]=...), in the order asked; hosts this server does not know are left out [HttpGet("/api/privapub/v1/instances"), Scope("read", requiresUser: false), AllowAnonymous] public async Task Instances(CancellationToken token) => Json(await Describe(Params.List("host"), token)); //the server's weekly snapshots, newest first (weeks=, at most MaxWeeks): its place as shown publicly, its CDN, its size [HttpGet("/api/privapub/v1/instances/{host}/history"), Scope("read", requiresUser: false), AllowAnonymous] public async Task History(string host, CancellationToken token) { host = host?.Trim().ToLowerInvariant(); if (string.IsNullOrEmpty(host) || host.Length > 253 || !await DB.Default.Find().Match(i => i.Host == host).ExecuteAnyAsync(token)) return NotFoundError(); var weeks = Math.Clamp(Params.Int("weeks") ?? 52, 1, MaxWeeks); return Json((await _places.History(host, weeks, token)).Select(s => new InstanceWeek { Week = s.Week, TakenAt = MastodonJson.Time(s.TakenAt), Reachable = s.Reachable, Software = s.Software, Version = s.SoftwareVersion, UsersTotal = s.UsersTotal, UsersActiveMonth = s.UsersActiveMonth, LocalPosts = s.LocalPosts, OpenRegistrations = s.OpenRegistrations, Geo = Located(PublicGeo.Project(new InstanceGeo { Country = s.Country, City = s.City, Latitude = s.Latitude, Longitude = s.Longitude, Asn = s.Asn, Cdn = s.Cdn, CdnDomain = s.CdnDomain, CdnSource = s.CdnSource, Ipv6 = s.Ipv6 }), new InstanceGeo { Cdn = s.Cdn, CdnDomain = s.CdnDomain ?? CdnCatalog.ByKey(s.Cdn)?.Key, CdnSource = s.CdnSource }) }).ToList()); } async Task> Describe(IEnumerable hosts, CancellationToken token) { var wanted = hosts.Select(h => h?.Trim().ToLowerInvariant()).Where(h => !string.IsNullOrEmpty(h) && h.Length <= 253) .Distinct().Take(MaxHosts).ToList(); var selfHost = Uri.TryCreate(_localActors.BaseAddress, UriKind.Absolute, out var self) ? self.Host.ToLowerInvariant() : default; var remote = await DB.Default.Find().Match(i => wanted.Contains(i.Host)).ExecuteAsync(token); var beforeCdn = await _places.BeforeCdn(remote.Where(i => i.Geo?.Cdn != default).Select(i => i.Host), token); var described = new List(); foreach (var host in wanted) { if (host == selfHost) described.Add(await DescribeSelf(host, token)); else if (remote.FirstOrDefault(i => i.Host == host) is { } instance) described.Add(Describe(instance, beforeCdn.GetValueOrDefault(host))); } return described; } async Task DescribeSelf(string host, CancellationToken token) { var geo = await _self.Get(token); return new InstanceDescription { Host = host, Software = "privapub", Version = BuildInfo.Ref, NodeName = "PrivaPub", Protocols = new() { "activitypub" }, OpenRegistrations = _registrations.CurrentValue.IsOpen, DescribedAt = MastodonJson.Time(DateTime.UtcNow), Geo = Located(PublicGeo.Project(geo), geo) }; } //the public projection of a server's place (PublicGeo); for a CDN-fronted one, the CDN's domain, how it was found and //where the server was before it static InstanceLocation Located(PublicLocation location, InstanceGeo geo, PlaceBeforeCdn beforeCdn = default) => location == default ? default : new InstanceLocation { Precision = location.Cdn != default ? "cdn" : location.Latitude != default ? "city" : "country", Country = location.Country, City = location.City, Latitude = location.Latitude, Longitude = location.Longitude, Network = location.Network, Cdn = location.Cdn, CdnDomain = location.Cdn != default ? ServerPlaces.KeyOf(geo) : default, CdnSource = location.Cdn != default ? geo?.CdnSource : default, BeforeCdn = location.Cdn == default || beforeCdn == default ? default : new InstancePlace { Week = beforeCdn.Week, Country = beforeCdn.Country, City = beforeCdn.City, Latitude = beforeCdn.Latitude, Longitude = beforeCdn.Longitude }, Source = geo?.Source, LocatedAt = geo?.LocatedAt is { } at ? MastodonJson.Time(at) : default }; Provenance Describe(PostEntity post, ObjectRecord record, RemoteInstance instance, PlaceBeforeCdn beforeCdn) { var raw = record?.Raw == default ? default : JsonNode.Parse(record.Raw) as JsonObject; return new Provenance { ObjectUri = post.ObjectURI, ObjectType = record?.ObjectType ?? post.ObjectType, Url = post.Url, Path = record == default ? "unrecorded" : record.Path == ObjectPath.Delivered ? "delivered" : "fetched", Refetched = record?.Refetched == true, Activity = record?.ActivityId == default ? default : new ProvenanceActivity { Id = record.ActivityId, Type = record.ActivityType, Actor = record.ActivityActorURI }, Inbox = record?.Inbox, FetchedBy = record?.Path == ObjectPath.Fetched ? "instance-actor" : default, Signature = record?.KeyId == default || record.Path == ObjectPath.Fetched ? default : new ProvenanceSignature { Scheme = record.SignatureScheme, KeyId = record.KeyId, Algorithm = record.Algorithm, Headers = record.SignedHeaders }, ReceivedAt = record == default ? default : MastodonJson.Time(record.ReceivedAt), Published = record?.Published is { } published ? MastodonJson.Time(published) : default, Updated = record?.Updated is { } updated ? MastodonJson.Time(updated) : default, Extensions = record?.Extensions ?? ObjectFeatures.Detect(raw), ContextNamespaces = record?.ContextNamespaces ?? ObjectFeatures.Namespaces(raw, record?.ActivityContext), Raw = raw, RawBytes = record?.RawBytes ?? 0, RawHash = record?.RawHash, RawTruncated = record?.RawTruncated == true, Revisions = record?.Revisions.Select(r => new ProvenanceRevision { ActivityId = r.ActivityId, Updated = r.Updated is { } at ? MastodonJson.Time(at) : default, ReceivedAt = MastodonJson.Time(r.ReceivedAt), RawHash = r.RawHash, Raw = r.Raw == default ? default : JsonNode.Parse(r.Raw) }).ToList() ?? new(), Instance = instance == default ? default : Describe(instance, beforeCdn) }; } static InstanceDescription Describe(RemoteInstance instance, PlaceBeforeCdn beforeCdn) => new() { Host = instance.Host, Software = instance.Software, Version = instance.SoftwareVersion, NodeName = instance.NodeName, Protocols = instance.Protocols ?? new(), OpenRegistrations = instance.OpenRegistrations, DescribedAt = instance.DescribedAt is { } described ? MastodonJson.Time(described) : default, DescriptionError = instance.DescriptionError, NodeInfo = instance.NodeInfo == default ? default : JsonNode.Parse(instance.NodeInfo), Delivery = new InstanceDelivery { ConsecutiveFailures = instance.ConsecutiveFailures, UnavailableUntil = instance.UnavailableUntil is { } until ? MastodonJson.Time(until) : default, LastSuccessAt = instance.LastSuccessAt is { } success ? MastodonJson.Time(success) : default, LastFailureAt = instance.LastFailureAt is { } failure ? MastodonJson.Time(failure) : default }, Geo = Located(PublicGeo.Project(instance.Geo), instance.Geo, beforeCdn) }; public class Provenance { public string ObjectUri { get; set; } public string ObjectType { get; set; } public string Url { get; set; } public string Path { get; set; } public bool Refetched { get; set; } public ProvenanceActivity Activity { get; set; } public string Inbox { get; set; } public string FetchedBy { get; set; } public ProvenanceSignature Signature { get; set; } public string ReceivedAt { get; set; } public string Published { get; set; } public string Updated { get; set; } public List Extensions { get; set; } = new(); public List ContextNamespaces { get; set; } = new(); public JsonNode Raw { get; set; } public int RawBytes { get; set; } public string RawHash { get; set; } public bool RawTruncated { get; set; } public List Revisions { get; set; } = new(); public InstanceDescription Instance { get; set; } } public class ProvenanceActivity { public string Id { get; set; } public string Type { get; set; } public string Actor { get; set; } } public class ProvenanceSignature { public string Scheme { get; set; } public string KeyId { get; set; } public string Algorithm { get; set; } public List Headers { get; set; } = new(); } public class ProvenanceRevision { public string ActivityId { get; set; } public string Updated { get; set; } public string ReceivedAt { get; set; } public string RawHash { get; set; } public JsonNode Raw { get; set; } } public class InstanceDescription { public string Host { get; set; } public string Software { get; set; } public string Version { get; set; } public string NodeName { get; set; } public List Protocols { get; set; } = new(); public bool? OpenRegistrations { get; set; } public string DescribedAt { get; set; } public string DescriptionError { get; set; } public JsonNode NodeInfo { get; set; } public InstanceDelivery Delivery { get; set; } public InstanceLocation Geo { get; set; } } //precision: "city" (coordinates to 0.1°, city and network), "country" (located only to a country) or "cdn" (the //CDN's name only) public class InstanceLocation { public string Precision { get; set; } public string Country { get; set; } public string City { get; set; } public double? Latitude { get; set; } public double? Longitude { get; set; } public string Network { get; set; } public string Cdn { get; set; } public string CdnDomain { get; set; } public string CdnSource { get; set; }//"ranges", "headers" or "asn" public InstancePlace BeforeCdn { get; set; }//where a CDN-fronted server was located before the CDN, by its snapshots public string Source { get; set; } public string LocatedAt { get; set; } } public class InstancePlace { public string Week { get; set; } public string Country { get; set; } public string City { get; set; } public double? Latitude { get; set; } public double? Longitude { get; set; } } public class InstanceWeek { public string Week { get; set; } public string TakenAt { get; set; } public bool Reachable { get; set; } public string Software { get; set; } public string Version { get; set; } public long? UsersTotal { get; set; } public long? UsersActiveMonth { get; set; } public long? LocalPosts { get; set; } public bool? OpenRegistrations { get; set; } public InstanceLocation Geo { get; set; } } public class InstanceDelivery { public int ConsecutiveFailures { get; set; } public string UnavailableUntil { get; set; } public string LastSuccessAt { get; set; } public string LastFailureAt { get; set; } } } }