From c8a305ae9262a7ad941b40c86f5fcea0084e89e3 Mon Sep 17 00:00:00 2001 From: thepra Date: Sat, 3 Oct 2026 12:07:57 +0200 Subject: [PATCH] M8: every server we touch is described, located and snapshotted weekly - Touches: the ledger marks a server as touched when it sends us a verified activity, when we exchange activities with it, or when we read its actors, keys, objects or WebFinger. It upserts RemoteInstance.Seen, FirstSeenAt and LastSeenAt at most hourly per server, and queues one DescribeInstance a week with the same dedupe key ObjectRecords uses. Suspended servers and pages behind link previews are never described. Migration _010 marks the servers already known as touched, with their dates. - InstanceDescriber.Describe(host, crawled, allowed) reads: - NodeInfo 2.2/2.1/2.0, now with its published user counts, posts, comments, description, languages and schema version; - for software with a Mastodon API, /api/v2/instance falling back to v1: title, languages, registration mode, character limit, API version, source URL. It never keeps a contact as a field; the raw document is kept for the admin only. It locates the server from the address our connection reached (DB-IP Lite city and ASN, the CDN named when fronted) and writes a RemoteInstanceSnapshot per ISO week, unreachable weeks included. A crawled server is upserted as crawled only on insert, so it never downgrades a touched one, and robots.txt can deny any path. - PublicGeo.Project is the only public form of a location: a CDN-fronted server shows its CDN only, a server reporting at least ten users shows its city, coordinates and network, any other only its country. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01ELjqpznMFMNrJoJUj6K5p2 --- PrivaPub.Tests/Http/InboxRouteTests.cs | 19 +- PrivaPub.Tests/Statistics/DescribeTests.cs | 261 ++++++++++++++++++ PrivaPub/Domain/Statistics/PublicGeo.cs | 23 ++ .../Federation/Objects/InstanceDescriber.cs | 208 ++++++++++---- .../Federation/Objects/InstanceDocuments.cs | 156 +++++++++++ PrivaPub/Federation/Objects/ObjectRecords.cs | 3 +- PrivaPub/Infrastructure/Data/Indexes.cs | 5 + .../_010_remote_instances_were_touched.cs | 24 ++ .../Statistics/InteractionLedger.cs | 67 ++++- PrivaPub/Models/Jobs/RemoteInstance.cs | 63 +++++ 10 files changed, 757 insertions(+), 72 deletions(-) create mode 100644 PrivaPub.Tests/Statistics/DescribeTests.cs create mode 100644 PrivaPub/Domain/Statistics/PublicGeo.cs create mode 100644 PrivaPub/Federation/Objects/InstanceDocuments.cs create mode 100644 PrivaPub/Infrastructure/Data/Migrations/_010_remote_instances_were_touched.cs diff --git a/PrivaPub.Tests/Http/InboxRouteTests.cs b/PrivaPub.Tests/Http/InboxRouteTests.cs index 10f1322..d2dce28 100644 --- a/PrivaPub.Tests/Http/InboxRouteTests.cs +++ b/PrivaPub.Tests/Http/InboxRouteTests.cs @@ -188,18 +188,15 @@ namespace PrivaPub.Tests.Http return response.StatusCode; } - var clock = System.Diagnostics.Stopwatch.StartNew(); - var first = new List(); - for (var i = 0; i < 300; i++) - first.Add(await Post()); - var limited = await Post(); - // The bucket refills 50 every ten seconds: a slow machine may have earned a few more before the limit bites. - for (var refilled = (int)((clock.Elapsed.TotalSeconds + 1) / 10) * 50; limited != HttpStatusCode.TooManyRequests && refilled > 0; refilled--) - limited = await Post(); + // The bucket holds 300 and refills 50 every ten seconds on its own timer, so a refill can land at any moment: + // the first refusal comes after at least 300 posts and within a few refills. + var answers = new List(); + while (answers.Count < 600 && (answers.Count == 0 || answers[^1] != HttpStatusCode.TooManyRequests)) + answers.Add(await Post()); - Assert.DoesNotContain(HttpStatusCode.TooManyRequests, first); - Assert.All(first, status => Assert.Equal(HttpStatusCode.Unauthorized, status)); - Assert.Equal(HttpStatusCode.TooManyRequests, limited); + Assert.Equal(HttpStatusCode.TooManyRequests, answers[^1]); + Assert.True(answers.Count > 300, $"limited after only {answers.Count - 1} posts"); + Assert.All(answers.SkipLast(1), status => Assert.Equal(HttpStatusCode.Unauthorized, status)); var polite = new RemoteActor(_peer, "polite"); using var signed = await client.SendAsync(polite.SignedPost("/human-centipede", Direct(polite)), token); Assert.Equal(HttpStatusCode.Accepted, signed.StatusCode); diff --git a/PrivaPub.Tests/Statistics/DescribeTests.cs b/PrivaPub.Tests/Statistics/DescribeTests.cs new file mode 100644 index 0000000..c634c13 --- /dev/null +++ b/PrivaPub.Tests/Statistics/DescribeTests.cs @@ -0,0 +1,261 @@ +using Microsoft.Extensions.Logging.Abstractions; + +using MongoDB.Entities; + +using PrivaPub.Domain.Statistics; +using PrivaPub.Federation.Moderation; +using PrivaPub.Federation.Objects; +using PrivaPub.Infrastructure.Data.Migrations; +using PrivaPub.Infrastructure.Geo; +using PrivaPub.Infrastructure.Http; +using PrivaPub.Infrastructure.Jobs; +using PrivaPub.Infrastructure.Statistics; +using PrivaPub.Models.Federation; +using PrivaPub.Models.Jobs; +using PrivaPub.Models.Statistics; +using PrivaPub.Tests.Support; + +using System.Net; +using System.Text.Json; + +namespace PrivaPub.Tests.Statistics +{ + public class InstanceDocumentsTests + { + static JsonElement Json(string text) => JsonDocument.Parse(text).RootElement; + + [Fact] + public void NodeInfo_gives_software_usage_and_languages_whatever_their_shape() + { + var facts = InstanceDocuments.NodeInfo(Json(""" + {"software":{"name":"mastodon","version":"4.7.3"},"protocols":["activitypub"],"openRegistrations":false, + "usage":{"users":{"total":"1234","activeMonth":56.0,"activeHalfyear":-3},"localPosts":99999999999,"localComments":7}, + "metadata":{"nodeName":"Example","nodeDescription":"

A small\n place

","languages":["en","DE","klingon!"]}} + """), "2.1"); + + Assert.Equal(("mastodon", "4.7.3", "Example", "2.1"), (facts.Software, facts.SoftwareVersion, facts.NodeName, facts.Version)); + Assert.Equal("A small place", facts.NodeDescription); + Assert.Equal(new[] { "activitypub" }, facts.Protocols); + Assert.False(facts.OpenRegistrations); + Assert.Equal(1234, facts.UsersTotal); + Assert.Equal(56, facts.UsersActiveMonth); + Assert.Equal(0, facts.UsersActiveHalfyear); + Assert.Equal(1_000_000_000, facts.LocalPosts); + Assert.Equal(7, facts.LocalComments); + Assert.Equal(new[] { "en", "de" }, facts.Languages); + Assert.Null(InstanceDocuments.NodeInfo(Json("{}"), "2.0").UsersTotal); + } + + [Fact] + public void The_newest_https_nodeinfo_link_is_chosen() + { + var links = Json(""" + {"links":[{"rel":"http://nodeinfo.diaspora.software/ns/schema/2.0","href":"https://a.example/2.0"}, + {"rel":"http://nodeinfo.diaspora.software/ns/schema/2.1","href":"http://a.example/2.1"}, + {"rel":"http://nodeinfo.diaspora.software/ns/schema/2.2","href":"https://a.example/2.2"}]} + """); + + Assert.Equal(("https://a.example/2.2", "2.2"), InstanceDocuments.NodeInfoLink(links)); + Assert.Equal(default, InstanceDocuments.NodeInfoLink(Json("[]"))); + } + + [Fact] + public void The_instance_api_gives_registration_mode_limits_and_never_a_contact() + { + var v2 = InstanceDocuments.InstanceApi(Json(""" + {"title":"Example","languages":["en"],"registrations":{"enabled":true,"approval_required":true}, + "configuration":{"statuses":{"max_characters":500}},"api_versions":{"mastodon":7},"source_url":"https://github.com/mastodon/mastodon", + "contact":{"email":"admin@a.example","account":{"username":"admin"}}} + """)); + var v1 = InstanceDocuments.InstanceApi(Json("""{"title":"Old","registrations":false,"max_toot_chars":"5000"}""")); + + Assert.Equal(("Example", "approval", 500, 7, "https://github.com/mastodon/mastodon"), (v2.Title, v2.RegistrationMode, v2.MaxPostChars!.Value, v2.ApiVersion!.Value, v2.SourceUrl)); + Assert.Equal(new[] { "en" }, v2.Languages); + Assert.DoesNotContain("admin", System.Text.Json.JsonSerializer.Serialize(v2)); + Assert.Equal(("closed", 5000), (v1.RegistrationMode, v1.MaxPostChars!.Value)); + Assert.True(InstanceDocuments.SpeaksMastodonApi("GoToSocial")); + Assert.False(InstanceDocuments.SpeaksMastodonApi("lemmy")); + } + + [Fact] + public void The_public_location_depends_on_size_and_on_a_cdn() + { + var geo = new InstanceGeo { Country = "DE", City = "Berlin", Latitude = 52.5, Longitude = 13.4, AsnOrg = "Hetzner", Asn = 24940 }; + + Assert.Equal(new PublicLocation("DE", "Berlin", 52.5, 13.4, "Hetzner", null), PublicGeo.Project(new RemoteInstance { Geo = geo, UsersTotal = 10 }, 10)); + Assert.Equal(new PublicLocation("DE", null, null, null, null, null), PublicGeo.Project(new RemoteInstance { Geo = geo, UsersTotal = 1 }, 10)); + Assert.Equal(new PublicLocation("DE", null, null, null, null, null), PublicGeo.Project(new RemoteInstance { Geo = geo }, 10)); + Assert.Equal(new PublicLocation(null, null, null, null, null, "Cloudflare"), + PublicGeo.Project(new RemoteInstance { Geo = new InstanceGeo { Country = "US", City = "SF", Cdn = "Cloudflare" }, UsersTotal = 5000 }, 10)); + Assert.Null(PublicGeo.Project(new RemoteInstance(), 10)); + } + } + + [Trait("Category", "Integration")] + public sealed class DescribeTests : IAsyncLifetime + { + Peer _peer; + + public async ValueTask InitializeAsync() + { + Assert.SkipUnless(MongoFixture.Enabled, MongoFixture.Skip); + _peer = await Peer.Start(); + } + + public async ValueTask DisposeAsync() + { + if (_peer != default) + await _peer.DisposeAsync(); + } + + sealed class PeerDescriber : InstanceDescriber + { + readonly string _peer; + + public PeerDescriber(IFederationHttp http, IGeoLocator geo, IConnectedAddresses addresses, string peer) : base(http, geo, addresses) => _peer = peer; + + protected override string Address(string url) => _peer + new Uri(url).PathAndQuery; + } + + sealed class FixedGeo : IGeoLocator + { + public GeoFix Locate(IPAddress address) => new("NL", "Amsterdam", 52.4, 4.9, 60781, "LeaseWeb"); + public string Source => "test"; + } + + void ServeServer(string prefix, string software, bool v2 = true) + { + _peer.Serve($"{prefix}/.well-known/nodeinfo", $$"""{"links":[{"rel":"http://nodeinfo.diaspora.software/ns/schema/2.1","href":"https://ignored.example/nodeinfo/2.1"}]}"""); + _peer.Serve($"{prefix}/nodeinfo/2.1", $$$"""{"software":{"name":"{{{software}}}","version":"1.0"},"usage":{"users":{"total":42,"activeMonth":7}},"openRegistrations":true}"""); + if (v2) + _peer.Serve($"{prefix}/api/v2/instance", """{"title":"A test server","languages":["it"],"registrations":{"enabled":true,"approval_required":false}}"""); + _peer.Serve($"{prefix}/api/v1/instance", """{"title":"An old test server","registrations":true}"""); + } + + [Fact] + public async Task A_server_is_described_located_and_snapshotted_weekly() + { + var token = TestContext.Current.CancellationToken; + var host = $"described{Guid.NewGuid():N}.example"; + ServeServer("/" + host, "mastodon"); + var addresses = new ConnectedAddresses(); + var describer = new PeerDescriber(Peer.Http(connected: addresses), new FixedGeo(), addresses, _peer.A + "/" + host); + + Assert.True(await describer.Describe(host, crawled: false, default, token)); + + var instance = await DB.Default.Find().Match(i => i.Host == host).ExecuteSingleAsync(token); + Assert.Equal(("mastodon", 42L, 7L, "2.1"), (instance.Software, instance.UsersTotal!.Value, instance.UsersActiveMonth!.Value, instance.NodeInfoVersion)); + Assert.Equal(("A test server", "open", "mastodon-v2"), (instance.Title, instance.RegistrationMode, instance.InstanceApi)); + Assert.Equal(new[] { "it" }, instance.Languages); + Assert.Equal(("NL", "Amsterdam", "LeaseWeb", "test"), (instance.Geo.Country, instance.Geo.City, instance.Geo.AsnOrg, instance.Geo.Source)); + Assert.False(instance.Geo.Ipv6); + var snapshot = await DB.Default.Find().Match(s => s.Host == host).ExecuteSingleAsync(token); + Assert.True(snapshot.Reachable); + Assert.Equal((42L, "NL"), (snapshot.UsersTotal!.Value, snapshot.Country)); + Assert.Equal(InstanceDescriber.Week(DateTime.UtcNow), snapshot.Week); + } + + [Fact] + public async Task Without_api_v2_the_v1_api_is_read_and_an_unreachable_week_is_recorded_too() + { + var token = TestContext.Current.CancellationToken; + var old = $"old{Guid.NewGuid():N}.example"; + var gone = $"gone{Guid.NewGuid():N}.example"; + ServeServer("/" + old, "pleroma", v2: false); + + await new PeerDescriber(Peer.Http(), default, default, _peer.A + "/" + old).Describe(old, crawled: false, default, token); + Assert.False(await new PeerDescriber(Peer.Http(), default, default, _peer.A + "/" + gone).Describe(gone, crawled: false, default, token)); + + var oldOne = await DB.Default.Find().Match(i => i.Host == old).ExecuteSingleAsync(token); + Assert.Equal(("mastodon-v1", "An old test server", "open"), (oldOne.InstanceApi, oldOne.Title, oldOne.RegistrationMode)); + Assert.Null(oldOne.Geo); + var goneOne = await DB.Default.Find().Match(i => i.Host == gone).ExecuteSingleAsync(token); + Assert.Equal("no NodeInfo", goneOne.DescriptionError); + Assert.False((await DB.Default.Find().Match(s => s.Host == gone).ExecuteSingleAsync(token)).Reachable); + } + + [Fact] + public async Task A_crawled_server_never_downgrades_a_touched_one_and_robots_are_obeyed() + { + var token = TestContext.Current.CancellationToken; + var touched = $"touched{Guid.NewGuid():N}.example"; + var shy = $"shy{Guid.NewGuid():N}.example"; + ServeServer("/" + touched, "gotosocial"); + ServeServer("/" + shy, "mastodon"); + await DB.Default.SaveAsync(new RemoteInstance { Host = touched, Seen = "touched" }, token); + + await new PeerDescriber(Peer.Http(), default, default, _peer.A + "/" + touched).Describe(touched, crawled: true, default, token); + await new PeerDescriber(Peer.Http(), default, default, _peer.A + "/" + shy).Describe(shy, crawled: true, path => !path.StartsWith("/api/"), token); + + Assert.Equal("touched", (await DB.Default.Find().Match(i => i.Host == touched).ExecuteSingleAsync(token)).Seen); + var shyOne = await DB.Default.Find().Match(i => i.Host == shy).ExecuteSingleAsync(token); + Assert.Equal(("crawled", "mastodon"), (shyOne.Seen, shyOne.Software)); + Assert.Null(shyOne.InstanceApi); + Assert.DoesNotContain(_peer.Requests, r => r.Path.StartsWith("/" + shy + "/api/", StringComparison.Ordinal)); + Assert.Contains(_peer.Requests, r => r.Path == "/" + shy + "/nodeinfo/2.1"); + } + } + + [Trait("Category", "Integration")] + public sealed class TouchTests : IAsyncLifetime + { + public ValueTask InitializeAsync() + { + Assert.SkipUnless(MongoFixture.Enabled, MongoFixture.Skip); + return ValueTask.CompletedTask; + } + + public ValueTask DisposeAsync() => ValueTask.CompletedTask; + + [Fact] + public async Task A_server_we_exchange_with_is_seen_and_described_once_a_week_unless_suspended() + { + var token = TestContext.Current.CancellationToken; + var friend = $"friend{Guid.NewGuid():N}.example"; + var spammer = $"spam{Guid.NewGuid():N}.example"; + var linked = $"linked{Guid.NewGuid():N}.example"; + var looked = $"looked{Guid.NewGuid():N}.example"; + var blocks = new DomainBlocks(NullLogger.Instance); + blocks.Load(new[] { new DomainBlock { Domain = spammer, Severity = DomainBlockSeverity.Suspend } }); + var ledger = new InteractionLedger(new InteractionSalts(), new StaticOptions(new StatisticsOptions()), NullLogger.Instance, + new JobQueue(j => j.DedupeKey == "never"), blocks); + + ledger.Record(new InteractionEvent { Host = friend, Channel = "in", Activity = "Create", Outcome = "accepted" }, "https://" + friend + "/users/a"); + ledger.Record(new InteractionEvent { Host = friend, Channel = "out", Activity = "Like", Outcome = "ok" }); + ledger.Record(new InteractionEvent { Host = spammer, Channel = "out", Activity = "Reject", Outcome = "ok" }); + ledger.Record(new InteractionEvent { Host = linked, Channel = "preview", Outcome = "ok" }); + ledger.Count(looked, "http:actor:ok"); + await ledger.Flush(token); + + var seen = await DB.Default.Find().Match(i => i.Host == friend).ExecuteSingleAsync(token); + Assert.Equal("touched", seen.Seen); + Assert.NotNull(seen.FirstSeenAt); + Assert.Single(await DB.Default.Find().Match(j => j.Kind == JobKind.DescribeInstance && j.Payload == friend).ExecuteAsync(token)); + Assert.True(await DB.Default.Find().Match(j => j.Kind == JobKind.DescribeInstance && j.Payload == looked).ExecuteAnyAsync(token)); + Assert.False(await DB.Default.Find().Match(j => j.Kind == JobKind.DescribeInstance && j.Payload == spammer).ExecuteAnyAsync(token)); + Assert.False(await DB.Default.Find().Match(i => i.Host == linked).ExecuteAnyAsync(token)); + } + } + + [Xunit.Collection(nameof(Exclusive))] + [Trait("Category", "Integration")] + public sealed class TouchedMigrationTests + { + [Fact] + public async Task Known_servers_become_touched_with_their_dates() + { + Assert.SkipUnless(MongoFixture.Enabled, MongoFixture.Skip); + var token = TestContext.Current.CancellationToken; + var host = $"legacy{Guid.NewGuid():N}.example"; + var success = DateTime.UtcNow.AddDays(-2); + await DB.Default.SaveAsync(new RemoteInstance { Host = host, LastSuccessAt = success, DescribedAt = success.AddDays(-1) }, token); + + await new _010_remote_instances_were_touched().UpgradeAsync(); + + var instance = await DB.Default.Find().Match(i => i.Host == host).ExecuteSingleAsync(token); + Assert.Equal("touched", instance.Seen); + Assert.NotNull(instance.FirstSeenAt); + Assert.InRange(instance.LastSeenAt!.Value, success.AddSeconds(-1), success.AddSeconds(1)); + } + } +} diff --git a/PrivaPub/Domain/Statistics/PublicGeo.cs b/PrivaPub/Domain/Statistics/PublicGeo.cs new file mode 100644 index 0000000..aaa3809 --- /dev/null +++ b/PrivaPub/Domain/Statistics/PublicGeo.cs @@ -0,0 +1,23 @@ +using PrivaPub.Models.Jobs; + +namespace PrivaPub.Domain.Statistics +{ + public sealed record PublicLocation(string Country, string City, double? Latitude, double? Longitude, string Network, string Cdn); + + public static class PublicGeo + { + //owner decision: a CDN-fronted server shows only its CDN (the address is an edge), a server reporting at least + //minUsers users its city, coordinates and network, any other only its country + public static PublicLocation Project(RemoteInstance instance, int minUsers) + { + var geo = instance?.Geo; + if (geo == default) + return default; + if (geo.Cdn != default) + return new PublicLocation(default, default, default, default, default, geo.Cdn); + if (instance.UsersTotal >= minUsers) + return new PublicLocation(geo.Country, geo.City, geo.Latitude, geo.Longitude, geo.AsnOrg, default); + return geo.Country == default ? default : new PublicLocation(geo.Country, default, default, default, default, default); + } + } +} diff --git a/PrivaPub/Federation/Objects/InstanceDescriber.cs b/PrivaPub/Federation/Objects/InstanceDescriber.cs index 8b0522f..384aaed 100644 --- a/PrivaPub/Federation/Objects/InstanceDescriber.cs +++ b/PrivaPub/Federation/Objects/InstanceDescriber.cs @@ -1,88 +1,180 @@ using MongoDB.Entities; +using PrivaPub.Infrastructure.Geo; using PrivaPub.Infrastructure.Http; using PrivaPub.Infrastructure.Jobs; using PrivaPub.Models.Jobs; +using System.Globalization; +using System.Net.Sockets; using System.Text.Json; namespace PrivaPub.Federation.Objects { public class InstanceDescriber : IJobHandler { - const int MaxNodeInfoChars = 64 * 1024; - static readonly string[] Schemas = - { - "http://nodeinfo.diaspora.software/ns/schema/2.1", - "http://nodeinfo.diaspora.software/ns/schema/2.0" - }; + const int MaxRawChars = 64 * 1024; readonly IFederationHttp _http; + readonly IGeoLocator _geo; + readonly IConnectedAddresses _addresses; - public InstanceDescriber(IFederationHttp http) => _http = http; + public InstanceDescriber(IFederationHttp http, IGeoLocator geo = default, IConnectedAddresses addresses = default) + { + _http = http; + _geo = geo; + _addresses = addresses; + } public JobKind Kind => JobKind.DescribeInstance; public int Concurrency => 1; public int MaxAttempts => 2; public int PerHostLimit => 1; + public static string Week(DateTime at) => $"{ISOWeek.GetYear(at)}-W{ISOWeek.GetWeekOfYear(at):00}"; + + public static string DedupeKey(string host, DateTime at) => $"describe|{host}|{ISOWeek.GetYear(at)}-{ISOWeek.GetWeekOfYear(at)}"; + public async Task Handle(Job job, CancellationToken token) { - var host = job.Payload; - using var scope = HttpScope.For("nodeinfo"); - using var links = await _http.GetJson(Address($"https://{host}/.well-known/nodeinfo"), "application/json", sign: default, token); - var href = links == default ? default : Schemas.Select(schema => LinkFor(links.Root, schema)).FirstOrDefault(link => link != default); - if (href == default) - return await Described(host, default, "no NodeInfo", token); + await Describe(job.Payload, crawled: false, default, token); + return JobOutcome.Done; + } - using var nodeInfo = await _http.GetJson(Address(href), "application/json", sign: default, token); - return nodeInfo == default ? await Described(host, default, "NodeInfo unreadable", token) : await Described(host, nodeInfo.Root, default, token); + //NodeInfo, the Mastodon instance API where the software has one, and where the address we reached sits; unsigned, + //because none of these are ActivityPub documents. `allowed` is the crawler's robots.txt. + public async Task Describe(string host, bool crawled, Func allowed, CancellationToken token) + { + allowed ??= _ => true; + NodeInfoFacts facts = default; + string rawNodeInfo = default, error; + using (HttpScope.For("nodeinfo")) + { + error = allowed("/.well-known/nodeinfo") ? "no NodeInfo" : "robots.txt disallows NodeInfo"; + using var links = allowed("/.well-known/nodeinfo") + ? await _http.GetJson(Address($"https://{host}/.well-known/nodeinfo"), "application/json", sign: default, token) + : default; + var (href, version) = links == default ? default : InstanceDocuments.NodeInfoLink(links.Root); + if (href != default && allowed(new Uri(href).AbsolutePath)) + { + using var nodeInfo = await _http.GetJson(Address(href), "application/json", sign: default, token); + if (nodeInfo != default) + { + facts = InstanceDocuments.NodeInfo(nodeInfo.Root, version); + rawNodeInfo = nodeInfo.Root.GetRawText(); + error = default; + } + else + error = "NodeInfo unreadable"; + } + } + + InstanceApiFacts api = default; + string apiKind = default, rawApi = default; + if (InstanceDocuments.SpeaksMastodonApi(facts?.Software)) + using (HttpScope.For("instance-api")) + foreach (var (path, kind) in new[] { ("/api/v2/instance", "mastodon-v2"), ("/api/v1/instance", "mastodon-v1") }) + { + if (!allowed(path)) + continue; + using var document = await _http.GetJson(Address($"https://{host}{path}"), "application/json", sign: default, token); + if (document == default || document.Root.ValueKind != JsonValueKind.Object) + continue; + api = InstanceDocuments.InstanceApi(document.Root); + apiKind = kind; + rawApi = document.Root.GetRawText(); + break; + } + + var geo = Locate(host); + var now = DateTime.UtcNow; + var update = DB.Default.Update() + .Match(i => i.Host == host) + .Modify(i => i.DescribedAt, now) + .Modify(i => i.DescriptionError, error) + .Option(o => o.IsUpsert = true); + if (crawled) + update = update + .Modify(b => b.SetOnInsert(i => i.Seen, "crawled")) + .Modify(b => b.SetOnInsert(i => i.FirstSeenAt, now)); + if (facts != default) + update = update + .Modify(i => i.Software, facts.Software) + .Modify(i => i.SoftwareVersion, facts.SoftwareVersion) + .Modify(i => i.NodeName, facts.NodeName) + .Modify(i => i.NodeDescription, facts.NodeDescription) + .Modify(i => i.NodeInfoVersion, facts.Version) + .Modify(i => i.Protocols, facts.Protocols) + .Modify(i => i.OpenRegistrations, facts.OpenRegistrations) + .Modify(i => i.UsersTotal, facts.UsersTotal) + .Modify(i => i.UsersActiveMonth, facts.UsersActiveMonth) + .Modify(i => i.UsersActiveHalfyear, facts.UsersActiveHalfyear) + .Modify(i => i.LocalPosts, facts.LocalPosts) + .Modify(i => i.LocalComments, facts.LocalComments) + .Modify(i => i.NodeInfo, rawNodeInfo.Length <= MaxRawChars ? rawNodeInfo : default); + if (api != default) + update = update + .Modify(i => i.InstanceApi, apiKind) + .Modify(i => i.InstanceApiRaw, rawApi.Length <= MaxRawChars ? rawApi : default) + .Modify(i => i.Title, api.Title) + .Modify(i => i.Languages, api.Languages ?? facts?.Languages) + .Modify(i => i.RegistrationMode, api.RegistrationMode) + .Modify(i => i.MaxPostChars, api.MaxPostChars) + .Modify(i => i.MastodonApiVersion, api.ApiVersion) + .Modify(i => i.SourceUrl, api.SourceUrl); + else if (facts?.Languages != default) + update = update.Modify(i => i.Languages, facts.Languages); + if (geo != default) + update = update.Modify(i => i.Geo, geo); + await update.ExecuteAsync(token); + var instance = await DB.Default.Find().Match(i => i.Host == host).ExecuteFirstAsync(token); + + var week = Week(now); + await DB.Default.Update() + .Match(s => s.Host == host && s.Week == week) + .Modify(s => s.TakenAt, now) + .Modify(s => s.Seen, instance?.Seen) + .Modify(s => s.Reachable, facts != default || api != default) + .Modify(s => s.Software, instance?.Software) + .Modify(s => s.SoftwareVersion, instance?.SoftwareVersion) + .Modify(s => s.UsersTotal, instance?.UsersTotal) + .Modify(s => s.UsersActiveMonth, instance?.UsersActiveMonth) + .Modify(s => s.UsersActiveHalfyear, instance?.UsersActiveHalfyear) + .Modify(s => s.LocalPosts, instance?.LocalPosts) + .Modify(s => s.OpenRegistrations, instance?.OpenRegistrations) + .Modify(s => s.RegistrationMode, instance?.RegistrationMode) + .Modify(s => s.Languages, instance?.Languages) + .Modify(s => s.Country, instance?.Geo?.Country) + .Modify(s => s.Asn, instance?.Geo?.Asn) + .Modify(s => s.Cdn, instance?.Geo?.Cdn) + .Modify(s => s.Ipv6, instance?.Geo?.Ipv6) + .Option(o => o.IsUpsert = true) + .ExecuteAsync(token); + return facts != default; + } + + InstanceGeo Locate(string host) + { + var reached = _addresses?.Of(new Uri(Address($"https://{host}/")).Host); + var fix = reached == default ? default : _geo?.Locate(reached); + if (fix == default) + return default; + return new InstanceGeo + { + Country = fix.Country, + City = fix.City, + Latitude = fix.Latitude, + Longitude = fix.Longitude, + Asn = fix.Asn, + AsnOrg = fix.AsnOrg, + Cdn = CdnNetworks.Of(fix.Asn), + Ipv6 = reached.AddressFamily == AddressFamily.InterNetworkV6, + Source = _geo.Source, + LocatedAt = DateTime.UtcNow + }; } //where an https address is fetched from: itself, except in tests, which point it at a plain-http peer protected virtual string Address(string url) => url; - - static async Task Described(string host, JsonElement? nodeInfo, string error, CancellationToken token) - { - var update = DB.Default.Update() - .Match(i => i.Host == host) - .Modify(i => i.DescribedAt, DateTime.UtcNow) - .Modify(i => i.DescriptionError, error) - .Option(o => o.IsUpsert = true); - if (nodeInfo is { } root) - { - var raw = root.GetRawText(); - update = update - .Modify(i => i.Software, Text(root, "software", "name")) - .Modify(i => i.SoftwareVersion, Text(root, "software", "version")) - .Modify(i => i.NodeName, Text(root, "metadata", "nodeName")) - .Modify(i => i.Protocols, root.TryGetProperty("protocols", out var protocols) && protocols.ValueKind == JsonValueKind.Array - ? protocols.EnumerateArray().Where(p => p.ValueKind == JsonValueKind.String).Select(p => p.GetString()).Take(16).ToList() - : new List()) - .Modify(i => i.OpenRegistrations, root.TryGetProperty("openRegistrations", out var open) && open.ValueKind is JsonValueKind.True or JsonValueKind.False - ? open.GetBoolean() - : default(bool?)) - .Modify(i => i.NodeInfo, raw.Length <= MaxNodeInfoChars ? raw : default); - } - await update.ExecuteAsync(token); - return JobOutcome.Done; - } - - static string LinkFor(JsonElement root, string schema) => - root.TryGetProperty("links", out var links) && links.ValueKind == JsonValueKind.Array - ? links.EnumerateArray() - .Where(l => l.ValueKind == JsonValueKind.Object && l.TryGetProperty("rel", out var rel) && rel.ValueKind == JsonValueKind.String && rel.GetString() == schema) - .Select(l => l.TryGetProperty("href", out var href) && href.ValueKind == JsonValueKind.String ? href.GetString() : default) - .FirstOrDefault(h => h != default && h.StartsWith("https://", StringComparison.OrdinalIgnoreCase)) - : default; - - static string Text(JsonElement root, string parent, string name) - { - if (!root.TryGetProperty(parent, out var section) || section.ValueKind != JsonValueKind.Object - || !section.TryGetProperty(name, out var value) || value.ValueKind != JsonValueKind.String) - return default; - var text = value.GetString(); - return text.Length <= 200 ? text : text[..200]; - } } } diff --git a/PrivaPub/Federation/Objects/InstanceDocuments.cs b/PrivaPub/Federation/Objects/InstanceDocuments.cs new file mode 100644 index 0000000..b51f6ea --- /dev/null +++ b/PrivaPub/Federation/Objects/InstanceDocuments.cs @@ -0,0 +1,156 @@ +using System.Globalization; +using System.Text.Json; +using System.Text.RegularExpressions; + +namespace PrivaPub.Federation.Objects +{ + public sealed record NodeInfoFacts(string Version, string Software, string SoftwareVersion, string NodeName, string NodeDescription, + List Protocols, bool? OpenRegistrations, long? UsersTotal, long? UsersActiveMonth, long? UsersActiveHalfyear, long? LocalPosts, + long? LocalComments, List Languages); + + public sealed record InstanceApiFacts(string Title, List Languages, string RegistrationMode, int? MaxPostChars, int? ApiVersion, string SourceUrl); + + public static partial class InstanceDocuments + { + public static readonly string[] Schemas = + { + "http://nodeinfo.diaspora.software/ns/schema/2.2", + "http://nodeinfo.diaspora.software/ns/schema/2.1", + "http://nodeinfo.diaspora.software/ns/schema/2.0" + }; + + static readonly HashSet MastodonLike = new(StringComparer.OrdinalIgnoreCase) + { + "mastodon", "hometown", "glitch-soc", "gotosocial", "pleroma", "akkoma", "sharkey", "iceshrimp", "iceshrimp.net", "firefish", "catodon", + "friendica", "pixelfed", "takahe", "snac", "honk", "mitra", "pachli" + }; + + const long MaxCount = 1_000_000_000; + + public static bool SpeaksMastodonApi(string software) => software != default && MastodonLike.Contains(software); + + public static (string Href, string Version) NodeInfoLink(JsonElement links) + { + if (links.ValueKind != JsonValueKind.Object || !links.TryGetProperty("links", out var list) || list.ValueKind != JsonValueKind.Array) + return default; + foreach (var schema in Schemas) + foreach (var link in list.EnumerateArray()) + if (link.ValueKind == JsonValueKind.Object && Text(link, "rel") == schema && Text(link, "href") is { } href + && href.StartsWith("https://", StringComparison.OrdinalIgnoreCase)) + return (href, schema[(schema.LastIndexOf('/') + 1)..]); + return default; + } + + public static NodeInfoFacts NodeInfo(JsonElement root, string version) + { + var software = Section(root, "software"); + var usage = Section(root, "usage"); + var users = Section(usage, "users"); + var metadata = Section(root, "metadata"); + return new NodeInfoFacts( + version, + Text(software, "name", 200), + Text(software, "version", 200), + Text(metadata, "nodeName", 200), + Plain(Text(metadata, "nodeDescription", 2000), 500), + Strings(root, "protocols", 16, value => value.Length <= 40), + root.ValueKind == JsonValueKind.Object && root.TryGetProperty("openRegistrations", out var open) && open.ValueKind is JsonValueKind.True or JsonValueKind.False + ? open.GetBoolean() + : (bool?)null, + Count(users, "total"), + Count(users, "activeMonth"), + Count(users, "activeHalfyear"), + Count(usage, "localPosts"), + Count(usage, "localComments"), + Languages(metadata, "languages") ?? Languages(metadata, "langs")); + } + + public static InstanceApiFacts InstanceApi(JsonElement root) + { + var registrations = root.ValueKind == JsonValueKind.Object && root.TryGetProperty("registrations", out var r) ? r : default; + string mode; + if (registrations.ValueKind == JsonValueKind.Object) + mode = Flag(registrations, "enabled") switch + { + true => Flag(registrations, "approval_required") == true ? "approval" : "open", + false => "closed", + _ => default + }; + else if (registrations.ValueKind is JsonValueKind.True or JsonValueKind.False) + mode = registrations.GetBoolean() ? Flag(root, "approval_required") == true ? "approval" : "open" : "closed"; + else + mode = default; + var statuses = Section(Section(root, "configuration"), "statuses"); + var maxChars = Count(statuses, "max_characters") ?? Count(root, "max_toot_chars"); + var apiVersion = Count(Section(root, "api_versions"), "mastodon"); + var source = Text(root, "source_url", 300); + return new InstanceApiFacts( + Plain(Text(root, "title", 300), 200), + Languages(root, "languages"), + mode, + maxChars is { } chars ? (int)Math.Min(chars, int.MaxValue) : (int?)null, + apiVersion is { } api ? (int)Math.Min(api, int.MaxValue) : (int?)null, + source != default && source.StartsWith("https://", StringComparison.OrdinalIgnoreCase) ? source : default); + } + + static JsonElement Section(JsonElement parent, string name) => + parent.ValueKind == JsonValueKind.Object && parent.TryGetProperty(name, out var value) && value.ValueKind == JsonValueKind.Object ? value : default; + + static string Text(JsonElement parent, string name, int max = 2048) + { + if (parent.ValueKind != JsonValueKind.Object || !parent.TryGetProperty(name, out var value) || value.ValueKind != JsonValueKind.String) + return default; + var text = value.GetString(); + return text.Length <= max ? text : text[..max]; + } + + static bool? Flag(JsonElement parent, string name) => + parent.ValueKind == JsonValueKind.Object && parent.TryGetProperty(name, out var value) && value.ValueKind is JsonValueKind.True or JsonValueKind.False + ? value.GetBoolean() + : (bool?)null; + + static long? Count(JsonElement parent, string name) + { + if (parent.ValueKind != JsonValueKind.Object || !parent.TryGetProperty(name, out var value)) + return default; + long? number = value.ValueKind switch + { + JsonValueKind.Number when value.TryGetInt64(out var whole) => whole, + JsonValueKind.Number when value.TryGetDouble(out var real) && real is >= 0 and < MaxCount => (long)real, + JsonValueKind.String when long.TryParse(value.GetString(), NumberStyles.Integer, CultureInfo.InvariantCulture, out var parsed) => parsed, + _ => (long?)null + }; + return number is { } n ? Math.Clamp(n, 0, MaxCount) : (long?)null; + } + + static List Strings(JsonElement parent, string name, int max, Func keep) => + parent.ValueKind == JsonValueKind.Object && parent.TryGetProperty(name, out var value) && value.ValueKind == JsonValueKind.Array + ? value.EnumerateArray().Where(v => v.ValueKind == JsonValueKind.String).Select(v => v.GetString()).Where(keep).Distinct().Take(max).ToList() + : new List(); + + static List Languages(JsonElement parent, string name) + { + var languages = Strings(parent, name, 16, language => LanguageCode().IsMatch(language.ToLowerInvariant())) + .Select(language => language.ToLowerInvariant()).Distinct().ToList(); + return languages.Count == 0 ? default : languages; + } + + static string Plain(string text, int max) + { + if (string.IsNullOrWhiteSpace(text)) + return default; + var plain = Tags().Replace(text, " ").Trim(); + plain = Spaces().Replace(plain, " "); + return plain.Length <= max ? plain : plain[..max]; + } + + [GeneratedRegex("^[a-z]{2,3}(-[a-z0-9]{2,8})?$")] + private static partial Regex LanguageCode(); + + [GeneratedRegex("<[^>]*>")] + private static partial Regex Tags(); + + [GeneratedRegex("\\s+")] + private static partial Regex Spaces(); + } +} diff --git a/PrivaPub/Federation/Objects/ObjectRecords.cs b/PrivaPub/Federation/Objects/ObjectRecords.cs index 3ebe25a..c1c8217 100644 --- a/PrivaPub/Federation/Objects/ObjectRecords.cs +++ b/PrivaPub/Federation/Objects/ObjectRecords.cs @@ -99,8 +99,7 @@ namespace PrivaPub.Federation.Objects var known = await DB.Default.Find().Match(i => i.Host == host).ExecuteFirstAsync(token); if (known?.DescribedAt > DateTime.UtcNow - DescriptionAge) return; - var week = ISOWeek.GetYear(DateTime.UtcNow) + "-" + ISOWeek.GetWeekOfYear(DateTime.UtcNow); - await _queue.Enqueue(JobKind.DescribeInstance, host, host, $"describe|{host}|{week}", token); + await _queue.Enqueue(JobKind.DescribeInstance, host, host, InstanceDescriber.DedupeKey(host, DateTime.UtcNow), token); } public static (string Text, string Hash, int Bytes, bool Truncated) Capture(JsonNode raw) diff --git a/PrivaPub/Infrastructure/Data/Indexes.cs b/PrivaPub/Infrastructure/Data/Indexes.cs index a06150d..3af9b9e 100644 --- a/PrivaPub/Infrastructure/Data/Indexes.cs +++ b/PrivaPub/Infrastructure/Data/Indexes.cs @@ -127,6 +127,11 @@ namespace PrivaPub.Infrastructure.Data await DB.Default.Index().Key(d => d.Day, KeyType.Ascending).Key(d => d.Host, KeyType.Ascending).Option(o => o.Unique = true).CreateAsync(token); await Plain(token, d => d.Host, d => d.Day); await DB.Default.Index().Key(d => d.Day, KeyType.Ascending).Option(o => o.Unique = true).CreateAsync(token); + await DB.Default.Index().Key(s => s.Host, KeyType.Ascending).Key(s => s.Week, KeyType.Ascending) + .Option(o => o.Unique = true).CreateAsync(token); + await Plain(token, s => s.Week); + await Plain(token, i => i.Seen); + await Plain(token, i => i.LastCrawledAt); } static async Task Unique(System.Linq.Expressions.Expression> key, FilterDefinition partial, diff --git a/PrivaPub/Infrastructure/Data/Migrations/_010_remote_instances_were_touched.cs b/PrivaPub/Infrastructure/Data/Migrations/_010_remote_instances_were_touched.cs new file mode 100644 index 0000000..4a8e8e8 --- /dev/null +++ b/PrivaPub/Infrastructure/Data/Migrations/_010_remote_instances_were_touched.cs @@ -0,0 +1,24 @@ +using MongoDB.Bson; +using MongoDB.Entities; + +using PrivaPub.Models.Jobs; + +namespace PrivaPub.Infrastructure.Data.Migrations +{ + public class _010_remote_instances_were_touched : IMigration + { + public async Task UpgradeAsync() + { + foreach (var instance in await DB.Default.Find().Match(i => i.Seen == null).ExecuteAsync()) + { + var firstSeen = ObjectId.TryParse(instance.ID, out var id) ? id.CreationTime : DateTime.UtcNow; + var lastSeen = new[] { instance.LastSuccessAt, instance.LastFailureAt, instance.DescribedAt }.Max() ?? firstSeen; + await DB.Default.Update().MatchID(instance.ID) + .Modify(i => i.Seen, "touched") + .Modify(i => i.FirstSeenAt, firstSeen) + .Modify(i => i.LastSeenAt, lastSeen) + .ExecuteAsync(); + } + } + } +} diff --git a/PrivaPub/Infrastructure/Statistics/InteractionLedger.cs b/PrivaPub/Infrastructure/Statistics/InteractionLedger.cs index a0b05b1..1ce8843 100644 --- a/PrivaPub/Infrastructure/Statistics/InteractionLedger.cs +++ b/PrivaPub/Infrastructure/Statistics/InteractionLedger.cs @@ -3,6 +3,8 @@ using Microsoft.Extensions.Options; using MongoDB.Driver; using MongoDB.Entities; +using PrivaPub.Federation.Moderation; +using PrivaPub.Infrastructure.Jobs; using PrivaPub.Models.Jobs; using PrivaPub.Models.Statistics; @@ -41,9 +43,16 @@ namespace PrivaPub.Infrastructure.Statistics SingleWriter = false, SingleReader = false }); + static readonly TimeSpan TouchInterval = TimeSpan.FromHours(1); + static readonly HashSet TouchingPurposes = new(StringComparer.Ordinal) { "actor", "key", "object", "webfinger", "context" }; + readonly InteractionSalts _salts; readonly IOptionsMonitor _options; readonly ILogger _logger; + readonly IJobQueue _queue; + readonly IDomainBlocks _blocks; + readonly ConcurrentDictionary _touches = new(StringComparer.Ordinal); + readonly ConcurrentDictionary _touchedAt = new(StringComparer.Ordinal); readonly SemaphoreSlim _flushing = new(1, 1); readonly ConcurrentDictionary<(DateTime Day, string Host, string Key), long> _reads = new(); readonly ConcurrentDictionary<(DateTime Day, string Field, string Key), long> _server = new(); @@ -54,11 +63,14 @@ namespace PrivaPub.Infrastructure.Statistics long _written; long _failed; - public InteractionLedger(InteractionSalts salts, IOptionsMonitor options, ILogger logger) + public InteractionLedger(InteractionSalts salts, IOptionsMonitor options, ILogger logger, + IJobQueue queue = default, IDomainBlocks blocks = default) { _salts = salts; _options = options; _logger = logger; + _queue = queue; + _blocks = blocks; } public long Dropped => Interlocked.Read(ref _dropped); @@ -85,6 +97,9 @@ namespace PrivaPub.Infrastructure.Statistics var day = DateTime.UtcNow.Date; host = Interactions.Host(host); key = Key(key); + if (key.StartsWith("http:", StringComparison.Ordinal) && key.EndsWith(":ok", StringComparison.Ordinal) + && TouchingPurposes.Contains(key.Split(':')[1])) + _touches[host] = 0; _reads.AddOrUpdate((day, host, key), 1, (_, count) => count + 1); if (bytes > 0) _reads.AddOrUpdate((day, host, key + ":bytes"), bytes, (_, count) => count + bytes); @@ -118,6 +133,7 @@ namespace PrivaPub.Infrastructure.Statistics { } await FlushEvents(stoppingToken); + await FlushTouches(stoppingToken); if (DateTime.UtcNow - countersAt >= CounterInterval) { countersAt = DateTime.UtcNow; @@ -152,9 +168,56 @@ namespace PrivaPub.Infrastructure.Statistics public async Task Flush(CancellationToken token) { await FlushEvents(token); + await FlushTouches(token); await FlushCounters(token); } + static bool Touches(InteractionEvent e, bool verified) => e.Channel switch + { + Interactions.Receive => verified && e.Outcome == Interactions.Queued, + Interactions.In or Interactions.Out => true, + Interactions.Http => e.Outcome == Interactions.Ok && TouchingPurposes.Contains(e.Purpose ?? string.Empty), + _ => false + }; + + //a server we exchanged something with is marked as seen, and described once a week + async Task FlushTouches(CancellationToken token) + { + await _flushing.WaitAsync(token); + try + { + await Touch(token); + } + finally + { + _flushing.Release(); + } + } + + async Task Touch(CancellationToken token) + { + foreach (var host in _touches.Keys.ToList()) + { + _touches.TryRemove(host, out _); + var now = DateTime.UtcNow; + if (host == Interactions.Unknown || _touchedAt.TryGetValue(host, out var at) && now - at < TouchInterval) + continue; + _touchedAt[host] = now; + await DB.Default.Update() + .Match(i => i.Host == host) + .Modify(i => i.Seen, "touched") + .Modify(i => i.LastSeenAt, now) + .Modify(b => b.SetOnInsert(i => i.FirstSeenAt, now)) + .Option(o => o.IsUpsert = true) + .ExecuteAsync(token); + _knownHosts.Add(host); + if (_queue != default && _blocks?.IsSuspended(host) != true) + await _queue.Enqueue(JobKind.DescribeInstance, host, host, Federation.Objects.InstanceDescriber.DedupeKey(host, now), token); + } + if (_touchedAt.Count > 100_000) + _touchedAt.Clear(); + } + async Task FlushEvents(CancellationToken token) { await _flushing.WaitAsync(token); @@ -192,6 +255,8 @@ namespace PrivaPub.Infrastructure.Statistics interaction.Host = Interactions.Unknown; if (pending.ActorUri != default) interaction.ActorHash = InteractionSalts.Hash(await _salts.For(interaction.At, token), pending.ActorUri); + if (interaction.Host != Interactions.Unknown && Touches(interaction, pending.ActorUri != default)) + _touches[interaction.Host] = 0; return interaction; } diff --git a/PrivaPub/Models/Jobs/RemoteInstance.cs b/PrivaPub/Models/Jobs/RemoteInstance.cs index 8a4b5ee..3ee4489 100644 --- a/PrivaPub/Models/Jobs/RemoteInstance.cs +++ b/PrivaPub/Models/Jobs/RemoteInstance.cs @@ -19,5 +19,68 @@ namespace PrivaPub.Models.Jobs public string NodeInfo { get; set; }//the raw document, when small enough public DateTime? DescribedAt { get; set; } public string DescriptionError { get; set; } + + public string Seen { get; set; }//"touched": we exchanged something with it; "crawled": only its peers list named it + public DateTime? FirstSeenAt { get; set; } + public DateTime? LastSeenAt { get; set; } + public DateTime? LastCrawledAt { get; set; } + public string CrawlError { get; set; } + public bool? RobotsDisallowed { get; set; } + public int? PeersCount { get; set; } + + public string NodeInfoVersion { get; set; } + public string NodeDescription { get; set; } + public long? UsersTotal { get; set; }//as the server publishes them in its NodeInfo + public long? UsersActiveMonth { get; set; } + public long? UsersActiveHalfyear { get; set; } + public long? LocalPosts { get; set; } + public long? LocalComments { get; set; } + + public string InstanceApi { get; set; }//"mastodon-v2" or "mastodon-v1" + public string InstanceApiRaw { get; set; }//admin only: it names maintainers and contact accounts + public string Title { get; set; } + public List Languages { get; set; } + public string RegistrationMode { get; set; }//open, approval, closed + public int? MaxPostChars { get; set; } + public int? MastodonApiVersion { get; set; } + public string SourceUrl { get; set; } + + public InstanceGeo Geo { get; set; } + } + + public class InstanceGeo + { + public string Country { get; set; } + public string City { get; set; } + public double? Latitude { get; set; }//rounded to one decimal + public double? Longitude { get; set; } + public int? Asn { get; set; } + public string AsnOrg { get; set; } + public string Cdn { get; set; }//the CDN in front of it, whose edge address says nothing about where it runs + public bool? Ipv6 { get; set; } + public string Source { get; set; } + public DateTime? LocatedAt { get; set; } + } + + public class RemoteInstanceSnapshot : Entity + { + public string Host { get; set; } + public string Week { get; set; }//ISO week, "2026-W40" + public DateTime TakenAt { get; set; } + public string Seen { get; set; } + public bool Reachable { get; set; } + public string Software { get; set; } + public string SoftwareVersion { get; set; } + public long? UsersTotal { get; set; } + public long? UsersActiveMonth { get; set; } + public long? UsersActiveHalfyear { get; set; } + public long? LocalPosts { get; set; } + public bool? OpenRegistrations { get; set; } + public string RegistrationMode { get; set; } + public List Languages { get; set; } + public string Country { get; set; } + public int? Asn { get; set; } + public string Cdn { get; set; } + public bool? Ipv6 { get; set; } } }