M8: every server we touch is described, located and snapshotted weekly

- Touches: the ledger marks a server as touched when it sends us a verified activity, when
  we exchange activities with it, or when we read its actors, keys, objects or WebFinger.
  It upserts RemoteInstance.Seen, FirstSeenAt and LastSeenAt at most hourly per server, and
  queues one DescribeInstance a week with the same dedupe key ObjectRecords uses. Suspended
  servers and pages behind link previews are never described. Migration _010 marks the
  servers already known as touched, with their dates.
- InstanceDescriber.Describe(host, crawled, allowed) reads:
  - NodeInfo 2.2/2.1/2.0, now with its published user counts, posts, comments,
    description, languages and schema version;
  - for software with a Mastodon API, /api/v2/instance falling back to v1: title,
    languages, registration mode, character limit, API version, source URL.

  It never keeps a contact as a field; the raw document is kept for the admin only. It
  locates the server from the address our connection reached (DB-IP Lite city and ASN, the
  CDN named when fronted) and writes a RemoteInstanceSnapshot per ISO week, unreachable
  weeks included. A crawled server is upserted as crawled only on insert, so it never
  downgrades a touched one, and robots.txt can deny any path.
- PublicGeo.Project is the only public form of a location: a CDN-fronted server shows its
  CDN only, a server reporting at least ten users shows its city, coordinates and network,
  any other only its country.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01ELjqpznMFMNrJoJUj6K5p2

PrivaPub.Tests/Http/InboxRouteTests.cs

@@ -188,18 +188,15 @@ namespace PrivaPub.Tests.Http
 				return response.StatusCode;
 			}
 
-			var clock = System.Diagnostics.Stopwatch.StartNew();
-			var first = new List<HttpStatusCode>();
-			for (var i = 0; i < 300; i++)
-				first.Add(await Post());
-			var limited = await Post();
-			// The bucket refills 50 every ten seconds: a slow machine may have earned a few more before the limit bites.
-			for (var refilled = (int)((clock.Elapsed.TotalSeconds + 1) / 10) * 50; limited != HttpStatusCode.TooManyRequests && refilled > 0; refilled--)
-				limited = await Post();
+			// The bucket holds 300 and refills 50 every ten seconds on its own timer, so a refill can land at any moment:
+			// the first refusal comes after at least 300 posts and within a few refills.
+			var answers = new List<HttpStatusCode>();
+			while (answers.Count < 600 && (answers.Count == 0 || answers[^1] != HttpStatusCode.TooManyRequests))
+				answers.Add(await Post());
 
-			Assert.DoesNotContain(HttpStatusCode.TooManyRequests, first);
-			Assert.All(first, status => Assert.Equal(HttpStatusCode.Unauthorized, status));
-			Assert.Equal(HttpStatusCode.TooManyRequests, limited);
+			Assert.Equal(HttpStatusCode.TooManyRequests, answers[^1]);
+			Assert.True(answers.Count > 300, $"limited after only {answers.Count - 1} posts");
+			Assert.All(answers.SkipLast(1), status => Assert.Equal(HttpStatusCode.Unauthorized, status));
 			var polite = new RemoteActor(_peer, "polite");
 			using var signed = await client.SendAsync(polite.SignedPost("/human-centipede", Direct(polite)), token);
 			Assert.Equal(HttpStatusCode.Accepted, signed.StatusCode);

PrivaPub.Tests/Statistics/DescribeTests.cs

@@ -0,0 +1,261 @@
+using Microsoft.Extensions.Logging.Abstractions;
+
+using MongoDB.Entities;
+
+using PrivaPub.Domain.Statistics;
+using PrivaPub.Federation.Moderation;
+using PrivaPub.Federation.Objects;
+using PrivaPub.Infrastructure.Data.Migrations;
+using PrivaPub.Infrastructure.Geo;
+using PrivaPub.Infrastructure.Http;
+using PrivaPub.Infrastructure.Jobs;
+using PrivaPub.Infrastructure.Statistics;
+using PrivaPub.Models.Federation;
+using PrivaPub.Models.Jobs;
+using PrivaPub.Models.Statistics;
+using PrivaPub.Tests.Support;
+
+using System.Net;
+using System.Text.Json;
+
+namespace PrivaPub.Tests.Statistics
+{
+	public class InstanceDocumentsTests
+	{
+		static JsonElement Json(string text) => JsonDocument.Parse(text).RootElement;
+
+		[Fact]
+		public void NodeInfo_gives_software_usage_and_languages_whatever_their_shape()
+		{
+			var facts = InstanceDocuments.NodeInfo(Json("""
+				{"software":{"name":"mastodon","version":"4.7.3"},"protocols":["activitypub"],"openRegistrations":false,
+				 "usage":{"users":{"total":"1234","activeMonth":56.0,"activeHalfyear":-3},"localPosts":99999999999,"localComments":7},
+				 "metadata":{"nodeName":"Example","nodeDescription":"<p>A <b>small</b>\n place</p>","languages":["en","DE","klingon!"]}}
+				"""), "2.1");
+
+			Assert.Equal(("mastodon", "4.7.3", "Example", "2.1"), (facts.Software, facts.SoftwareVersion, facts.NodeName, facts.Version));
+			Assert.Equal("A small place", facts.NodeDescription);
+			Assert.Equal(new[] { "activitypub" }, facts.Protocols);
+			Assert.False(facts.OpenRegistrations);
+			Assert.Equal(1234, facts.UsersTotal);
+			Assert.Equal(56, facts.UsersActiveMonth);
+			Assert.Equal(0, facts.UsersActiveHalfyear);
+			Assert.Equal(1_000_000_000, facts.LocalPosts);
+			Assert.Equal(7, facts.LocalComments);
+			Assert.Equal(new[] { "en", "de" }, facts.Languages);
+			Assert.Null(InstanceDocuments.NodeInfo(Json("{}"), "2.0").UsersTotal);
+		}
+
+		[Fact]
+		public void The_newest_https_nodeinfo_link_is_chosen()
+		{
+			var links = Json("""
+				{"links":[{"rel":"http://nodeinfo.diaspora.software/ns/schema/2.0","href":"https://a.example/2.0"},
+				          {"rel":"http://nodeinfo.diaspora.software/ns/schema/2.1","href":"http://a.example/2.1"},
+				          {"rel":"http://nodeinfo.diaspora.software/ns/schema/2.2","href":"https://a.example/2.2"}]}
+				""");
+
+			Assert.Equal(("https://a.example/2.2", "2.2"), InstanceDocuments.NodeInfoLink(links));
+			Assert.Equal(default, InstanceDocuments.NodeInfoLink(Json("[]")));
+		}
+
+		[Fact]
+		public void The_instance_api_gives_registration_mode_limits_and_never_a_contact()
+		{
+			var v2 = InstanceDocuments.InstanceApi(Json("""
+				{"title":"Example","languages":["en"],"registrations":{"enabled":true,"approval_required":true},
+				 "configuration":{"statuses":{"max_characters":500}},"api_versions":{"mastodon":7},"source_url":"https://github.com/mastodon/mastodon",
+				 "contact":{"email":"admin@a.example","account":{"username":"admin"}}}
+				"""));
+			var v1 = InstanceDocuments.InstanceApi(Json("""{"title":"Old","registrations":false,"max_toot_chars":"5000"}"""));
+
+			Assert.Equal(("Example", "approval", 500, 7, "https://github.com/mastodon/mastodon"), (v2.Title, v2.RegistrationMode, v2.MaxPostChars!.Value, v2.ApiVersion!.Value, v2.SourceUrl));
+			Assert.Equal(new[] { "en" }, v2.Languages);
+			Assert.DoesNotContain("admin", System.Text.Json.JsonSerializer.Serialize(v2));
+			Assert.Equal(("closed", 5000), (v1.RegistrationMode, v1.MaxPostChars!.Value));
+			Assert.True(InstanceDocuments.SpeaksMastodonApi("GoToSocial"));
+			Assert.False(InstanceDocuments.SpeaksMastodonApi("lemmy"));
+		}
+
+		[Fact]
+		public void The_public_location_depends_on_size_and_on_a_cdn()
+		{
+			var geo = new InstanceGeo { Country = "DE", City = "Berlin", Latitude = 52.5, Longitude = 13.4, AsnOrg = "Hetzner", Asn = 24940 };
+
+			Assert.Equal(new PublicLocation("DE", "Berlin", 52.5, 13.4, "Hetzner", null), PublicGeo.Project(new RemoteInstance { Geo = geo, UsersTotal = 10 }, 10));
+			Assert.Equal(new PublicLocation("DE", null, null, null, null, null), PublicGeo.Project(new RemoteInstance { Geo = geo, UsersTotal = 1 }, 10));
+			Assert.Equal(new PublicLocation("DE", null, null, null, null, null), PublicGeo.Project(new RemoteInstance { Geo = geo }, 10));
+			Assert.Equal(new PublicLocation(null, null, null, null, null, "Cloudflare"),
+				PublicGeo.Project(new RemoteInstance { Geo = new InstanceGeo { Country = "US", City = "SF", Cdn = "Cloudflare" }, UsersTotal = 5000 }, 10));
+			Assert.Null(PublicGeo.Project(new RemoteInstance(), 10));
+		}
+	}
+
+	[Trait("Category", "Integration")]
+	public sealed class DescribeTests : IAsyncLifetime
+	{
+		Peer _peer;
+
+		public async ValueTask InitializeAsync()
+		{
+			Assert.SkipUnless(MongoFixture.Enabled, MongoFixture.Skip);
+			_peer = await Peer.Start();
+		}
+
+		public async ValueTask DisposeAsync()
+		{
+			if (_peer != default)
+				await _peer.DisposeAsync();
+		}
+
+		sealed class PeerDescriber : InstanceDescriber
+		{
+			readonly string _peer;
+
+			public PeerDescriber(IFederationHttp http, IGeoLocator geo, IConnectedAddresses addresses, string peer) : base(http, geo, addresses) => _peer = peer;
+
+			protected override string Address(string url) => _peer + new Uri(url).PathAndQuery;
+		}
+
+		sealed class FixedGeo : IGeoLocator
+		{
+			public GeoFix Locate(IPAddress address) => new("NL", "Amsterdam", 52.4, 4.9, 60781, "LeaseWeb");
+			public string Source => "test";
+		}
+
+		void ServeServer(string prefix, string software, bool v2 = true)
+		{
+			_peer.Serve($"{prefix}/.well-known/nodeinfo", $$"""{"links":[{"rel":"http://nodeinfo.diaspora.software/ns/schema/2.1","href":"https://ignored.example/nodeinfo/2.1"}]}""");
+			_peer.Serve($"{prefix}/nodeinfo/2.1", $$$"""{"software":{"name":"{{{software}}}","version":"1.0"},"usage":{"users":{"total":42,"activeMonth":7}},"openRegistrations":true}""");
+			if (v2)
+				_peer.Serve($"{prefix}/api/v2/instance", """{"title":"A test server","languages":["it"],"registrations":{"enabled":true,"approval_required":false}}""");
+			_peer.Serve($"{prefix}/api/v1/instance", """{"title":"An old test server","registrations":true}""");
+		}
+
+		[Fact]
+		public async Task A_server_is_described_located_and_snapshotted_weekly()
+		{
+			var token = TestContext.Current.CancellationToken;
+			var host = $"described{Guid.NewGuid():N}.example";
+			ServeServer("/" + host, "mastodon");
+			var addresses = new ConnectedAddresses();
+			var describer = new PeerDescriber(Peer.Http(connected: addresses), new FixedGeo(), addresses, _peer.A + "/" + host);
+
+			Assert.True(await describer.Describe(host, crawled: false, default, token));
+
+			var instance = await DB.Default.Find<RemoteInstance>().Match(i => i.Host == host).ExecuteSingleAsync(token);
+			Assert.Equal(("mastodon", 42L, 7L, "2.1"), (instance.Software, instance.UsersTotal!.Value, instance.UsersActiveMonth!.Value, instance.NodeInfoVersion));
+			Assert.Equal(("A test server", "open", "mastodon-v2"), (instance.Title, instance.RegistrationMode, instance.InstanceApi));
+			Assert.Equal(new[] { "it" }, instance.Languages);
+			Assert.Equal(("NL", "Amsterdam", "LeaseWeb", "test"), (instance.Geo.Country, instance.Geo.City, instance.Geo.AsnOrg, instance.Geo.Source));
+			Assert.False(instance.Geo.Ipv6);
+			var snapshot = await DB.Default.Find<RemoteInstanceSnapshot>().Match(s => s.Host == host).ExecuteSingleAsync(token);
+			Assert.True(snapshot.Reachable);
+			Assert.Equal((42L, "NL"), (snapshot.UsersTotal!.Value, snapshot.Country));
+			Assert.Equal(InstanceDescriber.Week(DateTime.UtcNow), snapshot.Week);
+		}
+
+		[Fact]
+		public async Task Without_api_v2_the_v1_api_is_read_and_an_unreachable_week_is_recorded_too()
+		{
+			var token = TestContext.Current.CancellationToken;
+			var old = $"old{Guid.NewGuid():N}.example";
+			var gone = $"gone{Guid.NewGuid():N}.example";
+			ServeServer("/" + old, "pleroma", v2: false);
+
+			await new PeerDescriber(Peer.Http(), default, default, _peer.A + "/" + old).Describe(old, crawled: false, default, token);
+			Assert.False(await new PeerDescriber(Peer.Http(), default, default, _peer.A + "/" + gone).Describe(gone, crawled: false, default, token));
+
+			var oldOne = await DB.Default.Find<RemoteInstance>().Match(i => i.Host == old).ExecuteSingleAsync(token);
+			Assert.Equal(("mastodon-v1", "An old test server", "open"), (oldOne.InstanceApi, oldOne.Title, oldOne.RegistrationMode));
+			Assert.Null(oldOne.Geo);
+			var goneOne = await DB.Default.Find<RemoteInstance>().Match(i => i.Host == gone).ExecuteSingleAsync(token);
+			Assert.Equal("no NodeInfo", goneOne.DescriptionError);
+			Assert.False((await DB.Default.Find<RemoteInstanceSnapshot>().Match(s => s.Host == gone).ExecuteSingleAsync(token)).Reachable);
+		}
+
+		[Fact]
+		public async Task A_crawled_server_never_downgrades_a_touched_one_and_robots_are_obeyed()
+		{
+			var token = TestContext.Current.CancellationToken;
+			var touched = $"touched{Guid.NewGuid():N}.example";
+			var shy = $"shy{Guid.NewGuid():N}.example";
+			ServeServer("/" + touched, "gotosocial");
+			ServeServer("/" + shy, "mastodon");
+			await DB.Default.SaveAsync(new RemoteInstance { Host = touched, Seen = "touched" }, token);
+
+			await new PeerDescriber(Peer.Http(), default, default, _peer.A + "/" + touched).Describe(touched, crawled: true, default, token);
+			await new PeerDescriber(Peer.Http(), default, default, _peer.A + "/" + shy).Describe(shy, crawled: true, path => !path.StartsWith("/api/"), token);
+
+			Assert.Equal("touched", (await DB.Default.Find<RemoteInstance>().Match(i => i.Host == touched).ExecuteSingleAsync(token)).Seen);
+			var shyOne = await DB.Default.Find<RemoteInstance>().Match(i => i.Host == shy).ExecuteSingleAsync(token);
+			Assert.Equal(("crawled", "mastodon"), (shyOne.Seen, shyOne.Software));
+			Assert.Null(shyOne.InstanceApi);
+			Assert.DoesNotContain(_peer.Requests, r => r.Path.StartsWith("/" + shy + "/api/", StringComparison.Ordinal));
+			Assert.Contains(_peer.Requests, r => r.Path == "/" + shy + "/nodeinfo/2.1");
+		}
+	}
+
+	[Trait("Category", "Integration")]
+	public sealed class TouchTests : IAsyncLifetime
+	{
+		public ValueTask InitializeAsync()
+		{
+			Assert.SkipUnless(MongoFixture.Enabled, MongoFixture.Skip);
+			return ValueTask.CompletedTask;
+		}
+
+		public ValueTask DisposeAsync() => ValueTask.CompletedTask;
+
+		[Fact]
+		public async Task A_server_we_exchange_with_is_seen_and_described_once_a_week_unless_suspended()
+		{
+			var token = TestContext.Current.CancellationToken;
+			var friend = $"friend{Guid.NewGuid():N}.example";
+			var spammer = $"spam{Guid.NewGuid():N}.example";
+			var linked = $"linked{Guid.NewGuid():N}.example";
+			var looked = $"looked{Guid.NewGuid():N}.example";
+			var blocks = new DomainBlocks(NullLogger<DomainBlocks>.Instance);
+			blocks.Load(new[] { new DomainBlock { Domain = spammer, Severity = DomainBlockSeverity.Suspend } });
+			var ledger = new InteractionLedger(new InteractionSalts(), new StaticOptions<StatisticsOptions>(new StatisticsOptions()), NullLogger<InteractionLedger>.Instance,
+				new JobQueue(j => j.DedupeKey == "never"), blocks);
+
+			ledger.Record(new InteractionEvent { Host = friend, Channel = "in", Activity = "Create", Outcome = "accepted" }, "https://" + friend + "/users/a");
+			ledger.Record(new InteractionEvent { Host = friend, Channel = "out", Activity = "Like", Outcome = "ok" });
+			ledger.Record(new InteractionEvent { Host = spammer, Channel = "out", Activity = "Reject", Outcome = "ok" });
+			ledger.Record(new InteractionEvent { Host = linked, Channel = "preview", Outcome = "ok" });
+			ledger.Count(looked, "http:actor:ok");
+			await ledger.Flush(token);
+
+			var seen = await DB.Default.Find<RemoteInstance>().Match(i => i.Host == friend).ExecuteSingleAsync(token);
+			Assert.Equal("touched", seen.Seen);
+			Assert.NotNull(seen.FirstSeenAt);
+			Assert.Single(await DB.Default.Find<Job>().Match(j => j.Kind == JobKind.DescribeInstance && j.Payload == friend).ExecuteAsync(token));
+			Assert.True(await DB.Default.Find<Job>().Match(j => j.Kind == JobKind.DescribeInstance && j.Payload == looked).ExecuteAnyAsync(token));
+			Assert.False(await DB.Default.Find<Job>().Match(j => j.Kind == JobKind.DescribeInstance && j.Payload == spammer).ExecuteAnyAsync(token));
+			Assert.False(await DB.Default.Find<RemoteInstance>().Match(i => i.Host == linked).ExecuteAnyAsync(token));
+		}
+	}
+
+	[Xunit.Collection(nameof(Exclusive))]
+	[Trait("Category", "Integration")]
+	public sealed class TouchedMigrationTests
+	{
+		[Fact]
+		public async Task Known_servers_become_touched_with_their_dates()
+		{
+			Assert.SkipUnless(MongoFixture.Enabled, MongoFixture.Skip);
+			var token = TestContext.Current.CancellationToken;
+			var host = $"legacy{Guid.NewGuid():N}.example";
+			var success = DateTime.UtcNow.AddDays(-2);
+			await DB.Default.SaveAsync(new RemoteInstance { Host = host, LastSuccessAt = success, DescribedAt = success.AddDays(-1) }, token);
+
+			await new _010_remote_instances_were_touched().UpgradeAsync();
+
+			var instance = await DB.Default.Find<RemoteInstance>().Match(i => i.Host == host).ExecuteSingleAsync(token);
+			Assert.Equal("touched", instance.Seen);
+			Assert.NotNull(instance.FirstSeenAt);
+			Assert.InRange(instance.LastSeenAt!.Value, success.AddSeconds(-1), success.AddSeconds(1));
+		}
+	}
+}

PrivaPub/Domain/Statistics/PublicGeo.cs

@@ -0,0 +1,23 @@
+using PrivaPub.Models.Jobs;
+
+namespace PrivaPub.Domain.Statistics
+{
+	public sealed record PublicLocation(string Country, string City, double? Latitude, double? Longitude, string Network, string Cdn);
+
+	public static class PublicGeo
+	{
+		//owner decision: a CDN-fronted server shows only its CDN (the address is an edge), a server reporting at least
+		//minUsers users its city, coordinates and network, any other only its country
+		public static PublicLocation Project(RemoteInstance instance, int minUsers)
+		{
+			var geo = instance?.Geo;
+			if (geo == default)
+				return default;
+			if (geo.Cdn != default)
+				return new PublicLocation(default, default, default, default, default, geo.Cdn);
+			if (instance.UsersTotal >= minUsers)
+				return new PublicLocation(geo.Country, geo.City, geo.Latitude, geo.Longitude, geo.AsnOrg, default);
+			return geo.Country == default ? default : new PublicLocation(geo.Country, default, default, default, default, default);
+		}
+	}
+}

PrivaPub/Federation/Objects/InstanceDescriber.cs

@@ -1,88 +1,180 @@
 using MongoDB.Entities;
 
+using PrivaPub.Infrastructure.Geo;
 using PrivaPub.Infrastructure.Http;
 using PrivaPub.Infrastructure.Jobs;
 using PrivaPub.Models.Jobs;
 
+using System.Globalization;
+using System.Net.Sockets;
 using System.Text.Json;
 
 namespace PrivaPub.Federation.Objects
 {
 	public class InstanceDescriber : IJobHandler
 	{
-		const int MaxNodeInfoChars = 64 * 1024;
-		static readonly string[] Schemas =
-		{
-			"http://nodeinfo.diaspora.software/ns/schema/2.1",
-			"http://nodeinfo.diaspora.software/ns/schema/2.0"
-		};
+		const int MaxRawChars = 64 * 1024;
 
 		readonly IFederationHttp _http;
+		readonly IGeoLocator _geo;
+		readonly IConnectedAddresses _addresses;
 
-		public InstanceDescriber(IFederationHttp http) => _http = http;
+		public InstanceDescriber(IFederationHttp http, IGeoLocator geo = default, IConnectedAddresses addresses = default)
+		{
+			_http = http;
+			_geo = geo;
+			_addresses = addresses;
+		}
 
 		public JobKind Kind => JobKind.DescribeInstance;
 		public int Concurrency => 1;
 		public int MaxAttempts => 2;
 		public int PerHostLimit => 1;
 
+		public static string Week(DateTime at) => $"{ISOWeek.GetYear(at)}-W{ISOWeek.GetWeekOfYear(at):00}";
+
+		public static string DedupeKey(string host, DateTime at) => $"describe|{host}|{ISOWeek.GetYear(at)}-{ISOWeek.GetWeekOfYear(at)}";
+
 		public async Task<JobOutcome> Handle(Job job, CancellationToken token)
 		{
-			var host = job.Payload;
-			using var scope = HttpScope.For("nodeinfo");
-			using var links = await _http.GetJson(Address($"https://{host}/.well-known/nodeinfo"), "application/json", sign: default, token);
-			var href = links == default ? default : Schemas.Select(schema => LinkFor(links.Root, schema)).FirstOrDefault(link => link != default);
-			if (href == default)
-				return await Described(host, default, "no NodeInfo", token);
+			await Describe(job.Payload, crawled: false, default, token);
+			return JobOutcome.Done;
+		}
 
-			using var nodeInfo = await _http.GetJson(Address(href), "application/json", sign: default, token);
-			return nodeInfo == default ? await Described(host, default, "NodeInfo unreadable", token) : await Described(host, nodeInfo.Root, default, token);
+		//NodeInfo, the Mastodon instance API where the software has one, and where the address we reached sits; unsigned,
+		//because none of these are ActivityPub documents. `allowed` is the crawler's robots.txt.
+		public async Task<bool> Describe(string host, bool crawled, Func<string, bool> allowed, CancellationToken token)
+		{
+			allowed ??= _ => true;
+			NodeInfoFacts facts = default;
+			string rawNodeInfo = default, error;
+			using (HttpScope.For("nodeinfo"))
+			{
+				error = allowed("/.well-known/nodeinfo") ? "no NodeInfo" : "robots.txt disallows NodeInfo";
+				using var links = allowed("/.well-known/nodeinfo")
+					? await _http.GetJson(Address($"https://{host}/.well-known/nodeinfo"), "application/json", sign: default, token)
+					: default;
+				var (href, version) = links == default ? default : InstanceDocuments.NodeInfoLink(links.Root);
+				if (href != default && allowed(new Uri(href).AbsolutePath))
+				{
+					using var nodeInfo = await _http.GetJson(Address(href), "application/json", sign: default, token);
+					if (nodeInfo != default)
+					{
+						facts = InstanceDocuments.NodeInfo(nodeInfo.Root, version);
+						rawNodeInfo = nodeInfo.Root.GetRawText();
+						error = default;
+					}
+					else
+						error = "NodeInfo unreadable";
+				}
+			}
+
+			InstanceApiFacts api = default;
+			string apiKind = default, rawApi = default;
+			if (InstanceDocuments.SpeaksMastodonApi(facts?.Software))
+				using (HttpScope.For("instance-api"))
+					foreach (var (path, kind) in new[] { ("/api/v2/instance", "mastodon-v2"), ("/api/v1/instance", "mastodon-v1") })
+					{
+						if (!allowed(path))
+							continue;
+						using var document = await _http.GetJson(Address($"https://{host}{path}"), "application/json", sign: default, token);
+						if (document == default || document.Root.ValueKind != JsonValueKind.Object)
+							continue;
+						api = InstanceDocuments.InstanceApi(document.Root);
+						apiKind = kind;
+						rawApi = document.Root.GetRawText();
+						break;
+					}
+
+			var geo = Locate(host);
+			var now = DateTime.UtcNow;
+			var update = DB.Default.Update<RemoteInstance>()
+				.Match(i => i.Host == host)
+				.Modify(i => i.DescribedAt, now)
+				.Modify(i => i.DescriptionError, error)
+				.Option(o => o.IsUpsert = true);
+			if (crawled)
+				update = update
+					.Modify(b => b.SetOnInsert(i => i.Seen, "crawled"))
+					.Modify(b => b.SetOnInsert(i => i.FirstSeenAt, now));
+			if (facts != default)
+				update = update
+					.Modify(i => i.Software, facts.Software)
+					.Modify(i => i.SoftwareVersion, facts.SoftwareVersion)
+					.Modify(i => i.NodeName, facts.NodeName)
+					.Modify(i => i.NodeDescription, facts.NodeDescription)
+					.Modify(i => i.NodeInfoVersion, facts.Version)
+					.Modify(i => i.Protocols, facts.Protocols)
+					.Modify(i => i.OpenRegistrations, facts.OpenRegistrations)
+					.Modify(i => i.UsersTotal, facts.UsersTotal)
+					.Modify(i => i.UsersActiveMonth, facts.UsersActiveMonth)
+					.Modify(i => i.UsersActiveHalfyear, facts.UsersActiveHalfyear)
+					.Modify(i => i.LocalPosts, facts.LocalPosts)
+					.Modify(i => i.LocalComments, facts.LocalComments)
+					.Modify(i => i.NodeInfo, rawNodeInfo.Length <= MaxRawChars ? rawNodeInfo : default);
+			if (api != default)
+				update = update
+					.Modify(i => i.InstanceApi, apiKind)
+					.Modify(i => i.InstanceApiRaw, rawApi.Length <= MaxRawChars ? rawApi : default)
+					.Modify(i => i.Title, api.Title)
+					.Modify(i => i.Languages, api.Languages ?? facts?.Languages)
+					.Modify(i => i.RegistrationMode, api.RegistrationMode)
+					.Modify(i => i.MaxPostChars, api.MaxPostChars)
+					.Modify(i => i.MastodonApiVersion, api.ApiVersion)
+					.Modify(i => i.SourceUrl, api.SourceUrl);
+			else if (facts?.Languages != default)
+				update = update.Modify(i => i.Languages, facts.Languages);
+			if (geo != default)
+				update = update.Modify(i => i.Geo, geo);
+			await update.ExecuteAsync(token);
+			var instance = await DB.Default.Find<RemoteInstance>().Match(i => i.Host == host).ExecuteFirstAsync(token);
+
+			var week = Week(now);
+			await DB.Default.Update<RemoteInstanceSnapshot>()
+				.Match(s => s.Host == host && s.Week == week)
+				.Modify(s => s.TakenAt, now)
+				.Modify(s => s.Seen, instance?.Seen)
+				.Modify(s => s.Reachable, facts != default || api != default)
+				.Modify(s => s.Software, instance?.Software)
+				.Modify(s => s.SoftwareVersion, instance?.SoftwareVersion)
+				.Modify(s => s.UsersTotal, instance?.UsersTotal)
+				.Modify(s => s.UsersActiveMonth, instance?.UsersActiveMonth)
+				.Modify(s => s.UsersActiveHalfyear, instance?.UsersActiveHalfyear)
+				.Modify(s => s.LocalPosts, instance?.LocalPosts)
+				.Modify(s => s.OpenRegistrations, instance?.OpenRegistrations)
+				.Modify(s => s.RegistrationMode, instance?.RegistrationMode)
+				.Modify(s => s.Languages, instance?.Languages)
+				.Modify(s => s.Country, instance?.Geo?.Country)
+				.Modify(s => s.Asn, instance?.Geo?.Asn)
+				.Modify(s => s.Cdn, instance?.Geo?.Cdn)
+				.Modify(s => s.Ipv6, instance?.Geo?.Ipv6)
+				.Option(o => o.IsUpsert = true)
+				.ExecuteAsync(token);
+			return facts != default;
+		}
+
+		InstanceGeo Locate(string host)
+		{
+			var reached = _addresses?.Of(new Uri(Address($"https://{host}/")).Host);
+			var fix = reached == default ? default : _geo?.Locate(reached);
+			if (fix == default)
+				return default;
+			return new InstanceGeo
+			{
+				Country = fix.Country,
+				City = fix.City,
+				Latitude = fix.Latitude,
+				Longitude = fix.Longitude,
+				Asn = fix.Asn,
+				AsnOrg = fix.AsnOrg,
+				Cdn = CdnNetworks.Of(fix.Asn),
+				Ipv6 = reached.AddressFamily == AddressFamily.InterNetworkV6,
+				Source = _geo.Source,
+				LocatedAt = DateTime.UtcNow
+			};
 		}
 
 		//where an https address is fetched from: itself, except in tests, which point it at a plain-http peer
 		protected virtual string Address(string url) => url;
-
-		static async Task<JobOutcome> Described(string host, JsonElement? nodeInfo, string error, CancellationToken token)
-		{
-			var update = DB.Default.Update<RemoteInstance>()
-				.Match(i => i.Host == host)
-				.Modify(i => i.DescribedAt, DateTime.UtcNow)
-				.Modify(i => i.DescriptionError, error)
-				.Option(o => o.IsUpsert = true);
-			if (nodeInfo is { } root)
-			{
-				var raw = root.GetRawText();
-				update = update
-					.Modify(i => i.Software, Text(root, "software", "name"))
-					.Modify(i => i.SoftwareVersion, Text(root, "software", "version"))
-					.Modify(i => i.NodeName, Text(root, "metadata", "nodeName"))
-					.Modify(i => i.Protocols, root.TryGetProperty("protocols", out var protocols) && protocols.ValueKind == JsonValueKind.Array
-						? protocols.EnumerateArray().Where(p => p.ValueKind == JsonValueKind.String).Select(p => p.GetString()).Take(16).ToList()
-						: new List<string>())
-					.Modify(i => i.OpenRegistrations, root.TryGetProperty("openRegistrations", out var open) && open.ValueKind is JsonValueKind.True or JsonValueKind.False
-						? open.GetBoolean()
-						: default(bool?))
-					.Modify(i => i.NodeInfo, raw.Length <= MaxNodeInfoChars ? raw : default);
-			}
-			await update.ExecuteAsync(token);
-			return JobOutcome.Done;
-		}
-
-		static string LinkFor(JsonElement root, string schema) =>
-			root.TryGetProperty("links", out var links) && links.ValueKind == JsonValueKind.Array
-				? links.EnumerateArray()
-					.Where(l => l.ValueKind == JsonValueKind.Object && l.TryGetProperty("rel", out var rel) && rel.ValueKind == JsonValueKind.String && rel.GetString() == schema)
-					.Select(l => l.TryGetProperty("href", out var href) && href.ValueKind == JsonValueKind.String ? href.GetString() : default)
-					.FirstOrDefault(h => h != default && h.StartsWith("https://", StringComparison.OrdinalIgnoreCase))
-				: default;
-
-		static string Text(JsonElement root, string parent, string name)
-		{
-			if (!root.TryGetProperty(parent, out var section) || section.ValueKind != JsonValueKind.Object
-				|| !section.TryGetProperty(name, out var value) || value.ValueKind != JsonValueKind.String)
-				return default;
-			var text = value.GetString();
-			return text.Length <= 200 ? text : text[..200];
-		}
 	}
 }

PrivaPub/Federation/Objects/InstanceDocuments.cs

@@ -0,0 +1,156 @@
+using System.Globalization;
+using System.Text.Json;
+using System.Text.RegularExpressions;
+
+namespace PrivaPub.Federation.Objects
+{
+	public sealed record NodeInfoFacts(string Version, string Software, string SoftwareVersion, string NodeName, string NodeDescription,
+		List<string> Protocols, bool? OpenRegistrations, long? UsersTotal, long? UsersActiveMonth, long? UsersActiveHalfyear, long? LocalPosts,
+		long? LocalComments, List<string> Languages);
+
+	public sealed record InstanceApiFacts(string Title, List<string> Languages, string RegistrationMode, int? MaxPostChars, int? ApiVersion, string SourceUrl);
+
+	public static partial class InstanceDocuments
+	{
+		public static readonly string[] Schemas =
+		{
+			"http://nodeinfo.diaspora.software/ns/schema/2.2",
+			"http://nodeinfo.diaspora.software/ns/schema/2.1",
+			"http://nodeinfo.diaspora.software/ns/schema/2.0"
+		};
+
+		static readonly HashSet<string> MastodonLike = new(StringComparer.OrdinalIgnoreCase)
+		{
+			"mastodon", "hometown", "glitch-soc", "gotosocial", "pleroma", "akkoma", "sharkey", "iceshrimp", "iceshrimp.net", "firefish", "catodon",
+			"friendica", "pixelfed", "takahe", "snac", "honk", "mitra", "pachli"
+		};
+
+		const long MaxCount = 1_000_000_000;
+
+		public static bool SpeaksMastodonApi(string software) => software != default && MastodonLike.Contains(software);
+
+		public static (string Href, string Version) NodeInfoLink(JsonElement links)
+		{
+			if (links.ValueKind != JsonValueKind.Object || !links.TryGetProperty("links", out var list) || list.ValueKind != JsonValueKind.Array)
+				return default;
+			foreach (var schema in Schemas)
+				foreach (var link in list.EnumerateArray())
+					if (link.ValueKind == JsonValueKind.Object && Text(link, "rel") == schema && Text(link, "href") is { } href
+						&& href.StartsWith("https://", StringComparison.OrdinalIgnoreCase))
+						return (href, schema[(schema.LastIndexOf('/') + 1)..]);
+			return default;
+		}
+
+		public static NodeInfoFacts NodeInfo(JsonElement root, string version)
+		{
+			var software = Section(root, "software");
+			var usage = Section(root, "usage");
+			var users = Section(usage, "users");
+			var metadata = Section(root, "metadata");
+			return new NodeInfoFacts(
+				version,
+				Text(software, "name", 200),
+				Text(software, "version", 200),
+				Text(metadata, "nodeName", 200),
+				Plain(Text(metadata, "nodeDescription", 2000), 500),
+				Strings(root, "protocols", 16, value => value.Length <= 40),
+				root.ValueKind == JsonValueKind.Object && root.TryGetProperty("openRegistrations", out var open) && open.ValueKind is JsonValueKind.True or JsonValueKind.False
+					? open.GetBoolean()
+					: (bool?)null,
+				Count(users, "total"),
+				Count(users, "activeMonth"),
+				Count(users, "activeHalfyear"),
+				Count(usage, "localPosts"),
+				Count(usage, "localComments"),
+				Languages(metadata, "languages") ?? Languages(metadata, "langs"));
+		}
+
+		public static InstanceApiFacts InstanceApi(JsonElement root)
+		{
+			var registrations = root.ValueKind == JsonValueKind.Object && root.TryGetProperty("registrations", out var r) ? r : default;
+			string mode;
+			if (registrations.ValueKind == JsonValueKind.Object)
+				mode = Flag(registrations, "enabled") switch
+				{
+					true => Flag(registrations, "approval_required") == true ? "approval" : "open",
+					false => "closed",
+					_ => default
+				};
+			else if (registrations.ValueKind is JsonValueKind.True or JsonValueKind.False)
+				mode = registrations.GetBoolean() ? Flag(root, "approval_required") == true ? "approval" : "open" : "closed";
+			else
+				mode = default;
+			var statuses = Section(Section(root, "configuration"), "statuses");
+			var maxChars = Count(statuses, "max_characters") ?? Count(root, "max_toot_chars");
+			var apiVersion = Count(Section(root, "api_versions"), "mastodon");
+			var source = Text(root, "source_url", 300);
+			return new InstanceApiFacts(
+				Plain(Text(root, "title", 300), 200),
+				Languages(root, "languages"),
+				mode,
+				maxChars is { } chars ? (int)Math.Min(chars, int.MaxValue) : (int?)null,
+				apiVersion is { } api ? (int)Math.Min(api, int.MaxValue) : (int?)null,
+				source != default && source.StartsWith("https://", StringComparison.OrdinalIgnoreCase) ? source : default);
+		}
+
+		static JsonElement Section(JsonElement parent, string name) =>
+			parent.ValueKind == JsonValueKind.Object && parent.TryGetProperty(name, out var value) && value.ValueKind == JsonValueKind.Object ? value : default;
+
+		static string Text(JsonElement parent, string name, int max = 2048)
+		{
+			if (parent.ValueKind != JsonValueKind.Object || !parent.TryGetProperty(name, out var value) || value.ValueKind != JsonValueKind.String)
+				return default;
+			var text = value.GetString();
+			return text.Length <= max ? text : text[..max];
+		}
+
+		static bool? Flag(JsonElement parent, string name) =>
+			parent.ValueKind == JsonValueKind.Object && parent.TryGetProperty(name, out var value) && value.ValueKind is JsonValueKind.True or JsonValueKind.False
+				? value.GetBoolean()
+				: (bool?)null;
+
+		static long? Count(JsonElement parent, string name)
+		{
+			if (parent.ValueKind != JsonValueKind.Object || !parent.TryGetProperty(name, out var value))
+				return default;
+			long? number = value.ValueKind switch
+			{
+				JsonValueKind.Number when value.TryGetInt64(out var whole) => whole,
+				JsonValueKind.Number when value.TryGetDouble(out var real) && real is >= 0 and < MaxCount => (long)real,
+				JsonValueKind.String when long.TryParse(value.GetString(), NumberStyles.Integer, CultureInfo.InvariantCulture, out var parsed) => parsed,
+				_ => (long?)null
+			};
+			return number is { } n ? Math.Clamp(n, 0, MaxCount) : (long?)null;
+		}
+
+		static List<string> Strings(JsonElement parent, string name, int max, Func<string, bool> keep) =>
+			parent.ValueKind == JsonValueKind.Object && parent.TryGetProperty(name, out var value) && value.ValueKind == JsonValueKind.Array
+				? value.EnumerateArray().Where(v => v.ValueKind == JsonValueKind.String).Select(v => v.GetString()).Where(keep).Distinct().Take(max).ToList()
+				: new List<string>();
+
+		static List<string> Languages(JsonElement parent, string name)
+		{
+			var languages = Strings(parent, name, 16, language => LanguageCode().IsMatch(language.ToLowerInvariant()))
+				.Select(language => language.ToLowerInvariant()).Distinct().ToList();
+			return languages.Count == 0 ? default : languages;
+		}
+
+		static string Plain(string text, int max)
+		{
+			if (string.IsNullOrWhiteSpace(text))
+				return default;
+			var plain = Tags().Replace(text, " ").Trim();
+			plain = Spaces().Replace(plain, " ");
+			return plain.Length <= max ? plain : plain[..max];
+		}
+
+		[GeneratedRegex("^[a-z]{2,3}(-[a-z0-9]{2,8})?$")]
+		private static partial Regex LanguageCode();
+
+		[GeneratedRegex("<[^>]*>")]
+		private static partial Regex Tags();
+
+		[GeneratedRegex("\\s+")]
+		private static partial Regex Spaces();
+	}
+}

PrivaPub/Federation/Objects/ObjectRecords.cs

@@ -99,8 +99,7 @@ namespace PrivaPub.Federation.Objects
 			var known = await DB.Default.Find<RemoteInstance>().Match(i => i.Host == host).ExecuteFirstAsync(token);
 			if (known?.DescribedAt > DateTime.UtcNow - DescriptionAge)
 				return;
-			var week = ISOWeek.GetYear(DateTime.UtcNow) + "-" + ISOWeek.GetWeekOfYear(DateTime.UtcNow);
-			await _queue.Enqueue(JobKind.DescribeInstance, host, host, $"describe|{host}|{week}", token);
+			await _queue.Enqueue(JobKind.DescribeInstance, host, host, InstanceDescriber.DedupeKey(host, DateTime.UtcNow), token);
 		}
 
 		public static (string Text, string Hash, int Bytes, bool Truncated) Capture(JsonNode raw)

PrivaPub/Infrastructure/Data/Indexes.cs

@@ -127,6 +127,11 @@ namespace PrivaPub.Infrastructure.Data
 			await DB.Default.Index<InstanceDay>().Key(d => d.Day, KeyType.Ascending).Key(d => d.Host, KeyType.Ascending).Option(o => o.Unique = true).CreateAsync(token);
 			await Plain<InstanceDay>(token, d => d.Host, d => d.Day);
 			await DB.Default.Index<ServerDay>().Key(d => d.Day, KeyType.Ascending).Option(o => o.Unique = true).CreateAsync(token);
+			await DB.Default.Index<RemoteInstanceSnapshot>().Key(s => s.Host, KeyType.Ascending).Key(s => s.Week, KeyType.Ascending)
+				.Option(o => o.Unique = true).CreateAsync(token);
+			await Plain<RemoteInstanceSnapshot>(token, s => s.Week);
+			await Plain<RemoteInstance>(token, i => i.Seen);
+			await Plain<RemoteInstance>(token, i => i.LastCrawledAt);
 		}
 
 		static async Task Unique<T>(System.Linq.Expressions.Expression<Func<T, object>> key, FilterDefinition<T> partial,

PrivaPub/Infrastructure/Data/Migrations/_010_remote_instances_were_touched.cs

@@ -0,0 +1,24 @@
+using MongoDB.Bson;
+using MongoDB.Entities;
+
+using PrivaPub.Models.Jobs;
+
+namespace PrivaPub.Infrastructure.Data.Migrations
+{
+	public class _010_remote_instances_were_touched : IMigration
+	{
+		public async Task UpgradeAsync()
+		{
+			foreach (var instance in await DB.Default.Find<RemoteInstance>().Match(i => i.Seen == null).ExecuteAsync())
+			{
+				var firstSeen = ObjectId.TryParse(instance.ID, out var id) ? id.CreationTime : DateTime.UtcNow;
+				var lastSeen = new[] { instance.LastSuccessAt, instance.LastFailureAt, instance.DescribedAt }.Max() ?? firstSeen;
+				await DB.Default.Update<RemoteInstance>().MatchID(instance.ID)
+					.Modify(i => i.Seen, "touched")
+					.Modify(i => i.FirstSeenAt, firstSeen)
+					.Modify(i => i.LastSeenAt, lastSeen)
+					.ExecuteAsync();
+			}
+		}
+	}
+}

PrivaPub/Infrastructure/Statistics/InteractionLedger.cs

@@ -3,6 +3,8 @@ using Microsoft.Extensions.Options;
 using MongoDB.Driver;
 using MongoDB.Entities;
 
+using PrivaPub.Federation.Moderation;
+using PrivaPub.Infrastructure.Jobs;
 using PrivaPub.Models.Jobs;
 using PrivaPub.Models.Statistics;
 
@@ -41,9 +43,16 @@ namespace PrivaPub.Infrastructure.Statistics
 			SingleWriter = false,
 			SingleReader = false
 		});
+		static readonly TimeSpan TouchInterval = TimeSpan.FromHours(1);
+		static readonly HashSet<string> TouchingPurposes = new(StringComparer.Ordinal) { "actor", "key", "object", "webfinger", "context" };
+
 		readonly InteractionSalts _salts;
 		readonly IOptionsMonitor<StatisticsOptions> _options;
 		readonly ILogger<InteractionLedger> _logger;
+		readonly IJobQueue _queue;
+		readonly IDomainBlocks _blocks;
+		readonly ConcurrentDictionary<string, byte> _touches = new(StringComparer.Ordinal);
+		readonly ConcurrentDictionary<string, DateTime> _touchedAt = new(StringComparer.Ordinal);
 		readonly SemaphoreSlim _flushing = new(1, 1);
 		readonly ConcurrentDictionary<(DateTime Day, string Host, string Key), long> _reads = new();
 		readonly ConcurrentDictionary<(DateTime Day, string Field, string Key), long> _server = new();
@@ -54,11 +63,14 @@ namespace PrivaPub.Infrastructure.Statistics
 		long _written;
 		long _failed;
 
-		public InteractionLedger(InteractionSalts salts, IOptionsMonitor<StatisticsOptions> options, ILogger<InteractionLedger> logger)
+		public InteractionLedger(InteractionSalts salts, IOptionsMonitor<StatisticsOptions> options, ILogger<InteractionLedger> logger,
+			IJobQueue queue = default, IDomainBlocks blocks = default)
 		{
 			_salts = salts;
 			_options = options;
 			_logger = logger;
+			_queue = queue;
+			_blocks = blocks;
 		}
 
 		public long Dropped => Interlocked.Read(ref _dropped);
@@ -85,6 +97,9 @@ namespace PrivaPub.Infrastructure.Statistics
 			var day = DateTime.UtcNow.Date;
 			host = Interactions.Host(host);
 			key = Key(key);
+			if (key.StartsWith("http:", StringComparison.Ordinal) && key.EndsWith(":ok", StringComparison.Ordinal)
+				&& TouchingPurposes.Contains(key.Split(':')[1]))
+				_touches[host] = 0;
 			_reads.AddOrUpdate((day, host, key), 1, (_, count) => count + 1);
 			if (bytes > 0)
 				_reads.AddOrUpdate((day, host, key + ":bytes"), bytes, (_, count) => count + bytes);
@@ -118,6 +133,7 @@ namespace PrivaPub.Infrastructure.Statistics
 					{
 					}
 					await FlushEvents(stoppingToken);
+					await FlushTouches(stoppingToken);
 					if (DateTime.UtcNow - countersAt >= CounterInterval)
 					{
 						countersAt = DateTime.UtcNow;
@@ -152,9 +168,56 @@ namespace PrivaPub.Infrastructure.Statistics
 		public async Task Flush(CancellationToken token)
 		{
 			await FlushEvents(token);
+			await FlushTouches(token);
 			await FlushCounters(token);
 		}
 
+		static bool Touches(InteractionEvent e, bool verified) => e.Channel switch
+		{
+			Interactions.Receive => verified && e.Outcome == Interactions.Queued,
+			Interactions.In or Interactions.Out => true,
+			Interactions.Http => e.Outcome == Interactions.Ok && TouchingPurposes.Contains(e.Purpose ?? string.Empty),
+			_ => false
+		};
+
+		//a server we exchanged something with is marked as seen, and described once a week
+		async Task FlushTouches(CancellationToken token)
+		{
+			await _flushing.WaitAsync(token);
+			try
+			{
+				await Touch(token);
+			}
+			finally
+			{
+				_flushing.Release();
+			}
+		}
+
+		async Task Touch(CancellationToken token)
+		{
+			foreach (var host in _touches.Keys.ToList())
+			{
+				_touches.TryRemove(host, out _);
+				var now = DateTime.UtcNow;
+				if (host == Interactions.Unknown || _touchedAt.TryGetValue(host, out var at) && now - at < TouchInterval)
+					continue;
+				_touchedAt[host] = now;
+				await DB.Default.Update<RemoteInstance>()
+					.Match(i => i.Host == host)
+					.Modify(i => i.Seen, "touched")
+					.Modify(i => i.LastSeenAt, now)
+					.Modify(b => b.SetOnInsert(i => i.FirstSeenAt, now))
+					.Option(o => o.IsUpsert = true)
+					.ExecuteAsync(token);
+				_knownHosts.Add(host);
+				if (_queue != default && _blocks?.IsSuspended(host) != true)
+					await _queue.Enqueue(JobKind.DescribeInstance, host, host, Federation.Objects.InstanceDescriber.DedupeKey(host, now), token);
+			}
+			if (_touchedAt.Count > 100_000)
+				_touchedAt.Clear();
+		}
+
 		async Task FlushEvents(CancellationToken token)
 		{
 			await _flushing.WaitAsync(token);
@@ -192,6 +255,8 @@ namespace PrivaPub.Infrastructure.Statistics
 				interaction.Host = Interactions.Unknown;
 			if (pending.ActorUri != default)
 				interaction.ActorHash = InteractionSalts.Hash(await _salts.For(interaction.At, token), pending.ActorUri);
+			if (interaction.Host != Interactions.Unknown && Touches(interaction, pending.ActorUri != default))
+				_touches[interaction.Host] = 0;
 			return interaction;
 		}
 

PrivaPub/Models/Jobs/RemoteInstance.cs

@@ -19,5 +19,68 @@ namespace PrivaPub.Models.Jobs
 		public string NodeInfo { get; set; }//the raw document, when small enough
 		public DateTime? DescribedAt { get; set; }
 		public string DescriptionError { get; set; }
+
+		public string Seen { get; set; }//"touched": we exchanged something with it; "crawled": only its peers list named it
+		public DateTime? FirstSeenAt { get; set; }
+		public DateTime? LastSeenAt { get; set; }
+		public DateTime? LastCrawledAt { get; set; }
+		public string CrawlError { get; set; }
+		public bool? RobotsDisallowed { get; set; }
+		public int? PeersCount { get; set; }
+
+		public string NodeInfoVersion { get; set; }
+		public string NodeDescription { get; set; }
+		public long? UsersTotal { get; set; }//as the server publishes them in its NodeInfo
+		public long? UsersActiveMonth { get; set; }
+		public long? UsersActiveHalfyear { get; set; }
+		public long? LocalPosts { get; set; }
+		public long? LocalComments { get; set; }
+
+		public string InstanceApi { get; set; }//"mastodon-v2" or "mastodon-v1"
+		public string InstanceApiRaw { get; set; }//admin only: it names maintainers and contact accounts
+		public string Title { get; set; }
+		public List<string> Languages { get; set; }
+		public string RegistrationMode { get; set; }//open, approval, closed
+		public int? MaxPostChars { get; set; }
+		public int? MastodonApiVersion { get; set; }
+		public string SourceUrl { get; set; }
+
+		public InstanceGeo Geo { get; set; }
+	}
+
+	public class InstanceGeo
+	{
+		public string Country { get; set; }
+		public string City { get; set; }
+		public double? Latitude { get; set; }//rounded to one decimal
+		public double? Longitude { get; set; }
+		public int? Asn { get; set; }
+		public string AsnOrg { get; set; }
+		public string Cdn { get; set; }//the CDN in front of it, whose edge address says nothing about where it runs
+		public bool? Ipv6 { get; set; }
+		public string Source { get; set; }
+		public DateTime? LocatedAt { get; set; }
+	}
+
+	public class RemoteInstanceSnapshot : Entity
+	{
+		public string Host { get; set; }
+		public string Week { get; set; }//ISO week, "2026-W40"
+		public DateTime TakenAt { get; set; }
+		public string Seen { get; set; }
+		public bool Reachable { get; set; }
+		public string Software { get; set; }
+		public string SoftwareVersion { get; set; }
+		public long? UsersTotal { get; set; }
+		public long? UsersActiveMonth { get; set; }
+		public long? UsersActiveHalfyear { get; set; }
+		public long? LocalPosts { get; set; }
+		public bool? OpenRegistrations { get; set; }
+		public string RegistrationMode { get; set; }
+		public List<string> Languages { get; set; }
+		public string Country { get; set; }
+		public int? Asn { get; set; }
+		public string Cdn { get; set; }
+		public bool? Ipv6 { get; set; }
 	}
 }