M8: every server we touch is described, located and snapshotted weekly

- Touches: the ledger marks a server as touched when it sends us a verified activity, when
  we exchange activities with it, or when we read its actors, keys, objects or WebFinger.
  It upserts RemoteInstance.Seen, FirstSeenAt and LastSeenAt at most hourly per server, and
  queues one DescribeInstance a week with the same dedupe key ObjectRecords uses. Suspended
  servers and pages behind link previews are never described. Migration _010 marks the
  servers already known as touched, with their dates.
- InstanceDescriber.Describe(host, crawled, allowed) reads:
  - NodeInfo 2.2/2.1/2.0, now with its published user counts, posts, comments,
    description, languages and schema version;
  - for software with a Mastodon API, /api/v2/instance falling back to v1: title,
    languages, registration mode, character limit, API version, source URL.

  It never keeps a contact as a field; the raw document is kept for the admin only. It
  locates the server from the address our connection reached (DB-IP Lite city and ASN, the
  CDN named when fronted) and writes a RemoteInstanceSnapshot per ISO week, unreachable
  weeks included. A crawled server is upserted as crawled only on insert, so it never
  downgrades a touched one, and robots.txt can deny any path.
- PublicGeo.Project is the only public form of a location: a CDN-fronted server shows its
  CDN only, a server reporting at least ten users shows its city, coordinates and network,
  any other only its country.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01ELjqpznMFMNrJoJUj6K5p2

PrivaPub/Domain/Statistics/PublicGeo.cs

@@ -0,0 +1,23 @@
+using PrivaPub.Models.Jobs;
+
+namespace PrivaPub.Domain.Statistics
+{
+	public sealed record PublicLocation(string Country, string City, double? Latitude, double? Longitude, string Network, string Cdn);
+
+	public static class PublicGeo
+	{
+		//owner decision: a CDN-fronted server shows only its CDN (the address is an edge), a server reporting at least
+		//minUsers users its city, coordinates and network, any other only its country
+		public static PublicLocation Project(RemoteInstance instance, int minUsers)
+		{
+			var geo = instance?.Geo;
+			if (geo == default)
+				return default;
+			if (geo.Cdn != default)
+				return new PublicLocation(default, default, default, default, default, geo.Cdn);
+			if (instance.UsersTotal >= minUsers)
+				return new PublicLocation(geo.Country, geo.City, geo.Latitude, geo.Longitude, geo.AsnOrg, default);
+			return geo.Country == default ? default : new PublicLocation(geo.Country, default, default, default, default, default);
+		}
+	}
+}

PrivaPub/Federation/Objects/InstanceDescriber.cs

@@ -1,88 +1,180 @@
 using MongoDB.Entities;
 
+using PrivaPub.Infrastructure.Geo;
 using PrivaPub.Infrastructure.Http;
 using PrivaPub.Infrastructure.Jobs;
 using PrivaPub.Models.Jobs;
 
+using System.Globalization;
+using System.Net.Sockets;
 using System.Text.Json;
 
 namespace PrivaPub.Federation.Objects
 {
 	public class InstanceDescriber : IJobHandler
 	{
-		const int MaxNodeInfoChars = 64 * 1024;
-		static readonly string[] Schemas =
-		{
-			"http://nodeinfo.diaspora.software/ns/schema/2.1",
-			"http://nodeinfo.diaspora.software/ns/schema/2.0"
-		};
+		const int MaxRawChars = 64 * 1024;
 
 		readonly IFederationHttp _http;
+		readonly IGeoLocator _geo;
+		readonly IConnectedAddresses _addresses;
 
-		public InstanceDescriber(IFederationHttp http) => _http = http;
+		public InstanceDescriber(IFederationHttp http, IGeoLocator geo = default, IConnectedAddresses addresses = default)
+		{
+			_http = http;
+			_geo = geo;
+			_addresses = addresses;
+		}
 
 		public JobKind Kind => JobKind.DescribeInstance;
 		public int Concurrency => 1;
 		public int MaxAttempts => 2;
 		public int PerHostLimit => 1;
 
+		public static string Week(DateTime at) => $"{ISOWeek.GetYear(at)}-W{ISOWeek.GetWeekOfYear(at):00}";
+
+		public static string DedupeKey(string host, DateTime at) => $"describe|{host}|{ISOWeek.GetYear(at)}-{ISOWeek.GetWeekOfYear(at)}";
+
 		public async Task<JobOutcome> Handle(Job job, CancellationToken token)
 		{
-			var host = job.Payload;
-			using var scope = HttpScope.For("nodeinfo");
-			using var links = await _http.GetJson(Address($"https://{host}/.well-known/nodeinfo"), "application/json", sign: default, token);
-			var href = links == default ? default : Schemas.Select(schema => LinkFor(links.Root, schema)).FirstOrDefault(link => link != default);
-			if (href == default)
-				return await Described(host, default, "no NodeInfo", token);
+			await Describe(job.Payload, crawled: false, default, token);
+			return JobOutcome.Done;
+		}
 
-			using var nodeInfo = await _http.GetJson(Address(href), "application/json", sign: default, token);
-			return nodeInfo == default ? await Described(host, default, "NodeInfo unreadable", token) : await Described(host, nodeInfo.Root, default, token);
+		//NodeInfo, the Mastodon instance API where the software has one, and where the address we reached sits; unsigned,
+		//because none of these are ActivityPub documents. `allowed` is the crawler's robots.txt.
+		public async Task<bool> Describe(string host, bool crawled, Func<string, bool> allowed, CancellationToken token)
+		{
+			allowed ??= _ => true;
+			NodeInfoFacts facts = default;
+			string rawNodeInfo = default, error;
+			using (HttpScope.For("nodeinfo"))
+			{
+				error = allowed("/.well-known/nodeinfo") ? "no NodeInfo" : "robots.txt disallows NodeInfo";
+				using var links = allowed("/.well-known/nodeinfo")
+					? await _http.GetJson(Address($"https://{host}/.well-known/nodeinfo"), "application/json", sign: default, token)
+					: default;
+				var (href, version) = links == default ? default : InstanceDocuments.NodeInfoLink(links.Root);
+				if (href != default && allowed(new Uri(href).AbsolutePath))
+				{
+					using var nodeInfo = await _http.GetJson(Address(href), "application/json", sign: default, token);
+					if (nodeInfo != default)
+					{
+						facts = InstanceDocuments.NodeInfo(nodeInfo.Root, version);
+						rawNodeInfo = nodeInfo.Root.GetRawText();
+						error = default;
+					}
+					else
+						error = "NodeInfo unreadable";
+				}
+			}
+
+			InstanceApiFacts api = default;
+			string apiKind = default, rawApi = default;
+			if (InstanceDocuments.SpeaksMastodonApi(facts?.Software))
+				using (HttpScope.For("instance-api"))
+					foreach (var (path, kind) in new[] { ("/api/v2/instance", "mastodon-v2"), ("/api/v1/instance", "mastodon-v1") })
+					{
+						if (!allowed(path))
+							continue;
+						using var document = await _http.GetJson(Address($"https://{host}{path}"), "application/json", sign: default, token);
+						if (document == default || document.Root.ValueKind != JsonValueKind.Object)
+							continue;
+						api = InstanceDocuments.InstanceApi(document.Root);
+						apiKind = kind;
+						rawApi = document.Root.GetRawText();
+						break;
+					}
+
+			var geo = Locate(host);
+			var now = DateTime.UtcNow;
+			var update = DB.Default.Update<RemoteInstance>()
+				.Match(i => i.Host == host)
+				.Modify(i => i.DescribedAt, now)
+				.Modify(i => i.DescriptionError, error)
+				.Option(o => o.IsUpsert = true);
+			if (crawled)
+				update = update
+					.Modify(b => b.SetOnInsert(i => i.Seen, "crawled"))
+					.Modify(b => b.SetOnInsert(i => i.FirstSeenAt, now));
+			if (facts != default)
+				update = update
+					.Modify(i => i.Software, facts.Software)
+					.Modify(i => i.SoftwareVersion, facts.SoftwareVersion)
+					.Modify(i => i.NodeName, facts.NodeName)
+					.Modify(i => i.NodeDescription, facts.NodeDescription)
+					.Modify(i => i.NodeInfoVersion, facts.Version)
+					.Modify(i => i.Protocols, facts.Protocols)
+					.Modify(i => i.OpenRegistrations, facts.OpenRegistrations)
+					.Modify(i => i.UsersTotal, facts.UsersTotal)
+					.Modify(i => i.UsersActiveMonth, facts.UsersActiveMonth)
+					.Modify(i => i.UsersActiveHalfyear, facts.UsersActiveHalfyear)
+					.Modify(i => i.LocalPosts, facts.LocalPosts)
+					.Modify(i => i.LocalComments, facts.LocalComments)
+					.Modify(i => i.NodeInfo, rawNodeInfo.Length <= MaxRawChars ? rawNodeInfo : default);
+			if (api != default)
+				update = update
+					.Modify(i => i.InstanceApi, apiKind)
+					.Modify(i => i.InstanceApiRaw, rawApi.Length <= MaxRawChars ? rawApi : default)
+					.Modify(i => i.Title, api.Title)
+					.Modify(i => i.Languages, api.Languages ?? facts?.Languages)
+					.Modify(i => i.RegistrationMode, api.RegistrationMode)
+					.Modify(i => i.MaxPostChars, api.MaxPostChars)
+					.Modify(i => i.MastodonApiVersion, api.ApiVersion)
+					.Modify(i => i.SourceUrl, api.SourceUrl);
+			else if (facts?.Languages != default)
+				update = update.Modify(i => i.Languages, facts.Languages);
+			if (geo != default)
+				update = update.Modify(i => i.Geo, geo);
+			await update.ExecuteAsync(token);
+			var instance = await DB.Default.Find<RemoteInstance>().Match(i => i.Host == host).ExecuteFirstAsync(token);
+
+			var week = Week(now);
+			await DB.Default.Update<RemoteInstanceSnapshot>()
+				.Match(s => s.Host == host && s.Week == week)
+				.Modify(s => s.TakenAt, now)
+				.Modify(s => s.Seen, instance?.Seen)
+				.Modify(s => s.Reachable, facts != default || api != default)
+				.Modify(s => s.Software, instance?.Software)
+				.Modify(s => s.SoftwareVersion, instance?.SoftwareVersion)
+				.Modify(s => s.UsersTotal, instance?.UsersTotal)
+				.Modify(s => s.UsersActiveMonth, instance?.UsersActiveMonth)
+				.Modify(s => s.UsersActiveHalfyear, instance?.UsersActiveHalfyear)
+				.Modify(s => s.LocalPosts, instance?.LocalPosts)
+				.Modify(s => s.OpenRegistrations, instance?.OpenRegistrations)
+				.Modify(s => s.RegistrationMode, instance?.RegistrationMode)
+				.Modify(s => s.Languages, instance?.Languages)
+				.Modify(s => s.Country, instance?.Geo?.Country)
+				.Modify(s => s.Asn, instance?.Geo?.Asn)
+				.Modify(s => s.Cdn, instance?.Geo?.Cdn)
+				.Modify(s => s.Ipv6, instance?.Geo?.Ipv6)
+				.Option(o => o.IsUpsert = true)
+				.ExecuteAsync(token);
+			return facts != default;
+		}
+
+		InstanceGeo Locate(string host)
+		{
+			var reached = _addresses?.Of(new Uri(Address($"https://{host}/")).Host);
+			var fix = reached == default ? default : _geo?.Locate(reached);
+			if (fix == default)
+				return default;
+			return new InstanceGeo
+			{
+				Country = fix.Country,
+				City = fix.City,
+				Latitude = fix.Latitude,
+				Longitude = fix.Longitude,
+				Asn = fix.Asn,
+				AsnOrg = fix.AsnOrg,
+				Cdn = CdnNetworks.Of(fix.Asn),
+				Ipv6 = reached.AddressFamily == AddressFamily.InterNetworkV6,
+				Source = _geo.Source,
+				LocatedAt = DateTime.UtcNow
+			};
 		}
 
 		//where an https address is fetched from: itself, except in tests, which point it at a plain-http peer
 		protected virtual string Address(string url) => url;
-
-		static async Task<JobOutcome> Described(string host, JsonElement? nodeInfo, string error, CancellationToken token)
-		{
-			var update = DB.Default.Update<RemoteInstance>()
-				.Match(i => i.Host == host)
-				.Modify(i => i.DescribedAt, DateTime.UtcNow)
-				.Modify(i => i.DescriptionError, error)
-				.Option(o => o.IsUpsert = true);
-			if (nodeInfo is { } root)
-			{
-				var raw = root.GetRawText();
-				update = update
-					.Modify(i => i.Software, Text(root, "software", "name"))
-					.Modify(i => i.SoftwareVersion, Text(root, "software", "version"))
-					.Modify(i => i.NodeName, Text(root, "metadata", "nodeName"))
-					.Modify(i => i.Protocols, root.TryGetProperty("protocols", out var protocols) && protocols.ValueKind == JsonValueKind.Array
-						? protocols.EnumerateArray().Where(p => p.ValueKind == JsonValueKind.String).Select(p => p.GetString()).Take(16).ToList()
-						: new List<string>())
-					.Modify(i => i.OpenRegistrations, root.TryGetProperty("openRegistrations", out var open) && open.ValueKind is JsonValueKind.True or JsonValueKind.False
-						? open.GetBoolean()
-						: default(bool?))
-					.Modify(i => i.NodeInfo, raw.Length <= MaxNodeInfoChars ? raw : default);
-			}
-			await update.ExecuteAsync(token);
-			return JobOutcome.Done;
-		}
-
-		static string LinkFor(JsonElement root, string schema) =>
-			root.TryGetProperty("links", out var links) && links.ValueKind == JsonValueKind.Array
-				? links.EnumerateArray()
-					.Where(l => l.ValueKind == JsonValueKind.Object && l.TryGetProperty("rel", out var rel) && rel.ValueKind == JsonValueKind.String && rel.GetString() == schema)
-					.Select(l => l.TryGetProperty("href", out var href) && href.ValueKind == JsonValueKind.String ? href.GetString() : default)
-					.FirstOrDefault(h => h != default && h.StartsWith("https://", StringComparison.OrdinalIgnoreCase))
-				: default;
-
-		static string Text(JsonElement root, string parent, string name)
-		{
-			if (!root.TryGetProperty(parent, out var section) || section.ValueKind != JsonValueKind.Object
-				|| !section.TryGetProperty(name, out var value) || value.ValueKind != JsonValueKind.String)
-				return default;
-			var text = value.GetString();
-			return text.Length <= 200 ? text : text[..200];
-		}
 	}
 }

PrivaPub/Federation/Objects/InstanceDocuments.cs

@@ -0,0 +1,156 @@
+using System.Globalization;
+using System.Text.Json;
+using System.Text.RegularExpressions;
+
+namespace PrivaPub.Federation.Objects
+{
+	public sealed record NodeInfoFacts(string Version, string Software, string SoftwareVersion, string NodeName, string NodeDescription,
+		List<string> Protocols, bool? OpenRegistrations, long? UsersTotal, long? UsersActiveMonth, long? UsersActiveHalfyear, long? LocalPosts,
+		long? LocalComments, List<string> Languages);
+
+	public sealed record InstanceApiFacts(string Title, List<string> Languages, string RegistrationMode, int? MaxPostChars, int? ApiVersion, string SourceUrl);
+
+	public static partial class InstanceDocuments
+	{
+		public static readonly string[] Schemas =
+		{
+			"http://nodeinfo.diaspora.software/ns/schema/2.2",
+			"http://nodeinfo.diaspora.software/ns/schema/2.1",
+			"http://nodeinfo.diaspora.software/ns/schema/2.0"
+		};
+
+		static readonly HashSet<string> MastodonLike = new(StringComparer.OrdinalIgnoreCase)
+		{
+			"mastodon", "hometown", "glitch-soc", "gotosocial", "pleroma", "akkoma", "sharkey", "iceshrimp", "iceshrimp.net", "firefish", "catodon",
+			"friendica", "pixelfed", "takahe", "snac", "honk", "mitra", "pachli"
+		};
+
+		const long MaxCount = 1_000_000_000;
+
+		public static bool SpeaksMastodonApi(string software) => software != default && MastodonLike.Contains(software);
+
+		public static (string Href, string Version) NodeInfoLink(JsonElement links)
+		{
+			if (links.ValueKind != JsonValueKind.Object || !links.TryGetProperty("links", out var list) || list.ValueKind != JsonValueKind.Array)
+				return default;
+			foreach (var schema in Schemas)
+				foreach (var link in list.EnumerateArray())
+					if (link.ValueKind == JsonValueKind.Object && Text(link, "rel") == schema && Text(link, "href") is { } href
+						&& href.StartsWith("https://", StringComparison.OrdinalIgnoreCase))
+						return (href, schema[(schema.LastIndexOf('/') + 1)..]);
+			return default;
+		}
+
+		public static NodeInfoFacts NodeInfo(JsonElement root, string version)
+		{
+			var software = Section(root, "software");
+			var usage = Section(root, "usage");
+			var users = Section(usage, "users");
+			var metadata = Section(root, "metadata");
+			return new NodeInfoFacts(
+				version,
+				Text(software, "name", 200),
+				Text(software, "version", 200),
+				Text(metadata, "nodeName", 200),
+				Plain(Text(metadata, "nodeDescription", 2000), 500),
+				Strings(root, "protocols", 16, value => value.Length <= 40),
+				root.ValueKind == JsonValueKind.Object && root.TryGetProperty("openRegistrations", out var open) && open.ValueKind is JsonValueKind.True or JsonValueKind.False
+					? open.GetBoolean()
+					: (bool?)null,
+				Count(users, "total"),
+				Count(users, "activeMonth"),
+				Count(users, "activeHalfyear"),
+				Count(usage, "localPosts"),
+				Count(usage, "localComments"),
+				Languages(metadata, "languages") ?? Languages(metadata, "langs"));
+		}
+
+		public static InstanceApiFacts InstanceApi(JsonElement root)
+		{
+			var registrations = root.ValueKind == JsonValueKind.Object && root.TryGetProperty("registrations", out var r) ? r : default;
+			string mode;
+			if (registrations.ValueKind == JsonValueKind.Object)
+				mode = Flag(registrations, "enabled") switch
+				{
+					true => Flag(registrations, "approval_required") == true ? "approval" : "open",
+					false => "closed",
+					_ => default
+				};
+			else if (registrations.ValueKind is JsonValueKind.True or JsonValueKind.False)
+				mode = registrations.GetBoolean() ? Flag(root, "approval_required") == true ? "approval" : "open" : "closed";
+			else
+				mode = default;
+			var statuses = Section(Section(root, "configuration"), "statuses");
+			var maxChars = Count(statuses, "max_characters") ?? Count(root, "max_toot_chars");
+			var apiVersion = Count(Section(root, "api_versions"), "mastodon");
+			var source = Text(root, "source_url", 300);
+			return new InstanceApiFacts(
+				Plain(Text(root, "title", 300), 200),
+				Languages(root, "languages"),
+				mode,
+				maxChars is { } chars ? (int)Math.Min(chars, int.MaxValue) : (int?)null,
+				apiVersion is { } api ? (int)Math.Min(api, int.MaxValue) : (int?)null,
+				source != default && source.StartsWith("https://", StringComparison.OrdinalIgnoreCase) ? source : default);
+		}
+
+		static JsonElement Section(JsonElement parent, string name) =>
+			parent.ValueKind == JsonValueKind.Object && parent.TryGetProperty(name, out var value) && value.ValueKind == JsonValueKind.Object ? value : default;
+
+		static string Text(JsonElement parent, string name, int max = 2048)
+		{
+			if (parent.ValueKind != JsonValueKind.Object || !parent.TryGetProperty(name, out var value) || value.ValueKind != JsonValueKind.String)
+				return default;
+			var text = value.GetString();
+			return text.Length <= max ? text : text[..max];
+		}
+
+		static bool? Flag(JsonElement parent, string name) =>
+			parent.ValueKind == JsonValueKind.Object && parent.TryGetProperty(name, out var value) && value.ValueKind is JsonValueKind.True or JsonValueKind.False
+				? value.GetBoolean()
+				: (bool?)null;
+
+		static long? Count(JsonElement parent, string name)
+		{
+			if (parent.ValueKind != JsonValueKind.Object || !parent.TryGetProperty(name, out var value))
+				return default;
+			long? number = value.ValueKind switch
+			{
+				JsonValueKind.Number when value.TryGetInt64(out var whole) => whole,
+				JsonValueKind.Number when value.TryGetDouble(out var real) && real is >= 0 and < MaxCount => (long)real,
+				JsonValueKind.String when long.TryParse(value.GetString(), NumberStyles.Integer, CultureInfo.InvariantCulture, out var parsed) => parsed,
+				_ => (long?)null
+			};
+			return number is { } n ? Math.Clamp(n, 0, MaxCount) : (long?)null;
+		}
+
+		static List<string> Strings(JsonElement parent, string name, int max, Func<string, bool> keep) =>
+			parent.ValueKind == JsonValueKind.Object && parent.TryGetProperty(name, out var value) && value.ValueKind == JsonValueKind.Array
+				? value.EnumerateArray().Where(v => v.ValueKind == JsonValueKind.String).Select(v => v.GetString()).Where(keep).Distinct().Take(max).ToList()
+				: new List<string>();
+
+		static List<string> Languages(JsonElement parent, string name)
+		{
+			var languages = Strings(parent, name, 16, language => LanguageCode().IsMatch(language.ToLowerInvariant()))
+				.Select(language => language.ToLowerInvariant()).Distinct().ToList();
+			return languages.Count == 0 ? default : languages;
+		}
+
+		static string Plain(string text, int max)
+		{
+			if (string.IsNullOrWhiteSpace(text))
+				return default;
+			var plain = Tags().Replace(text, " ").Trim();
+			plain = Spaces().Replace(plain, " ");
+			return plain.Length <= max ? plain : plain[..max];
+		}
+
+		[GeneratedRegex("^[a-z]{2,3}(-[a-z0-9]{2,8})?$")]
+		private static partial Regex LanguageCode();
+
+		[GeneratedRegex("<[^>]*>")]
+		private static partial Regex Tags();
+
+		[GeneratedRegex("\\s+")]
+		private static partial Regex Spaces();
+	}
+}

PrivaPub/Federation/Objects/ObjectRecords.cs

@@ -99,8 +99,7 @@ namespace PrivaPub.Federation.Objects
 			var known = await DB.Default.Find<RemoteInstance>().Match(i => i.Host == host).ExecuteFirstAsync(token);
 			if (known?.DescribedAt > DateTime.UtcNow - DescriptionAge)
 				return;
-			var week = ISOWeek.GetYear(DateTime.UtcNow) + "-" + ISOWeek.GetWeekOfYear(DateTime.UtcNow);
-			await _queue.Enqueue(JobKind.DescribeInstance, host, host, $"describe|{host}|{week}", token);
+			await _queue.Enqueue(JobKind.DescribeInstance, host, host, InstanceDescriber.DedupeKey(host, DateTime.UtcNow), token);
 		}
 
 		public static (string Text, string Hash, int Bytes, bool Truncated) Capture(JsonNode raw)

PrivaPub/Infrastructure/Data/Indexes.cs

@@ -127,6 +127,11 @@ namespace PrivaPub.Infrastructure.Data
 			await DB.Default.Index<InstanceDay>().Key(d => d.Day, KeyType.Ascending).Key(d => d.Host, KeyType.Ascending).Option(o => o.Unique = true).CreateAsync(token);
 			await Plain<InstanceDay>(token, d => d.Host, d => d.Day);
 			await DB.Default.Index<ServerDay>().Key(d => d.Day, KeyType.Ascending).Option(o => o.Unique = true).CreateAsync(token);
+			await DB.Default.Index<RemoteInstanceSnapshot>().Key(s => s.Host, KeyType.Ascending).Key(s => s.Week, KeyType.Ascending)
+				.Option(o => o.Unique = true).CreateAsync(token);
+			await Plain<RemoteInstanceSnapshot>(token, s => s.Week);
+			await Plain<RemoteInstance>(token, i => i.Seen);
+			await Plain<RemoteInstance>(token, i => i.LastCrawledAt);
 		}
 
 		static async Task Unique<T>(System.Linq.Expressions.Expression<Func<T, object>> key, FilterDefinition<T> partial,

PrivaPub/Infrastructure/Data/Migrations/_010_remote_instances_were_touched.cs

@@ -0,0 +1,24 @@
+using MongoDB.Bson;
+using MongoDB.Entities;
+
+using PrivaPub.Models.Jobs;
+
+namespace PrivaPub.Infrastructure.Data.Migrations
+{
+	public class _010_remote_instances_were_touched : IMigration
+	{
+		public async Task UpgradeAsync()
+		{
+			foreach (var instance in await DB.Default.Find<RemoteInstance>().Match(i => i.Seen == null).ExecuteAsync())
+			{
+				var firstSeen = ObjectId.TryParse(instance.ID, out var id) ? id.CreationTime : DateTime.UtcNow;
+				var lastSeen = new[] { instance.LastSuccessAt, instance.LastFailureAt, instance.DescribedAt }.Max() ?? firstSeen;
+				await DB.Default.Update<RemoteInstance>().MatchID(instance.ID)
+					.Modify(i => i.Seen, "touched")
+					.Modify(i => i.FirstSeenAt, firstSeen)
+					.Modify(i => i.LastSeenAt, lastSeen)
+					.ExecuteAsync();
+			}
+		}
+	}
+}

PrivaPub/Infrastructure/Statistics/InteractionLedger.cs

@@ -3,6 +3,8 @@ using Microsoft.Extensions.Options;
 using MongoDB.Driver;
 using MongoDB.Entities;
 
+using PrivaPub.Federation.Moderation;
+using PrivaPub.Infrastructure.Jobs;
 using PrivaPub.Models.Jobs;
 using PrivaPub.Models.Statistics;
 
@@ -41,9 +43,16 @@ namespace PrivaPub.Infrastructure.Statistics
 			SingleWriter = false,
 			SingleReader = false
 		});
+		static readonly TimeSpan TouchInterval = TimeSpan.FromHours(1);
+		static readonly HashSet<string> TouchingPurposes = new(StringComparer.Ordinal) { "actor", "key", "object", "webfinger", "context" };
+
 		readonly InteractionSalts _salts;
 		readonly IOptionsMonitor<StatisticsOptions> _options;
 		readonly ILogger<InteractionLedger> _logger;
+		readonly IJobQueue _queue;
+		readonly IDomainBlocks _blocks;
+		readonly ConcurrentDictionary<string, byte> _touches = new(StringComparer.Ordinal);
+		readonly ConcurrentDictionary<string, DateTime> _touchedAt = new(StringComparer.Ordinal);
 		readonly SemaphoreSlim _flushing = new(1, 1);
 		readonly ConcurrentDictionary<(DateTime Day, string Host, string Key), long> _reads = new();
 		readonly ConcurrentDictionary<(DateTime Day, string Field, string Key), long> _server = new();
@@ -54,11 +63,14 @@ namespace PrivaPub.Infrastructure.Statistics
 		long _written;
 		long _failed;
 
-		public InteractionLedger(InteractionSalts salts, IOptionsMonitor<StatisticsOptions> options, ILogger<InteractionLedger> logger)
+		public InteractionLedger(InteractionSalts salts, IOptionsMonitor<StatisticsOptions> options, ILogger<InteractionLedger> logger,
+			IJobQueue queue = default, IDomainBlocks blocks = default)
 		{
 			_salts = salts;
 			_options = options;
 			_logger = logger;
+			_queue = queue;
+			_blocks = blocks;
 		}
 
 		public long Dropped => Interlocked.Read(ref _dropped);
@@ -85,6 +97,9 @@ namespace PrivaPub.Infrastructure.Statistics
 			var day = DateTime.UtcNow.Date;
 			host = Interactions.Host(host);
 			key = Key(key);
+			if (key.StartsWith("http:", StringComparison.Ordinal) && key.EndsWith(":ok", StringComparison.Ordinal)
+				&& TouchingPurposes.Contains(key.Split(':')[1]))
+				_touches[host] = 0;
 			_reads.AddOrUpdate((day, host, key), 1, (_, count) => count + 1);
 			if (bytes > 0)
 				_reads.AddOrUpdate((day, host, key + ":bytes"), bytes, (_, count) => count + bytes);
@@ -118,6 +133,7 @@ namespace PrivaPub.Infrastructure.Statistics
 					{
 					}
 					await FlushEvents(stoppingToken);
+					await FlushTouches(stoppingToken);
 					if (DateTime.UtcNow - countersAt >= CounterInterval)
 					{
 						countersAt = DateTime.UtcNow;
@@ -152,9 +168,56 @@ namespace PrivaPub.Infrastructure.Statistics
 		public async Task Flush(CancellationToken token)
 		{
 			await FlushEvents(token);
+			await FlushTouches(token);
 			await FlushCounters(token);
 		}
 
+		static bool Touches(InteractionEvent e, bool verified) => e.Channel switch
+		{
+			Interactions.Receive => verified && e.Outcome == Interactions.Queued,
+			Interactions.In or Interactions.Out => true,
+			Interactions.Http => e.Outcome == Interactions.Ok && TouchingPurposes.Contains(e.Purpose ?? string.Empty),
+			_ => false
+		};
+
+		//a server we exchanged something with is marked as seen, and described once a week
+		async Task FlushTouches(CancellationToken token)
+		{
+			await _flushing.WaitAsync(token);
+			try
+			{
+				await Touch(token);
+			}
+			finally
+			{
+				_flushing.Release();
+			}
+		}
+
+		async Task Touch(CancellationToken token)
+		{
+			foreach (var host in _touches.Keys.ToList())
+			{
+				_touches.TryRemove(host, out _);
+				var now = DateTime.UtcNow;
+				if (host == Interactions.Unknown || _touchedAt.TryGetValue(host, out var at) && now - at < TouchInterval)
+					continue;
+				_touchedAt[host] = now;
+				await DB.Default.Update<RemoteInstance>()
+					.Match(i => i.Host == host)
+					.Modify(i => i.Seen, "touched")
+					.Modify(i => i.LastSeenAt, now)
+					.Modify(b => b.SetOnInsert(i => i.FirstSeenAt, now))
+					.Option(o => o.IsUpsert = true)
+					.ExecuteAsync(token);
+				_knownHosts.Add(host);
+				if (_queue != default && _blocks?.IsSuspended(host) != true)
+					await _queue.Enqueue(JobKind.DescribeInstance, host, host, Federation.Objects.InstanceDescriber.DedupeKey(host, now), token);
+			}
+			if (_touchedAt.Count > 100_000)
+				_touchedAt.Clear();
+		}
+
 		async Task FlushEvents(CancellationToken token)
 		{
 			await _flushing.WaitAsync(token);
@@ -192,6 +255,8 @@ namespace PrivaPub.Infrastructure.Statistics
 				interaction.Host = Interactions.Unknown;
 			if (pending.ActorUri != default)
 				interaction.ActorHash = InteractionSalts.Hash(await _salts.For(interaction.At, token), pending.ActorUri);
+			if (interaction.Host != Interactions.Unknown && Touches(interaction, pending.ActorUri != default))
+				_touches[interaction.Host] = 0;
 			return interaction;
 		}
 

PrivaPub/Models/Jobs/RemoteInstance.cs

@@ -19,5 +19,68 @@ namespace PrivaPub.Models.Jobs
 		public string NodeInfo { get; set; }//the raw document, when small enough
 		public DateTime? DescribedAt { get; set; }
 		public string DescriptionError { get; set; }
+
+		public string Seen { get; set; }//"touched": we exchanged something with it; "crawled": only its peers list named it
+		public DateTime? FirstSeenAt { get; set; }
+		public DateTime? LastSeenAt { get; set; }
+		public DateTime? LastCrawledAt { get; set; }
+		public string CrawlError { get; set; }
+		public bool? RobotsDisallowed { get; set; }
+		public int? PeersCount { get; set; }
+
+		public string NodeInfoVersion { get; set; }
+		public string NodeDescription { get; set; }
+		public long? UsersTotal { get; set; }//as the server publishes them in its NodeInfo
+		public long? UsersActiveMonth { get; set; }
+		public long? UsersActiveHalfyear { get; set; }
+		public long? LocalPosts { get; set; }
+		public long? LocalComments { get; set; }
+
+		public string InstanceApi { get; set; }//"mastodon-v2" or "mastodon-v1"
+		public string InstanceApiRaw { get; set; }//admin only: it names maintainers and contact accounts
+		public string Title { get; set; }
+		public List<string> Languages { get; set; }
+		public string RegistrationMode { get; set; }//open, approval, closed
+		public int? MaxPostChars { get; set; }
+		public int? MastodonApiVersion { get; set; }
+		public string SourceUrl { get; set; }
+
+		public InstanceGeo Geo { get; set; }
+	}
+
+	public class InstanceGeo
+	{
+		public string Country { get; set; }
+		public string City { get; set; }
+		public double? Latitude { get; set; }//rounded to one decimal
+		public double? Longitude { get; set; }
+		public int? Asn { get; set; }
+		public string AsnOrg { get; set; }
+		public string Cdn { get; set; }//the CDN in front of it, whose edge address says nothing about where it runs
+		public bool? Ipv6 { get; set; }
+		public string Source { get; set; }
+		public DateTime? LocatedAt { get; set; }
+	}
+
+	public class RemoteInstanceSnapshot : Entity
+	{
+		public string Host { get; set; }
+		public string Week { get; set; }//ISO week, "2026-W40"
+		public DateTime TakenAt { get; set; }
+		public string Seen { get; set; }
+		public bool Reachable { get; set; }
+		public string Software { get; set; }
+		public string SoftwareVersion { get; set; }
+		public long? UsersTotal { get; set; }
+		public long? UsersActiveMonth { get; set; }
+		public long? UsersActiveHalfyear { get; set; }
+		public long? LocalPosts { get; set; }
+		public bool? OpenRegistrations { get; set; }
+		public string RegistrationMode { get; set; }
+		public List<string> Languages { get; set; }
+		public string Country { get; set; }
+		public int? Asn { get; set; }
+		public string Cdn { get; set; }
+		public bool? Ipv6 { get; set; }
 	}
 }