M8: every server we touch is described, located and snapshotted weekly

- Touches: the ledger marks a server as touched when it sends us a verified activity, when
  we exchange activities with it, or when we read its actors, keys, objects or WebFinger.
  It upserts RemoteInstance.Seen, FirstSeenAt and LastSeenAt at most hourly per server, and
  queues one DescribeInstance a week with the same dedupe key ObjectRecords uses. Suspended
  servers and pages behind link previews are never described. Migration _010 marks the
  servers already known as touched, with their dates.
- InstanceDescriber.Describe(host, crawled, allowed) reads:
  - NodeInfo 2.2/2.1/2.0, now with its published user counts, posts, comments,
    description, languages and schema version;
  - for software with a Mastodon API, /api/v2/instance falling back to v1: title,
    languages, registration mode, character limit, API version, source URL.

  It never keeps a contact as a field; the raw document is kept for the admin only. It
  locates the server from the address our connection reached (DB-IP Lite city and ASN, the
  CDN named when fronted) and writes a RemoteInstanceSnapshot per ISO week, unreachable
  weeks included. A crawled server is upserted as crawled only on insert, so it never
  downgrades a touched one, and robots.txt can deny any path.
- PublicGeo.Project is the only public form of a location: a CDN-fronted server shows its
  CDN only, a server reporting at least ten users shows its city, coordinates and network,
  any other only its country.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01ELjqpznMFMNrJoJUj6K5p2

PrivaPub/Federation/Objects/InstanceDocuments.cs

@@ -0,0 +1,156 @@
+using System.Globalization;
+using System.Text.Json;
+using System.Text.RegularExpressions;
+
+namespace PrivaPub.Federation.Objects
+{
+	public sealed record NodeInfoFacts(string Version, string Software, string SoftwareVersion, string NodeName, string NodeDescription,
+		List<string> Protocols, bool? OpenRegistrations, long? UsersTotal, long? UsersActiveMonth, long? UsersActiveHalfyear, long? LocalPosts,
+		long? LocalComments, List<string> Languages);
+
+	public sealed record InstanceApiFacts(string Title, List<string> Languages, string RegistrationMode, int? MaxPostChars, int? ApiVersion, string SourceUrl);
+
+	public static partial class InstanceDocuments
+	{
+		public static readonly string[] Schemas =
+		{
+			"http://nodeinfo.diaspora.software/ns/schema/2.2",
+			"http://nodeinfo.diaspora.software/ns/schema/2.1",
+			"http://nodeinfo.diaspora.software/ns/schema/2.0"
+		};
+
+		static readonly HashSet<string> MastodonLike = new(StringComparer.OrdinalIgnoreCase)
+		{
+			"mastodon", "hometown", "glitch-soc", "gotosocial", "pleroma", "akkoma", "sharkey", "iceshrimp", "iceshrimp.net", "firefish", "catodon",
+			"friendica", "pixelfed", "takahe", "snac", "honk", "mitra", "pachli"
+		};
+
+		const long MaxCount = 1_000_000_000;
+
+		public static bool SpeaksMastodonApi(string software) => software != default && MastodonLike.Contains(software);
+
+		public static (string Href, string Version) NodeInfoLink(JsonElement links)
+		{
+			if (links.ValueKind != JsonValueKind.Object || !links.TryGetProperty("links", out var list) || list.ValueKind != JsonValueKind.Array)
+				return default;
+			foreach (var schema in Schemas)
+				foreach (var link in list.EnumerateArray())
+					if (link.ValueKind == JsonValueKind.Object && Text(link, "rel") == schema && Text(link, "href") is { } href
+						&& href.StartsWith("https://", StringComparison.OrdinalIgnoreCase))
+						return (href, schema[(schema.LastIndexOf('/') + 1)..]);
+			return default;
+		}
+
+		public static NodeInfoFacts NodeInfo(JsonElement root, string version)
+		{
+			var software = Section(root, "software");
+			var usage = Section(root, "usage");
+			var users = Section(usage, "users");
+			var metadata = Section(root, "metadata");
+			return new NodeInfoFacts(
+				version,
+				Text(software, "name", 200),
+				Text(software, "version", 200),
+				Text(metadata, "nodeName", 200),
+				Plain(Text(metadata, "nodeDescription", 2000), 500),
+				Strings(root, "protocols", 16, value => value.Length <= 40),
+				root.ValueKind == JsonValueKind.Object && root.TryGetProperty("openRegistrations", out var open) && open.ValueKind is JsonValueKind.True or JsonValueKind.False
+					? open.GetBoolean()
+					: (bool?)null,
+				Count(users, "total"),
+				Count(users, "activeMonth"),
+				Count(users, "activeHalfyear"),
+				Count(usage, "localPosts"),
+				Count(usage, "localComments"),
+				Languages(metadata, "languages") ?? Languages(metadata, "langs"));
+		}
+
+		public static InstanceApiFacts InstanceApi(JsonElement root)
+		{
+			var registrations = root.ValueKind == JsonValueKind.Object && root.TryGetProperty("registrations", out var r) ? r : default;
+			string mode;
+			if (registrations.ValueKind == JsonValueKind.Object)
+				mode = Flag(registrations, "enabled") switch
+				{
+					true => Flag(registrations, "approval_required") == true ? "approval" : "open",
+					false => "closed",
+					_ => default
+				};
+			else if (registrations.ValueKind is JsonValueKind.True or JsonValueKind.False)
+				mode = registrations.GetBoolean() ? Flag(root, "approval_required") == true ? "approval" : "open" : "closed";
+			else
+				mode = default;
+			var statuses = Section(Section(root, "configuration"), "statuses");
+			var maxChars = Count(statuses, "max_characters") ?? Count(root, "max_toot_chars");
+			var apiVersion = Count(Section(root, "api_versions"), "mastodon");
+			var source = Text(root, "source_url", 300);
+			return new InstanceApiFacts(
+				Plain(Text(root, "title", 300), 200),
+				Languages(root, "languages"),
+				mode,
+				maxChars is { } chars ? (int)Math.Min(chars, int.MaxValue) : (int?)null,
+				apiVersion is { } api ? (int)Math.Min(api, int.MaxValue) : (int?)null,
+				source != default && source.StartsWith("https://", StringComparison.OrdinalIgnoreCase) ? source : default);
+		}
+
+		static JsonElement Section(JsonElement parent, string name) =>
+			parent.ValueKind == JsonValueKind.Object && parent.TryGetProperty(name, out var value) && value.ValueKind == JsonValueKind.Object ? value : default;
+
+		static string Text(JsonElement parent, string name, int max = 2048)
+		{
+			if (parent.ValueKind != JsonValueKind.Object || !parent.TryGetProperty(name, out var value) || value.ValueKind != JsonValueKind.String)
+				return default;
+			var text = value.GetString();
+			return text.Length <= max ? text : text[..max];
+		}
+
+		static bool? Flag(JsonElement parent, string name) =>
+			parent.ValueKind == JsonValueKind.Object && parent.TryGetProperty(name, out var value) && value.ValueKind is JsonValueKind.True or JsonValueKind.False
+				? value.GetBoolean()
+				: (bool?)null;
+
+		static long? Count(JsonElement parent, string name)
+		{
+			if (parent.ValueKind != JsonValueKind.Object || !parent.TryGetProperty(name, out var value))
+				return default;
+			long? number = value.ValueKind switch
+			{
+				JsonValueKind.Number when value.TryGetInt64(out var whole) => whole,
+				JsonValueKind.Number when value.TryGetDouble(out var real) && real is >= 0 and < MaxCount => (long)real,
+				JsonValueKind.String when long.TryParse(value.GetString(), NumberStyles.Integer, CultureInfo.InvariantCulture, out var parsed) => parsed,
+				_ => (long?)null
+			};
+			return number is { } n ? Math.Clamp(n, 0, MaxCount) : (long?)null;
+		}
+
+		static List<string> Strings(JsonElement parent, string name, int max, Func<string, bool> keep) =>
+			parent.ValueKind == JsonValueKind.Object && parent.TryGetProperty(name, out var value) && value.ValueKind == JsonValueKind.Array
+				? value.EnumerateArray().Where(v => v.ValueKind == JsonValueKind.String).Select(v => v.GetString()).Where(keep).Distinct().Take(max).ToList()
+				: new List<string>();
+
+		static List<string> Languages(JsonElement parent, string name)
+		{
+			var languages = Strings(parent, name, 16, language => LanguageCode().IsMatch(language.ToLowerInvariant()))
+				.Select(language => language.ToLowerInvariant()).Distinct().ToList();
+			return languages.Count == 0 ? default : languages;
+		}
+
+		static string Plain(string text, int max)
+		{
+			if (string.IsNullOrWhiteSpace(text))
+				return default;
+			var plain = Tags().Replace(text, " ").Trim();
+			plain = Spaces().Replace(plain, " ");
+			return plain.Length <= max ? plain : plain[..max];
+		}
+
+		[GeneratedRegex("^[a-z]{2,3}(-[a-z0-9]{2,8})?$")]
+		private static partial Regex LanguageCode();
+
+		[GeneratedRegex("<[^>]*>")]
+		private static partial Regex Tags();
+
+		[GeneratedRegex("\\s+")]
+		private static partial Regex Spaces();
+	}
+}