M9 (first part): the means to locate a server

- IConnectedAddresses remembers which address each host's last connection reached, set in
  SafeHttpHandlerFactory's connect callback. That is once per pooled connection, with no
  second DNS lookup, and the address is never stored.
- IGeoLocator / DbIpLocator reads the offline DB-IP Lite city and ASN databases (MaxMind
  .mmdb, via MaxMind.Db). It maps memory, swaps to new files within ten minutes, rounds
  coordinates to one decimal, never looks up a private address, and answers nothing
  when the files are missing. CdnNetworks names the CDNs whose edge addresses say nothing
  about where a server is.
- deploy/max/geo-update.sh fetches this or last month's databases, checks them and swaps
  them in atomically. The privapub-geo timer runs it monthly as www-data, and setup.sh
  installs the directory, the script, the units and a first download.

Describing servers will use these in M8.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01ELjqpznMFMNrJoJUj6K5p2

PrivaPub.Tests/Infrastructure/GeoLocatorTests.cs

@@ -0,0 +1,69 @@
+using Microsoft.Extensions.Logging.Abstractions;
+
+using PrivaPub.Infrastructure.Geo;
+using PrivaPub.Infrastructure.Http;
+using PrivaPub.Infrastructure.Statistics;
+using PrivaPub.Tests.Support;
+
+using System.Net;
+
+namespace PrivaPub.Tests.Infrastructure
+{
+	public class GeoLocatorTests
+	{
+		static DbIpLocator Locator(string directory) =>
+			new(new StaticOptions<StatisticsOptions>(new StatisticsOptions { GeoDirectory = directory }), NullLogger<DbIpLocator>.Instance);
+
+		[Fact]
+		public void Without_databases_nothing_is_located()
+		{
+			using var locator = Locator(Path.Combine(Path.GetTempPath(), $"no-geo-{Guid.NewGuid():N}"));
+
+			Assert.Null(locator.Locate(IPAddress.Parse("1.1.1.1")));
+			Assert.Null(locator.Source);
+		}
+
+		[Fact]
+		public void A_private_address_is_never_looked_up()
+		{
+			using var locator = Locator(Path.GetTempPath());
+
+			Assert.Null(locator.Locate(IPAddress.Parse("10.1.2.3")));
+			Assert.Null(locator.Locate(IPAddress.Loopback));
+			Assert.Null(locator.Locate(default));
+		}
+
+		[Fact]
+		public void The_real_databases_give_a_country_a_city_and_a_network()
+		{
+			var directory = Environment.GetEnvironmentVariable("PRIVAPUB_TEST_GEO");
+			Assert.SkipWhen(string.IsNullOrEmpty(directory), "set PRIVAPUB_TEST_GEO to a directory holding the DB-IP Lite city and ASN files");
+			using var locator = Locator(directory);
+
+			var fix = locator.Locate(IPAddress.Parse("1.1.1.1"));
+
+			Assert.NotNull(fix);
+			Assert.Equal(2, fix.Country.Length);
+			Assert.Equal(13335, fix.Asn);
+			Assert.Equal("Cloudflare", CdnNetworks.Of(fix.Asn));
+			Assert.NotNull(fix.AsnOrg);
+			Assert.NotNull(fix.Latitude);
+			Assert.Equal(Math.Round(fix.Latitude.Value, 1), fix.Latitude);
+			Assert.StartsWith("DB-IP Lite ", locator.Source);
+		}
+
+		[Fact]
+		public void A_connection_remembers_its_address_as_ipv4()
+		{
+			var connected = new ConnectedAddresses();
+
+			connected.Remember("Social.Example", IPAddress.Parse("::ffff:203.0.113.7"));
+
+			Assert.Equal(IPAddress.Parse("203.0.113.7"), connected.Of("social.example"));
+			Assert.Null(connected.Of("other.example"));
+			Assert.Equal("Fastly", CdnNetworks.Of(54113));
+			Assert.Null(CdnNetworks.Of(64496));
+			Assert.Null(CdnNetworks.Of(default));
+		}
+	}
+}

PrivaPub.Tests/Statistics/OutboundRequestTests.cs

@@ -126,5 +126,17 @@ namespace PrivaPub.Tests.Statistics
 			Assert.Empty(ledger.Events);
 			Assert.Equal(1, ledger.Counts[("127.0.0.1", "http:webfinger:ok")]);
 		}
+	
+		[Fact]
+		public async Task A_connection_remembers_the_address_it_reached_for_locating_the_server()
+		{
+			var connected = new ConnectedAddresses();
+			var http = Peer.Http(connected: connected);
+			_peer.Serve("/users/c", "{\"id\":\"{A}/users/c\",\"type\":\"Person\"}");
+
+			await http.GetJson($"{_peer.A}/users/c", "application/activity+json", default, TestContext.Current.CancellationToken);
+
+			Assert.Equal(System.Net.IPAddress.Loopback, connected.Of("127.0.0.1"));
+		}
 	}
 }

PrivaPub.Tests/Support/Peer.cs

@@ -80,12 +80,13 @@ namespace PrivaPub.Tests.Support
 
 		public void Answer(string path, int status, TimeSpan delay = default) => _answers[path] = (status, delay);
 
-		public static FederationHttp Http(IMemoryCache cache = default, IDomainBlocks blocks = default, IInteractionLedger ledger = default)
+		public static FederationHttp Http(IMemoryCache cache = default, IDomainBlocks blocks = default, IInteractionLedger ledger = default,
+			IConnectedAddresses connected = default)
 		{
 			var options = new FederationOptions { AllowPrivateNetworks = true, AllowPlainHttp = true };
 			var services = new ServiceCollection();
 			services.AddHttpClient(FederationHttp.ClientName)
-				.ConfigurePrimaryHttpMessageHandler(() => SafeHttpHandlerFactory.Create(options));
+				.ConfigurePrimaryHttpMessageHandler(() => SafeHttpHandlerFactory.Create(options, connected));
 			return new FederationHttp(services.BuildServiceProvider().GetRequiredService<IHttpClientFactory>(),
 				cache ?? new MemoryCache(new MemoryCacheOptions()), new StaticOptions<FederationOptions>(options),
 				blocks ?? new NoBlocks(), NullLogger<FederationHttp>.Instance, ledger);

PrivaPub/Infrastructure/Geo/GeoLocator.cs

@@ -0,0 +1,180 @@
+using MaxMind.Db;
+
+using Microsoft.Extensions.Options;
+
+using PrivaPub.Infrastructure.Http;
+using PrivaPub.Infrastructure.Statistics;
+
+using System.Net;
+
+namespace PrivaPub.Infrastructure.Geo
+{
+	public sealed record GeoFix(string Country, string City, double? Latitude, double? Longitude, int? Asn, string AsnOrg);
+
+	public interface IGeoLocator
+	{
+		GeoFix Locate(IPAddress address);
+		string Source { get; }
+	}
+
+	public sealed class NoGeoLocator : IGeoLocator
+	{
+		public GeoFix Locate(IPAddress address) => default;
+		public string Source => default;
+	}
+
+	public sealed class DbIpLocator : IGeoLocator, IDisposable
+	{
+		public const string CityFile = "dbip-city-lite.mmdb";
+		public const string AsnFile = "dbip-asn-lite.mmdb";
+		static readonly TimeSpan CheckInterval = TimeSpan.FromMinutes(10);
+
+		readonly IOptionsMonitor<StatisticsOptions> _options;
+		readonly ILogger<DbIpLocator> _logger;
+		readonly object _lock = new();
+		Reader _city;
+		Reader _asn;
+		DateTime _cityStamp;
+		DateTime _asnStamp;
+		DateTime _checkedAt = DateTime.MinValue;
+		bool _warned;
+
+		public DbIpLocator(IOptionsMonitor<StatisticsOptions> options, ILogger<DbIpLocator> logger)
+		{
+			_options = options;
+			_logger = logger;
+		}
+
+		public string Source
+		{
+			get
+			{
+				Refresh();
+				var built = _city?.Metadata.BuildDate ?? _asn?.Metadata.BuildDate;
+				return built == default ? default : $"DB-IP Lite {built.Value:yyyy-MM}";
+			}
+		}
+
+		public GeoFix Locate(IPAddress address)
+		{
+			if (address == default || !IpRangeGuard.IsPublic(address))
+				return default;
+			Refresh();
+			Reader city, asn;
+			lock (_lock)
+			{
+				city = _city;
+				asn = _asn;
+			}
+			if (city == default && asn == default)
+				return default;
+			try
+			{
+				var place = city?.Find<Dictionary<string, object>>(address);
+				var network = asn?.Find<Dictionary<string, object>>(address);
+				var location = Section(place, "location");
+				return new GeoFix(
+					Text(Section(place, "country"), "iso_code"),
+					Text(Section(Section(place, "city"), "names"), "en"),
+					Round(location?.GetValueOrDefault("latitude")),
+					Round(location?.GetValueOrDefault("longitude")),
+					Number(network?.GetValueOrDefault("autonomous_system_number")),
+					network?.GetValueOrDefault("autonomous_system_organization") as string);
+			}
+			catch (Exception ex) when (ex is InvalidDatabaseException or ArgumentException)
+			{
+				_logger.LogWarning(ex, "The geolocation databases could not be read");
+				return default;
+			}
+		}
+
+		void Refresh()
+		{
+			if (DateTime.UtcNow - _checkedAt < CheckInterval)
+				return;
+			lock (_lock)
+			{
+				if (DateTime.UtcNow - _checkedAt < CheckInterval)
+					return;
+				_checkedAt = DateTime.UtcNow;
+				var directory = _options.CurrentValue.GeoDirectory;
+				(_city, _cityStamp) = Open(Path.Combine(directory ?? string.Empty, CityFile), _city, _cityStamp);
+				(_asn, _asnStamp) = Open(Path.Combine(directory ?? string.Empty, AsnFile), _asn, _asnStamp);
+				if (_city == default && _asn == default && !_warned)
+				{
+					_warned = true;
+					_logger.LogInformation("No geolocation databases in {Directory}; servers are not located", directory);
+				}
+			}
+		}
+
+		(Reader, DateTime) Open(string path, Reader current, DateTime stamp)
+		{
+			if (!File.Exists(path))
+			{
+				current?.Dispose();
+				return (default, default);
+			}
+			var modified = File.GetLastWriteTimeUtc(path);
+			if (current != default && modified == stamp)
+				return (current, stamp);
+			try
+			{
+				var reader = new Reader(path, FileAccessMode.MemoryMapped);
+				current?.Dispose();
+				return (reader, modified);
+			}
+			catch (Exception ex) when (ex is InvalidDatabaseException or IOException)
+			{
+				_logger.LogWarning(ex, "{Path} is not a readable geolocation database", path);
+				return (current, stamp);
+			}
+		}
+
+		static Dictionary<string, object> Section(Dictionary<string, object> parent, string name) =>
+			parent?.GetValueOrDefault(name) as Dictionary<string, object>;
+
+		static string Text(Dictionary<string, object> parent, string name) => parent?.GetValueOrDefault(name) as string;
+
+		static double? Round(object value) => value switch
+		{
+			double d => Math.Round(d, 1),
+			float f => Math.Round(f, 1),
+			_ => (double?)null
+		};
+
+		static int? Number(object value) => value switch
+		{
+			long l => (int)l,
+			int i => i,
+			uint u => (int)u,
+			ulong ul => (int)ul,
+			_ => (int?)null
+		};
+
+		public void Dispose()
+		{
+			_city?.Dispose();
+			_asn?.Dispose();
+		}
+	}
+
+	public static class CdnNetworks
+	{
+		static readonly Dictionary<int, string> Known = new()
+		{
+			[13335] = "Cloudflare",
+			[209242] = "Cloudflare",
+			[54113] = "Fastly",
+			[20940] = "Akamai",
+			[16625] = "Akamai",
+			[16702] = "Akamai",
+			[21342] = "Akamai",
+			[200325] = "Bunny",
+			[60068] = "CDN77",
+			[15133] = "Edgio"
+		};
+
+		public static string Of(int? asn) => asn is { } number && Known.TryGetValue(number, out var name) ? name : default;
+	}
+}

PrivaPub/Infrastructure/Http/ConnectedAddresses.cs

@@ -0,0 +1,29 @@
+using System.Collections.Concurrent;
+using System.Net;
+
+namespace PrivaPub.Infrastructure.Http
+{
+	public interface IConnectedAddresses
+	{
+		void Remember(string host, IPAddress address);
+		IPAddress Of(string host);
+	}
+
+	public class ConnectedAddresses : IConnectedAddresses
+	{
+		const int MaxHosts = 50_000;
+
+		readonly ConcurrentDictionary<string, IPAddress> _addresses = new(StringComparer.OrdinalIgnoreCase);
+
+		public void Remember(string host, IPAddress address)
+		{
+			if (string.IsNullOrEmpty(host) || address == default)
+				return;
+			if (_addresses.Count >= MaxHosts)
+				_addresses.Clear();
+			_addresses[host] = address.IsIPv4MappedToIPv6 ? address.MapToIPv4() : address;
+		}
+
+		public IPAddress Of(string host) => host != default && _addresses.TryGetValue(host, out var address) ? address : default;
+	}
+}

PrivaPub/Infrastructure/Http/SafeHttpHandlerFactory.cs

@@ -11,7 +11,7 @@ namespace PrivaPub.Infrastructure.Http
 
 	public static class SafeHttpHandlerFactory
 	{
-		public static SocketsHttpHandler Create(FederationOptions options) => new()
+		public static SocketsHttpHandler Create(FederationOptions options, IConnectedAddresses connected = default) => new()
 		{
 			SslOptions = options.AcceptAnyCertificate
 				? new SslClientAuthenticationOptions { RemoteCertificateValidationCallback = (_, _, _, _) => true }
@@ -23,10 +23,11 @@ namespace PrivaPub.Infrastructure.Http
 			ConnectTimeout = TimeSpan.FromSeconds(10),
 			PooledConnectionLifetime = TimeSpan.FromMinutes(2),
 			MaxResponseHeadersLength = 64,
-			ConnectCallback = (context, token) => Connect(context.DnsEndPoint, options.AllowPrivateNetworks, token)
+			ConnectCallback = (context, token) => Connect(context.DnsEndPoint, options.AllowPrivateNetworks, token, connected)
 		};
 
-		public static async ValueTask<Stream> Connect(DnsEndPoint endPoint, bool allowPrivateNetworks, CancellationToken token)
+		public static async ValueTask<Stream> Connect(DnsEndPoint endPoint, bool allowPrivateNetworks, CancellationToken token,
+			IConnectedAddresses connected = default)
 		{
 			var addresses = await Resolve(endPoint.Host, token);
 			if (addresses.Length == 0 || !allowPrivateNetworks && !addresses.All(IpRangeGuard.IsPublic))
@@ -36,6 +37,7 @@ namespace PrivaPub.Infrastructure.Http
 			try
 			{
 				await socket.ConnectAsync(addresses, endPoint.Port, token);
+				connected?.Remember(endPoint.Host, (socket.RemoteEndPoint as IPEndPoint)?.Address);
 				return new NetworkStream(socket, ownsSocket: true);
 			}
 			catch

PrivaPub/Middleware/SocialPubConfigurations.cs

@@ -27,6 +27,7 @@ using PrivaPub.Domain.Statuses;
 using PrivaPub.Domain.Timelines;
 using PrivaPub.Infrastructure.Http;
 using PrivaPub.Infrastructure.Jobs;
+using PrivaPub.Infrastructure.Geo;
 using PrivaPub.Infrastructure.Statistics;
 using Microsoft.Extensions.Options;
 
@@ -59,8 +60,9 @@ namespace PrivaPub.Middleware
 					client.DefaultRequestHeaders.UserAgent.ParseAdd($"PrivaPub/{BuildInfo.Ref} (+{baseAddress}/)");
 				})
 				.ConfigurePrimaryHttpMessageHandler(provider =>
-					SafeHttpHandlerFactory.Create(provider.GetRequiredService<IOptions<FederationOptions>>().Value));
+					SafeHttpHandlerFactory.Create(provider.GetRequiredService<IOptions<FederationOptions>>().Value, provider.GetRequiredService<IConnectedAddresses>()));
 			return service
+				.AddSingleton<IConnectedAddresses, ConnectedAddresses>()
 				.AddSingleton<IFederationHttp, FederationHttp>()
 				.AddSingleton<IDomainBlocks, DomainBlocks>()
 				.AddSingleton<IContentRenderer, ContentRenderer>()
@@ -112,6 +114,7 @@ namespace PrivaPub.Middleware
 				.AddSingleton<IInteractionLedger>(services => services.GetRequiredService<InteractionLedger>())
 				.AddHostedService(services => services.GetRequiredService<InteractionLedger>())
 				.AddSingleton<IJobHandler, RollupJob>()
+				.AddSingleton<IGeoLocator, DbIpLocator>()
 				.AddHostedService<StatisticsSchedule>();
 
 		public static IServiceCollection PrivaPubAuthServicesConfiguration(this IServiceCollection service, IConfiguration configuration)

PrivaPub/PrivaPub.csproj

@@ -11,6 +11,7 @@
     <PackageReference Include="HtmlSanitizer" Version="9.2.1039" />
     <PackageReference Include="MailKit" Version="4.18.0" />
     <PackageReference Include="Markdig" Version="1.4.0" />
+    <PackageReference Include="MaxMind.Db" Version="5.2.0" />
     <PackageReference Include="Microsoft.AspNetCore.Authentication.JwtBearer" Version="10.0.9" />
     <PackageReference Include="MongoDB.Entities" Version="25.1.0" />
     <PackageReference Include="NetVips" Version="3.2.0" />

deploy/max/geo-update.sh

@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+# Fetches the month's DB-IP Lite city and ASN databases (CC BY 4.0, https://db-ip.com) into the directory PrivaPub reads
+# them from (Statistics:GeoDirectory). The app swaps to new files on its own; a failed download leaves the old ones.
+set -euo pipefail
+dir="${GEO_DIR:-/var/lib/privapub/geo}"
+tmp=$(mktemp -d); trap 'rm -rf "$tmp"' EXIT
+this=$(date -u +%Y-%m)
+last=$(date -u -d "$(date -u +%Y-%m-15) -1 month" +%Y-%m)
+for kind in city asn; do
+	got=""
+	for month in "$this" "$last"; do
+		if curl -fsS --max-time 900 -o "$tmp/$kind.gz" "https://download.db-ip.com/free/dbip-$kind-lite-$month.mmdb.gz"; then
+			got=$month; break
+		fi
+	done
+	[ -n "$got" ] || { echo "no $kind database for $this or $last" >&2; exit 1; }
+	gunzip -t "$tmp/$kind.gz"
+	gunzip -c "$tmp/$kind.gz" > "$tmp/dbip-$kind-lite.mmdb"
+	[ "$(stat -c %s "$tmp/dbip-$kind-lite.mmdb")" -gt 1000000 ] || { echo "the $kind database is too small" >&2; exit 1; }
+	install -m 640 "$tmp/dbip-$kind-lite.mmdb" "$dir/.dbip-$kind-lite.mmdb.new"
+	mv -f "$dir/.dbip-$kind-lite.mmdb.new" "$dir/dbip-$kind-lite.mmdb"
+	echo "$kind: DB-IP Lite $got"
+done

deploy/max/setup.sh

@@ -12,7 +12,7 @@ ACME=/root/.acme.sh/acme.sh
 echo "== directories"
 install -d -o "$RUNNER" -g www-data -m 755 /var/www/$HOST
 install -d -o "$RUNNER" -g "$RUNNER" -m 750 /var/backups/$HOST
-install -d -o www-data -g www-data -m 750 /var/lib/privapub /var/lib/privapub/mongo
+install -d -o www-data -g www-data -m 750 /var/lib/privapub /var/lib/privapub/mongo /var/lib/privapub/geo
 
 echo "== sudoers"
 SUDOERS=/etc/sudoers.d/$RUNNER
@@ -23,10 +23,15 @@ visudo -cf "$SUDOERS"
 echo "== units"
 install -m 644 "$SRC/systemd/privapub-mongod.service" /etc/systemd/system/privapub-mongod.service
 install -m 644 "$SRC/systemd/$UNIT.service" /etc/systemd/system/$UNIT.service
+install -m 755 "$SRC/max/geo-update.sh" /usr/local/bin/privapub-geo-update
+install -m 644 "$SRC/systemd/privapub-geo.service" /etc/systemd/system/privapub-geo.service
+install -m 644 "$SRC/systemd/privapub-geo.timer" /etc/systemd/system/privapub-geo.timer
 systemctl daemon-reload
 systemctl enable --now privapub-mongod >/dev/null
 systemctl enable $UNIT >/dev/null
+systemctl enable --now privapub-geo.timer >/dev/null
 systemctl is-active privapub-mongod
+[ -f /var/lib/privapub/geo/dbip-city-lite.mmdb ] || systemctl start privapub-geo.service || echo "geolocation databases not fetched yet; the timer retries"
 
 echo "== nginx snippet and bootstrap vhost"
 install -m 644 "$SRC/nginx/privapub-headers.conf" /etc/nginx/snippets/privapub-headers.conf

deploy/systemd/privapub-geo.service

@@ -0,0 +1,15 @@
+[Unit]
+Description=PrivaPub: fetch the DB-IP Lite geolocation databases
+After=network-online.target
+Wants=network-online.target
+
+[Service]
+Type=oneshot
+User=www-data
+Group=www-data
+ExecStart=/usr/local/bin/privapub-geo-update
+NoNewPrivileges=true
+ProtectSystem=strict
+ProtectHome=true
+PrivateTmp=true
+ReadWritePaths=/var/lib/privapub/geo

deploy/systemd/privapub-geo.timer

@@ -0,0 +1,10 @@
+[Unit]
+Description=PrivaPub: refresh the geolocation databases monthly
+
+[Timer]
+OnCalendar=*-*-03 04:00:00
+RandomizedDelaySec=6h
+Persistent=true
+
+[Install]
+WantedBy=timers.target