From cee85309b80c22dd3940592a260b133b3bdcdee2 Mon Sep 17 00:00:00 2001 From: thepra Date: Sat, 3 Oct 2026 11:33:30 +0200 Subject: [PATCH] M9 (first part): the means to locate a server - IConnectedAddresses remembers which address each host's last connection reached, set in SafeHttpHandlerFactory's connect callback. That is once per pooled connection, with no second DNS lookup, and the address is never stored. - IGeoLocator / DbIpLocator reads the offline DB-IP Lite city and ASN databases (MaxMind .mmdb, via MaxMind.Db). It maps memory, swaps to new files within ten minutes, rounds coordinates to one decimal, never looks up a private address, and answers nothing when the files are missing. CdnNetworks names the CDNs whose edge addresses say nothing about where a server is. - deploy/max/geo-update.sh fetches this or last month's databases, checks them and swaps them in atomically. The privapub-geo timer runs it monthly as www-data, and setup.sh installs the directory, the script, the units and a first download. Describing servers will use these in M8. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01ELjqpznMFMNrJoJUj6K5p2 --- .../Infrastructure/GeoLocatorTests.cs | 69 +++++++ .../Statistics/OutboundRequestTests.cs | 12 ++ PrivaPub.Tests/Support/Peer.cs | 5 +- PrivaPub/Infrastructure/Geo/GeoLocator.cs | 180 ++++++++++++++++++ .../Infrastructure/Http/ConnectedAddresses.cs | 29 +++ .../Http/SafeHttpHandlerFactory.cs | 8 +- .../Middleware/SocialPubConfigurations.cs | 5 +- PrivaPub/PrivaPub.csproj | 1 + deploy/max/geo-update.sh | 23 +++ deploy/max/setup.sh | 7 +- deploy/systemd/privapub-geo.service | 15 ++ deploy/systemd/privapub-geo.timer | 10 + 12 files changed, 357 insertions(+), 7 deletions(-) create mode 100644 PrivaPub.Tests/Infrastructure/GeoLocatorTests.cs create mode 100644 PrivaPub/Infrastructure/Geo/GeoLocator.cs create mode 100644 PrivaPub/Infrastructure/Http/ConnectedAddresses.cs create mode 100755 deploy/max/geo-update.sh create mode 100644 deploy/systemd/privapub-geo.service create mode 100644 deploy/systemd/privapub-geo.timer diff --git a/PrivaPub.Tests/Infrastructure/GeoLocatorTests.cs b/PrivaPub.Tests/Infrastructure/GeoLocatorTests.cs new file mode 100644 index 0000000..ffef383 --- /dev/null +++ b/PrivaPub.Tests/Infrastructure/GeoLocatorTests.cs @@ -0,0 +1,69 @@ +using Microsoft.Extensions.Logging.Abstractions; + +using PrivaPub.Infrastructure.Geo; +using PrivaPub.Infrastructure.Http; +using PrivaPub.Infrastructure.Statistics; +using PrivaPub.Tests.Support; + +using System.Net; + +namespace PrivaPub.Tests.Infrastructure +{ + public class GeoLocatorTests + { + static DbIpLocator Locator(string directory) => + new(new StaticOptions(new StatisticsOptions { GeoDirectory = directory }), NullLogger.Instance); + + [Fact] + public void Without_databases_nothing_is_located() + { + using var locator = Locator(Path.Combine(Path.GetTempPath(), $"no-geo-{Guid.NewGuid():N}")); + + Assert.Null(locator.Locate(IPAddress.Parse("1.1.1.1"))); + Assert.Null(locator.Source); + } + + [Fact] + public void A_private_address_is_never_looked_up() + { + using var locator = Locator(Path.GetTempPath()); + + Assert.Null(locator.Locate(IPAddress.Parse("10.1.2.3"))); + Assert.Null(locator.Locate(IPAddress.Loopback)); + Assert.Null(locator.Locate(default)); + } + + [Fact] + public void The_real_databases_give_a_country_a_city_and_a_network() + { + var directory = Environment.GetEnvironmentVariable("PRIVAPUB_TEST_GEO"); + Assert.SkipWhen(string.IsNullOrEmpty(directory), "set PRIVAPUB_TEST_GEO to a directory holding the DB-IP Lite city and ASN files"); + using var locator = Locator(directory); + + var fix = locator.Locate(IPAddress.Parse("1.1.1.1")); + + Assert.NotNull(fix); + Assert.Equal(2, fix.Country.Length); + Assert.Equal(13335, fix.Asn); + Assert.Equal("Cloudflare", CdnNetworks.Of(fix.Asn)); + Assert.NotNull(fix.AsnOrg); + Assert.NotNull(fix.Latitude); + Assert.Equal(Math.Round(fix.Latitude.Value, 1), fix.Latitude); + Assert.StartsWith("DB-IP Lite ", locator.Source); + } + + [Fact] + public void A_connection_remembers_its_address_as_ipv4() + { + var connected = new ConnectedAddresses(); + + connected.Remember("Social.Example", IPAddress.Parse("::ffff:203.0.113.7")); + + Assert.Equal(IPAddress.Parse("203.0.113.7"), connected.Of("social.example")); + Assert.Null(connected.Of("other.example")); + Assert.Equal("Fastly", CdnNetworks.Of(54113)); + Assert.Null(CdnNetworks.Of(64496)); + Assert.Null(CdnNetworks.Of(default)); + } + } +} diff --git a/PrivaPub.Tests/Statistics/OutboundRequestTests.cs b/PrivaPub.Tests/Statistics/OutboundRequestTests.cs index 2227e23..9b16360 100644 --- a/PrivaPub.Tests/Statistics/OutboundRequestTests.cs +++ b/PrivaPub.Tests/Statistics/OutboundRequestTests.cs @@ -126,5 +126,17 @@ namespace PrivaPub.Tests.Statistics Assert.Empty(ledger.Events); Assert.Equal(1, ledger.Counts[("127.0.0.1", "http:webfinger:ok")]); } + + [Fact] + public async Task A_connection_remembers_the_address_it_reached_for_locating_the_server() + { + var connected = new ConnectedAddresses(); + var http = Peer.Http(connected: connected); + _peer.Serve("/users/c", "{\"id\":\"{A}/users/c\",\"type\":\"Person\"}"); + + await http.GetJson($"{_peer.A}/users/c", "application/activity+json", default, TestContext.Current.CancellationToken); + + Assert.Equal(System.Net.IPAddress.Loopback, connected.Of("127.0.0.1")); + } } } diff --git a/PrivaPub.Tests/Support/Peer.cs b/PrivaPub.Tests/Support/Peer.cs index 2ef1217..0596dea 100644 --- a/PrivaPub.Tests/Support/Peer.cs +++ b/PrivaPub.Tests/Support/Peer.cs @@ -80,12 +80,13 @@ namespace PrivaPub.Tests.Support public void Answer(string path, int status, TimeSpan delay = default) => _answers[path] = (status, delay); - public static FederationHttp Http(IMemoryCache cache = default, IDomainBlocks blocks = default, IInteractionLedger ledger = default) + public static FederationHttp Http(IMemoryCache cache = default, IDomainBlocks blocks = default, IInteractionLedger ledger = default, + IConnectedAddresses connected = default) { var options = new FederationOptions { AllowPrivateNetworks = true, AllowPlainHttp = true }; var services = new ServiceCollection(); services.AddHttpClient(FederationHttp.ClientName) - .ConfigurePrimaryHttpMessageHandler(() => SafeHttpHandlerFactory.Create(options)); + .ConfigurePrimaryHttpMessageHandler(() => SafeHttpHandlerFactory.Create(options, connected)); return new FederationHttp(services.BuildServiceProvider().GetRequiredService(), cache ?? new MemoryCache(new MemoryCacheOptions()), new StaticOptions(options), blocks ?? new NoBlocks(), NullLogger.Instance, ledger); diff --git a/PrivaPub/Infrastructure/Geo/GeoLocator.cs b/PrivaPub/Infrastructure/Geo/GeoLocator.cs new file mode 100644 index 0000000..69469c8 --- /dev/null +++ b/PrivaPub/Infrastructure/Geo/GeoLocator.cs @@ -0,0 +1,180 @@ +using MaxMind.Db; + +using Microsoft.Extensions.Options; + +using PrivaPub.Infrastructure.Http; +using PrivaPub.Infrastructure.Statistics; + +using System.Net; + +namespace PrivaPub.Infrastructure.Geo +{ + public sealed record GeoFix(string Country, string City, double? Latitude, double? Longitude, int? Asn, string AsnOrg); + + public interface IGeoLocator + { + GeoFix Locate(IPAddress address); + string Source { get; } + } + + public sealed class NoGeoLocator : IGeoLocator + { + public GeoFix Locate(IPAddress address) => default; + public string Source => default; + } + + public sealed class DbIpLocator : IGeoLocator, IDisposable + { + public const string CityFile = "dbip-city-lite.mmdb"; + public const string AsnFile = "dbip-asn-lite.mmdb"; + static readonly TimeSpan CheckInterval = TimeSpan.FromMinutes(10); + + readonly IOptionsMonitor _options; + readonly ILogger _logger; + readonly object _lock = new(); + Reader _city; + Reader _asn; + DateTime _cityStamp; + DateTime _asnStamp; + DateTime _checkedAt = DateTime.MinValue; + bool _warned; + + public DbIpLocator(IOptionsMonitor options, ILogger logger) + { + _options = options; + _logger = logger; + } + + public string Source + { + get + { + Refresh(); + var built = _city?.Metadata.BuildDate ?? _asn?.Metadata.BuildDate; + return built == default ? default : $"DB-IP Lite {built.Value:yyyy-MM}"; + } + } + + public GeoFix Locate(IPAddress address) + { + if (address == default || !IpRangeGuard.IsPublic(address)) + return default; + Refresh(); + Reader city, asn; + lock (_lock) + { + city = _city; + asn = _asn; + } + if (city == default && asn == default) + return default; + try + { + var place = city?.Find>(address); + var network = asn?.Find>(address); + var location = Section(place, "location"); + return new GeoFix( + Text(Section(place, "country"), "iso_code"), + Text(Section(Section(place, "city"), "names"), "en"), + Round(location?.GetValueOrDefault("latitude")), + Round(location?.GetValueOrDefault("longitude")), + Number(network?.GetValueOrDefault("autonomous_system_number")), + network?.GetValueOrDefault("autonomous_system_organization") as string); + } + catch (Exception ex) when (ex is InvalidDatabaseException or ArgumentException) + { + _logger.LogWarning(ex, "The geolocation databases could not be read"); + return default; + } + } + + void Refresh() + { + if (DateTime.UtcNow - _checkedAt < CheckInterval) + return; + lock (_lock) + { + if (DateTime.UtcNow - _checkedAt < CheckInterval) + return; + _checkedAt = DateTime.UtcNow; + var directory = _options.CurrentValue.GeoDirectory; + (_city, _cityStamp) = Open(Path.Combine(directory ?? string.Empty, CityFile), _city, _cityStamp); + (_asn, _asnStamp) = Open(Path.Combine(directory ?? string.Empty, AsnFile), _asn, _asnStamp); + if (_city == default && _asn == default && !_warned) + { + _warned = true; + _logger.LogInformation("No geolocation databases in {Directory}; servers are not located", directory); + } + } + } + + (Reader, DateTime) Open(string path, Reader current, DateTime stamp) + { + if (!File.Exists(path)) + { + current?.Dispose(); + return (default, default); + } + var modified = File.GetLastWriteTimeUtc(path); + if (current != default && modified == stamp) + return (current, stamp); + try + { + var reader = new Reader(path, FileAccessMode.MemoryMapped); + current?.Dispose(); + return (reader, modified); + } + catch (Exception ex) when (ex is InvalidDatabaseException or IOException) + { + _logger.LogWarning(ex, "{Path} is not a readable geolocation database", path); + return (current, stamp); + } + } + + static Dictionary Section(Dictionary parent, string name) => + parent?.GetValueOrDefault(name) as Dictionary; + + static string Text(Dictionary parent, string name) => parent?.GetValueOrDefault(name) as string; + + static double? Round(object value) => value switch + { + double d => Math.Round(d, 1), + float f => Math.Round(f, 1), + _ => (double?)null + }; + + static int? Number(object value) => value switch + { + long l => (int)l, + int i => i, + uint u => (int)u, + ulong ul => (int)ul, + _ => (int?)null + }; + + public void Dispose() + { + _city?.Dispose(); + _asn?.Dispose(); + } + } + + public static class CdnNetworks + { + static readonly Dictionary Known = new() + { + [13335] = "Cloudflare", + [209242] = "Cloudflare", + [54113] = "Fastly", + [20940] = "Akamai", + [16625] = "Akamai", + [16702] = "Akamai", + [21342] = "Akamai", + [200325] = "Bunny", + [60068] = "CDN77", + [15133] = "Edgio" + }; + + public static string Of(int? asn) => asn is { } number && Known.TryGetValue(number, out var name) ? name : default; + } +} diff --git a/PrivaPub/Infrastructure/Http/ConnectedAddresses.cs b/PrivaPub/Infrastructure/Http/ConnectedAddresses.cs new file mode 100644 index 0000000..642e058 --- /dev/null +++ b/PrivaPub/Infrastructure/Http/ConnectedAddresses.cs @@ -0,0 +1,29 @@ +using System.Collections.Concurrent; +using System.Net; + +namespace PrivaPub.Infrastructure.Http +{ + public interface IConnectedAddresses + { + void Remember(string host, IPAddress address); + IPAddress Of(string host); + } + + public class ConnectedAddresses : IConnectedAddresses + { + const int MaxHosts = 50_000; + + readonly ConcurrentDictionary _addresses = new(StringComparer.OrdinalIgnoreCase); + + public void Remember(string host, IPAddress address) + { + if (string.IsNullOrEmpty(host) || address == default) + return; + if (_addresses.Count >= MaxHosts) + _addresses.Clear(); + _addresses[host] = address.IsIPv4MappedToIPv6 ? address.MapToIPv4() : address; + } + + public IPAddress Of(string host) => host != default && _addresses.TryGetValue(host, out var address) ? address : default; + } +} diff --git a/PrivaPub/Infrastructure/Http/SafeHttpHandlerFactory.cs b/PrivaPub/Infrastructure/Http/SafeHttpHandlerFactory.cs index 2f20b92..18717aa 100644 --- a/PrivaPub/Infrastructure/Http/SafeHttpHandlerFactory.cs +++ b/PrivaPub/Infrastructure/Http/SafeHttpHandlerFactory.cs @@ -11,7 +11,7 @@ namespace PrivaPub.Infrastructure.Http public static class SafeHttpHandlerFactory { - public static SocketsHttpHandler Create(FederationOptions options) => new() + public static SocketsHttpHandler Create(FederationOptions options, IConnectedAddresses connected = default) => new() { SslOptions = options.AcceptAnyCertificate ? new SslClientAuthenticationOptions { RemoteCertificateValidationCallback = (_, _, _, _) => true } @@ -23,10 +23,11 @@ namespace PrivaPub.Infrastructure.Http ConnectTimeout = TimeSpan.FromSeconds(10), PooledConnectionLifetime = TimeSpan.FromMinutes(2), MaxResponseHeadersLength = 64, - ConnectCallback = (context, token) => Connect(context.DnsEndPoint, options.AllowPrivateNetworks, token) + ConnectCallback = (context, token) => Connect(context.DnsEndPoint, options.AllowPrivateNetworks, token, connected) }; - public static async ValueTask Connect(DnsEndPoint endPoint, bool allowPrivateNetworks, CancellationToken token) + public static async ValueTask Connect(DnsEndPoint endPoint, bool allowPrivateNetworks, CancellationToken token, + IConnectedAddresses connected = default) { var addresses = await Resolve(endPoint.Host, token); if (addresses.Length == 0 || !allowPrivateNetworks && !addresses.All(IpRangeGuard.IsPublic)) @@ -36,6 +37,7 @@ namespace PrivaPub.Infrastructure.Http try { await socket.ConnectAsync(addresses, endPoint.Port, token); + connected?.Remember(endPoint.Host, (socket.RemoteEndPoint as IPEndPoint)?.Address); return new NetworkStream(socket, ownsSocket: true); } catch diff --git a/PrivaPub/Middleware/SocialPubConfigurations.cs b/PrivaPub/Middleware/SocialPubConfigurations.cs index a8f2018..478f216 100644 --- a/PrivaPub/Middleware/SocialPubConfigurations.cs +++ b/PrivaPub/Middleware/SocialPubConfigurations.cs @@ -27,6 +27,7 @@ using PrivaPub.Domain.Statuses; using PrivaPub.Domain.Timelines; using PrivaPub.Infrastructure.Http; using PrivaPub.Infrastructure.Jobs; +using PrivaPub.Infrastructure.Geo; using PrivaPub.Infrastructure.Statistics; using Microsoft.Extensions.Options; @@ -59,8 +60,9 @@ namespace PrivaPub.Middleware client.DefaultRequestHeaders.UserAgent.ParseAdd($"PrivaPub/{BuildInfo.Ref} (+{baseAddress}/)"); }) .ConfigurePrimaryHttpMessageHandler(provider => - SafeHttpHandlerFactory.Create(provider.GetRequiredService>().Value)); + SafeHttpHandlerFactory.Create(provider.GetRequiredService>().Value, provider.GetRequiredService())); return service + .AddSingleton() .AddSingleton() .AddSingleton() .AddSingleton() @@ -112,6 +114,7 @@ namespace PrivaPub.Middleware .AddSingleton(services => services.GetRequiredService()) .AddHostedService(services => services.GetRequiredService()) .AddSingleton() + .AddSingleton() .AddHostedService(); public static IServiceCollection PrivaPubAuthServicesConfiguration(this IServiceCollection service, IConfiguration configuration) diff --git a/PrivaPub/PrivaPub.csproj b/PrivaPub/PrivaPub.csproj index b433327..a32a0e0 100644 --- a/PrivaPub/PrivaPub.csproj +++ b/PrivaPub/PrivaPub.csproj @@ -11,6 +11,7 @@ + diff --git a/deploy/max/geo-update.sh b/deploy/max/geo-update.sh new file mode 100755 index 0000000..c574d9e --- /dev/null +++ b/deploy/max/geo-update.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +# Fetches the month's DB-IP Lite city and ASN databases (CC BY 4.0, https://db-ip.com) into the directory PrivaPub reads +# them from (Statistics:GeoDirectory). The app swaps to new files on its own; a failed download leaves the old ones. +set -euo pipefail +dir="${GEO_DIR:-/var/lib/privapub/geo}" +tmp=$(mktemp -d); trap 'rm -rf "$tmp"' EXIT +this=$(date -u +%Y-%m) +last=$(date -u -d "$(date -u +%Y-%m-15) -1 month" +%Y-%m) +for kind in city asn; do + got="" + for month in "$this" "$last"; do + if curl -fsS --max-time 900 -o "$tmp/$kind.gz" "https://download.db-ip.com/free/dbip-$kind-lite-$month.mmdb.gz"; then + got=$month; break + fi + done + [ -n "$got" ] || { echo "no $kind database for $this or $last" >&2; exit 1; } + gunzip -t "$tmp/$kind.gz" + gunzip -c "$tmp/$kind.gz" > "$tmp/dbip-$kind-lite.mmdb" + [ "$(stat -c %s "$tmp/dbip-$kind-lite.mmdb")" -gt 1000000 ] || { echo "the $kind database is too small" >&2; exit 1; } + install -m 640 "$tmp/dbip-$kind-lite.mmdb" "$dir/.dbip-$kind-lite.mmdb.new" + mv -f "$dir/.dbip-$kind-lite.mmdb.new" "$dir/dbip-$kind-lite.mmdb" + echo "$kind: DB-IP Lite $got" +done diff --git a/deploy/max/setup.sh b/deploy/max/setup.sh index a028224..832a516 100755 --- a/deploy/max/setup.sh +++ b/deploy/max/setup.sh @@ -12,7 +12,7 @@ ACME=/root/.acme.sh/acme.sh echo "== directories" install -d -o "$RUNNER" -g www-data -m 755 /var/www/$HOST install -d -o "$RUNNER" -g "$RUNNER" -m 750 /var/backups/$HOST -install -d -o www-data -g www-data -m 750 /var/lib/privapub /var/lib/privapub/mongo +install -d -o www-data -g www-data -m 750 /var/lib/privapub /var/lib/privapub/mongo /var/lib/privapub/geo echo "== sudoers" SUDOERS=/etc/sudoers.d/$RUNNER @@ -23,10 +23,15 @@ visudo -cf "$SUDOERS" echo "== units" install -m 644 "$SRC/systemd/privapub-mongod.service" /etc/systemd/system/privapub-mongod.service install -m 644 "$SRC/systemd/$UNIT.service" /etc/systemd/system/$UNIT.service +install -m 755 "$SRC/max/geo-update.sh" /usr/local/bin/privapub-geo-update +install -m 644 "$SRC/systemd/privapub-geo.service" /etc/systemd/system/privapub-geo.service +install -m 644 "$SRC/systemd/privapub-geo.timer" /etc/systemd/system/privapub-geo.timer systemctl daemon-reload systemctl enable --now privapub-mongod >/dev/null systemctl enable $UNIT >/dev/null +systemctl enable --now privapub-geo.timer >/dev/null systemctl is-active privapub-mongod +[ -f /var/lib/privapub/geo/dbip-city-lite.mmdb ] || systemctl start privapub-geo.service || echo "geolocation databases not fetched yet; the timer retries" echo "== nginx snippet and bootstrap vhost" install -m 644 "$SRC/nginx/privapub-headers.conf" /etc/nginx/snippets/privapub-headers.conf diff --git a/deploy/systemd/privapub-geo.service b/deploy/systemd/privapub-geo.service new file mode 100644 index 0000000..ac7dd72 --- /dev/null +++ b/deploy/systemd/privapub-geo.service @@ -0,0 +1,15 @@ +[Unit] +Description=PrivaPub: fetch the DB-IP Lite geolocation databases +After=network-online.target +Wants=network-online.target + +[Service] +Type=oneshot +User=www-data +Group=www-data +ExecStart=/usr/local/bin/privapub-geo-update +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true +ReadWritePaths=/var/lib/privapub/geo diff --git a/deploy/systemd/privapub-geo.timer b/deploy/systemd/privapub-geo.timer new file mode 100644 index 0000000..997a57e --- /dev/null +++ b/deploy/systemd/privapub-geo.timer @@ -0,0 +1,10 @@ +[Unit] +Description=PrivaPub: refresh the geolocation databases monthly + +[Timer] +OnCalendar=*-*-03 04:00:00 +RandomizedDelaySec=6h +Persistent=true + +[Install] +WantedBy=timers.target