diff --git a/PrivaPub.Tests/Infrastructure/GeoLocatorTests.cs b/PrivaPub.Tests/Infrastructure/GeoLocatorTests.cs new file mode 100644 index 0000000..ffef383 --- /dev/null +++ b/PrivaPub.Tests/Infrastructure/GeoLocatorTests.cs @@ -0,0 +1,69 @@ +using Microsoft.Extensions.Logging.Abstractions; + +using PrivaPub.Infrastructure.Geo; +using PrivaPub.Infrastructure.Http; +using PrivaPub.Infrastructure.Statistics; +using PrivaPub.Tests.Support; + +using System.Net; + +namespace PrivaPub.Tests.Infrastructure +{ + public class GeoLocatorTests + { + static DbIpLocator Locator(string directory) => + new(new StaticOptions(new StatisticsOptions { GeoDirectory = directory }), NullLogger.Instance); + + [Fact] + public void Without_databases_nothing_is_located() + { + using var locator = Locator(Path.Combine(Path.GetTempPath(), $"no-geo-{Guid.NewGuid():N}")); + + Assert.Null(locator.Locate(IPAddress.Parse("1.1.1.1"))); + Assert.Null(locator.Source); + } + + [Fact] + public void A_private_address_is_never_looked_up() + { + using var locator = Locator(Path.GetTempPath()); + + Assert.Null(locator.Locate(IPAddress.Parse("10.1.2.3"))); + Assert.Null(locator.Locate(IPAddress.Loopback)); + Assert.Null(locator.Locate(default)); + } + + [Fact] + public void The_real_databases_give_a_country_a_city_and_a_network() + { + var directory = Environment.GetEnvironmentVariable("PRIVAPUB_TEST_GEO"); + Assert.SkipWhen(string.IsNullOrEmpty(directory), "set PRIVAPUB_TEST_GEO to a directory holding the DB-IP Lite city and ASN files"); + using var locator = Locator(directory); + + var fix = locator.Locate(IPAddress.Parse("1.1.1.1")); + + Assert.NotNull(fix); + Assert.Equal(2, fix.Country.Length); + Assert.Equal(13335, fix.Asn); + Assert.Equal("Cloudflare", CdnNetworks.Of(fix.Asn)); + Assert.NotNull(fix.AsnOrg); + Assert.NotNull(fix.Latitude); + Assert.Equal(Math.Round(fix.Latitude.Value, 1), fix.Latitude); + Assert.StartsWith("DB-IP Lite ", locator.Source); + } + + [Fact] + public void A_connection_remembers_its_address_as_ipv4() + { + var connected = new ConnectedAddresses(); + + connected.Remember("Social.Example", IPAddress.Parse("::ffff:203.0.113.7")); + + Assert.Equal(IPAddress.Parse("203.0.113.7"), connected.Of("social.example")); + Assert.Null(connected.Of("other.example")); + Assert.Equal("Fastly", CdnNetworks.Of(54113)); + Assert.Null(CdnNetworks.Of(64496)); + Assert.Null(CdnNetworks.Of(default)); + } + } +} diff --git a/PrivaPub.Tests/Statistics/OutboundRequestTests.cs b/PrivaPub.Tests/Statistics/OutboundRequestTests.cs index 2227e23..9b16360 100644 --- a/PrivaPub.Tests/Statistics/OutboundRequestTests.cs +++ b/PrivaPub.Tests/Statistics/OutboundRequestTests.cs @@ -126,5 +126,17 @@ namespace PrivaPub.Tests.Statistics Assert.Empty(ledger.Events); Assert.Equal(1, ledger.Counts[("127.0.0.1", "http:webfinger:ok")]); } + + [Fact] + public async Task A_connection_remembers_the_address_it_reached_for_locating_the_server() + { + var connected = new ConnectedAddresses(); + var http = Peer.Http(connected: connected); + _peer.Serve("/users/c", "{\"id\":\"{A}/users/c\",\"type\":\"Person\"}"); + + await http.GetJson($"{_peer.A}/users/c", "application/activity+json", default, TestContext.Current.CancellationToken); + + Assert.Equal(System.Net.IPAddress.Loopback, connected.Of("127.0.0.1")); + } } } diff --git a/PrivaPub.Tests/Support/Peer.cs b/PrivaPub.Tests/Support/Peer.cs index 2ef1217..0596dea 100644 --- a/PrivaPub.Tests/Support/Peer.cs +++ b/PrivaPub.Tests/Support/Peer.cs @@ -80,12 +80,13 @@ namespace PrivaPub.Tests.Support public void Answer(string path, int status, TimeSpan delay = default) => _answers[path] = (status, delay); - public static FederationHttp Http(IMemoryCache cache = default, IDomainBlocks blocks = default, IInteractionLedger ledger = default) + public static FederationHttp Http(IMemoryCache cache = default, IDomainBlocks blocks = default, IInteractionLedger ledger = default, + IConnectedAddresses connected = default) { var options = new FederationOptions { AllowPrivateNetworks = true, AllowPlainHttp = true }; var services = new ServiceCollection(); services.AddHttpClient(FederationHttp.ClientName) - .ConfigurePrimaryHttpMessageHandler(() => SafeHttpHandlerFactory.Create(options)); + .ConfigurePrimaryHttpMessageHandler(() => SafeHttpHandlerFactory.Create(options, connected)); return new FederationHttp(services.BuildServiceProvider().GetRequiredService(), cache ?? new MemoryCache(new MemoryCacheOptions()), new StaticOptions(options), blocks ?? new NoBlocks(), NullLogger.Instance, ledger); diff --git a/PrivaPub/Infrastructure/Geo/GeoLocator.cs b/PrivaPub/Infrastructure/Geo/GeoLocator.cs new file mode 100644 index 0000000..69469c8 --- /dev/null +++ b/PrivaPub/Infrastructure/Geo/GeoLocator.cs @@ -0,0 +1,180 @@ +using MaxMind.Db; + +using Microsoft.Extensions.Options; + +using PrivaPub.Infrastructure.Http; +using PrivaPub.Infrastructure.Statistics; + +using System.Net; + +namespace PrivaPub.Infrastructure.Geo +{ + public sealed record GeoFix(string Country, string City, double? Latitude, double? Longitude, int? Asn, string AsnOrg); + + public interface IGeoLocator + { + GeoFix Locate(IPAddress address); + string Source { get; } + } + + public sealed class NoGeoLocator : IGeoLocator + { + public GeoFix Locate(IPAddress address) => default; + public string Source => default; + } + + public sealed class DbIpLocator : IGeoLocator, IDisposable + { + public const string CityFile = "dbip-city-lite.mmdb"; + public const string AsnFile = "dbip-asn-lite.mmdb"; + static readonly TimeSpan CheckInterval = TimeSpan.FromMinutes(10); + + readonly IOptionsMonitor _options; + readonly ILogger _logger; + readonly object _lock = new(); + Reader _city; + Reader _asn; + DateTime _cityStamp; + DateTime _asnStamp; + DateTime _checkedAt = DateTime.MinValue; + bool _warned; + + public DbIpLocator(IOptionsMonitor options, ILogger logger) + { + _options = options; + _logger = logger; + } + + public string Source + { + get + { + Refresh(); + var built = _city?.Metadata.BuildDate ?? _asn?.Metadata.BuildDate; + return built == default ? default : $"DB-IP Lite {built.Value:yyyy-MM}"; + } + } + + public GeoFix Locate(IPAddress address) + { + if (address == default || !IpRangeGuard.IsPublic(address)) + return default; + Refresh(); + Reader city, asn; + lock (_lock) + { + city = _city; + asn = _asn; + } + if (city == default && asn == default) + return default; + try + { + var place = city?.Find>(address); + var network = asn?.Find>(address); + var location = Section(place, "location"); + return new GeoFix( + Text(Section(place, "country"), "iso_code"), + Text(Section(Section(place, "city"), "names"), "en"), + Round(location?.GetValueOrDefault("latitude")), + Round(location?.GetValueOrDefault("longitude")), + Number(network?.GetValueOrDefault("autonomous_system_number")), + network?.GetValueOrDefault("autonomous_system_organization") as string); + } + catch (Exception ex) when (ex is InvalidDatabaseException or ArgumentException) + { + _logger.LogWarning(ex, "The geolocation databases could not be read"); + return default; + } + } + + void Refresh() + { + if (DateTime.UtcNow - _checkedAt < CheckInterval) + return; + lock (_lock) + { + if (DateTime.UtcNow - _checkedAt < CheckInterval) + return; + _checkedAt = DateTime.UtcNow; + var directory = _options.CurrentValue.GeoDirectory; + (_city, _cityStamp) = Open(Path.Combine(directory ?? string.Empty, CityFile), _city, _cityStamp); + (_asn, _asnStamp) = Open(Path.Combine(directory ?? string.Empty, AsnFile), _asn, _asnStamp); + if (_city == default && _asn == default && !_warned) + { + _warned = true; + _logger.LogInformation("No geolocation databases in {Directory}; servers are not located", directory); + } + } + } + + (Reader, DateTime) Open(string path, Reader current, DateTime stamp) + { + if (!File.Exists(path)) + { + current?.Dispose(); + return (default, default); + } + var modified = File.GetLastWriteTimeUtc(path); + if (current != default && modified == stamp) + return (current, stamp); + try + { + var reader = new Reader(path, FileAccessMode.MemoryMapped); + current?.Dispose(); + return (reader, modified); + } + catch (Exception ex) when (ex is InvalidDatabaseException or IOException) + { + _logger.LogWarning(ex, "{Path} is not a readable geolocation database", path); + return (current, stamp); + } + } + + static Dictionary Section(Dictionary parent, string name) => + parent?.GetValueOrDefault(name) as Dictionary; + + static string Text(Dictionary parent, string name) => parent?.GetValueOrDefault(name) as string; + + static double? Round(object value) => value switch + { + double d => Math.Round(d, 1), + float f => Math.Round(f, 1), + _ => (double?)null + }; + + static int? Number(object value) => value switch + { + long l => (int)l, + int i => i, + uint u => (int)u, + ulong ul => (int)ul, + _ => (int?)null + }; + + public void Dispose() + { + _city?.Dispose(); + _asn?.Dispose(); + } + } + + public static class CdnNetworks + { + static readonly Dictionary Known = new() + { + [13335] = "Cloudflare", + [209242] = "Cloudflare", + [54113] = "Fastly", + [20940] = "Akamai", + [16625] = "Akamai", + [16702] = "Akamai", + [21342] = "Akamai", + [200325] = "Bunny", + [60068] = "CDN77", + [15133] = "Edgio" + }; + + public static string Of(int? asn) => asn is { } number && Known.TryGetValue(number, out var name) ? name : default; + } +} diff --git a/PrivaPub/Infrastructure/Http/ConnectedAddresses.cs b/PrivaPub/Infrastructure/Http/ConnectedAddresses.cs new file mode 100644 index 0000000..642e058 --- /dev/null +++ b/PrivaPub/Infrastructure/Http/ConnectedAddresses.cs @@ -0,0 +1,29 @@ +using System.Collections.Concurrent; +using System.Net; + +namespace PrivaPub.Infrastructure.Http +{ + public interface IConnectedAddresses + { + void Remember(string host, IPAddress address); + IPAddress Of(string host); + } + + public class ConnectedAddresses : IConnectedAddresses + { + const int MaxHosts = 50_000; + + readonly ConcurrentDictionary _addresses = new(StringComparer.OrdinalIgnoreCase); + + public void Remember(string host, IPAddress address) + { + if (string.IsNullOrEmpty(host) || address == default) + return; + if (_addresses.Count >= MaxHosts) + _addresses.Clear(); + _addresses[host] = address.IsIPv4MappedToIPv6 ? address.MapToIPv4() : address; + } + + public IPAddress Of(string host) => host != default && _addresses.TryGetValue(host, out var address) ? address : default; + } +} diff --git a/PrivaPub/Infrastructure/Http/SafeHttpHandlerFactory.cs b/PrivaPub/Infrastructure/Http/SafeHttpHandlerFactory.cs index 2f20b92..18717aa 100644 --- a/PrivaPub/Infrastructure/Http/SafeHttpHandlerFactory.cs +++ b/PrivaPub/Infrastructure/Http/SafeHttpHandlerFactory.cs @@ -11,7 +11,7 @@ namespace PrivaPub.Infrastructure.Http public static class SafeHttpHandlerFactory { - public static SocketsHttpHandler Create(FederationOptions options) => new() + public static SocketsHttpHandler Create(FederationOptions options, IConnectedAddresses connected = default) => new() { SslOptions = options.AcceptAnyCertificate ? new SslClientAuthenticationOptions { RemoteCertificateValidationCallback = (_, _, _, _) => true } @@ -23,10 +23,11 @@ namespace PrivaPub.Infrastructure.Http ConnectTimeout = TimeSpan.FromSeconds(10), PooledConnectionLifetime = TimeSpan.FromMinutes(2), MaxResponseHeadersLength = 64, - ConnectCallback = (context, token) => Connect(context.DnsEndPoint, options.AllowPrivateNetworks, token) + ConnectCallback = (context, token) => Connect(context.DnsEndPoint, options.AllowPrivateNetworks, token, connected) }; - public static async ValueTask Connect(DnsEndPoint endPoint, bool allowPrivateNetworks, CancellationToken token) + public static async ValueTask Connect(DnsEndPoint endPoint, bool allowPrivateNetworks, CancellationToken token, + IConnectedAddresses connected = default) { var addresses = await Resolve(endPoint.Host, token); if (addresses.Length == 0 || !allowPrivateNetworks && !addresses.All(IpRangeGuard.IsPublic)) @@ -36,6 +37,7 @@ namespace PrivaPub.Infrastructure.Http try { await socket.ConnectAsync(addresses, endPoint.Port, token); + connected?.Remember(endPoint.Host, (socket.RemoteEndPoint as IPEndPoint)?.Address); return new NetworkStream(socket, ownsSocket: true); } catch diff --git a/PrivaPub/Middleware/SocialPubConfigurations.cs b/PrivaPub/Middleware/SocialPubConfigurations.cs index a8f2018..478f216 100644 --- a/PrivaPub/Middleware/SocialPubConfigurations.cs +++ b/PrivaPub/Middleware/SocialPubConfigurations.cs @@ -27,6 +27,7 @@ using PrivaPub.Domain.Statuses; using PrivaPub.Domain.Timelines; using PrivaPub.Infrastructure.Http; using PrivaPub.Infrastructure.Jobs; +using PrivaPub.Infrastructure.Geo; using PrivaPub.Infrastructure.Statistics; using Microsoft.Extensions.Options; @@ -59,8 +60,9 @@ namespace PrivaPub.Middleware client.DefaultRequestHeaders.UserAgent.ParseAdd($"PrivaPub/{BuildInfo.Ref} (+{baseAddress}/)"); }) .ConfigurePrimaryHttpMessageHandler(provider => - SafeHttpHandlerFactory.Create(provider.GetRequiredService>().Value)); + SafeHttpHandlerFactory.Create(provider.GetRequiredService>().Value, provider.GetRequiredService())); return service + .AddSingleton() .AddSingleton() .AddSingleton() .AddSingleton() @@ -112,6 +114,7 @@ namespace PrivaPub.Middleware .AddSingleton(services => services.GetRequiredService()) .AddHostedService(services => services.GetRequiredService()) .AddSingleton() + .AddSingleton() .AddHostedService(); public static IServiceCollection PrivaPubAuthServicesConfiguration(this IServiceCollection service, IConfiguration configuration) diff --git a/PrivaPub/PrivaPub.csproj b/PrivaPub/PrivaPub.csproj index b433327..a32a0e0 100644 --- a/PrivaPub/PrivaPub.csproj +++ b/PrivaPub/PrivaPub.csproj @@ -11,6 +11,7 @@ + diff --git a/deploy/max/geo-update.sh b/deploy/max/geo-update.sh new file mode 100755 index 0000000..c574d9e --- /dev/null +++ b/deploy/max/geo-update.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +# Fetches the month's DB-IP Lite city and ASN databases (CC BY 4.0, https://db-ip.com) into the directory PrivaPub reads +# them from (Statistics:GeoDirectory). The app swaps to new files on its own; a failed download leaves the old ones. +set -euo pipefail +dir="${GEO_DIR:-/var/lib/privapub/geo}" +tmp=$(mktemp -d); trap 'rm -rf "$tmp"' EXIT +this=$(date -u +%Y-%m) +last=$(date -u -d "$(date -u +%Y-%m-15) -1 month" +%Y-%m) +for kind in city asn; do + got="" + for month in "$this" "$last"; do + if curl -fsS --max-time 900 -o "$tmp/$kind.gz" "https://download.db-ip.com/free/dbip-$kind-lite-$month.mmdb.gz"; then + got=$month; break + fi + done + [ -n "$got" ] || { echo "no $kind database for $this or $last" >&2; exit 1; } + gunzip -t "$tmp/$kind.gz" + gunzip -c "$tmp/$kind.gz" > "$tmp/dbip-$kind-lite.mmdb" + [ "$(stat -c %s "$tmp/dbip-$kind-lite.mmdb")" -gt 1000000 ] || { echo "the $kind database is too small" >&2; exit 1; } + install -m 640 "$tmp/dbip-$kind-lite.mmdb" "$dir/.dbip-$kind-lite.mmdb.new" + mv -f "$dir/.dbip-$kind-lite.mmdb.new" "$dir/dbip-$kind-lite.mmdb" + echo "$kind: DB-IP Lite $got" +done diff --git a/deploy/max/setup.sh b/deploy/max/setup.sh index a028224..832a516 100755 --- a/deploy/max/setup.sh +++ b/deploy/max/setup.sh @@ -12,7 +12,7 @@ ACME=/root/.acme.sh/acme.sh echo "== directories" install -d -o "$RUNNER" -g www-data -m 755 /var/www/$HOST install -d -o "$RUNNER" -g "$RUNNER" -m 750 /var/backups/$HOST -install -d -o www-data -g www-data -m 750 /var/lib/privapub /var/lib/privapub/mongo +install -d -o www-data -g www-data -m 750 /var/lib/privapub /var/lib/privapub/mongo /var/lib/privapub/geo echo "== sudoers" SUDOERS=/etc/sudoers.d/$RUNNER @@ -23,10 +23,15 @@ visudo -cf "$SUDOERS" echo "== units" install -m 644 "$SRC/systemd/privapub-mongod.service" /etc/systemd/system/privapub-mongod.service install -m 644 "$SRC/systemd/$UNIT.service" /etc/systemd/system/$UNIT.service +install -m 755 "$SRC/max/geo-update.sh" /usr/local/bin/privapub-geo-update +install -m 644 "$SRC/systemd/privapub-geo.service" /etc/systemd/system/privapub-geo.service +install -m 644 "$SRC/systemd/privapub-geo.timer" /etc/systemd/system/privapub-geo.timer systemctl daemon-reload systemctl enable --now privapub-mongod >/dev/null systemctl enable $UNIT >/dev/null +systemctl enable --now privapub-geo.timer >/dev/null systemctl is-active privapub-mongod +[ -f /var/lib/privapub/geo/dbip-city-lite.mmdb ] || systemctl start privapub-geo.service || echo "geolocation databases not fetched yet; the timer retries" echo "== nginx snippet and bootstrap vhost" install -m 644 "$SRC/nginx/privapub-headers.conf" /etc/nginx/snippets/privapub-headers.conf diff --git a/deploy/systemd/privapub-geo.service b/deploy/systemd/privapub-geo.service new file mode 100644 index 0000000..ac7dd72 --- /dev/null +++ b/deploy/systemd/privapub-geo.service @@ -0,0 +1,15 @@ +[Unit] +Description=PrivaPub: fetch the DB-IP Lite geolocation databases +After=network-online.target +Wants=network-online.target + +[Service] +Type=oneshot +User=www-data +Group=www-data +ExecStart=/usr/local/bin/privapub-geo-update +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true +ReadWritePaths=/var/lib/privapub/geo diff --git a/deploy/systemd/privapub-geo.timer b/deploy/systemd/privapub-geo.timer new file mode 100644 index 0000000..997a57e --- /dev/null +++ b/deploy/systemd/privapub-geo.timer @@ -0,0 +1,10 @@ +[Unit] +Description=PrivaPub: refresh the geolocation databases monthly + +[Timer] +OnCalendar=*-*-03 04:00:00 +RandomizedDelaySec=6h +Persistent=true + +[Install] +WantedBy=timers.target