diff --git a/.gitea/workflows/build.yml b/.gitea/workflows/build.yml index fbbb448..9298f96 100644 --- a/.gitea/workflows/build.yml +++ b/.gitea/workflows/build.yml @@ -14,3 +14,6 @@ jobs: - name: Build run: dotnet build PrivaPub.sln -c Release + + - name: Test + run: dotnet test PrivaPub.sln -c Release --no-build diff --git a/.gitea/workflows/deploy.yml b/.gitea/workflows/deploy.yml index de22703..1f269b3 100644 --- a/.gitea/workflows/deploy.yml +++ b/.gitea/workflows/deploy.yml @@ -26,6 +26,9 @@ jobs: echo "BUILD_REF=${GITHUB_REF_NAME:-master}" >> "$GITHUB_ENV" echo "BUILD_TIME=$(date -u +%Y-%m-%dT%H:%M:%SZ)" >> "$GITHUB_ENV" + - name: Test + run: dotnet test PrivaPub.sln -c Release + - name: Publish run: | rm -rf "$GITHUB_WORKSPACE/publish" diff --git a/PrivaPub.Tests/Infrastructure/FederationHttpTests.cs b/PrivaPub.Tests/Infrastructure/FederationHttpTests.cs new file mode 100644 index 0000000..e641ace --- /dev/null +++ b/PrivaPub.Tests/Infrastructure/FederationHttpTests.cs @@ -0,0 +1,102 @@ +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Hosting; +using Microsoft.AspNetCore.Http; +using Microsoft.Extensions.Caching.Memory; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; + +using PrivaPub.Infrastructure.Http; + +namespace PrivaPub.Tests.Infrastructure +{ + public sealed class FederationHttpTests : IAsyncLifetime + { + WebApplication _peer; + string _base; + + public async ValueTask InitializeAsync() + { + var builder = WebApplication.CreateSlimBuilder(); + builder.WebHost.UseUrls("http://127.0.0.1:0"); + _peer = builder.Build(); + _peer.MapGet("/actor", () => Results.Text("{\"id\":\"x\"}", "application/activity+json")); + _peer.MapGet("/html", () => Results.Text("", "text/html")); + _peer.MapGet("/big", () => Results.Text("{\"a\":\"" + new string('a', FederationHttp.MaxResponseBytes) + "\"}", "application/activity+json")); + _peer.MapGet("/hop/{n:int}", (int n) => Results.Redirect(n == 0 ? "/actor" : $"/hop/{n - 1}")); + _peer.MapGet("/gone", () => Results.StatusCode(410)); + await _peer.StartAsync(); + _base = _peer.Urls.First(); + } + + public async ValueTask DisposeAsync() => await _peer.DisposeAsync(); + + static FederationHttp Client(bool allowTestNetwork = true) + { + var options = new FederationOptions { AllowPrivateNetworks = allowTestNetwork, AllowPlainHttp = allowTestNetwork }; + var services = new ServiceCollection(); + services.AddHttpClient(FederationHttp.ClientName) + .ConfigurePrimaryHttpMessageHandler(() => SafeHttpHandlerFactory.Create(options)); + var provider = services.BuildServiceProvider(); + return new FederationHttp(provider.GetRequiredService(), new MemoryCache(new MemoryCacheOptions()), + new StaticOptionsMonitor(options), NullLogger.Instance); + } + + [Fact] + public async Task Reads_a_json_document() + { + using var fetched = await Client().GetJson($"{_base}/actor", "application/activity+json", default, TestContext.Current.CancellationToken); + Assert.NotNull(fetched); + Assert.Equal("x", fetched.Root.GetProperty("id").GetString()); + } + + [Fact] + public async Task Follows_up_to_three_redirects_and_reports_the_final_url() + { + using var fetched = await Client().GetJson($"{_base}/hop/2", "application/activity+json", default, TestContext.Current.CancellationToken); + Assert.NotNull(fetched); + Assert.Equal($"{_base}/actor", fetched.FinalUri.ToString()); + } + + [Fact] + public async Task Refuses_a_fourth_redirect() => + Assert.Null(await Client().GetJson($"{_base}/hop/3", "application/activity+json", default, TestContext.Current.CancellationToken)); + + [Fact] + public async Task Refuses_html() => + Assert.Null(await Client().GetJson($"{_base}/html", "application/activity+json", default, TestContext.Current.CancellationToken)); + + [Fact] + public async Task Refuses_a_body_over_the_limit() => + Assert.Null(await Client().GetJson($"{_base}/big", "application/activity+json", default, TestContext.Current.CancellationToken)); + + [Fact] + public async Task Refuses_an_error_status() => + Assert.Null(await Client().GetJson($"{_base}/gone", "application/activity+json", default, TestContext.Current.CancellationToken)); + + [Fact] + public async Task Refuses_a_private_network_in_production_mode() => + Assert.Null(await Client(allowTestNetwork: false).GetJson($"{_base}/actor", "application/activity+json", default, TestContext.Current.CancellationToken)); + + [Theory] + [InlineData("https://mastodon.social/users/Gargron", true)] + [InlineData("http://mastodon.social/users/Gargron", false)] + [InlineData("https://user:pass@mastodon.social/", false)] + [InlineData("https://localhost/", false)] + [InlineData("https://printer.local/", false)] + [InlineData("https://metadata.google.internal/", false)] + [InlineData("https://127.0.0.1/", false)] + [InlineData("https://[::1]/", false)] + [InlineData("ftp://example.org/", false)] + public void IsAllowed_takes_https_dns_names_only(string url, bool allowed) => + Assert.Equal(allowed, Client(allowTestNetwork: false).IsAllowed(new Uri(url))); + + sealed class StaticOptionsMonitor : IOptionsMonitor + { + public StaticOptionsMonitor(FederationOptions value) => CurrentValue = value; + public FederationOptions CurrentValue { get; } + public FederationOptions Get(string name) => CurrentValue; + public IDisposable OnChange(Action listener) => default; + } + } +} diff --git a/PrivaPub.Tests/Infrastructure/IpRangeGuardTests.cs b/PrivaPub.Tests/Infrastructure/IpRangeGuardTests.cs new file mode 100644 index 0000000..f5c7707 --- /dev/null +++ b/PrivaPub.Tests/Infrastructure/IpRangeGuardTests.cs @@ -0,0 +1,86 @@ +using System.Net; + +using PrivaPub.Infrastructure.Http; + +namespace PrivaPub.Tests.Infrastructure +{ + public class IpRangeGuardTests + { + [Theory] + [InlineData("1.1.1.1")] + [InlineData("8.8.8.8")] + [InlineData("93.184.215.14")] + [InlineData("172.15.255.255")] + [InlineData("172.32.0.1")] + [InlineData("100.63.255.255")] + [InlineData("100.128.0.0")] + [InlineData("2a01:4f8::1")] + [InlineData("2606:4700:4700::1111")] + [InlineData("::ffff:1.1.1.1")] + public void Public_addresses_are_allowed(string address) => + Assert.True(IpRangeGuard.IsPublic(IPAddress.Parse(address))); + + [Theory] + [InlineData("0.0.0.0")] + [InlineData("10.1.2.3")] + [InlineData("100.64.0.1")] + [InlineData("100.127.255.255")] + [InlineData("127.0.0.1")] + [InlineData("127.255.255.254")] + [InlineData("169.254.169.254")] + [InlineData("172.16.0.1")] + [InlineData("172.31.255.255")] + [InlineData("192.0.0.170")] + [InlineData("192.0.2.1")] + [InlineData("192.88.99.1")] + [InlineData("192.168.1.1")] + [InlineData("198.18.0.1")] + [InlineData("198.51.100.7")] + [InlineData("203.0.113.9")] + [InlineData("224.0.0.251")] + [InlineData("239.255.255.250")] + [InlineData("240.0.0.1")] + [InlineData("255.255.255.255")] + [InlineData("::")] + [InlineData("::1")] + [InlineData("::127.0.0.1")] + [InlineData("::ffff:127.0.0.1")] + [InlineData("::ffff:10.0.0.1")] + [InlineData("::ffff:169.254.169.254")] + [InlineData("64:ff9b::7f00:1")] + [InlineData("64:ff9b::808:808")] + [InlineData("64:ff9b:1::1")] + [InlineData("100::1")] + [InlineData("2001::1")] + [InlineData("2001:0:4136:e378:8000:63bf:3fff:fdd2")] + [InlineData("2001:db8::1")] + [InlineData("2001:10::1")] + [InlineData("2002:7f00:1::1")] + [InlineData("2002:c0a8:101::1")] + [InlineData("3fff::1")] + [InlineData("fc00::1")] + [InlineData("fd12:3456:789a::1")] + [InlineData("fe80::1")] + [InlineData("fec0::1")] + [InlineData("ff02::1")] + public void Private_and_special_addresses_are_refused(string address) => + Assert.False(IpRangeGuard.IsPublic(IPAddress.Parse(address))); + + [Fact] + public async Task Connect_refuses_a_name_that_resolves_to_loopback() + { + await Assert.ThrowsAsync(async () => + await SafeHttpHandlerFactory.Connect(new DnsEndPoint("localhost", 443), allowPrivateNetworks: false, TestContext.Current.CancellationToken)); + } + + [Theory] + [InlineData("127.0.0.1")] + [InlineData("[::1]")] + [InlineData("169.254.169.254")] + public async Task Connect_refuses_a_private_literal(string host) + { + await Assert.ThrowsAsync(async () => + await SafeHttpHandlerFactory.Connect(new DnsEndPoint(host, 443), allowPrivateNetworks: false, TestContext.Current.CancellationToken)); + } + } +} diff --git a/PrivaPub.Tests/PrivaPub.Tests.csproj b/PrivaPub.Tests/PrivaPub.Tests.csproj new file mode 100644 index 0000000..d4dd227 --- /dev/null +++ b/PrivaPub.Tests/PrivaPub.Tests.csproj @@ -0,0 +1,29 @@ + + + + net10.0 + disable + enable + Exe + false + + + + + + + + + + + + + + + + + + + + + diff --git a/PrivaPub.sln b/PrivaPub.sln index d71a0c8..ee22584 100644 --- a/PrivaPub.sln +++ b/PrivaPub.sln @@ -7,20 +7,54 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "PrivaPub", "PrivaPub\PrivaP EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "PrivaPub.ClientModels", "PrivaPub.ClientModels\PrivaPub.ClientModels.csproj", "{5E63C68C-0E60-4418-A53A-FF4EF0007080}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PrivaPub.Tests", "PrivaPub.Tests\PrivaPub.Tests.csproj", "{BA20AFE4-A348-4BDF-A2F4-36E0710D61A2}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU + Debug|x64 = Debug|x64 + Debug|x86 = Debug|x86 Release|Any CPU = Release|Any CPU + Release|x64 = Release|x64 + Release|x86 = Release|x86 EndGlobalSection GlobalSection(ProjectConfigurationPlatforms) = postSolution {EB2A0BD2-0150-405A-939E-5B0F6F642539}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {EB2A0BD2-0150-405A-939E-5B0F6F642539}.Debug|Any CPU.Build.0 = Debug|Any CPU + {EB2A0BD2-0150-405A-939E-5B0F6F642539}.Debug|x64.ActiveCfg = Debug|Any CPU + {EB2A0BD2-0150-405A-939E-5B0F6F642539}.Debug|x64.Build.0 = Debug|Any CPU + {EB2A0BD2-0150-405A-939E-5B0F6F642539}.Debug|x86.ActiveCfg = Debug|Any CPU + {EB2A0BD2-0150-405A-939E-5B0F6F642539}.Debug|x86.Build.0 = Debug|Any CPU {EB2A0BD2-0150-405A-939E-5B0F6F642539}.Release|Any CPU.ActiveCfg = Release|Any CPU {EB2A0BD2-0150-405A-939E-5B0F6F642539}.Release|Any CPU.Build.0 = Release|Any CPU + {EB2A0BD2-0150-405A-939E-5B0F6F642539}.Release|x64.ActiveCfg = Release|Any CPU + {EB2A0BD2-0150-405A-939E-5B0F6F642539}.Release|x64.Build.0 = Release|Any CPU + {EB2A0BD2-0150-405A-939E-5B0F6F642539}.Release|x86.ActiveCfg = Release|Any CPU + {EB2A0BD2-0150-405A-939E-5B0F6F642539}.Release|x86.Build.0 = Release|Any CPU {5E63C68C-0E60-4418-A53A-FF4EF0007080}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {5E63C68C-0E60-4418-A53A-FF4EF0007080}.Debug|Any CPU.Build.0 = Debug|Any CPU + {5E63C68C-0E60-4418-A53A-FF4EF0007080}.Debug|x64.ActiveCfg = Debug|Any CPU + {5E63C68C-0E60-4418-A53A-FF4EF0007080}.Debug|x64.Build.0 = Debug|Any CPU + {5E63C68C-0E60-4418-A53A-FF4EF0007080}.Debug|x86.ActiveCfg = Debug|Any CPU + {5E63C68C-0E60-4418-A53A-FF4EF0007080}.Debug|x86.Build.0 = Debug|Any CPU {5E63C68C-0E60-4418-A53A-FF4EF0007080}.Release|Any CPU.ActiveCfg = Release|Any CPU {5E63C68C-0E60-4418-A53A-FF4EF0007080}.Release|Any CPU.Build.0 = Release|Any CPU + {5E63C68C-0E60-4418-A53A-FF4EF0007080}.Release|x64.ActiveCfg = Release|Any CPU + {5E63C68C-0E60-4418-A53A-FF4EF0007080}.Release|x64.Build.0 = Release|Any CPU + {5E63C68C-0E60-4418-A53A-FF4EF0007080}.Release|x86.ActiveCfg = Release|Any CPU + {5E63C68C-0E60-4418-A53A-FF4EF0007080}.Release|x86.Build.0 = Release|Any CPU + {BA20AFE4-A348-4BDF-A2F4-36E0710D61A2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {BA20AFE4-A348-4BDF-A2F4-36E0710D61A2}.Debug|Any CPU.Build.0 = Debug|Any CPU + {BA20AFE4-A348-4BDF-A2F4-36E0710D61A2}.Debug|x64.ActiveCfg = Debug|Any CPU + {BA20AFE4-A348-4BDF-A2F4-36E0710D61A2}.Debug|x64.Build.0 = Debug|Any CPU + {BA20AFE4-A348-4BDF-A2F4-36E0710D61A2}.Debug|x86.ActiveCfg = Debug|Any CPU + {BA20AFE4-A348-4BDF-A2F4-36E0710D61A2}.Debug|x86.Build.0 = Debug|Any CPU + {BA20AFE4-A348-4BDF-A2F4-36E0710D61A2}.Release|Any CPU.ActiveCfg = Release|Any CPU + {BA20AFE4-A348-4BDF-A2F4-36E0710D61A2}.Release|Any CPU.Build.0 = Release|Any CPU + {BA20AFE4-A348-4BDF-A2F4-36E0710D61A2}.Release|x64.ActiveCfg = Release|Any CPU + {BA20AFE4-A348-4BDF-A2F4-36E0710D61A2}.Release|x64.Build.0 = Release|Any CPU + {BA20AFE4-A348-4BDF-A2F4-36E0710D61A2}.Release|x86.ActiveCfg = Release|Any CPU + {BA20AFE4-A348-4BDF-A2F4-36E0710D61A2}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/PrivaPub/Federation/Actors/RemoteActorService.cs b/PrivaPub/Federation/Actors/RemoteActorService.cs index 82907b6..f7d67b0 100644 --- a/PrivaPub/Federation/Actors/RemoteActorService.cs +++ b/PrivaPub/Federation/Actors/RemoteActorService.cs @@ -3,9 +3,10 @@ using MongoDB.Entities; using PrivaPub.Models.User; using PrivaPub.StaticServices; -using System.Net.Http.Headers; using System.Text.Json; + using PrivaPub.Federation.Signing; +using PrivaPub.Infrastructure.Http; namespace PrivaPub.Federation.Actors { @@ -19,41 +20,24 @@ namespace PrivaPub.Federation.Actors public class RemoteActorService : IRemoteActorService { - public const string HttpClientName = "ActivityPub"; public const string ActivityJson = "application/activity+json"; const string Accept = "application/activity+json, application/ld+json; profile=\"https://www.w3.org/ns/activitystreams\""; static readonly TimeSpan CacheLifetime = TimeSpan.FromDays(1); - readonly IHttpClientFactory _httpClientFactory; + readonly IFederationHttp _http; readonly DbEntities _dbEntities; - readonly ILogger _logger; - public RemoteActorService(IHttpClientFactory httpClientFactory, DbEntities dbEntities, ILogger logger) + public RemoteActorService(IFederationHttp http, DbEntities dbEntities) { - _httpClientFactory = httpClientFactory; + _http = http; _dbEntities = dbEntities; - _logger = logger; } public async Task Fetch(string uri, LocalActor signAs, CancellationToken token) { - if (!Uri.TryCreate(uri, UriKind.Absolute, out var target) || !IsFetchable(target)) - return default; - - using var request = new HttpRequestMessage(HttpMethod.Get, target); - request.Headers.Accept.ParseAdd(Accept); - if (signAs != default) - HttpSignatures.Sign(request, signAs, body: null); - - using var response = await _httpClientFactory.CreateClient(HttpClientName).SendAsync(request, token); - if (!response.IsSuccessStatusCode) - { - _logger.LogInformation("GET {Uri} answered {Status}", uri, (int)response.StatusCode); - return default; - } - - await using var stream = await response.Content.ReadAsStreamAsync(token); - return await JsonDocument.ParseAsync(stream, cancellationToken: token); + var fetched = await _http.GetJson(uri, Accept, + signAs == default ? default : request => HttpSignatures.Sign(request, signAs, body: null), token); + return fetched?.Document; } public async Task GetActor(string actorUri, LocalActor signAs, bool refresh, CancellationToken token) @@ -108,17 +92,10 @@ namespace PrivaPub.Federation.Actors return default; var url = $"https://{parts[1]}/.well-known/webfinger?resource={Uri.EscapeDataString($"acct:{parts[0]}@{parts[1]}")}"; - if (!Uri.TryCreate(url, UriKind.Absolute, out var target) || !IsFetchable(target)) + using var fetched = await _http.GetJson(url, "application/jrd+json, application/json", sign: default, token); + if (fetched == default) return default; - using var request = new HttpRequestMessage(HttpMethod.Get, target); - request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue("application/jrd+json")); - request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json")); - using var response = await _httpClientFactory.CreateClient(HttpClientName).SendAsync(request, token); - if (!response.IsSuccessStatusCode) - return default; - - await using var stream = await response.Content.ReadAsStreamAsync(token); - using var document = await JsonDocument.ParseAsync(stream, cancellationToken: token); + var document = fetched.Document; if (!document.RootElement.TryGetProperty("links", out var links) || links.ValueKind != JsonValueKind.Array) return default; @@ -163,13 +140,6 @@ namespace PrivaPub.Federation.Actors return avatar; } - public static bool IsFetchable(Uri target) => - target.Scheme == Uri.UriSchemeHttps - && target.HostNameType == UriHostNameType.Dns - && !target.IsLoopback - && !target.Host.Equals("localhost", StringComparison.OrdinalIgnoreCase) - && target.Host.Contains('.'); - public static string StripFragment(string uri) { var hash = uri.IndexOf('#'); diff --git a/PrivaPub/Federation/Outbox/DeliveryService.cs b/PrivaPub/Federation/Outbox/DeliveryService.cs index 4554451..9d13f73 100644 --- a/PrivaPub/Federation/Outbox/DeliveryService.cs +++ b/PrivaPub/Federation/Outbox/DeliveryService.cs @@ -9,6 +9,7 @@ using System.Text; using System.Text.Json.Nodes; using PrivaPub.Federation.Actors; using PrivaPub.Federation.Signing; +using PrivaPub.Infrastructure.Http; namespace PrivaPub.Federation.Outbox { @@ -69,13 +70,13 @@ namespace PrivaPub.Federation.Outbox static readonly TimeSpan Poll = TimeSpan.FromSeconds(3); readonly IServiceProvider _services; - readonly IHttpClientFactory _httpClientFactory; + readonly IFederationHttp _http; readonly ILogger _logger; - public DeliveryWorker(IServiceProvider services, IHttpClientFactory httpClientFactory, ILogger logger) + public DeliveryWorker(IServiceProvider services, IFederationHttp http, ILogger logger) { _services = services; - _httpClientFactory = httpClientFactory; + _http = http; _logger = logger; } @@ -131,7 +132,7 @@ namespace PrivaPub.Federation.Outbox delivery.Attempts++; try { - if (!Uri.TryCreate(delivery.InboxURL, UriKind.Absolute, out var inbox) || !RemoteActorService.IsFetchable(inbox)) + if (!Uri.TryCreate(delivery.InboxURL, UriKind.Absolute, out var inbox) || !_http.IsAllowed(inbox)) { delivery.AbandonedAt = DateTime.UtcNow; delivery.LastError = "not a deliverable inbox"; @@ -146,7 +147,7 @@ namespace PrivaPub.Federation.Outbox request.Content.Headers.ContentType = MediaTypeHeaderValue.Parse(RemoteActorService.ActivityJson); HttpSignatures.Sign(request, signer, body); - using var response = await _httpClientFactory.CreateClient(RemoteActorService.HttpClientName).SendAsync(request, token); + using var response = await _http.Send(request, token); if (response.IsSuccessStatusCode) { delivery.DeliveredAt = DateTime.UtcNow; @@ -162,7 +163,7 @@ namespace PrivaPub.Federation.Outbox return; } } - catch (Exception ex) when (ex is HttpRequestException or TaskCanceledException && !token.IsCancellationRequested) + catch (Exception ex) when (ex is HttpRequestException or BlockedDestinationException or TaskCanceledException && !token.IsCancellationRequested) { delivery.LastError = ex.Message; } diff --git a/PrivaPub/Infrastructure/Http/FederationHttp.cs b/PrivaPub/Infrastructure/Http/FederationHttp.cs new file mode 100644 index 0000000..832a2a1 --- /dev/null +++ b/PrivaPub/Infrastructure/Http/FederationHttp.cs @@ -0,0 +1,164 @@ +using Microsoft.Extensions.Caching.Memory; +using Microsoft.Extensions.Options; + +using System.Net; +using System.Text.Json; + +namespace PrivaPub.Infrastructure.Http +{ + public sealed class FetchedJson : IDisposable + { + public Uri FinalUri { get; init; } + public JsonDocument Document { get; init; } + public JsonElement Root => Document.RootElement; + + public void Dispose() => Document?.Dispose(); + } + + public interface IFederationHttp + { + bool IsAllowed(Uri target); + Task GetJson(string url, string accept, Action sign, CancellationToken token); + Task Send(HttpRequestMessage request, CancellationToken token); + } + + public class FederationHttp : IFederationHttp + { + public const string ClientName = "Federation"; + public const int MaxResponseBytes = 1024 * 1024; + public static readonly TimeSpan RequestTimeout = TimeSpan.FromSeconds(15); + + const int MaxRedirects = 3; + static readonly TimeSpan NegativeCacheLifetime = TimeSpan.FromMinutes(5); + static readonly string[] JsonMediaTypes = + { + "application/activity+json", + "application/ld+json", + "application/jrd+json", + "application/json" + }; + + readonly IHttpClientFactory _httpClientFactory; + readonly IMemoryCache _cache; + readonly IOptionsMonitor _options; + readonly ILogger _logger; + + public FederationHttp(IHttpClientFactory httpClientFactory, IMemoryCache cache, IOptionsMonitor options, + ILogger logger) + { + _httpClientFactory = httpClientFactory; + _cache = cache; + _options = options; + _logger = logger; + } + + public bool IsAllowed(Uri target) + { + if (target is not { IsAbsoluteUri: true } || !string.IsNullOrEmpty(target.UserInfo)) + return false; + var options = _options.CurrentValue; + if (target.Scheme != Uri.UriSchemeHttps && !(options.AllowPlainHttp && target.Scheme == Uri.UriSchemeHttp)) + return false; + if (options.AllowPrivateNetworks) + return true; + return target.HostNameType == UriHostNameType.Dns + && target.Host.Contains('.') + && !target.Host.EndsWith(".localhost", StringComparison.OrdinalIgnoreCase) + && !target.Host.EndsWith(".local", StringComparison.OrdinalIgnoreCase) + && !target.Host.EndsWith(".internal", StringComparison.OrdinalIgnoreCase); + } + + public async Task GetJson(string url, string accept, Action sign, CancellationToken token) + { + if (!Uri.TryCreate(url, UriKind.Absolute, out var target) || !IsAllowed(target)) + return default; + var negativeKey = NegativeKey(target); + if (_cache.TryGetValue(negativeKey, out _)) + return default; + + using var timeout = CancellationTokenSource.CreateLinkedTokenSource(token); + timeout.CancelAfter(RequestTimeout); + try + { + for (var hop = 0; hop <= MaxRedirects; hop++) + { + using var request = new HttpRequestMessage(HttpMethod.Get, target); + request.Headers.Accept.ParseAdd(accept); + sign?.Invoke(request); + + using var response = await _httpClientFactory.CreateClient(ClientName) + .SendAsync(request, HttpCompletionOption.ResponseHeadersRead, timeout.Token); + + if (IsRedirect(response.StatusCode)) + { + var location = response.Headers.Location; + var next = location == default ? default : location.IsAbsoluteUri ? location : new Uri(target, location); + if (!IsAllowed(next)) + return Refuse(negativeKey, url, "a redirect to a disallowed location"); + target = next; + continue; + } + + if (!response.IsSuccessStatusCode) + return Refuse(negativeKey, url, $"status {(int)response.StatusCode}"); + + var mediaType = response.Content.Headers.ContentType?.MediaType; + if (mediaType == default || !JsonMediaTypes.Contains(mediaType, StringComparer.OrdinalIgnoreCase)) + return Refuse(negativeKey, url, $"content type '{mediaType}'"); + if (response.Content.Headers.ContentLength > MaxResponseBytes) + return Refuse(negativeKey, url, "a body over the size limit"); + + var body = await ReadBounded(response.Content, MaxResponseBytes, timeout.Token); + if (body == default) + return Refuse(negativeKey, url, "a body over the size limit"); + + return new FetchedJson { FinalUri = target, Document = JsonDocument.Parse(body) }; + } + return Refuse(negativeKey, url, "too many redirects"); + } + catch (OperationCanceledException) when (!token.IsCancellationRequested) + { + return Refuse(negativeKey, url, "a timeout"); + } + catch (Exception ex) when (ex is HttpRequestException or JsonException or BlockedDestinationException) + { + return Refuse(negativeKey, url, ex.Message); + } + } + + public async Task Send(HttpRequestMessage request, CancellationToken token) + { + if (!IsAllowed(request.RequestUri)) + throw new BlockedDestinationException(request.RequestUri?.Host); + return await _httpClientFactory.CreateClient(ClientName).SendAsync(request, HttpCompletionOption.ResponseHeadersRead, token); + } + + public static async Task ReadBounded(HttpContent content, int limit, CancellationToken token) + { + await using var stream = await content.ReadAsStreamAsync(token); + using var buffer = new MemoryStream(); + var chunk = new byte[16 * 1024]; + int read; + while ((read = await stream.ReadAsync(chunk, token)) > 0) + { + if (buffer.Length + read > limit) + return default; + buffer.Write(chunk, 0, read); + } + return buffer.ToArray(); + } + + static bool IsRedirect(HttpStatusCode status) => + status is HttpStatusCode.MovedPermanently or HttpStatusCode.Found or HttpStatusCode.SeeOther + or HttpStatusCode.TemporaryRedirect or HttpStatusCode.PermanentRedirect; + + static string NegativeKey(Uri target) => "federation-http:refused:" + target.AbsoluteUri; + + FetchedJson Refuse(string negativeKey, string url, string reason) + { + _cache.Set(negativeKey, true, NegativeCacheLifetime); + _logger.LogInformation("GET {Url} refused: {Reason}", url, reason); + return default; + } + } +} diff --git a/PrivaPub/Infrastructure/Http/FederationOptions.cs b/PrivaPub/Infrastructure/Http/FederationOptions.cs new file mode 100644 index 0000000..c68a23c --- /dev/null +++ b/PrivaPub/Infrastructure/Http/FederationOptions.cs @@ -0,0 +1,8 @@ +namespace PrivaPub.Infrastructure.Http +{ + public class FederationOptions + { + public bool AllowPrivateNetworks { get; set; } + public bool AllowPlainHttp { get; set; } + } +} diff --git a/PrivaPub/Infrastructure/Http/IpRangeGuard.cs b/PrivaPub/Infrastructure/Http/IpRangeGuard.cs new file mode 100644 index 0000000..ff5a7b5 --- /dev/null +++ b/PrivaPub/Infrastructure/Http/IpRangeGuard.cs @@ -0,0 +1,55 @@ +using System.Net; +using System.Net.Sockets; + +namespace PrivaPub.Infrastructure.Http +{ + public static class IpRangeGuard + { + static readonly IPNetwork[] BlockedV4 = + { + IPNetwork.Parse("0.0.0.0/8"), + IPNetwork.Parse("10.0.0.0/8"), + IPNetwork.Parse("100.64.0.0/10"), + IPNetwork.Parse("127.0.0.0/8"), + IPNetwork.Parse("169.254.0.0/16"), + IPNetwork.Parse("172.16.0.0/12"), + IPNetwork.Parse("192.0.0.0/24"), + IPNetwork.Parse("192.0.2.0/24"), + IPNetwork.Parse("192.88.99.0/24"), + IPNetwork.Parse("192.168.0.0/16"), + IPNetwork.Parse("198.18.0.0/15"), + IPNetwork.Parse("198.51.100.0/24"), + IPNetwork.Parse("203.0.113.0/24"), + IPNetwork.Parse("224.0.0.0/4"), + IPNetwork.Parse("240.0.0.0/4") + }; + + static readonly IPNetwork GlobalUnicastV6 = IPNetwork.Parse("2000::/3"); + + static readonly IPNetwork[] BlockedV6 = + { + IPNetwork.Parse("2001::/32"),//Teredo + IPNetwork.Parse("2001:2::/48"), + IPNetwork.Parse("2001:10::/28"), + IPNetwork.Parse("2001:20::/28"), + IPNetwork.Parse("2001:db8::/32"), + IPNetwork.Parse("2002::/16"),//6to4 + IPNetwork.Parse("3fff::/20") + }; + + public static bool IsPublic(IPAddress address) + { + if (address == default) + return false; + if (address.IsIPv4MappedToIPv6) + address = address.MapToIPv4(); + + return address.AddressFamily switch + { + AddressFamily.InterNetwork => !BlockedV4.Any(range => range.Contains(address)), + AddressFamily.InterNetworkV6 => GlobalUnicastV6.Contains(address) && !BlockedV6.Any(range => range.Contains(address)), + _ => false + }; + } + } +} diff --git a/PrivaPub/Infrastructure/Http/SafeHttpHandlerFactory.cs b/PrivaPub/Infrastructure/Http/SafeHttpHandlerFactory.cs new file mode 100644 index 0000000..df38b7e --- /dev/null +++ b/PrivaPub/Infrastructure/Http/SafeHttpHandlerFactory.cs @@ -0,0 +1,58 @@ +using System.Net; +using System.Net.Sockets; + +namespace PrivaPub.Infrastructure.Http +{ + public sealed class BlockedDestinationException : Exception + { + public BlockedDestinationException(string host) : base($"'{host}' does not resolve to a public address") { } + } + + public static class SafeHttpHandlerFactory + { + public static SocketsHttpHandler Create(FederationOptions options) => new() + { + AllowAutoRedirect = false, + UseProxy = false, + UseCookies = false, + AutomaticDecompression = DecompressionMethods.All, + ConnectTimeout = TimeSpan.FromSeconds(10), + PooledConnectionLifetime = TimeSpan.FromMinutes(2), + MaxResponseHeadersLength = 64, + ConnectCallback = (context, token) => Connect(context.DnsEndPoint, options.AllowPrivateNetworks, token) + }; + + public static async ValueTask Connect(DnsEndPoint endPoint, bool allowPrivateNetworks, CancellationToken token) + { + var addresses = await Resolve(endPoint.Host, token); + if (addresses.Length == 0 || !allowPrivateNetworks && !addresses.All(IpRangeGuard.IsPublic)) + throw new BlockedDestinationException(endPoint.Host); + + var socket = new Socket(SocketType.Stream, ProtocolType.Tcp) { NoDelay = true }; + try + { + await socket.ConnectAsync(addresses, endPoint.Port, token); + return new NetworkStream(socket, ownsSocket: true); + } + catch + { + socket.Dispose(); + throw; + } + } + + static async Task Resolve(string host, CancellationToken token) + { + if (IPAddress.TryParse(host.Trim('[', ']'), out var literal)) + return new[] { literal }; + try + { + return await Dns.GetHostAddressesAsync(host, token); + } + catch (SocketException) + { + return Array.Empty(); + } + } + } +} diff --git a/PrivaPub/Middleware/SocialPubConfigurations.cs b/PrivaPub/Middleware/SocialPubConfigurations.cs index 4e754b0..acf60d0 100644 --- a/PrivaPub/Middleware/SocialPubConfigurations.cs +++ b/PrivaPub/Middleware/SocialPubConfigurations.cs @@ -14,6 +14,8 @@ using PrivaPub.Services.ClientToServer.Public; using PrivaPub.Federation.Actors; using PrivaPub.Federation.Outbox; using PrivaPub.Federation.Inbox; +using PrivaPub.Infrastructure.Http; +using Microsoft.Extensions.Options; namespace PrivaPub.Middleware { @@ -32,14 +34,19 @@ namespace PrivaPub.Middleware //.AddHostedService() //.AddHostedService(); } - public static IServiceCollection PrivaPubFederationConfiguration(this IServiceCollection service) + public static IServiceCollection PrivaPubFederationConfiguration(this IServiceCollection service, IConfiguration configuration) { - service.AddHttpClient(RemoteActorService.HttpClientName, client => - { - client.Timeout = TimeSpan.FromSeconds(20); - client.DefaultRequestHeaders.UserAgent.ParseAdd($"PrivaPub/{BuildInfo.Ref}"); - }); + service.Configure(configuration.GetSection("Federation")); + service.AddHttpClient(FederationHttp.ClientName, (provider, client) => + { + var baseAddress = provider.GetRequiredService>().CurrentValue.BackendBaseAddress?.TrimEnd('/'); + client.Timeout = FederationHttp.RequestTimeout; + client.DefaultRequestHeaders.UserAgent.ParseAdd($"PrivaPub/{BuildInfo.Ref} (+{baseAddress}/)"); + }) + .ConfigurePrimaryHttpMessageHandler(provider => + SafeHttpHandlerFactory.Create(provider.GetRequiredService>().Value)); return service + .AddSingleton() .AddSingleton() .AddSingleton() .AddSingleton() diff --git a/PrivaPub/Program.cs b/PrivaPub/Program.cs index 0fbe2cd..a8f055e 100644 --- a/PrivaPub/Program.cs +++ b/PrivaPub/Program.cs @@ -11,6 +11,7 @@ using Serilog; using PrivaPub.Data; using PrivaPub.Extensions; +using PrivaPub.Infrastructure.Http; using PrivaPub.Middleware; using PrivaPub.Models; using PrivaPub.Services; @@ -47,7 +48,7 @@ try .PrivaPubOptimizationConfiguration() .PrivaPubDataBaseConfiguration() .PrivaPubServicesConfiguration() - .PrivaPubFederationConfiguration() + .PrivaPubFederationConfiguration(builder.Configuration) .PrivaPubCORSConfiguration() .PrivaPubMiddlewareConfiguration(); } @@ -57,6 +58,10 @@ try throw; } + var federationOptions = builder.Configuration.GetSection("Federation").Get() ?? new(); + if (builder.Environment.IsProduction() && (federationOptions.AllowPrivateNetworks || federationOptions.AllowPlainHttp)) + throw new InvalidOperationException("Federation:AllowPrivateNetworks and Federation:AllowPlainHttp are for test networks and must stay off in Production."); + try { BsonSerializer.RegisterSerializer(new GuidSerializer(GuidRepresentation.Standard));