Every outbound federation request goes through one guarded client
Infrastructure/Http adds the client the roadmap's S3 and S4 ask for: - the connect callback resolves the name itself and refuses loopback, private, link-local, CGNAT, documentation, multicast, ULA, NAT64, 6to4, Teredo and IPv4-mapped/compatible forms, then connects to the vetted address, so DNS rebinding cannot swap it afterwards; - redirects are followed by hand, at most three, each one re-checked; - bodies are capped at 1 MB after decompression, only JSON media types are read, every request has a 15 s budget, and a refused URL is not asked again for five minutes. Actor and WebFinger fetches and inbox deliveries all use it. Test networks can switch on Federation:AllowPrivateNetworks/AllowPlainHttp; startup refuses both in Production. PrivaPub.Tests (xUnit v3) starts with the address table and the fetcher's limits against an in-process peer; build.yml and deploy.yml run it. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_012CzABvBkbcFqoHdmi8b9WB
This commit is contained in:
1 parent
034b792801
commit
ccc3597699
14 files changed
+579
-54
No files matched your search
@@ -0,0 +1,164 @@
|
||||
using Microsoft.Extensions.Caching.Memory;
|
||||
using Microsoft.Extensions.Options;
|
||||
|
||||
using System.Net;
|
||||
using System.Text.Json;
|
||||
|
||||
namespace PrivaPub.Infrastructure.Http
|
||||
{
|
||||
public sealed class FetchedJson : IDisposable
|
||||
{
|
||||
public Uri FinalUri { get; init; }
|
||||
public JsonDocument Document { get; init; }
|
||||
public JsonElement Root => Document.RootElement;
|
||||
|
||||
public void Dispose() => Document?.Dispose();
|
||||
}
|
||||
|
||||
public interface IFederationHttp
|
||||
{
|
||||
bool IsAllowed(Uri target);
|
||||
Task<FetchedJson> GetJson(string url, string accept, Action<HttpRequestMessage> sign, CancellationToken token);
|
||||
Task<HttpResponseMessage> Send(HttpRequestMessage request, CancellationToken token);
|
||||
}
|
||||
|
||||
public class FederationHttp : IFederationHttp
|
||||
{
|
||||
public const string ClientName = "Federation";
|
||||
public const int MaxResponseBytes = 1024 * 1024;
|
||||
public static readonly TimeSpan RequestTimeout = TimeSpan.FromSeconds(15);
|
||||
|
||||
const int MaxRedirects = 3;
|
||||
static readonly TimeSpan NegativeCacheLifetime = TimeSpan.FromMinutes(5);
|
||||
static readonly string[] JsonMediaTypes =
|
||||
{
|
||||
"application/activity+json",
|
||||
"application/ld+json",
|
||||
"application/jrd+json",
|
||||
"application/json"
|
||||
};
|
||||
|
||||
readonly IHttpClientFactory _httpClientFactory;
|
||||
readonly IMemoryCache _cache;
|
||||
readonly IOptionsMonitor<FederationOptions> _options;
|
||||
readonly ILogger<FederationHttp> _logger;
|
||||
|
||||
public FederationHttp(IHttpClientFactory httpClientFactory, IMemoryCache cache, IOptionsMonitor<FederationOptions> options,
|
||||
ILogger<FederationHttp> logger)
|
||||
{
|
||||
_httpClientFactory = httpClientFactory;
|
||||
_cache = cache;
|
||||
_options = options;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
public bool IsAllowed(Uri target)
|
||||
{
|
||||
if (target is not { IsAbsoluteUri: true } || !string.IsNullOrEmpty(target.UserInfo))
|
||||
return false;
|
||||
var options = _options.CurrentValue;
|
||||
if (target.Scheme != Uri.UriSchemeHttps && !(options.AllowPlainHttp && target.Scheme == Uri.UriSchemeHttp))
|
||||
return false;
|
||||
if (options.AllowPrivateNetworks)
|
||||
return true;
|
||||
return target.HostNameType == UriHostNameType.Dns
|
||||
&& target.Host.Contains('.')
|
||||
&& !target.Host.EndsWith(".localhost", StringComparison.OrdinalIgnoreCase)
|
||||
&& !target.Host.EndsWith(".local", StringComparison.OrdinalIgnoreCase)
|
||||
&& !target.Host.EndsWith(".internal", StringComparison.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
public async Task<FetchedJson> GetJson(string url, string accept, Action<HttpRequestMessage> sign, CancellationToken token)
|
||||
{
|
||||
if (!Uri.TryCreate(url, UriKind.Absolute, out var target) || !IsAllowed(target))
|
||||
return default;
|
||||
var negativeKey = NegativeKey(target);
|
||||
if (_cache.TryGetValue(negativeKey, out _))
|
||||
return default;
|
||||
|
||||
using var timeout = CancellationTokenSource.CreateLinkedTokenSource(token);
|
||||
timeout.CancelAfter(RequestTimeout);
|
||||
try
|
||||
{
|
||||
for (var hop = 0; hop <= MaxRedirects; hop++)
|
||||
{
|
||||
using var request = new HttpRequestMessage(HttpMethod.Get, target);
|
||||
request.Headers.Accept.ParseAdd(accept);
|
||||
sign?.Invoke(request);
|
||||
|
||||
using var response = await _httpClientFactory.CreateClient(ClientName)
|
||||
.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, timeout.Token);
|
||||
|
||||
if (IsRedirect(response.StatusCode))
|
||||
{
|
||||
var location = response.Headers.Location;
|
||||
var next = location == default ? default : location.IsAbsoluteUri ? location : new Uri(target, location);
|
||||
if (!IsAllowed(next))
|
||||
return Refuse(negativeKey, url, "a redirect to a disallowed location");
|
||||
target = next;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!response.IsSuccessStatusCode)
|
||||
return Refuse(negativeKey, url, $"status {(int)response.StatusCode}");
|
||||
|
||||
var mediaType = response.Content.Headers.ContentType?.MediaType;
|
||||
if (mediaType == default || !JsonMediaTypes.Contains(mediaType, StringComparer.OrdinalIgnoreCase))
|
||||
return Refuse(negativeKey, url, $"content type '{mediaType}'");
|
||||
if (response.Content.Headers.ContentLength > MaxResponseBytes)
|
||||
return Refuse(negativeKey, url, "a body over the size limit");
|
||||
|
||||
var body = await ReadBounded(response.Content, MaxResponseBytes, timeout.Token);
|
||||
if (body == default)
|
||||
return Refuse(negativeKey, url, "a body over the size limit");
|
||||
|
||||
return new FetchedJson { FinalUri = target, Document = JsonDocument.Parse(body) };
|
||||
}
|
||||
return Refuse(negativeKey, url, "too many redirects");
|
||||
}
|
||||
catch (OperationCanceledException) when (!token.IsCancellationRequested)
|
||||
{
|
||||
return Refuse(negativeKey, url, "a timeout");
|
||||
}
|
||||
catch (Exception ex) when (ex is HttpRequestException or JsonException or BlockedDestinationException)
|
||||
{
|
||||
return Refuse(negativeKey, url, ex.Message);
|
||||
}
|
||||
}
|
||||
|
||||
public async Task<HttpResponseMessage> Send(HttpRequestMessage request, CancellationToken token)
|
||||
{
|
||||
if (!IsAllowed(request.RequestUri))
|
||||
throw new BlockedDestinationException(request.RequestUri?.Host);
|
||||
return await _httpClientFactory.CreateClient(ClientName).SendAsync(request, HttpCompletionOption.ResponseHeadersRead, token);
|
||||
}
|
||||
|
||||
public static async Task<byte[]> ReadBounded(HttpContent content, int limit, CancellationToken token)
|
||||
{
|
||||
await using var stream = await content.ReadAsStreamAsync(token);
|
||||
using var buffer = new MemoryStream();
|
||||
var chunk = new byte[16 * 1024];
|
||||
int read;
|
||||
while ((read = await stream.ReadAsync(chunk, token)) > 0)
|
||||
{
|
||||
if (buffer.Length + read > limit)
|
||||
return default;
|
||||
buffer.Write(chunk, 0, read);
|
||||
}
|
||||
return buffer.ToArray();
|
||||
}
|
||||
|
||||
static bool IsRedirect(HttpStatusCode status) =>
|
||||
status is HttpStatusCode.MovedPermanently or HttpStatusCode.Found or HttpStatusCode.SeeOther
|
||||
or HttpStatusCode.TemporaryRedirect or HttpStatusCode.PermanentRedirect;
|
||||
|
||||
static string NegativeKey(Uri target) => "federation-http:refused:" + target.AbsoluteUri;
|
||||
|
||||
FetchedJson Refuse(string negativeKey, string url, string reason)
|
||||
{
|
||||
_cache.Set(negativeKey, true, NegativeCacheLifetime);
|
||||
_logger.LogInformation("GET {Url} refused: {Reason}", url, reason);
|
||||
return default;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
namespace PrivaPub.Infrastructure.Http
|
||||
{
|
||||
public class FederationOptions
|
||||
{
|
||||
public bool AllowPrivateNetworks { get; set; }
|
||||
public bool AllowPlainHttp { get; set; }
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
using System.Net;
|
||||
using System.Net.Sockets;
|
||||
|
||||
namespace PrivaPub.Infrastructure.Http
|
||||
{
|
||||
public static class IpRangeGuard
|
||||
{
|
||||
static readonly IPNetwork[] BlockedV4 =
|
||||
{
|
||||
IPNetwork.Parse("0.0.0.0/8"),
|
||||
IPNetwork.Parse("10.0.0.0/8"),
|
||||
IPNetwork.Parse("100.64.0.0/10"),
|
||||
IPNetwork.Parse("127.0.0.0/8"),
|
||||
IPNetwork.Parse("169.254.0.0/16"),
|
||||
IPNetwork.Parse("172.16.0.0/12"),
|
||||
IPNetwork.Parse("192.0.0.0/24"),
|
||||
IPNetwork.Parse("192.0.2.0/24"),
|
||||
IPNetwork.Parse("192.88.99.0/24"),
|
||||
IPNetwork.Parse("192.168.0.0/16"),
|
||||
IPNetwork.Parse("198.18.0.0/15"),
|
||||
IPNetwork.Parse("198.51.100.0/24"),
|
||||
IPNetwork.Parse("203.0.113.0/24"),
|
||||
IPNetwork.Parse("224.0.0.0/4"),
|
||||
IPNetwork.Parse("240.0.0.0/4")
|
||||
};
|
||||
|
||||
static readonly IPNetwork GlobalUnicastV6 = IPNetwork.Parse("2000::/3");
|
||||
|
||||
static readonly IPNetwork[] BlockedV6 =
|
||||
{
|
||||
IPNetwork.Parse("2001::/32"),//Teredo
|
||||
IPNetwork.Parse("2001:2::/48"),
|
||||
IPNetwork.Parse("2001:10::/28"),
|
||||
IPNetwork.Parse("2001:20::/28"),
|
||||
IPNetwork.Parse("2001:db8::/32"),
|
||||
IPNetwork.Parse("2002::/16"),//6to4
|
||||
IPNetwork.Parse("3fff::/20")
|
||||
};
|
||||
|
||||
public static bool IsPublic(IPAddress address)
|
||||
{
|
||||
if (address == default)
|
||||
return false;
|
||||
if (address.IsIPv4MappedToIPv6)
|
||||
address = address.MapToIPv4();
|
||||
|
||||
return address.AddressFamily switch
|
||||
{
|
||||
AddressFamily.InterNetwork => !BlockedV4.Any(range => range.Contains(address)),
|
||||
AddressFamily.InterNetworkV6 => GlobalUnicastV6.Contains(address) && !BlockedV6.Any(range => range.Contains(address)),
|
||||
_ => false
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
using System.Net;
|
||||
using System.Net.Sockets;
|
||||
|
||||
namespace PrivaPub.Infrastructure.Http
|
||||
{
|
||||
public sealed class BlockedDestinationException : Exception
|
||||
{
|
||||
public BlockedDestinationException(string host) : base($"'{host}' does not resolve to a public address") { }
|
||||
}
|
||||
|
||||
public static class SafeHttpHandlerFactory
|
||||
{
|
||||
public static SocketsHttpHandler Create(FederationOptions options) => new()
|
||||
{
|
||||
AllowAutoRedirect = false,
|
||||
UseProxy = false,
|
||||
UseCookies = false,
|
||||
AutomaticDecompression = DecompressionMethods.All,
|
||||
ConnectTimeout = TimeSpan.FromSeconds(10),
|
||||
PooledConnectionLifetime = TimeSpan.FromMinutes(2),
|
||||
MaxResponseHeadersLength = 64,
|
||||
ConnectCallback = (context, token) => Connect(context.DnsEndPoint, options.AllowPrivateNetworks, token)
|
||||
};
|
||||
|
||||
public static async ValueTask<Stream> Connect(DnsEndPoint endPoint, bool allowPrivateNetworks, CancellationToken token)
|
||||
{
|
||||
var addresses = await Resolve(endPoint.Host, token);
|
||||
if (addresses.Length == 0 || !allowPrivateNetworks && !addresses.All(IpRangeGuard.IsPublic))
|
||||
throw new BlockedDestinationException(endPoint.Host);
|
||||
|
||||
var socket = new Socket(SocketType.Stream, ProtocolType.Tcp) { NoDelay = true };
|
||||
try
|
||||
{
|
||||
await socket.ConnectAsync(addresses, endPoint.Port, token);
|
||||
return new NetworkStream(socket, ownsSocket: true);
|
||||
}
|
||||
catch
|
||||
{
|
||||
socket.Dispose();
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
static async Task<IPAddress[]> Resolve(string host, CancellationToken token)
|
||||
{
|
||||
if (IPAddress.TryParse(host.Trim('[', ']'), out var literal))
|
||||
return new[] { literal };
|
||||
try
|
||||
{
|
||||
return await Dns.GetHostAddressesAsync(host, token);
|
||||
}
|
||||
catch (SocketException)
|
||||
{
|
||||
return Array.Empty<IPAddress>();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user