Every outbound federation request goes through one guarded client
Infrastructure/Http adds the client the roadmap's S3 and S4 ask for: - the connect callback resolves the name itself and refuses loopback, private, link-local, CGNAT, documentation, multicast, ULA, NAT64, 6to4, Teredo and IPv4-mapped/compatible forms, then connects to the vetted address, so DNS rebinding cannot swap it afterwards; - redirects are followed by hand, at most three, each one re-checked; - bodies are capped at 1 MB after decompression, only JSON media types are read, every request has a 15 s budget, and a refused URL is not asked again for five minutes. Actor and WebFinger fetches and inbox deliveries all use it. Test networks can switch on Federation:AllowPrivateNetworks/AllowPlainHttp; startup refuses both in Production. PrivaPub.Tests (xUnit v3) starts with the address table and the fetcher's limits against an in-process peer; build.yml and deploy.yml run it. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_012CzABvBkbcFqoHdmi8b9WB
This commit is contained in:
1 parent
034b792801
commit
ccc3597699
14 files changed
+579
-54
No files matched your search
@@ -0,0 +1,164 @@
|
||||
using Microsoft.Extensions.Caching.Memory;
|
||||
using Microsoft.Extensions.Options;
|
||||
|
||||
using System.Net;
|
||||
using System.Text.Json;
|
||||
|
||||
namespace PrivaPub.Infrastructure.Http
|
||||
{
|
||||
public sealed class FetchedJson : IDisposable
|
||||
{
|
||||
public Uri FinalUri { get; init; }
|
||||
public JsonDocument Document { get; init; }
|
||||
public JsonElement Root => Document.RootElement;
|
||||
|
||||
public void Dispose() => Document?.Dispose();
|
||||
}
|
||||
|
||||
public interface IFederationHttp
|
||||
{
|
||||
bool IsAllowed(Uri target);
|
||||
Task<FetchedJson> GetJson(string url, string accept, Action<HttpRequestMessage> sign, CancellationToken token);
|
||||
Task<HttpResponseMessage> Send(HttpRequestMessage request, CancellationToken token);
|
||||
}
|
||||
|
||||
public class FederationHttp : IFederationHttp
|
||||
{
|
||||
public const string ClientName = "Federation";
|
||||
public const int MaxResponseBytes = 1024 * 1024;
|
||||
public static readonly TimeSpan RequestTimeout = TimeSpan.FromSeconds(15);
|
||||
|
||||
const int MaxRedirects = 3;
|
||||
static readonly TimeSpan NegativeCacheLifetime = TimeSpan.FromMinutes(5);
|
||||
static readonly string[] JsonMediaTypes =
|
||||
{
|
||||
"application/activity+json",
|
||||
"application/ld+json",
|
||||
"application/jrd+json",
|
||||
"application/json"
|
||||
};
|
||||
|
||||
readonly IHttpClientFactory _httpClientFactory;
|
||||
readonly IMemoryCache _cache;
|
||||
readonly IOptionsMonitor<FederationOptions> _options;
|
||||
readonly ILogger<FederationHttp> _logger;
|
||||
|
||||
public FederationHttp(IHttpClientFactory httpClientFactory, IMemoryCache cache, IOptionsMonitor<FederationOptions> options,
|
||||
ILogger<FederationHttp> logger)
|
||||
{
|
||||
_httpClientFactory = httpClientFactory;
|
||||
_cache = cache;
|
||||
_options = options;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
public bool IsAllowed(Uri target)
|
||||
{
|
||||
if (target is not { IsAbsoluteUri: true } || !string.IsNullOrEmpty(target.UserInfo))
|
||||
return false;
|
||||
var options = _options.CurrentValue;
|
||||
if (target.Scheme != Uri.UriSchemeHttps && !(options.AllowPlainHttp && target.Scheme == Uri.UriSchemeHttp))
|
||||
return false;
|
||||
if (options.AllowPrivateNetworks)
|
||||
return true;
|
||||
return target.HostNameType == UriHostNameType.Dns
|
||||
&& target.Host.Contains('.')
|
||||
&& !target.Host.EndsWith(".localhost", StringComparison.OrdinalIgnoreCase)
|
||||
&& !target.Host.EndsWith(".local", StringComparison.OrdinalIgnoreCase)
|
||||
&& !target.Host.EndsWith(".internal", StringComparison.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
public async Task<FetchedJson> GetJson(string url, string accept, Action<HttpRequestMessage> sign, CancellationToken token)
|
||||
{
|
||||
if (!Uri.TryCreate(url, UriKind.Absolute, out var target) || !IsAllowed(target))
|
||||
return default;
|
||||
var negativeKey = NegativeKey(target);
|
||||
if (_cache.TryGetValue(negativeKey, out _))
|
||||
return default;
|
||||
|
||||
using var timeout = CancellationTokenSource.CreateLinkedTokenSource(token);
|
||||
timeout.CancelAfter(RequestTimeout);
|
||||
try
|
||||
{
|
||||
for (var hop = 0; hop <= MaxRedirects; hop++)
|
||||
{
|
||||
using var request = new HttpRequestMessage(HttpMethod.Get, target);
|
||||
request.Headers.Accept.ParseAdd(accept);
|
||||
sign?.Invoke(request);
|
||||
|
||||
using var response = await _httpClientFactory.CreateClient(ClientName)
|
||||
.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, timeout.Token);
|
||||
|
||||
if (IsRedirect(response.StatusCode))
|
||||
{
|
||||
var location = response.Headers.Location;
|
||||
var next = location == default ? default : location.IsAbsoluteUri ? location : new Uri(target, location);
|
||||
if (!IsAllowed(next))
|
||||
return Refuse(negativeKey, url, "a redirect to a disallowed location");
|
||||
target = next;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!response.IsSuccessStatusCode)
|
||||
return Refuse(negativeKey, url, $"status {(int)response.StatusCode}");
|
||||
|
||||
var mediaType = response.Content.Headers.ContentType?.MediaType;
|
||||
if (mediaType == default || !JsonMediaTypes.Contains(mediaType, StringComparer.OrdinalIgnoreCase))
|
||||
return Refuse(negativeKey, url, $"content type '{mediaType}'");
|
||||
if (response.Content.Headers.ContentLength > MaxResponseBytes)
|
||||
return Refuse(negativeKey, url, "a body over the size limit");
|
||||
|
||||
var body = await ReadBounded(response.Content, MaxResponseBytes, timeout.Token);
|
||||
if (body == default)
|
||||
return Refuse(negativeKey, url, "a body over the size limit");
|
||||
|
||||
return new FetchedJson { FinalUri = target, Document = JsonDocument.Parse(body) };
|
||||
}
|
||||
return Refuse(negativeKey, url, "too many redirects");
|
||||
}
|
||||
catch (OperationCanceledException) when (!token.IsCancellationRequested)
|
||||
{
|
||||
return Refuse(negativeKey, url, "a timeout");
|
||||
}
|
||||
catch (Exception ex) when (ex is HttpRequestException or JsonException or BlockedDestinationException)
|
||||
{
|
||||
return Refuse(negativeKey, url, ex.Message);
|
||||
}
|
||||
}
|
||||
|
||||
public async Task<HttpResponseMessage> Send(HttpRequestMessage request, CancellationToken token)
|
||||
{
|
||||
if (!IsAllowed(request.RequestUri))
|
||||
throw new BlockedDestinationException(request.RequestUri?.Host);
|
||||
return await _httpClientFactory.CreateClient(ClientName).SendAsync(request, HttpCompletionOption.ResponseHeadersRead, token);
|
||||
}
|
||||
|
||||
public static async Task<byte[]> ReadBounded(HttpContent content, int limit, CancellationToken token)
|
||||
{
|
||||
await using var stream = await content.ReadAsStreamAsync(token);
|
||||
using var buffer = new MemoryStream();
|
||||
var chunk = new byte[16 * 1024];
|
||||
int read;
|
||||
while ((read = await stream.ReadAsync(chunk, token)) > 0)
|
||||
{
|
||||
if (buffer.Length + read > limit)
|
||||
return default;
|
||||
buffer.Write(chunk, 0, read);
|
||||
}
|
||||
return buffer.ToArray();
|
||||
}
|
||||
|
||||
static bool IsRedirect(HttpStatusCode status) =>
|
||||
status is HttpStatusCode.MovedPermanently or HttpStatusCode.Found or HttpStatusCode.SeeOther
|
||||
or HttpStatusCode.TemporaryRedirect or HttpStatusCode.PermanentRedirect;
|
||||
|
||||
static string NegativeKey(Uri target) => "federation-http:refused:" + target.AbsoluteUri;
|
||||
|
||||
FetchedJson Refuse(string negativeKey, string url, string reason)
|
||||
{
|
||||
_cache.Set(negativeKey, true, NegativeCacheLifetime);
|
||||
_logger.LogInformation("GET {Url} refused: {Reason}", url, reason);
|
||||
return default;
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user