using Microsoft.Extensions.Caching.Memory; using Microsoft.Extensions.Options; using PrivaPub.Federation.Moderation; using System.Net; using System.Text.Json; namespace PrivaPub.Infrastructure.Http { public sealed class FetchedJson : IDisposable { public Uri FinalUri { get; init; } public JsonDocument Document { get; init; } public JsonElement Root => Document.RootElement; public void Dispose() => Document?.Dispose(); } public interface IFederationHttp { bool IsAllowed(Uri target); Task GetJson(string url, string accept, Action sign, CancellationToken token); bool FailedTemporarily(string url); Task Send(HttpRequestMessage request, CancellationToken token); Task<(byte[] Bytes, string ContentType)> GetMedia(string url, long maxBytes, CancellationToken token); } public class FederationHttp : IFederationHttp { public const string ClientName = "Federation"; public const int MaxResponseBytes = 1024 * 1024; public static readonly TimeSpan RequestTimeout = TimeSpan.FromSeconds(15); const int MaxRedirects = 3; static readonly TimeSpan NegativeCacheLifetime = TimeSpan.FromMinutes(5); static readonly string[] JsonMediaTypes = { "application/activity+json", "application/ld+json", "application/jrd+json", "application/json" }; readonly IHttpClientFactory _httpClientFactory; readonly IMemoryCache _cache; readonly IOptionsMonitor _options; readonly IDomainBlocks _domainBlocks; readonly ILogger _logger; public FederationHttp(IHttpClientFactory httpClientFactory, IMemoryCache cache, IOptionsMonitor options, IDomainBlocks domainBlocks, ILogger logger) { _httpClientFactory = httpClientFactory; _cache = cache; _options = options; _domainBlocks = domainBlocks; _logger = logger; } public bool IsAllowed(Uri target) { if (target is not { IsAbsoluteUri: true } || !string.IsNullOrEmpty(target.UserInfo)) return false; if (_domainBlocks?.IsSuspended(target.Host) == true) return false; var options = _options.CurrentValue; if (target.Scheme != Uri.UriSchemeHttps && !(options.AllowPlainHttp && target.Scheme == Uri.UriSchemeHttp)) return false; if (options.AllowPrivateNetworks) return true; return target.HostNameType == UriHostNameType.Dns && target.Host.Contains('.') && !target.Host.EndsWith(".localhost", StringComparison.OrdinalIgnoreCase) && !target.Host.EndsWith(".local", StringComparison.OrdinalIgnoreCase) && !target.Host.EndsWith(".internal", StringComparison.OrdinalIgnoreCase); } public async Task GetJson(string url, string accept, Action sign, CancellationToken token) { if (!Uri.TryCreate(url, UriKind.Absolute, out var target) || !IsAllowed(target)) return default; var negativeKey = NegativeKey(target); if (_cache.TryGetValue(negativeKey, out _)) return default; using var timeout = CancellationTokenSource.CreateLinkedTokenSource(token); timeout.CancelAfter(RequestTimeout); try { for (var hop = 0; hop <= MaxRedirects; hop++) { using var request = new HttpRequestMessage(HttpMethod.Get, target); request.Headers.Accept.ParseAdd(accept); sign?.Invoke(request); using var response = await _httpClientFactory.CreateClient(ClientName) .SendAsync(request, HttpCompletionOption.ResponseHeadersRead, timeout.Token); if (IsRedirect(response.StatusCode)) { var location = response.Headers.Location; var next = location == default ? default : location.IsAbsoluteUri ? location : new Uri(target, location); if (!IsAllowed(next)) return Refuse(negativeKey, url, "a redirect to a disallowed location"); target = next; continue; } if (!response.IsSuccessStatusCode) return Refuse(negativeKey, url, $"status {(int)response.StatusCode}", transient: (int)response.StatusCode is >= 500 or 429 or 408); var mediaType = response.Content.Headers.ContentType?.MediaType; if (mediaType == default || !JsonMediaTypes.Contains(mediaType, StringComparer.OrdinalIgnoreCase)) return Refuse(negativeKey, url, $"content type '{mediaType}'"); if (response.Content.Headers.ContentLength > MaxResponseBytes) return Refuse(negativeKey, url, "a body over the size limit"); var body = await ReadBounded(response.Content, MaxResponseBytes, timeout.Token); if (body == default) return Refuse(negativeKey, url, "a body over the size limit"); return new FetchedJson { FinalUri = target, Document = JsonDocument.Parse(body) }; } return Refuse(negativeKey, url, "too many redirects"); } catch (OperationCanceledException) when (!token.IsCancellationRequested) { return Refuse(negativeKey, url, "a timeout", transient: true); } catch (HttpRequestException ex) { return Refuse(negativeKey, url, ex.Message, transient: true); } catch (Exception ex) when (ex is JsonException or BlockedDestinationException) { return Refuse(negativeKey, url, ex.Message); } } public bool FailedTemporarily(string url) => Uri.TryCreate(url, UriKind.Absolute, out var target) && _cache.TryGetValue(NegativeKey(target), out bool transient) && transient; public async Task<(byte[] Bytes, string ContentType)> GetMedia(string url, long maxBytes, CancellationToken token) { if (!Uri.TryCreate(url, UriKind.Absolute, out var target) || !IsAllowed(target)) return default; using var timeout = CancellationTokenSource.CreateLinkedTokenSource(token); timeout.CancelAfter(TimeSpan.FromSeconds(60)); try { for (var hop = 0; hop <= MaxRedirects; hop++) { using var request = new HttpRequestMessage(HttpMethod.Get, target); request.Headers.Accept.ParseAdd("image/*, video/*, audio/*"); using var response = await _httpClientFactory.CreateClient(ClientName).SendAsync(request, HttpCompletionOption.ResponseHeadersRead, timeout.Token); if (IsRedirect(response.StatusCode)) { var location = response.Headers.Location; var next = location == default ? default : location.IsAbsoluteUri ? location : new Uri(target, location); if (!IsAllowed(next)) return default; target = next; continue; } var mediaType = response.Content.Headers.ContentType?.MediaType?.ToLowerInvariant(); if (!response.IsSuccessStatusCode || mediaType == default || !(mediaType.StartsWith("image/") || mediaType.StartsWith("video/") || mediaType.StartsWith("audio/")) || mediaType.Contains("svg") || response.Content.Headers.ContentLength > maxBytes) return default; var bytes = await ReadBounded(response.Content, (int)Math.Min(maxBytes, int.MaxValue), timeout.Token); return bytes == default ? default : (bytes, mediaType); } return default; } catch (Exception ex) when (ex is HttpRequestException or BlockedDestinationException or OperationCanceledException && !token.IsCancellationRequested) { _logger.LogInformation("Media {Url} refused: {Reason}", url, ex.Message); return default; } } public async Task Send(HttpRequestMessage request, CancellationToken token) { if (!IsAllowed(request.RequestUri)) throw new BlockedDestinationException(request.RequestUri?.Host); return await _httpClientFactory.CreateClient(ClientName).SendAsync(request, HttpCompletionOption.ResponseHeadersRead, token); } public static async Task ReadBounded(HttpContent content, int limit, CancellationToken token) { await using var stream = await content.ReadAsStreamAsync(token); using var buffer = new MemoryStream(); var chunk = new byte[16 * 1024]; int read; while ((read = await stream.ReadAsync(chunk, token)) > 0) { if (buffer.Length + read > limit) return default; buffer.Write(chunk, 0, read); } return buffer.ToArray(); } static bool IsRedirect(HttpStatusCode status) => status is HttpStatusCode.MovedPermanently or HttpStatusCode.Found or HttpStatusCode.SeeOther or HttpStatusCode.TemporaryRedirect or HttpStatusCode.PermanentRedirect; static string NegativeKey(Uri target) => "federation-http:refused:" + target.AbsoluteUri; FetchedJson Refuse(string negativeKey, string url, string reason, bool transient = false) { _cache.Set(negativeKey, transient, NegativeCacheLifetime); _logger.LogInformation("GET {Url} refused: {Reason}", url, reason); return default; } } }