SocialPub/PrivaPub/Infrastructure/Http/FederationHttp.cs

using Microsoft.Extensions.Caching.Memory;
using Microsoft.Extensions.Options;

using PrivaPub.Federation.Moderation;

using System.Net;
using System.Text.Json;

namespace PrivaPub.Infrastructure.Http
{
	public sealed class FetchedJson : IDisposable
	{
		public Uri FinalUri { get; init; }
		public JsonDocument Document { get; init; }
		public JsonElement Root => Document.RootElement;

		public void Dispose() => Document?.Dispose();
	}

	public interface IFederationHttp
	{
		bool IsAllowed(Uri target);
		Task<FetchedJson> GetJson(string url, string accept, Action<HttpRequestMessage> sign, CancellationToken token);
		bool FailedTemporarily(string url);
		Task<(Uri FinalUri, string Html)> GetPage(string url, CancellationToken token);
		Task<HttpResponseMessage> OpenMedia(string url, System.Net.Http.Headers.RangeHeaderValue range, CancellationToken token);
		Task<HttpResponseMessage> Send(HttpRequestMessage request, CancellationToken token);
		Task<(byte[] Bytes, string ContentType)> GetMedia(string url, long maxBytes, CancellationToken token);
	}

	public class FederationHttp : IFederationHttp
	{
		public const string ClientName = "Federation";
		public const int MaxResponseBytes = 1024 * 1024;
		public static readonly TimeSpan RequestTimeout = TimeSpan.FromSeconds(15);

		const int MaxRedirects = 3;
		static readonly TimeSpan NegativeCacheLifetime = TimeSpan.FromMinutes(5);
		static readonly string[] JsonMediaTypes =
		{
			"application/activity+json",
			"application/ld+json",
			"application/jrd+json",
			"application/json"
		};

		readonly IHttpClientFactory _httpClientFactory;
		readonly IMemoryCache _cache;
		readonly IOptionsMonitor<FederationOptions> _options;
		readonly IDomainBlocks _domainBlocks;
		readonly ILogger<FederationHttp> _logger;

		public FederationHttp(IHttpClientFactory httpClientFactory, IMemoryCache cache, IOptionsMonitor<FederationOptions> options,
			IDomainBlocks domainBlocks, ILogger<FederationHttp> logger)
		{
			_httpClientFactory = httpClientFactory;
			_cache = cache;
			_options = options;
			_domainBlocks = domainBlocks;
			_logger = logger;
		}

		public bool IsAllowed(Uri target)
		{
			if (target is not { IsAbsoluteUri: true } || !string.IsNullOrEmpty(target.UserInfo))
				return false;
			if (_domainBlocks?.IsSuspended(target.Host) == true)
				return false;
			var options = _options.CurrentValue;
			if (target.Scheme != Uri.UriSchemeHttps && !(options.AllowPlainHttp && target.Scheme == Uri.UriSchemeHttp))
				return false;
			if (options.AllowPrivateNetworks)
				return true;
			return target.HostNameType == UriHostNameType.Dns
				&& target.Host.Contains('.')
				&& !target.Host.EndsWith(".localhost", StringComparison.OrdinalIgnoreCase)
				&& !target.Host.EndsWith(".local", StringComparison.OrdinalIgnoreCase)
				&& !target.Host.EndsWith(".internal", StringComparison.OrdinalIgnoreCase);
		}

		public async Task<FetchedJson> GetJson(string url, string accept, Action<HttpRequestMessage> sign, CancellationToken token)
		{
			if (!Uri.TryCreate(url, UriKind.Absolute, out var target) || !IsAllowed(target))
				return default;
			var negativeKey = NegativeKey(target);
			if (_cache.TryGetValue(negativeKey, out _))
				return default;

			using var timeout = CancellationTokenSource.CreateLinkedTokenSource(token);
			timeout.CancelAfter(RequestTimeout);
			try
			{
				for (var hop = 0; hop <= MaxRedirects; hop++)
				{
					using var request = new HttpRequestMessage(HttpMethod.Get, target);
					request.Headers.Accept.ParseAdd(accept);
					sign?.Invoke(request);

					using var response = await _httpClientFactory.CreateClient(ClientName)
						.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, timeout.Token);

					if (IsRedirect(response.StatusCode))
					{
						var location = response.Headers.Location;
						var next = location == default ? default : location.IsAbsoluteUri ? location : new Uri(target, location);
						if (!IsAllowed(next))
							return Refuse(negativeKey, url, "a redirect to a disallowed location");
						target = next;
						continue;
					}

					if (!response.IsSuccessStatusCode)
						return Refuse(negativeKey, url, $"status {(int)response.StatusCode}",
							transient: (int)response.StatusCode is >= 500 or 429 or 408);

					var mediaType = response.Content.Headers.ContentType?.MediaType;
					if (mediaType == default || !JsonMediaTypes.Contains(mediaType, StringComparer.OrdinalIgnoreCase))
						return Refuse(negativeKey, url, $"content type '{mediaType}'");
					if (response.Content.Headers.ContentLength > MaxResponseBytes)
						return Refuse(negativeKey, url, "a body over the size limit");

					var body = await ReadBounded(response.Content, MaxResponseBytes, timeout.Token);
					if (body == default)
						return Refuse(negativeKey, url, "a body over the size limit");

					return new FetchedJson { FinalUri = target, Document = JsonDocument.Parse(body) };
				}
				return Refuse(negativeKey, url, "too many redirects");
			}
			catch (OperationCanceledException) when (!token.IsCancellationRequested)
			{
				return Refuse(negativeKey, url, "a timeout", transient: true);
			}
			catch (HttpRequestException ex)
			{
				return Refuse(negativeKey, url, ex.Message, transient: true);
			}
			catch (Exception ex) when (ex is JsonException or BlockedDestinationException)
			{
				return Refuse(negativeKey, url, ex.Message);
			}
		}

		public bool FailedTemporarily(string url) =>
			Uri.TryCreate(url, UriKind.Absolute, out var target) && _cache.TryGetValue(NegativeKey(target), out bool transient) && transient;

		public async Task<(byte[] Bytes, string ContentType)> GetMedia(string url, long maxBytes, CancellationToken token)
		{
			if (!Uri.TryCreate(url, UriKind.Absolute, out var target) || !IsAllowed(target))
				return default;
			using var timeout = CancellationTokenSource.CreateLinkedTokenSource(token);
			timeout.CancelAfter(TimeSpan.FromSeconds(60));
			try
			{
				for (var hop = 0; hop <= MaxRedirects; hop++)
				{
					using var request = new HttpRequestMessage(HttpMethod.Get, target);
					request.Headers.Accept.ParseAdd("image/*, video/*, audio/*");
					using var response = await _httpClientFactory.CreateClient(ClientName).SendAsync(request, HttpCompletionOption.ResponseHeadersRead, timeout.Token);
					if (IsRedirect(response.StatusCode))
					{
						var location = response.Headers.Location;
						var next = location == default ? default : location.IsAbsoluteUri ? location : new Uri(target, location);
						if (!IsAllowed(next))
							return default;
						target = next;
						continue;
					}
					var mediaType = response.Content.Headers.ContentType?.MediaType?.ToLowerInvariant();
					if (!response.IsSuccessStatusCode || mediaType == default
						|| !(mediaType.StartsWith("image/") || mediaType.StartsWith("video/") || mediaType.StartsWith("audio/"))
						|| mediaType.Contains("svg") || response.Content.Headers.ContentLength > maxBytes)
						return default;
					var bytes = await ReadBounded(response.Content, (int)Math.Min(maxBytes, int.MaxValue), timeout.Token);
					return bytes == default ? default : (bytes, mediaType);
				}
				return default;
			}
			catch (Exception ex) when (ex is HttpRequestException or BlockedDestinationException or OperationCanceledException && !token.IsCancellationRequested)
			{
				_logger.LogInformation("Media {Url} refused: {Reason}", url, ex.Message);
				return default;
			}
		}

		public async Task<(Uri FinalUri, string Html)> GetPage(string url, CancellationToken token)
		{
			if (!Uri.TryCreate(url, UriKind.Absolute, out var target) || !IsAllowed(target))
				return default;
			using var timeout = CancellationTokenSource.CreateLinkedTokenSource(token);
			timeout.CancelAfter(RequestTimeout);
			try
			{
				for (var hop = 0; hop <= MaxRedirects; hop++)
				{
					using var request = new HttpRequestMessage(HttpMethod.Get, target);
					request.Headers.Accept.ParseAdd("text/html, application/xhtml+xml");
					using var response = await _httpClientFactory.CreateClient(ClientName).SendAsync(request, HttpCompletionOption.ResponseHeadersRead, timeout.Token);
					if (IsRedirect(response.StatusCode))
					{
						var location = response.Headers.Location;
						var next = location == default ? default : location.IsAbsoluteUri ? location : new Uri(target, location);
						if (!IsAllowed(next))
							return default;
						target = next;
						continue;
					}
					var mediaType = response.Content.Headers.ContentType?.MediaType?.ToLowerInvariant();
					if (!response.IsSuccessStatusCode || mediaType is not ("text/html" or "application/xhtml+xml"))
						return default;
					var bytes = await ReadPrefix(response.Content, MaxPageBytes, timeout.Token);
					var charset = response.Content.Headers.ContentType?.CharSet?.Trim('"');
					var encoding = System.Text.Encoding.UTF8;
					try
					{
						if (!string.IsNullOrEmpty(charset))
							encoding = System.Text.Encoding.GetEncoding(charset);
					}
					catch (ArgumentException)
					{
					}
					return (target, encoding.GetString(bytes));
				}
				return default;
			}
			catch (Exception ex) when (ex is HttpRequestException or BlockedDestinationException or OperationCanceledException && !token.IsCancellationRequested)
			{
				_logger.LogInformation("Page {Url} refused: {Reason}", url, ex.Message);
				return default;
			}
		}

		public async Task<HttpResponseMessage> OpenMedia(string url, System.Net.Http.Headers.RangeHeaderValue range, CancellationToken token)
		{
			if (!Uri.TryCreate(url, UriKind.Absolute, out var target) || !IsAllowed(target))
				return default;
			try
			{
				for (var hop = 0; hop <= MaxRedirects; hop++)
				{
					using var request = new HttpRequestMessage(HttpMethod.Get, target);
					request.Headers.Accept.ParseAdd("video/*, audio/*, image/*");
					request.Headers.Range = range;
					var response = await _httpClientFactory.CreateClient(ClientName).SendAsync(request, HttpCompletionOption.ResponseHeadersRead, token);
					if (IsRedirect(response.StatusCode))
					{
						var location = response.Headers.Location;
						response.Dispose();
						var next = location == default ? default : location.IsAbsoluteUri ? location : new Uri(target, location);
						if (!IsAllowed(next))
							return default;
						target = next;
						continue;
					}
					var mediaType = response.Content.Headers.ContentType?.MediaType?.ToLowerInvariant();
					if (!response.IsSuccessStatusCode || mediaType == default || mediaType.Contains("svg")
						|| !(mediaType.StartsWith("video/") || mediaType.StartsWith("audio/") || mediaType.StartsWith("image/") || mediaType == "application/octet-stream"))
					{
						response.Dispose();
						return default;
					}
					return response;
				}
				return default;
			}
			catch (Exception ex) when (ex is HttpRequestException or BlockedDestinationException or OperationCanceledException && !token.IsCancellationRequested)
			{
				_logger.LogInformation("Media stream {Url} refused: {Reason}", url, ex.Message);
				return default;
			}
		}

		const int MaxPageBytes = 512 * 1024;

		static async Task<byte[]> ReadPrefix(HttpContent content, int limit, CancellationToken token)
		{
			await using var stream = await content.ReadAsStreamAsync(token);
			var buffer = new byte[limit];
			var total = 0;
			int read;
			while (total < limit && (read = await stream.ReadAsync(buffer.AsMemory(total, limit - total), token)) > 0)
				total += read;
			return buffer[..total];
		}

		public async Task<HttpResponseMessage> Send(HttpRequestMessage request, CancellationToken token)
		{
			if (!IsAllowed(request.RequestUri))
				throw new BlockedDestinationException(request.RequestUri?.Host);
			return await _httpClientFactory.CreateClient(ClientName).SendAsync(request, HttpCompletionOption.ResponseHeadersRead, token);
		}

		public static async Task<byte[]> ReadBounded(HttpContent content, int limit, CancellationToken token)
		{
			await using var stream = await content.ReadAsStreamAsync(token);
			using var buffer = new MemoryStream();
			var chunk = new byte[16 * 1024];
			int read;
			while ((read = await stream.ReadAsync(chunk, token)) > 0)
			{
				if (buffer.Length + read > limit)
					return default;
				buffer.Write(chunk, 0, read);
			}
			return buffer.ToArray();
		}

		static bool IsRedirect(HttpStatusCode status) =>
			status is HttpStatusCode.MovedPermanently or HttpStatusCode.Found or HttpStatusCode.SeeOther
				or HttpStatusCode.TemporaryRedirect or HttpStatusCode.PermanentRedirect;

		static string NegativeKey(Uri target) => "federation-http:refused:" + target.AbsoluteUri;

		FetchedJson Refuse(string negativeKey, string url, string reason, bool transient = false)
		{
			_cache.Set(negativeKey, transient, NegativeCacheLifetime);
			_logger.LogInformation("GET {Url} refused: {Reason}", url, reason);
			return default;
		}
	}
}