thepraandClaude Opus 5.5
4f5df743ae
Off by default (Statistics:Crawler:Enabled), as the owner decided. When it is on:
- CrawlPlan runs hourly, from StatisticsSchedule. It inserts the configured seeds and
queues up to HostsPerHour servers whose last visit is older than RevisitDays, are not
paused by the breaker and are not domain-blocked, spread across the hour.
- CrawlInstance visits one server at a time as
"PrivaPub-Stargazer/<ref> (+<base>/stargazing)":
- it reads robots.txt (RFC 9309: its own group first, then PrivaPub, then *; longest
rule wins; a 4xx allows everything; a 5xx or no answer keeps it out);
- it describes servers that only crawling ever found, through
InstanceDescriber.Describe with robots.txt as the path filter;
- it reads /api/v1/instance/peers through the new GetStringArray, which keeps what it
read from the first 1 MB instead of refusing a large list;
- it adds the names a server could ever be reached at as "crawled": DNS only,
punycode, no addresses, ports or hidden services, and the reserved test names only on
a test network. Never more than MaxNewHostsPerCrawl per visit or MaxHosts in all, and
never over a touched server.
- It reads nothing but robots.txt, NodeInfo, the instance API and the peers list.
IFederationHttp.GetText serves robots.txt, and HttpScope.Crawl carries the
User-Agent.
- /stargazing explains all this and how to keep the crawler out, says whether it is on,
and credits DB-IP. GET /clientapi/admin/statistics/crawler shows the frontier.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01ELjqpznMFMNrJoJUj6K5p2
625 lines
22 KiB
C#
625 lines
22 KiB
C#
using Microsoft.Extensions.Caching.Memory;
|
|
using Microsoft.Extensions.Options;
|
|
|
|
using PrivaPub.Federation.Moderation;
|
|
using PrivaPub.Infrastructure.Statistics;
|
|
using PrivaPub.Models.Statistics;
|
|
|
|
using System.Net;
|
|
using System.Text.Json;
|
|
|
|
namespace PrivaPub.Infrastructure.Http
|
|
{
|
|
public sealed class FetchedJson : IDisposable
|
|
{
|
|
public Uri FinalUri { get; init; }
|
|
public JsonDocument Document { get; init; }
|
|
public JsonElement Root => Document.RootElement;
|
|
|
|
public void Dispose() => Document?.Dispose();
|
|
}
|
|
|
|
public interface IFederationHttp
|
|
{
|
|
bool IsAllowed(Uri target);
|
|
Task<FetchedJson> GetJson(string url, string accept, Action<HttpRequestMessage> sign, CancellationToken token);
|
|
bool FailedTemporarily(string url);
|
|
Task<(Uri FinalUri, string Html)> GetPage(string url, CancellationToken token);
|
|
Task<HttpResponseMessage> OpenMedia(string url, System.Net.Http.Headers.RangeHeaderValue range, CancellationToken token);
|
|
Task<HttpResponseMessage> Send(HttpRequestMessage request, CancellationToken token);
|
|
Task<(byte[] Bytes, string ContentType)> GetMedia(string url, long maxBytes, CancellationToken token);
|
|
Task<(int Status, string Text)> GetText(string url, int maxBytes, CancellationToken token);
|
|
Task<IReadOnlyList<string>> GetStringArray(string url, int maxItems, CancellationToken token);
|
|
}
|
|
|
|
public class FederationHttp : IFederationHttp
|
|
{
|
|
public const string ClientName = "Federation";
|
|
public const int MaxResponseBytes = 1024 * 1024;
|
|
public static readonly TimeSpan RequestTimeout = TimeSpan.FromSeconds(15);
|
|
|
|
const int MaxRedirects = 3;
|
|
static readonly TimeSpan NegativeCacheLifetime = TimeSpan.FromMinutes(5);
|
|
static readonly string[] JsonMediaTypes =
|
|
{
|
|
"application/activity+json",
|
|
"application/ld+json",
|
|
"application/jrd+json",
|
|
"application/json"
|
|
};
|
|
|
|
readonly IHttpClientFactory _httpClientFactory;
|
|
readonly IMemoryCache _cache;
|
|
readonly IOptionsMonitor<FederationOptions> _options;
|
|
readonly IDomainBlocks _domainBlocks;
|
|
readonly ILogger<FederationHttp> _logger;
|
|
readonly IInteractionLedger _ledger;
|
|
|
|
public FederationHttp(IHttpClientFactory httpClientFactory, IMemoryCache cache, IOptionsMonitor<FederationOptions> options,
|
|
IDomainBlocks domainBlocks, ILogger<FederationHttp> logger, IInteractionLedger ledger = default)
|
|
{
|
|
_httpClientFactory = httpClientFactory;
|
|
_cache = cache;
|
|
_options = options;
|
|
_domainBlocks = domainBlocks;
|
|
_logger = logger;
|
|
_ledger = ledger;
|
|
}
|
|
|
|
sealed class Exchange
|
|
{
|
|
public Exchange(string url, string purpose)
|
|
{
|
|
Host = Interactions.HostOf(url);
|
|
Purpose = purpose;
|
|
}
|
|
|
|
public readonly string Host;
|
|
public readonly string Purpose;
|
|
public readonly long Started = System.Diagnostics.Stopwatch.GetTimestamp();
|
|
public int? Status;
|
|
public long? Bytes;
|
|
public int Hops;
|
|
public string Outcome = Interactions.Ok;
|
|
public string Reason;
|
|
|
|
public void Refused(string reason) => (Outcome, Reason) = (Interactions.Refused, reason);
|
|
|
|
public void Failed(string reason) => (Outcome, Reason) = (Interactions.Failed, reason);
|
|
|
|
public void Answered(HttpResponseMessage response) => Status = (int)response.StatusCode;
|
|
}
|
|
|
|
void Record(Exchange exchange)
|
|
{
|
|
if (_ledger == default)
|
|
return;
|
|
var trigger = HttpScope.Trigger;
|
|
if (trigger == HttpScope.Request)
|
|
{
|
|
_ledger.Count(exchange.Host, $"http:{exchange.Purpose}:{exchange.Outcome}", exchange.Bytes ?? 0);
|
|
return;
|
|
}
|
|
_ledger.Record(new InteractionEvent
|
|
{
|
|
Channel = Interactions.Http,
|
|
Host = exchange.Host,
|
|
Purpose = exchange.Purpose,
|
|
Trigger = trigger,
|
|
Outcome = exchange.Outcome,
|
|
Reason = exchange.Reason,
|
|
Status = exchange.Status,
|
|
LatencyMs = (int)System.Diagnostics.Stopwatch.GetElapsedTime(exchange.Started).TotalMilliseconds,
|
|
Bytes = exchange.Bytes,
|
|
Redirects = exchange.Hops,
|
|
Crawl = HttpScope.Crawling
|
|
});
|
|
}
|
|
|
|
static HttpRequestMessage Request(Uri target, string accept)
|
|
{
|
|
var request = new HttpRequestMessage(HttpMethod.Get, target);
|
|
request.Headers.Accept.ParseAdd(accept);
|
|
if (HttpScope.UserAgent is { } userAgent)
|
|
request.Headers.UserAgent.ParseAdd(userAgent);
|
|
return request;
|
|
}
|
|
|
|
//robots.txt: the status alone matters when it is not 2xx (RFC 9309), so it is returned whatever it is; 0 when unreachable
|
|
public async Task<(int Status, string Text)> GetText(string url, int maxBytes, CancellationToken token)
|
|
{
|
|
var exchange = new Exchange(url, HttpScope.Purpose ?? "text");
|
|
try
|
|
{
|
|
if (!Uri.TryCreate(url, UriKind.Absolute, out var target) || !IsAllowed(target))
|
|
{
|
|
exchange.Refused("disallowed");
|
|
return default;
|
|
}
|
|
using var timeout = CancellationTokenSource.CreateLinkedTokenSource(token);
|
|
timeout.CancelAfter(RequestTimeout);
|
|
for (var hop = 0; hop <= MaxRedirects; hop++)
|
|
{
|
|
using var request = Request(target, "text/plain");
|
|
using var response = await _httpClientFactory.CreateClient(ClientName).SendAsync(request, HttpCompletionOption.ResponseHeadersRead, timeout.Token);
|
|
exchange.Answered(response);
|
|
if (IsRedirect(response.StatusCode))
|
|
{
|
|
var location = response.Headers.Location;
|
|
var next = location == default ? default : location.IsAbsoluteUri ? location : new Uri(target, location);
|
|
if (!IsAllowed(next))
|
|
{
|
|
exchange.Refused("bad-redirect");
|
|
return ((int)response.StatusCode, default);
|
|
}
|
|
target = next;
|
|
exchange.Hops++;
|
|
continue;
|
|
}
|
|
if (!response.IsSuccessStatusCode)
|
|
{
|
|
exchange.Refused(StatusReason(response));
|
|
return ((int)response.StatusCode, default);
|
|
}
|
|
var bytes = await ReadPrefix(response.Content, maxBytes, timeout.Token);
|
|
exchange.Bytes = bytes.Length;
|
|
return ((int)response.StatusCode, System.Text.Encoding.UTF8.GetString(bytes));
|
|
}
|
|
exchange.Refused("too-many-redirects");
|
|
return default;
|
|
}
|
|
catch (Exception ex) when (ex is HttpRequestException or BlockedDestinationException or OperationCanceledException && !token.IsCancellationRequested)
|
|
{
|
|
exchange.Failed(ex switch { BlockedDestinationException => "private-address", OperationCanceledException => "timeout", _ => "network" });
|
|
return default;
|
|
}
|
|
finally
|
|
{
|
|
Record(exchange);
|
|
}
|
|
}
|
|
|
|
//a JSON array of strings such as /api/v1/instance/peers, read from at most the first MaxResponseBytes: a large server's
|
|
//list is cut short rather than refused
|
|
public async Task<IReadOnlyList<string>> GetStringArray(string url, int maxItems, CancellationToken token)
|
|
{
|
|
var exchange = new Exchange(url, HttpScope.Purpose ?? "list");
|
|
var items = new List<string>();
|
|
try
|
|
{
|
|
if (!Uri.TryCreate(url, UriKind.Absolute, out var target) || !IsAllowed(target))
|
|
{
|
|
exchange.Refused("disallowed");
|
|
return items;
|
|
}
|
|
using var timeout = CancellationTokenSource.CreateLinkedTokenSource(token);
|
|
timeout.CancelAfter(RequestTimeout);
|
|
for (var hop = 0; hop <= MaxRedirects; hop++)
|
|
{
|
|
using var request = Request(target, "application/json");
|
|
using var response = await _httpClientFactory.CreateClient(ClientName).SendAsync(request, HttpCompletionOption.ResponseHeadersRead, timeout.Token);
|
|
exchange.Answered(response);
|
|
if (IsRedirect(response.StatusCode))
|
|
{
|
|
var location = response.Headers.Location;
|
|
var next = location == default ? default : location.IsAbsoluteUri ? location : new Uri(target, location);
|
|
if (!IsAllowed(next))
|
|
{
|
|
exchange.Refused("bad-redirect");
|
|
return items;
|
|
}
|
|
target = next;
|
|
exchange.Hops++;
|
|
continue;
|
|
}
|
|
if (!response.IsSuccessStatusCode)
|
|
{
|
|
exchange.Refused(StatusReason(response));
|
|
return items;
|
|
}
|
|
var mediaType = response.Content.Headers.ContentType?.MediaType;
|
|
if (mediaType == default || !JsonMediaTypes.Contains(mediaType, StringComparer.OrdinalIgnoreCase))
|
|
{
|
|
exchange.Refused("content-type");
|
|
return items;
|
|
}
|
|
var bytes = await ReadPrefix(response.Content, MaxResponseBytes, timeout.Token);
|
|
exchange.Bytes = bytes.Length;
|
|
ReadStrings(bytes, maxItems, items);
|
|
return items;
|
|
}
|
|
exchange.Refused("too-many-redirects");
|
|
return items;
|
|
}
|
|
catch (Exception ex) when (ex is HttpRequestException or BlockedDestinationException or OperationCanceledException && !token.IsCancellationRequested)
|
|
{
|
|
exchange.Failed(ex switch { BlockedDestinationException => "private-address", OperationCanceledException => "timeout", _ => "network" });
|
|
return items;
|
|
}
|
|
finally
|
|
{
|
|
Record(exchange);
|
|
}
|
|
}
|
|
|
|
public static void ReadStrings(byte[] json, int maxItems, List<string> items)
|
|
{
|
|
var reader = new Utf8JsonReader(json, isFinalBlock: false, state: default);
|
|
try
|
|
{
|
|
if (!reader.Read() || reader.TokenType != JsonTokenType.StartArray)
|
|
return;
|
|
while (items.Count < maxItems && reader.Read() && reader.TokenType != JsonTokenType.EndArray)
|
|
{
|
|
if (reader.TokenType == JsonTokenType.String)
|
|
items.Add(reader.GetString());
|
|
else if (!reader.TrySkip())
|
|
return;
|
|
}
|
|
}
|
|
catch (JsonException)
|
|
{
|
|
}
|
|
}
|
|
|
|
static string StatusReason(HttpResponseMessage response) => ((int)response.StatusCode).ToString(System.Globalization.CultureInfo.InvariantCulture);
|
|
|
|
public bool IsAllowed(Uri target)
|
|
{
|
|
if (target is not { IsAbsoluteUri: true } || !string.IsNullOrEmpty(target.UserInfo))
|
|
return false;
|
|
if (_domainBlocks?.IsSuspended(target.Host) == true)
|
|
return false;
|
|
var options = _options.CurrentValue;
|
|
if (target.Scheme != Uri.UriSchemeHttps && !(options.AllowPlainHttp && target.Scheme == Uri.UriSchemeHttp))
|
|
return false;
|
|
if (options.AllowPrivateNetworks)
|
|
return true;
|
|
return target.HostNameType == UriHostNameType.Dns
|
|
&& target.Host.Contains('.')
|
|
&& !target.Host.EndsWith(".localhost", StringComparison.OrdinalIgnoreCase)
|
|
&& !target.Host.EndsWith(".local", StringComparison.OrdinalIgnoreCase)
|
|
&& !target.Host.EndsWith(".internal", StringComparison.OrdinalIgnoreCase);
|
|
}
|
|
|
|
public async Task<FetchedJson> GetJson(string url, string accept, Action<HttpRequestMessage> sign, CancellationToken token)
|
|
{
|
|
var exchange = new Exchange(url, HttpScope.Purpose ?? "object");
|
|
try
|
|
{
|
|
return await GetJson(url, accept, sign, exchange, token);
|
|
}
|
|
finally
|
|
{
|
|
Record(exchange);
|
|
}
|
|
}
|
|
|
|
async Task<FetchedJson> GetJson(string url, string accept, Action<HttpRequestMessage> sign, Exchange exchange, CancellationToken token)
|
|
{
|
|
if (!Uri.TryCreate(url, UriKind.Absolute, out var target) || !IsAllowed(target))
|
|
{
|
|
exchange.Refused("disallowed");
|
|
return default;
|
|
}
|
|
var negativeKey = NegativeKey(target);
|
|
if (_cache.TryGetValue(negativeKey, out _))
|
|
{
|
|
exchange.Refused("remembered");
|
|
return default;
|
|
}
|
|
|
|
using var timeout = CancellationTokenSource.CreateLinkedTokenSource(token);
|
|
timeout.CancelAfter(RequestTimeout);
|
|
try
|
|
{
|
|
for (var hop = 0; hop <= MaxRedirects; hop++)
|
|
{
|
|
using var request = Request(target, accept);
|
|
sign?.Invoke(request);
|
|
|
|
using var response = await _httpClientFactory.CreateClient(ClientName)
|
|
.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, timeout.Token);
|
|
exchange.Answered(response);
|
|
|
|
if (IsRedirect(response.StatusCode))
|
|
{
|
|
var location = response.Headers.Location;
|
|
var next = location == default ? default : location.IsAbsoluteUri ? location : new Uri(target, location);
|
|
if (!IsAllowed(next))
|
|
return Refuse(negativeKey, url, "a redirect to a disallowed location", exchange, "bad-redirect");
|
|
target = next;
|
|
exchange.Hops++;
|
|
continue;
|
|
}
|
|
|
|
if (!response.IsSuccessStatusCode)
|
|
return Refuse(negativeKey, url, $"status {(int)response.StatusCode}", exchange, StatusReason(response),
|
|
transient: (int)response.StatusCode is >= 500 or 429 or 408);
|
|
|
|
var mediaType = response.Content.Headers.ContentType?.MediaType;
|
|
if (mediaType == default || !JsonMediaTypes.Contains(mediaType, StringComparer.OrdinalIgnoreCase))
|
|
return Refuse(negativeKey, url, $"content type '{mediaType}'", exchange, "content-type");
|
|
if (response.Content.Headers.ContentLength > MaxResponseBytes)
|
|
return Refuse(negativeKey, url, "a body over the size limit", exchange, "too-large");
|
|
|
|
var body = await ReadBounded(response.Content, MaxResponseBytes, timeout.Token);
|
|
if (body == default)
|
|
return Refuse(negativeKey, url, "a body over the size limit", exchange, "too-large");
|
|
exchange.Bytes = body.Length;
|
|
|
|
return new FetchedJson { FinalUri = target, Document = JsonDocument.Parse(body) };
|
|
}
|
|
return Refuse(negativeKey, url, "too many redirects", exchange, "too-many-redirects");
|
|
}
|
|
catch (OperationCanceledException) when (!token.IsCancellationRequested)
|
|
{
|
|
return Refuse(negativeKey, url, "a timeout", exchange, "timeout", transient: true);
|
|
}
|
|
catch (HttpRequestException ex)
|
|
{
|
|
return Refuse(negativeKey, url, ex.Message, exchange, "network", transient: true);
|
|
}
|
|
catch (Exception ex) when (ex is JsonException or BlockedDestinationException)
|
|
{
|
|
return Refuse(negativeKey, url, ex.Message, exchange, ex is JsonException ? "bad-json" : "private-address");
|
|
}
|
|
}
|
|
|
|
public bool FailedTemporarily(string url) =>
|
|
Uri.TryCreate(url, UriKind.Absolute, out var target) && _cache.TryGetValue(NegativeKey(target), out bool transient) && transient;
|
|
|
|
public async Task<(byte[] Bytes, string ContentType)> GetMedia(string url, long maxBytes, CancellationToken token)
|
|
{
|
|
var exchange = new Exchange(url, "media");
|
|
try
|
|
{
|
|
var media = await GetMedia(url, maxBytes, exchange, token);
|
|
if (media.Bytes == default && exchange.Outcome == Interactions.Ok)
|
|
exchange.Refused("unusable");
|
|
return media;
|
|
}
|
|
finally
|
|
{
|
|
Record(exchange);
|
|
}
|
|
}
|
|
|
|
async Task<(byte[] Bytes, string ContentType)> GetMedia(string url, long maxBytes, Exchange exchange, CancellationToken token)
|
|
{
|
|
if (!Uri.TryCreate(url, UriKind.Absolute, out var target) || !IsAllowed(target))
|
|
{
|
|
exchange.Refused("disallowed");
|
|
return default;
|
|
}
|
|
using var timeout = CancellationTokenSource.CreateLinkedTokenSource(token);
|
|
timeout.CancelAfter(TimeSpan.FromSeconds(60));
|
|
try
|
|
{
|
|
for (var hop = 0; hop <= MaxRedirects; hop++)
|
|
{
|
|
using var request = new HttpRequestMessage(HttpMethod.Get, target);
|
|
request.Headers.Accept.ParseAdd("image/*, video/*, audio/*");
|
|
using var response = await _httpClientFactory.CreateClient(ClientName).SendAsync(request, HttpCompletionOption.ResponseHeadersRead, timeout.Token);
|
|
exchange.Answered(response);
|
|
if (IsRedirect(response.StatusCode))
|
|
{
|
|
var location = response.Headers.Location;
|
|
var next = location == default ? default : location.IsAbsoluteUri ? location : new Uri(target, location);
|
|
if (!IsAllowed(next))
|
|
{
|
|
exchange.Refused("bad-redirect");
|
|
return default;
|
|
}
|
|
target = next;
|
|
exchange.Hops++;
|
|
continue;
|
|
}
|
|
var mediaType = response.Content.Headers.ContentType?.MediaType?.ToLowerInvariant();
|
|
if (!response.IsSuccessStatusCode)
|
|
{
|
|
exchange.Failed(StatusReason(response));
|
|
return default;
|
|
}
|
|
if (mediaType == default || !(mediaType.StartsWith("image/") || mediaType.StartsWith("video/") || mediaType.StartsWith("audio/"))
|
|
|| mediaType.Contains("svg"))
|
|
{
|
|
exchange.Refused("content-type");
|
|
return default;
|
|
}
|
|
if (response.Content.Headers.ContentLength > maxBytes)
|
|
{
|
|
exchange.Refused("too-large");
|
|
return default;
|
|
}
|
|
var bytes = await ReadBounded(response.Content, (int)Math.Min(maxBytes, int.MaxValue), timeout.Token);
|
|
if (bytes == default)
|
|
{
|
|
exchange.Refused("too-large");
|
|
return default;
|
|
}
|
|
exchange.Bytes = bytes.Length;
|
|
return (bytes, mediaType);
|
|
}
|
|
exchange.Refused("too-many-redirects");
|
|
return default;
|
|
}
|
|
catch (Exception ex) when (ex is HttpRequestException or BlockedDestinationException or OperationCanceledException && !token.IsCancellationRequested)
|
|
{
|
|
exchange.Failed(ex switch { BlockedDestinationException => "private-address", OperationCanceledException => "timeout", _ => "network" });
|
|
_logger.LogInformation("Media {Url} refused: {Reason}", url, ex.Message);
|
|
return default;
|
|
}
|
|
}
|
|
|
|
public async Task<(Uri FinalUri, string Html)> GetPage(string url, CancellationToken token)
|
|
{
|
|
if (!Uri.TryCreate(url, UriKind.Absolute, out var target) || !IsAllowed(target))
|
|
return default;
|
|
using var timeout = CancellationTokenSource.CreateLinkedTokenSource(token);
|
|
timeout.CancelAfter(RequestTimeout);
|
|
try
|
|
{
|
|
for (var hop = 0; hop <= MaxRedirects; hop++)
|
|
{
|
|
using var request = new HttpRequestMessage(HttpMethod.Get, target);
|
|
request.Headers.Accept.ParseAdd("text/html, application/xhtml+xml");
|
|
using var response = await _httpClientFactory.CreateClient(ClientName).SendAsync(request, HttpCompletionOption.ResponseHeadersRead, timeout.Token);
|
|
if (IsRedirect(response.StatusCode))
|
|
{
|
|
var location = response.Headers.Location;
|
|
var next = location == default ? default : location.IsAbsoluteUri ? location : new Uri(target, location);
|
|
if (!IsAllowed(next))
|
|
return default;
|
|
target = next;
|
|
continue;
|
|
}
|
|
var mediaType = response.Content.Headers.ContentType?.MediaType?.ToLowerInvariant();
|
|
if (!response.IsSuccessStatusCode || mediaType is not ("text/html" or "application/xhtml+xml"))
|
|
return default;
|
|
var bytes = await ReadPrefix(response.Content, MaxPageBytes, timeout.Token);
|
|
var charset = response.Content.Headers.ContentType?.CharSet?.Trim('"');
|
|
var encoding = System.Text.Encoding.UTF8;
|
|
try
|
|
{
|
|
if (!string.IsNullOrEmpty(charset))
|
|
encoding = System.Text.Encoding.GetEncoding(charset);
|
|
}
|
|
catch (ArgumentException)
|
|
{
|
|
}
|
|
return (target, encoding.GetString(bytes));
|
|
}
|
|
return default;
|
|
}
|
|
catch (Exception ex) when (ex is HttpRequestException or BlockedDestinationException or OperationCanceledException && !token.IsCancellationRequested)
|
|
{
|
|
_logger.LogInformation("Page {Url} refused: {Reason}", url, ex.Message);
|
|
return default;
|
|
}
|
|
}
|
|
|
|
public async Task<HttpResponseMessage> OpenMedia(string url, System.Net.Http.Headers.RangeHeaderValue range, CancellationToken token)
|
|
{
|
|
var exchange = new Exchange(url, "stream");
|
|
try
|
|
{
|
|
var response = await OpenMedia(url, range, exchange, token);
|
|
if (response == default && exchange.Outcome == Interactions.Ok)
|
|
exchange.Refused("unusable");
|
|
exchange.Bytes = response?.Content.Headers.ContentLength;
|
|
return response;
|
|
}
|
|
finally
|
|
{
|
|
Record(exchange);
|
|
}
|
|
}
|
|
|
|
async Task<HttpResponseMessage> OpenMedia(string url, System.Net.Http.Headers.RangeHeaderValue range, Exchange exchange, CancellationToken token)
|
|
{
|
|
if (!Uri.TryCreate(url, UriKind.Absolute, out var target) || !IsAllowed(target))
|
|
{
|
|
exchange.Refused("disallowed");
|
|
return default;
|
|
}
|
|
try
|
|
{
|
|
for (var hop = 0; hop <= MaxRedirects; hop++)
|
|
{
|
|
using var request = new HttpRequestMessage(HttpMethod.Get, target);
|
|
request.Headers.Accept.ParseAdd("video/*, audio/*, image/*");
|
|
request.Headers.Range = range;
|
|
var response = await _httpClientFactory.CreateClient(ClientName).SendAsync(request, HttpCompletionOption.ResponseHeadersRead, token);
|
|
exchange.Answered(response);
|
|
if (IsRedirect(response.StatusCode))
|
|
{
|
|
var location = response.Headers.Location;
|
|
response.Dispose();
|
|
var next = location == default ? default : location.IsAbsoluteUri ? location : new Uri(target, location);
|
|
if (!IsAllowed(next))
|
|
{
|
|
exchange.Refused("bad-redirect");
|
|
return default;
|
|
}
|
|
target = next;
|
|
exchange.Hops++;
|
|
continue;
|
|
}
|
|
var mediaType = response.Content.Headers.ContentType?.MediaType?.ToLowerInvariant();
|
|
if (!response.IsSuccessStatusCode || mediaType == default || mediaType.Contains("svg")
|
|
|| !(mediaType.StartsWith("video/") || mediaType.StartsWith("audio/") || mediaType.StartsWith("image/") || mediaType == "application/octet-stream"))
|
|
{
|
|
if (response.IsSuccessStatusCode)
|
|
exchange.Refused("content-type");
|
|
else
|
|
exchange.Failed(StatusReason(response));
|
|
response.Dispose();
|
|
return default;
|
|
}
|
|
return response;
|
|
}
|
|
return default;
|
|
}
|
|
catch (Exception ex) when (ex is HttpRequestException or BlockedDestinationException or OperationCanceledException && !token.IsCancellationRequested)
|
|
{
|
|
exchange.Failed(ex switch { BlockedDestinationException => "private-address", OperationCanceledException => "timeout", _ => "network" });
|
|
_logger.LogInformation("Media stream {Url} refused: {Reason}", url, ex.Message);
|
|
return default;
|
|
}
|
|
}
|
|
|
|
const int MaxPageBytes = 512 * 1024;
|
|
|
|
static async Task<byte[]> ReadPrefix(HttpContent content, int limit, CancellationToken token)
|
|
{
|
|
await using var stream = await content.ReadAsStreamAsync(token);
|
|
var buffer = new byte[limit];
|
|
var total = 0;
|
|
int read;
|
|
while (total < limit && (read = await stream.ReadAsync(buffer.AsMemory(total, limit - total), token)) > 0)
|
|
total += read;
|
|
return buffer[..total];
|
|
}
|
|
|
|
public async Task<HttpResponseMessage> Send(HttpRequestMessage request, CancellationToken token)
|
|
{
|
|
if (!IsAllowed(request.RequestUri))
|
|
throw new BlockedDestinationException(request.RequestUri?.Host);
|
|
return await _httpClientFactory.CreateClient(ClientName).SendAsync(request, HttpCompletionOption.ResponseHeadersRead, token);
|
|
}
|
|
|
|
public static async Task<byte[]> ReadBounded(HttpContent content, int limit, CancellationToken token)
|
|
{
|
|
await using var stream = await content.ReadAsStreamAsync(token);
|
|
using var buffer = new MemoryStream();
|
|
var chunk = new byte[16 * 1024];
|
|
int read;
|
|
while ((read = await stream.ReadAsync(chunk, token)) > 0)
|
|
{
|
|
if (buffer.Length + read > limit)
|
|
return default;
|
|
buffer.Write(chunk, 0, read);
|
|
}
|
|
return buffer.ToArray();
|
|
}
|
|
|
|
static bool IsRedirect(HttpStatusCode status) =>
|
|
status is HttpStatusCode.MovedPermanently or HttpStatusCode.Found or HttpStatusCode.SeeOther
|
|
or HttpStatusCode.TemporaryRedirect or HttpStatusCode.PermanentRedirect;
|
|
|
|
static string NegativeKey(Uri target) => "federation-http:refused:" + target.AbsoluteUri;
|
|
|
|
FetchedJson Refuse(string negativeKey, string url, string reason, Exchange exchange, string code, bool transient = false)
|
|
{
|
|
if (transient)
|
|
exchange.Failed(code);
|
|
else
|
|
exchange.Refused(code);
|
|
_cache.Set(negativeKey, transient, NegativeCacheLifetime);
|
|
_logger.LogInformation("GET {Url} refused: {Reason}", url, reason);
|
|
return default;
|
|
}
|
|
}
|
|
}
|