M11: the opt-in crawler, PrivaPub-Stargazer, and /stargazing

Off by default (Statistics:Crawler:Enabled), as the owner decided. When it is on:
- CrawlPlan runs hourly, from StatisticsSchedule. It inserts the configured seeds and
  queues up to HostsPerHour servers whose last visit is older than RevisitDays, are not
  paused by the breaker and are not domain-blocked, spread across the hour.
- CrawlInstance visits one server at a time as
  "PrivaPub-Stargazer/<ref> (+<base>/stargazing)":
  - it reads robots.txt (RFC 9309: its own group first, then PrivaPub, then *; longest
    rule wins; a 4xx allows everything; a 5xx or no answer keeps it out);
  - it describes servers that only crawling ever found, through
    InstanceDescriber.Describe with robots.txt as the path filter;
  - it reads /api/v1/instance/peers through the new GetStringArray, which keeps what it
    read from the first 1 MB instead of refusing a large list;
  - it adds the names a server could ever be reached at as "crawled": DNS only,
    punycode, no addresses, ports or hidden services, and the reserved test names only on
    a test network. Never more than MaxNewHostsPerCrawl per visit or MaxHosts in all, and
    never over a touched server.
- It reads nothing but robots.txt, NodeInfo, the instance API and the peers list.
  IFederationHttp.GetText serves robots.txt, and HttpScope.Crawl carries the
  User-Agent.
- /stargazing explains all this and how to keep the crawler out, says whether it is on,
  and credits DB-IP. GET /clientapi/admin/statistics/crawler shows the frontier.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01ELjqpznMFMNrJoJUj6K5p2

PrivaPub/Infrastructure/Http/FederationHttp.cs

@@ -28,6 +28,8 @@ namespace PrivaPub.Infrastructure.Http
 		Task<HttpResponseMessage> OpenMedia(string url, System.Net.Http.Headers.RangeHeaderValue range, CancellationToken token);
 		Task<HttpResponseMessage> Send(HttpRequestMessage request, CancellationToken token);
 		Task<(byte[] Bytes, string ContentType)> GetMedia(string url, long maxBytes, CancellationToken token);
+		Task<(int Status, string Text)> GetText(string url, int maxBytes, CancellationToken token);
+		Task<IReadOnlyList<string>> GetStringArray(string url, int maxItems, CancellationToken token);
 	}
 
 	public class FederationHttp : IFederationHttp
@@ -114,6 +116,152 @@ namespace PrivaPub.Infrastructure.Http
 			});
 		}
 
+		static HttpRequestMessage Request(Uri target, string accept)
+		{
+			var request = new HttpRequestMessage(HttpMethod.Get, target);
+			request.Headers.Accept.ParseAdd(accept);
+			if (HttpScope.UserAgent is { } userAgent)
+				request.Headers.UserAgent.ParseAdd(userAgent);
+			return request;
+		}
+
+		//robots.txt: the status alone matters when it is not 2xx (RFC 9309), so it is returned whatever it is; 0 when unreachable
+		public async Task<(int Status, string Text)> GetText(string url, int maxBytes, CancellationToken token)
+		{
+			var exchange = new Exchange(url, HttpScope.Purpose ?? "text");
+			try
+			{
+				if (!Uri.TryCreate(url, UriKind.Absolute, out var target) || !IsAllowed(target))
+				{
+					exchange.Refused("disallowed");
+					return default;
+				}
+				using var timeout = CancellationTokenSource.CreateLinkedTokenSource(token);
+				timeout.CancelAfter(RequestTimeout);
+				for (var hop = 0; hop <= MaxRedirects; hop++)
+				{
+					using var request = Request(target, "text/plain");
+					using var response = await _httpClientFactory.CreateClient(ClientName).SendAsync(request, HttpCompletionOption.ResponseHeadersRead, timeout.Token);
+					exchange.Answered(response);
+					if (IsRedirect(response.StatusCode))
+					{
+						var location = response.Headers.Location;
+						var next = location == default ? default : location.IsAbsoluteUri ? location : new Uri(target, location);
+						if (!IsAllowed(next))
+						{
+							exchange.Refused("bad-redirect");
+							return ((int)response.StatusCode, default);
+						}
+						target = next;
+						exchange.Hops++;
+						continue;
+					}
+					if (!response.IsSuccessStatusCode)
+					{
+						exchange.Refused(StatusReason(response));
+						return ((int)response.StatusCode, default);
+					}
+					var bytes = await ReadPrefix(response.Content, maxBytes, timeout.Token);
+					exchange.Bytes = bytes.Length;
+					return ((int)response.StatusCode, System.Text.Encoding.UTF8.GetString(bytes));
+				}
+				exchange.Refused("too-many-redirects");
+				return default;
+			}
+			catch (Exception ex) when (ex is HttpRequestException or BlockedDestinationException or OperationCanceledException && !token.IsCancellationRequested)
+			{
+				exchange.Failed(ex switch { BlockedDestinationException => "private-address", OperationCanceledException => "timeout", _ => "network" });
+				return default;
+			}
+			finally
+			{
+				Record(exchange);
+			}
+		}
+
+		//a JSON array of strings such as /api/v1/instance/peers, read from at most the first MaxResponseBytes: a large server's
+		//list is cut short rather than refused
+		public async Task<IReadOnlyList<string>> GetStringArray(string url, int maxItems, CancellationToken token)
+		{
+			var exchange = new Exchange(url, HttpScope.Purpose ?? "list");
+			var items = new List<string>();
+			try
+			{
+				if (!Uri.TryCreate(url, UriKind.Absolute, out var target) || !IsAllowed(target))
+				{
+					exchange.Refused("disallowed");
+					return items;
+				}
+				using var timeout = CancellationTokenSource.CreateLinkedTokenSource(token);
+				timeout.CancelAfter(RequestTimeout);
+				for (var hop = 0; hop <= MaxRedirects; hop++)
+				{
+					using var request = Request(target, "application/json");
+					using var response = await _httpClientFactory.CreateClient(ClientName).SendAsync(request, HttpCompletionOption.ResponseHeadersRead, timeout.Token);
+					exchange.Answered(response);
+					if (IsRedirect(response.StatusCode))
+					{
+						var location = response.Headers.Location;
+						var next = location == default ? default : location.IsAbsoluteUri ? location : new Uri(target, location);
+						if (!IsAllowed(next))
+						{
+							exchange.Refused("bad-redirect");
+							return items;
+						}
+						target = next;
+						exchange.Hops++;
+						continue;
+					}
+					if (!response.IsSuccessStatusCode)
+					{
+						exchange.Refused(StatusReason(response));
+						return items;
+					}
+					var mediaType = response.Content.Headers.ContentType?.MediaType;
+					if (mediaType == default || !JsonMediaTypes.Contains(mediaType, StringComparer.OrdinalIgnoreCase))
+					{
+						exchange.Refused("content-type");
+						return items;
+					}
+					var bytes = await ReadPrefix(response.Content, MaxResponseBytes, timeout.Token);
+					exchange.Bytes = bytes.Length;
+					ReadStrings(bytes, maxItems, items);
+					return items;
+				}
+				exchange.Refused("too-many-redirects");
+				return items;
+			}
+			catch (Exception ex) when (ex is HttpRequestException or BlockedDestinationException or OperationCanceledException && !token.IsCancellationRequested)
+			{
+				exchange.Failed(ex switch { BlockedDestinationException => "private-address", OperationCanceledException => "timeout", _ => "network" });
+				return items;
+			}
+			finally
+			{
+				Record(exchange);
+			}
+		}
+
+		public static void ReadStrings(byte[] json, int maxItems, List<string> items)
+		{
+			var reader = new Utf8JsonReader(json, isFinalBlock: false, state: default);
+			try
+			{
+				if (!reader.Read() || reader.TokenType != JsonTokenType.StartArray)
+					return;
+				while (items.Count < maxItems && reader.Read() && reader.TokenType != JsonTokenType.EndArray)
+				{
+					if (reader.TokenType == JsonTokenType.String)
+						items.Add(reader.GetString());
+					else if (!reader.TrySkip())
+						return;
+				}
+			}
+			catch (JsonException)
+			{
+			}
+		}
+
 		static string StatusReason(HttpResponseMessage response) => ((int)response.StatusCode).ToString(System.Globalization.CultureInfo.InvariantCulture);
 
 		public bool IsAllowed(Uri target)
@@ -167,8 +315,7 @@ namespace PrivaPub.Infrastructure.Http
 			{
 				for (var hop = 0; hop <= MaxRedirects; hop++)
 				{
-					using var request = new HttpRequestMessage(HttpMethod.Get, target);
-					request.Headers.Accept.ParseAdd(accept);
+					using var request = Request(target, accept);
 					sign?.Invoke(request);
 
 					using var response = await _httpClientFactory.CreateClient(ClientName)

PrivaPub/Infrastructure/Http/HttpScope.cs

@@ -4,13 +4,14 @@ namespace PrivaPub.Infrastructure.Http
 	{
 		public const string Request = "request";
 
-		sealed record Tags(string Purpose, string Trigger, bool Crawl);
+		sealed record Tags(string Purpose, string Trigger, bool Crawl, string UserAgent = default);
 
 		static readonly AsyncLocal<Tags> current = new();
 
 		public static string Purpose => current.Value?.Purpose;
 		public static string Trigger => current.Value?.Trigger ?? Request;
 		public static bool Crawling => current.Value?.Crawl == true;
+		public static string UserAgent => current.Value?.UserAgent;
 
 		public static IDisposable For(string purpose) => Push(tags => tags with { Purpose = purpose });
 
@@ -18,7 +19,7 @@ namespace PrivaPub.Infrastructure.Http
 
 		public static IDisposable Triggered(string trigger) => Push(tags => tags with { Trigger = trigger });
 
-		public static IDisposable Crawl() => Push(tags => tags with { Crawl = true });
+		public static IDisposable Crawl(string userAgent = default) => Push(tags => tags with { Crawl = true, UserAgent = userAgent ?? tags.UserAgent });
 
 		static IDisposable Push(Func<Tags, Tags> change)
 		{