M11: the opt-in crawler, PrivaPub-Stargazer, and /stargazing

Off by default (Statistics:Crawler:Enabled), as the owner decided. When it is on:
- CrawlPlan runs hourly, from StatisticsSchedule. It inserts the configured seeds and
  queues up to HostsPerHour servers whose last visit is older than RevisitDays, are not
  paused by the breaker and are not domain-blocked, spread across the hour.
- CrawlInstance visits one server at a time as
  "PrivaPub-Stargazer/<ref> (+<base>/stargazing)":
  - it reads robots.txt (RFC 9309: its own group first, then PrivaPub, then *; longest
    rule wins; a 4xx allows everything; a 5xx or no answer keeps it out);
  - it describes servers that only crawling ever found, through
    InstanceDescriber.Describe with robots.txt as the path filter;
  - it reads /api/v1/instance/peers through the new GetStringArray, which keeps what it
    read from the first 1 MB instead of refusing a large list;
  - it adds the names a server could ever be reached at as "crawled": DNS only,
    punycode, no addresses, ports or hidden services, and the reserved test names only on
    a test network. Never more than MaxNewHostsPerCrawl per visit or MaxHosts in all, and
    never over a touched server.
- It reads nothing but robots.txt, NodeInfo, the instance API and the peers list.
  IFederationHttp.GetText serves robots.txt, and HttpScope.Crawl carries the
  User-Agent.
- /stargazing explains all this and how to keep the crawler out, says whether it is on,
  and credits DB-IP. GET /clientapi/admin/statistics/crawler shows the frontier.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01ELjqpznMFMNrJoJUj6K5p2

PrivaPub/Controllers/ClientToServer/StatisticsController.cs

@@ -18,13 +18,19 @@ namespace PrivaPub.Controllers.ClientToServer
 	{
 		readonly StatisticsQueries _queries;
 		readonly IJobQueue _queue;
+		readonly Microsoft.Extensions.Options.IOptionsMonitor<StatisticsOptions> _options;
 
-		public StatisticsController(StatisticsQueries queries, IJobQueue queue)
+		public StatisticsController(StatisticsQueries queries, IJobQueue queue, Microsoft.Extensions.Options.IOptionsMonitor<StatisticsOptions> options)
 		{
 			_queries = queries;
 			_queue = queue;
+			_options = options;
 		}
 
+		[HttpGet, Route("/clientapi/admin/statistics/crawler")]
+		public async Task<IActionResult> Crawler(CancellationToken token) =>
+			Ok(await _queries.Crawler(_options.CurrentValue.Crawler.Enabled, _options.CurrentValue.Crawler.RevisitDays, token));
+
 		[HttpGet, Route("/clientapi/admin/statistics/overview")]
 		public async Task<IActionResult> Overview([FromQuery] int days = 30, CancellationToken token = default) =>
 			Ok(await _queries.Overview(days, token));

PrivaPub/Domain/Statistics/StatisticsQueries.cs

@@ -147,6 +147,25 @@ namespace PrivaPub.Domain.Statistics
 				.ExecuteAsync(token);
 		}
 
+		public async Task<object> Crawler(bool enabled, int revisitDays, CancellationToken token)
+		{
+			var due = DateTime.UtcNow.AddDays(-Math.Max(revisitDays, 1));
+			var recent = await DB.Default.Find<RemoteInstance>()
+				.Match(i => i.LastCrawledAt != null)
+				.Sort(i => i.LastCrawledAt, Order.Descending)
+				.Limit(20)
+				.ExecuteAsync(token);
+			return new
+			{
+				Enabled = enabled,
+				Touched = await DB.Default.CountAsync<RemoteInstance>(i => i.Seen == "touched", token),
+				Crawled = await DB.Default.CountAsync<RemoteInstance>(i => i.Seen == "crawled", token),
+				Due = await DB.Default.CountAsync<RemoteInstance>(i => i.LastCrawledAt == null || i.LastCrawledAt < due, token),
+				RobotsDisallowed = await DB.Default.CountAsync<RemoteInstance>(i => i.RobotsDisallowed == true, token),
+				Recent = recent.Select(i => new { i.Host, i.Seen, i.LastCrawledAt, i.PeersCount, i.RobotsDisallowed, i.CrawlError, i.Software })
+			};
+		}
+
 		public async Task<List<ServerDay>> Server(int days, CancellationToken token)
 		{
 			var since = DateTime.UtcNow.Date.AddDays(1 - Math.Clamp(days, 1, 3660));

PrivaPub/Federation/Crawler/Crawler.cs

@@ -0,0 +1,206 @@
+using Microsoft.Extensions.Options;
+
+using MongoDB.Entities;
+
+using PrivaPub.Federation.Moderation;
+using PrivaPub.Federation.Objects;
+using PrivaPub.Infrastructure.Http;
+using PrivaPub.Infrastructure.Jobs;
+using PrivaPub.Infrastructure.Statistics;
+using PrivaPub.Models;
+using PrivaPub.Models.Jobs;
+using PrivaPub.Models.Statistics;
+using PrivaPub.StaticServices;
+
+namespace PrivaPub.Federation.Crawler
+{
+	public static class Stargazer
+	{
+		public const string Token = "PrivaPub-Stargazer";
+		public const string Page = "/stargazing";
+
+		public static string UserAgent(string baseAddress) => $"{Token}/{BuildInfo.Ref} (+{baseAddress?.TrimEnd('/')}{Page})";
+	}
+
+	//each hour, the servers due for a visit, spread across the hour so the crawler reaches one at a time
+	public class CrawlPlanner : IJobHandler
+	{
+		readonly IJobQueue _queue;
+		readonly IDomainBlocks _blocks;
+		readonly IOptionsMonitor<StatisticsOptions> _options;
+
+		public CrawlPlanner(IJobQueue queue, IDomainBlocks blocks, IOptionsMonitor<StatisticsOptions> options)
+		{
+			_queue = queue;
+			_blocks = blocks;
+			_options = options;
+		}
+
+		public JobKind Kind => JobKind.CrawlPlan;
+		public int Concurrency => 1;
+		public int MaxAttempts => 1;
+		public int PerHostLimit => 1;
+
+		public static Job For(DateTime hour) => new()
+		{
+			Kind = JobKind.CrawlPlan,
+			Payload = hour.ToString("yyyy-MM-ddTHH", System.Globalization.CultureInfo.InvariantCulture),
+			DedupeKey = "crawl-plan|" + hour.ToString("yyyy-MM-ddTHH", System.Globalization.CultureInfo.InvariantCulture),
+			RunAt = hour
+		};
+
+		public async Task<JobOutcome> Handle(Job job, CancellationToken token)
+		{
+			var crawler = _options.CurrentValue.Crawler;
+			if (!crawler.Enabled)
+				return JobOutcome.Done;
+			var now = DateTime.UtcNow;
+			foreach (var seed in PeerHosts.Filter(crawler.Seeds ?? new(), host => !_blocks.IsSuspended(host)))
+				await DB.Default.Update<RemoteInstance>()
+					.Match(i => i.Host == seed)
+					.Modify(b => b.SetOnInsert(i => i.Seen, "crawled"))
+					.Modify(b => b.SetOnInsert(i => i.FirstSeenAt, now))
+					.Option(o => o.IsUpsert = true)
+					.ExecuteAsync(token);
+
+			var due = now.AddDays(-Math.Max(crawler.RevisitDays, 1));
+			var perHour = Math.Clamp(crawler.HostsPerHour, 1, 600);
+			var candidates = await DB.Default.Find<RemoteInstance>()
+				.Match(i => (i.LastCrawledAt == null || i.LastCrawledAt < due) && (i.UnavailableUntil == null || i.UnavailableUntil < now))
+				.Sort(i => i.LastCrawledAt, Order.Ascending)
+				.Limit(perHour * 3)
+				.ExecuteAsync(token);
+			var week = InstanceDescriber.Week(now);
+			var jobs = candidates.Where(i => _blocks.Find(i.Host) == default && PeerHosts.Normalise(i.Host) != default)
+				.Take(perHour)
+				.Select((instance, index) => new Job
+				{
+					Kind = JobKind.CrawlInstance,
+					Payload = instance.Host,
+					Host = instance.Host,
+					DedupeKey = $"crawl|{instance.Host}|{week}",
+					RunAt = now.AddSeconds(index * 3600.0 / perHour)
+				});
+			await _queue.EnqueueMany(jobs, token);
+			return JobOutcome.Done;
+		}
+	}
+
+	//one server: its robots.txt, a description if only crawling ever found it, then the servers its peers list names
+	public class InstanceCrawler : IJobHandler
+	{
+		const int MaxPeers = 20_000;
+		const int MaxRobotsBytes = 64 * 1024;
+
+		readonly IFederationHttp _http;
+		readonly InstanceDescriber _describer;
+		readonly IDomainBlocks _blocks;
+		readonly IOptionsMonitor<StatisticsOptions> _options;
+		readonly IOptionsMonitor<AppConfiguration> _app;
+		readonly IOptionsMonitor<FederationOptions> _federation;
+		readonly IInteractionLedger _ledger;
+
+		public InstanceCrawler(IFederationHttp http, InstanceDescriber describer, IDomainBlocks blocks, IOptionsMonitor<StatisticsOptions> options,
+			IOptionsMonitor<AppConfiguration> app, IOptionsMonitor<FederationOptions> federation, IInteractionLedger ledger = default)
+		{
+			_http = http;
+			_describer = describer;
+			_blocks = blocks;
+			_options = options;
+			_app = app;
+			_federation = federation;
+			_ledger = ledger;
+		}
+
+		public JobKind Kind => JobKind.CrawlInstance;
+		public int Concurrency => 1;
+		public int MaxAttempts => 1;
+		public int PerHostLimit => 1;
+
+		protected virtual string Address(string url) => url;
+
+		public async Task<JobOutcome> Handle(Job job, CancellationToken token)
+		{
+			var crawler = _options.CurrentValue.Crawler;
+			var host = job.Payload;
+			if (!crawler.Enabled || _blocks.Find(host) != default)
+				return JobOutcome.Done;
+			using var crawl = HttpScope.Crawl(Stargazer.UserAgent(_app.CurrentValue.BackendBaseAddress));
+
+			RobotsTxt robots;
+			using (HttpScope.For("robots"))
+			{
+				var (status, text) = await _http.GetText(Address($"https://{host}/robots.txt"), MaxRobotsBytes, token);
+				robots = RobotsTxt.From(status, text, Stargazer.Token, "PrivaPub");
+			}
+			var instance = await DB.Default.Find<RemoteInstance>().Match(i => i.Host == host).ExecuteFirstAsync(token);
+			var now = DateTime.UtcNow;
+			if (!robots.Allows("/.well-known/nodeinfo") && !robots.Allows("/api/v1/instance/peers"))
+			{
+				await Visited(host, robotsDisallowed: true, peers: default, "robots.txt keeps us out", token);
+				Record(host, Interactions.Refused, "robots-disallowed");
+				return JobOutcome.Done;
+			}
+
+			var reachable = true;
+			if (instance?.Seen != "touched")
+				reachable = await _describer.Describe(host, crawled: true, robots.Allows, token);
+
+			int? peersCount = default;
+			if (robots.Allows("/api/v1/instance/peers"))
+			{
+				IReadOnlyList<string> peers;
+				using (HttpScope.For("peers"))
+					peers = await _http.GetStringArray(Address($"https://{host}/api/v1/instance/peers"), MaxPeers, token);
+				peersCount = peers.Count;
+				await Discover(peers, crawler, token);
+			}
+			await Visited(host, robotsDisallowed: false, peersCount, reachable ? default : "no NodeInfo", token);
+			Record(host, reachable ? Interactions.Ok : Interactions.Failed, reachable ? default : "unreachable");
+			return JobOutcome.Done;
+		}
+
+		async Task Discover(IReadOnlyList<string> peers, CrawlerOptions crawler, CancellationToken token)
+		{
+			var room = Math.Min(crawler.MaxNewHostsPerCrawl, crawler.MaxHosts - (int)await DB.Default.CountAsync<RemoteInstance>(i => i.Seen == "crawled", token));
+			if (room <= 0)
+				return;
+			var candidates = PeerHosts.Filter(peers, host => _http.IsAllowed(new Uri($"https://{host}/")) && _blocks.Find(host) == default,
+				_federation.CurrentValue.AllowPrivateNetworks).ToList();
+			var now = DateTime.UtcNow;
+			foreach (var batch in candidates.Chunk(500))
+			{
+				if (room <= 0)
+					return;
+				var known = (await DB.Default.Find<RemoteInstance, string>().Match(i => batch.Contains(i.Host)).Project(i => i.Host).ExecuteAsync(token)).ToHashSet();
+				foreach (var host in batch.Where(h => !known.Contains(h)).Take(room))
+				{
+					await DB.Default.Update<RemoteInstance>()
+						.Match(i => i.Host == host)
+						.Modify(b => b.SetOnInsert(i => i.Seen, "crawled"))
+						.Modify(b => b.SetOnInsert(i => i.FirstSeenAt, now))
+						.Option(o => o.IsUpsert = true)
+						.ExecuteAsync(token);
+					room--;
+				}
+			}
+		}
+
+		static async Task Visited(string host, bool robotsDisallowed, int? peers, string error, CancellationToken token)
+		{
+			var update = DB.Default.Update<RemoteInstance>()
+				.Match(i => i.Host == host)
+				.Modify(i => i.LastCrawledAt, DateTime.UtcNow)
+				.Modify(i => i.RobotsDisallowed, robotsDisallowed)
+				.Modify(i => i.CrawlError, error)
+				.Modify(b => b.SetOnInsert(i => i.Seen, "crawled"))
+				.Option(o => o.IsUpsert = true);
+			if (peers is { } count)
+				update = update.Modify(i => i.PeersCount, count);
+			await update.ExecuteAsync(token);
+		}
+
+		void Record(string host, string outcome, string reason) =>
+			_ledger?.Record(new InteractionEvent { Channel = Interactions.Crawl, Host = host, Outcome = outcome, Reason = reason, Crawl = true });
+	}
+}

PrivaPub/Federation/Crawler/PeerHosts.cs

@@ -0,0 +1,37 @@
+using System.Globalization;
+
+namespace PrivaPub.Federation.Crawler
+{
+	public static class PeerHosts
+	{
+		static readonly IdnMapping Idn = new();
+		static readonly string[] Unreachable = { ".onion", ".i2p", ".local", ".localhost", ".internal" };
+		static readonly string[] Reserved = { ".test", ".invalid", ".example" };
+
+		//names from a peers list we could ever talk to: DNS names, ASCII (punycode), no addresses, ports or hidden services;
+		//the reserved test names only on a test network
+		public static IEnumerable<string> Filter(IEnumerable<string> names, Func<string, bool> allowed, bool testNetwork = false) =>
+			names
+				.Select(Normalise)
+				.Where(host => host != default && (testNetwork || !Reserved.Any(suffix => host.EndsWith(suffix, StringComparison.Ordinal))) && allowed(host))
+				.Distinct(StringComparer.Ordinal);
+
+		public static string Normalise(string name)
+		{
+			var host = name?.Trim().TrimEnd('.').ToLowerInvariant();
+			if (string.IsNullOrEmpty(host) || host.Length > 253 || !host.Contains('.') || host.Contains(':') || host.Contains('/'))
+				return default;
+			try
+			{
+				host = Idn.GetAscii(host);
+			}
+			catch (ArgumentException)
+			{
+				return default;
+			}
+			if (Uri.CheckHostName(host) != UriHostNameType.Dns || Unreachable.Any(suffix => host.EndsWith(suffix, StringComparison.Ordinal)))
+				return default;
+			return host;
+		}
+	}
+}

PrivaPub/Federation/Crawler/RobotsTxt.cs

@@ -0,0 +1,90 @@
+using System.Text.RegularExpressions;
+
+namespace PrivaPub.Federation.Crawler
+{
+	//RFC 9309: the most specific group naming the crawler (or "*"), the longest matching rule, Allow winning a tie.
+	//A 4xx means no rules at all; a 5xx or no answer means keep out.
+	public sealed class RobotsTxt
+	{
+		sealed record Rule(bool Allow, string Pattern, Regex Match);
+
+		readonly List<Rule> _rules;
+		readonly bool _everythingDisallowed;
+
+		RobotsTxt(List<Rule> rules, bool everythingDisallowed)
+		{
+			_rules = rules;
+			_everythingDisallowed = everythingDisallowed;
+		}
+
+		public static readonly RobotsTxt AllowAll = new(new List<Rule>(), false);
+		public static readonly RobotsTxt DisallowAll = new(new List<Rule>(), true);
+
+		public static RobotsTxt From(int status, string text, params string[] agents) => status switch
+		{
+			>= 200 and < 300 => Parse(text, agents),
+			>= 400 and < 500 => AllowAll,
+			_ => DisallowAll
+		};
+
+		public static RobotsTxt Parse(string text, params string[] agents)
+		{
+			var groups = new List<(List<string> Agents, List<Rule> Rules)>();
+			(List<string> Agents, List<Rule> Rules) current = default;
+			var collectingAgents = false;
+			foreach (var raw in (text ?? string.Empty).Split('\n'))
+			{
+				var line = raw.Split('#')[0].Trim();
+				var colon = line.IndexOf(':');
+				if (colon <= 0)
+					continue;
+				var field = line[..colon].Trim().ToLowerInvariant();
+				var value = line[(colon + 1)..].Trim();
+				if (field == "user-agent")
+				{
+					if (!collectingAgents)
+					{
+						current = (new List<string>(), new List<Rule>());
+						groups.Add(current);
+						collectingAgents = true;
+					}
+					current.Agents.Add(value.ToLowerInvariant());
+					continue;
+				}
+				if (field is not ("allow" or "disallow") || current.Agents == default)
+					continue;
+				collectingAgents = false;
+				if (value.Length == 0)
+					continue;
+				current.Rules.Add(new Rule(field == "allow", value, Pattern(value)));
+			}
+
+			foreach (var agent in agents.Select(a => a.ToLowerInvariant()))
+			{
+				var named = groups.Where(g => g.Agents.Contains(agent)).SelectMany(g => g.Rules).ToList();
+				if (groups.Any(g => g.Agents.Contains(agent)))
+					return new RobotsTxt(named, false);
+			}
+			return new RobotsTxt(groups.Where(g => g.Agents.Contains("*")).SelectMany(g => g.Rules).ToList(), false);
+		}
+
+		public bool Allows(string path)
+		{
+			if (_everythingDisallowed)
+				return false;
+			path = string.IsNullOrEmpty(path) ? "/" : path;
+			Rule best = default;
+			foreach (var rule in _rules.Where(r => r.Match.IsMatch(path)))
+				if (best == default || rule.Pattern.Length > best.Pattern.Length || rule.Pattern.Length == best.Pattern.Length && rule.Allow)
+					best = rule;
+			return best?.Allow != false;
+		}
+
+		static Regex Pattern(string value)
+		{
+			var anchored = value.EndsWith('$');
+			var body = Regex.Escape(anchored ? value[..^1] : value).Replace("\\*", ".*");
+			return new Regex("^" + body + (anchored ? "$" : string.Empty), RegexOptions.CultureInvariant, TimeSpan.FromMilliseconds(100));
+		}
+	}
+}

PrivaPub/Infrastructure/Http/FederationHttp.cs

@@ -28,6 +28,8 @@ namespace PrivaPub.Infrastructure.Http
 		Task<HttpResponseMessage> OpenMedia(string url, System.Net.Http.Headers.RangeHeaderValue range, CancellationToken token);
 		Task<HttpResponseMessage> Send(HttpRequestMessage request, CancellationToken token);
 		Task<(byte[] Bytes, string ContentType)> GetMedia(string url, long maxBytes, CancellationToken token);
+		Task<(int Status, string Text)> GetText(string url, int maxBytes, CancellationToken token);
+		Task<IReadOnlyList<string>> GetStringArray(string url, int maxItems, CancellationToken token);
 	}
 
 	public class FederationHttp : IFederationHttp
@@ -114,6 +116,152 @@ namespace PrivaPub.Infrastructure.Http
 			});
 		}
 
+		static HttpRequestMessage Request(Uri target, string accept)
+		{
+			var request = new HttpRequestMessage(HttpMethod.Get, target);
+			request.Headers.Accept.ParseAdd(accept);
+			if (HttpScope.UserAgent is { } userAgent)
+				request.Headers.UserAgent.ParseAdd(userAgent);
+			return request;
+		}
+
+		//robots.txt: the status alone matters when it is not 2xx (RFC 9309), so it is returned whatever it is; 0 when unreachable
+		public async Task<(int Status, string Text)> GetText(string url, int maxBytes, CancellationToken token)
+		{
+			var exchange = new Exchange(url, HttpScope.Purpose ?? "text");
+			try
+			{
+				if (!Uri.TryCreate(url, UriKind.Absolute, out var target) || !IsAllowed(target))
+				{
+					exchange.Refused("disallowed");
+					return default;
+				}
+				using var timeout = CancellationTokenSource.CreateLinkedTokenSource(token);
+				timeout.CancelAfter(RequestTimeout);
+				for (var hop = 0; hop <= MaxRedirects; hop++)
+				{
+					using var request = Request(target, "text/plain");
+					using var response = await _httpClientFactory.CreateClient(ClientName).SendAsync(request, HttpCompletionOption.ResponseHeadersRead, timeout.Token);
+					exchange.Answered(response);
+					if (IsRedirect(response.StatusCode))
+					{
+						var location = response.Headers.Location;
+						var next = location == default ? default : location.IsAbsoluteUri ? location : new Uri(target, location);
+						if (!IsAllowed(next))
+						{
+							exchange.Refused("bad-redirect");
+							return ((int)response.StatusCode, default);
+						}
+						target = next;
+						exchange.Hops++;
+						continue;
+					}
+					if (!response.IsSuccessStatusCode)
+					{
+						exchange.Refused(StatusReason(response));
+						return ((int)response.StatusCode, default);
+					}
+					var bytes = await ReadPrefix(response.Content, maxBytes, timeout.Token);
+					exchange.Bytes = bytes.Length;
+					return ((int)response.StatusCode, System.Text.Encoding.UTF8.GetString(bytes));
+				}
+				exchange.Refused("too-many-redirects");
+				return default;
+			}
+			catch (Exception ex) when (ex is HttpRequestException or BlockedDestinationException or OperationCanceledException && !token.IsCancellationRequested)
+			{
+				exchange.Failed(ex switch { BlockedDestinationException => "private-address", OperationCanceledException => "timeout", _ => "network" });
+				return default;
+			}
+			finally
+			{
+				Record(exchange);
+			}
+		}
+
+		//a JSON array of strings such as /api/v1/instance/peers, read from at most the first MaxResponseBytes: a large server's
+		//list is cut short rather than refused
+		public async Task<IReadOnlyList<string>> GetStringArray(string url, int maxItems, CancellationToken token)
+		{
+			var exchange = new Exchange(url, HttpScope.Purpose ?? "list");
+			var items = new List<string>();
+			try
+			{
+				if (!Uri.TryCreate(url, UriKind.Absolute, out var target) || !IsAllowed(target))
+				{
+					exchange.Refused("disallowed");
+					return items;
+				}
+				using var timeout = CancellationTokenSource.CreateLinkedTokenSource(token);
+				timeout.CancelAfter(RequestTimeout);
+				for (var hop = 0; hop <= MaxRedirects; hop++)
+				{
+					using var request = Request(target, "application/json");
+					using var response = await _httpClientFactory.CreateClient(ClientName).SendAsync(request, HttpCompletionOption.ResponseHeadersRead, timeout.Token);
+					exchange.Answered(response);
+					if (IsRedirect(response.StatusCode))
+					{
+						var location = response.Headers.Location;
+						var next = location == default ? default : location.IsAbsoluteUri ? location : new Uri(target, location);
+						if (!IsAllowed(next))
+						{
+							exchange.Refused("bad-redirect");
+							return items;
+						}
+						target = next;
+						exchange.Hops++;
+						continue;
+					}
+					if (!response.IsSuccessStatusCode)
+					{
+						exchange.Refused(StatusReason(response));
+						return items;
+					}
+					var mediaType = response.Content.Headers.ContentType?.MediaType;
+					if (mediaType == default || !JsonMediaTypes.Contains(mediaType, StringComparer.OrdinalIgnoreCase))
+					{
+						exchange.Refused("content-type");
+						return items;
+					}
+					var bytes = await ReadPrefix(response.Content, MaxResponseBytes, timeout.Token);
+					exchange.Bytes = bytes.Length;
+					ReadStrings(bytes, maxItems, items);
+					return items;
+				}
+				exchange.Refused("too-many-redirects");
+				return items;
+			}
+			catch (Exception ex) when (ex is HttpRequestException or BlockedDestinationException or OperationCanceledException && !token.IsCancellationRequested)
+			{
+				exchange.Failed(ex switch { BlockedDestinationException => "private-address", OperationCanceledException => "timeout", _ => "network" });
+				return items;
+			}
+			finally
+			{
+				Record(exchange);
+			}
+		}
+
+		public static void ReadStrings(byte[] json, int maxItems, List<string> items)
+		{
+			var reader = new Utf8JsonReader(json, isFinalBlock: false, state: default);
+			try
+			{
+				if (!reader.Read() || reader.TokenType != JsonTokenType.StartArray)
+					return;
+				while (items.Count < maxItems && reader.Read() && reader.TokenType != JsonTokenType.EndArray)
+				{
+					if (reader.TokenType == JsonTokenType.String)
+						items.Add(reader.GetString());
+					else if (!reader.TrySkip())
+						return;
+				}
+			}
+			catch (JsonException)
+			{
+			}
+		}
+
 		static string StatusReason(HttpResponseMessage response) => ((int)response.StatusCode).ToString(System.Globalization.CultureInfo.InvariantCulture);
 
 		public bool IsAllowed(Uri target)
@@ -167,8 +315,7 @@ namespace PrivaPub.Infrastructure.Http
 			{
 				for (var hop = 0; hop <= MaxRedirects; hop++)
 				{
-					using var request = new HttpRequestMessage(HttpMethod.Get, target);
-					request.Headers.Accept.ParseAdd(accept);
+					using var request = Request(target, accept);
 					sign?.Invoke(request);
 
 					using var response = await _httpClientFactory.CreateClient(ClientName)

PrivaPub/Infrastructure/Http/HttpScope.cs

@@ -4,13 +4,14 @@ namespace PrivaPub.Infrastructure.Http
 	{
 		public const string Request = "request";
 
-		sealed record Tags(string Purpose, string Trigger, bool Crawl);
+		sealed record Tags(string Purpose, string Trigger, bool Crawl, string UserAgent = default);
 
 		static readonly AsyncLocal<Tags> current = new();
 
 		public static string Purpose => current.Value?.Purpose;
 		public static string Trigger => current.Value?.Trigger ?? Request;
 		public static bool Crawling => current.Value?.Crawl == true;
+		public static string UserAgent => current.Value?.UserAgent;
 
 		public static IDisposable For(string purpose) => Push(tags => tags with { Purpose = purpose });
 
@@ -18,7 +19,7 @@ namespace PrivaPub.Infrastructure.Http
 
 		public static IDisposable Triggered(string trigger) => Push(tags => tags with { Trigger = trigger });
 
-		public static IDisposable Crawl() => Push(tags => tags with { Crawl = true });
+		public static IDisposable Crawl(string userAgent = default) => Push(tags => tags with { Crawl = true, UserAgent = userAgent ?? tags.UserAgent });
 
 		static IDisposable Push(Func<Tags, Tags> change)
 		{

PrivaPub/Infrastructure/Statistics/RollupJob.cs

@@ -1,3 +1,5 @@
+using Microsoft.Extensions.Options;
+
 using MongoDB.Driver;
 using MongoDB.Entities;
 
@@ -100,11 +102,13 @@ namespace PrivaPub.Infrastructure.Statistics
 
 		readonly IJobQueue _queue;
 		readonly ILogger<StatisticsSchedule> _logger;
+		readonly IOptionsMonitor<StatisticsOptions> _options;
 
-		public StatisticsSchedule(IJobQueue queue, ILogger<StatisticsSchedule> logger)
+		public StatisticsSchedule(IJobQueue queue, ILogger<StatisticsSchedule> logger, IOptionsMonitor<StatisticsOptions> options = default)
 		{
 			_queue = queue;
 			_logger = logger;
+			_options = options;
 		}
 
 		protected override async Task ExecuteAsync(CancellationToken stoppingToken)
@@ -142,6 +146,11 @@ namespace PrivaPub.Infrastructure.Statistics
 				if (!rolled && await DB.Default.Find<InteractionEvent>().Match(e => e.At >= day && e.At < end).ExecuteAnyAsync(token))
 					jobs.Add(RollupJob.For(day));
 			}
+			if (_options?.CurrentValue.Crawler.Enabled == true)
+			{
+				var now = DateTime.UtcNow;
+				jobs.Add(Federation.Crawler.CrawlPlanner.For(new DateTime(now.Year, now.Month, now.Day, now.Hour, 0, 0, DateTimeKind.Utc)));
+			}
 			await _queue.EnqueueMany(jobs, token);
 		}
 	}

PrivaPub/Infrastructure/Statistics/StatisticsOptions.cs

@@ -5,5 +5,17 @@ namespace PrivaPub.Infrastructure.Statistics
 		public bool Enabled { get; set; } = true;
 		public string GeoDirectory { get; set; } = "/var/lib/privapub/geo";
 		public int PublicCityMinUsers { get; set; } = 10;
+		public CrawlerOptions Crawler { get; set; } = new();
+	}
+
+	//owner decision: off by default; when on, one server a minute, each at most weekly, at most MaxHosts servers
+	public class CrawlerOptions
+	{
+		public bool Enabled { get; set; }
+		public int MaxHosts { get; set; } = 5000;
+		public int HostsPerHour { get; set; } = 60;
+		public int RevisitDays { get; set; } = 7;
+		public int MaxNewHostsPerCrawl { get; set; } = 500;
+		public List<string> Seeds { get; set; } = new();
 	}
 }

PrivaPub/Middleware/SocialPubConfigurations.cs

@@ -99,7 +99,8 @@ namespace PrivaPub.Middleware
 				.AddSingleton<IQuoteService, QuoteService>()
 				.AddSingleton<IJobHandler, PollRefreshJob>()
 				.AddSingleton<IJobHandler, PollCloseJob>()
-				.AddSingleton<IJobHandler, InstanceDescriber>()
+				.AddSingleton<InstanceDescriber>()
+				.AddSingleton<IJobHandler>(services => services.GetRequiredService<InstanceDescriber>())
 				.AddSingleton<IJobHandler, AncestorsJobHandler>()
 				.AddSingleton<IJobQueue, JobQueue>()
 				.AddSingleton<IHostCircuitBreaker, HostCircuitBreaker>()
@@ -116,6 +117,8 @@ namespace PrivaPub.Middleware
 				.AddSingleton<IJobHandler, RollupJob>()
 				.AddSingleton<IGeoLocator, DbIpLocator>()
 				.AddSingleton<Domain.Statistics.StatisticsQueries>()
+				.AddSingleton<IJobHandler, Federation.Crawler.CrawlPlanner>()
+				.AddSingleton<IJobHandler, Federation.Crawler.InstanceCrawler>()
 				.AddHostedService<StatisticsSchedule>();
 
 		public static IServiceCollection PrivaPubAuthServicesConfiguration(this IServiceCollection service, IConfiguration configuration)

PrivaPub/Models/Jobs/Job.cs

@@ -27,7 +27,9 @@ namespace PrivaPub.Models.Jobs
 		PollRefresh,
 		PollClose,
 		FetchPreview,
-		RollupDay
+		RollupDay,
+		CrawlPlan,
+		CrawlInstance
 	}
 
 	public enum JobState

PrivaPub/Web/Pages/Pages.cs

@@ -127,4 +127,22 @@ namespace PrivaPub.Web.Pages
 			return Page();
 		}
 	}
+
+	public class StargazingModel : PublicPageModel
+	{
+		readonly Microsoft.Extensions.Options.IOptionsMonitor<Infrastructure.Statistics.StatisticsOptions> _options;
+		readonly Microsoft.Extensions.Options.IOptionsMonitor<Models.AppConfiguration> _app;
+
+		public StargazingModel(Microsoft.Extensions.Options.IOptionsMonitor<Infrastructure.Statistics.StatisticsOptions> options,
+			Microsoft.Extensions.Options.IOptionsMonitor<Models.AppConfiguration> app)
+		{
+			_options = options;
+			_app = app;
+		}
+
+		public bool CrawlerEnabled => _options.CurrentValue.Crawler.Enabled;
+		public string UserAgent => Federation.Crawler.Stargazer.UserAgent(_app.CurrentValue.BackendBaseAddress);
+
+		public void OnGet() => Harden();
+	}
 }

PrivaPub/Web/Pages/Stargazing.cshtml

@@ -0,0 +1,48 @@
+@page "/stargazing"
+@model PrivaPub.Web.Pages.StargazingModel
+@{
+	ViewData["Title"] = "Stargazing";
+}
+<header>
+	<h1>Stargazing</h1>
+	<div class="meta">What this server records about other servers, and how to keep its crawler out.</div>
+</header>
+<article>
+	<h2>Servers, never people</h2>
+	<p>This server keeps statistics about the fediverse for teaching and curiosity. They name <strong>servers</strong>,
+	never accounts: what kind of software a server runs, what it exchanges with us, and how reliably. Distinct accounts are
+	only counted, through a key that is destroyed at the end of each day.</p>
+	<p>A server we exchange activities with is described once a week, from its public NodeInfo and, when it has one, its
+	Mastodon instance API. Its location comes from the address we reached, looked up in an offline database: only the
+	country is shown publicly for small servers, and only the CDN for servers behind one.</p>
+</article>
+<article>
+	<h2>The crawler</h2>
+	@if (Model.CrawlerEnabled)
+	{
+		<p>The crawler is <strong>on</strong> on this server.</p>
+	}
+	else
+	{
+		<p>The crawler is <strong>off</strong> on this server: it only learns about servers it already exchanges with.</p>
+	}
+	<p>When on, it identifies itself as</p>
+	<p><code>@Model.UserAgent</code></p>
+	<p>It visits one server a minute, each at most once a week, and reads only:</p>
+	<ul>
+		<li><code>/robots.txt</code></li>
+		<li><code>/.well-known/nodeinfo</code> and the NodeInfo document it points to</li>
+		<li><code>/api/v2/instance</code> or <code>/api/v1/instance</code></li>
+		<li><code>/api/v1/instance/peers</code>, to find other servers</li>
+	</ul>
+	<p>It never reads accounts, posts, timelines or directories.</p>
+	<h2>Keeping it out</h2>
+	<p>Add this to your server's <code>robots.txt</code>:</p>
+	<pre>User-agent: @StargazerToken
+Disallow: /</pre>
+	<p>If your <code>robots.txt</code> cannot be read because of a server error or a timeout, the crawler stays out too.</p>
+</article>
+<footer class="meta">IP geolocation by <a href="https://db-ip.com" rel="nofollow noopener noreferrer">DB-IP</a>, CC BY 4.0.</footer>
+@functions {
+	const string StargazerToken = PrivaPub.Federation.Crawler.Stargazer.Token;
+}