M11: the opt-in crawler, PrivaPub-Stargazer, and /stargazing

Off by default (Statistics:Crawler:Enabled), as the owner decided. When it is on:
- CrawlPlan runs hourly, from StatisticsSchedule. It inserts the configured seeds and
  queues up to HostsPerHour servers whose last visit is older than RevisitDays, are not
  paused by the breaker and are not domain-blocked, spread across the hour.
- CrawlInstance visits one server at a time as
  "PrivaPub-Stargazer/<ref> (+<base>/stargazing)":
  - it reads robots.txt (RFC 9309: its own group first, then PrivaPub, then *; longest
    rule wins; a 4xx allows everything; a 5xx or no answer keeps it out);
  - it describes servers that only crawling ever found, through
    InstanceDescriber.Describe with robots.txt as the path filter;
  - it reads /api/v1/instance/peers through the new GetStringArray, which keeps what it
    read from the first 1 MB instead of refusing a large list;
  - it adds the names a server could ever be reached at as "crawled": DNS only,
    punycode, no addresses, ports or hidden services, and the reserved test names only on
    a test network. Never more than MaxNewHostsPerCrawl per visit or MaxHosts in all, and
    never over a touched server.
- It reads nothing but robots.txt, NodeInfo, the instance API and the peers list.
  IFederationHttp.GetText serves robots.txt, and HttpScope.Crawl carries the
  User-Agent.
- /stargazing explains all this and how to keep the crawler out, says whether it is on,
  and credits DB-IP. GET /clientapi/admin/statistics/crawler shows the frontier.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01ELjqpznMFMNrJoJUj6K5p2

PrivaPub/Federation/Crawler/RobotsTxt.cs

@@ -0,0 +1,90 @@
+using System.Text.RegularExpressions;
+
+namespace PrivaPub.Federation.Crawler
+{
+	//RFC 9309: the most specific group naming the crawler (or "*"), the longest matching rule, Allow winning a tie.
+	//A 4xx means no rules at all; a 5xx or no answer means keep out.
+	public sealed class RobotsTxt
+	{
+		sealed record Rule(bool Allow, string Pattern, Regex Match);
+
+		readonly List<Rule> _rules;
+		readonly bool _everythingDisallowed;
+
+		RobotsTxt(List<Rule> rules, bool everythingDisallowed)
+		{
+			_rules = rules;
+			_everythingDisallowed = everythingDisallowed;
+		}
+
+		public static readonly RobotsTxt AllowAll = new(new List<Rule>(), false);
+		public static readonly RobotsTxt DisallowAll = new(new List<Rule>(), true);
+
+		public static RobotsTxt From(int status, string text, params string[] agents) => status switch
+		{
+			>= 200 and < 300 => Parse(text, agents),
+			>= 400 and < 500 => AllowAll,
+			_ => DisallowAll
+		};
+
+		public static RobotsTxt Parse(string text, params string[] agents)
+		{
+			var groups = new List<(List<string> Agents, List<Rule> Rules)>();
+			(List<string> Agents, List<Rule> Rules) current = default;
+			var collectingAgents = false;
+			foreach (var raw in (text ?? string.Empty).Split('\n'))
+			{
+				var line = raw.Split('#')[0].Trim();
+				var colon = line.IndexOf(':');
+				if (colon <= 0)
+					continue;
+				var field = line[..colon].Trim().ToLowerInvariant();
+				var value = line[(colon + 1)..].Trim();
+				if (field == "user-agent")
+				{
+					if (!collectingAgents)
+					{
+						current = (new List<string>(), new List<Rule>());
+						groups.Add(current);
+						collectingAgents = true;
+					}
+					current.Agents.Add(value.ToLowerInvariant());
+					continue;
+				}
+				if (field is not ("allow" or "disallow") || current.Agents == default)
+					continue;
+				collectingAgents = false;
+				if (value.Length == 0)
+					continue;
+				current.Rules.Add(new Rule(field == "allow", value, Pattern(value)));
+			}
+
+			foreach (var agent in agents.Select(a => a.ToLowerInvariant()))
+			{
+				var named = groups.Where(g => g.Agents.Contains(agent)).SelectMany(g => g.Rules).ToList();
+				if (groups.Any(g => g.Agents.Contains(agent)))
+					return new RobotsTxt(named, false);
+			}
+			return new RobotsTxt(groups.Where(g => g.Agents.Contains("*")).SelectMany(g => g.Rules).ToList(), false);
+		}
+
+		public bool Allows(string path)
+		{
+			if (_everythingDisallowed)
+				return false;
+			path = string.IsNullOrEmpty(path) ? "/" : path;
+			Rule best = default;
+			foreach (var rule in _rules.Where(r => r.Match.IsMatch(path)))
+				if (best == default || rule.Pattern.Length > best.Pattern.Length || rule.Pattern.Length == best.Pattern.Length && rule.Allow)
+					best = rule;
+			return best?.Allow != false;
+		}
+
+		static Regex Pattern(string value)
+		{
+			var anchored = value.EndsWith('$');
+			var body = Regex.Escape(anchored ? value[..^1] : value).Replace("\\*", ".*");
+			return new Regex("^" + body + (anchored ? "$" : string.Empty), RegexOptions.CultureInvariant, TimeSpan.FromMilliseconds(100));
+		}
+	}
+}