using System.Text.RegularExpressions; namespace PrivaPub.Federation.Crawler { //RFC 9309: the most specific group naming the crawler (or "*"), the longest matching rule, Allow winning a tie. //A 4xx means no rules at all; a 5xx or no answer means keep out. public sealed class RobotsTxt { sealed record Rule(bool Allow, string Pattern, Regex Match); readonly List _rules; readonly bool _everythingDisallowed; RobotsTxt(List rules, bool everythingDisallowed) { _rules = rules; _everythingDisallowed = everythingDisallowed; } public static readonly RobotsTxt AllowAll = new(new List(), false); public static readonly RobotsTxt DisallowAll = new(new List(), true); public static RobotsTxt From(int status, string text, params string[] agents) => status switch { >= 200 and < 300 => Parse(text, agents), >= 400 and < 500 => AllowAll, _ => DisallowAll }; public static RobotsTxt Parse(string text, params string[] agents) { var groups = new List<(List Agents, List Rules)>(); (List Agents, List Rules) current = default; var collectingAgents = false; foreach (var raw in (text ?? string.Empty).Split('\n')) { var line = raw.Split('#')[0].Trim(); var colon = line.IndexOf(':'); if (colon <= 0) continue; var field = line[..colon].Trim().ToLowerInvariant(); var value = line[(colon + 1)..].Trim(); if (field == "user-agent") { if (!collectingAgents) { current = (new List(), new List()); groups.Add(current); collectingAgents = true; } current.Agents.Add(value.ToLowerInvariant()); continue; } if (field is not ("allow" or "disallow") || current.Agents == default) continue; collectingAgents = false; if (value.Length == 0) continue; current.Rules.Add(new Rule(field == "allow", value, Pattern(value))); } foreach (var agent in agents.Select(a => a.ToLowerInvariant())) { var named = groups.Where(g => g.Agents.Contains(agent)).SelectMany(g => g.Rules).ToList(); if (groups.Any(g => g.Agents.Contains(agent))) return new RobotsTxt(named, false); } return new RobotsTxt(groups.Where(g => g.Agents.Contains("*")).SelectMany(g => g.Rules).ToList(), false); } public bool Allows(string path) { if (_everythingDisallowed) return false; path = string.IsNullOrEmpty(path) ? "/" : path; Rule best = default; foreach (var rule in _rules.Where(r => r.Match.IsMatch(path))) if (best == default || rule.Pattern.Length > best.Pattern.Length || rule.Pattern.Length == best.Pattern.Length && rule.Allow) best = rule; return best?.Allow != false; } static Regex Pattern(string value) { var anchored = value.EndsWith('$'); var body = Regex.Escape(anchored ? value[..^1] : value).Replace("\\*", ".*"); return new Regex("^" + body + (anchored ? "$" : string.Empty), RegexOptions.CultureInvariant, TimeSpan.FromMilliseconds(100)); } } }