From 4f5df743ae9dfdd6ea18b94bb081c710f76ec601 Mon Sep 17 00:00:00 2001 From: thepra Date: Sat, 3 Oct 2026 12:19:29 +0200 Subject: [PATCH] M11: the opt-in crawler, PrivaPub-Stargazer, and /stargazing Off by default (Statistics:Crawler:Enabled), as the owner decided. When it is on: - CrawlPlan runs hourly, from StatisticsSchedule. It inserts the configured seeds and queues up to HostsPerHour servers whose last visit is older than RevisitDays, are not paused by the breaker and are not domain-blocked, spread across the hour. - CrawlInstance visits one server at a time as "PrivaPub-Stargazer/ (+/stargazing)": - it reads robots.txt (RFC 9309: its own group first, then PrivaPub, then *; longest rule wins; a 4xx allows everything; a 5xx or no answer keeps it out); - it describes servers that only crawling ever found, through InstanceDescriber.Describe with robots.txt as the path filter; - it reads /api/v1/instance/peers through the new GetStringArray, which keeps what it read from the first 1 MB instead of refusing a large list; - it adds the names a server could ever be reached at as "crawled": DNS only, punycode, no addresses, ports or hidden services, and the reserved test names only on a test network. Never more than MaxNewHostsPerCrawl per visit or MaxHosts in all, and never over a touched server. - It reads nothing but robots.txt, NodeInfo, the instance API and the peers list. IFederationHttp.GetText serves robots.txt, and HttpScope.Crawl carries the User-Agent. - /stargazing explains all this and how to keep the crawler out, says whether it is on, and credits DB-IP. GET /clientapi/admin/statistics/crawler shows the frontier. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01ELjqpznMFMNrJoJUj6K5p2 --- PrivaPub.Tests/Statistics/CrawlerTests.cs | 276 ++++++++++++++++++ .../ClientToServer/StatisticsController.cs | 8 +- .../Domain/Statistics/StatisticsQueries.cs | 19 ++ PrivaPub/Federation/Crawler/Crawler.cs | 206 +++++++++++++ PrivaPub/Federation/Crawler/PeerHosts.cs | 37 +++ PrivaPub/Federation/Crawler/RobotsTxt.cs | 90 ++++++ .../Infrastructure/Http/FederationHttp.cs | 151 +++++++++- PrivaPub/Infrastructure/Http/HttpScope.cs | 5 +- .../Infrastructure/Statistics/RollupJob.cs | 11 +- .../Statistics/StatisticsOptions.cs | 12 + .../Middleware/SocialPubConfigurations.cs | 5 +- PrivaPub/Models/Jobs/Job.cs | 4 +- PrivaPub/Web/Pages/Pages.cs | 18 ++ PrivaPub/Web/Pages/Stargazing.cshtml | 48 +++ 14 files changed, 882 insertions(+), 8 deletions(-) create mode 100644 PrivaPub.Tests/Statistics/CrawlerTests.cs create mode 100644 PrivaPub/Federation/Crawler/Crawler.cs create mode 100644 PrivaPub/Federation/Crawler/PeerHosts.cs create mode 100644 PrivaPub/Federation/Crawler/RobotsTxt.cs create mode 100644 PrivaPub/Web/Pages/Stargazing.cshtml diff --git a/PrivaPub.Tests/Statistics/CrawlerTests.cs b/PrivaPub.Tests/Statistics/CrawlerTests.cs new file mode 100644 index 0000000..6f873fe --- /dev/null +++ b/PrivaPub.Tests/Statistics/CrawlerTests.cs @@ -0,0 +1,276 @@ +using Microsoft.Extensions.Logging.Abstractions; + +using MongoDB.Entities; + +using PrivaPub.Federation.Crawler; +using PrivaPub.Federation.Moderation; +using PrivaPub.Federation.Objects; +using PrivaPub.Infrastructure.Http; +using PrivaPub.Infrastructure.Jobs; +using PrivaPub.Infrastructure.Statistics; +using PrivaPub.Models; +using PrivaPub.Models.Federation; +using PrivaPub.Models.Jobs; +using PrivaPub.Tests.Support; +using PrivaPub.Tests.Support.Host; + +using System.Net; +using System.Text; + +namespace PrivaPub.Tests.Statistics +{ + public class RobotsTxtTests + { + const string Robots = """ + # comments are ignored + User-agent: * + Disallow: /private + Allow: /private/ok + + User-agent: PrivaPub-Stargazer + User-agent: other + Disallow: /api/ + Allow: /api/v1/instance$ + + User-agent: greedy + Disallow: /*.json$ + """; + + [Fact] + public void The_crawlers_own_group_wins_and_the_longest_rule_decides() + { + var mine = RobotsTxt.Parse(Robots, "PrivaPub-Stargazer"); + var anyone = RobotsTxt.Parse(Robots, "SomethingElse"); + var greedy = RobotsTxt.Parse(Robots, "greedy"); + + Assert.False(mine.Allows("/api/v1/instance/peers")); + Assert.True(mine.Allows("/api/v1/instance")); + Assert.True(mine.Allows("/private")); + Assert.True(mine.Allows("/.well-known/nodeinfo")); + Assert.False(anyone.Allows("/private/nope")); + Assert.True(anyone.Allows("/private/ok/yes")); + Assert.True(anyone.Allows("/api/v1/instance/peers")); + Assert.False(greedy.Allows("/nodeinfo/2.1.json")); + Assert.True(greedy.Allows("/nodeinfo/2.1.json?x=1")); + } + + [Fact] + public void An_error_keeps_the_crawler_out_and_a_missing_file_lets_it_in() + { + Assert.True(RobotsTxt.From(404, default, Stargazer.Token).Allows("/anything")); + Assert.False(RobotsTxt.From(503, default, Stargazer.Token).Allows("/anything")); + Assert.False(RobotsTxt.From(0, default, Stargazer.Token).Allows("/anything")); + Assert.False(RobotsTxt.From(200, "User-agent: *\nDisallow: /", Stargazer.Token).Allows("/robots.txt")); + Assert.True(RobotsTxt.From(200, "User-agent: *\nDisallow:", Stargazer.Token).Allows("/x")); + } + + [Fact] + public void Peer_lists_yield_only_names_we_could_ever_reach() + { + var names = new[] { "Social.Example.org.", "10.0.0.1", "host.example.org:8443", "x.onion", "printer.local", "bücher.example.org", "a.test", "noDot", "social.example.org" }; + + Assert.Equal(new[] { "social.example.org", "xn--bcher-kva.example.org" }, PeerHosts.Filter(names, _ => true)); + Assert.Contains("a.test", PeerHosts.Filter(names, _ => true, testNetwork: true)); + Assert.Empty(PeerHosts.Filter(names, _ => false)); + } + + [Fact] + public void A_truncated_peers_list_still_gives_what_was_read() + { + var items = new List(); + FederationHttp.ReadStrings(Encoding.UTF8.GetBytes("[\"a.example\", 3, {\"x\":1}, \"b.example\", \"c.exa"), 10, items); + var capped = new List(); + FederationHttp.ReadStrings(Encoding.UTF8.GetBytes("[\"a\",\"b\",\"c\"]"), 2, capped); + + Assert.Equal(new[] { "a.example", "b.example" }, items); + Assert.Equal(new[] { "a", "b" }, capped); + } + } + + [Trait("Category", "Integration")] + public sealed class CrawlerTests : IAsyncLifetime + { + Peer _peer; + readonly string _run = Guid.NewGuid().ToString("N")[..10]; + + public async ValueTask InitializeAsync() + { + Assert.SkipUnless(MongoFixture.Enabled, MongoFixture.Skip); + _peer = await Peer.Start(); + } + + public async ValueTask DisposeAsync() + { + if (_peer != default) + await _peer.DisposeAsync(); + } + + sealed class PeerDescriber : InstanceDescriber + { + readonly string _peer; + + public PeerDescriber(IFederationHttp http, string peer) : base(http) => _peer = peer; + + protected override string Address(string url) => $"{_peer}/{new Uri(url).Host}{new Uri(url).PathAndQuery}"; + } + + sealed class PeerCrawler : InstanceCrawler + { + readonly string _peer; + + public PeerCrawler(IFederationHttp http, InstanceDescriber describer, IDomainBlocks blocks, CrawlerOptions crawler, string peer, MemoryLedger ledger) + : base(http, describer, blocks, new StaticOptions(new StatisticsOptions { Crawler = crawler }), + new StaticOptions(new AppConfiguration { BackendBaseAddress = "https://privapub.test" }), + new StaticOptions(new FederationOptions { AllowPrivateNetworks = true }), ledger) => _peer = peer; + + protected override string Address(string url) => $"{_peer}/{new Uri(url).Host}{new Uri(url).PathAndQuery}"; + } + + (PeerCrawler Crawler, MemoryLedger Ledger) Crawler(CrawlerOptions options, IDomainBlocks blocks = default) + { + var http = Peer.Http(); + var ledger = new MemoryLedger(); + return (new PeerCrawler(http, new PeerDescriber(http, _peer.A), blocks ?? new NoBlocks(), options, _peer.A, ledger), ledger); + } + + string Host(string name) => $"{name}{_run}.example.org"; + + void Serve(string host, string robots = default, int robotsStatus = 200, params string[] peers) + { + if (robots != default) + _peer.ServeText($"/{host}/robots.txt", robots, "text/plain"); + else + _peer.Answer($"/{host}/robots.txt", robotsStatus); + _peer.Serve($"/{host}/.well-known/nodeinfo", $$"""{"links":[{"rel":"http://nodeinfo.diaspora.software/ns/schema/2.0","href":"https://{{host}}/nodeinfo/2.0"}]}"""); + _peer.Serve($"/{host}/nodeinfo/2.0", """{"software":{"name":"lemmy","version":"0.19.9"},"usage":{"users":{"total":3}}}"""); + _peer.Serve($"/{host}/api/v1/instance/peers", "[" + string.Join(",", peers.Select(p => $"\"{p}\"")) + "]"); + } + + [Fact] + public async Task A_crawl_describes_a_crawled_server_and_learns_its_peers_politely() + { + var token = TestContext.Current.CancellationToken; + var host = Host("seed"); + var blocked = Host("blocked"); + Serve(host, "User-agent: *\nDisallow: /private", 200, Host("new1"), Host("new2"), blocked, "10.0.0.1", Host("new1")); + var blocks = new DomainBlocks(NullLogger.Instance); + blocks.Load(new[] { new DomainBlock { Domain = blocked, Severity = DomainBlockSeverity.Silence } }); + await DB.Default.SaveAsync(new RemoteInstance { Host = host, Seen = "crawled" }, token); + var (crawler, ledger) = Crawler(new CrawlerOptions { Enabled = true }, blocks); + + await crawler.Handle(new Job { Kind = JobKind.CrawlInstance, Payload = host }, token); + + var seed = await DB.Default.Find().Match(i => i.Host == host).ExecuteSingleAsync(token); + Assert.Equal(("lemmy", 5, false), (seed.Software, seed.PeersCount!.Value, seed.RobotsDisallowed!.Value)); + Assert.NotNull(seed.LastCrawledAt); + var found = await DB.Default.Find().Match(i => i.Host == Host("new1") || i.Host == Host("new2") || i.Host == blocked).ExecuteAsync(token); + Assert.Equal(new[] { Host("new1"), Host("new2") }, found.Select(i => i.Host).OrderBy(h => h)); + Assert.All(found, i => Assert.Equal("crawled", i.Seen)); + var visited = _peer.Requests.Where(r => r.Path.StartsWith("/" + host + "/")).ToList(); + Assert.All(visited, r => Assert.Contains(r.Path[(host.Length + 1)..], new[] { "/robots.txt", "/.well-known/nodeinfo", "/nodeinfo/2.0", "/api/v1/instance/peers" })); + Assert.All(visited, r => Assert.StartsWith("PrivaPub-Stargazer/", r.Headers["User-Agent"])); + Assert.All(visited, r => Assert.Contains("/stargazing)", r.Headers["User-Agent"])); + Assert.Equal(("crawl", "ok"), (Assert.Single(ledger.Events).Event.Channel, ledger.Events.Single().Event.Outcome)); + } + + [Fact] + public async Task Robots_txt_and_its_absence_are_obeyed_and_a_touched_server_is_not_described_again() + { + var token = TestContext.Current.CancellationToken; + var shy = Host("shy"); + var broken = Host("broken"); + var touched = Host("touched"); + Serve(shy, "User-agent: PrivaPub-Stargazer\nDisallow: /api/", 200, Host("hidden")); + Serve(broken, default, 503, Host("never")); + Serve(touched, "", 200); + await DB.Default.SaveAsync(new[] { new RemoteInstance { Host = shy, Seen = "crawled" }, new RemoteInstance { Host = touched, Seen = "touched" } }, token); + var (crawler, ledger) = Crawler(new CrawlerOptions { Enabled = true }); + + await crawler.Handle(new Job { Payload = shy }, token); + await crawler.Handle(new Job { Payload = broken }, token); + await crawler.Handle(new Job { Payload = touched }, token); + + Assert.DoesNotContain(_peer.Requests, r => r.Path == $"/{shy}/api/v1/instance/peers"); + Assert.Contains(_peer.Requests, r => r.Path == $"/{shy}/nodeinfo/2.0"); + Assert.Equal(new[] { $"/{broken}/robots.txt" }, _peer.Requests.Where(r => r.Path.StartsWith($"/{broken}/")).Select(r => r.Path)); + Assert.True((await DB.Default.Find().Match(i => i.Host == broken).ExecuteSingleAsync(token)).RobotsDisallowed); + Assert.DoesNotContain(_peer.Requests, r => r.Path.StartsWith($"/{touched}/.well-known/")); + Assert.Contains(ledger.Events, e => e.Event.Host == broken && e.Event.Reason == "robots-disallowed"); + Assert.False(await DB.Default.Find().Match(i => i.Host == Host("hidden") || i.Host == Host("never")).ExecuteAnyAsync(token)); + } + + [Fact] + public async Task A_disabled_crawler_does_nothing_and_the_caps_hold() + { + var token = TestContext.Current.CancellationToken; + var host = Host("capped"); + Serve(host, "", 200, Enumerable.Range(0, 20).Select(i => Host($"many{i}-")).ToArray()); + await DB.Default.SaveAsync(new RemoteInstance { Host = host, Seen = "touched" }, token); + + await Crawler(new CrawlerOptions { Enabled = false }).Crawler.Handle(new Job { Payload = host }, token); + Assert.DoesNotContain(_peer.Requests, r => r.Path.StartsWith($"/{host}/")); + + await Crawler(new CrawlerOptions { Enabled = true, MaxNewHostsPerCrawl = 3 }).Crawler.Handle(new Job { Payload = host }, token); + Assert.Equal(3, await DB.Default.CountAsync(i => i.Host.StartsWith("many") && i.Host.EndsWith($"{_run}.example.org"), cancellation: token)); + } + + [Fact] + public async Task The_planner_spreads_due_servers_across_the_hour_and_skips_fresh_and_blocked_ones() + { + var token = TestContext.Current.CancellationToken; + var due = Host("due"); + var fresh = Host("fresh"); + var blocked = Host("plannedblock"); + await DB.Default.SaveAsync(new[] + { + new RemoteInstance { Host = due, Seen = "crawled", LastCrawledAt = new DateTime(2001, 1, 1, 0, 0, 0, DateTimeKind.Utc) }, + new RemoteInstance { Host = fresh, Seen = "crawled", LastCrawledAt = DateTime.UtcNow }, + new RemoteInstance { Host = blocked, Seen = "crawled", LastCrawledAt = new DateTime(2001, 1, 1, 0, 0, 0, DateTimeKind.Utc) } + }, token); + var blocks = new DomainBlocks(NullLogger.Instance); + blocks.Load(new[] { new DomainBlock { Domain = blocked, Severity = DomainBlockSeverity.Suspend } }); + var options = new StatisticsOptions { Crawler = new CrawlerOptions { Enabled = true, HostsPerHour = 600, Seeds = new() { Host("seedling"), "10.1.1.1" } } }; + + await new CrawlPlanner(new JobQueue(j => j.DedupeKey == "never"), blocks, new StaticOptions(options)).Handle(new Job(), token); + await new CrawlPlanner(new JobQueue(j => j.DedupeKey == "never"), blocks, new StaticOptions(new StatisticsOptions())).Handle(new Job(), token); + + var planned = (await DB.Default.Find().Match(j => j.Kind == JobKind.CrawlInstance && j.Payload.EndsWith($"{_run}.example.org")).ExecuteAsync(token)).Select(j => j.Payload).ToList(); + Assert.Contains(due, planned); + Assert.Contains(Host("seedling"), planned); + Assert.DoesNotContain(fresh, planned); + Assert.DoesNotContain(blocked, planned); + Assert.Equal("crawled", (await DB.Default.Find().Match(i => i.Host == Host("seedling")).ExecuteSingleAsync(token)).Seen); + } + } + + [Trait("Category", "Integration")] + public sealed class StargazingPageTests : IAsyncLifetime + { + PrivaPubHost _host; + + public async ValueTask InitializeAsync() + { + Assert.SkipUnless(MongoFixture.Enabled, MongoFixture.Skip); + _host = await PrivaPubHost.Shared(); + } + + public ValueTask DisposeAsync() => ValueTask.CompletedTask; + + [Fact] + public async Task The_page_names_the_crawler_says_it_is_off_and_how_to_keep_it_out() + { + using var client = _host.Client(); + + using var response = await client.GetAsync("/stargazing", TestContext.Current.CancellationToken); + var page = await response.Content.ReadAsStringAsync(TestContext.Current.CancellationToken); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + Assert.Contains("PrivaPub-Stargazer/", page); + Assert.Contains("https://privapub.test/stargazing", page); + Assert.Contains("off", page); + Assert.Contains("Disallow: /", page); + Assert.Contains("DB-IP", page); + Assert.StartsWith("default-src 'none'", response.Headers.GetValues("Content-Security-Policy").Single()); + } + } +} diff --git a/PrivaPub/Controllers/ClientToServer/StatisticsController.cs b/PrivaPub/Controllers/ClientToServer/StatisticsController.cs index 7f40b31..7f64098 100644 --- a/PrivaPub/Controllers/ClientToServer/StatisticsController.cs +++ b/PrivaPub/Controllers/ClientToServer/StatisticsController.cs @@ -18,13 +18,19 @@ namespace PrivaPub.Controllers.ClientToServer { readonly StatisticsQueries _queries; readonly IJobQueue _queue; + readonly Microsoft.Extensions.Options.IOptionsMonitor _options; - public StatisticsController(StatisticsQueries queries, IJobQueue queue) + public StatisticsController(StatisticsQueries queries, IJobQueue queue, Microsoft.Extensions.Options.IOptionsMonitor options) { _queries = queries; _queue = queue; + _options = options; } + [HttpGet, Route("/clientapi/admin/statistics/crawler")] + public async Task Crawler(CancellationToken token) => + Ok(await _queries.Crawler(_options.CurrentValue.Crawler.Enabled, _options.CurrentValue.Crawler.RevisitDays, token)); + [HttpGet, Route("/clientapi/admin/statistics/overview")] public async Task Overview([FromQuery] int days = 30, CancellationToken token = default) => Ok(await _queries.Overview(days, token)); diff --git a/PrivaPub/Domain/Statistics/StatisticsQueries.cs b/PrivaPub/Domain/Statistics/StatisticsQueries.cs index 4ba30a7..4d9595c 100644 --- a/PrivaPub/Domain/Statistics/StatisticsQueries.cs +++ b/PrivaPub/Domain/Statistics/StatisticsQueries.cs @@ -147,6 +147,25 @@ namespace PrivaPub.Domain.Statistics .ExecuteAsync(token); } + public async Task Crawler(bool enabled, int revisitDays, CancellationToken token) + { + var due = DateTime.UtcNow.AddDays(-Math.Max(revisitDays, 1)); + var recent = await DB.Default.Find() + .Match(i => i.LastCrawledAt != null) + .Sort(i => i.LastCrawledAt, Order.Descending) + .Limit(20) + .ExecuteAsync(token); + return new + { + Enabled = enabled, + Touched = await DB.Default.CountAsync(i => i.Seen == "touched", token), + Crawled = await DB.Default.CountAsync(i => i.Seen == "crawled", token), + Due = await DB.Default.CountAsync(i => i.LastCrawledAt == null || i.LastCrawledAt < due, token), + RobotsDisallowed = await DB.Default.CountAsync(i => i.RobotsDisallowed == true, token), + Recent = recent.Select(i => new { i.Host, i.Seen, i.LastCrawledAt, i.PeersCount, i.RobotsDisallowed, i.CrawlError, i.Software }) + }; + } + public async Task> Server(int days, CancellationToken token) { var since = DateTime.UtcNow.Date.AddDays(1 - Math.Clamp(days, 1, 3660)); diff --git a/PrivaPub/Federation/Crawler/Crawler.cs b/PrivaPub/Federation/Crawler/Crawler.cs new file mode 100644 index 0000000..116bee2 --- /dev/null +++ b/PrivaPub/Federation/Crawler/Crawler.cs @@ -0,0 +1,206 @@ +using Microsoft.Extensions.Options; + +using MongoDB.Entities; + +using PrivaPub.Federation.Moderation; +using PrivaPub.Federation.Objects; +using PrivaPub.Infrastructure.Http; +using PrivaPub.Infrastructure.Jobs; +using PrivaPub.Infrastructure.Statistics; +using PrivaPub.Models; +using PrivaPub.Models.Jobs; +using PrivaPub.Models.Statistics; +using PrivaPub.StaticServices; + +namespace PrivaPub.Federation.Crawler +{ + public static class Stargazer + { + public const string Token = "PrivaPub-Stargazer"; + public const string Page = "/stargazing"; + + public static string UserAgent(string baseAddress) => $"{Token}/{BuildInfo.Ref} (+{baseAddress?.TrimEnd('/')}{Page})"; + } + + //each hour, the servers due for a visit, spread across the hour so the crawler reaches one at a time + public class CrawlPlanner : IJobHandler + { + readonly IJobQueue _queue; + readonly IDomainBlocks _blocks; + readonly IOptionsMonitor _options; + + public CrawlPlanner(IJobQueue queue, IDomainBlocks blocks, IOptionsMonitor options) + { + _queue = queue; + _blocks = blocks; + _options = options; + } + + public JobKind Kind => JobKind.CrawlPlan; + public int Concurrency => 1; + public int MaxAttempts => 1; + public int PerHostLimit => 1; + + public static Job For(DateTime hour) => new() + { + Kind = JobKind.CrawlPlan, + Payload = hour.ToString("yyyy-MM-ddTHH", System.Globalization.CultureInfo.InvariantCulture), + DedupeKey = "crawl-plan|" + hour.ToString("yyyy-MM-ddTHH", System.Globalization.CultureInfo.InvariantCulture), + RunAt = hour + }; + + public async Task Handle(Job job, CancellationToken token) + { + var crawler = _options.CurrentValue.Crawler; + if (!crawler.Enabled) + return JobOutcome.Done; + var now = DateTime.UtcNow; + foreach (var seed in PeerHosts.Filter(crawler.Seeds ?? new(), host => !_blocks.IsSuspended(host))) + await DB.Default.Update() + .Match(i => i.Host == seed) + .Modify(b => b.SetOnInsert(i => i.Seen, "crawled")) + .Modify(b => b.SetOnInsert(i => i.FirstSeenAt, now)) + .Option(o => o.IsUpsert = true) + .ExecuteAsync(token); + + var due = now.AddDays(-Math.Max(crawler.RevisitDays, 1)); + var perHour = Math.Clamp(crawler.HostsPerHour, 1, 600); + var candidates = await DB.Default.Find() + .Match(i => (i.LastCrawledAt == null || i.LastCrawledAt < due) && (i.UnavailableUntil == null || i.UnavailableUntil < now)) + .Sort(i => i.LastCrawledAt, Order.Ascending) + .Limit(perHour * 3) + .ExecuteAsync(token); + var week = InstanceDescriber.Week(now); + var jobs = candidates.Where(i => _blocks.Find(i.Host) == default && PeerHosts.Normalise(i.Host) != default) + .Take(perHour) + .Select((instance, index) => new Job + { + Kind = JobKind.CrawlInstance, + Payload = instance.Host, + Host = instance.Host, + DedupeKey = $"crawl|{instance.Host}|{week}", + RunAt = now.AddSeconds(index * 3600.0 / perHour) + }); + await _queue.EnqueueMany(jobs, token); + return JobOutcome.Done; + } + } + + //one server: its robots.txt, a description if only crawling ever found it, then the servers its peers list names + public class InstanceCrawler : IJobHandler + { + const int MaxPeers = 20_000; + const int MaxRobotsBytes = 64 * 1024; + + readonly IFederationHttp _http; + readonly InstanceDescriber _describer; + readonly IDomainBlocks _blocks; + readonly IOptionsMonitor _options; + readonly IOptionsMonitor _app; + readonly IOptionsMonitor _federation; + readonly IInteractionLedger _ledger; + + public InstanceCrawler(IFederationHttp http, InstanceDescriber describer, IDomainBlocks blocks, IOptionsMonitor options, + IOptionsMonitor app, IOptionsMonitor federation, IInteractionLedger ledger = default) + { + _http = http; + _describer = describer; + _blocks = blocks; + _options = options; + _app = app; + _federation = federation; + _ledger = ledger; + } + + public JobKind Kind => JobKind.CrawlInstance; + public int Concurrency => 1; + public int MaxAttempts => 1; + public int PerHostLimit => 1; + + protected virtual string Address(string url) => url; + + public async Task Handle(Job job, CancellationToken token) + { + var crawler = _options.CurrentValue.Crawler; + var host = job.Payload; + if (!crawler.Enabled || _blocks.Find(host) != default) + return JobOutcome.Done; + using var crawl = HttpScope.Crawl(Stargazer.UserAgent(_app.CurrentValue.BackendBaseAddress)); + + RobotsTxt robots; + using (HttpScope.For("robots")) + { + var (status, text) = await _http.GetText(Address($"https://{host}/robots.txt"), MaxRobotsBytes, token); + robots = RobotsTxt.From(status, text, Stargazer.Token, "PrivaPub"); + } + var instance = await DB.Default.Find().Match(i => i.Host == host).ExecuteFirstAsync(token); + var now = DateTime.UtcNow; + if (!robots.Allows("/.well-known/nodeinfo") && !robots.Allows("/api/v1/instance/peers")) + { + await Visited(host, robotsDisallowed: true, peers: default, "robots.txt keeps us out", token); + Record(host, Interactions.Refused, "robots-disallowed"); + return JobOutcome.Done; + } + + var reachable = true; + if (instance?.Seen != "touched") + reachable = await _describer.Describe(host, crawled: true, robots.Allows, token); + + int? peersCount = default; + if (robots.Allows("/api/v1/instance/peers")) + { + IReadOnlyList peers; + using (HttpScope.For("peers")) + peers = await _http.GetStringArray(Address($"https://{host}/api/v1/instance/peers"), MaxPeers, token); + peersCount = peers.Count; + await Discover(peers, crawler, token); + } + await Visited(host, robotsDisallowed: false, peersCount, reachable ? default : "no NodeInfo", token); + Record(host, reachable ? Interactions.Ok : Interactions.Failed, reachable ? default : "unreachable"); + return JobOutcome.Done; + } + + async Task Discover(IReadOnlyList peers, CrawlerOptions crawler, CancellationToken token) + { + var room = Math.Min(crawler.MaxNewHostsPerCrawl, crawler.MaxHosts - (int)await DB.Default.CountAsync(i => i.Seen == "crawled", token)); + if (room <= 0) + return; + var candidates = PeerHosts.Filter(peers, host => _http.IsAllowed(new Uri($"https://{host}/")) && _blocks.Find(host) == default, + _federation.CurrentValue.AllowPrivateNetworks).ToList(); + var now = DateTime.UtcNow; + foreach (var batch in candidates.Chunk(500)) + { + if (room <= 0) + return; + var known = (await DB.Default.Find().Match(i => batch.Contains(i.Host)).Project(i => i.Host).ExecuteAsync(token)).ToHashSet(); + foreach (var host in batch.Where(h => !known.Contains(h)).Take(room)) + { + await DB.Default.Update() + .Match(i => i.Host == host) + .Modify(b => b.SetOnInsert(i => i.Seen, "crawled")) + .Modify(b => b.SetOnInsert(i => i.FirstSeenAt, now)) + .Option(o => o.IsUpsert = true) + .ExecuteAsync(token); + room--; + } + } + } + + static async Task Visited(string host, bool robotsDisallowed, int? peers, string error, CancellationToken token) + { + var update = DB.Default.Update() + .Match(i => i.Host == host) + .Modify(i => i.LastCrawledAt, DateTime.UtcNow) + .Modify(i => i.RobotsDisallowed, robotsDisallowed) + .Modify(i => i.CrawlError, error) + .Modify(b => b.SetOnInsert(i => i.Seen, "crawled")) + .Option(o => o.IsUpsert = true); + if (peers is { } count) + update = update.Modify(i => i.PeersCount, count); + await update.ExecuteAsync(token); + } + + void Record(string host, string outcome, string reason) => + _ledger?.Record(new InteractionEvent { Channel = Interactions.Crawl, Host = host, Outcome = outcome, Reason = reason, Crawl = true }); + } +} diff --git a/PrivaPub/Federation/Crawler/PeerHosts.cs b/PrivaPub/Federation/Crawler/PeerHosts.cs new file mode 100644 index 0000000..bec0dcf --- /dev/null +++ b/PrivaPub/Federation/Crawler/PeerHosts.cs @@ -0,0 +1,37 @@ +using System.Globalization; + +namespace PrivaPub.Federation.Crawler +{ + public static class PeerHosts + { + static readonly IdnMapping Idn = new(); + static readonly string[] Unreachable = { ".onion", ".i2p", ".local", ".localhost", ".internal" }; + static readonly string[] Reserved = { ".test", ".invalid", ".example" }; + + //names from a peers list we could ever talk to: DNS names, ASCII (punycode), no addresses, ports or hidden services; + //the reserved test names only on a test network + public static IEnumerable Filter(IEnumerable names, Func allowed, bool testNetwork = false) => + names + .Select(Normalise) + .Where(host => host != default && (testNetwork || !Reserved.Any(suffix => host.EndsWith(suffix, StringComparison.Ordinal))) && allowed(host)) + .Distinct(StringComparer.Ordinal); + + public static string Normalise(string name) + { + var host = name?.Trim().TrimEnd('.').ToLowerInvariant(); + if (string.IsNullOrEmpty(host) || host.Length > 253 || !host.Contains('.') || host.Contains(':') || host.Contains('/')) + return default; + try + { + host = Idn.GetAscii(host); + } + catch (ArgumentException) + { + return default; + } + if (Uri.CheckHostName(host) != UriHostNameType.Dns || Unreachable.Any(suffix => host.EndsWith(suffix, StringComparison.Ordinal))) + return default; + return host; + } + } +} diff --git a/PrivaPub/Federation/Crawler/RobotsTxt.cs b/PrivaPub/Federation/Crawler/RobotsTxt.cs new file mode 100644 index 0000000..3455de5 --- /dev/null +++ b/PrivaPub/Federation/Crawler/RobotsTxt.cs @@ -0,0 +1,90 @@ +using System.Text.RegularExpressions; + +namespace PrivaPub.Federation.Crawler +{ + //RFC 9309: the most specific group naming the crawler (or "*"), the longest matching rule, Allow winning a tie. + //A 4xx means no rules at all; a 5xx or no answer means keep out. + public sealed class RobotsTxt + { + sealed record Rule(bool Allow, string Pattern, Regex Match); + + readonly List _rules; + readonly bool _everythingDisallowed; + + RobotsTxt(List rules, bool everythingDisallowed) + { + _rules = rules; + _everythingDisallowed = everythingDisallowed; + } + + public static readonly RobotsTxt AllowAll = new(new List(), false); + public static readonly RobotsTxt DisallowAll = new(new List(), true); + + public static RobotsTxt From(int status, string text, params string[] agents) => status switch + { + >= 200 and < 300 => Parse(text, agents), + >= 400 and < 500 => AllowAll, + _ => DisallowAll + }; + + public static RobotsTxt Parse(string text, params string[] agents) + { + var groups = new List<(List Agents, List Rules)>(); + (List Agents, List Rules) current = default; + var collectingAgents = false; + foreach (var raw in (text ?? string.Empty).Split('\n')) + { + var line = raw.Split('#')[0].Trim(); + var colon = line.IndexOf(':'); + if (colon <= 0) + continue; + var field = line[..colon].Trim().ToLowerInvariant(); + var value = line[(colon + 1)..].Trim(); + if (field == "user-agent") + { + if (!collectingAgents) + { + current = (new List(), new List()); + groups.Add(current); + collectingAgents = true; + } + current.Agents.Add(value.ToLowerInvariant()); + continue; + } + if (field is not ("allow" or "disallow") || current.Agents == default) + continue; + collectingAgents = false; + if (value.Length == 0) + continue; + current.Rules.Add(new Rule(field == "allow", value, Pattern(value))); + } + + foreach (var agent in agents.Select(a => a.ToLowerInvariant())) + { + var named = groups.Where(g => g.Agents.Contains(agent)).SelectMany(g => g.Rules).ToList(); + if (groups.Any(g => g.Agents.Contains(agent))) + return new RobotsTxt(named, false); + } + return new RobotsTxt(groups.Where(g => g.Agents.Contains("*")).SelectMany(g => g.Rules).ToList(), false); + } + + public bool Allows(string path) + { + if (_everythingDisallowed) + return false; + path = string.IsNullOrEmpty(path) ? "/" : path; + Rule best = default; + foreach (var rule in _rules.Where(r => r.Match.IsMatch(path))) + if (best == default || rule.Pattern.Length > best.Pattern.Length || rule.Pattern.Length == best.Pattern.Length && rule.Allow) + best = rule; + return best?.Allow != false; + } + + static Regex Pattern(string value) + { + var anchored = value.EndsWith('$'); + var body = Regex.Escape(anchored ? value[..^1] : value).Replace("\\*", ".*"); + return new Regex("^" + body + (anchored ? "$" : string.Empty), RegexOptions.CultureInvariant, TimeSpan.FromMilliseconds(100)); + } + } +} diff --git a/PrivaPub/Infrastructure/Http/FederationHttp.cs b/PrivaPub/Infrastructure/Http/FederationHttp.cs index e49a24b..b6a6e9f 100644 --- a/PrivaPub/Infrastructure/Http/FederationHttp.cs +++ b/PrivaPub/Infrastructure/Http/FederationHttp.cs @@ -28,6 +28,8 @@ namespace PrivaPub.Infrastructure.Http Task OpenMedia(string url, System.Net.Http.Headers.RangeHeaderValue range, CancellationToken token); Task Send(HttpRequestMessage request, CancellationToken token); Task<(byte[] Bytes, string ContentType)> GetMedia(string url, long maxBytes, CancellationToken token); + Task<(int Status, string Text)> GetText(string url, int maxBytes, CancellationToken token); + Task> GetStringArray(string url, int maxItems, CancellationToken token); } public class FederationHttp : IFederationHttp @@ -114,6 +116,152 @@ namespace PrivaPub.Infrastructure.Http }); } + static HttpRequestMessage Request(Uri target, string accept) + { + var request = new HttpRequestMessage(HttpMethod.Get, target); + request.Headers.Accept.ParseAdd(accept); + if (HttpScope.UserAgent is { } userAgent) + request.Headers.UserAgent.ParseAdd(userAgent); + return request; + } + + //robots.txt: the status alone matters when it is not 2xx (RFC 9309), so it is returned whatever it is; 0 when unreachable + public async Task<(int Status, string Text)> GetText(string url, int maxBytes, CancellationToken token) + { + var exchange = new Exchange(url, HttpScope.Purpose ?? "text"); + try + { + if (!Uri.TryCreate(url, UriKind.Absolute, out var target) || !IsAllowed(target)) + { + exchange.Refused("disallowed"); + return default; + } + using var timeout = CancellationTokenSource.CreateLinkedTokenSource(token); + timeout.CancelAfter(RequestTimeout); + for (var hop = 0; hop <= MaxRedirects; hop++) + { + using var request = Request(target, "text/plain"); + using var response = await _httpClientFactory.CreateClient(ClientName).SendAsync(request, HttpCompletionOption.ResponseHeadersRead, timeout.Token); + exchange.Answered(response); + if (IsRedirect(response.StatusCode)) + { + var location = response.Headers.Location; + var next = location == default ? default : location.IsAbsoluteUri ? location : new Uri(target, location); + if (!IsAllowed(next)) + { + exchange.Refused("bad-redirect"); + return ((int)response.StatusCode, default); + } + target = next; + exchange.Hops++; + continue; + } + if (!response.IsSuccessStatusCode) + { + exchange.Refused(StatusReason(response)); + return ((int)response.StatusCode, default); + } + var bytes = await ReadPrefix(response.Content, maxBytes, timeout.Token); + exchange.Bytes = bytes.Length; + return ((int)response.StatusCode, System.Text.Encoding.UTF8.GetString(bytes)); + } + exchange.Refused("too-many-redirects"); + return default; + } + catch (Exception ex) when (ex is HttpRequestException or BlockedDestinationException or OperationCanceledException && !token.IsCancellationRequested) + { + exchange.Failed(ex switch { BlockedDestinationException => "private-address", OperationCanceledException => "timeout", _ => "network" }); + return default; + } + finally + { + Record(exchange); + } + } + + //a JSON array of strings such as /api/v1/instance/peers, read from at most the first MaxResponseBytes: a large server's + //list is cut short rather than refused + public async Task> GetStringArray(string url, int maxItems, CancellationToken token) + { + var exchange = new Exchange(url, HttpScope.Purpose ?? "list"); + var items = new List(); + try + { + if (!Uri.TryCreate(url, UriKind.Absolute, out var target) || !IsAllowed(target)) + { + exchange.Refused("disallowed"); + return items; + } + using var timeout = CancellationTokenSource.CreateLinkedTokenSource(token); + timeout.CancelAfter(RequestTimeout); + for (var hop = 0; hop <= MaxRedirects; hop++) + { + using var request = Request(target, "application/json"); + using var response = await _httpClientFactory.CreateClient(ClientName).SendAsync(request, HttpCompletionOption.ResponseHeadersRead, timeout.Token); + exchange.Answered(response); + if (IsRedirect(response.StatusCode)) + { + var location = response.Headers.Location; + var next = location == default ? default : location.IsAbsoluteUri ? location : new Uri(target, location); + if (!IsAllowed(next)) + { + exchange.Refused("bad-redirect"); + return items; + } + target = next; + exchange.Hops++; + continue; + } + if (!response.IsSuccessStatusCode) + { + exchange.Refused(StatusReason(response)); + return items; + } + var mediaType = response.Content.Headers.ContentType?.MediaType; + if (mediaType == default || !JsonMediaTypes.Contains(mediaType, StringComparer.OrdinalIgnoreCase)) + { + exchange.Refused("content-type"); + return items; + } + var bytes = await ReadPrefix(response.Content, MaxResponseBytes, timeout.Token); + exchange.Bytes = bytes.Length; + ReadStrings(bytes, maxItems, items); + return items; + } + exchange.Refused("too-many-redirects"); + return items; + } + catch (Exception ex) when (ex is HttpRequestException or BlockedDestinationException or OperationCanceledException && !token.IsCancellationRequested) + { + exchange.Failed(ex switch { BlockedDestinationException => "private-address", OperationCanceledException => "timeout", _ => "network" }); + return items; + } + finally + { + Record(exchange); + } + } + + public static void ReadStrings(byte[] json, int maxItems, List items) + { + var reader = new Utf8JsonReader(json, isFinalBlock: false, state: default); + try + { + if (!reader.Read() || reader.TokenType != JsonTokenType.StartArray) + return; + while (items.Count < maxItems && reader.Read() && reader.TokenType != JsonTokenType.EndArray) + { + if (reader.TokenType == JsonTokenType.String) + items.Add(reader.GetString()); + else if (!reader.TrySkip()) + return; + } + } + catch (JsonException) + { + } + } + static string StatusReason(HttpResponseMessage response) => ((int)response.StatusCode).ToString(System.Globalization.CultureInfo.InvariantCulture); public bool IsAllowed(Uri target) @@ -167,8 +315,7 @@ namespace PrivaPub.Infrastructure.Http { for (var hop = 0; hop <= MaxRedirects; hop++) { - using var request = new HttpRequestMessage(HttpMethod.Get, target); - request.Headers.Accept.ParseAdd(accept); + using var request = Request(target, accept); sign?.Invoke(request); using var response = await _httpClientFactory.CreateClient(ClientName) diff --git a/PrivaPub/Infrastructure/Http/HttpScope.cs b/PrivaPub/Infrastructure/Http/HttpScope.cs index ba5d6cd..37bb61e 100644 --- a/PrivaPub/Infrastructure/Http/HttpScope.cs +++ b/PrivaPub/Infrastructure/Http/HttpScope.cs @@ -4,13 +4,14 @@ namespace PrivaPub.Infrastructure.Http { public const string Request = "request"; - sealed record Tags(string Purpose, string Trigger, bool Crawl); + sealed record Tags(string Purpose, string Trigger, bool Crawl, string UserAgent = default); static readonly AsyncLocal current = new(); public static string Purpose => current.Value?.Purpose; public static string Trigger => current.Value?.Trigger ?? Request; public static bool Crawling => current.Value?.Crawl == true; + public static string UserAgent => current.Value?.UserAgent; public static IDisposable For(string purpose) => Push(tags => tags with { Purpose = purpose }); @@ -18,7 +19,7 @@ namespace PrivaPub.Infrastructure.Http public static IDisposable Triggered(string trigger) => Push(tags => tags with { Trigger = trigger }); - public static IDisposable Crawl() => Push(tags => tags with { Crawl = true }); + public static IDisposable Crawl(string userAgent = default) => Push(tags => tags with { Crawl = true, UserAgent = userAgent ?? tags.UserAgent }); static IDisposable Push(Func change) { diff --git a/PrivaPub/Infrastructure/Statistics/RollupJob.cs b/PrivaPub/Infrastructure/Statistics/RollupJob.cs index c1e74c1..0b044ba 100644 --- a/PrivaPub/Infrastructure/Statistics/RollupJob.cs +++ b/PrivaPub/Infrastructure/Statistics/RollupJob.cs @@ -1,3 +1,5 @@ +using Microsoft.Extensions.Options; + using MongoDB.Driver; using MongoDB.Entities; @@ -100,11 +102,13 @@ namespace PrivaPub.Infrastructure.Statistics readonly IJobQueue _queue; readonly ILogger _logger; + readonly IOptionsMonitor _options; - public StatisticsSchedule(IJobQueue queue, ILogger logger) + public StatisticsSchedule(IJobQueue queue, ILogger logger, IOptionsMonitor options = default) { _queue = queue; _logger = logger; + _options = options; } protected override async Task ExecuteAsync(CancellationToken stoppingToken) @@ -142,6 +146,11 @@ namespace PrivaPub.Infrastructure.Statistics if (!rolled && await DB.Default.Find().Match(e => e.At >= day && e.At < end).ExecuteAnyAsync(token)) jobs.Add(RollupJob.For(day)); } + if (_options?.CurrentValue.Crawler.Enabled == true) + { + var now = DateTime.UtcNow; + jobs.Add(Federation.Crawler.CrawlPlanner.For(new DateTime(now.Year, now.Month, now.Day, now.Hour, 0, 0, DateTimeKind.Utc))); + } await _queue.EnqueueMany(jobs, token); } } diff --git a/PrivaPub/Infrastructure/Statistics/StatisticsOptions.cs b/PrivaPub/Infrastructure/Statistics/StatisticsOptions.cs index fb109d7..6b7d067 100644 --- a/PrivaPub/Infrastructure/Statistics/StatisticsOptions.cs +++ b/PrivaPub/Infrastructure/Statistics/StatisticsOptions.cs @@ -5,5 +5,17 @@ namespace PrivaPub.Infrastructure.Statistics public bool Enabled { get; set; } = true; public string GeoDirectory { get; set; } = "/var/lib/privapub/geo"; public int PublicCityMinUsers { get; set; } = 10; + public CrawlerOptions Crawler { get; set; } = new(); + } + + //owner decision: off by default; when on, one server a minute, each at most weekly, at most MaxHosts servers + public class CrawlerOptions + { + public bool Enabled { get; set; } + public int MaxHosts { get; set; } = 5000; + public int HostsPerHour { get; set; } = 60; + public int RevisitDays { get; set; } = 7; + public int MaxNewHostsPerCrawl { get; set; } = 500; + public List Seeds { get; set; } = new(); } } diff --git a/PrivaPub/Middleware/SocialPubConfigurations.cs b/PrivaPub/Middleware/SocialPubConfigurations.cs index 19bafe1..dcdabd3 100644 --- a/PrivaPub/Middleware/SocialPubConfigurations.cs +++ b/PrivaPub/Middleware/SocialPubConfigurations.cs @@ -99,7 +99,8 @@ namespace PrivaPub.Middleware .AddSingleton() .AddSingleton() .AddSingleton() - .AddSingleton() + .AddSingleton() + .AddSingleton(services => services.GetRequiredService()) .AddSingleton() .AddSingleton() .AddSingleton() @@ -116,6 +117,8 @@ namespace PrivaPub.Middleware .AddSingleton() .AddSingleton() .AddSingleton() + .AddSingleton() + .AddSingleton() .AddHostedService(); public static IServiceCollection PrivaPubAuthServicesConfiguration(this IServiceCollection service, IConfiguration configuration) diff --git a/PrivaPub/Models/Jobs/Job.cs b/PrivaPub/Models/Jobs/Job.cs index 8b9e6cc..936166b 100644 --- a/PrivaPub/Models/Jobs/Job.cs +++ b/PrivaPub/Models/Jobs/Job.cs @@ -27,7 +27,9 @@ namespace PrivaPub.Models.Jobs PollRefresh, PollClose, FetchPreview, - RollupDay + RollupDay, + CrawlPlan, + CrawlInstance } public enum JobState diff --git a/PrivaPub/Web/Pages/Pages.cs b/PrivaPub/Web/Pages/Pages.cs index 7f66d97..95933e7 100644 --- a/PrivaPub/Web/Pages/Pages.cs +++ b/PrivaPub/Web/Pages/Pages.cs @@ -127,4 +127,22 @@ namespace PrivaPub.Web.Pages return Page(); } } + + public class StargazingModel : PublicPageModel + { + readonly Microsoft.Extensions.Options.IOptionsMonitor _options; + readonly Microsoft.Extensions.Options.IOptionsMonitor _app; + + public StargazingModel(Microsoft.Extensions.Options.IOptionsMonitor options, + Microsoft.Extensions.Options.IOptionsMonitor app) + { + _options = options; + _app = app; + } + + public bool CrawlerEnabled => _options.CurrentValue.Crawler.Enabled; + public string UserAgent => Federation.Crawler.Stargazer.UserAgent(_app.CurrentValue.BackendBaseAddress); + + public void OnGet() => Harden(); + } } diff --git a/PrivaPub/Web/Pages/Stargazing.cshtml b/PrivaPub/Web/Pages/Stargazing.cshtml new file mode 100644 index 0000000..074b49f --- /dev/null +++ b/PrivaPub/Web/Pages/Stargazing.cshtml @@ -0,0 +1,48 @@ +@page "/stargazing" +@model PrivaPub.Web.Pages.StargazingModel +@{ + ViewData["Title"] = "Stargazing"; +} +
+

Stargazing

+
What this server records about other servers, and how to keep its crawler out.
+
+
+

Servers, never people

+

This server keeps statistics about the fediverse for teaching and curiosity. They name servers, + never accounts: what kind of software a server runs, what it exchanges with us, and how reliably. Distinct accounts are + only counted, through a key that is destroyed at the end of each day.

+

A server we exchange activities with is described once a week, from its public NodeInfo and, when it has one, its + Mastodon instance API. Its location comes from the address we reached, looked up in an offline database: only the + country is shown publicly for small servers, and only the CDN for servers behind one.

+
+
+

The crawler

+ @if (Model.CrawlerEnabled) + { +

The crawler is on on this server.

+ } + else + { +

The crawler is off on this server: it only learns about servers it already exchanges with.

+ } +

When on, it identifies itself as

+

@Model.UserAgent

+

It visits one server a minute, each at most once a week, and reads only:

+
    +
  • /robots.txt
    • +
  • /.well-known/nodeinfo and the NodeInfo document it points to
    • +
  • /api/v2/instance or /api/v1/instance
    • +
  • /api/v1/instance/peers, to find other servers
    • +
+

It never reads accounts, posts, timelines or directories.

+

Keeping it out

+

Add this to your server's robots.txt:

+ 
User-agent: @StargazerToken
+Disallow: /
+

If your robots.txt cannot be read because of a server error or a timeout, the crawler stays out too.

+
+ +@functions { + const string StargazerToken = PrivaPub.Federation.Crawler.Stargazer.Token; +}