diff --git a/PrivaPub.Tests/Statistics/CrawlerTests.cs b/PrivaPub.Tests/Statistics/CrawlerTests.cs new file mode 100644 index 0000000..6f873fe --- /dev/null +++ b/PrivaPub.Tests/Statistics/CrawlerTests.cs @@ -0,0 +1,276 @@ +using Microsoft.Extensions.Logging.Abstractions; + +using MongoDB.Entities; + +using PrivaPub.Federation.Crawler; +using PrivaPub.Federation.Moderation; +using PrivaPub.Federation.Objects; +using PrivaPub.Infrastructure.Http; +using PrivaPub.Infrastructure.Jobs; +using PrivaPub.Infrastructure.Statistics; +using PrivaPub.Models; +using PrivaPub.Models.Federation; +using PrivaPub.Models.Jobs; +using PrivaPub.Tests.Support; +using PrivaPub.Tests.Support.Host; + +using System.Net; +using System.Text; + +namespace PrivaPub.Tests.Statistics +{ + public class RobotsTxtTests + { + const string Robots = """ + # comments are ignored + User-agent: * + Disallow: /private + Allow: /private/ok + + User-agent: PrivaPub-Stargazer + User-agent: other + Disallow: /api/ + Allow: /api/v1/instance$ + + User-agent: greedy + Disallow: /*.json$ + """; + + [Fact] + public void The_crawlers_own_group_wins_and_the_longest_rule_decides() + { + var mine = RobotsTxt.Parse(Robots, "PrivaPub-Stargazer"); + var anyone = RobotsTxt.Parse(Robots, "SomethingElse"); + var greedy = RobotsTxt.Parse(Robots, "greedy"); + + Assert.False(mine.Allows("/api/v1/instance/peers")); + Assert.True(mine.Allows("/api/v1/instance")); + Assert.True(mine.Allows("/private")); + Assert.True(mine.Allows("/.well-known/nodeinfo")); + Assert.False(anyone.Allows("/private/nope")); + Assert.True(anyone.Allows("/private/ok/yes")); + Assert.True(anyone.Allows("/api/v1/instance/peers")); + Assert.False(greedy.Allows("/nodeinfo/2.1.json")); + Assert.True(greedy.Allows("/nodeinfo/2.1.json?x=1")); + } + + [Fact] + public void An_error_keeps_the_crawler_out_and_a_missing_file_lets_it_in() + { + Assert.True(RobotsTxt.From(404, default, Stargazer.Token).Allows("/anything")); + Assert.False(RobotsTxt.From(503, default, Stargazer.Token).Allows("/anything")); + Assert.False(RobotsTxt.From(0, default, Stargazer.Token).Allows("/anything")); + Assert.False(RobotsTxt.From(200, "User-agent: *\nDisallow: /", Stargazer.Token).Allows("/robots.txt")); + Assert.True(RobotsTxt.From(200, "User-agent: *\nDisallow:", Stargazer.Token).Allows("/x")); + } + + [Fact] + public void Peer_lists_yield_only_names_we_could_ever_reach() + { + var names = new[] { "Social.Example.org.", "10.0.0.1", "host.example.org:8443", "x.onion", "printer.local", "bücher.example.org", "a.test", "noDot", "social.example.org" }; + + Assert.Equal(new[] { "social.example.org", "xn--bcher-kva.example.org" }, PeerHosts.Filter(names, _ => true)); + Assert.Contains("a.test", PeerHosts.Filter(names, _ => true, testNetwork: true)); + Assert.Empty(PeerHosts.Filter(names, _ => false)); + } + + [Fact] + public void A_truncated_peers_list_still_gives_what_was_read() + { + var items = new List(); + FederationHttp.ReadStrings(Encoding.UTF8.GetBytes("[\"a.example\", 3, {\"x\":1}, \"b.example\", \"c.exa"), 10, items); + var capped = new List(); + FederationHttp.ReadStrings(Encoding.UTF8.GetBytes("[\"a\",\"b\",\"c\"]"), 2, capped); + + Assert.Equal(new[] { "a.example", "b.example" }, items); + Assert.Equal(new[] { "a", "b" }, capped); + } + } + + [Trait("Category", "Integration")] + public sealed class CrawlerTests : IAsyncLifetime + { + Peer _peer; + readonly string _run = Guid.NewGuid().ToString("N")[..10]; + + public async ValueTask InitializeAsync() + { + Assert.SkipUnless(MongoFixture.Enabled, MongoFixture.Skip); + _peer = await Peer.Start(); + } + + public async ValueTask DisposeAsync() + { + if (_peer != default) + await _peer.DisposeAsync(); + } + + sealed class PeerDescriber : InstanceDescriber + { + readonly string _peer; + + public PeerDescriber(IFederationHttp http, string peer) : base(http) => _peer = peer; + + protected override string Address(string url) => $"{_peer}/{new Uri(url).Host}{new Uri(url).PathAndQuery}"; + } + + sealed class PeerCrawler : InstanceCrawler + { + readonly string _peer; + + public PeerCrawler(IFederationHttp http, InstanceDescriber describer, IDomainBlocks blocks, CrawlerOptions crawler, string peer, MemoryLedger ledger) + : base(http, describer, blocks, new StaticOptions(new StatisticsOptions { Crawler = crawler }), + new StaticOptions(new AppConfiguration { BackendBaseAddress = "https://privapub.test" }), + new StaticOptions(new FederationOptions { AllowPrivateNetworks = true }), ledger) => _peer = peer; + + protected override string Address(string url) => $"{_peer}/{new Uri(url).Host}{new Uri(url).PathAndQuery}"; + } + + (PeerCrawler Crawler, MemoryLedger Ledger) Crawler(CrawlerOptions options, IDomainBlocks blocks = default) + { + var http = Peer.Http(); + var ledger = new MemoryLedger(); + return (new PeerCrawler(http, new PeerDescriber(http, _peer.A), blocks ?? new NoBlocks(), options, _peer.A, ledger), ledger); + } + + string Host(string name) => $"{name}{_run}.example.org"; + + void Serve(string host, string robots = default, int robotsStatus = 200, params string[] peers) + { + if (robots != default) + _peer.ServeText($"/{host}/robots.txt", robots, "text/plain"); + else + _peer.Answer($"/{host}/robots.txt", robotsStatus); + _peer.Serve($"/{host}/.well-known/nodeinfo", $$"""{"links":[{"rel":"http://nodeinfo.diaspora.software/ns/schema/2.0","href":"https://{{host}}/nodeinfo/2.0"}]}"""); + _peer.Serve($"/{host}/nodeinfo/2.0", """{"software":{"name":"lemmy","version":"0.19.9"},"usage":{"users":{"total":3}}}"""); + _peer.Serve($"/{host}/api/v1/instance/peers", "[" + string.Join(",", peers.Select(p => $"\"{p}\"")) + "]"); + } + + [Fact] + public async Task A_crawl_describes_a_crawled_server_and_learns_its_peers_politely() + { + var token = TestContext.Current.CancellationToken; + var host = Host("seed"); + var blocked = Host("blocked"); + Serve(host, "User-agent: *\nDisallow: /private", 200, Host("new1"), Host("new2"), blocked, "10.0.0.1", Host("new1")); + var blocks = new DomainBlocks(NullLogger.Instance); + blocks.Load(new[] { new DomainBlock { Domain = blocked, Severity = DomainBlockSeverity.Silence } }); + await DB.Default.SaveAsync(new RemoteInstance { Host = host, Seen = "crawled" }, token); + var (crawler, ledger) = Crawler(new CrawlerOptions { Enabled = true }, blocks); + + await crawler.Handle(new Job { Kind = JobKind.CrawlInstance, Payload = host }, token); + + var seed = await DB.Default.Find().Match(i => i.Host == host).ExecuteSingleAsync(token); + Assert.Equal(("lemmy", 5, false), (seed.Software, seed.PeersCount!.Value, seed.RobotsDisallowed!.Value)); + Assert.NotNull(seed.LastCrawledAt); + var found = await DB.Default.Find().Match(i => i.Host == Host("new1") || i.Host == Host("new2") || i.Host == blocked).ExecuteAsync(token); + Assert.Equal(new[] { Host("new1"), Host("new2") }, found.Select(i => i.Host).OrderBy(h => h)); + Assert.All(found, i => Assert.Equal("crawled", i.Seen)); + var visited = _peer.Requests.Where(r => r.Path.StartsWith("/" + host + "/")).ToList(); + Assert.All(visited, r => Assert.Contains(r.Path[(host.Length + 1)..], new[] { "/robots.txt", "/.well-known/nodeinfo", "/nodeinfo/2.0", "/api/v1/instance/peers" })); + Assert.All(visited, r => Assert.StartsWith("PrivaPub-Stargazer/", r.Headers["User-Agent"])); + Assert.All(visited, r => Assert.Contains("/stargazing)", r.Headers["User-Agent"])); + Assert.Equal(("crawl", "ok"), (Assert.Single(ledger.Events).Event.Channel, ledger.Events.Single().Event.Outcome)); + } + + [Fact] + public async Task Robots_txt_and_its_absence_are_obeyed_and_a_touched_server_is_not_described_again() + { + var token = TestContext.Current.CancellationToken; + var shy = Host("shy"); + var broken = Host("broken"); + var touched = Host("touched"); + Serve(shy, "User-agent: PrivaPub-Stargazer\nDisallow: /api/", 200, Host("hidden")); + Serve(broken, default, 503, Host("never")); + Serve(touched, "", 200); + await DB.Default.SaveAsync(new[] { new RemoteInstance { Host = shy, Seen = "crawled" }, new RemoteInstance { Host = touched, Seen = "touched" } }, token); + var (crawler, ledger) = Crawler(new CrawlerOptions { Enabled = true }); + + await crawler.Handle(new Job { Payload = shy }, token); + await crawler.Handle(new Job { Payload = broken }, token); + await crawler.Handle(new Job { Payload = touched }, token); + + Assert.DoesNotContain(_peer.Requests, r => r.Path == $"/{shy}/api/v1/instance/peers"); + Assert.Contains(_peer.Requests, r => r.Path == $"/{shy}/nodeinfo/2.0"); + Assert.Equal(new[] { $"/{broken}/robots.txt" }, _peer.Requests.Where(r => r.Path.StartsWith($"/{broken}/")).Select(r => r.Path)); + Assert.True((await DB.Default.Find().Match(i => i.Host == broken).ExecuteSingleAsync(token)).RobotsDisallowed); + Assert.DoesNotContain(_peer.Requests, r => r.Path.StartsWith($"/{touched}/.well-known/")); + Assert.Contains(ledger.Events, e => e.Event.Host == broken && e.Event.Reason == "robots-disallowed"); + Assert.False(await DB.Default.Find().Match(i => i.Host == Host("hidden") || i.Host == Host("never")).ExecuteAnyAsync(token)); + } + + [Fact] + public async Task A_disabled_crawler_does_nothing_and_the_caps_hold() + { + var token = TestContext.Current.CancellationToken; + var host = Host("capped"); + Serve(host, "", 200, Enumerable.Range(0, 20).Select(i => Host($"many{i}-")).ToArray()); + await DB.Default.SaveAsync(new RemoteInstance { Host = host, Seen = "touched" }, token); + + await Crawler(new CrawlerOptions { Enabled = false }).Crawler.Handle(new Job { Payload = host }, token); + Assert.DoesNotContain(_peer.Requests, r => r.Path.StartsWith($"/{host}/")); + + await Crawler(new CrawlerOptions { Enabled = true, MaxNewHostsPerCrawl = 3 }).Crawler.Handle(new Job { Payload = host }, token); + Assert.Equal(3, await DB.Default.CountAsync(i => i.Host.StartsWith("many") && i.Host.EndsWith($"{_run}.example.org"), cancellation: token)); + } + + [Fact] + public async Task The_planner_spreads_due_servers_across_the_hour_and_skips_fresh_and_blocked_ones() + { + var token = TestContext.Current.CancellationToken; + var due = Host("due"); + var fresh = Host("fresh"); + var blocked = Host("plannedblock"); + await DB.Default.SaveAsync(new[] + { + new RemoteInstance { Host = due, Seen = "crawled", LastCrawledAt = new DateTime(2001, 1, 1, 0, 0, 0, DateTimeKind.Utc) }, + new RemoteInstance { Host = fresh, Seen = "crawled", LastCrawledAt = DateTime.UtcNow }, + new RemoteInstance { Host = blocked, Seen = "crawled", LastCrawledAt = new DateTime(2001, 1, 1, 0, 0, 0, DateTimeKind.Utc) } + }, token); + var blocks = new DomainBlocks(NullLogger.Instance); + blocks.Load(new[] { new DomainBlock { Domain = blocked, Severity = DomainBlockSeverity.Suspend } }); + var options = new StatisticsOptions { Crawler = new CrawlerOptions { Enabled = true, HostsPerHour = 600, Seeds = new() { Host("seedling"), "10.1.1.1" } } }; + + await new CrawlPlanner(new JobQueue(j => j.DedupeKey == "never"), blocks, new StaticOptions(options)).Handle(new Job(), token); + await new CrawlPlanner(new JobQueue(j => j.DedupeKey == "never"), blocks, new StaticOptions(new StatisticsOptions())).Handle(new Job(), token); + + var planned = (await DB.Default.Find().Match(j => j.Kind == JobKind.CrawlInstance && j.Payload.EndsWith($"{_run}.example.org")).ExecuteAsync(token)).Select(j => j.Payload).ToList(); + Assert.Contains(due, planned); + Assert.Contains(Host("seedling"), planned); + Assert.DoesNotContain(fresh, planned); + Assert.DoesNotContain(blocked, planned); + Assert.Equal("crawled", (await DB.Default.Find().Match(i => i.Host == Host("seedling")).ExecuteSingleAsync(token)).Seen); + } + } + + [Trait("Category", "Integration")] + public sealed class StargazingPageTests : IAsyncLifetime + { + PrivaPubHost _host; + + public async ValueTask InitializeAsync() + { + Assert.SkipUnless(MongoFixture.Enabled, MongoFixture.Skip); + _host = await PrivaPubHost.Shared(); + } + + public ValueTask DisposeAsync() => ValueTask.CompletedTask; + + [Fact] + public async Task The_page_names_the_crawler_says_it_is_off_and_how_to_keep_it_out() + { + using var client = _host.Client(); + + using var response = await client.GetAsync("/stargazing", TestContext.Current.CancellationToken); + var page = await response.Content.ReadAsStringAsync(TestContext.Current.CancellationToken); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + Assert.Contains("PrivaPub-Stargazer/", page); + Assert.Contains("https://privapub.test/stargazing", page); + Assert.Contains("off", page); + Assert.Contains("Disallow: /", page); + Assert.Contains("DB-IP", page); + Assert.StartsWith("default-src 'none'", response.Headers.GetValues("Content-Security-Policy").Single()); + } + } +} diff --git a/PrivaPub/Controllers/ClientToServer/StatisticsController.cs b/PrivaPub/Controllers/ClientToServer/StatisticsController.cs index 7f40b31..7f64098 100644 --- a/PrivaPub/Controllers/ClientToServer/StatisticsController.cs +++ b/PrivaPub/Controllers/ClientToServer/StatisticsController.cs @@ -18,13 +18,19 @@ namespace PrivaPub.Controllers.ClientToServer { readonly StatisticsQueries _queries; readonly IJobQueue _queue; + readonly Microsoft.Extensions.Options.IOptionsMonitor _options; - public StatisticsController(StatisticsQueries queries, IJobQueue queue) + public StatisticsController(StatisticsQueries queries, IJobQueue queue, Microsoft.Extensions.Options.IOptionsMonitor options) { _queries = queries; _queue = queue; + _options = options; } + [HttpGet, Route("/clientapi/admin/statistics/crawler")] + public async Task Crawler(CancellationToken token) => + Ok(await _queries.Crawler(_options.CurrentValue.Crawler.Enabled, _options.CurrentValue.Crawler.RevisitDays, token)); + [HttpGet, Route("/clientapi/admin/statistics/overview")] public async Task Overview([FromQuery] int days = 30, CancellationToken token = default) => Ok(await _queries.Overview(days, token)); diff --git a/PrivaPub/Domain/Statistics/StatisticsQueries.cs b/PrivaPub/Domain/Statistics/StatisticsQueries.cs index 4ba30a7..4d9595c 100644 --- a/PrivaPub/Domain/Statistics/StatisticsQueries.cs +++ b/PrivaPub/Domain/Statistics/StatisticsQueries.cs @@ -147,6 +147,25 @@ namespace PrivaPub.Domain.Statistics .ExecuteAsync(token); } + public async Task Crawler(bool enabled, int revisitDays, CancellationToken token) + { + var due = DateTime.UtcNow.AddDays(-Math.Max(revisitDays, 1)); + var recent = await DB.Default.Find() + .Match(i => i.LastCrawledAt != null) + .Sort(i => i.LastCrawledAt, Order.Descending) + .Limit(20) + .ExecuteAsync(token); + return new + { + Enabled = enabled, + Touched = await DB.Default.CountAsync(i => i.Seen == "touched", token), + Crawled = await DB.Default.CountAsync(i => i.Seen == "crawled", token), + Due = await DB.Default.CountAsync(i => i.LastCrawledAt == null || i.LastCrawledAt < due, token), + RobotsDisallowed = await DB.Default.CountAsync(i => i.RobotsDisallowed == true, token), + Recent = recent.Select(i => new { i.Host, i.Seen, i.LastCrawledAt, i.PeersCount, i.RobotsDisallowed, i.CrawlError, i.Software }) + }; + } + public async Task> Server(int days, CancellationToken token) { var since = DateTime.UtcNow.Date.AddDays(1 - Math.Clamp(days, 1, 3660)); diff --git a/PrivaPub/Federation/Crawler/Crawler.cs b/PrivaPub/Federation/Crawler/Crawler.cs new file mode 100644 index 0000000..116bee2 --- /dev/null +++ b/PrivaPub/Federation/Crawler/Crawler.cs @@ -0,0 +1,206 @@ +using Microsoft.Extensions.Options; + +using MongoDB.Entities; + +using PrivaPub.Federation.Moderation; +using PrivaPub.Federation.Objects; +using PrivaPub.Infrastructure.Http; +using PrivaPub.Infrastructure.Jobs; +using PrivaPub.Infrastructure.Statistics; +using PrivaPub.Models; +using PrivaPub.Models.Jobs; +using PrivaPub.Models.Statistics; +using PrivaPub.StaticServices; + +namespace PrivaPub.Federation.Crawler +{ + public static class Stargazer + { + public const string Token = "PrivaPub-Stargazer"; + public const string Page = "/stargazing"; + + public static string UserAgent(string baseAddress) => $"{Token}/{BuildInfo.Ref} (+{baseAddress?.TrimEnd('/')}{Page})"; + } + + //each hour, the servers due for a visit, spread across the hour so the crawler reaches one at a time + public class CrawlPlanner : IJobHandler + { + readonly IJobQueue _queue; + readonly IDomainBlocks _blocks; + readonly IOptionsMonitor _options; + + public CrawlPlanner(IJobQueue queue, IDomainBlocks blocks, IOptionsMonitor options) + { + _queue = queue; + _blocks = blocks; + _options = options; + } + + public JobKind Kind => JobKind.CrawlPlan; + public int Concurrency => 1; + public int MaxAttempts => 1; + public int PerHostLimit => 1; + + public static Job For(DateTime hour) => new() + { + Kind = JobKind.CrawlPlan, + Payload = hour.ToString("yyyy-MM-ddTHH", System.Globalization.CultureInfo.InvariantCulture), + DedupeKey = "crawl-plan|" + hour.ToString("yyyy-MM-ddTHH", System.Globalization.CultureInfo.InvariantCulture), + RunAt = hour + }; + + public async Task Handle(Job job, CancellationToken token) + { + var crawler = _options.CurrentValue.Crawler; + if (!crawler.Enabled) + return JobOutcome.Done; + var now = DateTime.UtcNow; + foreach (var seed in PeerHosts.Filter(crawler.Seeds ?? new(), host => !_blocks.IsSuspended(host))) + await DB.Default.Update() + .Match(i => i.Host == seed) + .Modify(b => b.SetOnInsert(i => i.Seen, "crawled")) + .Modify(b => b.SetOnInsert(i => i.FirstSeenAt, now)) + .Option(o => o.IsUpsert = true) + .ExecuteAsync(token); + + var due = now.AddDays(-Math.Max(crawler.RevisitDays, 1)); + var perHour = Math.Clamp(crawler.HostsPerHour, 1, 600); + var candidates = await DB.Default.Find() + .Match(i => (i.LastCrawledAt == null || i.LastCrawledAt < due) && (i.UnavailableUntil == null || i.UnavailableUntil < now)) + .Sort(i => i.LastCrawledAt, Order.Ascending) + .Limit(perHour * 3) + .ExecuteAsync(token); + var week = InstanceDescriber.Week(now); + var jobs = candidates.Where(i => _blocks.Find(i.Host) == default && PeerHosts.Normalise(i.Host) != default) + .Take(perHour) + .Select((instance, index) => new Job + { + Kind = JobKind.CrawlInstance, + Payload = instance.Host, + Host = instance.Host, + DedupeKey = $"crawl|{instance.Host}|{week}", + RunAt = now.AddSeconds(index * 3600.0 / perHour) + }); + await _queue.EnqueueMany(jobs, token); + return JobOutcome.Done; + } + } + + //one server: its robots.txt, a description if only crawling ever found it, then the servers its peers list names + public class InstanceCrawler : IJobHandler + { + const int MaxPeers = 20_000; + const int MaxRobotsBytes = 64 * 1024; + + readonly IFederationHttp _http; + readonly InstanceDescriber _describer; + readonly IDomainBlocks _blocks; + readonly IOptionsMonitor _options; + readonly IOptionsMonitor _app; + readonly IOptionsMonitor _federation; + readonly IInteractionLedger _ledger; + + public InstanceCrawler(IFederationHttp http, InstanceDescriber describer, IDomainBlocks blocks, IOptionsMonitor options, + IOptionsMonitor app, IOptionsMonitor federation, IInteractionLedger ledger = default) + { + _http = http; + _describer = describer; + _blocks = blocks; + _options = options; + _app = app; + _federation = federation; + _ledger = ledger; + } + + public JobKind Kind => JobKind.CrawlInstance; + public int Concurrency => 1; + public int MaxAttempts => 1; + public int PerHostLimit => 1; + + protected virtual string Address(string url) => url; + + public async Task Handle(Job job, CancellationToken token) + { + var crawler = _options.CurrentValue.Crawler; + var host = job.Payload; + if (!crawler.Enabled || _blocks.Find(host) != default) + return JobOutcome.Done; + using var crawl = HttpScope.Crawl(Stargazer.UserAgent(_app.CurrentValue.BackendBaseAddress)); + + RobotsTxt robots; + using (HttpScope.For("robots")) + { + var (status, text) = await _http.GetText(Address($"https://{host}/robots.txt"), MaxRobotsBytes, token); + robots = RobotsTxt.From(status, text, Stargazer.Token, "PrivaPub"); + } + var instance = await DB.Default.Find().Match(i => i.Host == host).ExecuteFirstAsync(token); + var now = DateTime.UtcNow; + if (!robots.Allows("/.well-known/nodeinfo") && !robots.Allows("/api/v1/instance/peers")) + { + await Visited(host, robotsDisallowed: true, peers: default, "robots.txt keeps us out", token); + Record(host, Interactions.Refused, "robots-disallowed"); + return JobOutcome.Done; + } + + var reachable = true; + if (instance?.Seen != "touched") + reachable = await _describer.Describe(host, crawled: true, robots.Allows, token); + + int? peersCount = default; + if (robots.Allows("/api/v1/instance/peers")) + { + IReadOnlyList peers; + using (HttpScope.For("peers")) + peers = await _http.GetStringArray(Address($"https://{host}/api/v1/instance/peers"), MaxPeers, token); + peersCount = peers.Count; + await Discover(peers, crawler, token); + } + await Visited(host, robotsDisallowed: false, peersCount, reachable ? default : "no NodeInfo", token); + Record(host, reachable ? Interactions.Ok : Interactions.Failed, reachable ? default : "unreachable"); + return JobOutcome.Done; + } + + async Task Discover(IReadOnlyList peers, CrawlerOptions crawler, CancellationToken token) + { + var room = Math.Min(crawler.MaxNewHostsPerCrawl, crawler.MaxHosts - (int)await DB.Default.CountAsync(i => i.Seen == "crawled", token)); + if (room <= 0) + return; + var candidates = PeerHosts.Filter(peers, host => _http.IsAllowed(new Uri($"https://{host}/")) && _blocks.Find(host) == default, + _federation.CurrentValue.AllowPrivateNetworks).ToList(); + var now = DateTime.UtcNow; + foreach (var batch in candidates.Chunk(500)) + { + if (room <= 0) + return; + var known = (await DB.Default.Find().Match(i => batch.Contains(i.Host)).Project(i => i.Host).ExecuteAsync(token)).ToHashSet(); + foreach (var host in batch.Where(h => !known.Contains(h)).Take(room)) + { + await DB.Default.Update() + .Match(i => i.Host == host) + .Modify(b => b.SetOnInsert(i => i.Seen, "crawled")) + .Modify(b => b.SetOnInsert(i => i.FirstSeenAt, now)) + .Option(o => o.IsUpsert = true) + .ExecuteAsync(token); + room--; + } + } + } + + static async Task Visited(string host, bool robotsDisallowed, int? peers, string error, CancellationToken token) + { + var update = DB.Default.Update() + .Match(i => i.Host == host) + .Modify(i => i.LastCrawledAt, DateTime.UtcNow) + .Modify(i => i.RobotsDisallowed, robotsDisallowed) + .Modify(i => i.CrawlError, error) + .Modify(b => b.SetOnInsert(i => i.Seen, "crawled")) + .Option(o => o.IsUpsert = true); + if (peers is { } count) + update = update.Modify(i => i.PeersCount, count); + await update.ExecuteAsync(token); + } + + void Record(string host, string outcome, string reason) => + _ledger?.Record(new InteractionEvent { Channel = Interactions.Crawl, Host = host, Outcome = outcome, Reason = reason, Crawl = true }); + } +} diff --git a/PrivaPub/Federation/Crawler/PeerHosts.cs b/PrivaPub/Federation/Crawler/PeerHosts.cs new file mode 100644 index 0000000..bec0dcf --- /dev/null +++ b/PrivaPub/Federation/Crawler/PeerHosts.cs @@ -0,0 +1,37 @@ +using System.Globalization; + +namespace PrivaPub.Federation.Crawler +{ + public static class PeerHosts + { + static readonly IdnMapping Idn = new(); + static readonly string[] Unreachable = { ".onion", ".i2p", ".local", ".localhost", ".internal" }; + static readonly string[] Reserved = { ".test", ".invalid", ".example" }; + + //names from a peers list we could ever talk to: DNS names, ASCII (punycode), no addresses, ports or hidden services; + //the reserved test names only on a test network + public static IEnumerable Filter(IEnumerable names, Func allowed, bool testNetwork = false) => + names + .Select(Normalise) + .Where(host => host != default && (testNetwork || !Reserved.Any(suffix => host.EndsWith(suffix, StringComparison.Ordinal))) && allowed(host)) + .Distinct(StringComparer.Ordinal); + + public static string Normalise(string name) + { + var host = name?.Trim().TrimEnd('.').ToLowerInvariant(); + if (string.IsNullOrEmpty(host) || host.Length > 253 || !host.Contains('.') || host.Contains(':') || host.Contains('/')) + return default; + try + { + host = Idn.GetAscii(host); + } + catch (ArgumentException) + { + return default; + } + if (Uri.CheckHostName(host) != UriHostNameType.Dns || Unreachable.Any(suffix => host.EndsWith(suffix, StringComparison.Ordinal))) + return default; + return host; + } + } +} diff --git a/PrivaPub/Federation/Crawler/RobotsTxt.cs b/PrivaPub/Federation/Crawler/RobotsTxt.cs new file mode 100644 index 0000000..3455de5 --- /dev/null +++ b/PrivaPub/Federation/Crawler/RobotsTxt.cs @@ -0,0 +1,90 @@ +using System.Text.RegularExpressions; + +namespace PrivaPub.Federation.Crawler +{ + //RFC 9309: the most specific group naming the crawler (or "*"), the longest matching rule, Allow winning a tie. + //A 4xx means no rules at all; a 5xx or no answer means keep out. + public sealed class RobotsTxt + { + sealed record Rule(bool Allow, string Pattern, Regex Match); + + readonly List _rules; + readonly bool _everythingDisallowed; + + RobotsTxt(List rules, bool everythingDisallowed) + { + _rules = rules; + _everythingDisallowed = everythingDisallowed; + } + + public static readonly RobotsTxt AllowAll = new(new List(), false); + public static readonly RobotsTxt DisallowAll = new(new List(), true); + + public static RobotsTxt From(int status, string text, params string[] agents) => status switch + { + >= 200 and < 300 => Parse(text, agents), + >= 400 and < 500 => AllowAll, + _ => DisallowAll + }; + + public static RobotsTxt Parse(string text, params string[] agents) + { + var groups = new List<(List Agents, List Rules)>(); + (List Agents, List Rules) current = default; + var collectingAgents = false; + foreach (var raw in (text ?? string.Empty).Split('\n')) + { + var line = raw.Split('#')[0].Trim(); + var colon = line.IndexOf(':'); + if (colon <= 0) + continue; + var field = line[..colon].Trim().ToLowerInvariant(); + var value = line[(colon + 1)..].Trim(); + if (field == "user-agent") + { + if (!collectingAgents) + { + current = (new List(), new List()); + groups.Add(current); + collectingAgents = true; + } + current.Agents.Add(value.ToLowerInvariant()); + continue; + } + if (field is not ("allow" or "disallow") || current.Agents == default) + continue; + collectingAgents = false; + if (value.Length == 0) + continue; + current.Rules.Add(new Rule(field == "allow", value, Pattern(value))); + } + + foreach (var agent in agents.Select(a => a.ToLowerInvariant())) + { + var named = groups.Where(g => g.Agents.Contains(agent)).SelectMany(g => g.Rules).ToList(); + if (groups.Any(g => g.Agents.Contains(agent))) + return new RobotsTxt(named, false); + } + return new RobotsTxt(groups.Where(g => g.Agents.Contains("*")).SelectMany(g => g.Rules).ToList(), false); + } + + public bool Allows(string path) + { + if (_everythingDisallowed) + return false; + path = string.IsNullOrEmpty(path) ? "/" : path; + Rule best = default; + foreach (var rule in _rules.Where(r => r.Match.IsMatch(path))) + if (best == default || rule.Pattern.Length > best.Pattern.Length || rule.Pattern.Length == best.Pattern.Length && rule.Allow) + best = rule; + return best?.Allow != false; + } + + static Regex Pattern(string value) + { + var anchored = value.EndsWith('$'); + var body = Regex.Escape(anchored ? value[..^1] : value).Replace("\\*", ".*"); + return new Regex("^" + body + (anchored ? "$" : string.Empty), RegexOptions.CultureInvariant, TimeSpan.FromMilliseconds(100)); + } + } +} diff --git a/PrivaPub/Infrastructure/Http/FederationHttp.cs b/PrivaPub/Infrastructure/Http/FederationHttp.cs index e49a24b..b6a6e9f 100644 --- a/PrivaPub/Infrastructure/Http/FederationHttp.cs +++ b/PrivaPub/Infrastructure/Http/FederationHttp.cs @@ -28,6 +28,8 @@ namespace PrivaPub.Infrastructure.Http Task OpenMedia(string url, System.Net.Http.Headers.RangeHeaderValue range, CancellationToken token); Task Send(HttpRequestMessage request, CancellationToken token); Task<(byte[] Bytes, string ContentType)> GetMedia(string url, long maxBytes, CancellationToken token); + Task<(int Status, string Text)> GetText(string url, int maxBytes, CancellationToken token); + Task> GetStringArray(string url, int maxItems, CancellationToken token); } public class FederationHttp : IFederationHttp @@ -114,6 +116,152 @@ namespace PrivaPub.Infrastructure.Http }); } + static HttpRequestMessage Request(Uri target, string accept) + { + var request = new HttpRequestMessage(HttpMethod.Get, target); + request.Headers.Accept.ParseAdd(accept); + if (HttpScope.UserAgent is { } userAgent) + request.Headers.UserAgent.ParseAdd(userAgent); + return request; + } + + //robots.txt: the status alone matters when it is not 2xx (RFC 9309), so it is returned whatever it is; 0 when unreachable + public async Task<(int Status, string Text)> GetText(string url, int maxBytes, CancellationToken token) + { + var exchange = new Exchange(url, HttpScope.Purpose ?? "text"); + try + { + if (!Uri.TryCreate(url, UriKind.Absolute, out var target) || !IsAllowed(target)) + { + exchange.Refused("disallowed"); + return default; + } + using var timeout = CancellationTokenSource.CreateLinkedTokenSource(token); + timeout.CancelAfter(RequestTimeout); + for (var hop = 0; hop <= MaxRedirects; hop++) + { + using var request = Request(target, "text/plain"); + using var response = await _httpClientFactory.CreateClient(ClientName).SendAsync(request, HttpCompletionOption.ResponseHeadersRead, timeout.Token); + exchange.Answered(response); + if (IsRedirect(response.StatusCode)) + { + var location = response.Headers.Location; + var next = location == default ? default : location.IsAbsoluteUri ? location : new Uri(target, location); + if (!IsAllowed(next)) + { + exchange.Refused("bad-redirect"); + return ((int)response.StatusCode, default); + } + target = next; + exchange.Hops++; + continue; + } + if (!response.IsSuccessStatusCode) + { + exchange.Refused(StatusReason(response)); + return ((int)response.StatusCode, default); + } + var bytes = await ReadPrefix(response.Content, maxBytes, timeout.Token); + exchange.Bytes = bytes.Length; + return ((int)response.StatusCode, System.Text.Encoding.UTF8.GetString(bytes)); + } + exchange.Refused("too-many-redirects"); + return default; + } + catch (Exception ex) when (ex is HttpRequestException or BlockedDestinationException or OperationCanceledException && !token.IsCancellationRequested) + { + exchange.Failed(ex switch { BlockedDestinationException => "private-address", OperationCanceledException => "timeout", _ => "network" }); + return default; + } + finally + { + Record(exchange); + } + } + + //a JSON array of strings such as /api/v1/instance/peers, read from at most the first MaxResponseBytes: a large server's + //list is cut short rather than refused + public async Task> GetStringArray(string url, int maxItems, CancellationToken token) + { + var exchange = new Exchange(url, HttpScope.Purpose ?? "list"); + var items = new List(); + try + { + if (!Uri.TryCreate(url, UriKind.Absolute, out var target) || !IsAllowed(target)) + { + exchange.Refused("disallowed"); + return items; + } + using var timeout = CancellationTokenSource.CreateLinkedTokenSource(token); + timeout.CancelAfter(RequestTimeout); + for (var hop = 0; hop <= MaxRedirects; hop++) + { + using var request = Request(target, "application/json"); + using var response = await _httpClientFactory.CreateClient(ClientName).SendAsync(request, HttpCompletionOption.ResponseHeadersRead, timeout.Token); + exchange.Answered(response); + if (IsRedirect(response.StatusCode)) + { + var location = response.Headers.Location; + var next = location == default ? default : location.IsAbsoluteUri ? location : new Uri(target, location); + if (!IsAllowed(next)) + { + exchange.Refused("bad-redirect"); + return items; + } + target = next; + exchange.Hops++; + continue; + } + if (!response.IsSuccessStatusCode) + { + exchange.Refused(StatusReason(response)); + return items; + } + var mediaType = response.Content.Headers.ContentType?.MediaType; + if (mediaType == default || !JsonMediaTypes.Contains(mediaType, StringComparer.OrdinalIgnoreCase)) + { + exchange.Refused("content-type"); + return items; + } + var bytes = await ReadPrefix(response.Content, MaxResponseBytes, timeout.Token); + exchange.Bytes = bytes.Length; + ReadStrings(bytes, maxItems, items); + return items; + } + exchange.Refused("too-many-redirects"); + return items; + } + catch (Exception ex) when (ex is HttpRequestException or BlockedDestinationException or OperationCanceledException && !token.IsCancellationRequested) + { + exchange.Failed(ex switch { BlockedDestinationException => "private-address", OperationCanceledException => "timeout", _ => "network" }); + return items; + } + finally + { + Record(exchange); + } + } + + public static void ReadStrings(byte[] json, int maxItems, List items) + { + var reader = new Utf8JsonReader(json, isFinalBlock: false, state: default); + try + { + if (!reader.Read() || reader.TokenType != JsonTokenType.StartArray) + return; + while (items.Count < maxItems && reader.Read() && reader.TokenType != JsonTokenType.EndArray) + { + if (reader.TokenType == JsonTokenType.String) + items.Add(reader.GetString()); + else if (!reader.TrySkip()) + return; + } + } + catch (JsonException) + { + } + } + static string StatusReason(HttpResponseMessage response) => ((int)response.StatusCode).ToString(System.Globalization.CultureInfo.InvariantCulture); public bool IsAllowed(Uri target) @@ -167,8 +315,7 @@ namespace PrivaPub.Infrastructure.Http { for (var hop = 0; hop <= MaxRedirects; hop++) { - using var request = new HttpRequestMessage(HttpMethod.Get, target); - request.Headers.Accept.ParseAdd(accept); + using var request = Request(target, accept); sign?.Invoke(request); using var response = await _httpClientFactory.CreateClient(ClientName) diff --git a/PrivaPub/Infrastructure/Http/HttpScope.cs b/PrivaPub/Infrastructure/Http/HttpScope.cs index ba5d6cd..37bb61e 100644 --- a/PrivaPub/Infrastructure/Http/HttpScope.cs +++ b/PrivaPub/Infrastructure/Http/HttpScope.cs @@ -4,13 +4,14 @@ namespace PrivaPub.Infrastructure.Http { public const string Request = "request"; - sealed record Tags(string Purpose, string Trigger, bool Crawl); + sealed record Tags(string Purpose, string Trigger, bool Crawl, string UserAgent = default); static readonly AsyncLocal current = new(); public static string Purpose => current.Value?.Purpose; public static string Trigger => current.Value?.Trigger ?? Request; public static bool Crawling => current.Value?.Crawl == true; + public static string UserAgent => current.Value?.UserAgent; public static IDisposable For(string purpose) => Push(tags => tags with { Purpose = purpose }); @@ -18,7 +19,7 @@ namespace PrivaPub.Infrastructure.Http public static IDisposable Triggered(string trigger) => Push(tags => tags with { Trigger = trigger }); - public static IDisposable Crawl() => Push(tags => tags with { Crawl = true }); + public static IDisposable Crawl(string userAgent = default) => Push(tags => tags with { Crawl = true, UserAgent = userAgent ?? tags.UserAgent }); static IDisposable Push(Func change) { diff --git a/PrivaPub/Infrastructure/Statistics/RollupJob.cs b/PrivaPub/Infrastructure/Statistics/RollupJob.cs index c1e74c1..0b044ba 100644 --- a/PrivaPub/Infrastructure/Statistics/RollupJob.cs +++ b/PrivaPub/Infrastructure/Statistics/RollupJob.cs @@ -1,3 +1,5 @@ +using Microsoft.Extensions.Options; + using MongoDB.Driver; using MongoDB.Entities; @@ -100,11 +102,13 @@ namespace PrivaPub.Infrastructure.Statistics readonly IJobQueue _queue; readonly ILogger _logger; + readonly IOptionsMonitor _options; - public StatisticsSchedule(IJobQueue queue, ILogger logger) + public StatisticsSchedule(IJobQueue queue, ILogger logger, IOptionsMonitor options = default) { _queue = queue; _logger = logger; + _options = options; } protected override async Task ExecuteAsync(CancellationToken stoppingToken) @@ -142,6 +146,11 @@ namespace PrivaPub.Infrastructure.Statistics if (!rolled && await DB.Default.Find().Match(e => e.At >= day && e.At < end).ExecuteAnyAsync(token)) jobs.Add(RollupJob.For(day)); } + if (_options?.CurrentValue.Crawler.Enabled == true) + { + var now = DateTime.UtcNow; + jobs.Add(Federation.Crawler.CrawlPlanner.For(new DateTime(now.Year, now.Month, now.Day, now.Hour, 0, 0, DateTimeKind.Utc))); + } await _queue.EnqueueMany(jobs, token); } } diff --git a/PrivaPub/Infrastructure/Statistics/StatisticsOptions.cs b/PrivaPub/Infrastructure/Statistics/StatisticsOptions.cs index fb109d7..6b7d067 100644 --- a/PrivaPub/Infrastructure/Statistics/StatisticsOptions.cs +++ b/PrivaPub/Infrastructure/Statistics/StatisticsOptions.cs @@ -5,5 +5,17 @@ namespace PrivaPub.Infrastructure.Statistics public bool Enabled { get; set; } = true; public string GeoDirectory { get; set; } = "/var/lib/privapub/geo"; public int PublicCityMinUsers { get; set; } = 10; + public CrawlerOptions Crawler { get; set; } = new(); + } + + //owner decision: off by default; when on, one server a minute, each at most weekly, at most MaxHosts servers + public class CrawlerOptions + { + public bool Enabled { get; set; } + public int MaxHosts { get; set; } = 5000; + public int HostsPerHour { get; set; } = 60; + public int RevisitDays { get; set; } = 7; + public int MaxNewHostsPerCrawl { get; set; } = 500; + public List Seeds { get; set; } = new(); } } diff --git a/PrivaPub/Middleware/SocialPubConfigurations.cs b/PrivaPub/Middleware/SocialPubConfigurations.cs index 19bafe1..dcdabd3 100644 --- a/PrivaPub/Middleware/SocialPubConfigurations.cs +++ b/PrivaPub/Middleware/SocialPubConfigurations.cs @@ -99,7 +99,8 @@ namespace PrivaPub.Middleware .AddSingleton() .AddSingleton() .AddSingleton() - .AddSingleton() + .AddSingleton() + .AddSingleton(services => services.GetRequiredService()) .AddSingleton() .AddSingleton() .AddSingleton() @@ -116,6 +117,8 @@ namespace PrivaPub.Middleware .AddSingleton() .AddSingleton() .AddSingleton() + .AddSingleton() + .AddSingleton() .AddHostedService(); public static IServiceCollection PrivaPubAuthServicesConfiguration(this IServiceCollection service, IConfiguration configuration) diff --git a/PrivaPub/Models/Jobs/Job.cs b/PrivaPub/Models/Jobs/Job.cs index 8b9e6cc..936166b 100644 --- a/PrivaPub/Models/Jobs/Job.cs +++ b/PrivaPub/Models/Jobs/Job.cs @@ -27,7 +27,9 @@ namespace PrivaPub.Models.Jobs PollRefresh, PollClose, FetchPreview, - RollupDay + RollupDay, + CrawlPlan, + CrawlInstance } public enum JobState diff --git a/PrivaPub/Web/Pages/Pages.cs b/PrivaPub/Web/Pages/Pages.cs index 7f66d97..95933e7 100644 --- a/PrivaPub/Web/Pages/Pages.cs +++ b/PrivaPub/Web/Pages/Pages.cs @@ -127,4 +127,22 @@ namespace PrivaPub.Web.Pages return Page(); } } + + public class StargazingModel : PublicPageModel + { + readonly Microsoft.Extensions.Options.IOptionsMonitor _options; + readonly Microsoft.Extensions.Options.IOptionsMonitor _app; + + public StargazingModel(Microsoft.Extensions.Options.IOptionsMonitor options, + Microsoft.Extensions.Options.IOptionsMonitor app) + { + _options = options; + _app = app; + } + + public bool CrawlerEnabled => _options.CurrentValue.Crawler.Enabled; + public string UserAgent => Federation.Crawler.Stargazer.UserAgent(_app.CurrentValue.BackendBaseAddress); + + public void OnGet() => Harden(); + } } diff --git a/PrivaPub/Web/Pages/Stargazing.cshtml b/PrivaPub/Web/Pages/Stargazing.cshtml new file mode 100644 index 0000000..074b49f --- /dev/null +++ b/PrivaPub/Web/Pages/Stargazing.cshtml @@ -0,0 +1,48 @@ +@page "/stargazing" +@model PrivaPub.Web.Pages.StargazingModel +@{ + ViewData["Title"] = "Stargazing"; +} +
+

Stargazing

+
What this server records about other servers, and how to keep its crawler out.
+
+
+

Servers, never people

+

This server keeps statistics about the fediverse for teaching and curiosity. They name servers, + never accounts: what kind of software a server runs, what it exchanges with us, and how reliably. Distinct accounts are + only counted, through a key that is destroyed at the end of each day.

+

A server we exchange activities with is described once a week, from its public NodeInfo and, when it has one, its + Mastodon instance API. Its location comes from the address we reached, looked up in an offline database: only the + country is shown publicly for small servers, and only the CDN for servers behind one.

+
+
+

The crawler

+ @if (Model.CrawlerEnabled) + { +

The crawler is on on this server.

+ } + else + { +

The crawler is off on this server: it only learns about servers it already exchanges with.

+ } +

When on, it identifies itself as

+

@Model.UserAgent

+

It visits one server a minute, each at most once a week, and reads only:

+
    +
  • /robots.txt
    • +
  • /.well-known/nodeinfo and the NodeInfo document it points to
    • +
  • /api/v2/instance or /api/v1/instance
    • +
  • /api/v1/instance/peers, to find other servers
    • +
+

It never reads accounts, posts, timelines or directories.

+

Keeping it out

+

Add this to your server's robots.txt:

+ 
User-agent: @StargazerToken
+Disallow: /
+

If your robots.txt cannot be read because of a server error or a timeout, the crawler stays out too.

+
+ +@functions { + const string StargazerToken = PrivaPub.Federation.Crawler.Stargazer.Token; +}