M11: the opt-in crawler, PrivaPub-Stargazer, and /stargazing
Off by default (Statistics:Crawler:Enabled), as the owner decided. When it is on:
- CrawlPlan runs hourly, from StatisticsSchedule. It inserts the configured seeds and
queues up to HostsPerHour servers whose last visit is older than RevisitDays, are not
paused by the breaker and are not domain-blocked, spread across the hour.
- CrawlInstance visits one server at a time as
"PrivaPub-Stargazer/<ref> (+<base>/stargazing)":
- it reads robots.txt (RFC 9309: its own group first, then PrivaPub, then *; longest
rule wins; a 4xx allows everything; a 5xx or no answer keeps it out);
- it describes servers that only crawling ever found, through
InstanceDescriber.Describe with robots.txt as the path filter;
- it reads /api/v1/instance/peers through the new GetStringArray, which keeps what it
read from the first 1 MB instead of refusing a large list;
- it adds the names a server could ever be reached at as "crawled": DNS only,
punycode, no addresses, ports or hidden services, and the reserved test names only on
a test network. Never more than MaxNewHostsPerCrawl per visit or MaxHosts in all, and
never over a touched server.
- It reads nothing but robots.txt, NodeInfo, the instance API and the peers list.
IFederationHttp.GetText serves robots.txt, and HttpScope.Crawl carries the
User-Agent.
- /stargazing explains all this and how to keep the crawler out, says whether it is on,
and credits DB-IP. GET /clientapi/admin/statistics/crawler shows the frontier.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01ELjqpznMFMNrJoJUj6K5p2
This commit is contained in:
thepraandClaude Opus 5.5 committed
1 parent
c8a305ae92
commit
4f5df743ae
14 files changed
+882
-8
No files matched your search
@@ -0,0 +1,276 @@
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
|
||||
using MongoDB.Entities;
|
||||
|
||||
using PrivaPub.Federation.Crawler;
|
||||
using PrivaPub.Federation.Moderation;
|
||||
using PrivaPub.Federation.Objects;
|
||||
using PrivaPub.Infrastructure.Http;
|
||||
using PrivaPub.Infrastructure.Jobs;
|
||||
using PrivaPub.Infrastructure.Statistics;
|
||||
using PrivaPub.Models;
|
||||
using PrivaPub.Models.Federation;
|
||||
using PrivaPub.Models.Jobs;
|
||||
using PrivaPub.Tests.Support;
|
||||
using PrivaPub.Tests.Support.Host;
|
||||
|
||||
using System.Net;
|
||||
using System.Text;
|
||||
|
||||
namespace PrivaPub.Tests.Statistics
|
||||
{
|
||||
public class RobotsTxtTests
|
||||
{
|
||||
const string Robots = """
|
||||
# comments are ignored
|
||||
User-agent: *
|
||||
Disallow: /private
|
||||
Allow: /private/ok
|
||||
|
||||
User-agent: PrivaPub-Stargazer
|
||||
User-agent: other
|
||||
Disallow: /api/
|
||||
Allow: /api/v1/instance$
|
||||
|
||||
User-agent: greedy
|
||||
Disallow: /*.json$
|
||||
""";
|
||||
|
||||
[Fact]
|
||||
public void The_crawlers_own_group_wins_and_the_longest_rule_decides()
|
||||
{
|
||||
var mine = RobotsTxt.Parse(Robots, "PrivaPub-Stargazer");
|
||||
var anyone = RobotsTxt.Parse(Robots, "SomethingElse");
|
||||
var greedy = RobotsTxt.Parse(Robots, "greedy");
|
||||
|
||||
Assert.False(mine.Allows("/api/v1/instance/peers"));
|
||||
Assert.True(mine.Allows("/api/v1/instance"));
|
||||
Assert.True(mine.Allows("/private"));
|
||||
Assert.True(mine.Allows("/.well-known/nodeinfo"));
|
||||
Assert.False(anyone.Allows("/private/nope"));
|
||||
Assert.True(anyone.Allows("/private/ok/yes"));
|
||||
Assert.True(anyone.Allows("/api/v1/instance/peers"));
|
||||
Assert.False(greedy.Allows("/nodeinfo/2.1.json"));
|
||||
Assert.True(greedy.Allows("/nodeinfo/2.1.json?x=1"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void An_error_keeps_the_crawler_out_and_a_missing_file_lets_it_in()
|
||||
{
|
||||
Assert.True(RobotsTxt.From(404, default, Stargazer.Token).Allows("/anything"));
|
||||
Assert.False(RobotsTxt.From(503, default, Stargazer.Token).Allows("/anything"));
|
||||
Assert.False(RobotsTxt.From(0, default, Stargazer.Token).Allows("/anything"));
|
||||
Assert.False(RobotsTxt.From(200, "User-agent: *\nDisallow: /", Stargazer.Token).Allows("/robots.txt"));
|
||||
Assert.True(RobotsTxt.From(200, "User-agent: *\nDisallow:", Stargazer.Token).Allows("/x"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Peer_lists_yield_only_names_we_could_ever_reach()
|
||||
{
|
||||
var names = new[] { "Social.Example.org.", "10.0.0.1", "host.example.org:8443", "x.onion", "printer.local", "bücher.example.org", "a.test", "noDot", "social.example.org" };
|
||||
|
||||
Assert.Equal(new[] { "social.example.org", "xn--bcher-kva.example.org" }, PeerHosts.Filter(names, _ => true));
|
||||
Assert.Contains("a.test", PeerHosts.Filter(names, _ => true, testNetwork: true));
|
||||
Assert.Empty(PeerHosts.Filter(names, _ => false));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_truncated_peers_list_still_gives_what_was_read()
|
||||
{
|
||||
var items = new List<string>();
|
||||
FederationHttp.ReadStrings(Encoding.UTF8.GetBytes("[\"a.example\", 3, {\"x\":1}, \"b.example\", \"c.exa"), 10, items);
|
||||
var capped = new List<string>();
|
||||
FederationHttp.ReadStrings(Encoding.UTF8.GetBytes("[\"a\",\"b\",\"c\"]"), 2, capped);
|
||||
|
||||
Assert.Equal(new[] { "a.example", "b.example" }, items);
|
||||
Assert.Equal(new[] { "a", "b" }, capped);
|
||||
}
|
||||
}
|
||||
|
||||
[Trait("Category", "Integration")]
|
||||
public sealed class CrawlerTests : IAsyncLifetime
|
||||
{
|
||||
Peer _peer;
|
||||
readonly string _run = Guid.NewGuid().ToString("N")[..10];
|
||||
|
||||
public async ValueTask InitializeAsync()
|
||||
{
|
||||
Assert.SkipUnless(MongoFixture.Enabled, MongoFixture.Skip);
|
||||
_peer = await Peer.Start();
|
||||
}
|
||||
|
||||
public async ValueTask DisposeAsync()
|
||||
{
|
||||
if (_peer != default)
|
||||
await _peer.DisposeAsync();
|
||||
}
|
||||
|
||||
sealed class PeerDescriber : InstanceDescriber
|
||||
{
|
||||
readonly string _peer;
|
||||
|
||||
public PeerDescriber(IFederationHttp http, string peer) : base(http) => _peer = peer;
|
||||
|
||||
protected override string Address(string url) => $"{_peer}/{new Uri(url).Host}{new Uri(url).PathAndQuery}";
|
||||
}
|
||||
|
||||
sealed class PeerCrawler : InstanceCrawler
|
||||
{
|
||||
readonly string _peer;
|
||||
|
||||
public PeerCrawler(IFederationHttp http, InstanceDescriber describer, IDomainBlocks blocks, CrawlerOptions crawler, string peer, MemoryLedger ledger)
|
||||
: base(http, describer, blocks, new StaticOptions<StatisticsOptions>(new StatisticsOptions { Crawler = crawler }),
|
||||
new StaticOptions<AppConfiguration>(new AppConfiguration { BackendBaseAddress = "https://privapub.test" }),
|
||||
new StaticOptions<FederationOptions>(new FederationOptions { AllowPrivateNetworks = true }), ledger) => _peer = peer;
|
||||
|
||||
protected override string Address(string url) => $"{_peer}/{new Uri(url).Host}{new Uri(url).PathAndQuery}";
|
||||
}
|
||||
|
||||
(PeerCrawler Crawler, MemoryLedger Ledger) Crawler(CrawlerOptions options, IDomainBlocks blocks = default)
|
||||
{
|
||||
var http = Peer.Http();
|
||||
var ledger = new MemoryLedger();
|
||||
return (new PeerCrawler(http, new PeerDescriber(http, _peer.A), blocks ?? new NoBlocks(), options, _peer.A, ledger), ledger);
|
||||
}
|
||||
|
||||
string Host(string name) => $"{name}{_run}.example.org";
|
||||
|
||||
void Serve(string host, string robots = default, int robotsStatus = 200, params string[] peers)
|
||||
{
|
||||
if (robots != default)
|
||||
_peer.ServeText($"/{host}/robots.txt", robots, "text/plain");
|
||||
else
|
||||
_peer.Answer($"/{host}/robots.txt", robotsStatus);
|
||||
_peer.Serve($"/{host}/.well-known/nodeinfo", $$"""{"links":[{"rel":"http://nodeinfo.diaspora.software/ns/schema/2.0","href":"https://{{host}}/nodeinfo/2.0"}]}""");
|
||||
_peer.Serve($"/{host}/nodeinfo/2.0", """{"software":{"name":"lemmy","version":"0.19.9"},"usage":{"users":{"total":3}}}""");
|
||||
_peer.Serve($"/{host}/api/v1/instance/peers", "[" + string.Join(",", peers.Select(p => $"\"{p}\"")) + "]");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_crawl_describes_a_crawled_server_and_learns_its_peers_politely()
|
||||
{
|
||||
var token = TestContext.Current.CancellationToken;
|
||||
var host = Host("seed");
|
||||
var blocked = Host("blocked");
|
||||
Serve(host, "User-agent: *\nDisallow: /private", 200, Host("new1"), Host("new2"), blocked, "10.0.0.1", Host("new1"));
|
||||
var blocks = new DomainBlocks(NullLogger<DomainBlocks>.Instance);
|
||||
blocks.Load(new[] { new DomainBlock { Domain = blocked, Severity = DomainBlockSeverity.Silence } });
|
||||
await DB.Default.SaveAsync(new RemoteInstance { Host = host, Seen = "crawled" }, token);
|
||||
var (crawler, ledger) = Crawler(new CrawlerOptions { Enabled = true }, blocks);
|
||||
|
||||
await crawler.Handle(new Job { Kind = JobKind.CrawlInstance, Payload = host }, token);
|
||||
|
||||
var seed = await DB.Default.Find<RemoteInstance>().Match(i => i.Host == host).ExecuteSingleAsync(token);
|
||||
Assert.Equal(("lemmy", 5, false), (seed.Software, seed.PeersCount!.Value, seed.RobotsDisallowed!.Value));
|
||||
Assert.NotNull(seed.LastCrawledAt);
|
||||
var found = await DB.Default.Find<RemoteInstance>().Match(i => i.Host == Host("new1") || i.Host == Host("new2") || i.Host == blocked).ExecuteAsync(token);
|
||||
Assert.Equal(new[] { Host("new1"), Host("new2") }, found.Select(i => i.Host).OrderBy(h => h));
|
||||
Assert.All(found, i => Assert.Equal("crawled", i.Seen));
|
||||
var visited = _peer.Requests.Where(r => r.Path.StartsWith("/" + host + "/")).ToList();
|
||||
Assert.All(visited, r => Assert.Contains(r.Path[(host.Length + 1)..], new[] { "/robots.txt", "/.well-known/nodeinfo", "/nodeinfo/2.0", "/api/v1/instance/peers" }));
|
||||
Assert.All(visited, r => Assert.StartsWith("PrivaPub-Stargazer/", r.Headers["User-Agent"]));
|
||||
Assert.All(visited, r => Assert.Contains("/stargazing)", r.Headers["User-Agent"]));
|
||||
Assert.Equal(("crawl", "ok"), (Assert.Single(ledger.Events).Event.Channel, ledger.Events.Single().Event.Outcome));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Robots_txt_and_its_absence_are_obeyed_and_a_touched_server_is_not_described_again()
|
||||
{
|
||||
var token = TestContext.Current.CancellationToken;
|
||||
var shy = Host("shy");
|
||||
var broken = Host("broken");
|
||||
var touched = Host("touched");
|
||||
Serve(shy, "User-agent: PrivaPub-Stargazer\nDisallow: /api/", 200, Host("hidden"));
|
||||
Serve(broken, default, 503, Host("never"));
|
||||
Serve(touched, "", 200);
|
||||
await DB.Default.SaveAsync(new[] { new RemoteInstance { Host = shy, Seen = "crawled" }, new RemoteInstance { Host = touched, Seen = "touched" } }, token);
|
||||
var (crawler, ledger) = Crawler(new CrawlerOptions { Enabled = true });
|
||||
|
||||
await crawler.Handle(new Job { Payload = shy }, token);
|
||||
await crawler.Handle(new Job { Payload = broken }, token);
|
||||
await crawler.Handle(new Job { Payload = touched }, token);
|
||||
|
||||
Assert.DoesNotContain(_peer.Requests, r => r.Path == $"/{shy}/api/v1/instance/peers");
|
||||
Assert.Contains(_peer.Requests, r => r.Path == $"/{shy}/nodeinfo/2.0");
|
||||
Assert.Equal(new[] { $"/{broken}/robots.txt" }, _peer.Requests.Where(r => r.Path.StartsWith($"/{broken}/")).Select(r => r.Path));
|
||||
Assert.True((await DB.Default.Find<RemoteInstance>().Match(i => i.Host == broken).ExecuteSingleAsync(token)).RobotsDisallowed);
|
||||
Assert.DoesNotContain(_peer.Requests, r => r.Path.StartsWith($"/{touched}/.well-known/"));
|
||||
Assert.Contains(ledger.Events, e => e.Event.Host == broken && e.Event.Reason == "robots-disallowed");
|
||||
Assert.False(await DB.Default.Find<RemoteInstance>().Match(i => i.Host == Host("hidden") || i.Host == Host("never")).ExecuteAnyAsync(token));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_disabled_crawler_does_nothing_and_the_caps_hold()
|
||||
{
|
||||
var token = TestContext.Current.CancellationToken;
|
||||
var host = Host("capped");
|
||||
Serve(host, "", 200, Enumerable.Range(0, 20).Select(i => Host($"many{i}-")).ToArray());
|
||||
await DB.Default.SaveAsync(new RemoteInstance { Host = host, Seen = "touched" }, token);
|
||||
|
||||
await Crawler(new CrawlerOptions { Enabled = false }).Crawler.Handle(new Job { Payload = host }, token);
|
||||
Assert.DoesNotContain(_peer.Requests, r => r.Path.StartsWith($"/{host}/"));
|
||||
|
||||
await Crawler(new CrawlerOptions { Enabled = true, MaxNewHostsPerCrawl = 3 }).Crawler.Handle(new Job { Payload = host }, token);
|
||||
Assert.Equal(3, await DB.Default.CountAsync<RemoteInstance>(i => i.Host.StartsWith("many") && i.Host.EndsWith($"{_run}.example.org"), cancellation: token));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task The_planner_spreads_due_servers_across_the_hour_and_skips_fresh_and_blocked_ones()
|
||||
{
|
||||
var token = TestContext.Current.CancellationToken;
|
||||
var due = Host("due");
|
||||
var fresh = Host("fresh");
|
||||
var blocked = Host("plannedblock");
|
||||
await DB.Default.SaveAsync(new[]
|
||||
{
|
||||
new RemoteInstance { Host = due, Seen = "crawled", LastCrawledAt = new DateTime(2001, 1, 1, 0, 0, 0, DateTimeKind.Utc) },
|
||||
new RemoteInstance { Host = fresh, Seen = "crawled", LastCrawledAt = DateTime.UtcNow },
|
||||
new RemoteInstance { Host = blocked, Seen = "crawled", LastCrawledAt = new DateTime(2001, 1, 1, 0, 0, 0, DateTimeKind.Utc) }
|
||||
}, token);
|
||||
var blocks = new DomainBlocks(NullLogger<DomainBlocks>.Instance);
|
||||
blocks.Load(new[] { new DomainBlock { Domain = blocked, Severity = DomainBlockSeverity.Suspend } });
|
||||
var options = new StatisticsOptions { Crawler = new CrawlerOptions { Enabled = true, HostsPerHour = 600, Seeds = new() { Host("seedling"), "10.1.1.1" } } };
|
||||
|
||||
await new CrawlPlanner(new JobQueue(j => j.DedupeKey == "never"), blocks, new StaticOptions<StatisticsOptions>(options)).Handle(new Job(), token);
|
||||
await new CrawlPlanner(new JobQueue(j => j.DedupeKey == "never"), blocks, new StaticOptions<StatisticsOptions>(new StatisticsOptions())).Handle(new Job(), token);
|
||||
|
||||
var planned = (await DB.Default.Find<Job>().Match(j => j.Kind == JobKind.CrawlInstance && j.Payload.EndsWith($"{_run}.example.org")).ExecuteAsync(token)).Select(j => j.Payload).ToList();
|
||||
Assert.Contains(due, planned);
|
||||
Assert.Contains(Host("seedling"), planned);
|
||||
Assert.DoesNotContain(fresh, planned);
|
||||
Assert.DoesNotContain(blocked, planned);
|
||||
Assert.Equal("crawled", (await DB.Default.Find<RemoteInstance>().Match(i => i.Host == Host("seedling")).ExecuteSingleAsync(token)).Seen);
|
||||
}
|
||||
}
|
||||
|
||||
[Trait("Category", "Integration")]
|
||||
public sealed class StargazingPageTests : IAsyncLifetime
|
||||
{
|
||||
PrivaPubHost _host;
|
||||
|
||||
public async ValueTask InitializeAsync()
|
||||
{
|
||||
Assert.SkipUnless(MongoFixture.Enabled, MongoFixture.Skip);
|
||||
_host = await PrivaPubHost.Shared();
|
||||
}
|
||||
|
||||
public ValueTask DisposeAsync() => ValueTask.CompletedTask;
|
||||
|
||||
[Fact]
|
||||
public async Task The_page_names_the_crawler_says_it_is_off_and_how_to_keep_it_out()
|
||||
{
|
||||
using var client = _host.Client();
|
||||
|
||||
using var response = await client.GetAsync("/stargazing", TestContext.Current.CancellationToken);
|
||||
var page = await response.Content.ReadAsStringAsync(TestContext.Current.CancellationToken);
|
||||
|
||||
Assert.Equal(HttpStatusCode.OK, response.StatusCode);
|
||||
Assert.Contains("PrivaPub-Stargazer/", page);
|
||||
Assert.Contains("https://privapub.test/stargazing", page);
|
||||
Assert.Contains("<strong>off</strong>", page);
|
||||
Assert.Contains("Disallow: /", page);
|
||||
Assert.Contains("DB-IP", page);
|
||||
Assert.StartsWith("default-src 'none'", response.Headers.GetValues("Content-Security-Policy").Single());
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user