M8: every server we touch is described, located and snapshotted weekly
- Touches: the ledger marks a server as touched when it sends us a verified activity, when
we exchange activities with it, or when we read its actors, keys, objects or WebFinger.
It upserts RemoteInstance.Seen, FirstSeenAt and LastSeenAt at most hourly per server, and
queues one DescribeInstance a week with the same dedupe key ObjectRecords uses. Suspended
servers and pages behind link previews are never described. Migration _010 marks the
servers already known as touched, with their dates.
- InstanceDescriber.Describe(host, crawled, allowed) reads:
- NodeInfo 2.2/2.1/2.0, now with its published user counts, posts, comments,
description, languages and schema version;
- for software with a Mastodon API, /api/v2/instance falling back to v1: title,
languages, registration mode, character limit, API version, source URL.
It never keeps a contact as a field; the raw document is kept for the admin only. It
locates the server from the address our connection reached (DB-IP Lite city and ASN, the
CDN named when fronted) and writes a RemoteInstanceSnapshot per ISO week, unreachable
weeks included. A crawled server is upserted as crawled only on insert, so it never
downgrades a touched one, and robots.txt can deny any path.
- PublicGeo.Project is the only public form of a location: a CDN-fronted server shows its
CDN only, a server reporting at least ten users shows its city, coordinates and network,
any other only its country.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01ELjqpznMFMNrJoJUj6K5p2
This commit is contained in:
thepraandClaude Opus 5.5 committed
1 parent
fc5bb9511f
commit
c8a305ae92
10 files changed
+757
-72
No files matched your search
@@ -0,0 +1,261 @@
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
|
||||
using MongoDB.Entities;
|
||||
|
||||
using PrivaPub.Domain.Statistics;
|
||||
using PrivaPub.Federation.Moderation;
|
||||
using PrivaPub.Federation.Objects;
|
||||
using PrivaPub.Infrastructure.Data.Migrations;
|
||||
using PrivaPub.Infrastructure.Geo;
|
||||
using PrivaPub.Infrastructure.Http;
|
||||
using PrivaPub.Infrastructure.Jobs;
|
||||
using PrivaPub.Infrastructure.Statistics;
|
||||
using PrivaPub.Models.Federation;
|
||||
using PrivaPub.Models.Jobs;
|
||||
using PrivaPub.Models.Statistics;
|
||||
using PrivaPub.Tests.Support;
|
||||
|
||||
using System.Net;
|
||||
using System.Text.Json;
|
||||
|
||||
namespace PrivaPub.Tests.Statistics
|
||||
{
|
||||
public class InstanceDocumentsTests
|
||||
{
|
||||
static JsonElement Json(string text) => JsonDocument.Parse(text).RootElement;
|
||||
|
||||
[Fact]
|
||||
public void NodeInfo_gives_software_usage_and_languages_whatever_their_shape()
|
||||
{
|
||||
var facts = InstanceDocuments.NodeInfo(Json("""
|
||||
{"software":{"name":"mastodon","version":"4.7.3"},"protocols":["activitypub"],"openRegistrations":false,
|
||||
"usage":{"users":{"total":"1234","activeMonth":56.0,"activeHalfyear":-3},"localPosts":99999999999,"localComments":7},
|
||||
"metadata":{"nodeName":"Example","nodeDescription":"<p>A <b>small</b>\n place</p>","languages":["en","DE","klingon!"]}}
|
||||
"""), "2.1");
|
||||
|
||||
Assert.Equal(("mastodon", "4.7.3", "Example", "2.1"), (facts.Software, facts.SoftwareVersion, facts.NodeName, facts.Version));
|
||||
Assert.Equal("A small place", facts.NodeDescription);
|
||||
Assert.Equal(new[] { "activitypub" }, facts.Protocols);
|
||||
Assert.False(facts.OpenRegistrations);
|
||||
Assert.Equal(1234, facts.UsersTotal);
|
||||
Assert.Equal(56, facts.UsersActiveMonth);
|
||||
Assert.Equal(0, facts.UsersActiveHalfyear);
|
||||
Assert.Equal(1_000_000_000, facts.LocalPosts);
|
||||
Assert.Equal(7, facts.LocalComments);
|
||||
Assert.Equal(new[] { "en", "de" }, facts.Languages);
|
||||
Assert.Null(InstanceDocuments.NodeInfo(Json("{}"), "2.0").UsersTotal);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void The_newest_https_nodeinfo_link_is_chosen()
|
||||
{
|
||||
var links = Json("""
|
||||
{"links":[{"rel":"http://nodeinfo.diaspora.software/ns/schema/2.0","href":"https://a.example/2.0"},
|
||||
{"rel":"http://nodeinfo.diaspora.software/ns/schema/2.1","href":"http://a.example/2.1"},
|
||||
{"rel":"http://nodeinfo.diaspora.software/ns/schema/2.2","href":"https://a.example/2.2"}]}
|
||||
""");
|
||||
|
||||
Assert.Equal(("https://a.example/2.2", "2.2"), InstanceDocuments.NodeInfoLink(links));
|
||||
Assert.Equal(default, InstanceDocuments.NodeInfoLink(Json("[]")));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void The_instance_api_gives_registration_mode_limits_and_never_a_contact()
|
||||
{
|
||||
var v2 = InstanceDocuments.InstanceApi(Json("""
|
||||
{"title":"Example","languages":["en"],"registrations":{"enabled":true,"approval_required":true},
|
||||
"configuration":{"statuses":{"max_characters":500}},"api_versions":{"mastodon":7},"source_url":"https://github.com/mastodon/mastodon",
|
||||
"contact":{"email":"admin@a.example","account":{"username":"admin"}}}
|
||||
"""));
|
||||
var v1 = InstanceDocuments.InstanceApi(Json("""{"title":"Old","registrations":false,"max_toot_chars":"5000"}"""));
|
||||
|
||||
Assert.Equal(("Example", "approval", 500, 7, "https://github.com/mastodon/mastodon"), (v2.Title, v2.RegistrationMode, v2.MaxPostChars!.Value, v2.ApiVersion!.Value, v2.SourceUrl));
|
||||
Assert.Equal(new[] { "en" }, v2.Languages);
|
||||
Assert.DoesNotContain("admin", System.Text.Json.JsonSerializer.Serialize(v2));
|
||||
Assert.Equal(("closed", 5000), (v1.RegistrationMode, v1.MaxPostChars!.Value));
|
||||
Assert.True(InstanceDocuments.SpeaksMastodonApi("GoToSocial"));
|
||||
Assert.False(InstanceDocuments.SpeaksMastodonApi("lemmy"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void The_public_location_depends_on_size_and_on_a_cdn()
|
||||
{
|
||||
var geo = new InstanceGeo { Country = "DE", City = "Berlin", Latitude = 52.5, Longitude = 13.4, AsnOrg = "Hetzner", Asn = 24940 };
|
||||
|
||||
Assert.Equal(new PublicLocation("DE", "Berlin", 52.5, 13.4, "Hetzner", null), PublicGeo.Project(new RemoteInstance { Geo = geo, UsersTotal = 10 }, 10));
|
||||
Assert.Equal(new PublicLocation("DE", null, null, null, null, null), PublicGeo.Project(new RemoteInstance { Geo = geo, UsersTotal = 1 }, 10));
|
||||
Assert.Equal(new PublicLocation("DE", null, null, null, null, null), PublicGeo.Project(new RemoteInstance { Geo = geo }, 10));
|
||||
Assert.Equal(new PublicLocation(null, null, null, null, null, "Cloudflare"),
|
||||
PublicGeo.Project(new RemoteInstance { Geo = new InstanceGeo { Country = "US", City = "SF", Cdn = "Cloudflare" }, UsersTotal = 5000 }, 10));
|
||||
Assert.Null(PublicGeo.Project(new RemoteInstance(), 10));
|
||||
}
|
||||
}
|
||||
|
||||
[Trait("Category", "Integration")]
|
||||
public sealed class DescribeTests : IAsyncLifetime
|
||||
{
|
||||
Peer _peer;
|
||||
|
||||
public async ValueTask InitializeAsync()
|
||||
{
|
||||
Assert.SkipUnless(MongoFixture.Enabled, MongoFixture.Skip);
|
||||
_peer = await Peer.Start();
|
||||
}
|
||||
|
||||
public async ValueTask DisposeAsync()
|
||||
{
|
||||
if (_peer != default)
|
||||
await _peer.DisposeAsync();
|
||||
}
|
||||
|
||||
sealed class PeerDescriber : InstanceDescriber
|
||||
{
|
||||
readonly string _peer;
|
||||
|
||||
public PeerDescriber(IFederationHttp http, IGeoLocator geo, IConnectedAddresses addresses, string peer) : base(http, geo, addresses) => _peer = peer;
|
||||
|
||||
protected override string Address(string url) => _peer + new Uri(url).PathAndQuery;
|
||||
}
|
||||
|
||||
sealed class FixedGeo : IGeoLocator
|
||||
{
|
||||
public GeoFix Locate(IPAddress address) => new("NL", "Amsterdam", 52.4, 4.9, 60781, "LeaseWeb");
|
||||
public string Source => "test";
|
||||
}
|
||||
|
||||
void ServeServer(string prefix, string software, bool v2 = true)
|
||||
{
|
||||
_peer.Serve($"{prefix}/.well-known/nodeinfo", $$"""{"links":[{"rel":"http://nodeinfo.diaspora.software/ns/schema/2.1","href":"https://ignored.example/nodeinfo/2.1"}]}""");
|
||||
_peer.Serve($"{prefix}/nodeinfo/2.1", $$$"""{"software":{"name":"{{{software}}}","version":"1.0"},"usage":{"users":{"total":42,"activeMonth":7}},"openRegistrations":true}""");
|
||||
if (v2)
|
||||
_peer.Serve($"{prefix}/api/v2/instance", """{"title":"A test server","languages":["it"],"registrations":{"enabled":true,"approval_required":false}}""");
|
||||
_peer.Serve($"{prefix}/api/v1/instance", """{"title":"An old test server","registrations":true}""");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_server_is_described_located_and_snapshotted_weekly()
|
||||
{
|
||||
var token = TestContext.Current.CancellationToken;
|
||||
var host = $"described{Guid.NewGuid():N}.example";
|
||||
ServeServer("/" + host, "mastodon");
|
||||
var addresses = new ConnectedAddresses();
|
||||
var describer = new PeerDescriber(Peer.Http(connected: addresses), new FixedGeo(), addresses, _peer.A + "/" + host);
|
||||
|
||||
Assert.True(await describer.Describe(host, crawled: false, default, token));
|
||||
|
||||
var instance = await DB.Default.Find<RemoteInstance>().Match(i => i.Host == host).ExecuteSingleAsync(token);
|
||||
Assert.Equal(("mastodon", 42L, 7L, "2.1"), (instance.Software, instance.UsersTotal!.Value, instance.UsersActiveMonth!.Value, instance.NodeInfoVersion));
|
||||
Assert.Equal(("A test server", "open", "mastodon-v2"), (instance.Title, instance.RegistrationMode, instance.InstanceApi));
|
||||
Assert.Equal(new[] { "it" }, instance.Languages);
|
||||
Assert.Equal(("NL", "Amsterdam", "LeaseWeb", "test"), (instance.Geo.Country, instance.Geo.City, instance.Geo.AsnOrg, instance.Geo.Source));
|
||||
Assert.False(instance.Geo.Ipv6);
|
||||
var snapshot = await DB.Default.Find<RemoteInstanceSnapshot>().Match(s => s.Host == host).ExecuteSingleAsync(token);
|
||||
Assert.True(snapshot.Reachable);
|
||||
Assert.Equal((42L, "NL"), (snapshot.UsersTotal!.Value, snapshot.Country));
|
||||
Assert.Equal(InstanceDescriber.Week(DateTime.UtcNow), snapshot.Week);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Without_api_v2_the_v1_api_is_read_and_an_unreachable_week_is_recorded_too()
|
||||
{
|
||||
var token = TestContext.Current.CancellationToken;
|
||||
var old = $"old{Guid.NewGuid():N}.example";
|
||||
var gone = $"gone{Guid.NewGuid():N}.example";
|
||||
ServeServer("/" + old, "pleroma", v2: false);
|
||||
|
||||
await new PeerDescriber(Peer.Http(), default, default, _peer.A + "/" + old).Describe(old, crawled: false, default, token);
|
||||
Assert.False(await new PeerDescriber(Peer.Http(), default, default, _peer.A + "/" + gone).Describe(gone, crawled: false, default, token));
|
||||
|
||||
var oldOne = await DB.Default.Find<RemoteInstance>().Match(i => i.Host == old).ExecuteSingleAsync(token);
|
||||
Assert.Equal(("mastodon-v1", "An old test server", "open"), (oldOne.InstanceApi, oldOne.Title, oldOne.RegistrationMode));
|
||||
Assert.Null(oldOne.Geo);
|
||||
var goneOne = await DB.Default.Find<RemoteInstance>().Match(i => i.Host == gone).ExecuteSingleAsync(token);
|
||||
Assert.Equal("no NodeInfo", goneOne.DescriptionError);
|
||||
Assert.False((await DB.Default.Find<RemoteInstanceSnapshot>().Match(s => s.Host == gone).ExecuteSingleAsync(token)).Reachable);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_crawled_server_never_downgrades_a_touched_one_and_robots_are_obeyed()
|
||||
{
|
||||
var token = TestContext.Current.CancellationToken;
|
||||
var touched = $"touched{Guid.NewGuid():N}.example";
|
||||
var shy = $"shy{Guid.NewGuid():N}.example";
|
||||
ServeServer("/" + touched, "gotosocial");
|
||||
ServeServer("/" + shy, "mastodon");
|
||||
await DB.Default.SaveAsync(new RemoteInstance { Host = touched, Seen = "touched" }, token);
|
||||
|
||||
await new PeerDescriber(Peer.Http(), default, default, _peer.A + "/" + touched).Describe(touched, crawled: true, default, token);
|
||||
await new PeerDescriber(Peer.Http(), default, default, _peer.A + "/" + shy).Describe(shy, crawled: true, path => !path.StartsWith("/api/"), token);
|
||||
|
||||
Assert.Equal("touched", (await DB.Default.Find<RemoteInstance>().Match(i => i.Host == touched).ExecuteSingleAsync(token)).Seen);
|
||||
var shyOne = await DB.Default.Find<RemoteInstance>().Match(i => i.Host == shy).ExecuteSingleAsync(token);
|
||||
Assert.Equal(("crawled", "mastodon"), (shyOne.Seen, shyOne.Software));
|
||||
Assert.Null(shyOne.InstanceApi);
|
||||
Assert.DoesNotContain(_peer.Requests, r => r.Path.StartsWith("/" + shy + "/api/", StringComparison.Ordinal));
|
||||
Assert.Contains(_peer.Requests, r => r.Path == "/" + shy + "/nodeinfo/2.1");
|
||||
}
|
||||
}
|
||||
|
||||
[Trait("Category", "Integration")]
|
||||
public sealed class TouchTests : IAsyncLifetime
|
||||
{
|
||||
public ValueTask InitializeAsync()
|
||||
{
|
||||
Assert.SkipUnless(MongoFixture.Enabled, MongoFixture.Skip);
|
||||
return ValueTask.CompletedTask;
|
||||
}
|
||||
|
||||
public ValueTask DisposeAsync() => ValueTask.CompletedTask;
|
||||
|
||||
[Fact]
|
||||
public async Task A_server_we_exchange_with_is_seen_and_described_once_a_week_unless_suspended()
|
||||
{
|
||||
var token = TestContext.Current.CancellationToken;
|
||||
var friend = $"friend{Guid.NewGuid():N}.example";
|
||||
var spammer = $"spam{Guid.NewGuid():N}.example";
|
||||
var linked = $"linked{Guid.NewGuid():N}.example";
|
||||
var looked = $"looked{Guid.NewGuid():N}.example";
|
||||
var blocks = new DomainBlocks(NullLogger<DomainBlocks>.Instance);
|
||||
blocks.Load(new[] { new DomainBlock { Domain = spammer, Severity = DomainBlockSeverity.Suspend } });
|
||||
var ledger = new InteractionLedger(new InteractionSalts(), new StaticOptions<StatisticsOptions>(new StatisticsOptions()), NullLogger<InteractionLedger>.Instance,
|
||||
new JobQueue(j => j.DedupeKey == "never"), blocks);
|
||||
|
||||
ledger.Record(new InteractionEvent { Host = friend, Channel = "in", Activity = "Create", Outcome = "accepted" }, "https://" + friend + "/users/a");
|
||||
ledger.Record(new InteractionEvent { Host = friend, Channel = "out", Activity = "Like", Outcome = "ok" });
|
||||
ledger.Record(new InteractionEvent { Host = spammer, Channel = "out", Activity = "Reject", Outcome = "ok" });
|
||||
ledger.Record(new InteractionEvent { Host = linked, Channel = "preview", Outcome = "ok" });
|
||||
ledger.Count(looked, "http:actor:ok");
|
||||
await ledger.Flush(token);
|
||||
|
||||
var seen = await DB.Default.Find<RemoteInstance>().Match(i => i.Host == friend).ExecuteSingleAsync(token);
|
||||
Assert.Equal("touched", seen.Seen);
|
||||
Assert.NotNull(seen.FirstSeenAt);
|
||||
Assert.Single(await DB.Default.Find<Job>().Match(j => j.Kind == JobKind.DescribeInstance && j.Payload == friend).ExecuteAsync(token));
|
||||
Assert.True(await DB.Default.Find<Job>().Match(j => j.Kind == JobKind.DescribeInstance && j.Payload == looked).ExecuteAnyAsync(token));
|
||||
Assert.False(await DB.Default.Find<Job>().Match(j => j.Kind == JobKind.DescribeInstance && j.Payload == spammer).ExecuteAnyAsync(token));
|
||||
Assert.False(await DB.Default.Find<RemoteInstance>().Match(i => i.Host == linked).ExecuteAnyAsync(token));
|
||||
}
|
||||
}
|
||||
|
||||
[Xunit.Collection(nameof(Exclusive))]
|
||||
[Trait("Category", "Integration")]
|
||||
public sealed class TouchedMigrationTests
|
||||
{
|
||||
[Fact]
|
||||
public async Task Known_servers_become_touched_with_their_dates()
|
||||
{
|
||||
Assert.SkipUnless(MongoFixture.Enabled, MongoFixture.Skip);
|
||||
var token = TestContext.Current.CancellationToken;
|
||||
var host = $"legacy{Guid.NewGuid():N}.example";
|
||||
var success = DateTime.UtcNow.AddDays(-2);
|
||||
await DB.Default.SaveAsync(new RemoteInstance { Host = host, LastSuccessAt = success, DescribedAt = success.AddDays(-1) }, token);
|
||||
|
||||
await new _010_remote_instances_were_touched().UpgradeAsync();
|
||||
|
||||
var instance = await DB.Default.Find<RemoteInstance>().Match(i => i.Host == host).ExecuteSingleAsync(token);
|
||||
Assert.Equal("touched", instance.Seen);
|
||||
Assert.NotNull(instance.FirstSeenAt);
|
||||
Assert.InRange(instance.LastSeenAt!.Value, success.AddSeconds(-1), success.AddSeconds(1));
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user