using Microsoft.Extensions.Logging.Abstractions; using MongoDB.Entities; using PrivaPub.Domain.Statistics; using PrivaPub.Federation.Moderation; using PrivaPub.Federation.Objects; using PrivaPub.Infrastructure.Data.Migrations; using PrivaPub.Infrastructure.Geo; using PrivaPub.Infrastructure.Http; using PrivaPub.Infrastructure.Jobs; using PrivaPub.Infrastructure.Statistics; using PrivaPub.Models.Federation; using PrivaPub.Models.Jobs; using PrivaPub.Models.Statistics; using PrivaPub.Tests.Support; using System.Net; using System.Text.Json; namespace PrivaPub.Tests.Statistics { public class InstanceDocumentsTests { static JsonElement Json(string text) => JsonDocument.Parse(text).RootElement; [Fact] public void NodeInfo_gives_software_usage_and_languages_whatever_their_shape() { var facts = InstanceDocuments.NodeInfo(Json(""" {"software":{"name":"mastodon","version":"4.7.3"},"protocols":["activitypub"],"openRegistrations":false, "usage":{"users":{"total":"1234","activeMonth":56.0,"activeHalfyear":-3},"localPosts":99999999999,"localComments":7}, "metadata":{"nodeName":"Example","nodeDescription":"

A small\n place

","languages":["en","DE","klingon!"]}} """), "2.1"); Assert.Equal(("mastodon", "4.7.3", "Example", "2.1"), (facts.Software, facts.SoftwareVersion, facts.NodeName, facts.Version)); Assert.Equal("A small place", facts.NodeDescription); Assert.Equal(new[] { "activitypub" }, facts.Protocols); Assert.False(facts.OpenRegistrations); Assert.Equal(1234, facts.UsersTotal); Assert.Equal(56, facts.UsersActiveMonth); Assert.Equal(0, facts.UsersActiveHalfyear); Assert.Equal(1_000_000_000, facts.LocalPosts); Assert.Equal(7, facts.LocalComments); Assert.Equal(new[] { "en", "de" }, facts.Languages); Assert.Null(InstanceDocuments.NodeInfo(Json("{}"), "2.0").UsersTotal); } [Fact] public void The_newest_https_nodeinfo_link_is_chosen() { var links = Json(""" {"links":[{"rel":"http://nodeinfo.diaspora.software/ns/schema/2.0","href":"https://a.example/2.0"}, {"rel":"http://nodeinfo.diaspora.software/ns/schema/2.1","href":"http://a.example/2.1"}, {"rel":"http://nodeinfo.diaspora.software/ns/schema/2.2","href":"https://a.example/2.2"}]} """); Assert.Equal(("https://a.example/2.2", "2.2"), InstanceDocuments.NodeInfoLink(links)); Assert.Equal(default, InstanceDocuments.NodeInfoLink(Json("[]"))); } [Fact] public void The_instance_api_gives_registration_mode_limits_and_never_a_contact() { var v2 = InstanceDocuments.InstanceApi(Json(""" {"title":"Example","languages":["en"],"registrations":{"enabled":true,"approval_required":true}, "configuration":{"statuses":{"max_characters":500}},"api_versions":{"mastodon":7},"source_url":"https://github.com/mastodon/mastodon", "contact":{"email":"admin@a.example","account":{"username":"admin"}}} """)); var v1 = InstanceDocuments.InstanceApi(Json("""{"title":"Old","registrations":false,"max_toot_chars":"5000"}""")); Assert.Equal(("Example", "approval", 500, 7, "https://github.com/mastodon/mastodon"), (v2.Title, v2.RegistrationMode, v2.MaxPostChars!.Value, v2.ApiVersion!.Value, v2.SourceUrl)); Assert.Equal(new[] { "en" }, v2.Languages); Assert.DoesNotContain("admin", System.Text.Json.JsonSerializer.Serialize(v2)); Assert.Equal(("closed", 5000), (v1.RegistrationMode, v1.MaxPostChars!.Value)); Assert.True(InstanceDocuments.SpeaksMastodonApi("GoToSocial")); Assert.False(InstanceDocuments.SpeaksMastodonApi("lemmy")); } [Fact] public void The_public_location_depends_on_size_and_on_a_cdn() { var geo = new InstanceGeo { Country = "DE", City = "Berlin", Latitude = 52.5, Longitude = 13.4, AsnOrg = "Hetzner", Asn = 24940 }; Assert.Equal(new PublicLocation("DE", "Berlin", 52.5, 13.4, "Hetzner", null), PublicGeo.Project(new RemoteInstance { Geo = geo, UsersTotal = 10 }, 10)); Assert.Equal(new PublicLocation("DE", null, null, null, null, null), PublicGeo.Project(new RemoteInstance { Geo = geo, UsersTotal = 1 }, 10)); Assert.Equal(new PublicLocation("DE", null, null, null, null, null), PublicGeo.Project(new RemoteInstance { Geo = geo }, 10)); Assert.Equal(new PublicLocation(null, null, null, null, null, "Cloudflare"), PublicGeo.Project(new RemoteInstance { Geo = new InstanceGeo { Country = "US", City = "SF", Cdn = "Cloudflare" }, UsersTotal = 5000 }, 10)); Assert.Null(PublicGeo.Project(new RemoteInstance(), 10)); } } [Trait("Category", "Integration")] public sealed class DescribeTests : IAsyncLifetime { Peer _peer; public async ValueTask InitializeAsync() { Assert.SkipUnless(MongoFixture.Enabled, MongoFixture.Skip); _peer = await Peer.Start(); } public async ValueTask DisposeAsync() { if (_peer != default) await _peer.DisposeAsync(); } sealed class PeerDescriber : InstanceDescriber { readonly string _peer; public PeerDescriber(IFederationHttp http, IGeoLocator geo, IConnectedAddresses addresses, string peer) : base(http, geo, addresses) => _peer = peer; protected override string Address(string url) => _peer + new Uri(url).PathAndQuery; } sealed class FixedGeo : IGeoLocator { public GeoFix Locate(IPAddress address) => new("NL", "Amsterdam", 52.4, 4.9, 60781, "LeaseWeb"); public string Source => "test"; } void ServeServer(string prefix, string software, bool v2 = true) { _peer.Serve($"{prefix}/.well-known/nodeinfo", $$"""{"links":[{"rel":"http://nodeinfo.diaspora.software/ns/schema/2.1","href":"https://ignored.example/nodeinfo/2.1"}]}"""); _peer.Serve($"{prefix}/nodeinfo/2.1", $$$"""{"software":{"name":"{{{software}}}","version":"1.0"},"usage":{"users":{"total":42,"activeMonth":7}},"openRegistrations":true}"""); if (v2) _peer.Serve($"{prefix}/api/v2/instance", """{"title":"A test server","languages":["it"],"registrations":{"enabled":true,"approval_required":false}}"""); _peer.Serve($"{prefix}/api/v1/instance", """{"title":"An old test server","registrations":true}"""); } [Fact] public async Task A_server_is_described_located_and_snapshotted_weekly() { var token = TestContext.Current.CancellationToken; var host = $"described{Guid.NewGuid():N}.example"; ServeServer("/" + host, "mastodon"); var addresses = new ConnectedAddresses(); var describer = new PeerDescriber(Peer.Http(connected: addresses), new FixedGeo(), addresses, _peer.A + "/" + host); Assert.True(await describer.Describe(host, crawled: false, default, token)); var instance = await DB.Default.Find().Match(i => i.Host == host).ExecuteSingleAsync(token); Assert.Equal(("mastodon", 42L, 7L, "2.1"), (instance.Software, instance.UsersTotal!.Value, instance.UsersActiveMonth!.Value, instance.NodeInfoVersion)); Assert.Equal(("A test server", "open", "mastodon-v2"), (instance.Title, instance.RegistrationMode, instance.InstanceApi)); Assert.Equal(new[] { "it" }, instance.Languages); Assert.Equal(("NL", "Amsterdam", "LeaseWeb", "test"), (instance.Geo.Country, instance.Geo.City, instance.Geo.AsnOrg, instance.Geo.Source)); Assert.False(instance.Geo.Ipv6); var snapshot = await DB.Default.Find().Match(s => s.Host == host).ExecuteSingleAsync(token); Assert.True(snapshot.Reachable); Assert.Equal((42L, "NL"), (snapshot.UsersTotal!.Value, snapshot.Country)); Assert.Equal(InstanceDescriber.Week(DateTime.UtcNow), snapshot.Week); } [Fact] public async Task Without_api_v2_the_v1_api_is_read_and_an_unreachable_week_is_recorded_too() { var token = TestContext.Current.CancellationToken; var old = $"old{Guid.NewGuid():N}.example"; var gone = $"gone{Guid.NewGuid():N}.example"; ServeServer("/" + old, "pleroma", v2: false); await new PeerDescriber(Peer.Http(), default, default, _peer.A + "/" + old).Describe(old, crawled: false, default, token); Assert.False(await new PeerDescriber(Peer.Http(), default, default, _peer.A + "/" + gone).Describe(gone, crawled: false, default, token)); var oldOne = await DB.Default.Find().Match(i => i.Host == old).ExecuteSingleAsync(token); Assert.Equal(("mastodon-v1", "An old test server", "open"), (oldOne.InstanceApi, oldOne.Title, oldOne.RegistrationMode)); Assert.Null(oldOne.Geo); var goneOne = await DB.Default.Find().Match(i => i.Host == gone).ExecuteSingleAsync(token); Assert.Equal("no NodeInfo", goneOne.DescriptionError); Assert.False((await DB.Default.Find().Match(s => s.Host == gone).ExecuteSingleAsync(token)).Reachable); } [Fact] public async Task A_crawled_server_never_downgrades_a_touched_one_and_robots_are_obeyed() { var token = TestContext.Current.CancellationToken; var touched = $"touched{Guid.NewGuid():N}.example"; var shy = $"shy{Guid.NewGuid():N}.example"; ServeServer("/" + touched, "gotosocial"); ServeServer("/" + shy, "mastodon"); await DB.Default.SaveAsync(new RemoteInstance { Host = touched, Seen = "touched" }, token); await new PeerDescriber(Peer.Http(), default, default, _peer.A + "/" + touched).Describe(touched, crawled: true, default, token); await new PeerDescriber(Peer.Http(), default, default, _peer.A + "/" + shy).Describe(shy, crawled: true, path => !path.StartsWith("/api/"), token); Assert.Equal("touched", (await DB.Default.Find().Match(i => i.Host == touched).ExecuteSingleAsync(token)).Seen); var shyOne = await DB.Default.Find().Match(i => i.Host == shy).ExecuteSingleAsync(token); Assert.Equal(("crawled", "mastodon"), (shyOne.Seen, shyOne.Software)); Assert.Null(shyOne.InstanceApi); Assert.DoesNotContain(_peer.Requests, r => r.Path.StartsWith("/" + shy + "/api/", StringComparison.Ordinal)); Assert.Contains(_peer.Requests, r => r.Path == "/" + shy + "/nodeinfo/2.1"); } } [Trait("Category", "Integration")] public sealed class TouchTests : IAsyncLifetime { public ValueTask InitializeAsync() { Assert.SkipUnless(MongoFixture.Enabled, MongoFixture.Skip); return ValueTask.CompletedTask; } public ValueTask DisposeAsync() => ValueTask.CompletedTask; [Fact] public async Task A_server_we_exchange_with_is_seen_and_described_once_a_week_unless_suspended() { var token = TestContext.Current.CancellationToken; var friend = $"friend{Guid.NewGuid():N}.example"; var spammer = $"spam{Guid.NewGuid():N}.example"; var linked = $"linked{Guid.NewGuid():N}.example"; var looked = $"looked{Guid.NewGuid():N}.example"; var blocks = new DomainBlocks(NullLogger.Instance); blocks.Load(new[] { new DomainBlock { Domain = spammer, Severity = DomainBlockSeverity.Suspend } }); var ledger = new InteractionLedger(new InteractionSalts(), new StaticOptions(new StatisticsOptions()), NullLogger.Instance, new JobQueue(j => j.DedupeKey == "never"), blocks); ledger.Record(new InteractionEvent { Host = friend, Channel = "in", Activity = "Create", Outcome = "accepted" }, "https://" + friend + "/users/a"); ledger.Record(new InteractionEvent { Host = friend, Channel = "out", Activity = "Like", Outcome = "ok" }); ledger.Record(new InteractionEvent { Host = spammer, Channel = "out", Activity = "Reject", Outcome = "ok" }); ledger.Record(new InteractionEvent { Host = linked, Channel = "preview", Outcome = "ok" }); ledger.Count(looked, "http:actor:ok"); await ledger.Flush(token); var seen = await DB.Default.Find().Match(i => i.Host == friend).ExecuteSingleAsync(token); Assert.Equal("touched", seen.Seen); Assert.NotNull(seen.FirstSeenAt); Assert.Single(await DB.Default.Find().Match(j => j.Kind == JobKind.DescribeInstance && j.Payload == friend).ExecuteAsync(token)); Assert.True(await DB.Default.Find().Match(j => j.Kind == JobKind.DescribeInstance && j.Payload == looked).ExecuteAnyAsync(token)); Assert.False(await DB.Default.Find().Match(j => j.Kind == JobKind.DescribeInstance && j.Payload == spammer).ExecuteAnyAsync(token)); Assert.False(await DB.Default.Find().Match(i => i.Host == linked).ExecuteAnyAsync(token)); } } [Xunit.Collection(nameof(Exclusive))] [Trait("Category", "Integration")] public sealed class TouchedMigrationTests { [Fact] public async Task Known_servers_become_touched_with_their_dates() { Assert.SkipUnless(MongoFixture.Enabled, MongoFixture.Skip); var token = TestContext.Current.CancellationToken; var host = $"legacy{Guid.NewGuid():N}.example"; var success = DateTime.UtcNow.AddDays(-2); await DB.Default.SaveAsync(new RemoteInstance { Host = host, LastSuccessAt = success, DescribedAt = success.AddDays(-1) }, token); await new _010_remote_instances_were_touched().UpgradeAsync(); var instance = await DB.Default.Find().Match(i => i.Host == host).ExecuteSingleAsync(token); Assert.Equal("touched", instance.Seen); Assert.NotNull(instance.FirstSeenAt); Assert.InRange(instance.LastSeenAt!.Value, success.AddSeconds(-1), success.AddSeconds(1)); } } }