SocialPub/PrivaPub.Tests/Statistics/DescribeTests.cs

using Microsoft.Extensions.Logging.Abstractions;

using MongoDB.Entities;

using PrivaPub.Domain.Statistics;
using PrivaPub.Federation.Moderation;
using PrivaPub.Federation.Objects;
using PrivaPub.Infrastructure.Data.Migrations;
using PrivaPub.Infrastructure.Geo;
using PrivaPub.Infrastructure.Http;
using PrivaPub.Infrastructure.Jobs;
using PrivaPub.Infrastructure.Statistics;
using PrivaPub.Models.Federation;
using PrivaPub.Models.Jobs;
using PrivaPub.Models.Statistics;
using PrivaPub.Tests.Support;

using System.Net;
using System.Text.Json;

namespace PrivaPub.Tests.Statistics
{
	public class InstanceDocumentsTests
	{
		static JsonElement Json(string text) => JsonDocument.Parse(text).RootElement;

		[Fact]
		public void NodeInfo_gives_software_usage_and_languages_whatever_their_shape()
		{
			var facts = InstanceDocuments.NodeInfo(Json("""
				{"software":{"name":"mastodon","version":"4.7.3"},"protocols":["activitypub"],"openRegistrations":false,
				 "usage":{"users":{"total":"1234","activeMonth":56.0,"activeHalfyear":-3},"localPosts":99999999999,"localComments":7},
				 "metadata":{"nodeName":"Example","nodeDescription":"<p>A <b>small</b>\n place</p>","languages":["en","DE","klingon!"]}}
				"""), "2.1");

			Assert.Equal(("mastodon", "4.7.3", "Example", "2.1"), (facts.Software, facts.SoftwareVersion, facts.NodeName, facts.Version));
			Assert.Equal("A small place", facts.NodeDescription);
			Assert.Equal(new[] { "activitypub" }, facts.Protocols);
			Assert.False(facts.OpenRegistrations);
			Assert.Equal(1234, facts.UsersTotal);
			Assert.Equal(56, facts.UsersActiveMonth);
			Assert.Equal(0, facts.UsersActiveHalfyear);
			Assert.Equal(1_000_000_000, facts.LocalPosts);
			Assert.Equal(7, facts.LocalComments);
			Assert.Equal(new[] { "en", "de" }, facts.Languages);
			Assert.Null(InstanceDocuments.NodeInfo(Json("{}"), "2.0").UsersTotal);
			// NodeInfo wants the name in lower case, which Mobilizon does not keep
			Assert.Equal("mobilizon", InstanceDocuments.NodeInfo(Json("""{"software":{"name":"Mobilizon"}}"""), "2.1").Software);
		}

		[Fact]
		public void The_newest_https_nodeinfo_link_is_chosen()
		{
			var links = Json("""
				{"links":[{"rel":"http://nodeinfo.diaspora.software/ns/schema/2.0","href":"https://a.example/2.0"},
				          {"rel":"http://nodeinfo.diaspora.software/ns/schema/2.1","href":"http://a.example/2.1"},
				          {"rel":"http://nodeinfo.diaspora.software/ns/schema/2.2","href":"https://a.example/2.2"}]}
				""");

			Assert.Equal(("https://a.example/2.2", "2.2"), InstanceDocuments.NodeInfoLink(links));
			Assert.Equal(default, InstanceDocuments.NodeInfoLink(Json("[]")));
		}

		[Fact]
		public void The_instance_api_gives_registration_mode_limits_and_never_a_contact()
		{
			var v2 = InstanceDocuments.InstanceApi(Json("""
				{"title":"Example","languages":["en"],"registrations":{"enabled":true,"approval_required":true},
				 "configuration":{"statuses":{"max_characters":500}},"api_versions":{"mastodon":7},"source_url":"https://github.com/mastodon/mastodon",
				 "contact":{"email":"admin@a.example","account":{"username":"admin"}}}
				"""));
			var v1 = InstanceDocuments.InstanceApi(Json("""{"title":"Old","registrations":false,"max_toot_chars":"5000"}"""));

			Assert.Equal(("Example", "approval", 500, 7, "https://github.com/mastodon/mastodon"), (v2.Title, v2.RegistrationMode, v2.MaxPostChars!.Value, v2.ApiVersion!.Value, v2.SourceUrl));
			Assert.Equal(new[] { "en" }, v2.Languages);
			Assert.DoesNotContain("admin", System.Text.Json.JsonSerializer.Serialize(v2));
			Assert.Equal(("closed", 5000), (v1.RegistrationMode, v1.MaxPostChars!.Value));
			Assert.True(InstanceDocuments.SpeaksMastodonApi("GoToSocial"));
			Assert.False(InstanceDocuments.SpeaksMastodonApi("lemmy"));
		}

		[Fact]
		public void The_public_location_is_where_the_server_was_located_unless_a_cdn_hides_it()
		{
			var geo = new InstanceGeo { Country = "DE", City = "Berlin", Latitude = 52.5, Longitude = 13.4, AsnOrg = "Hetzner", Asn = 24940 };

			Assert.Equal(new PublicLocation("DE", "Berlin", 52.5, 13.4, "Hetzner", null), PublicGeo.Project(geo));
			Assert.Equal(new PublicLocation("DE", null, null, null, null, null), PublicGeo.Project(new InstanceGeo { Country = "DE" }));
			Assert.Equal(new PublicLocation(null, null, null, null, null, "Cloudflare"),
				PublicGeo.Project(new InstanceGeo { Country = "US", City = "SF", Latitude = 37.8, Longitude = -122.4, Cdn = "Cloudflare" }));
			Assert.Null(PublicGeo.Project(new InstanceGeo()));
			Assert.Null(PublicGeo.Project(null));
		}
	}

	[Trait("Category", "Integration")]
	public sealed class DescribeTests : IAsyncLifetime
	{
		Peer _peer;

		public async ValueTask InitializeAsync()
		{
			Assert.SkipUnless(MongoFixture.Enabled, MongoFixture.Skip);
			_peer = await Peer.Start();
		}

		public async ValueTask DisposeAsync()
		{
			if (_peer != default)
				await _peer.DisposeAsync();
		}

		sealed class PeerDescriber : InstanceDescriber
		{
			readonly string _peer;

			public PeerDescriber(IFederationHttp http, IGeoLocator geo, IConnectedAddresses addresses, string peer, ICdnRanges ranges = default,
				IEdgeHints edges = default) : base(http, geo, addresses, ranges, edges) => _peer = peer;

			protected override string Address(string url) => _peer + new Uri(url).PathAndQuery;
		}

		sealed class FixedGeo : IGeoLocator
		{
			public GeoFix Locate(IPAddress address) => new("NL", "Amsterdam", 52.4, 4.9, 60781, "LeaseWeb");
			public string Source => "test";
		}

		void ServeServer(string prefix, string software, bool v2 = true)
		{
			_peer.Serve($"{prefix}/.well-known/nodeinfo", $$"""{"links":[{"rel":"http://nodeinfo.diaspora.software/ns/schema/2.1","href":"https://ignored.example/nodeinfo/2.1"}]}""");
			_peer.Serve($"{prefix}/nodeinfo/2.1", $$$"""{"software":{"name":"{{{software}}}","version":"1.0"},"usage":{"users":{"total":42,"activeMonth":7}},"openRegistrations":true}""");
			if (v2)
				_peer.Serve($"{prefix}/api/v2/instance", """{"title":"A test server","languages":["it"],"registrations":{"enabled":true,"approval_required":false}}""");
			_peer.Serve($"{prefix}/api/v1/instance", """{"title":"An old test server","registrations":true}""");
		}

		[Fact]
		public async Task A_server_is_described_located_and_snapshotted_weekly()
		{
			var token = TestContext.Current.CancellationToken;
			var host = $"described{Guid.NewGuid():N}.example";
			ServeServer("/" + host, "mastodon");
			var addresses = new ConnectedAddresses();
			var describer = new PeerDescriber(Peer.Http(connected: addresses), new FixedGeo(), addresses, _peer.A + "/" + host);

			Assert.True(await describer.Describe(host, crawled: false, default, token));

			var instance = await DB.Default.Find<RemoteInstance>().Match(i => i.Host == host).ExecuteSingleAsync(token);
			Assert.Equal(("mastodon", 42L, 7L, "2.1"), (instance.Software, instance.UsersTotal!.Value, instance.UsersActiveMonth!.Value, instance.NodeInfoVersion));
			Assert.Equal(("A test server", "open", "mastodon-v2"), (instance.Title, instance.RegistrationMode, instance.InstanceApi));
			Assert.Equal(new[] { "it" }, instance.Languages);
			Assert.Equal(("NL", "Amsterdam", "LeaseWeb", "test"), (instance.Geo.Country, instance.Geo.City, instance.Geo.AsnOrg, instance.Geo.Source));
			Assert.False(instance.Geo.Ipv6);
			var snapshot = await DB.Default.Find<RemoteInstanceSnapshot>().Match(s => s.Host == host).ExecuteSingleAsync(token);
			Assert.True(snapshot.Reachable);
			Assert.Equal((42L, "NL"), (snapshot.UsersTotal!.Value, snapshot.Country));
			Assert.Equal(InstanceDescriber.Week(DateTime.UtcNow), snapshot.Week);
		}

		[Fact]
		public async Task A_cdn_is_found_by_its_published_ranges_first_then_by_its_headers()
		{
			var token = TestContext.Current.CancellationToken;
			var listed = $"listed{Guid.NewGuid():N}.example";
			var hinted = $"hinted{Guid.NewGuid():N}.example";
			ServeServer("/" + listed, "mastodon");
			ServeServer("/" + hinted, "mastodon");
			var addresses = new ConnectedAddresses();
			var ranges = new CdnRanges();
			ranges.Replace(new CdnRangeSet { Key = "cloudflare.com", Name = "Cloudflare", Ranges = new() { "127.0.0.0/8" }, FetchedAt = DateTime.UtcNow });
			var edges = new EdgeHints();
			edges.Remember("127.0.0.1", CdnCatalog.ByKey("fastly.com"));

			await new PeerDescriber(Peer.Http(connected: addresses), new FixedGeo(), addresses, _peer.A + "/" + listed, ranges, edges).Describe(listed, crawled: false, default, token);
			await new PeerDescriber(Peer.Http(connected: addresses), new FixedGeo(), addresses, _peer.A + "/" + hinted, new CdnRanges(), edges).Describe(hinted, crawled: false, default, token);

			var listedOne = await DB.Default.Find<RemoteInstance>().Match(i => i.Host == listed).ExecuteSingleAsync(token);
			Assert.Equal(("Cloudflare", "cloudflare.com", "ranges"), (listedOne.Geo.Cdn, listedOne.Geo.CdnDomain, listedOne.Geo.CdnSource));
			var hintedOne = await DB.Default.Find<RemoteInstance>().Match(i => i.Host == hinted).ExecuteSingleAsync(token);
			Assert.Equal(("Fastly", "fastly.com", "headers"), (hintedOne.Geo.Cdn, hintedOne.Geo.CdnDomain, hintedOne.Geo.CdnSource));
			var snapshot = await DB.Default.Find<RemoteInstanceSnapshot>().Match(s => s.Host == listed).ExecuteSingleAsync(token);
			Assert.Equal(("Cloudflare", "cloudflare.com", "ranges", "Amsterdam", 52.4), (snapshot.Cdn, snapshot.CdnDomain, snapshot.CdnSource, snapshot.City, snapshot.Latitude!.Value));
		}

		[Fact]
		public async Task Without_api_v2_the_v1_api_is_read_and_an_unreachable_week_is_recorded_too()
		{
			var token = TestContext.Current.CancellationToken;
			var old = $"old{Guid.NewGuid():N}.example";
			var gone = $"gone{Guid.NewGuid():N}.example";
			ServeServer("/" + old, "pleroma", v2: false);

			await new PeerDescriber(Peer.Http(), default, default, _peer.A + "/" + old).Describe(old, crawled: false, default, token);
			Assert.False(await new PeerDescriber(Peer.Http(), default, default, _peer.A + "/" + gone).Describe(gone, crawled: false, default, token));

			var oldOne = await DB.Default.Find<RemoteInstance>().Match(i => i.Host == old).ExecuteSingleAsync(token);
			Assert.Equal(("mastodon-v1", "An old test server", "open"), (oldOne.InstanceApi, oldOne.Title, oldOne.RegistrationMode));
			Assert.Null(oldOne.Geo);
			var goneOne = await DB.Default.Find<RemoteInstance>().Match(i => i.Host == gone).ExecuteSingleAsync(token);
			Assert.Equal("no NodeInfo", goneOne.DescriptionError);
			Assert.False((await DB.Default.Find<RemoteInstanceSnapshot>().Match(s => s.Host == gone).ExecuteSingleAsync(token)).Reachable);
		}

		[Fact]
		public async Task A_crawled_server_never_downgrades_a_touched_one_and_robots_are_obeyed()
		{
			var token = TestContext.Current.CancellationToken;
			var touched = $"touched{Guid.NewGuid():N}.example";
			var shy = $"shy{Guid.NewGuid():N}.example";
			ServeServer("/" + touched, "gotosocial");
			ServeServer("/" + shy, "mastodon");
			await DB.Default.SaveAsync(new RemoteInstance { Host = touched, Seen = "touched" }, token);

			await new PeerDescriber(Peer.Http(), default, default, _peer.A + "/" + touched).Describe(touched, crawled: true, default, token);
			await new PeerDescriber(Peer.Http(), default, default, _peer.A + "/" + shy).Describe(shy, crawled: true, path => !path.StartsWith("/api/"), token);

			Assert.Equal("touched", (await DB.Default.Find<RemoteInstance>().Match(i => i.Host == touched).ExecuteSingleAsync(token)).Seen);
			var shyOne = await DB.Default.Find<RemoteInstance>().Match(i => i.Host == shy).ExecuteSingleAsync(token);
			Assert.Equal(("crawled", "mastodon"), (shyOne.Seen, shyOne.Software));
			Assert.Null(shyOne.InstanceApi);
			Assert.DoesNotContain(_peer.Requests, r => r.Path.StartsWith("/" + shy + "/api/", StringComparison.Ordinal));
			Assert.Contains(_peer.Requests, r => r.Path == "/" + shy + "/nodeinfo/2.1");
		}
	}

	[Trait("Category", "Integration")]
	public sealed class TouchTests : IAsyncLifetime
	{
		public ValueTask InitializeAsync()
		{
			Assert.SkipUnless(MongoFixture.Enabled, MongoFixture.Skip);
			return ValueTask.CompletedTask;
		}

		public ValueTask DisposeAsync() => ValueTask.CompletedTask;

		[Fact]
		public async Task A_server_we_exchange_with_is_seen_and_described_once_a_week_unless_suspended()
		{
			var token = TestContext.Current.CancellationToken;
			var friend = $"friend{Guid.NewGuid():N}.example";
			var spammer = $"spam{Guid.NewGuid():N}.example";
			var linked = $"linked{Guid.NewGuid():N}.example";
			var looked = $"looked{Guid.NewGuid():N}.example";
			var blocks = new DomainBlocks(NullLogger<DomainBlocks>.Instance);
			blocks.Load(new[] { new DomainBlock { Domain = spammer, Severity = DomainBlockSeverity.Suspend } });
			var ledger = new InteractionLedger(new InteractionSalts(), new StaticOptions<StatisticsOptions>(new StatisticsOptions()), NullLogger<InteractionLedger>.Instance,
				new JobQueue(j => j.DedupeKey == "never"), blocks);

			ledger.Record(new InteractionEvent { Host = friend, Channel = "in", Activity = "Create", Outcome = "accepted" }, "https://" + friend + "/users/a");
			ledger.Record(new InteractionEvent { Host = friend, Channel = "out", Activity = "Like", Outcome = "ok" });
			ledger.Record(new InteractionEvent { Host = spammer, Channel = "out", Activity = "Reject", Outcome = "ok" });
			ledger.Record(new InteractionEvent { Host = linked, Channel = "preview", Outcome = "ok" });
			ledger.Count(looked, "http:actor:ok");
			await ledger.Flush(token);

			var seen = await DB.Default.Find<RemoteInstance>().Match(i => i.Host == friend).ExecuteSingleAsync(token);
			Assert.Equal("touched", seen.Seen);
			Assert.NotNull(seen.FirstSeenAt);
			Assert.Single(await DB.Default.Find<Job>().Match(j => j.Kind == JobKind.DescribeInstance && j.Payload == friend).ExecuteAsync(token));
			Assert.True(await DB.Default.Find<Job>().Match(j => j.Kind == JobKind.DescribeInstance && j.Payload == looked).ExecuteAnyAsync(token));
			Assert.False(await DB.Default.Find<Job>().Match(j => j.Kind == JobKind.DescribeInstance && j.Payload == spammer).ExecuteAnyAsync(token));
			Assert.False(await DB.Default.Find<RemoteInstance>().Match(i => i.Host == linked).ExecuteAnyAsync(token));
		}
	}

	[Xunit.Collection(nameof(Exclusive))]
	[Trait("Category", "Integration")]
	public sealed class TouchedMigrationTests
	{
		[Fact]
		public async Task Known_servers_become_touched_with_their_dates()
		{
			Assert.SkipUnless(MongoFixture.Enabled, MongoFixture.Skip);
			var token = TestContext.Current.CancellationToken;
			var host = $"legacy{Guid.NewGuid():N}.example";
			var success = DateTime.UtcNow.AddDays(-2);
			await DB.Default.SaveAsync(new RemoteInstance { Host = host, LastSuccessAt = success, DescribedAt = success.AddDays(-1) }, token);

			await new _010_remote_instances_were_touched().UpgradeAsync();

			var instance = await DB.Default.Find<RemoteInstance>().Match(i => i.Host == host).ExecuteSingleAsync(token);
			Assert.Equal("touched", instance.Seen);
			Assert.NotNull(instance.FirstSeenAt);
			Assert.InRange(instance.LastSeenAt!.Value, success.AddSeconds(-1), success.AddSeconds(1));
		}
	}
}