diff --git a/PrivaPub.Tests/Http/InboxRouteTests.cs b/PrivaPub.Tests/Http/InboxRouteTests.cs index 10f1322..d2dce28 100644 --- a/PrivaPub.Tests/Http/InboxRouteTests.cs +++ b/PrivaPub.Tests/Http/InboxRouteTests.cs @@ -188,18 +188,15 @@ namespace PrivaPub.Tests.Http return response.StatusCode; } - var clock = System.Diagnostics.Stopwatch.StartNew(); - var first = new List(); - for (var i = 0; i < 300; i++) - first.Add(await Post()); - var limited = await Post(); - // The bucket refills 50 every ten seconds: a slow machine may have earned a few more before the limit bites. - for (var refilled = (int)((clock.Elapsed.TotalSeconds + 1) / 10) * 50; limited != HttpStatusCode.TooManyRequests && refilled > 0; refilled--) - limited = await Post(); + // The bucket holds 300 and refills 50 every ten seconds on its own timer, so a refill can land at any moment: + // the first refusal comes after at least 300 posts and within a few refills. + var answers = new List(); + while (answers.Count < 600 && (answers.Count == 0 || answers[^1] != HttpStatusCode.TooManyRequests)) + answers.Add(await Post()); - Assert.DoesNotContain(HttpStatusCode.TooManyRequests, first); - Assert.All(first, status => Assert.Equal(HttpStatusCode.Unauthorized, status)); - Assert.Equal(HttpStatusCode.TooManyRequests, limited); + Assert.Equal(HttpStatusCode.TooManyRequests, answers[^1]); + Assert.True(answers.Count > 300, $"limited after only {answers.Count - 1} posts"); + Assert.All(answers.SkipLast(1), status => Assert.Equal(HttpStatusCode.Unauthorized, status)); var polite = new RemoteActor(_peer, "polite"); using var signed = await client.SendAsync(polite.SignedPost("/human-centipede", Direct(polite)), token); Assert.Equal(HttpStatusCode.Accepted, signed.StatusCode); diff --git a/PrivaPub.Tests/Statistics/DescribeTests.cs b/PrivaPub.Tests/Statistics/DescribeTests.cs new file mode 100644 index 0000000..c634c13 --- /dev/null +++ b/PrivaPub.Tests/Statistics/DescribeTests.cs @@ -0,0 +1,261 @@ +using Microsoft.Extensions.Logging.Abstractions; + +using MongoDB.Entities; + +using PrivaPub.Domain.Statistics; +using PrivaPub.Federation.Moderation; +using PrivaPub.Federation.Objects; +using PrivaPub.Infrastructure.Data.Migrations; +using PrivaPub.Infrastructure.Geo; +using PrivaPub.Infrastructure.Http; +using PrivaPub.Infrastructure.Jobs; +using PrivaPub.Infrastructure.Statistics; +using PrivaPub.Models.Federation; +using PrivaPub.Models.Jobs; +using PrivaPub.Models.Statistics; +using PrivaPub.Tests.Support; + +using System.Net; +using System.Text.Json; + +namespace PrivaPub.Tests.Statistics +{ + public class InstanceDocumentsTests + { + static JsonElement Json(string text) => JsonDocument.Parse(text).RootElement; + + [Fact] + public void NodeInfo_gives_software_usage_and_languages_whatever_their_shape() + { + var facts = InstanceDocuments.NodeInfo(Json(""" + {"software":{"name":"mastodon","version":"4.7.3"},"protocols":["activitypub"],"openRegistrations":false, + "usage":{"users":{"total":"1234","activeMonth":56.0,"activeHalfyear":-3},"localPosts":99999999999,"localComments":7}, + "metadata":{"nodeName":"Example","nodeDescription":"

A small\n place

","languages":["en","DE","klingon!"]}} + """), "2.1"); + + Assert.Equal(("mastodon", "4.7.3", "Example", "2.1"), (facts.Software, facts.SoftwareVersion, facts.NodeName, facts.Version)); + Assert.Equal("A small place", facts.NodeDescription); + Assert.Equal(new[] { "activitypub" }, facts.Protocols); + Assert.False(facts.OpenRegistrations); + Assert.Equal(1234, facts.UsersTotal); + Assert.Equal(56, facts.UsersActiveMonth); + Assert.Equal(0, facts.UsersActiveHalfyear); + Assert.Equal(1_000_000_000, facts.LocalPosts); + Assert.Equal(7, facts.LocalComments); + Assert.Equal(new[] { "en", "de" }, facts.Languages); + Assert.Null(InstanceDocuments.NodeInfo(Json("{}"), "2.0").UsersTotal); + } + + [Fact] + public void The_newest_https_nodeinfo_link_is_chosen() + { + var links = Json(""" + {"links":[{"rel":"http://nodeinfo.diaspora.software/ns/schema/2.0","href":"https://a.example/2.0"}, + {"rel":"http://nodeinfo.diaspora.software/ns/schema/2.1","href":"http://a.example/2.1"}, + {"rel":"http://nodeinfo.diaspora.software/ns/schema/2.2","href":"https://a.example/2.2"}]} + """); + + Assert.Equal(("https://a.example/2.2", "2.2"), InstanceDocuments.NodeInfoLink(links)); + Assert.Equal(default, InstanceDocuments.NodeInfoLink(Json("[]"))); + } + + [Fact] + public void The_instance_api_gives_registration_mode_limits_and_never_a_contact() + { + var v2 = InstanceDocuments.InstanceApi(Json(""" + {"title":"Example","languages":["en"],"registrations":{"enabled":true,"approval_required":true}, + "configuration":{"statuses":{"max_characters":500}},"api_versions":{"mastodon":7},"source_url":"https://github.com/mastodon/mastodon", + "contact":{"email":"admin@a.example","account":{"username":"admin"}}} + """)); + var v1 = InstanceDocuments.InstanceApi(Json("""{"title":"Old","registrations":false,"max_toot_chars":"5000"}""")); + + Assert.Equal(("Example", "approval", 500, 7, "https://github.com/mastodon/mastodon"), (v2.Title, v2.RegistrationMode, v2.MaxPostChars!.Value, v2.ApiVersion!.Value, v2.SourceUrl)); + Assert.Equal(new[] { "en" }, v2.Languages); + Assert.DoesNotContain("admin", System.Text.Json.JsonSerializer.Serialize(v2)); + Assert.Equal(("closed", 5000), (v1.RegistrationMode, v1.MaxPostChars!.Value)); + Assert.True(InstanceDocuments.SpeaksMastodonApi("GoToSocial")); + Assert.False(InstanceDocuments.SpeaksMastodonApi("lemmy")); + } + + [Fact] + public void The_public_location_depends_on_size_and_on_a_cdn() + { + var geo = new InstanceGeo { Country = "DE", City = "Berlin", Latitude = 52.5, Longitude = 13.4, AsnOrg = "Hetzner", Asn = 24940 }; + + Assert.Equal(new PublicLocation("DE", "Berlin", 52.5, 13.4, "Hetzner", null), PublicGeo.Project(new RemoteInstance { Geo = geo, UsersTotal = 10 }, 10)); + Assert.Equal(new PublicLocation("DE", null, null, null, null, null), PublicGeo.Project(new RemoteInstance { Geo = geo, UsersTotal = 1 }, 10)); + Assert.Equal(new PublicLocation("DE", null, null, null, null, null), PublicGeo.Project(new RemoteInstance { Geo = geo }, 10)); + Assert.Equal(new PublicLocation(null, null, null, null, null, "Cloudflare"), + PublicGeo.Project(new RemoteInstance { Geo = new InstanceGeo { Country = "US", City = "SF", Cdn = "Cloudflare" }, UsersTotal = 5000 }, 10)); + Assert.Null(PublicGeo.Project(new RemoteInstance(), 10)); + } + } + + [Trait("Category", "Integration")] + public sealed class DescribeTests : IAsyncLifetime + { + Peer _peer; + + public async ValueTask InitializeAsync() + { + Assert.SkipUnless(MongoFixture.Enabled, MongoFixture.Skip); + _peer = await Peer.Start(); + } + + public async ValueTask DisposeAsync() + { + if (_peer != default) + await _peer.DisposeAsync(); + } + + sealed class PeerDescriber : InstanceDescriber + { + readonly string _peer; + + public PeerDescriber(IFederationHttp http, IGeoLocator geo, IConnectedAddresses addresses, string peer) : base(http, geo, addresses) => _peer = peer; + + protected override string Address(string url) => _peer + new Uri(url).PathAndQuery; + } + + sealed class FixedGeo : IGeoLocator + { + public GeoFix Locate(IPAddress address) => new("NL", "Amsterdam", 52.4, 4.9, 60781, "LeaseWeb"); + public string Source => "test"; + } + + void ServeServer(string prefix, string software, bool v2 = true) + { + _peer.Serve($"{prefix}/.well-known/nodeinfo", $$"""{"links":[{"rel":"http://nodeinfo.diaspora.software/ns/schema/2.1","href":"https://ignored.example/nodeinfo/2.1"}]}"""); + _peer.Serve($"{prefix}/nodeinfo/2.1", $$$"""{"software":{"name":"{{{software}}}","version":"1.0"},"usage":{"users":{"total":42,"activeMonth":7}},"openRegistrations":true}"""); + if (v2) + _peer.Serve($"{prefix}/api/v2/instance", """{"title":"A test server","languages":["it"],"registrations":{"enabled":true,"approval_required":false}}"""); + _peer.Serve($"{prefix}/api/v1/instance", """{"title":"An old test server","registrations":true}"""); + } + + [Fact] + public async Task A_server_is_described_located_and_snapshotted_weekly() + { + var token = TestContext.Current.CancellationToken; + var host = $"described{Guid.NewGuid():N}.example"; + ServeServer("/" + host, "mastodon"); + var addresses = new ConnectedAddresses(); + var describer = new PeerDescriber(Peer.Http(connected: addresses), new FixedGeo(), addresses, _peer.A + "/" + host); + + Assert.True(await describer.Describe(host, crawled: false, default, token)); + + var instance = await DB.Default.Find().Match(i => i.Host == host).ExecuteSingleAsync(token); + Assert.Equal(("mastodon", 42L, 7L, "2.1"), (instance.Software, instance.UsersTotal!.Value, instance.UsersActiveMonth!.Value, instance.NodeInfoVersion)); + Assert.Equal(("A test server", "open", "mastodon-v2"), (instance.Title, instance.RegistrationMode, instance.InstanceApi)); + Assert.Equal(new[] { "it" }, instance.Languages); + Assert.Equal(("NL", "Amsterdam", "LeaseWeb", "test"), (instance.Geo.Country, instance.Geo.City, instance.Geo.AsnOrg, instance.Geo.Source)); + Assert.False(instance.Geo.Ipv6); + var snapshot = await DB.Default.Find().Match(s => s.Host == host).ExecuteSingleAsync(token); + Assert.True(snapshot.Reachable); + Assert.Equal((42L, "NL"), (snapshot.UsersTotal!.Value, snapshot.Country)); + Assert.Equal(InstanceDescriber.Week(DateTime.UtcNow), snapshot.Week); + } + + [Fact] + public async Task Without_api_v2_the_v1_api_is_read_and_an_unreachable_week_is_recorded_too() + { + var token = TestContext.Current.CancellationToken; + var old = $"old{Guid.NewGuid():N}.example"; + var gone = $"gone{Guid.NewGuid():N}.example"; + ServeServer("/" + old, "pleroma", v2: false); + + await new PeerDescriber(Peer.Http(), default, default, _peer.A + "/" + old).Describe(old, crawled: false, default, token); + Assert.False(await new PeerDescriber(Peer.Http(), default, default, _peer.A + "/" + gone).Describe(gone, crawled: false, default, token)); + + var oldOne = await DB.Default.Find().Match(i => i.Host == old).ExecuteSingleAsync(token); + Assert.Equal(("mastodon-v1", "An old test server", "open"), (oldOne.InstanceApi, oldOne.Title, oldOne.RegistrationMode)); + Assert.Null(oldOne.Geo); + var goneOne = await DB.Default.Find().Match(i => i.Host == gone).ExecuteSingleAsync(token); + Assert.Equal("no NodeInfo", goneOne.DescriptionError); + Assert.False((await DB.Default.Find().Match(s => s.Host == gone).ExecuteSingleAsync(token)).Reachable); + } + + [Fact] + public async Task A_crawled_server_never_downgrades_a_touched_one_and_robots_are_obeyed() + { + var token = TestContext.Current.CancellationToken; + var touched = $"touched{Guid.NewGuid():N}.example"; + var shy = $"shy{Guid.NewGuid():N}.example"; + ServeServer("/" + touched, "gotosocial"); + ServeServer("/" + shy, "mastodon"); + await DB.Default.SaveAsync(new RemoteInstance { Host = touched, Seen = "touched" }, token); + + await new PeerDescriber(Peer.Http(), default, default, _peer.A + "/" + touched).Describe(touched, crawled: true, default, token); + await new PeerDescriber(Peer.Http(), default, default, _peer.A + "/" + shy).Describe(shy, crawled: true, path => !path.StartsWith("/api/"), token); + + Assert.Equal("touched", (await DB.Default.Find().Match(i => i.Host == touched).ExecuteSingleAsync(token)).Seen); + var shyOne = await DB.Default.Find().Match(i => i.Host == shy).ExecuteSingleAsync(token); + Assert.Equal(("crawled", "mastodon"), (shyOne.Seen, shyOne.Software)); + Assert.Null(shyOne.InstanceApi); + Assert.DoesNotContain(_peer.Requests, r => r.Path.StartsWith("/" + shy + "/api/", StringComparison.Ordinal)); + Assert.Contains(_peer.Requests, r => r.Path == "/" + shy + "/nodeinfo/2.1"); + } + } + + [Trait("Category", "Integration")] + public sealed class TouchTests : IAsyncLifetime + { + public ValueTask InitializeAsync() + { + Assert.SkipUnless(MongoFixture.Enabled, MongoFixture.Skip); + return ValueTask.CompletedTask; + } + + public ValueTask DisposeAsync() => ValueTask.CompletedTask; + + [Fact] + public async Task A_server_we_exchange_with_is_seen_and_described_once_a_week_unless_suspended() + { + var token = TestContext.Current.CancellationToken; + var friend = $"friend{Guid.NewGuid():N}.example"; + var spammer = $"spam{Guid.NewGuid():N}.example"; + var linked = $"linked{Guid.NewGuid():N}.example"; + var looked = $"looked{Guid.NewGuid():N}.example"; + var blocks = new DomainBlocks(NullLogger.Instance); + blocks.Load(new[] { new DomainBlock { Domain = spammer, Severity = DomainBlockSeverity.Suspend } }); + var ledger = new InteractionLedger(new InteractionSalts(), new StaticOptions(new StatisticsOptions()), NullLogger.Instance, + new JobQueue(j => j.DedupeKey == "never"), blocks); + + ledger.Record(new InteractionEvent { Host = friend, Channel = "in", Activity = "Create", Outcome = "accepted" }, "https://" + friend + "/users/a"); + ledger.Record(new InteractionEvent { Host = friend, Channel = "out", Activity = "Like", Outcome = "ok" }); + ledger.Record(new InteractionEvent { Host = spammer, Channel = "out", Activity = "Reject", Outcome = "ok" }); + ledger.Record(new InteractionEvent { Host = linked, Channel = "preview", Outcome = "ok" }); + ledger.Count(looked, "http:actor:ok"); + await ledger.Flush(token); + + var seen = await DB.Default.Find().Match(i => i.Host == friend).ExecuteSingleAsync(token); + Assert.Equal("touched", seen.Seen); + Assert.NotNull(seen.FirstSeenAt); + Assert.Single(await DB.Default.Find().Match(j => j.Kind == JobKind.DescribeInstance && j.Payload == friend).ExecuteAsync(token)); + Assert.True(await DB.Default.Find().Match(j => j.Kind == JobKind.DescribeInstance && j.Payload == looked).ExecuteAnyAsync(token)); + Assert.False(await DB.Default.Find().Match(j => j.Kind == JobKind.DescribeInstance && j.Payload == spammer).ExecuteAnyAsync(token)); + Assert.False(await DB.Default.Find().Match(i => i.Host == linked).ExecuteAnyAsync(token)); + } + } + + [Xunit.Collection(nameof(Exclusive))] + [Trait("Category", "Integration")] + public sealed class TouchedMigrationTests + { + [Fact] + public async Task Known_servers_become_touched_with_their_dates() + { + Assert.SkipUnless(MongoFixture.Enabled, MongoFixture.Skip); + var token = TestContext.Current.CancellationToken; + var host = $"legacy{Guid.NewGuid():N}.example"; + var success = DateTime.UtcNow.AddDays(-2); + await DB.Default.SaveAsync(new RemoteInstance { Host = host, LastSuccessAt = success, DescribedAt = success.AddDays(-1) }, token); + + await new _010_remote_instances_were_touched().UpgradeAsync(); + + var instance = await DB.Default.Find().Match(i => i.Host == host).ExecuteSingleAsync(token); + Assert.Equal("touched", instance.Seen); + Assert.NotNull(instance.FirstSeenAt); + Assert.InRange(instance.LastSeenAt!.Value, success.AddSeconds(-1), success.AddSeconds(1)); + } + } +} diff --git a/PrivaPub/Domain/Statistics/PublicGeo.cs b/PrivaPub/Domain/Statistics/PublicGeo.cs new file mode 100644 index 0000000..aaa3809 --- /dev/null +++ b/PrivaPub/Domain/Statistics/PublicGeo.cs @@ -0,0 +1,23 @@ +using PrivaPub.Models.Jobs; + +namespace PrivaPub.Domain.Statistics +{ + public sealed record PublicLocation(string Country, string City, double? Latitude, double? Longitude, string Network, string Cdn); + + public static class PublicGeo + { + //owner decision: a CDN-fronted server shows only its CDN (the address is an edge), a server reporting at least + //minUsers users its city, coordinates and network, any other only its country + public static PublicLocation Project(RemoteInstance instance, int minUsers) + { + var geo = instance?.Geo; + if (geo == default) + return default; + if (geo.Cdn != default) + return new PublicLocation(default, default, default, default, default, geo.Cdn); + if (instance.UsersTotal >= minUsers) + return new PublicLocation(geo.Country, geo.City, geo.Latitude, geo.Longitude, geo.AsnOrg, default); + return geo.Country == default ? default : new PublicLocation(geo.Country, default, default, default, default, default); + } + } +} diff --git a/PrivaPub/Federation/Objects/InstanceDescriber.cs b/PrivaPub/Federation/Objects/InstanceDescriber.cs index 8b0522f..384aaed 100644 --- a/PrivaPub/Federation/Objects/InstanceDescriber.cs +++ b/PrivaPub/Federation/Objects/InstanceDescriber.cs @@ -1,88 +1,180 @@ using MongoDB.Entities; +using PrivaPub.Infrastructure.Geo; using PrivaPub.Infrastructure.Http; using PrivaPub.Infrastructure.Jobs; using PrivaPub.Models.Jobs; +using System.Globalization; +using System.Net.Sockets; using System.Text.Json; namespace PrivaPub.Federation.Objects { public class InstanceDescriber : IJobHandler { - const int MaxNodeInfoChars = 64 * 1024; - static readonly string[] Schemas = - { - "http://nodeinfo.diaspora.software/ns/schema/2.1", - "http://nodeinfo.diaspora.software/ns/schema/2.0" - }; + const int MaxRawChars = 64 * 1024; readonly IFederationHttp _http; + readonly IGeoLocator _geo; + readonly IConnectedAddresses _addresses; - public InstanceDescriber(IFederationHttp http) => _http = http; + public InstanceDescriber(IFederationHttp http, IGeoLocator geo = default, IConnectedAddresses addresses = default) + { + _http = http; + _geo = geo; + _addresses = addresses; + } public JobKind Kind => JobKind.DescribeInstance; public int Concurrency => 1; public int MaxAttempts => 2; public int PerHostLimit => 1; + public static string Week(DateTime at) => $"{ISOWeek.GetYear(at)}-W{ISOWeek.GetWeekOfYear(at):00}"; + + public static string DedupeKey(string host, DateTime at) => $"describe|{host}|{ISOWeek.GetYear(at)}-{ISOWeek.GetWeekOfYear(at)}"; + public async Task Handle(Job job, CancellationToken token) { - var host = job.Payload; - using var scope = HttpScope.For("nodeinfo"); - using var links = await _http.GetJson(Address($"https://{host}/.well-known/nodeinfo"), "application/json", sign: default, token); - var href = links == default ? default : Schemas.Select(schema => LinkFor(links.Root, schema)).FirstOrDefault(link => link != default); - if (href == default) - return await Described(host, default, "no NodeInfo", token); + await Describe(job.Payload, crawled: false, default, token); + return JobOutcome.Done; + } - using var nodeInfo = await _http.GetJson(Address(href), "application/json", sign: default, token); - return nodeInfo == default ? await Described(host, default, "NodeInfo unreadable", token) : await Described(host, nodeInfo.Root, default, token); + //NodeInfo, the Mastodon instance API where the software has one, and where the address we reached sits; unsigned, + //because none of these are ActivityPub documents. `allowed` is the crawler's robots.txt. + public async Task Describe(string host, bool crawled, Func allowed, CancellationToken token) + { + allowed ??= _ => true; + NodeInfoFacts facts = default; + string rawNodeInfo = default, error; + using (HttpScope.For("nodeinfo")) + { + error = allowed("/.well-known/nodeinfo") ? "no NodeInfo" : "robots.txt disallows NodeInfo"; + using var links = allowed("/.well-known/nodeinfo") + ? await _http.GetJson(Address($"https://{host}/.well-known/nodeinfo"), "application/json", sign: default, token) + : default; + var (href, version) = links == default ? default : InstanceDocuments.NodeInfoLink(links.Root); + if (href != default && allowed(new Uri(href).AbsolutePath)) + { + using var nodeInfo = await _http.GetJson(Address(href), "application/json", sign: default, token); + if (nodeInfo != default) + { + facts = InstanceDocuments.NodeInfo(nodeInfo.Root, version); + rawNodeInfo = nodeInfo.Root.GetRawText(); + error = default; + } + else + error = "NodeInfo unreadable"; + } + } + + InstanceApiFacts api = default; + string apiKind = default, rawApi = default; + if (InstanceDocuments.SpeaksMastodonApi(facts?.Software)) + using (HttpScope.For("instance-api")) + foreach (var (path, kind) in new[] { ("/api/v2/instance", "mastodon-v2"), ("/api/v1/instance", "mastodon-v1") }) + { + if (!allowed(path)) + continue; + using var document = await _http.GetJson(Address($"https://{host}{path}"), "application/json", sign: default, token); + if (document == default || document.Root.ValueKind != JsonValueKind.Object) + continue; + api = InstanceDocuments.InstanceApi(document.Root); + apiKind = kind; + rawApi = document.Root.GetRawText(); + break; + } + + var geo = Locate(host); + var now = DateTime.UtcNow; + var update = DB.Default.Update() + .Match(i => i.Host == host) + .Modify(i => i.DescribedAt, now) + .Modify(i => i.DescriptionError, error) + .Option(o => o.IsUpsert = true); + if (crawled) + update = update + .Modify(b => b.SetOnInsert(i => i.Seen, "crawled")) + .Modify(b => b.SetOnInsert(i => i.FirstSeenAt, now)); + if (facts != default) + update = update + .Modify(i => i.Software, facts.Software) + .Modify(i => i.SoftwareVersion, facts.SoftwareVersion) + .Modify(i => i.NodeName, facts.NodeName) + .Modify(i => i.NodeDescription, facts.NodeDescription) + .Modify(i => i.NodeInfoVersion, facts.Version) + .Modify(i => i.Protocols, facts.Protocols) + .Modify(i => i.OpenRegistrations, facts.OpenRegistrations) + .Modify(i => i.UsersTotal, facts.UsersTotal) + .Modify(i => i.UsersActiveMonth, facts.UsersActiveMonth) + .Modify(i => i.UsersActiveHalfyear, facts.UsersActiveHalfyear) + .Modify(i => i.LocalPosts, facts.LocalPosts) + .Modify(i => i.LocalComments, facts.LocalComments) + .Modify(i => i.NodeInfo, rawNodeInfo.Length <= MaxRawChars ? rawNodeInfo : default); + if (api != default) + update = update + .Modify(i => i.InstanceApi, apiKind) + .Modify(i => i.InstanceApiRaw, rawApi.Length <= MaxRawChars ? rawApi : default) + .Modify(i => i.Title, api.Title) + .Modify(i => i.Languages, api.Languages ?? facts?.Languages) + .Modify(i => i.RegistrationMode, api.RegistrationMode) + .Modify(i => i.MaxPostChars, api.MaxPostChars) + .Modify(i => i.MastodonApiVersion, api.ApiVersion) + .Modify(i => i.SourceUrl, api.SourceUrl); + else if (facts?.Languages != default) + update = update.Modify(i => i.Languages, facts.Languages); + if (geo != default) + update = update.Modify(i => i.Geo, geo); + await update.ExecuteAsync(token); + var instance = await DB.Default.Find().Match(i => i.Host == host).ExecuteFirstAsync(token); + + var week = Week(now); + await DB.Default.Update() + .Match(s => s.Host == host && s.Week == week) + .Modify(s => s.TakenAt, now) + .Modify(s => s.Seen, instance?.Seen) + .Modify(s => s.Reachable, facts != default || api != default) + .Modify(s => s.Software, instance?.Software) + .Modify(s => s.SoftwareVersion, instance?.SoftwareVersion) + .Modify(s => s.UsersTotal, instance?.UsersTotal) + .Modify(s => s.UsersActiveMonth, instance?.UsersActiveMonth) + .Modify(s => s.UsersActiveHalfyear, instance?.UsersActiveHalfyear) + .Modify(s => s.LocalPosts, instance?.LocalPosts) + .Modify(s => s.OpenRegistrations, instance?.OpenRegistrations) + .Modify(s => s.RegistrationMode, instance?.RegistrationMode) + .Modify(s => s.Languages, instance?.Languages) + .Modify(s => s.Country, instance?.Geo?.Country) + .Modify(s => s.Asn, instance?.Geo?.Asn) + .Modify(s => s.Cdn, instance?.Geo?.Cdn) + .Modify(s => s.Ipv6, instance?.Geo?.Ipv6) + .Option(o => o.IsUpsert = true) + .ExecuteAsync(token); + return facts != default; + } + + InstanceGeo Locate(string host) + { + var reached = _addresses?.Of(new Uri(Address($"https://{host}/")).Host); + var fix = reached == default ? default : _geo?.Locate(reached); + if (fix == default) + return default; + return new InstanceGeo + { + Country = fix.Country, + City = fix.City, + Latitude = fix.Latitude, + Longitude = fix.Longitude, + Asn = fix.Asn, + AsnOrg = fix.AsnOrg, + Cdn = CdnNetworks.Of(fix.Asn), + Ipv6 = reached.AddressFamily == AddressFamily.InterNetworkV6, + Source = _geo.Source, + LocatedAt = DateTime.UtcNow + }; } //where an https address is fetched from: itself, except in tests, which point it at a plain-http peer protected virtual string Address(string url) => url; - - static async Task Described(string host, JsonElement? nodeInfo, string error, CancellationToken token) - { - var update = DB.Default.Update() - .Match(i => i.Host == host) - .Modify(i => i.DescribedAt, DateTime.UtcNow) - .Modify(i => i.DescriptionError, error) - .Option(o => o.IsUpsert = true); - if (nodeInfo is { } root) - { - var raw = root.GetRawText(); - update = update - .Modify(i => i.Software, Text(root, "software", "name")) - .Modify(i => i.SoftwareVersion, Text(root, "software", "version")) - .Modify(i => i.NodeName, Text(root, "metadata", "nodeName")) - .Modify(i => i.Protocols, root.TryGetProperty("protocols", out var protocols) && protocols.ValueKind == JsonValueKind.Array - ? protocols.EnumerateArray().Where(p => p.ValueKind == JsonValueKind.String).Select(p => p.GetString()).Take(16).ToList() - : new List()) - .Modify(i => i.OpenRegistrations, root.TryGetProperty("openRegistrations", out var open) && open.ValueKind is JsonValueKind.True or JsonValueKind.False - ? open.GetBoolean() - : default(bool?)) - .Modify(i => i.NodeInfo, raw.Length <= MaxNodeInfoChars ? raw : default); - } - await update.ExecuteAsync(token); - return JobOutcome.Done; - } - - static string LinkFor(JsonElement root, string schema) => - root.TryGetProperty("links", out var links) && links.ValueKind == JsonValueKind.Array - ? links.EnumerateArray() - .Where(l => l.ValueKind == JsonValueKind.Object && l.TryGetProperty("rel", out var rel) && rel.ValueKind == JsonValueKind.String && rel.GetString() == schema) - .Select(l => l.TryGetProperty("href", out var href) && href.ValueKind == JsonValueKind.String ? href.GetString() : default) - .FirstOrDefault(h => h != default && h.StartsWith("https://", StringComparison.OrdinalIgnoreCase)) - : default; - - static string Text(JsonElement root, string parent, string name) - { - if (!root.TryGetProperty(parent, out var section) || section.ValueKind != JsonValueKind.Object - || !section.TryGetProperty(name, out var value) || value.ValueKind != JsonValueKind.String) - return default; - var text = value.GetString(); - return text.Length <= 200 ? text : text[..200]; - } } } diff --git a/PrivaPub/Federation/Objects/InstanceDocuments.cs b/PrivaPub/Federation/Objects/InstanceDocuments.cs new file mode 100644 index 0000000..b51f6ea --- /dev/null +++ b/PrivaPub/Federation/Objects/InstanceDocuments.cs @@ -0,0 +1,156 @@ +using System.Globalization; +using System.Text.Json; +using System.Text.RegularExpressions; + +namespace PrivaPub.Federation.Objects +{ + public sealed record NodeInfoFacts(string Version, string Software, string SoftwareVersion, string NodeName, string NodeDescription, + List Protocols, bool? OpenRegistrations, long? UsersTotal, long? UsersActiveMonth, long? UsersActiveHalfyear, long? LocalPosts, + long? LocalComments, List Languages); + + public sealed record InstanceApiFacts(string Title, List Languages, string RegistrationMode, int? MaxPostChars, int? ApiVersion, string SourceUrl); + + public static partial class InstanceDocuments + { + public static readonly string[] Schemas = + { + "http://nodeinfo.diaspora.software/ns/schema/2.2", + "http://nodeinfo.diaspora.software/ns/schema/2.1", + "http://nodeinfo.diaspora.software/ns/schema/2.0" + }; + + static readonly HashSet MastodonLike = new(StringComparer.OrdinalIgnoreCase) + { + "mastodon", "hometown", "glitch-soc", "gotosocial", "pleroma", "akkoma", "sharkey", "iceshrimp", "iceshrimp.net", "firefish", "catodon", + "friendica", "pixelfed", "takahe", "snac", "honk", "mitra", "pachli" + }; + + const long MaxCount = 1_000_000_000; + + public static bool SpeaksMastodonApi(string software) => software != default && MastodonLike.Contains(software); + + public static (string Href, string Version) NodeInfoLink(JsonElement links) + { + if (links.ValueKind != JsonValueKind.Object || !links.TryGetProperty("links", out var list) || list.ValueKind != JsonValueKind.Array) + return default; + foreach (var schema in Schemas) + foreach (var link in list.EnumerateArray()) + if (link.ValueKind == JsonValueKind.Object && Text(link, "rel") == schema && Text(link, "href") is { } href + && href.StartsWith("https://", StringComparison.OrdinalIgnoreCase)) + return (href, schema[(schema.LastIndexOf('/') + 1)..]); + return default; + } + + public static NodeInfoFacts NodeInfo(JsonElement root, string version) + { + var software = Section(root, "software"); + var usage = Section(root, "usage"); + var users = Section(usage, "users"); + var metadata = Section(root, "metadata"); + return new NodeInfoFacts( + version, + Text(software, "name", 200), + Text(software, "version", 200), + Text(metadata, "nodeName", 200), + Plain(Text(metadata, "nodeDescription", 2000), 500), + Strings(root, "protocols", 16, value => value.Length <= 40), + root.ValueKind == JsonValueKind.Object && root.TryGetProperty("openRegistrations", out var open) && open.ValueKind is JsonValueKind.True or JsonValueKind.False + ? open.GetBoolean() + : (bool?)null, + Count(users, "total"), + Count(users, "activeMonth"), + Count(users, "activeHalfyear"), + Count(usage, "localPosts"), + Count(usage, "localComments"), + Languages(metadata, "languages") ?? Languages(metadata, "langs")); + } + + public static InstanceApiFacts InstanceApi(JsonElement root) + { + var registrations = root.ValueKind == JsonValueKind.Object && root.TryGetProperty("registrations", out var r) ? r : default; + string mode; + if (registrations.ValueKind == JsonValueKind.Object) + mode = Flag(registrations, "enabled") switch + { + true => Flag(registrations, "approval_required") == true ? "approval" : "open", + false => "closed", + _ => default + }; + else if (registrations.ValueKind is JsonValueKind.True or JsonValueKind.False) + mode = registrations.GetBoolean() ? Flag(root, "approval_required") == true ? "approval" : "open" : "closed"; + else + mode = default; + var statuses = Section(Section(root, "configuration"), "statuses"); + var maxChars = Count(statuses, "max_characters") ?? Count(root, "max_toot_chars"); + var apiVersion = Count(Section(root, "api_versions"), "mastodon"); + var source = Text(root, "source_url", 300); + return new InstanceApiFacts( + Plain(Text(root, "title", 300), 200), + Languages(root, "languages"), + mode, + maxChars is { } chars ? (int)Math.Min(chars, int.MaxValue) : (int?)null, + apiVersion is { } api ? (int)Math.Min(api, int.MaxValue) : (int?)null, + source != default && source.StartsWith("https://", StringComparison.OrdinalIgnoreCase) ? source : default); + } + + static JsonElement Section(JsonElement parent, string name) => + parent.ValueKind == JsonValueKind.Object && parent.TryGetProperty(name, out var value) && value.ValueKind == JsonValueKind.Object ? value : default; + + static string Text(JsonElement parent, string name, int max = 2048) + { + if (parent.ValueKind != JsonValueKind.Object || !parent.TryGetProperty(name, out var value) || value.ValueKind != JsonValueKind.String) + return default; + var text = value.GetString(); + return text.Length <= max ? text : text[..max]; + } + + static bool? Flag(JsonElement parent, string name) => + parent.ValueKind == JsonValueKind.Object && parent.TryGetProperty(name, out var value) && value.ValueKind is JsonValueKind.True or JsonValueKind.False + ? value.GetBoolean() + : (bool?)null; + + static long? Count(JsonElement parent, string name) + { + if (parent.ValueKind != JsonValueKind.Object || !parent.TryGetProperty(name, out var value)) + return default; + long? number = value.ValueKind switch + { + JsonValueKind.Number when value.TryGetInt64(out var whole) => whole, + JsonValueKind.Number when value.TryGetDouble(out var real) && real is >= 0 and < MaxCount => (long)real, + JsonValueKind.String when long.TryParse(value.GetString(), NumberStyles.Integer, CultureInfo.InvariantCulture, out var parsed) => parsed, + _ => (long?)null + }; + return number is { } n ? Math.Clamp(n, 0, MaxCount) : (long?)null; + } + + static List Strings(JsonElement parent, string name, int max, Func keep) => + parent.ValueKind == JsonValueKind.Object && parent.TryGetProperty(name, out var value) && value.ValueKind == JsonValueKind.Array + ? value.EnumerateArray().Where(v => v.ValueKind == JsonValueKind.String).Select(v => v.GetString()).Where(keep).Distinct().Take(max).ToList() + : new List(); + + static List Languages(JsonElement parent, string name) + { + var languages = Strings(parent, name, 16, language => LanguageCode().IsMatch(language.ToLowerInvariant())) + .Select(language => language.ToLowerInvariant()).Distinct().ToList(); + return languages.Count == 0 ? default : languages; + } + + static string Plain(string text, int max) + { + if (string.IsNullOrWhiteSpace(text)) + return default; + var plain = Tags().Replace(text, " ").Trim(); + plain = Spaces().Replace(plain, " "); + return plain.Length <= max ? plain : plain[..max]; + } + + [GeneratedRegex("^[a-z]{2,3}(-[a-z0-9]{2,8})?$")] + private static partial Regex LanguageCode(); + + [GeneratedRegex("<[^>]*>")] + private static partial Regex Tags(); + + [GeneratedRegex("\\s+")] + private static partial Regex Spaces(); + } +} diff --git a/PrivaPub/Federation/Objects/ObjectRecords.cs b/PrivaPub/Federation/Objects/ObjectRecords.cs index 3ebe25a..c1c8217 100644 --- a/PrivaPub/Federation/Objects/ObjectRecords.cs +++ b/PrivaPub/Federation/Objects/ObjectRecords.cs @@ -99,8 +99,7 @@ namespace PrivaPub.Federation.Objects var known = await DB.Default.Find().Match(i => i.Host == host).ExecuteFirstAsync(token); if (known?.DescribedAt > DateTime.UtcNow - DescriptionAge) return; - var week = ISOWeek.GetYear(DateTime.UtcNow) + "-" + ISOWeek.GetWeekOfYear(DateTime.UtcNow); - await _queue.Enqueue(JobKind.DescribeInstance, host, host, $"describe|{host}|{week}", token); + await _queue.Enqueue(JobKind.DescribeInstance, host, host, InstanceDescriber.DedupeKey(host, DateTime.UtcNow), token); } public static (string Text, string Hash, int Bytes, bool Truncated) Capture(JsonNode raw) diff --git a/PrivaPub/Infrastructure/Data/Indexes.cs b/PrivaPub/Infrastructure/Data/Indexes.cs index a06150d..3af9b9e 100644 --- a/PrivaPub/Infrastructure/Data/Indexes.cs +++ b/PrivaPub/Infrastructure/Data/Indexes.cs @@ -127,6 +127,11 @@ namespace PrivaPub.Infrastructure.Data await DB.Default.Index().Key(d => d.Day, KeyType.Ascending).Key(d => d.Host, KeyType.Ascending).Option(o => o.Unique = true).CreateAsync(token); await Plain(token, d => d.Host, d => d.Day); await DB.Default.Index().Key(d => d.Day, KeyType.Ascending).Option(o => o.Unique = true).CreateAsync(token); + await DB.Default.Index().Key(s => s.Host, KeyType.Ascending).Key(s => s.Week, KeyType.Ascending) + .Option(o => o.Unique = true).CreateAsync(token); + await Plain(token, s => s.Week); + await Plain(token, i => i.Seen); + await Plain(token, i => i.LastCrawledAt); } static async Task Unique(System.Linq.Expressions.Expression> key, FilterDefinition partial, diff --git a/PrivaPub/Infrastructure/Data/Migrations/_010_remote_instances_were_touched.cs b/PrivaPub/Infrastructure/Data/Migrations/_010_remote_instances_were_touched.cs new file mode 100644 index 0000000..4a8e8e8 --- /dev/null +++ b/PrivaPub/Infrastructure/Data/Migrations/_010_remote_instances_were_touched.cs @@ -0,0 +1,24 @@ +using MongoDB.Bson; +using MongoDB.Entities; + +using PrivaPub.Models.Jobs; + +namespace PrivaPub.Infrastructure.Data.Migrations +{ + public class _010_remote_instances_were_touched : IMigration + { + public async Task UpgradeAsync() + { + foreach (var instance in await DB.Default.Find().Match(i => i.Seen == null).ExecuteAsync()) + { + var firstSeen = ObjectId.TryParse(instance.ID, out var id) ? id.CreationTime : DateTime.UtcNow; + var lastSeen = new[] { instance.LastSuccessAt, instance.LastFailureAt, instance.DescribedAt }.Max() ?? firstSeen; + await DB.Default.Update().MatchID(instance.ID) + .Modify(i => i.Seen, "touched") + .Modify(i => i.FirstSeenAt, firstSeen) + .Modify(i => i.LastSeenAt, lastSeen) + .ExecuteAsync(); + } + } + } +} diff --git a/PrivaPub/Infrastructure/Statistics/InteractionLedger.cs b/PrivaPub/Infrastructure/Statistics/InteractionLedger.cs index a0b05b1..1ce8843 100644 --- a/PrivaPub/Infrastructure/Statistics/InteractionLedger.cs +++ b/PrivaPub/Infrastructure/Statistics/InteractionLedger.cs @@ -3,6 +3,8 @@ using Microsoft.Extensions.Options; using MongoDB.Driver; using MongoDB.Entities; +using PrivaPub.Federation.Moderation; +using PrivaPub.Infrastructure.Jobs; using PrivaPub.Models.Jobs; using PrivaPub.Models.Statistics; @@ -41,9 +43,16 @@ namespace PrivaPub.Infrastructure.Statistics SingleWriter = false, SingleReader = false }); + static readonly TimeSpan TouchInterval = TimeSpan.FromHours(1); + static readonly HashSet TouchingPurposes = new(StringComparer.Ordinal) { "actor", "key", "object", "webfinger", "context" }; + readonly InteractionSalts _salts; readonly IOptionsMonitor _options; readonly ILogger _logger; + readonly IJobQueue _queue; + readonly IDomainBlocks _blocks; + readonly ConcurrentDictionary _touches = new(StringComparer.Ordinal); + readonly ConcurrentDictionary _touchedAt = new(StringComparer.Ordinal); readonly SemaphoreSlim _flushing = new(1, 1); readonly ConcurrentDictionary<(DateTime Day, string Host, string Key), long> _reads = new(); readonly ConcurrentDictionary<(DateTime Day, string Field, string Key), long> _server = new(); @@ -54,11 +63,14 @@ namespace PrivaPub.Infrastructure.Statistics long _written; long _failed; - public InteractionLedger(InteractionSalts salts, IOptionsMonitor options, ILogger logger) + public InteractionLedger(InteractionSalts salts, IOptionsMonitor options, ILogger logger, + IJobQueue queue = default, IDomainBlocks blocks = default) { _salts = salts; _options = options; _logger = logger; + _queue = queue; + _blocks = blocks; } public long Dropped => Interlocked.Read(ref _dropped); @@ -85,6 +97,9 @@ namespace PrivaPub.Infrastructure.Statistics var day = DateTime.UtcNow.Date; host = Interactions.Host(host); key = Key(key); + if (key.StartsWith("http:", StringComparison.Ordinal) && key.EndsWith(":ok", StringComparison.Ordinal) + && TouchingPurposes.Contains(key.Split(':')[1])) + _touches[host] = 0; _reads.AddOrUpdate((day, host, key), 1, (_, count) => count + 1); if (bytes > 0) _reads.AddOrUpdate((day, host, key + ":bytes"), bytes, (_, count) => count + bytes); @@ -118,6 +133,7 @@ namespace PrivaPub.Infrastructure.Statistics { } await FlushEvents(stoppingToken); + await FlushTouches(stoppingToken); if (DateTime.UtcNow - countersAt >= CounterInterval) { countersAt = DateTime.UtcNow; @@ -152,9 +168,56 @@ namespace PrivaPub.Infrastructure.Statistics public async Task Flush(CancellationToken token) { await FlushEvents(token); + await FlushTouches(token); await FlushCounters(token); } + static bool Touches(InteractionEvent e, bool verified) => e.Channel switch + { + Interactions.Receive => verified && e.Outcome == Interactions.Queued, + Interactions.In or Interactions.Out => true, + Interactions.Http => e.Outcome == Interactions.Ok && TouchingPurposes.Contains(e.Purpose ?? string.Empty), + _ => false + }; + + //a server we exchanged something with is marked as seen, and described once a week + async Task FlushTouches(CancellationToken token) + { + await _flushing.WaitAsync(token); + try + { + await Touch(token); + } + finally + { + _flushing.Release(); + } + } + + async Task Touch(CancellationToken token) + { + foreach (var host in _touches.Keys.ToList()) + { + _touches.TryRemove(host, out _); + var now = DateTime.UtcNow; + if (host == Interactions.Unknown || _touchedAt.TryGetValue(host, out var at) && now - at < TouchInterval) + continue; + _touchedAt[host] = now; + await DB.Default.Update() + .Match(i => i.Host == host) + .Modify(i => i.Seen, "touched") + .Modify(i => i.LastSeenAt, now) + .Modify(b => b.SetOnInsert(i => i.FirstSeenAt, now)) + .Option(o => o.IsUpsert = true) + .ExecuteAsync(token); + _knownHosts.Add(host); + if (_queue != default && _blocks?.IsSuspended(host) != true) + await _queue.Enqueue(JobKind.DescribeInstance, host, host, Federation.Objects.InstanceDescriber.DedupeKey(host, now), token); + } + if (_touchedAt.Count > 100_000) + _touchedAt.Clear(); + } + async Task FlushEvents(CancellationToken token) { await _flushing.WaitAsync(token); @@ -192,6 +255,8 @@ namespace PrivaPub.Infrastructure.Statistics interaction.Host = Interactions.Unknown; if (pending.ActorUri != default) interaction.ActorHash = InteractionSalts.Hash(await _salts.For(interaction.At, token), pending.ActorUri); + if (interaction.Host != Interactions.Unknown && Touches(interaction, pending.ActorUri != default)) + _touches[interaction.Host] = 0; return interaction; } diff --git a/PrivaPub/Models/Jobs/RemoteInstance.cs b/PrivaPub/Models/Jobs/RemoteInstance.cs index 8a4b5ee..3ee4489 100644 --- a/PrivaPub/Models/Jobs/RemoteInstance.cs +++ b/PrivaPub/Models/Jobs/RemoteInstance.cs @@ -19,5 +19,68 @@ namespace PrivaPub.Models.Jobs public string NodeInfo { get; set; }//the raw document, when small enough public DateTime? DescribedAt { get; set; } public string DescriptionError { get; set; } + + public string Seen { get; set; }//"touched": we exchanged something with it; "crawled": only its peers list named it + public DateTime? FirstSeenAt { get; set; } + public DateTime? LastSeenAt { get; set; } + public DateTime? LastCrawledAt { get; set; } + public string CrawlError { get; set; } + public bool? RobotsDisallowed { get; set; } + public int? PeersCount { get; set; } + + public string NodeInfoVersion { get; set; } + public string NodeDescription { get; set; } + public long? UsersTotal { get; set; }//as the server publishes them in its NodeInfo + public long? UsersActiveMonth { get; set; } + public long? UsersActiveHalfyear { get; set; } + public long? LocalPosts { get; set; } + public long? LocalComments { get; set; } + + public string InstanceApi { get; set; }//"mastodon-v2" or "mastodon-v1" + public string InstanceApiRaw { get; set; }//admin only: it names maintainers and contact accounts + public string Title { get; set; } + public List Languages { get; set; } + public string RegistrationMode { get; set; }//open, approval, closed + public int? MaxPostChars { get; set; } + public int? MastodonApiVersion { get; set; } + public string SourceUrl { get; set; } + + public InstanceGeo Geo { get; set; } + } + + public class InstanceGeo + { + public string Country { get; set; } + public string City { get; set; } + public double? Latitude { get; set; }//rounded to one decimal + public double? Longitude { get; set; } + public int? Asn { get; set; } + public string AsnOrg { get; set; } + public string Cdn { get; set; }//the CDN in front of it, whose edge address says nothing about where it runs + public bool? Ipv6 { get; set; } + public string Source { get; set; } + public DateTime? LocatedAt { get; set; } + } + + public class RemoteInstanceSnapshot : Entity + { + public string Host { get; set; } + public string Week { get; set; }//ISO week, "2026-W40" + public DateTime TakenAt { get; set; } + public string Seen { get; set; } + public bool Reachable { get; set; } + public string Software { get; set; } + public string SoftwareVersion { get; set; } + public long? UsersTotal { get; set; } + public long? UsersActiveMonth { get; set; } + public long? UsersActiveHalfyear { get; set; } + public long? LocalPosts { get; set; } + public bool? OpenRegistrations { get; set; } + public string RegistrationMode { get; set; } + public List Languages { get; set; } + public string Country { get; set; } + public int? Asn { get; set; } + public string Cdn { get; set; } + public bool? Ipv6 { get; set; } } }