diff --git a/CLAUDE.md b/CLAUDE.md index af3896f..45e721a 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -319,7 +319,12 @@ group www-data and reaches the private mongod; `sudo -u www-data` works too. revision. Delivery details reach them through `Arrival.Current`, which `InboxProcessor` sets for the handler's duration. A fetched record has no signature, key or `@context` of the activity that caused the fetch (its activity fields name that trigger), and every record stores its `Extensions` and `ContextNamespaces` for statistics. The raw form lives outside `Post` so timelines never load it. Read through - `/api/privapub/v1/statuses/:id/provenance` and `/api/privapub/v1/instances/:host`. + `/api/privapub/v1/statuses/:id/provenance` and `/api/privapub/v1/instances/:host` (or `?host[]=`, up to 40; this + server answers too). +- **A server's place leaves only through `PublicGeo.Project`:** city, coordinates (0.1°) and network for servers + reporting at least `Statistics:PublicCityMinUsers` users and not behind a CDN, the country otherwise, the CDN's name + for a CDN-fronted one. This server's own place is its host's address located once a day (`SelfLocation`), or + `Statistics:Geo:Self` when the owner sets it. - **Remote objects are parsed for every shape in `ObjectShapes`:** `url`/`icon`/`image` as a value, an object or an array; Markdown `content`; missing `mediaType`s inferred; thumbnails from any of their five places; and the typed `Link`, `Video`, `Audio` and `Event` details. A Mastodon API card is built from those, never by fetching the linked page diff --git a/PrivaPub.Tests/Http/InstanceLocationTests.cs b/PrivaPub.Tests/Http/InstanceLocationTests.cs new file mode 100644 index 0000000..c4740ea --- /dev/null +++ b/PrivaPub.Tests/Http/InstanceLocationTests.cs @@ -0,0 +1,204 @@ +using MongoDB.Entities; + +using PrivaPub.Models.Jobs; +using PrivaPub.Tests.Support; +using PrivaPub.Tests.Support.Host; + +using System.Net; +using System.Text.Json.Nodes; + +namespace PrivaPub.Tests.Http +{ + // Where a server is, on the public instance API: the projection decided for public server locations (city and network + // only for servers reporting at least 10 users and not behind a CDN, the country otherwise, the CDN's name for a + // CDN-fronted one), the batch form, and this server itself. + [Trait("Category", "Integration")] + public sealed class InstanceLocationTests : IAsyncLifetime + { + PrivaPubHost _host; + + public async ValueTask InitializeAsync() + { + Assert.SkipUnless(MongoFixture.Enabled, MongoFixture.Skip); + _host = await PrivaPubHost.Shared(); + } + + public ValueTask DisposeAsync() => ValueTask.CompletedTask; + + static CancellationToken Token => TestContext.Current.CancellationToken; + + static async Task Instance(int users, InstanceGeo geo, string name = "located") + { + var host = $"{name}{Guid.NewGuid():N}.example"; + await DB.Default.SaveAsync(new RemoteInstance { Host = host, Software = "mastodon", UsersTotal = users, Geo = geo }, Token); + return host; + } + + static InstanceGeo Milan(string cdn = default) => new() + { + Country = "IT", + City = "Milan", + Latitude = 45.5, + Longitude = 9.2, + Asn = 12345, + AsnOrg = "Example Hosting", + Cdn = cdn, + Ipv6 = true, + Source = "DB-IP Lite 2026-10", + LocatedAt = DateTime.UtcNow.AddDays(-2) + }; + + async Task Geo(string host) => (await _host.Client().Get($"/api/privapub/v1/instances/{host}")).Ok().Body["geo"]; + + [Fact] + public async Task A_server_with_enough_users_shows_its_city_coordinates_and_network() + { + var geo = await Geo(await Instance(50, Milan())); + + Assert.Equal("city", geo.Text("precision")); + Assert.Equal(("IT", "Milan", "Example Hosting"), (geo.Text("country"), geo.Text("city"), geo.Text("network"))); + Assert.Equal(45.5, geo["latitude"]!.GetValue()); + Assert.Equal(9.2, geo["longitude"]!.GetValue()); + Assert.Equal("DB-IP Lite 2026-10", geo.Text("source")); + Assert.NotNull(geo.Text("located_at")); + Assert.Null(geo["asn"]); + Assert.Null(geo["ipv6"]); + } + + [Fact] + public async Task A_small_server_shows_only_its_country() + { + var geo = await Geo(await Instance(3, Milan())); + + Assert.Equal("country", geo.Text("precision")); + Assert.Equal("IT", geo.Text("country")); + Assert.Null(geo["city"]); + Assert.Null(geo["latitude"]); + Assert.Null(geo["longitude"]); + Assert.Null(geo["network"]); + } + + [Fact] + public async Task A_cdn_fronted_server_shows_only_its_cdn() + { + var geo = await Geo(await Instance(5000, Milan(cdn: "Cloudflare"))); + + Assert.Equal("cdn", geo.Text("precision")); + Assert.Equal("Cloudflare", geo.Text("cdn")); + Assert.Null(geo["country"]); + Assert.Null(geo["latitude"]); + Assert.Null(geo["network"]); + } + + [Fact] + public async Task An_unlocated_server_has_no_place() + { + Assert.Null(await Geo(await Instance(50, default))); + } + + [Fact] + public async Task The_provenance_of_a_post_carries_the_same_projection() + { + var host = await Instance(3, Milan()); + var described = (await _host.Client().Get($"/api/privapub/v1/instances/{host}")).Ok().Body; + + Assert.Equal("country", described["geo"].Text("precision")); + Assert.Equal(described["geo"]!.ToJsonString(), (await Geo(host))!.ToJsonString()); + } + + [Fact] + public async Task Several_servers_answer_at_once_in_the_order_asked_without_the_unknown_ones() + { + var big = await Instance(50, Milan(), "big"); + var small = await Instance(3, Milan(), "small"); + var unknown = $"unknown{Guid.NewGuid():N}.example"; + + var answer = (await _host.Client().Get($"/api/privapub/v1/instances?host[]={small}&host[]={unknown}&host[]={big.ToUpperInvariant()}&host[]={small}")).Ok(); + var hosts = answer.Body!.AsArray().Select(i => i.Text("host")).ToList(); + + Assert.Equal(new[] { small, big }, hosts); + Assert.Equal("country", answer.Body![0]["geo"].Text("precision")); + Assert.Equal("city", answer.Body![1]["geo"].Text("precision")); + Assert.Empty((await _host.Client().Get("/api/privapub/v1/instances")).Ok().Body!.AsArray()); + } + + [Fact] + public async Task At_most_forty_servers_answer_at_once() + { + var hosts = new List(); + for (var i = 0; i < 42; i++) + hosts.Add(await Instance(1, default, "many")); + + var answer = (await _host.Client().Get("/api/privapub/v1/instances?" + string.Join("&", hosts.Select(h => "host[]=" + h)))).Ok(); + + Assert.Equal(hosts.Take(40), answer.Body!.AsArray().Select(i => i.Text("host"))); + } + + [Fact] + public async Task This_server_describes_itself() + { + var self = (await _host.Client().Get($"/api/privapub/v1/instances/{PrivaPubHost.Host}")).Ok().Body; + + Assert.Equal(PrivaPubHost.Host, self.Text("host")); + Assert.Equal("privapub", self.Text("software")); + Assert.Equal("activitypub", Assert.Single(self["protocols"]!.AsArray())!.GetValue()); + Assert.Null(self["geo"]); + var batch = (await _host.Client().Get($"/api/privapub/v1/instances?host[]={PrivaPubHost.Host}")).Ok().Body!.AsArray(); + Assert.Equal("privapub", Assert.Single(batch).Text("software")); + } + + [Fact] + public async Task The_owner_may_place_this_server() + { + var placed = await SelfPlacedHost.Shared(); + + var geo = (await placed.Client().Get($"/api/privapub/v1/instances/{PrivaPubHost.Host}")).Ok().Body["geo"]; + + Assert.Equal("city", geo.Text("precision")); + Assert.Equal(("IT", "Milan", "configured"), (geo.Text("country"), geo.Text("city"), geo.Text("source"))); + Assert.Equal(45.5, geo["latitude"]!.GetValue()); + Assert.Equal(9.2, geo["longitude"]!.GetValue()); + } + + [Fact] + public async Task Junk_hosts_are_not_an_error() + { + Assert.Equal(HttpStatusCode.NotFound, (await _host.Client().Get($"/api/privapub/v1/instances/{new string('a', 300)}")).Status); + Assert.Empty((await _host.Client().Get($"/api/privapub/v1/instances?host[]=&host[]={new string('a', 300)}")).Ok().Body!.AsArray()); + } + } + + public sealed class SelfPlacedHost : PrivaPubHost + { + static readonly SemaphoreSlim Boot = new(1, 1); + static SelfPlacedHost _shared; + + public static new async Task Shared() + { + await PrivaPubHost.Shared(); + await Boot.WaitAsync(); + try + { + if (_shared == default) + { + var host = new SelfPlacedHost(); + _ = host.Services; + _shared = host; + } + return _shared; + } + finally + { + Boot.Release(); + } + } + + protected override IEnumerable> Settings() => base.Settings().Concat(new Dictionary + { + ["Statistics:Geo:Self:Latitude"] = "45.4642", + ["Statistics:Geo:Self:Longitude"] = "9.19", + ["Statistics:Geo:Self:City"] = "Milan", + ["Statistics:Geo:Self:Country"] = "IT" + }); + } +} diff --git a/PrivaPub/Api/Mastodon/Controllers/ProvenanceController.cs b/PrivaPub/Api/Mastodon/Controllers/ProvenanceController.cs index 7a0cdbf..0ca96a0 100644 --- a/PrivaPub/Api/Mastodon/Controllers/ProvenanceController.cs +++ b/PrivaPub/Api/Mastodon/Controllers/ProvenanceController.cs @@ -1,11 +1,16 @@ -using Microsoft.AspNetCore.Authorization; +using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Mvc; +using Microsoft.Extensions.Options; using MongoDB.Entities; using PrivaPub.Api.Mastodon.Infrastructure; using PrivaPub.Domain.Privacy; +using PrivaPub.Domain.Statistics; +using PrivaPub.Federation.Actors; using PrivaPub.Federation.Objects; +using PrivaPub.Infrastructure; +using PrivaPub.Infrastructure.Statistics; using PrivaPub.Models.Federation; using PrivaPub.Models.Jobs; using PrivaPub.Models.Post; @@ -19,9 +24,23 @@ namespace PrivaPub.Api.Mastodon.Controllers { public class ProvenanceController : MastodonController { - readonly DbEntities _dbEntities; + const int MaxHosts = 40; - public ProvenanceController(DbEntities dbEntities) => _dbEntities = dbEntities; + readonly DbEntities _dbEntities; + readonly ISelfLocation _self; + readonly ILocalActorService _localActors; + readonly IOptionsMonitor _statistics; + readonly IOptionsMonitor _registrations; + + public ProvenanceController(DbEntities dbEntities, ISelfLocation self, ILocalActorService localActors, + IOptionsMonitor statistics, IOptionsMonitor registrations) + { + _dbEntities = dbEntities; + _self = self; + _localActors = localActors; + _statistics = statistics; + _registrations = registrations; + } [HttpGet("/api/privapub/v1/statuses/{id}/provenance"), Scope("read:statuses", requiresUser: false), AllowAnonymous] public async Task Status(string id, CancellationToken token) @@ -44,12 +63,65 @@ namespace PrivaPub.Api.Mastodon.Controllers [HttpGet("/api/privapub/v1/instances/{host}"), Scope("read", requiresUser: false), AllowAnonymous] public async Task Instance(string host, CancellationToken token) { - host = host?.Trim().ToLowerInvariant(); - var instance = await DB.Default.Find().Match(i => i.Host == host).ExecuteFirstAsync(token); - return instance == default ? NotFoundError() : Json(Describe(instance)); + var described = await Describe(new[] { host }, token); + return described.Count == 0 ? NotFoundError() : Json(described[0]); } - static Provenance Describe(PostEntity post, ObjectRecord record, RemoteInstance instance) + //up to MaxHosts at once (host[]=...), in the order asked; hosts this server does not know are left out + [HttpGet("/api/privapub/v1/instances"), Scope("read", requiresUser: false), AllowAnonymous] + public async Task Instances(CancellationToken token) => Json(await Describe(Params.List("host"), token)); + + async Task> Describe(IEnumerable hosts, CancellationToken token) + { + var wanted = hosts.Select(h => h?.Trim().ToLowerInvariant()).Where(h => !string.IsNullOrEmpty(h) && h.Length <= 253) + .Distinct().Take(MaxHosts).ToList(); + var selfHost = Uri.TryCreate(_localActors.BaseAddress, UriKind.Absolute, out var self) ? self.Host.ToLowerInvariant() : default; + var remote = await DB.Default.Find().Match(i => wanted.Contains(i.Host)).ExecuteAsync(token); + var described = new List(); + foreach (var host in wanted) + { + if (host == selfHost) + described.Add(await DescribeSelf(host, token)); + else if (remote.FirstOrDefault(i => i.Host == host) is { } instance) + described.Add(Describe(instance, _statistics.CurrentValue.PublicCityMinUsers)); + } + return described; + } + + async Task DescribeSelf(string host, CancellationToken token) + { + var (geo, configured) = await _self.Get(token); + var users = await Counted.Users(token); + return new InstanceDescription + { + Host = host, + Software = "privapub", + Version = BuildInfo.Ref, + NodeName = "PrivaPub", + Protocols = new() { "activitypub" }, + OpenRegistrations = _registrations.CurrentValue.IsOpen, + DescribedAt = MastodonJson.Time(DateTime.UtcNow), + Geo = configured + ? Located(new PublicLocation(geo.Country, geo.City, geo.Latitude, geo.Longitude, default, default), geo) + : Located(PublicGeo.Project(new RemoteInstance { Geo = geo, UsersTotal = users }, _statistics.CurrentValue.PublicCityMinUsers), geo) + }; + } + + //the public projection of a server's place (owner decision on server locations, ROADMAP 2026-10-03) + static InstanceLocation Located(PublicLocation location, InstanceGeo geo) => location == default ? default : new InstanceLocation + { + Precision = location.Cdn != default ? "cdn" : location.Latitude != default ? "city" : "country", + Country = location.Country, + City = location.City, + Latitude = location.Latitude, + Longitude = location.Longitude, + Network = location.Network, + Cdn = location.Cdn, + Source = geo?.Source, + LocatedAt = geo?.LocatedAt is { } at ? MastodonJson.Time(at) : default + }; + + Provenance Describe(PostEntity post, ObjectRecord record, RemoteInstance instance) { var raw = record?.Raw == default ? default : JsonNode.Parse(record.Raw) as JsonObject; return new Provenance @@ -86,11 +158,11 @@ namespace PrivaPub.Api.Mastodon.Controllers RawHash = r.RawHash, Raw = r.Raw == default ? default : JsonNode.Parse(r.Raw) }).ToList() ?? new(), - Instance = instance == default ? default : Describe(instance) + Instance = instance == default ? default : Describe(instance, _statistics.CurrentValue.PublicCityMinUsers) }; } - static InstanceDescription Describe(RemoteInstance instance) => new() + static InstanceDescription Describe(RemoteInstance instance, int minUsers) => new() { Host = instance.Host, Software = instance.Software, @@ -107,7 +179,8 @@ namespace PrivaPub.Api.Mastodon.Controllers UnavailableUntil = instance.UnavailableUntil is { } until ? MastodonJson.Time(until) : default, LastSuccessAt = instance.LastSuccessAt is { } success ? MastodonJson.Time(success) : default, LastFailureAt = instance.LastFailureAt is { } failure ? MastodonJson.Time(failure) : default - } + }, + Geo = Located(PublicGeo.Project(instance, minUsers), instance.Geo) }; public class Provenance @@ -170,6 +243,21 @@ namespace PrivaPub.Api.Mastodon.Controllers public string DescriptionError { get; set; } public JsonNode NodeInfo { get; set; } public InstanceDelivery Delivery { get; set; } + public InstanceLocation Geo { get; set; } + } + + //precision: "city" (coordinates to 0.1°, city and network), "country" (the country only) or "cdn" (the CDN's name only) + public class InstanceLocation + { + public string Precision { get; set; } + public string Country { get; set; } + public string City { get; set; } + public double? Latitude { get; set; } + public double? Longitude { get; set; } + public string Network { get; set; } + public string Cdn { get; set; } + public string Source { get; set; } + public string LocatedAt { get; set; } } public class InstanceDelivery diff --git a/PrivaPub/Domain/Statistics/SelfLocation.cs b/PrivaPub/Domain/Statistics/SelfLocation.cs new file mode 100644 index 0000000..f20677d --- /dev/null +++ b/PrivaPub/Domain/Statistics/SelfLocation.cs @@ -0,0 +1,103 @@ +using Microsoft.Extensions.Options; + +using PrivaPub.Infrastructure.Geo; +using PrivaPub.Infrastructure.Statistics; +using PrivaPub.Models; +using PrivaPub.Models.Jobs; + +using System.Net; +using System.Net.Sockets; + +namespace PrivaPub.Domain.Statistics +{ + public interface ISelfLocation + { + //where this server is, or default when it cannot tell; Configured when the owner set it (Statistics:Geo:Self) + Task<(InstanceGeo Geo, bool Configured)> Get(CancellationToken token); + } + + // This server's own place, for the instance API (the client's globe draws every server from it). The owner may set it, + // for a server behind a proxy or a CDN; otherwise the server's own host name is resolved and located like any other + // server's, once a day. Never the address of a request. + public class SelfLocation : ISelfLocation + { + static readonly TimeSpan Lifetime = TimeSpan.FromDays(1); + + readonly IGeoLocator _geo; + readonly IOptionsMonitor _options; + readonly IOptionsMonitor _app; + readonly ILogger _logger; + readonly SemaphoreSlim _gate = new(1, 1); + + InstanceGeo _located; + DateTime _locatedAt = DateTime.MinValue; + + public SelfLocation(IGeoLocator geo, IOptionsMonitor options, IOptionsMonitor app, ILogger logger) + { + _geo = geo; + _options = options; + _app = app; + _logger = logger; + } + + public async Task<(InstanceGeo Geo, bool Configured)> Get(CancellationToken token) + { + if (_options.CurrentValue.Geo.Self is { Latitude: { } latitude, Longitude: { } longitude } self) + return (new InstanceGeo + { + Country = self.Country, + City = self.City, + Latitude = Math.Round(latitude, 1), + Longitude = Math.Round(longitude, 1), + Source = "configured" + }, true); + + if (DateTime.UtcNow - _locatedAt < Lifetime) + return (_located, false); + await _gate.WaitAsync(token); + try + { + if (DateTime.UtcNow - _locatedAt >= Lifetime) + { + _located = await Locate(token); + _locatedAt = DateTime.UtcNow; + } + return (_located, false); + } + finally + { + _gate.Release(); + } + } + + async Task Locate(CancellationToken token) + { + try + { + if (!Uri.TryCreate(_app.CurrentValue.BackendBaseAddress, UriKind.Absolute, out var self)) + return default; + var addresses = await Dns.GetHostAddressesAsync(self.Host, token); + var address = addresses.FirstOrDefault(a => a.AddressFamily == AddressFamily.InterNetwork) ?? addresses.FirstOrDefault(); + var fix = address == default ? default : _geo.Locate(address); + return fix == default ? default : new InstanceGeo + { + Country = fix.Country, + City = fix.City, + Latitude = fix.Latitude, + Longitude = fix.Longitude, + Asn = fix.Asn, + AsnOrg = fix.AsnOrg, + Cdn = CdnNetworks.Of(fix.Asn), + Ipv6 = address.AddressFamily == AddressFamily.InterNetworkV6, + Source = _geo.Source, + LocatedAt = DateTime.UtcNow + }; + } + catch (Exception ex) when (ex is SocketException or ArgumentException) + { + _logger.LogInformation("{Service}: this server's host does not resolve", nameof(SelfLocation)); + return default; + } + } + } +} diff --git a/PrivaPub/Infrastructure/Statistics/StatisticsOptions.cs b/PrivaPub/Infrastructure/Statistics/StatisticsOptions.cs index 0a71a51..f38a909 100644 --- a/PrivaPub/Infrastructure/Statistics/StatisticsOptions.cs +++ b/PrivaPub/Infrastructure/Statistics/StatisticsOptions.cs @@ -1,4 +1,4 @@ -namespace PrivaPub.Infrastructure.Statistics +namespace PrivaPub.Infrastructure.Statistics { public class StatisticsOptions { @@ -16,6 +16,16 @@ namespace PrivaPub.Infrastructure.Statistics public string DownloadBase { get; set; } = "https://download.db-ip.com/free"; public long MaxDownloadBytes { get; set; } = 300L * 1024 * 1024; public long MaxDatabaseBytes { get; set; } = 1024L * 1024 * 1024; + //this server's place, when locating its own address would be wrong (behind a proxy or a CDN): the owner's call + public SelfGeoOptions Self { get; set; } = new(); + } + + public class SelfGeoOptions + { + public double? Latitude { get; set; } + public double? Longitude { get; set; } + public string City { get; set; } + public string Country { get; set; } } //owner decision: off by default (on in production since 2026-10-04); when on, one server a minute, each at most weekly, at most MaxHosts servers diff --git a/PrivaPub/Middleware/SocialPubConfigurations.cs b/PrivaPub/Middleware/SocialPubConfigurations.cs index a9b4b7d..8025109 100644 --- a/PrivaPub/Middleware/SocialPubConfigurations.cs +++ b/PrivaPub/Middleware/SocialPubConfigurations.cs @@ -1,4 +1,4 @@ -using Microsoft.AspNetCore.Authentication.JwtBearer; +using Microsoft.AspNetCore.Authentication.JwtBearer; using Microsoft.AspNetCore.ResponseCompression; using OpenIddict.Validation.AspNetCore; @@ -127,6 +127,7 @@ namespace PrivaPub.Middleware .AddHostedService(services => services.GetRequiredService()) .AddSingleton() .AddSingleton() + .AddSingleton() .AddHostedService() .AddSingleton() .AddSingleton() diff --git a/docs/INTEROP.md b/docs/INTEROP.md index 7318aa2..217f211 100644 --- a/docs/INTEROP.md +++ b/docs/INTEROP.md @@ -1,4 +1,4 @@ -# Interop: what every peer sends, what it expects, and what PrivaPub still drops +# Interop: what every peer sends, what it expects, and what PrivaPub still drops Research as of **2026-10-01**: the platforms' source on their default branches, live ActivityPub fetches from large instances, release notes and the FEP repository. Version numbers are what was current that day. Claims the @@ -807,7 +807,9 @@ a remote host. `GET /api/privapub/v1/instances/:host` returns cached NodeInfo (`metadata` extras: Misskey `themeColor`/`maxNoteTextLength`, Pleroma `features[]`/`federation.mrf_policies`), the instance API's icon and description, the delivery health our -circuit breaker sees, and which signature scheme worked. +circuit breaker sees, and which signature scheme worked. Its `geo` is the public projection of where the server is +(`precision` `city`, `country` or `cdn`, ROADMAP owner decision on server locations); `?host[]=` answers up to 40 at +once, and this server describes itself the same way. `software.name` → family, for display: diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index 5e2fa4e..c315163 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -202,6 +202,7 @@ and circles (see Owner decisions). | Question | Decision | |---|---| | Sign-in | **One login.** decePubClient signs in on `/clientapi` and exchanges that JWT for one persona's Mastodon token (RFC 8693 token exchange on `/oauth/token`, `PersonaExchange`), one token per persona it uses. Only the seeded first-party application may exchange; the token names the persona, never the root, like any other. | +| Server locations on the instance API | **Public, as the projection already decided for public server locations** (2026-10-03): city, coordinates to 0.1° and network only for servers reporting at least 10 users and not behind a CDN, the country otherwise, the CDN's name for a CDN-fronted one, with DB-IP's attribution. decePubClient's globe draws posts at their author's server. This server's own place comes from its host's address, or from `Statistics:Geo:Self` when the owner sets it (a server behind a proxy). | ## Libraries (researched; no maintained .NET ActivityPub library exists, so Letterbook and Iceshrimp.NET both wrote their own)