diff --git a/CLAUDE.md b/CLAUDE.md index 102e8e4..2e515d3 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -322,9 +322,8 @@ group www-data and reaches the private mongod; `sudo -u www-data` works too. fields name that trigger), and every record stores its `Extensions` and `ContextNamespaces` for statistics. The raw form lives outside `Post` so timelines never load it. Read through `/api/privapub/v1/statuses/:id/provenance` and `/api/privapub/v1/instances/:host` (or `?host[]=`, up to 40; this server answers too). -- **A server's place leaves only through `PublicGeo.Project`:** city, coordinates (0.1°) and network for servers - reporting at least `Statistics:PublicCityMinUsers` users and not behind a CDN, the country otherwise, the CDN's name - for a CDN-fronted one. This server's own place is its host's address located once a day (`SelfLocation`), or +- **A server's place leaves only through `PublicGeo.Project`:** the city, coordinates (0.1°) and network it was located + to, whatever its size (there is no user threshold), and only the CDN's name for a CDN-fronted one. This server's own place is its host's address located once a day (`SelfLocation`), or `Statistics:Geo:Self` when the owner sets it. - **Remote objects are parsed for every shape in `ObjectShapes`:** `url`/`icon`/`image` as a value, an object or an array; Markdown `content`; missing `mediaType`s inferred; thumbnails from any of their five places; and the typed diff --git a/PrivaPub.Tests/Http/InstanceLocationTests.cs b/PrivaPub.Tests/Http/InstanceLocationTests.cs index c4740ea..6d727d7 100644 --- a/PrivaPub.Tests/Http/InstanceLocationTests.cs +++ b/PrivaPub.Tests/Http/InstanceLocationTests.cs @@ -1,4 +1,4 @@ -using MongoDB.Entities; +using MongoDB.Entities; using PrivaPub.Models.Jobs; using PrivaPub.Tests.Support; @@ -9,9 +9,8 @@ using System.Text.Json.Nodes; namespace PrivaPub.Tests.Http { - // Where a server is, on the public instance API: the projection decided for public server locations (city and network - // only for servers reporting at least 10 users and not behind a CDN, the country otherwise, the CDN's name for a - // CDN-fronted one), the batch form, and this server itself. + // Where a server is, on the public instance API: where it was located (city, coordinates and network, whatever its size), + // only the CDN's name for a CDN-fronted one, the batch form, and this server itself. [Trait("Category", "Integration")] public sealed class InstanceLocationTests : IAsyncLifetime { @@ -51,7 +50,7 @@ namespace PrivaPub.Tests.Http async Task Geo(string host) => (await _host.Client().Get($"/api/privapub/v1/instances/{host}")).Ok().Body["geo"]; [Fact] - public async Task A_server_with_enough_users_shows_its_city_coordinates_and_network() + public async Task A_located_server_shows_its_city_coordinates_and_network() { var geo = await Geo(await Instance(50, Milan())); @@ -66,16 +65,24 @@ namespace PrivaPub.Tests.Http } [Fact] - public async Task A_small_server_shows_only_its_country() + public async Task A_one_user_server_shows_its_city_too() { - var geo = await Geo(await Instance(3, Milan())); + var geo = await Geo(await Instance(1, Milan())); + + Assert.Equal("city", geo.Text("precision")); + Assert.Equal(("IT", "Milan", "Example Hosting"), (geo.Text("country"), geo.Text("city"), geo.Text("network"))); + Assert.Equal(45.5, geo["latitude"]!.GetValue()); + } + + [Fact] + public async Task A_server_located_only_to_its_country_shows_the_country() + { + var geo = await Geo(await Instance(50, new InstanceGeo { Country = "IT", Source = "DB-IP Lite 2026-10", LocatedAt = DateTime.UtcNow })); Assert.Equal("country", geo.Text("precision")); Assert.Equal("IT", geo.Text("country")); Assert.Null(geo["city"]); Assert.Null(geo["latitude"]); - Assert.Null(geo["longitude"]); - Assert.Null(geo["network"]); } [Fact] @@ -102,7 +109,7 @@ namespace PrivaPub.Tests.Http var host = await Instance(3, Milan()); var described = (await _host.Client().Get($"/api/privapub/v1/instances/{host}")).Ok().Body; - Assert.Equal("country", described["geo"].Text("precision")); + Assert.Equal("city", described["geo"].Text("precision")); Assert.Equal(described["geo"]!.ToJsonString(), (await Geo(host))!.ToJsonString()); } @@ -117,7 +124,7 @@ namespace PrivaPub.Tests.Http var hosts = answer.Body!.AsArray().Select(i => i.Text("host")).ToList(); Assert.Equal(new[] { small, big }, hosts); - Assert.Equal("country", answer.Body![0]["geo"].Text("precision")); + Assert.Equal("city", answer.Body![0]["geo"].Text("precision")); Assert.Equal("city", answer.Body![1]["geo"].Text("precision")); Assert.Empty((await _host.Client().Get("/api/privapub/v1/instances")).Ok().Body!.AsArray()); } diff --git a/PrivaPub.Tests/Statistics/DescribeTests.cs b/PrivaPub.Tests/Statistics/DescribeTests.cs index c634c13..5670309 100644 --- a/PrivaPub.Tests/Statistics/DescribeTests.cs +++ b/PrivaPub.Tests/Statistics/DescribeTests.cs @@ -1,4 +1,4 @@ -using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Logging.Abstractions; using MongoDB.Entities; @@ -78,16 +78,16 @@ namespace PrivaPub.Tests.Statistics } [Fact] - public void The_public_location_depends_on_size_and_on_a_cdn() + public void The_public_location_is_where_the_server_was_located_unless_a_cdn_hides_it() { var geo = new InstanceGeo { Country = "DE", City = "Berlin", Latitude = 52.5, Longitude = 13.4, AsnOrg = "Hetzner", Asn = 24940 }; - Assert.Equal(new PublicLocation("DE", "Berlin", 52.5, 13.4, "Hetzner", null), PublicGeo.Project(new RemoteInstance { Geo = geo, UsersTotal = 10 }, 10)); - Assert.Equal(new PublicLocation("DE", null, null, null, null, null), PublicGeo.Project(new RemoteInstance { Geo = geo, UsersTotal = 1 }, 10)); - Assert.Equal(new PublicLocation("DE", null, null, null, null, null), PublicGeo.Project(new RemoteInstance { Geo = geo }, 10)); + Assert.Equal(new PublicLocation("DE", "Berlin", 52.5, 13.4, "Hetzner", null), PublicGeo.Project(geo)); + Assert.Equal(new PublicLocation("DE", null, null, null, null, null), PublicGeo.Project(new InstanceGeo { Country = "DE" })); Assert.Equal(new PublicLocation(null, null, null, null, null, "Cloudflare"), - PublicGeo.Project(new RemoteInstance { Geo = new InstanceGeo { Country = "US", City = "SF", Cdn = "Cloudflare" }, UsersTotal = 5000 }, 10)); - Assert.Null(PublicGeo.Project(new RemoteInstance(), 10)); + PublicGeo.Project(new InstanceGeo { Country = "US", City = "SF", Latitude = 37.8, Longitude = -122.4, Cdn = "Cloudflare" })); + Assert.Null(PublicGeo.Project(new InstanceGeo())); + Assert.Null(PublicGeo.Project(null)); } } diff --git a/PrivaPub/Api/Mastodon/Controllers/ProvenanceController.cs b/PrivaPub/Api/Mastodon/Controllers/ProvenanceController.cs index 0ca96a0..98ef32a 100644 --- a/PrivaPub/Api/Mastodon/Controllers/ProvenanceController.cs +++ b/PrivaPub/Api/Mastodon/Controllers/ProvenanceController.cs @@ -10,7 +10,6 @@ using PrivaPub.Domain.Statistics; using PrivaPub.Federation.Actors; using PrivaPub.Federation.Objects; using PrivaPub.Infrastructure; -using PrivaPub.Infrastructure.Statistics; using PrivaPub.Models.Federation; using PrivaPub.Models.Jobs; using PrivaPub.Models.Post; @@ -29,16 +28,14 @@ namespace PrivaPub.Api.Mastodon.Controllers readonly DbEntities _dbEntities; readonly ISelfLocation _self; readonly ILocalActorService _localActors; - readonly IOptionsMonitor _statistics; readonly IOptionsMonitor _registrations; public ProvenanceController(DbEntities dbEntities, ISelfLocation self, ILocalActorService localActors, - IOptionsMonitor statistics, IOptionsMonitor registrations) + IOptionsMonitor registrations) { _dbEntities = dbEntities; _self = self; _localActors = localActors; - _statistics = statistics; _registrations = registrations; } @@ -83,15 +80,14 @@ namespace PrivaPub.Api.Mastodon.Controllers if (host == selfHost) described.Add(await DescribeSelf(host, token)); else if (remote.FirstOrDefault(i => i.Host == host) is { } instance) - described.Add(Describe(instance, _statistics.CurrentValue.PublicCityMinUsers)); + described.Add(Describe(instance)); } return described; } async Task DescribeSelf(string host, CancellationToken token) { - var (geo, configured) = await _self.Get(token); - var users = await Counted.Users(token); + var geo = await _self.Get(token); return new InstanceDescription { Host = host, @@ -101,13 +97,11 @@ namespace PrivaPub.Api.Mastodon.Controllers Protocols = new() { "activitypub" }, OpenRegistrations = _registrations.CurrentValue.IsOpen, DescribedAt = MastodonJson.Time(DateTime.UtcNow), - Geo = configured - ? Located(new PublicLocation(geo.Country, geo.City, geo.Latitude, geo.Longitude, default, default), geo) - : Located(PublicGeo.Project(new RemoteInstance { Geo = geo, UsersTotal = users }, _statistics.CurrentValue.PublicCityMinUsers), geo) + Geo = Located(PublicGeo.Project(geo), geo) }; } - //the public projection of a server's place (owner decision on server locations, ROADMAP 2026-10-03) + //the public projection of a server's place (PublicGeo) static InstanceLocation Located(PublicLocation location, InstanceGeo geo) => location == default ? default : new InstanceLocation { Precision = location.Cdn != default ? "cdn" : location.Latitude != default ? "city" : "country", @@ -158,11 +152,11 @@ namespace PrivaPub.Api.Mastodon.Controllers RawHash = r.RawHash, Raw = r.Raw == default ? default : JsonNode.Parse(r.Raw) }).ToList() ?? new(), - Instance = instance == default ? default : Describe(instance, _statistics.CurrentValue.PublicCityMinUsers) + Instance = instance == default ? default : Describe(instance) }; } - static InstanceDescription Describe(RemoteInstance instance, int minUsers) => new() + static InstanceDescription Describe(RemoteInstance instance) => new() { Host = instance.Host, Software = instance.Software, @@ -180,7 +174,7 @@ namespace PrivaPub.Api.Mastodon.Controllers LastSuccessAt = instance.LastSuccessAt is { } success ? MastodonJson.Time(success) : default, LastFailureAt = instance.LastFailureAt is { } failure ? MastodonJson.Time(failure) : default }, - Geo = Located(PublicGeo.Project(instance, minUsers), instance.Geo) + Geo = Located(PublicGeo.Project(instance.Geo), instance.Geo) }; public class Provenance @@ -246,7 +240,8 @@ namespace PrivaPub.Api.Mastodon.Controllers public InstanceLocation Geo { get; set; } } - //precision: "city" (coordinates to 0.1°, city and network), "country" (the country only) or "cdn" (the CDN's name only) + //precision: "city" (coordinates to 0.1°, city and network), "country" (located only to a country) or "cdn" (the + //CDN's name only) public class InstanceLocation { public string Precision { get; set; } diff --git a/PrivaPub/Domain/Statistics/PublicGeo.cs b/PrivaPub/Domain/Statistics/PublicGeo.cs index aaa3809..afa22d8 100644 --- a/PrivaPub/Domain/Statistics/PublicGeo.cs +++ b/PrivaPub/Domain/Statistics/PublicGeo.cs @@ -6,18 +6,17 @@ namespace PrivaPub.Domain.Statistics public static class PublicGeo { - //owner decision: a CDN-fronted server shows only its CDN (the address is an edge), a server reporting at least - //minUsers users its city, coordinates and network, any other only its country - public static PublicLocation Project(RemoteInstance instance, int minUsers) + //a server's place as it is shown publicly: where it was located (country, city, coordinates to 0.1°, network), + //except that a CDN-fronted server shows only its CDN, because the address we reached is the CDN's edge + public static PublicLocation Project(InstanceGeo geo) { - var geo = instance?.Geo; if (geo == default) return default; if (geo.Cdn != default) return new PublicLocation(default, default, default, default, default, geo.Cdn); - if (instance.UsersTotal >= minUsers) - return new PublicLocation(geo.Country, geo.City, geo.Latitude, geo.Longitude, geo.AsnOrg, default); - return geo.Country == default ? default : new PublicLocation(geo.Country, default, default, default, default, default); + if (geo.Country == default && geo.Latitude == default) + return default; + return new PublicLocation(geo.Country, geo.City, geo.Latitude, geo.Longitude, geo.AsnOrg, default); } } } diff --git a/PrivaPub/Domain/Statistics/SelfLocation.cs b/PrivaPub/Domain/Statistics/SelfLocation.cs index f20677d..4bc8592 100644 --- a/PrivaPub/Domain/Statistics/SelfLocation.cs +++ b/PrivaPub/Domain/Statistics/SelfLocation.cs @@ -1,4 +1,4 @@ -using Microsoft.Extensions.Options; +using Microsoft.Extensions.Options; using PrivaPub.Infrastructure.Geo; using PrivaPub.Infrastructure.Statistics; @@ -12,8 +12,8 @@ namespace PrivaPub.Domain.Statistics { public interface ISelfLocation { - //where this server is, or default when it cannot tell; Configured when the owner set it (Statistics:Geo:Self) - Task<(InstanceGeo Geo, bool Configured)> Get(CancellationToken token); + //where this server is, or default when it cannot tell; Source is "configured" when the owner set it (Statistics:Geo:Self) + Task Get(CancellationToken token); } // This server's own place, for the instance API (the client's globe draws every server from it). The owner may set it, @@ -40,20 +40,20 @@ namespace PrivaPub.Domain.Statistics _logger = logger; } - public async Task<(InstanceGeo Geo, bool Configured)> Get(CancellationToken token) + public async Task Get(CancellationToken token) { if (_options.CurrentValue.Geo.Self is { Latitude: { } latitude, Longitude: { } longitude } self) - return (new InstanceGeo + return new InstanceGeo { Country = self.Country, City = self.City, Latitude = Math.Round(latitude, 1), Longitude = Math.Round(longitude, 1), Source = "configured" - }, true); + }; if (DateTime.UtcNow - _locatedAt < Lifetime) - return (_located, false); + return _located; await _gate.WaitAsync(token); try { @@ -62,7 +62,7 @@ namespace PrivaPub.Domain.Statistics _located = await Locate(token); _locatedAt = DateTime.UtcNow; } - return (_located, false); + return _located; } finally { diff --git a/PrivaPub/Infrastructure/Statistics/StatisticsOptions.cs b/PrivaPub/Infrastructure/Statistics/StatisticsOptions.cs index f38a909..4db3190 100644 --- a/PrivaPub/Infrastructure/Statistics/StatisticsOptions.cs +++ b/PrivaPub/Infrastructure/Statistics/StatisticsOptions.cs @@ -4,7 +4,6 @@ { public bool Enabled { get; set; } = true; public string GeoDirectory { get; set; } = "/var/lib/privapub/geo"; - public int PublicCityMinUsers { get; set; } = 10; public CrawlerOptions Crawler { get; set; } = new(); public GeoOptions Geo { get; set; } = new(); } diff --git a/PrivaPub/Web/Pages/Stargazing.cshtml b/PrivaPub/Web/Pages/Stargazing.cshtml index 110d761..feb2f24 100644 --- a/PrivaPub/Web/Pages/Stargazing.cshtml +++ b/PrivaPub/Web/Pages/Stargazing.cshtml @@ -1,4 +1,4 @@ -@page "/stargazing" +@page "/stargazing" @model PrivaPub.Web.Pages.StargazingModel @{ ViewData["Title"] = "Stargazing"; @@ -13,8 +13,8 @@ never accounts: what kind of software a server runs, what it exchanges with us, and how reliably. Distinct accounts are only counted, through a key that is destroyed at the end of each day.

A server we exchange activities with is described once a week, from its public NodeInfo and, when it has one, its - Mastodon instance API. Its location comes from the address we reached, looked up in an offline database: only the - country is shown publicly for small servers, and only the CDN for servers behind one.

+ Mastodon instance API. Its location comes from the address we reached, looked up in an offline database, and is shown + to the city; for a server behind a CDN, only the CDN is shown, since the address is the CDN's.

@if (Model.GeoSource is { } geo) {

Locations come from @geo, licensed under diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index 593535e..c043b6f 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -178,7 +178,7 @@ and circles (see Owner decisions). | Local side in statistics | **Only the kind of local actor** (person, group, application), **and only on public and unlisted traffic.** DMs, followers-only and circle traffic are one "private" class, never broken out per server in public. Circles are never named, whether as a kind or as a reason. Fetches of our own documents are counted per day, never per server. | | Reading-driven traffic | **Counted per day, never logged per event:** the media proxy, lookups a client asks for, and the client API per endpoint group (admin only). Client app names are not recorded. | | Describing servers | **Every server we exchange activities with is described weekly**, from its NodeInfo (including the user counts it publishes) and its Mastodon instance API, never its contact account. These requests are unsigned, because they are not ActivityPub documents. Never on read. | -| Server locations | **City and network (ASN) from the offline DB-IP Lite databases** (CC BY 4.0, attributed), downloaded monthly (by the server itself since 2026-10-04). The location comes from the address we connected to; an inbound sender's address is never recorded, and no address is stored. In public: city and network only for servers reporting at least 10 users and not behind a CDN; the country otherwise; only the CDN's name for CDN-fronted servers. The admin sees everything. | +| Server locations | **City and network (ASN) from the offline DB-IP Lite databases** (CC BY 4.0, attributed), downloaded monthly (by the server itself since 2026-10-04). The location comes from the address we connected to; an inbound sender's address is never recorded, and no address is stored. In public: the city, coordinates and network the server was located to; only the CDN's name for CDN-fronted servers. The admin sees everything. *Corrected 2026-10-04: this row used to show only the country for servers reporting fewer than 10 users; the owner never decided that threshold, and it is gone.* | | Crawler | **Off by default** (`Statistics:Crawler:Enabled`); **on in production since 2026-10-04**, see below. When on, it identifies as `PrivaPub-Stargazer/ (+https://privapub.thepra.dev/stargazing)`, where `/stargazing` explains it and how to opt out. It honours robots.txt (an unreachable robots.txt means "keep out") and domain blocks. It visits one server a minute, each at most weekly, and at most 5000 servers. It reads only robots.txt, NodeInfo, the instance API and the peers list, never accounts, posts or directories. Crawled servers stay marked as crawled. | | A remote account deletes itself | **Its posts are kept but hidden everywhere** (`Post.AuthorGone`): from timelines, profiles, search and lookups by id. Its follows and timeline rows go, as before. | | Signed-in smoke check in production | **An undiscoverable persona**, the deploy checks `verify_credentials`, home and notifications with it. *Superseded 2026-10-04: the deploy makes and keeps the persona itself, below.* | @@ -202,7 +202,7 @@ and circles (see Owner decisions). | Question | Decision | |---|---| | Sign-in | **One login.** decePubClient signs in on `/clientapi` and exchanges that JWT for one persona's Mastodon token (RFC 8693 token exchange on `/oauth/token`, `PersonaExchange`), one token per persona it uses. Only the seeded first-party application may exchange; the token names the persona, never the root, like any other. | -| Server locations on the instance API | **Public, as the projection already decided for public server locations** (2026-10-03): city, coordinates to 0.1° and network only for servers reporting at least 10 users and not behind a CDN, the country otherwise, the CDN's name for a CDN-fronted one, with DB-IP's attribution. decePubClient's globe draws posts at their author's server. This server's own place comes from its host's address, or from `Statistics:Geo:Self` when the owner sets it (a server behind a proxy). | +| Server locations on the instance API | **Public, as decided for public server locations** (2026-10-03, corrected): city, coordinates to 0.1° and network for every located server whatever its size, the CDN's name for a CDN-fronted one, with DB-IP's attribution. decePubClient's globe draws posts at their author's server. This server's own place comes from its host's address, or from `Statistics:Geo:Self` when the owner sets it (a server behind a proxy). | ## Libraries (researched; no maintained .NET ActivityPub library exists, so Letterbook and Iceshrimp.NET both wrote their own)