diff --git a/CLAUDE.md b/CLAUDE.md index 2e515d3..a8e2795 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -323,8 +323,16 @@ group www-data and reaches the private mongod; `sudo -u www-data` works too. `/api/privapub/v1/statuses/:id/provenance` and `/api/privapub/v1/instances/:host` (or `?host[]=`, up to 40; this server answers too). - **A server's place leaves only through `PublicGeo.Project`:** the city, coordinates (0.1°) and network it was located - to, whatever its size (there is no user threshold), and only the CDN's name for a CDN-fronted one. This server's own place is its host's address located once a day (`SelfLocation`), or - `Statistics:Geo:Self` when the owner sets it. + to, whatever its size (there is no user threshold), and only the CDN's name for a CDN-fronted one. This server's own + place is its host's address located once a day (`SelfLocation`), or `Statistics:Geo:Self` when the owner sets it. +- **CDNs are found three ways, best first** (`Infrastructure/Geo/CdnCatalog.cs`): the ranges a CDN publishes + (`CdnUpdater` downloads them daily into `CdnRangeSet`, keeping the old set when a download or parse fails), the CDN's + fingerprint in the responses PrivaPub already gets (`EdgeHintsHandler` on the federation client, memory only), and + a network that carries nothing but a CDN. Never list an ASN shared with plain hosting (AWS, Google, Microsoft, + DataPacket). A server's `Geo` records the CDN, its domain and how it was found. +- **Servers through time** (`Domain/Statistics/ServerPlaces.cs`, from the weekly `RemoteInstanceSnapshot`s): + `/api/privapub/v1/instances/:host/history`, a CDN-fronted server's `geo.before_cdn`, and the CDNs grouped by their + own domain (`/api/privapub/v1/cdns`, `/cdns/:domain` with weekly joined/left). - **Remote objects are parsed for every shape in `ObjectShapes`:** `url`/`icon`/`image` as a value, an object or an array; Markdown `content`; missing `mediaType`s inferred; thumbnails from any of their five places; and the typed `Link`, `Video`, `Audio` and `Event` details. A Mastodon API card is built from those, never by fetching the linked page diff --git a/PrivaPub.Tests/Http/InstanceLocationTests.cs b/PrivaPub.Tests/Http/InstanceLocationTests.cs index 6d727d7..4d99b61 100644 --- a/PrivaPub.Tests/Http/InstanceLocationTests.cs +++ b/PrivaPub.Tests/Http/InstanceLocationTests.cs @@ -167,6 +167,86 @@ namespace PrivaPub.Tests.Http Assert.Equal(9.2, geo["longitude"]!.GetValue()); } + static string WeeksAgo(int weeks) => PrivaPub.Federation.Objects.InstanceDescriber.Week(DateTime.UtcNow.AddDays(-7 * weeks)); + + static Task Snapshot(string host, int weeksAgo, string cdn = default, string cdnDomain = default, string city = default, double? latitude = default, + double? longitude = default, string country = default) => + DB.Default.SaveAsync(new RemoteInstanceSnapshot + { + Host = host, Week = WeeksAgo(weeksAgo), TakenAt = DateTime.UtcNow.AddDays(-7 * weeksAgo), Reachable = true, Software = "mastodon", UsersTotal = 3, + Cdn = cdn, CdnDomain = cdnDomain, CdnSource = cdn == default ? default : "ranges", Country = country, City = city, Latitude = latitude, Longitude = longitude + }, Token); + + [Fact] + public async Task A_cdn_fronted_server_names_its_cdn_and_where_it_was_before_it() + { + var host = await Instance(3, new InstanceGeo + { + Country = "US", City = "San Francisco", Latitude = 37.8, Longitude = -122.4, Cdn = "Cloudflare", CdnDomain = "cloudflare.com", CdnSource = "ranges", + Source = "DB-IP Lite 2026-10", LocatedAt = DateTime.UtcNow + }, "fronted"); + await Snapshot(host, 3, city: "Berlin", latitude: 52.5, longitude: 13.4, country: "DE"); + await Snapshot(host, 2, city: "Hamburg", latitude: 53.6, longitude: 10.0, country: "DE"); + await Snapshot(host, 1, cdn: "Cloudflare", cdnDomain: "cloudflare.com", city: "San Francisco", latitude: 37.8, longitude: -122.4, country: "US"); + + var geo = await Geo(host); + + Assert.Equal(("cdn", "Cloudflare", "cloudflare.com", "ranges"), (geo.Text("precision"), geo.Text("cdn"), geo.Text("cdn_domain"), geo.Text("cdn_source"))); + Assert.Null(geo["latitude"]); + Assert.Equal((WeeksAgo(2), "Hamburg", "DE"), (geo["before_cdn"].Text("week"), geo["before_cdn"].Text("city"), geo["before_cdn"].Text("country"))); + Assert.Equal(53.6, geo["before_cdn"]!["latitude"]!.GetValue()); + } + + [Fact] + public async Task A_server_history_is_weekly_newest_first_and_a_cdn_hides_its_place() + { + var host = await Instance(3, Milan(), "historic"); + await Snapshot(host, 2, city: "Milan", latitude: 45.5, longitude: 9.2, country: "IT"); + await Snapshot(host, 1, cdn: "Fastly", cdnDomain: "fastly.com", city: "Frankfurt", latitude: 50.1, longitude: 8.7, country: "DE"); + + var weeks = (await _host.Client().Get($"/api/privapub/v1/instances/{host}/history")).Ok().Body!.AsArray(); + + Assert.Equal(new[] { WeeksAgo(1), WeeksAgo(2) }, weeks.Select(w => w.Text("week"))); + Assert.Equal(("cdn", "fastly.com"), (weeks[0]!["geo"].Text("precision"), weeks[0]!["geo"].Text("cdn_domain"))); + Assert.Null(weeks[0]!["geo"]!["city"]); + Assert.Equal(("city", "Milan"), (weeks[1]!["geo"].Text("precision"), weeks[1]!["geo"].Text("city"))); + Assert.Equal(3, weeks[1]!["users_total"]!.GetValue()); + Assert.Single((await _host.Client().Get($"/api/privapub/v1/instances/{host}/history?weeks=1")).Ok().Body!.AsArray()); + Assert.Equal(HttpStatusCode.NotFound, (await _host.Client().Get($"/api/privapub/v1/instances/unknown{Guid.NewGuid():N}.example/history")).Status); + } + + [Fact] + public async Task Servers_are_grouped_by_their_cdn_week_by_week() + { + var cdn = $"Edge{Guid.NewGuid():N}"[..20]; + var key = cdn.ToLowerInvariant(); + var stayed = await Instance(3, new InstanceGeo { Country = "FR", Cdn = cdn, CdnSource = "headers" }, "stayed"); + var moved = await Instance(3, Milan(), "moved"); + await Snapshot(moved, 3, cdn: cdn); + await Snapshot(moved, 2, cdn: cdn); + await Snapshot(stayed, 2, cdn: cdn); + await Snapshot(moved, 1, city: "Milan", latitude: 45.5, longitude: 9.2, country: "IT"); + await Snapshot(stayed, 1, cdn: cdn); + + var list = (await _host.Client().Get("/api/privapub/v1/cdns")).Ok().Body!.AsArray(); + var group = list.Single(g => g.Text("key") == key); + Assert.Equal((cdn, 1), (group.Text("name"), group["servers"]!.GetValue())); + var cloudflare = list.Single(g => g.Text("key") == "cloudflare.com"); + Assert.Equal(new[] { "ranges", "headers", "asn" }, cloudflare["detected_by"]!.AsArray().Select(v => v!.GetValue())); + + var detail = (await _host.Client().Get($"/api/privapub/v1/cdns/{key}")).Ok().Body; + Assert.Equal(new[] { stayed }, detail["hosts"]!.AsArray().Select(h => h!.GetValue())); + var weeks = detail["weeks"]!.AsArray(); + Assert.Equal(new[] { WeeksAgo(1), WeeksAgo(2), WeeksAgo(3) }, weeks.Select(w => w.Text("week"))); + Assert.Equal(new[] { 1, 2, 1 }, weeks.Select(w => w!["servers"]!.GetValue())); + Assert.Equal(new[] { moved }, weeks[0]!["left"]!.AsArray().Select(h => h!.GetValue())); + Assert.Equal(new[] { stayed }, weeks[1]!["joined"]!.AsArray().Select(h => h!.GetValue())); + Assert.Equal(new[] { moved }, weeks[2]!["joined"]!.AsArray().Select(h => h!.GetValue())); + + Assert.Equal("Cloudflare", (await _host.Client().Get("/api/privapub/v1/cdns/cloudflare.com")).Ok().Body.Text("name")); + Assert.Equal(HttpStatusCode.NotFound, (await _host.Client().Get($"/api/privapub/v1/cdns/none{Guid.NewGuid():N}")).Status); + } + [Fact] public async Task Junk_hosts_are_not_an_error() { diff --git a/PrivaPub.Tests/Infrastructure/CdnCatalogTests.cs b/PrivaPub.Tests/Infrastructure/CdnCatalogTests.cs new file mode 100644 index 0000000..48a29df --- /dev/null +++ b/PrivaPub.Tests/Infrastructure/CdnCatalogTests.cs @@ -0,0 +1,111 @@ +using PrivaPub.Infrastructure.Geo; +using PrivaPub.Infrastructure.Http; +using PrivaPub.Models.Jobs; + +using System.Net; + +namespace PrivaPub.Tests.Infrastructure +{ + public class CdnCatalogTests + { + static HttpResponseMessage Response(params (string Name, string Value)[] headers) + { + var response = new HttpResponseMessage(HttpStatusCode.OK) { Content = new StringContent("{}") }; + foreach (var (name, value) in headers) + response.Headers.TryAddWithoutValidation(name, value); + return response; + } + + [Theory] + [InlineData("cf-ray", "8c1f0e2b4d5a1234-FRA", "cloudflare.com")] + [InlineData("server", "cloudflare", "cloudflare.com")] + [InlineData("x-amz-cf-id", "abc==", "cloudfront.net")] + [InlineData("via", "1.1 5a1b.cloudfront.net (CloudFront)", "cloudfront.net")] + [InlineData("x-served-by", "cache-fra-eddf8230045-FRA", "fastly.com")] + [InlineData("server", "BunnyCDN-DE1-1078", "bunny.net")] + [InlineData("akamai-grn", "0.1234", "akamai.com")] + [InlineData("x-azure-ref", "20261004T101010Z-abc", "azure.microsoft.com")] + [InlineData("x-vercel-id", "fra1::abc", "vercel.com")] + [InlineData("x-iinfo", "1-2-3", "imperva.com")] + public void A_cdn_is_recognised_by_its_fingerprint(string header, string value, string domain) + { + Assert.Equal(domain, CdnCatalog.FromResponse(Response((header, value)))?.Domain); + } + + [Fact] + public void A_plain_server_has_no_cdn_and_an_unknown_cdn_names_itself() + { + Assert.Null(CdnCatalog.FromResponse(Response(("server", "nginx"), ("via", "1.1 varnish")))); + var named = CdnCatalog.FromResponse(Response(("x-cdn", "EdgeOne"))); + Assert.Equal(("EdgeOne", null, "edgeone"), (named.Name, named.Domain, named.Key)); + Assert.Null(CdnCatalog.FromResponse(Response(("x-cdn", "