CDNs found by themselves, and servers followed through time
PrivaPub now finds CDNs three ways, best first: the address ranges the CDNs publish (Cloudflare, Fastly, Amazon CloudFront, Bunny, Gcore, Imperva), downloaded daily by CdnUpdater and kept in CdnRangeSet; the CDN's fingerprint in the responses it already gets from a server (EdgeHintsHandler on the federation client); and the networks that carry only a CDN. The fixed ASN list is gone; ASNs shared with plain hosting (AWS, DataPacket) no longer hide a server. A server's Geo records the CDN, its domain and how it was found, and weekly snapshots now keep the city and coordinates too. Servers through time (ServerPlaces): /instances/:host/history lists a server's weekly snapshots, a CDN-fronted server's geo names the CDN's domain and where the server was before it (before_cdn), and /api/privapub/v1/cdns and /cdns/:domain group servers by CDN with week by week who joined and who left. Owner decisions recorded in ROADMAP. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01LsXgEaXee4GCU1hwYgPJXw
This commit is contained in:
1 parent
5d7edc4a4c
commit
436f7da464
26 files changed
+1159
-62
No files matched your search
@@ -167,6 +167,86 @@ namespace PrivaPub.Tests.Http
|
||||
Assert.Equal(9.2, geo["longitude"]!.GetValue<double>());
|
||||
}
|
||||
|
||||
static string WeeksAgo(int weeks) => PrivaPub.Federation.Objects.InstanceDescriber.Week(DateTime.UtcNow.AddDays(-7 * weeks));
|
||||
|
||||
static Task Snapshot(string host, int weeksAgo, string cdn = default, string cdnDomain = default, string city = default, double? latitude = default,
|
||||
double? longitude = default, string country = default) =>
|
||||
DB.Default.SaveAsync(new RemoteInstanceSnapshot
|
||||
{
|
||||
Host = host, Week = WeeksAgo(weeksAgo), TakenAt = DateTime.UtcNow.AddDays(-7 * weeksAgo), Reachable = true, Software = "mastodon", UsersTotal = 3,
|
||||
Cdn = cdn, CdnDomain = cdnDomain, CdnSource = cdn == default ? default : "ranges", Country = country, City = city, Latitude = latitude, Longitude = longitude
|
||||
}, Token);
|
||||
|
||||
[Fact]
|
||||
public async Task A_cdn_fronted_server_names_its_cdn_and_where_it_was_before_it()
|
||||
{
|
||||
var host = await Instance(3, new InstanceGeo
|
||||
{
|
||||
Country = "US", City = "San Francisco", Latitude = 37.8, Longitude = -122.4, Cdn = "Cloudflare", CdnDomain = "cloudflare.com", CdnSource = "ranges",
|
||||
Source = "DB-IP Lite 2026-10", LocatedAt = DateTime.UtcNow
|
||||
}, "fronted");
|
||||
await Snapshot(host, 3, city: "Berlin", latitude: 52.5, longitude: 13.4, country: "DE");
|
||||
await Snapshot(host, 2, city: "Hamburg", latitude: 53.6, longitude: 10.0, country: "DE");
|
||||
await Snapshot(host, 1, cdn: "Cloudflare", cdnDomain: "cloudflare.com", city: "San Francisco", latitude: 37.8, longitude: -122.4, country: "US");
|
||||
|
||||
var geo = await Geo(host);
|
||||
|
||||
Assert.Equal(("cdn", "Cloudflare", "cloudflare.com", "ranges"), (geo.Text("precision"), geo.Text("cdn"), geo.Text("cdn_domain"), geo.Text("cdn_source")));
|
||||
Assert.Null(geo["latitude"]);
|
||||
Assert.Equal((WeeksAgo(2), "Hamburg", "DE"), (geo["before_cdn"].Text("week"), geo["before_cdn"].Text("city"), geo["before_cdn"].Text("country")));
|
||||
Assert.Equal(53.6, geo["before_cdn"]!["latitude"]!.GetValue<double>());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_server_history_is_weekly_newest_first_and_a_cdn_hides_its_place()
|
||||
{
|
||||
var host = await Instance(3, Milan(), "historic");
|
||||
await Snapshot(host, 2, city: "Milan", latitude: 45.5, longitude: 9.2, country: "IT");
|
||||
await Snapshot(host, 1, cdn: "Fastly", cdnDomain: "fastly.com", city: "Frankfurt", latitude: 50.1, longitude: 8.7, country: "DE");
|
||||
|
||||
var weeks = (await _host.Client().Get($"/api/privapub/v1/instances/{host}/history")).Ok().Body!.AsArray();
|
||||
|
||||
Assert.Equal(new[] { WeeksAgo(1), WeeksAgo(2) }, weeks.Select(w => w.Text("week")));
|
||||
Assert.Equal(("cdn", "fastly.com"), (weeks[0]!["geo"].Text("precision"), weeks[0]!["geo"].Text("cdn_domain")));
|
||||
Assert.Null(weeks[0]!["geo"]!["city"]);
|
||||
Assert.Equal(("city", "Milan"), (weeks[1]!["geo"].Text("precision"), weeks[1]!["geo"].Text("city")));
|
||||
Assert.Equal(3, weeks[1]!["users_total"]!.GetValue<long>());
|
||||
Assert.Single((await _host.Client().Get($"/api/privapub/v1/instances/{host}/history?weeks=1")).Ok().Body!.AsArray());
|
||||
Assert.Equal(HttpStatusCode.NotFound, (await _host.Client().Get($"/api/privapub/v1/instances/unknown{Guid.NewGuid():N}.example/history")).Status);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Servers_are_grouped_by_their_cdn_week_by_week()
|
||||
{
|
||||
var cdn = $"Edge{Guid.NewGuid():N}"[..20];
|
||||
var key = cdn.ToLowerInvariant();
|
||||
var stayed = await Instance(3, new InstanceGeo { Country = "FR", Cdn = cdn, CdnSource = "headers" }, "stayed");
|
||||
var moved = await Instance(3, Milan(), "moved");
|
||||
await Snapshot(moved, 3, cdn: cdn);
|
||||
await Snapshot(moved, 2, cdn: cdn);
|
||||
await Snapshot(stayed, 2, cdn: cdn);
|
||||
await Snapshot(moved, 1, city: "Milan", latitude: 45.5, longitude: 9.2, country: "IT");
|
||||
await Snapshot(stayed, 1, cdn: cdn);
|
||||
|
||||
var list = (await _host.Client().Get("/api/privapub/v1/cdns")).Ok().Body!.AsArray();
|
||||
var group = list.Single(g => g.Text("key") == key);
|
||||
Assert.Equal((cdn, 1), (group.Text("name"), group["servers"]!.GetValue<int>()));
|
||||
var cloudflare = list.Single(g => g.Text("key") == "cloudflare.com");
|
||||
Assert.Equal(new[] { "ranges", "headers", "asn" }, cloudflare["detected_by"]!.AsArray().Select(v => v!.GetValue<string>()));
|
||||
|
||||
var detail = (await _host.Client().Get($"/api/privapub/v1/cdns/{key}")).Ok().Body;
|
||||
Assert.Equal(new[] { stayed }, detail["hosts"]!.AsArray().Select(h => h!.GetValue<string>()));
|
||||
var weeks = detail["weeks"]!.AsArray();
|
||||
Assert.Equal(new[] { WeeksAgo(1), WeeksAgo(2), WeeksAgo(3) }, weeks.Select(w => w.Text("week")));
|
||||
Assert.Equal(new[] { 1, 2, 1 }, weeks.Select(w => w!["servers"]!.GetValue<int>()));
|
||||
Assert.Equal(new[] { moved }, weeks[0]!["left"]!.AsArray().Select(h => h!.GetValue<string>()));
|
||||
Assert.Equal(new[] { stayed }, weeks[1]!["joined"]!.AsArray().Select(h => h!.GetValue<string>()));
|
||||
Assert.Equal(new[] { moved }, weeks[2]!["joined"]!.AsArray().Select(h => h!.GetValue<string>()));
|
||||
|
||||
Assert.Equal("Cloudflare", (await _host.Client().Get("/api/privapub/v1/cdns/cloudflare.com")).Ok().Body.Text("name"));
|
||||
Assert.Equal(HttpStatusCode.NotFound, (await _host.Client().Get($"/api/privapub/v1/cdns/none{Guid.NewGuid():N}")).Status);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Junk_hosts_are_not_an_error()
|
||||
{
|
||||
|
||||
Reference in new issue
Block a user