CDNs found by themselves, and servers followed through time
Build / Build (push) Successful in 5m11s
Deploy / privapub.thepra.dev (push) Successful in 5m48s

PrivaPub now finds CDNs three ways, best first: the address ranges the
CDNs publish (Cloudflare, Fastly, Amazon CloudFront, Bunny, Gcore,
Imperva), downloaded daily by CdnUpdater and kept in CdnRangeSet; the
CDN's fingerprint in the responses it already gets from a server
(EdgeHintsHandler on the federation client); and the networks that carry
only a CDN. The fixed ASN list is gone; ASNs shared with plain hosting
(AWS, DataPacket) no longer hide a server. A server's Geo records the
CDN, its domain and how it was found, and weekly snapshots now keep the
city and coordinates too.

Servers through time (ServerPlaces): /instances/:host/history lists a
server's weekly snapshots, a CDN-fronted server's geo names the CDN's
domain and where the server was before it (before_cdn), and
/api/privapub/v1/cdns and /cdns/:domain group servers by CDN with week
by week who joined and who left. Owner decisions recorded in ROADMAP.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01LsXgEaXee4GCU1hwYgPJXw
This commit is contained in:
thepraandClaude Opus 5.5 committed 2026-10-04 11:33:39 +02:00
1 parent 5d7edc4a4c
commit 436f7da464
26 files changed
+1159 -62

No files matched your search

@@ -167,6 +167,86 @@ namespace PrivaPub.Tests.Http
Assert.Equal(9.2, geo["longitude"]!.GetValue<double>());
}
static string WeeksAgo(int weeks) => PrivaPub.Federation.Objects.InstanceDescriber.Week(DateTime.UtcNow.AddDays(-7 * weeks));
static Task Snapshot(string host, int weeksAgo, string cdn = default, string cdnDomain = default, string city = default, double? latitude = default,
double? longitude = default, string country = default) =>
DB.Default.SaveAsync(new RemoteInstanceSnapshot
{
Host = host, Week = WeeksAgo(weeksAgo), TakenAt = DateTime.UtcNow.AddDays(-7 * weeksAgo), Reachable = true, Software = "mastodon", UsersTotal = 3,
Cdn = cdn, CdnDomain = cdnDomain, CdnSource = cdn == default ? default : "ranges", Country = country, City = city, Latitude = latitude, Longitude = longitude
}, Token);
[Fact]
public async Task A_cdn_fronted_server_names_its_cdn_and_where_it_was_before_it()
{
var host = await Instance(3, new InstanceGeo
{
Country = "US", City = "San Francisco", Latitude = 37.8, Longitude = -122.4, Cdn = "Cloudflare", CdnDomain = "cloudflare.com", CdnSource = "ranges",
Source = "DB-IP Lite 2026-10", LocatedAt = DateTime.UtcNow
}, "fronted");
await Snapshot(host, 3, city: "Berlin", latitude: 52.5, longitude: 13.4, country: "DE");
await Snapshot(host, 2, city: "Hamburg", latitude: 53.6, longitude: 10.0, country: "DE");
await Snapshot(host, 1, cdn: "Cloudflare", cdnDomain: "cloudflare.com", city: "San Francisco", latitude: 37.8, longitude: -122.4, country: "US");
var geo = await Geo(host);
Assert.Equal(("cdn", "Cloudflare", "cloudflare.com", "ranges"), (geo.Text("precision"), geo.Text("cdn"), geo.Text("cdn_domain"), geo.Text("cdn_source")));
Assert.Null(geo["latitude"]);
Assert.Equal((WeeksAgo(2), "Hamburg", "DE"), (geo["before_cdn"].Text("week"), geo["before_cdn"].Text("city"), geo["before_cdn"].Text("country")));
Assert.Equal(53.6, geo["before_cdn"]!["latitude"]!.GetValue<double>());
}
[Fact]
public async Task A_server_history_is_weekly_newest_first_and_a_cdn_hides_its_place()
{
var host = await Instance(3, Milan(), "historic");
await Snapshot(host, 2, city: "Milan", latitude: 45.5, longitude: 9.2, country: "IT");
await Snapshot(host, 1, cdn: "Fastly", cdnDomain: "fastly.com", city: "Frankfurt", latitude: 50.1, longitude: 8.7, country: "DE");
var weeks = (await _host.Client().Get($"/api/privapub/v1/instances/{host}/history")).Ok().Body!.AsArray();
Assert.Equal(new[] { WeeksAgo(1), WeeksAgo(2) }, weeks.Select(w => w.Text("week")));
Assert.Equal(("cdn", "fastly.com"), (weeks[0]!["geo"].Text("precision"), weeks[0]!["geo"].Text("cdn_domain")));
Assert.Null(weeks[0]!["geo"]!["city"]);
Assert.Equal(("city", "Milan"), (weeks[1]!["geo"].Text("precision"), weeks[1]!["geo"].Text("city")));
Assert.Equal(3, weeks[1]!["users_total"]!.GetValue<long>());
Assert.Single((await _host.Client().Get($"/api/privapub/v1/instances/{host}/history?weeks=1")).Ok().Body!.AsArray());
Assert.Equal(HttpStatusCode.NotFound, (await _host.Client().Get($"/api/privapub/v1/instances/unknown{Guid.NewGuid():N}.example/history")).Status);
}
[Fact]
public async Task Servers_are_grouped_by_their_cdn_week_by_week()
{
var cdn = $"Edge{Guid.NewGuid():N}"[..20];
var key = cdn.ToLowerInvariant();
var stayed = await Instance(3, new InstanceGeo { Country = "FR", Cdn = cdn, CdnSource = "headers" }, "stayed");
var moved = await Instance(3, Milan(), "moved");
await Snapshot(moved, 3, cdn: cdn);
await Snapshot(moved, 2, cdn: cdn);
await Snapshot(stayed, 2, cdn: cdn);
await Snapshot(moved, 1, city: "Milan", latitude: 45.5, longitude: 9.2, country: "IT");
await Snapshot(stayed, 1, cdn: cdn);
var list = (await _host.Client().Get("/api/privapub/v1/cdns")).Ok().Body!.AsArray();
var group = list.Single(g => g.Text("key") == key);
Assert.Equal((cdn, 1), (group.Text("name"), group["servers"]!.GetValue<int>()));
var cloudflare = list.Single(g => g.Text("key") == "cloudflare.com");
Assert.Equal(new[] { "ranges", "headers", "asn" }, cloudflare["detected_by"]!.AsArray().Select(v => v!.GetValue<string>()));
var detail = (await _host.Client().Get($"/api/privapub/v1/cdns/{key}")).Ok().Body;
Assert.Equal(new[] { stayed }, detail["hosts"]!.AsArray().Select(h => h!.GetValue<string>()));
var weeks = detail["weeks"]!.AsArray();
Assert.Equal(new[] { WeeksAgo(1), WeeksAgo(2), WeeksAgo(3) }, weeks.Select(w => w.Text("week")));
Assert.Equal(new[] { 1, 2, 1 }, weeks.Select(w => w!["servers"]!.GetValue<int>()));
Assert.Equal(new[] { moved }, weeks[0]!["left"]!.AsArray().Select(h => h!.GetValue<string>()));
Assert.Equal(new[] { stayed }, weeks[1]!["joined"]!.AsArray().Select(h => h!.GetValue<string>()));
Assert.Equal(new[] { moved }, weeks[2]!["joined"]!.AsArray().Select(h => h!.GetValue<string>()));
Assert.Equal("Cloudflare", (await _host.Client().Get("/api/privapub/v1/cdns/cloudflare.com")).Ok().Body.Text("name"));
Assert.Equal(HttpStatusCode.NotFound, (await _host.Client().Get($"/api/privapub/v1/cdns/none{Guid.NewGuid():N}")).Status);
}
[Fact]
public async Task Junk_hosts_are_not_an_error()
{
@@ -0,0 +1,111 @@
using PrivaPub.Infrastructure.Geo;
using PrivaPub.Infrastructure.Http;
using PrivaPub.Models.Jobs;
using System.Net;
namespace PrivaPub.Tests.Infrastructure
{
public class CdnCatalogTests
{
static HttpResponseMessage Response(params (string Name, string Value)[] headers)
{
var response = new HttpResponseMessage(HttpStatusCode.OK) { Content = new StringContent("{}") };
foreach (var (name, value) in headers)
response.Headers.TryAddWithoutValidation(name, value);
return response;
}
[Theory]
[InlineData("cf-ray", "8c1f0e2b4d5a1234-FRA", "cloudflare.com")]
[InlineData("server", "cloudflare", "cloudflare.com")]
[InlineData("x-amz-cf-id", "abc==", "cloudfront.net")]
[InlineData("via", "1.1 5a1b.cloudfront.net (CloudFront)", "cloudfront.net")]
[InlineData("x-served-by", "cache-fra-eddf8230045-FRA", "fastly.com")]
[InlineData("server", "BunnyCDN-DE1-1078", "bunny.net")]
[InlineData("akamai-grn", "0.1234", "akamai.com")]
[InlineData("x-azure-ref", "20261004T101010Z-abc", "azure.microsoft.com")]
[InlineData("x-vercel-id", "fra1::abc", "vercel.com")]
[InlineData("x-iinfo", "1-2-3", "imperva.com")]
public void A_cdn_is_recognised_by_its_fingerprint(string header, string value, string domain)
{
Assert.Equal(domain, CdnCatalog.FromResponse(Response((header, value)))?.Domain);
}
[Fact]
public void A_plain_server_has_no_cdn_and_an_unknown_cdn_names_itself()
{
Assert.Null(CdnCatalog.FromResponse(Response(("server", "nginx"), ("via", "1.1 varnish"))));
var named = CdnCatalog.FromResponse(Response(("x-cdn", "EdgeOne")));
Assert.Equal(("EdgeOne", null, "edgeone"), (named.Name, named.Domain, named.Key));
Assert.Null(CdnCatalog.FromResponse(Response(("x-cdn", "<script>"))));
}
[Fact]
public void Only_networks_that_carry_nothing_but_a_cdn_name_it()
{
Assert.Equal("Cloudflare", CdnCatalog.OfAsn(13335)?.Name);
Assert.Equal("Akamai", CdnCatalog.OfAsn(20940)?.Name);
Assert.Null(CdnCatalog.OfAsn(16509));//AWS: CloudFront only by its ranges or its headers
Assert.Null(CdnCatalog.OfAsn(60068));//DataPacket also rents servers
}
[Fact]
public void Every_published_list_format_is_read()
{
Assert.Equal(new[] { "173.245.48.0/20", "2400:cb00::/32" },
CdnCatalog.Parse(CdnListFormat.Lines, "173.245.48.0/20\n# comment\n\n2400:cb00::/32\n").Select(n => n.ToString()));
Assert.Equal(new[] { "89.187.188.227/32", "2400:52e0:1500::714:1/128" },
CdnCatalog.Parse(CdnListFormat.StringArray, """["89.187.188.227","2400:52e0:1500::714:1"]""").Select(n => n.ToString()));
Assert.Equal(new[] { "23.235.32.0/20", "2a04:4e40::/32" },
CdnCatalog.Parse(CdnListFormat.AddressesJson, """{"addresses":["23.235.32.0/20"],"ipv6_addresses":["2a04:4e40::/32"]}""").Select(n => n.ToString()));
Assert.Equal(new[] { "92.223.124.12/32", "2a03:90c0::/32" },
CdnCatalog.Parse(CdnListFormat.AddressesJson, """{"addresses":["92.223.124.12/32"],"addresses_v6":["2a03:90c0::/32"]}""").Select(n => n.ToString()));
Assert.Equal(new[] { "199.83.128.0/21", "2a02:e980::/29" },
CdnCatalog.Parse(CdnListFormat.ImpervaJson, """{"ipRanges":["199.83.128.0/21"],"ipv6Ranges":["2a02:e980::/29"],"res":0}""").Select(n => n.ToString()));
Assert.Equal(new[] { "13.32.0.0/15", "2600:9000::/28" },
CdnCatalog.Parse(CdnListFormat.AwsCloudFront, """
{"prefixes":[{"ip_prefix":"3.4.12.4/32","service":"AMAZON"},{"ip_prefix":"13.32.0.0/15","service":"CLOUDFRONT"}],
"ipv6_prefixes":[{"ipv6_prefix":"2600:9000::/28","service":"CLOUDFRONT"},{"ipv6_prefix":"2406:daba::/40","service":"EC2"}]}
""").Select(n => n.ToString()));
Assert.Throws<FormatException>(() => CdnCatalog.Parse(CdnListFormat.Lines, "173.245.48.0/20\nnot an address"));
}
[Fact]
public void The_published_ranges_name_the_cdn_of_an_address()
{
var ranges = new CdnRanges();
ranges.Replace(new CdnRangeSet { Key = "cloudflare.com", Name = "Cloudflare", Ranges = new() { "104.16.0.0/13", "2606:4700::/32" }, FetchedAt = DateTime.UtcNow });
Assert.Equal("Cloudflare", ranges.Of(IPAddress.Parse("104.18.2.3"))?.Name);
Assert.Equal("Cloudflare", ranges.Of(IPAddress.Parse("::ffff:104.18.2.3"))?.Name);
Assert.Equal("Cloudflare", ranges.Of(IPAddress.Parse("2606:4700::1"))?.Name);
Assert.Null(ranges.Of(IPAddress.Parse("203.0.113.7")));
Assert.Equal(2, ranges.Status["cloudflare.com"].Ranges);
}
[Fact]
public async Task Responses_leave_a_cdn_hint_for_their_host_until_one_comes_without()
{
var hints = new EdgeHints();
var answers = new Queue<HttpResponseMessage>(new[] { Response(("cf-ray", "1-FRA")), Response(("server", "nginx")) });
using var client = new HttpClient(new EdgeHintsHandler(hints) { InnerHandler = new Answering(answers) });
await client.GetAsync("https://social.example/.well-known/nodeinfo", TestContext.Current.CancellationToken);
Assert.Equal("Cloudflare", hints.Of("social.example")?.Name);
await client.GetAsync("https://social.example/nodeinfo/2.1", TestContext.Current.CancellationToken);
Assert.Null(hints.Of("social.example"));
}
sealed class Answering : HttpMessageHandler
{
readonly Queue<HttpResponseMessage> _answers;
public Answering(Queue<HttpResponseMessage> answers) => _answers = answers;
protected override Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) =>
Task.FromResult(_answers.Dequeue());
}
}
}
@@ -0,0 +1,105 @@
using Microsoft.Extensions.Logging.Abstractions;
using MongoDB.Entities;
using PrivaPub.Infrastructure.Geo;
using PrivaPub.Infrastructure.Statistics;
using PrivaPub.Models.Jobs;
using PrivaPub.Tests.Support;
using System.Net;
namespace PrivaPub.Tests.Infrastructure
{
[Trait("Category", "Integration")]
public sealed class CdnUpdaterTests : IAsyncLifetime
{
Peer _peer;
CdnRanges _ranges;
CdnUpdater _updater;
string _key;
public async ValueTask InitializeAsync()
{
Assert.SkipUnless(MongoFixture.Enabled, MongoFixture.Skip);
_peer = await Peer.Start();
_ranges = new CdnRanges();
_updater = new CdnUpdater(new Clients(), new StaticOptions<StatisticsOptions>(new StatisticsOptions()), _ranges, NullLogger<CdnUpdater>.Instance);
_key = $"edge{Guid.NewGuid():N}.example";
}
public async ValueTask DisposeAsync()
{
if (_peer != default)
await _peer.DisposeAsync();
}
sealed class Clients : IHttpClientFactory
{
public HttpClient CreateClient(string name) => new();
}
CdnProvider Provider() => new("Test Edge", _key, Array.Empty<int>(), new[]
{
new CdnList($"{_peer.A}/edge/v4", CdnListFormat.Lines),
new CdnList($"{_peer.A}/edge/v6", CdnListFormat.StringArray)
}, _ => false);
static CancellationToken Token => TestContext.Current.CancellationToken;
[Fact]
public async Task The_published_ranges_are_downloaded_stored_and_used()
{
_peer.ServeText("/edge/v4", "198.51.100.0/24\n", "text/plain");
_peer.ServeText("/edge/v6", """["2001:db8::/32","192.0.2.9"]""", "application/json");
Assert.Equal(1, await _updater.Update(new[] { Provider() }, Token));
var stored = await DB.Default.Find<CdnRangeSet>().Match(s => s.Key == _key).ExecuteSingleAsync(Token);
Assert.Equal(new[] { "198.51.100.0/24", "2001:db8::/32", "192.0.2.9/32" }, stored.Ranges);
Assert.Equal("Test Edge", _ranges.Of(IPAddress.Parse("198.51.100.20"))?.Name);
Assert.Equal("Test Edge", _ranges.Of(IPAddress.Parse("192.0.2.9"))?.Name);
var restarted = new CdnRanges();
await restarted.Load(Token);
Assert.Equal("Test Edge", restarted.Of(IPAddress.Parse("2001:db8::7"))?.Name);
}
[Theory]
[InlineData("not a network")]
[InlineData("")]
public async Task A_broken_or_empty_list_keeps_the_stored_ranges(string broken)
{
_peer.ServeText("/edge/v4", "198.51.100.0/24", "text/plain");
_peer.ServeText("/edge/v6", "[]", "application/json");
Assert.Equal(1, await _updater.Update(new[] { Provider() }, Token));
_peer.ServeText("/edge/v4", broken, "text/plain");
Assert.Equal(0, await _updater.Update(new[] { Provider() }, Token));
Assert.Equal(new[] { "198.51.100.0/24" }, (await DB.Default.Find<CdnRangeSet>().Match(s => s.Key == _key).ExecuteSingleAsync(Token)).Ranges);
Assert.Equal("Test Edge", _ranges.Of(IPAddress.Parse("198.51.100.1"))?.Name);
}
// The real lists, from the CDNs themselves: set PRIVAPUB_TEST_CDN_LIVE=1 to check that every format still parses.
[Fact]
public async Task Every_cdns_real_list_still_parses()
{
Assert.SkipUnless(Environment.GetEnvironmentVariable("PRIVAPUB_TEST_CDN_LIVE") == "1", "set PRIVAPUB_TEST_CDN_LIVE=1 to download the real lists");
var listed = CdnCatalog.All.Where(p => p.Lists.Length > 0).ToList();
Assert.Equal(listed.Count, await _updater.Update(listed, Token));
Assert.Equal("Cloudflare", _ranges.Of(IPAddress.Parse("104.16.0.1"))?.Name);
foreach (var provider in listed)
Assert.True(_ranges.Status[provider.Key].Ranges > 0, provider.Name);
}
[Fact]
public async Task An_unreachable_list_changes_nothing()
{
Assert.Equal(0, await _updater.Update(new[] { Provider() }, Token));
Assert.False(await DB.Default.Find<CdnRangeSet>().Match(s => s.Key == _key).ExecuteAnyAsync(Token));
}
}
}
@@ -1,4 +1,4 @@
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Logging.Abstractions;
using PrivaPub.Infrastructure.Geo;
using PrivaPub.Infrastructure.Http;
@@ -45,7 +45,7 @@ namespace PrivaPub.Tests.Infrastructure
Assert.NotNull(fix);
Assert.Equal(2, fix.Country.Length);
Assert.Equal(13335, fix.Asn);
Assert.Equal("Cloudflare", CdnNetworks.Of(fix.Asn));
Assert.Equal("Cloudflare", CdnCatalog.OfAsn(fix.Asn)?.Name);
Assert.NotNull(fix.AsnOrg);
Assert.NotNull(fix.Latitude);
Assert.Equal(Math.Round(fix.Latitude.Value, 1), fix.Latitude);
@@ -61,9 +61,9 @@ namespace PrivaPub.Tests.Infrastructure
Assert.Equal(IPAddress.Parse("203.0.113.7"), connected.Of("social.example"));
Assert.Null(connected.Of("other.example"));
Assert.Equal("Fastly", CdnNetworks.Of(54113));
Assert.Null(CdnNetworks.Of(64496));
Assert.Null(CdnNetworks.Of(default));
Assert.Equal("Fastly", CdnCatalog.OfAsn(54113)?.Name);
Assert.Null(CdnCatalog.OfAsn(64496));
Assert.Null(CdnCatalog.OfAsn(default));
}
}
}
+27 -1
View File
@@ -112,7 +112,8 @@ namespace PrivaPub.Tests.Statistics
{
readonly string _peer;
public PeerDescriber(IFederationHttp http, IGeoLocator geo, IConnectedAddresses addresses, string peer) : base(http, geo, addresses) => _peer = peer;
public PeerDescriber(IFederationHttp http, IGeoLocator geo, IConnectedAddresses addresses, string peer, ICdnRanges ranges = default,
IEdgeHints edges = default) : base(http, geo, addresses, ranges, edges) => _peer = peer;
protected override string Address(string url) => _peer + new Uri(url).PathAndQuery;
}
@@ -155,6 +156,31 @@ namespace PrivaPub.Tests.Statistics
Assert.Equal(InstanceDescriber.Week(DateTime.UtcNow), snapshot.Week);
}
[Fact]
public async Task A_cdn_is_found_by_its_published_ranges_first_then_by_its_headers()
{
var token = TestContext.Current.CancellationToken;
var listed = $"listed{Guid.NewGuid():N}.example";
var hinted = $"hinted{Guid.NewGuid():N}.example";
ServeServer("/" + listed, "mastodon");
ServeServer("/" + hinted, "mastodon");
var addresses = new ConnectedAddresses();
var ranges = new CdnRanges();
ranges.Replace(new CdnRangeSet { Key = "cloudflare.com", Name = "Cloudflare", Ranges = new() { "127.0.0.0/8" }, FetchedAt = DateTime.UtcNow });
var edges = new EdgeHints();
edges.Remember("127.0.0.1", CdnCatalog.ByKey("fastly.com"));
await new PeerDescriber(Peer.Http(connected: addresses), new FixedGeo(), addresses, _peer.A + "/" + listed, ranges, edges).Describe(listed, crawled: false, default, token);
await new PeerDescriber(Peer.Http(connected: addresses), new FixedGeo(), addresses, _peer.A + "/" + hinted, new CdnRanges(), edges).Describe(hinted, crawled: false, default, token);
var listedOne = await DB.Default.Find<RemoteInstance>().Match(i => i.Host == listed).ExecuteSingleAsync(token);
Assert.Equal(("Cloudflare", "cloudflare.com", "ranges"), (listedOne.Geo.Cdn, listedOne.Geo.CdnDomain, listedOne.Geo.CdnSource));
var hintedOne = await DB.Default.Find<RemoteInstance>().Match(i => i.Host == hinted).ExecuteSingleAsync(token);
Assert.Equal(("Fastly", "fastly.com", "headers"), (hintedOne.Geo.Cdn, hintedOne.Geo.CdnDomain, hintedOne.Geo.CdnSource));
var snapshot = await DB.Default.Find<RemoteInstanceSnapshot>().Match(s => s.Host == listed).ExecuteSingleAsync(token);
Assert.Equal(("Cloudflare", "cloudflare.com", "ranges", "Amsterdam", 52.4), (snapshot.Cdn, snapshot.CdnDomain, snapshot.CdnSource, snapshot.City, snapshot.Latitude!.Value));
}
[Fact]
public async Task Without_api_v2_the_v1_api_is_read_and_an_unreachable_week_is_recorded_too()
{
+2 -1
View File
@@ -1,4 +1,4 @@
using Microsoft.AspNetCore.Builder;
using Microsoft.AspNetCore.Builder;
using Microsoft.AspNetCore.DataProtection;
using Microsoft.AspNetCore.Hosting;
using Microsoft.AspNetCore.Http;
@@ -75,6 +75,7 @@ namespace PrivaPub.Tests.Support.Host
["Federation:FetchLinkPreviews"] = "false",
["Registrations:Mode"] = "Open",
["Statistics:Geo:AutoUpdate"] = "false",
["Statistics:Cdn:AutoUpdate"] = "false",
["Media:Root"] = _mediaRoot,
["Logging:LogLevel:Default"] = "Warning",
["Serilog:MinimumLevel:Default"] = Environment.GetEnvironmentVariable("PRIVAPUB_TEST_LOGS") == "1" ? "Information" : "Fatal"