CDNs found by themselves, and servers followed through time

PrivaPub now finds CDNs three ways, best first: the address ranges the
CDNs publish (Cloudflare, Fastly, Amazon CloudFront, Bunny, Gcore,
Imperva), downloaded daily by CdnUpdater and kept in CdnRangeSet; the
CDN's fingerprint in the responses it already gets from a server
(EdgeHintsHandler on the federation client); and the networks that carry
only a CDN. The fixed ASN list is gone; ASNs shared with plain hosting
(AWS, DataPacket) no longer hide a server. A server's Geo records the
CDN, its domain and how it was found, and weekly snapshots now keep the
city and coordinates too.

Servers through time (ServerPlaces): /instances/:host/history lists a
server's weekly snapshots, a CDN-fronted server's geo names the CDN's
domain and where the server was before it (before_cdn), and
/api/privapub/v1/cdns and /cdns/:domain group servers by CDN with week
by week who joined and who left. Owner decisions recorded in ROADMAP.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01LsXgEaXee4GCU1hwYgPJXw

PrivaPub.Tests/Infrastructure/CdnCatalogTests.cs

@@ -0,0 +1,111 @@
+using PrivaPub.Infrastructure.Geo;
+using PrivaPub.Infrastructure.Http;
+using PrivaPub.Models.Jobs;
+
+using System.Net;
+
+namespace PrivaPub.Tests.Infrastructure
+{
+	public class CdnCatalogTests
+	{
+		static HttpResponseMessage Response(params (string Name, string Value)[] headers)
+		{
+			var response = new HttpResponseMessage(HttpStatusCode.OK) { Content = new StringContent("{}") };
+			foreach (var (name, value) in headers)
+				response.Headers.TryAddWithoutValidation(name, value);
+			return response;
+		}
+
+		[Theory]
+		[InlineData("cf-ray", "8c1f0e2b4d5a1234-FRA", "cloudflare.com")]
+		[InlineData("server", "cloudflare", "cloudflare.com")]
+		[InlineData("x-amz-cf-id", "abc==", "cloudfront.net")]
+		[InlineData("via", "1.1 5a1b.cloudfront.net (CloudFront)", "cloudfront.net")]
+		[InlineData("x-served-by", "cache-fra-eddf8230045-FRA", "fastly.com")]
+		[InlineData("server", "BunnyCDN-DE1-1078", "bunny.net")]
+		[InlineData("akamai-grn", "0.1234", "akamai.com")]
+		[InlineData("x-azure-ref", "20261004T101010Z-abc", "azure.microsoft.com")]
+		[InlineData("x-vercel-id", "fra1::abc", "vercel.com")]
+		[InlineData("x-iinfo", "1-2-3", "imperva.com")]
+		public void A_cdn_is_recognised_by_its_fingerprint(string header, string value, string domain)
+		{
+			Assert.Equal(domain, CdnCatalog.FromResponse(Response((header, value)))?.Domain);
+		}
+
+		[Fact]
+		public void A_plain_server_has_no_cdn_and_an_unknown_cdn_names_itself()
+		{
+			Assert.Null(CdnCatalog.FromResponse(Response(("server", "nginx"), ("via", "1.1 varnish"))));
+			var named = CdnCatalog.FromResponse(Response(("x-cdn", "EdgeOne")));
+			Assert.Equal(("EdgeOne", null, "edgeone"), (named.Name, named.Domain, named.Key));
+			Assert.Null(CdnCatalog.FromResponse(Response(("x-cdn", "<script>"))));
+		}
+
+		[Fact]
+		public void Only_networks_that_carry_nothing_but_a_cdn_name_it()
+		{
+			Assert.Equal("Cloudflare", CdnCatalog.OfAsn(13335)?.Name);
+			Assert.Equal("Akamai", CdnCatalog.OfAsn(20940)?.Name);
+			Assert.Null(CdnCatalog.OfAsn(16509));//AWS: CloudFront only by its ranges or its headers
+			Assert.Null(CdnCatalog.OfAsn(60068));//DataPacket also rents servers
+		}
+
+		[Fact]
+		public void Every_published_list_format_is_read()
+		{
+			Assert.Equal(new[] { "173.245.48.0/20", "2400:cb00::/32" },
+				CdnCatalog.Parse(CdnListFormat.Lines, "173.245.48.0/20\n# comment\n\n2400:cb00::/32\n").Select(n => n.ToString()));
+			Assert.Equal(new[] { "89.187.188.227/32", "2400:52e0:1500::714:1/128" },
+				CdnCatalog.Parse(CdnListFormat.StringArray, """["89.187.188.227","2400:52e0:1500::714:1"]""").Select(n => n.ToString()));
+			Assert.Equal(new[] { "23.235.32.0/20", "2a04:4e40::/32" },
+				CdnCatalog.Parse(CdnListFormat.AddressesJson, """{"addresses":["23.235.32.0/20"],"ipv6_addresses":["2a04:4e40::/32"]}""").Select(n => n.ToString()));
+			Assert.Equal(new[] { "92.223.124.12/32", "2a03:90c0::/32" },
+				CdnCatalog.Parse(CdnListFormat.AddressesJson, """{"addresses":["92.223.124.12/32"],"addresses_v6":["2a03:90c0::/32"]}""").Select(n => n.ToString()));
+			Assert.Equal(new[] { "199.83.128.0/21", "2a02:e980::/29" },
+				CdnCatalog.Parse(CdnListFormat.ImpervaJson, """{"ipRanges":["199.83.128.0/21"],"ipv6Ranges":["2a02:e980::/29"],"res":0}""").Select(n => n.ToString()));
+			Assert.Equal(new[] { "13.32.0.0/15", "2600:9000::/28" },
+				CdnCatalog.Parse(CdnListFormat.AwsCloudFront, """
+					{"prefixes":[{"ip_prefix":"3.4.12.4/32","service":"AMAZON"},{"ip_prefix":"13.32.0.0/15","service":"CLOUDFRONT"}],
+					 "ipv6_prefixes":[{"ipv6_prefix":"2600:9000::/28","service":"CLOUDFRONT"},{"ipv6_prefix":"2406:daba::/40","service":"EC2"}]}
+					""").Select(n => n.ToString()));
+			Assert.Throws<FormatException>(() => CdnCatalog.Parse(CdnListFormat.Lines, "173.245.48.0/20\nnot an address"));
+		}
+
+		[Fact]
+		public void The_published_ranges_name_the_cdn_of_an_address()
+		{
+			var ranges = new CdnRanges();
+			ranges.Replace(new CdnRangeSet { Key = "cloudflare.com", Name = "Cloudflare", Ranges = new() { "104.16.0.0/13", "2606:4700::/32" }, FetchedAt = DateTime.UtcNow });
+
+			Assert.Equal("Cloudflare", ranges.Of(IPAddress.Parse("104.18.2.3"))?.Name);
+			Assert.Equal("Cloudflare", ranges.Of(IPAddress.Parse("::ffff:104.18.2.3"))?.Name);
+			Assert.Equal("Cloudflare", ranges.Of(IPAddress.Parse("2606:4700::1"))?.Name);
+			Assert.Null(ranges.Of(IPAddress.Parse("203.0.113.7")));
+			Assert.Equal(2, ranges.Status["cloudflare.com"].Ranges);
+		}
+
+		[Fact]
+		public async Task Responses_leave_a_cdn_hint_for_their_host_until_one_comes_without()
+		{
+			var hints = new EdgeHints();
+			var answers = new Queue<HttpResponseMessage>(new[] { Response(("cf-ray", "1-FRA")), Response(("server", "nginx")) });
+			using var client = new HttpClient(new EdgeHintsHandler(hints) { InnerHandler = new Answering(answers) });
+
+			await client.GetAsync("https://social.example/.well-known/nodeinfo", TestContext.Current.CancellationToken);
+			Assert.Equal("Cloudflare", hints.Of("social.example")?.Name);
+
+			await client.GetAsync("https://social.example/nodeinfo/2.1", TestContext.Current.CancellationToken);
+			Assert.Null(hints.Of("social.example"));
+		}
+
+		sealed class Answering : HttpMessageHandler
+		{
+			readonly Queue<HttpResponseMessage> _answers;
+
+			public Answering(Queue<HttpResponseMessage> answers) => _answers = answers;
+
+			protected override Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) =>
+				Task.FromResult(_answers.Dequeue());
+		}
+	}
+}

PrivaPub.Tests/Infrastructure/CdnUpdaterTests.cs

@@ -0,0 +1,105 @@
+using Microsoft.Extensions.Logging.Abstractions;
+
+using MongoDB.Entities;
+
+using PrivaPub.Infrastructure.Geo;
+using PrivaPub.Infrastructure.Statistics;
+using PrivaPub.Models.Jobs;
+using PrivaPub.Tests.Support;
+
+using System.Net;
+
+namespace PrivaPub.Tests.Infrastructure
+{
+	[Trait("Category", "Integration")]
+	public sealed class CdnUpdaterTests : IAsyncLifetime
+	{
+		Peer _peer;
+		CdnRanges _ranges;
+		CdnUpdater _updater;
+		string _key;
+
+		public async ValueTask InitializeAsync()
+		{
+			Assert.SkipUnless(MongoFixture.Enabled, MongoFixture.Skip);
+			_peer = await Peer.Start();
+			_ranges = new CdnRanges();
+			_updater = new CdnUpdater(new Clients(), new StaticOptions<StatisticsOptions>(new StatisticsOptions()), _ranges, NullLogger<CdnUpdater>.Instance);
+			_key = $"edge{Guid.NewGuid():N}.example";
+		}
+
+		public async ValueTask DisposeAsync()
+		{
+			if (_peer != default)
+				await _peer.DisposeAsync();
+		}
+
+		sealed class Clients : IHttpClientFactory
+		{
+			public HttpClient CreateClient(string name) => new();
+		}
+
+		CdnProvider Provider() => new("Test Edge", _key, Array.Empty<int>(), new[]
+		{
+			new CdnList($"{_peer.A}/edge/v4", CdnListFormat.Lines),
+			new CdnList($"{_peer.A}/edge/v6", CdnListFormat.StringArray)
+		}, _ => false);
+
+		static CancellationToken Token => TestContext.Current.CancellationToken;
+
+		[Fact]
+		public async Task The_published_ranges_are_downloaded_stored_and_used()
+		{
+			_peer.ServeText("/edge/v4", "198.51.100.0/24\n", "text/plain");
+			_peer.ServeText("/edge/v6", """["2001:db8::/32","192.0.2.9"]""", "application/json");
+
+			Assert.Equal(1, await _updater.Update(new[] { Provider() }, Token));
+
+			var stored = await DB.Default.Find<CdnRangeSet>().Match(s => s.Key == _key).ExecuteSingleAsync(Token);
+			Assert.Equal(new[] { "198.51.100.0/24", "2001:db8::/32", "192.0.2.9/32" }, stored.Ranges);
+			Assert.Equal("Test Edge", _ranges.Of(IPAddress.Parse("198.51.100.20"))?.Name);
+			Assert.Equal("Test Edge", _ranges.Of(IPAddress.Parse("192.0.2.9"))?.Name);
+
+			var restarted = new CdnRanges();
+			await restarted.Load(Token);
+			Assert.Equal("Test Edge", restarted.Of(IPAddress.Parse("2001:db8::7"))?.Name);
+		}
+
+		[Theory]
+		[InlineData("not a network")]
+		[InlineData("")]
+		public async Task A_broken_or_empty_list_keeps_the_stored_ranges(string broken)
+		{
+			_peer.ServeText("/edge/v4", "198.51.100.0/24", "text/plain");
+			_peer.ServeText("/edge/v6", "[]", "application/json");
+			Assert.Equal(1, await _updater.Update(new[] { Provider() }, Token));
+
+			_peer.ServeText("/edge/v4", broken, "text/plain");
+			Assert.Equal(0, await _updater.Update(new[] { Provider() }, Token));
+
+			Assert.Equal(new[] { "198.51.100.0/24" }, (await DB.Default.Find<CdnRangeSet>().Match(s => s.Key == _key).ExecuteSingleAsync(Token)).Ranges);
+			Assert.Equal("Test Edge", _ranges.Of(IPAddress.Parse("198.51.100.1"))?.Name);
+		}
+
+		// The real lists, from the CDNs themselves: set PRIVAPUB_TEST_CDN_LIVE=1 to check that every format still parses.
+		[Fact]
+		public async Task Every_cdns_real_list_still_parses()
+		{
+			Assert.SkipUnless(Environment.GetEnvironmentVariable("PRIVAPUB_TEST_CDN_LIVE") == "1", "set PRIVAPUB_TEST_CDN_LIVE=1 to download the real lists");
+			var listed = CdnCatalog.All.Where(p => p.Lists.Length > 0).ToList();
+
+			Assert.Equal(listed.Count, await _updater.Update(listed, Token));
+
+			Assert.Equal("Cloudflare", _ranges.Of(IPAddress.Parse("104.16.0.1"))?.Name);
+			foreach (var provider in listed)
+				Assert.True(_ranges.Status[provider.Key].Ranges > 0, provider.Name);
+		}
+
+		[Fact]
+		public async Task An_unreachable_list_changes_nothing()
+		{
+			Assert.Equal(0, await _updater.Update(new[] { Provider() }, Token));
+			Assert.False(await DB.Default.Find<CdnRangeSet>().Match(s => s.Key == _key).ExecuteAnyAsync(Token));
+		}
+	}
+}

PrivaPub.Tests/Infrastructure/GeoLocatorTests.cs

@@ -1,4 +1,4 @@
-using Microsoft.Extensions.Logging.Abstractions;
+using Microsoft.Extensions.Logging.Abstractions;
 
 using PrivaPub.Infrastructure.Geo;
 using PrivaPub.Infrastructure.Http;
@@ -45,7 +45,7 @@ namespace PrivaPub.Tests.Infrastructure
 			Assert.NotNull(fix);
 			Assert.Equal(2, fix.Country.Length);
 			Assert.Equal(13335, fix.Asn);
-			Assert.Equal("Cloudflare", CdnNetworks.Of(fix.Asn));
+			Assert.Equal("Cloudflare", CdnCatalog.OfAsn(fix.Asn)?.Name);
 			Assert.NotNull(fix.AsnOrg);
 			Assert.NotNull(fix.Latitude);
 			Assert.Equal(Math.Round(fix.Latitude.Value, 1), fix.Latitude);
@@ -61,9 +61,9 @@ namespace PrivaPub.Tests.Infrastructure
 
 			Assert.Equal(IPAddress.Parse("203.0.113.7"), connected.Of("social.example"));
 			Assert.Null(connected.Of("other.example"));
-			Assert.Equal("Fastly", CdnNetworks.Of(54113));
-			Assert.Null(CdnNetworks.Of(64496));
-			Assert.Null(CdnNetworks.Of(default));
+			Assert.Equal("Fastly", CdnCatalog.OfAsn(54113)?.Name);
+			Assert.Null(CdnCatalog.OfAsn(64496));
+			Assert.Null(CdnCatalog.OfAsn(default));
 		}
 	}
 }