CDNs found by themselves, and servers followed through time

PrivaPub now finds CDNs three ways, best first: the address ranges the
CDNs publish (Cloudflare, Fastly, Amazon CloudFront, Bunny, Gcore,
Imperva), downloaded daily by CdnUpdater and kept in CdnRangeSet; the
CDN's fingerprint in the responses it already gets from a server
(EdgeHintsHandler on the federation client); and the networks that carry
only a CDN. The fixed ASN list is gone; ASNs shared with plain hosting
(AWS, DataPacket) no longer hide a server. A server's Geo records the
CDN, its domain and how it was found, and weekly snapshots now keep the
city and coordinates too.

Servers through time (ServerPlaces): /instances/:host/history lists a
server's weekly snapshots, a CDN-fronted server's geo names the CDN's
domain and where the server was before it (before_cdn), and
/api/privapub/v1/cdns and /cdns/:domain group servers by CDN with week
by week who joined and who left. Owner decisions recorded in ROADMAP.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01LsXgEaXee4GCU1hwYgPJXw

PrivaPub/Infrastructure/Geo/CdnUpdater.cs

@@ -0,0 +1,114 @@
+using Microsoft.Extensions.Options;
+
+using MongoDB.Entities;
+
+using PrivaPub.Infrastructure.Statistics;
+using PrivaPub.Models.Jobs;
+
+using System.Text;
+
+namespace PrivaPub.Infrastructure.Geo
+{
+	// Keeps the CDNs' published address ranges current by itself (owner decision 2026-10-04): at start it loads the stored
+	// ones, then once a day it downloads each list again. A list that fails to download or to parse, or comes back empty,
+	// leaves the stored one in place and is tried the next day. Like GeoUpdater, plain HTTPS to fixed hosts, not federation.
+	public sealed class CdnUpdater : BackgroundService
+	{
+		public const string ClientName = "cdn";
+		static readonly TimeSpan FirstWait = TimeSpan.FromMinutes(1);
+		static readonly TimeSpan Interval = TimeSpan.FromHours(24);
+		const int MaxBytes = 16 * 1024 * 1024;
+		const int MaxRanges = 200_000;
+
+		readonly IHttpClientFactory _http;
+		readonly IOptionsMonitor<StatisticsOptions> _options;
+		readonly ICdnRanges _ranges;
+		readonly ILogger<CdnUpdater> _logger;
+
+		public CdnUpdater(IHttpClientFactory http, IOptionsMonitor<StatisticsOptions> options, ICdnRanges ranges, ILogger<CdnUpdater> logger)
+		{
+			_http = http;
+			_options = options;
+			_ranges = ranges;
+			_logger = logger;
+		}
+
+		protected override async Task ExecuteAsync(CancellationToken stoppingToken)
+		{
+			try
+			{
+				await _ranges.Load(stoppingToken);
+				await Task.Delay(FirstWait, stoppingToken);
+				while (!stoppingToken.IsCancellationRequested)
+				{
+					if (_options.CurrentValue.Cdn.AutoUpdate)
+						await Update(CdnCatalog.All, stoppingToken);
+					await Task.Delay(Interval + TimeSpan.FromMinutes(Random.Shared.Next(60)), stoppingToken);
+				}
+			}
+			catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
+			{
+			}
+			catch (Exception ex)
+			{
+				_logger.LogWarning(ex, "{Service} stopped", nameof(CdnUpdater));
+			}
+		}
+
+		// The number of CDNs whose ranges were replaced.
+		public async Task<int> Update(IEnumerable<CdnProvider> providers, CancellationToken token)
+		{
+			var replaced = 0;
+			foreach (var provider in providers.Where(p => p.Lists.Length > 0))
+			{
+				try
+				{
+					var ranges = new List<string>();
+					foreach (var list in provider.Lists)
+						ranges.AddRange(CdnCatalog.Parse(list.Format, await Download(list, token)).Select(n => n.ToString()));
+					ranges = ranges.Distinct().ToList();
+					if (ranges.Count == 0 || ranges.Count > MaxRanges)
+						throw new InvalidDataException($"{ranges.Count} ranges");
+
+					var set = await DB.Default.Find<CdnRangeSet>().Match(s => s.Key == provider.Key).ExecuteFirstAsync(token) ?? new CdnRangeSet { Key = provider.Key };
+					set.Name = provider.Name;
+					set.Ranges = ranges;
+					set.Sources = provider.Lists.Select(l => l.Url).ToList();
+					set.FetchedAt = DateTime.UtcNow;
+					await DB.Default.SaveAsync(set, token);
+					_ranges.Replace(set);
+					replaced++;
+				}
+				catch (Exception ex) when (ex is HttpRequestException or InvalidDataException or FormatException or System.Text.Json.JsonException
+					or TaskCanceledException && !token.IsCancellationRequested)
+				{
+					_logger.LogWarning(ex, "Could not update the published ranges of {Cdn}", provider.Name);
+				}
+			}
+			return replaced;
+		}
+
+		async Task<string> Download(CdnList list, CancellationToken token)
+		{
+			using var request = new HttpRequestMessage(list.Method == "POST" ? HttpMethod.Post : HttpMethod.Get, list.Url);
+			request.Headers.Accept.ParseAdd("application/json, text/plain;q=0.9");
+			if (list.Body != default)
+				request.Content = new StringContent(list.Body, Encoding.UTF8, "application/x-www-form-urlencoded");
+			using var response = await _http.CreateClient(ClientName).SendAsync(request, HttpCompletionOption.ResponseHeadersRead, token);
+			response.EnsureSuccessStatusCode();
+			if (response.Content.Headers.ContentLength > MaxBytes)
+				throw new InvalidDataException($"more than {MaxBytes} bytes");
+			await using var stream = await response.Content.ReadAsStreamAsync(token);
+			using var buffer = new MemoryStream();
+			var chunk = new byte[81920];
+			int read;
+			while ((read = await stream.ReadAsync(chunk, token)) > 0)
+			{
+				if (buffer.Length + read > MaxBytes)
+					throw new InvalidDataException($"more than {MaxBytes} bytes");
+				buffer.Write(chunk, 0, read);
+			}
+			return Encoding.UTF8.GetString(buffer.ToArray());
+		}
+	}
+}