CDNs found by themselves, and servers followed through time

PrivaPub now finds CDNs three ways, best first: the address ranges the
CDNs publish (Cloudflare, Fastly, Amazon CloudFront, Bunny, Gcore,
Imperva), downloaded daily by CdnUpdater and kept in CdnRangeSet; the
CDN's fingerprint in the responses it already gets from a server
(EdgeHintsHandler on the federation client); and the networks that carry
only a CDN. The fixed ASN list is gone; ASNs shared with plain hosting
(AWS, DataPacket) no longer hide a server. A server's Geo records the
CDN, its domain and how it was found, and weekly snapshots now keep the
city and coordinates too.

Servers through time (ServerPlaces): /instances/:host/history lists a
server's weekly snapshots, a CDN-fronted server's geo names the CDN's
domain and where the server was before it (before_cdn), and
/api/privapub/v1/cdns and /cdns/:domain group servers by CDN with week
by week who joined and who left. Owner decisions recorded in ROADMAP.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01LsXgEaXee4GCU1hwYgPJXw

PrivaPub/Domain/Statistics/SelfLocation.cs

@@ -24,6 +24,7 @@ namespace PrivaPub.Domain.Statistics
 		static readonly TimeSpan Lifetime = TimeSpan.FromDays(1);
 
 		readonly IGeoLocator _geo;
+		readonly ICdnRanges _cdnRanges;
 		readonly IOptionsMonitor<StatisticsOptions> _options;
 		readonly IOptionsMonitor<AppConfiguration> _app;
 		readonly ILogger<SelfLocation> _logger;
@@ -32,9 +33,11 @@ namespace PrivaPub.Domain.Statistics
 		InstanceGeo _located;
 		DateTime _locatedAt = DateTime.MinValue;
 
-		public SelfLocation(IGeoLocator geo, IOptionsMonitor<StatisticsOptions> options, IOptionsMonitor<AppConfiguration> app, ILogger<SelfLocation> logger)
+		public SelfLocation(IGeoLocator geo, IOptionsMonitor<StatisticsOptions> options, IOptionsMonitor<AppConfiguration> app, ILogger<SelfLocation> logger,
+			ICdnRanges cdnRanges = default)
 		{
 			_geo = geo;
+			_cdnRanges = cdnRanges;
 			_options = options;
 			_app = app;
 			_logger = logger;
@@ -79,6 +82,7 @@ namespace PrivaPub.Domain.Statistics
 				var addresses = await Dns.GetHostAddressesAsync(self.Host, token);
 				var address = addresses.FirstOrDefault(a => a.AddressFamily == AddressFamily.InterNetwork) ?? addresses.FirstOrDefault();
 				var fix = address == default ? default : _geo.Locate(address);
+				var (cdn, cdnSource) = Federation.Objects.InstanceDescriber.Cdn(address, self.Host, fix?.Asn, _cdnRanges, default);
 				return fix == default ? default : new InstanceGeo
 				{
 					Country = fix.Country,
@@ -87,7 +91,9 @@ namespace PrivaPub.Domain.Statistics
 					Longitude = fix.Longitude,
 					Asn = fix.Asn,
 					AsnOrg = fix.AsnOrg,
-					Cdn = CdnNetworks.Of(fix.Asn),
+					Cdn = cdn?.Name,
+					CdnDomain = cdn?.Domain,
+					CdnSource = cdnSource,
 					Ipv6 = address.AddressFamily == AddressFamily.InterNetworkV6,
 					Source = _geo.Source,
 					LocatedAt = DateTime.UtcNow

PrivaPub/Domain/Statistics/ServerPlaces.cs

@@ -0,0 +1,163 @@
+using Microsoft.Extensions.Caching.Memory;
+
+using MongoDB.Entities;
+
+using PrivaPub.Federation.Objects;
+using PrivaPub.Infrastructure.Geo;
+using PrivaPub.Models.Jobs;
+
+namespace PrivaPub.Domain.Statistics
+{
+	// Where a server was before it went behind a CDN, as its weekly snapshots last saw it.
+	public sealed record PlaceBeforeCdn(string Week, string Country, string City, double? Latitude, double? Longitude);
+
+	// A CDN as a group of servers: those behind it now, and week by week how many were behind it, who joined and who left.
+	public sealed record CdnGroup(string Key, string Name, string Domain, int Servers, int Ranges, DateTime? RangesUpdatedAt, List<string> DetectedBy);
+
+	public sealed record CdnWeek(string Week, int Servers, List<string> Joined, List<string> Left);
+
+	public sealed record CdnHistory(CdnGroup Group, List<string> Hosts, List<CdnWeek> Weeks);
+
+	public interface IServerPlaces
+	{
+		Task<Dictionary<string, PlaceBeforeCdn>> BeforeCdn(IEnumerable<string> hosts, CancellationToken token);
+		Task<List<RemoteInstanceSnapshot>> History(string host, int weeks, CancellationToken token);
+		Task<List<CdnGroup>> Cdns(CancellationToken token);
+		Task<CdnHistory> Cdn(string key, int weeks, CancellationToken token);
+	}
+
+	// Servers through time (owner decision 2026-10-04): each server's weekly snapshots (place, CDN, size), and the servers
+	// grouped by the CDN in front of them, named by the CDN's own domain. Read from the weekly snapshots InstanceDescriber
+	// writes; the CDN pages are kept for half an hour, because a large CDN reads many snapshots.
+	public class ServerPlaces : IServerPlaces
+	{
+		public const int MaxHosts = 500;
+		const int MaxChanges = 50;
+		static readonly TimeSpan CacheFor = TimeSpan.FromMinutes(30);
+
+		readonly ICdnRanges _ranges;
+		readonly IMemoryCache _cache;
+
+		public ServerPlaces(ICdnRanges ranges, IMemoryCache cache)
+		{
+			_ranges = ranges;
+			_cache = cache;
+		}
+
+		// the CDN key a server's place belongs to: the CDN's domain, or its name for a CDN that has none
+		public static string KeyOf(InstanceGeo geo) =>
+			geo?.Cdn == default ? default : geo.CdnDomain ?? CdnCatalog.ByKey(geo.Cdn)?.Key ?? geo.Cdn.ToLowerInvariant();
+
+		static string KeyOf(RemoteInstanceSnapshot snapshot) =>
+			snapshot.Cdn == default ? default : snapshot.CdnDomain ?? CdnCatalog.ByKey(snapshot.Cdn)?.Key ?? snapshot.Cdn.ToLowerInvariant();
+
+		public async Task<Dictionary<string, PlaceBeforeCdn>> BeforeCdn(IEnumerable<string> hosts, CancellationToken token)
+		{
+			var wanted = hosts.Where(h => h != default).Distinct().ToList();
+			if (wanted.Count == 0)
+				return new();
+			var located = await DB.Default.Find<RemoteInstanceSnapshot>()
+				.Match(s => wanted.Contains(s.Host) && s.Cdn == null && (s.Country != null || s.Latitude != null))
+				.Sort(s => s.Week, Order.Descending)
+				.ExecuteAsync(token);
+			return located.GroupBy(s => s.Host)
+				.Select(g => g.First())
+				.ToDictionary(s => s.Host, s => new PlaceBeforeCdn(s.Week, s.Country, s.City, s.Latitude, s.Longitude));
+		}
+
+		public async Task<List<RemoteInstanceSnapshot>> History(string host, int weeks, CancellationToken token) =>
+			await DB.Default.Find<RemoteInstanceSnapshot>()
+				.Match(s => s.Host == host)
+				.Sort(s => s.Week, Order.Descending)
+				.Limit(weeks)
+				.ExecuteAsync(token);
+
+		public async Task<List<CdnGroup>> Cdns(CancellationToken token)
+		{
+			var fronted = await Fronted(token);
+			var status = _ranges.Status;
+			var keys = CdnCatalog.All.Select(p => p.Key).Concat(fronted.Select(i => KeyOf(i.Geo))).Distinct(StringComparer.OrdinalIgnoreCase);
+			return keys.Select(key => Group(key, fronted, status)).OrderByDescending(g => g.Servers).ThenBy(g => g.Name).ToList();
+		}
+
+		public async Task<CdnHistory> Cdn(string key, int weeks, CancellationToken token)
+		{
+			key = key?.Trim().ToLowerInvariant();
+			if (string.IsNullOrEmpty(key) || key.Length > 100)
+				return default;
+			return await _cache.GetOrCreateAsync($"cdn-history|{key}|{weeks}", async entry =>
+			{
+				entry.AbsoluteExpirationRelativeToNow = CacheFor;
+				var fronted = await Fronted(token);
+				var provider = CdnCatalog.ByKey(key);
+				if (provider == default && fronted.All(i => KeyOf(i.Geo) != key))
+					return default;
+				var group = Group(provider?.Key ?? key, fronted, _ranges.Status);
+				var hosts = fronted.Where(i => KeyOf(i.Geo) == group.Key).Select(i => i.Host).OrderBy(h => h).Take(MaxHosts).ToList();
+				return new CdnHistory(group, hosts, await Weeks(group, weeks, token));
+			});
+		}
+
+		async Task<List<CdnWeek>> Weeks(CdnGroup group, int weeks, CancellationToken token)
+		{
+			var since = InstanceDescriber.Week(DateTime.UtcNow.AddDays(-7 * weeks));
+			var everBehind = (await DB.Default.Find<RemoteInstanceSnapshot>()
+					.Match(f => f.Gte(s => s.Week, since) & (f.Eq(s => s.CdnDomain, group.Key) | f.Eq(s => s.Cdn, group.Name)))
+					.Project(s => new RemoteInstanceSnapshot { Host = s.Host })
+					.ExecuteAsync(token))
+				.Select(s => s.Host).Distinct().ToList();
+			if (everBehind.Count == 0)
+				return new();
+
+			var snapshots = await DB.Default.Find<RemoteInstanceSnapshot>()
+				.Match(f => f.In(s => s.Host, everBehind) & f.Gte(s => s.Week, since))
+				.Project(s => new RemoteInstanceSnapshot { Host = s.Host, Week = s.Week, Cdn = s.Cdn, CdnDomain = s.CdnDomain })
+				.ExecuteAsync(token);
+
+			// a server is behind the CDN from the week a snapshot shows it there until one shows it elsewhere
+			var behind = new Dictionary<string, bool>();
+			var history = new List<CdnWeek>();
+			foreach (var week in snapshots.GroupBy(s => s.Week).OrderBy(w => w.Key, StringComparer.Ordinal))
+			{
+				var joined = new List<string>();
+				var left = new List<string>();
+				foreach (var snapshot in week)
+				{
+					var now = KeyOf(snapshot) == group.Key;
+					var before = behind.GetValueOrDefault(snapshot.Host);
+					if (now && !before)
+						joined.Add(snapshot.Host);
+					else if (!now && before)
+						left.Add(snapshot.Host);
+					behind[snapshot.Host] = now;
+				}
+				history.Add(new CdnWeek(week.Key, behind.Count(b => b.Value), joined.OrderBy(h => h).Take(MaxChanges).ToList(),
+					left.OrderBy(h => h).Take(MaxChanges).ToList()));
+			}
+			history.Reverse();
+			return history;
+		}
+
+		static async Task<List<RemoteInstance>> Fronted(CancellationToken token) =>
+			await DB.Default.Find<RemoteInstance>()
+				.Match(i => i.Geo.Cdn != null)
+				.Project(i => new RemoteInstance { Host = i.Host, Geo = i.Geo })
+				.ExecuteAsync(token);
+
+		static CdnGroup Group(string key, List<RemoteInstance> fronted, IReadOnlyDictionary<string, (int Ranges, DateTime FetchedAt)> status)
+		{
+			var provider = CdnCatalog.ByKey(key);
+			var servers = fronted.Where(i => KeyOf(i.Geo) == key).ToList();
+			var detectedBy = new List<string>();
+			if (provider?.Lists.Length > 0)
+				detectedBy.Add("ranges");
+			if (provider != default)
+				detectedBy.Add("headers");
+			if (provider?.Asns.Length > 0)
+				detectedBy.Add("asn");
+			var ranges = status.TryGetValue(key, out var known) ? known : default;
+			return new CdnGroup(key, provider?.Name ?? servers.Select(i => i.Geo.Cdn).FirstOrDefault() ?? key, provider?.Domain, servers.Count, ranges.Ranges,
+				ranges.Ranges > 0 ? ranges.FetchedAt : null, detectedBy);
+		}
+	}
+}