thepra/SocialPub
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

CDNs found by themselves, and servers followed through time
Build / Build (push) Successful in 5m11s
Details
Deploy / privapub.thepra.dev (push) Successful in 5m48s
Details

Browse Source 
PrivaPub now finds CDNs three ways, best first: the address ranges the
CDNs publish (Cloudflare, Fastly, Amazon CloudFront, Bunny, Gcore,
Imperva), downloaded daily by CdnUpdater and kept in CdnRangeSet; the
CDN's fingerprint in the responses it already gets from a server
(EdgeHintsHandler on the federation client); and the networks that carry
only a CDN. The fixed ASN list is gone; ASNs shared with plain hosting
(AWS, DataPacket) no longer hide a server. A server's Geo records the
CDN, its domain and how it was found, and weekly snapshots now keep the
city and coordinates too.

Servers through time (ServerPlaces): /instances/:host/history lists a
server's weekly snapshots, a CDN-fronted server's geo names the CDN's
domain and where the server was before it (before_cdn), and
/api/privapub/v1/cdns and /cdns/:domain group servers by CDN with week
by week who joined and who left. Owner decisions recorded in ROADMAP.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01LsXgEaXee4GCU1hwYgPJXw
This commit is contained in:
thepraandClaude Opus 5.5 committed 2026-10-04 11:33:39 +02:00
1 parent 5d7edc4a4c
commit 436f7da464
26 files changed
+1159 -62
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

PrivaPub/Infrastructure/Geo/CdnCatalog.cs
+156
View File
@@ -0,0 +1,156 @@
using System.Net;
using System.Text.Json;
namespace PrivaPub.Infrastructure.Geo
{
// How a published list of a CDN's addresses is written.
public enum CdnListFormat
{
Lines,//one CIDR per line (Cloudflare)
StringArray,//a JSON array of addresses or CIDRs (Bunny)
AddressesJson,//{"addresses": [...], "ipv6_addresses"/"addresses_v6": [...]} (Fastly, Gcore)
AwsCloudFront,//AWS ip-ranges.json, CLOUDFRONT entries only
ImpervaJson//{"ipRanges": [...], "ipv6Ranges": [...]}
}
public sealed record CdnList(string Url, CdnListFormat Format, string Method = "GET", string Body = default);
// A CDN, or any edge network whose addresses say nothing about where a server runs. Domain names it in statistics.
public sealed record CdnProvider(string Name, string Domain, int[] Asns, CdnList[] Lists, Func<HttpResponseMessage, bool> Fingerprint)
{
public string Key => Domain ?? Name.ToLowerInvariant();
}
// owner decision (2026-10-04): PrivaPub finds CDNs by itself, three ways, best first. The address ranges a CDN
// publishes (downloaded daily by CdnUpdater), the CDN's fingerprint in the responses PrivaPub already gets from a server
// (EdgeHintsHandler), and the networks (ASN) that carry nothing but a CDN. A network shared with plain hosting (AWS,
// Google, Microsoft, DataPacket) is never listed by ASN: only ranges or fingerprints name those CDNs.
public static class CdnCatalog
{
public static readonly IReadOnlyList<CdnProvider> All = new CdnProvider[]
{
new("Cloudflare", "cloudflare.com", new[] { 13335, 209242 },
new[] { new CdnList("https://www.cloudflare.com/ips-v4", CdnListFormat.Lines), new CdnList("https://www.cloudflare.com/ips-v6", CdnListFormat.Lines) },
r => Has(r, "cf-ray") || ServerIs(r, "cloudflare")),
new("Fastly", "fastly.com", new[] { 54113 },
new[] { new CdnList("https://api.fastly.com/public-ip-list", CdnListFormat.AddressesJson) },
r => Has(r, "x-fastly-request-id") || Has(r, "fastly-restarts") || Value(r, "x-served-by")?.StartsWith("cache-", StringComparison.OrdinalIgnoreCase) == true),
new("Amazon CloudFront", "cloudfront.net", Array.Empty<int>(),
new[] { new CdnList("https://ip-ranges.amazonaws.com/ip-ranges.json", CdnListFormat.AwsCloudFront) },
r => Has(r, "x-amz-cf-id") || Has(r, "x-amz-cf-pop") || Value(r, "via")?.Contains("cloudfront", StringComparison.OrdinalIgnoreCase) == true),
new("Akamai", "akamai.com", new[] { 20940, 16625, 16702, 21342 }, Array.Empty<CdnList>(),
r => Has(r, "akamai-grn") || Has(r, "x-akamai-transformed") || Has(r, "akamai-cache-status") || ServerStarts(r, "AkamaiGHost") || ServerStarts(r, "AkamaiNetStorage")),
new("Bunny", "bunny.net", new[] { 200325 },
new[] { new CdnList("https://api.bunny.net/system/edgeserverlist", CdnListFormat.StringArray), new CdnList("https://api.bunny.net/system/edgeserverlist/IPv6", CdnListFormat.StringArray) },
r => ServerStarts(r, "BunnyCDN") || Has(r, "cdn-pullzone") || Has(r, "cdn-requestid")),
new("Gcore", "gcore.com", Array.Empty<int>(),
new[] { new CdnList("https://api.gcore.com/cdn/public-ip-list", CdnListFormat.AddressesJson) },
r => ServerStarts(r, "gcore")),
new("Imperva", "imperva.com", new[] { 19551 },
new[] { new CdnList("https://my.imperva.com/api/integration/v1/ips", CdnListFormat.ImpervaJson, "POST", "resp_format=json") },
r => Has(r, "x-iinfo") || Value(r, "x-cdn") is "Imperva" or "Incapsula"),
new("CDN77", "cdn77.com", Array.Empty<int>(), Array.Empty<CdnList>(),
r => Has(r, "x-77-nzt") || Has(r, "x-77-cache") || ServerStarts(r, "CDN77")),
new("Edgio", "edg.io", new[] { 15133 }, Array.Empty<CdnList>(),
r => ServerStarts(r, "ECAcc") || ServerStarts(r, "ECS (") || Has(r, "x-ec-custom-error")),
new("Azure Front Door", "azure.microsoft.com", Array.Empty<int>(), Array.Empty<CdnList>(),
r => Has(r, "x-azure-ref")),
new("Google Cloud", "cloud.google.com", Array.Empty<int>(), Array.Empty<CdnList>(),
r => Value(r, "via")?.Contains("google", StringComparison.OrdinalIgnoreCase) == true),
new("Vercel", "vercel.com", Array.Empty<int>(), Array.Empty<CdnList>(),
r => Has(r, "x-vercel-id") || ServerIs(r, "Vercel")),
new("Netlify", "netlify.com", Array.Empty<int>(), Array.Empty<CdnList>(),
r => Has(r, "x-nf-request-id") || ServerIs(r, "Netlify")),
new("Sucuri", "sucuri.net", Array.Empty<int>(), Array.Empty<CdnList>(),
r => Has(r, "x-sucuri-id") || ServerStarts(r, "Sucuri")),
new("DDoS-Guard", "ddos-guard.net", Array.Empty<int>(), Array.Empty<CdnList>(),
r => ServerIs(r, "ddos-guard"))
};
public static CdnProvider ByKey(string key) =>
key == default ? default : All.FirstOrDefault(p => string.Equals(p.Key, key, StringComparison.OrdinalIgnoreCase) || string.Equals(p.Name, key, StringComparison.OrdinalIgnoreCase));
public static CdnProvider OfAsn(int? asn) => asn is { } number ? All.FirstOrDefault(p => p.Asns.Contains(number)) : default;
// The CDN a response passed through: a known fingerprint, or a CDN that names itself in X-CDN.
public static CdnProvider FromResponse(HttpResponseMessage response)
{
if (response == default)
return default;
var known = All.FirstOrDefault(p => p.Fingerprint(response));
if (known != default)
return known;
var named = Value(response, "x-cdn")?.Trim();
return named is { Length: > 1 and <= 40 } && named.All(c => char.IsLetterOrDigit(c) || c is ' ' or '-' or '.' or '_')
? new CdnProvider(named, default, Array.Empty<int>(), Array.Empty<CdnList>(), _ => false)
: default;
}
// The networks a published list holds; an entry that is not an address or a network is an error, never skipped.
public static List<IPNetwork> Parse(CdnListFormat format, string body)
{
IEnumerable<string> entries = format switch
{
CdnListFormat.Lines => body.Split('\n', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries).Where(l => !l.StartsWith('#')),
CdnListFormat.StringArray => Root(body),
CdnListFormat.AddressesJson => Json(body, "addresses", "ipv6_addresses", "addresses_v6"),
CdnListFormat.ImpervaJson => Json(body, "ipRanges", "ipv6Ranges"),
CdnListFormat.AwsCloudFront => CloudFront(body),
_ => Enumerable.Empty<string>()
};
return entries.Select(Network).ToList();
}
static IPNetwork Network(string entry)
{
if (IPNetwork.TryParse(entry, out var network))
return network;
if (IPAddress.TryParse(entry, out var address))
return new IPNetwork(address, address.AddressFamily == System.Net.Sockets.AddressFamily.InterNetwork ? 32 : 128);
throw new FormatException($"not an address or a network: {entry}");
}
static List<string> Root(string body)
{
using var document = JsonDocument.Parse(body);
return Strings(document.RootElement);
}
static IEnumerable<string> Json(string body, params string[] properties)
{
using var document = JsonDocument.Parse(body);
var entries = new List<string>();
foreach (var property in properties)
if (document.RootElement.TryGetProperty(property, out var array))
entries.AddRange(Strings(array));
return entries;
}
static IEnumerable<string> CloudFront(string body)
{
using var document = JsonDocument.Parse(body);
var entries = new List<string>();
foreach (var (array, field) in new[] { ("prefixes", "ip_prefix"), ("ipv6_prefixes", "ipv6_prefix") })
if (document.RootElement.TryGetProperty(array, out var prefixes))
foreach (var prefix in prefixes.EnumerateArray())
if (prefix.TryGetProperty("service", out var service) && service.GetString() == "CLOUDFRONT" && prefix.TryGetProperty(field, out var value))
entries.Add(value.GetString());
return entries;
}
static List<string> Strings(JsonElement array) =>
array.ValueKind == JsonValueKind.Array ? array.EnumerateArray().Select(e => e.GetString()).ToList() : throw new FormatException("not a list");
static bool Has(HttpResponseMessage response, string header) =>
response.Headers.Contains(header) || response.Content?.Headers.Contains(header) == true;
static string Value(HttpResponseMessage response, string header) =>
response.Headers.TryGetValues(header, out var values) || response.Content?.Headers.TryGetValues(header, out values) == true
? string.Join(", ", values)
: default;
static bool ServerIs(HttpResponseMessage response, string name) => string.Equals(Value(response, "server")?.Trim(), name, StringComparison.OrdinalIgnoreCase);
static bool ServerStarts(HttpResponseMessage response, string prefix) => Value(response, "server")?.TrimStart().StartsWith(prefix, StringComparison.OrdinalIgnoreCase) == true;
}
}
PrivaPub/Infrastructure/Geo/CdnRanges.cs
+65
View File
@@ -0,0 +1,65 @@
using MongoDB.Entities;
using PrivaPub.Models.Jobs;
using System.Net;
namespace PrivaPub.Infrastructure.Geo
{
public interface ICdnRanges
{
// the CDN whose published ranges hold the address, or default
CdnProvider Of(IPAddress address);
void Replace(CdnRangeSet set);
Task Load(CancellationToken token);
// per CDN key: how many ranges are known and when they were downloaded
IReadOnlyDictionary<string, (int Ranges, DateTime FetchedAt)> Status { get; }
}
// The published CDN ranges in memory, rebuilt from the stored CdnRangeSets at start and replaced set by set as
// CdnUpdater downloads them. A few thousand networks: a linear scan per describe is cheap, and describes are weekly.
public class CdnRanges : ICdnRanges
{
readonly object _gate = new();
Dictionary<string, (CdnProvider Provider, IPNetwork[] Networks, DateTime FetchedAt)> _sets = new(StringComparer.OrdinalIgnoreCase);
public IReadOnlyDictionary<string, (int Ranges, DateTime FetchedAt)> Status
{
get
{
lock (_gate)
return _sets.ToDictionary(s => s.Key, s => (s.Value.Networks.Length, s.Value.FetchedAt), StringComparer.OrdinalIgnoreCase);
}
}
public CdnProvider Of(IPAddress address)
{
if (address == default)
return default;
if (address.IsIPv4MappedToIPv6)
address = address.MapToIPv4();
Dictionary<string, (CdnProvider Provider, IPNetwork[] Networks, DateTime FetchedAt)> sets;
lock (_gate)
sets = _sets;
foreach (var (provider, networks, _) in sets.Values)
foreach (var network in networks)
if (network.BaseAddress.AddressFamily == address.AddressFamily && network.Contains(address))
return provider;
return default;
}
public void Replace(CdnRangeSet set)
{
var provider = CdnCatalog.ByKey(set.Key) ?? new CdnProvider(set.Name, set.Key, Array.Empty<int>(), Array.Empty<CdnList>(), _ => false);
var networks = set.Ranges.Select(r => IPNetwork.TryParse(r, out var n) ? n : (IPNetwork?)null).Where(n => n != default).Select(n => n!.Value).ToArray();
lock (_gate)
_sets = new(_sets, StringComparer.OrdinalIgnoreCase) { [set.Key] = (provider, networks, set.FetchedAt) };
}
public async Task Load(CancellationToken token)
{
foreach (var set in await DB.Default.Find<CdnRangeSet>().ExecuteAsync(token))
Replace(set);
}
}
}
PrivaPub/Infrastructure/Geo/CdnUpdater.cs
+114
View File
@@ -0,0 +1,114 @@
using Microsoft.Extensions.Options;
using MongoDB.Entities;
using PrivaPub.Infrastructure.Statistics;
using PrivaPub.Models.Jobs;
using System.Text;
namespace PrivaPub.Infrastructure.Geo
{
// Keeps the CDNs' published address ranges current by itself (owner decision 2026-10-04): at start it loads the stored
// ones, then once a day it downloads each list again. A list that fails to download or to parse, or comes back empty,
// leaves the stored one in place and is tried the next day. Like GeoUpdater, plain HTTPS to fixed hosts, not federation.
public sealed class CdnUpdater : BackgroundService
{
public const string ClientName = "cdn";
static readonly TimeSpan FirstWait = TimeSpan.FromMinutes(1);
static readonly TimeSpan Interval = TimeSpan.FromHours(24);
const int MaxBytes = 16 * 1024 * 1024;
const int MaxRanges = 200_000;
readonly IHttpClientFactory _http;
readonly IOptionsMonitor<StatisticsOptions> _options;
readonly ICdnRanges _ranges;
readonly ILogger<CdnUpdater> _logger;
public CdnUpdater(IHttpClientFactory http, IOptionsMonitor<StatisticsOptions> options, ICdnRanges ranges, ILogger<CdnUpdater> logger)
{
_http = http;
_options = options;
_ranges = ranges;
_logger = logger;
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
try
{
await _ranges.Load(stoppingToken);
await Task.Delay(FirstWait, stoppingToken);
while (!stoppingToken.IsCancellationRequested)
{
if (_options.CurrentValue.Cdn.AutoUpdate)
await Update(CdnCatalog.All, stoppingToken);
await Task.Delay(Interval + TimeSpan.FromMinutes(Random.Shared.Next(60)), stoppingToken);
}
}
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
{
}
catch (Exception ex)
{
_logger.LogWarning(ex, "{Service} stopped", nameof(CdnUpdater));
}
}
// The number of CDNs whose ranges were replaced.
public async Task<int> Update(IEnumerable<CdnProvider> providers, CancellationToken token)
{
var replaced = 0;
foreach (var provider in providers.Where(p => p.Lists.Length > 0))
{
try
{
var ranges = new List<string>();
foreach (var list in provider.Lists)
ranges.AddRange(CdnCatalog.Parse(list.Format, await Download(list, token)).Select(n => n.ToString()));
ranges = ranges.Distinct().ToList();
if (ranges.Count == 0 || ranges.Count > MaxRanges)
throw new InvalidDataException($"{ranges.Count} ranges");
var set = await DB.Default.Find<CdnRangeSet>().Match(s => s.Key == provider.Key).ExecuteFirstAsync(token) ?? new CdnRangeSet { Key = provider.Key };
set.Name = provider.Name;
set.Ranges = ranges;
set.Sources = provider.Lists.Select(l => l.Url).ToList();
set.FetchedAt = DateTime.UtcNow;
await DB.Default.SaveAsync(set, token);
_ranges.Replace(set);
replaced++;
}
catch (Exception ex) when (ex is HttpRequestException or InvalidDataException or FormatException or System.Text.Json.JsonException
or TaskCanceledException && !token.IsCancellationRequested)
{
_logger.LogWarning(ex, "Could not update the published ranges of {Cdn}", provider.Name);
}
}
return replaced;
}
async Task<string> Download(CdnList list, CancellationToken token)
{
using var request = new HttpRequestMessage(list.Method == "POST" ? HttpMethod.Post : HttpMethod.Get, list.Url);
request.Headers.Accept.ParseAdd("application/json, text/plain;q=0.9");
if (list.Body != default)
request.Content = new StringContent(list.Body, Encoding.UTF8, "application/x-www-form-urlencoded");
using var response = await _http.CreateClient(ClientName).SendAsync(request, HttpCompletionOption.ResponseHeadersRead, token);
response.EnsureSuccessStatusCode();
if (response.Content.Headers.ContentLength > MaxBytes)
throw new InvalidDataException($"more than {MaxBytes} bytes");
await using var stream = await response.Content.ReadAsStreamAsync(token);
using var buffer = new MemoryStream();
var chunk = new byte[81920];
int read;
while ((read = await stream.ReadAsync(chunk, token)) > 0)
{
if (buffer.Length + read > MaxBytes)
throw new InvalidDataException($"more than {MaxBytes} bytes");
buffer.Write(chunk, 0, read);
}
return Encoding.UTF8.GetString(buffer.ToArray());
}
}
}
PrivaPub/Infrastructure/Geo/GeoLocator.cs
+1 -20
View File
@@ -1,4 +1,4 @@
using MaxMind.Db;
using MaxMind.Db;
using Microsoft.Extensions.Options;
@@ -165,23 +165,4 @@ namespace PrivaPub.Infrastructure.Geo
_asn?.Dispose();
}
}
public static class CdnNetworks
{
static readonly Dictionary<int, string> Known = new()
{
[13335] = "Cloudflare",
[209242] = "Cloudflare",
[54113] = "Fastly",
[20940] = "Akamai",
[16625] = "Akamai",
[16702] = "Akamai",
[21342] = "Akamai",
[200325] = "Bunny",
[60068] = "CDN77",
[15133] = "Edgio"
};
public static string Of(int? asn) => asn is { } number && Known.TryGetValue(number, out var name) ? name : default;
}
}
PrivaPub/Infrastructure/Geo/GeoUpdater.cs
+3 -3
View File
@@ -1,4 +1,4 @@
using MaxMind.Db;
using MaxMind.Db;
using Microsoft.Extensions.Options;
@@ -11,8 +11,8 @@ namespace PrivaPub.Infrastructure.Geo
{
// Keeps the DB-IP Lite databases current by itself, so a deploy needs no timer and no root: once a day it checks
// whether each database was built this month and, if not, fetches this month's (or, early in the month, last month's),
// checks it opens as the right kind of database and swaps it in. This and SMTP are the server's only traffic that is not
// federation, which is why it has its own client instead of IFederationHttp.
// checks it opens as the right kind of database and swaps it in. This, CdnUpdater and SMTP are the server's only traffic
// that is not federation, which is why each has its own client instead of IFederationHttp.
public sealed class GeoUpdater : BackgroundService
{
public const string ClientName = "geo";