CDNs found by themselves, and servers followed through time
PrivaPub now finds CDNs three ways, best first: the address ranges the CDNs publish (Cloudflare, Fastly, Amazon CloudFront, Bunny, Gcore, Imperva), downloaded daily by CdnUpdater and kept in CdnRangeSet; the CDN's fingerprint in the responses it already gets from a server (EdgeHintsHandler on the federation client); and the networks that carry only a CDN. The fixed ASN list is gone; ASNs shared with plain hosting (AWS, DataPacket) no longer hide a server. A server's Geo records the CDN, its domain and how it was found, and weekly snapshots now keep the city and coordinates too. Servers through time (ServerPlaces): /instances/:host/history lists a server's weekly snapshots, a CDN-fronted server's geo names the CDN's domain and where the server was before it (before_cdn), and /api/privapub/v1/cdns and /cdns/:domain group servers by CDN with week by week who joined and who left. Owner decisions recorded in ROADMAP. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01LsXgEaXee4GCU1hwYgPJXw
This commit is contained in:
thepraandClaude Opus 5.5 committed
1 parent
5d7edc4a4c
commit
436f7da464
26 files changed
+1159
-62
No files matched your search
@@ -0,0 +1,163 @@
|
||||
using Microsoft.Extensions.Caching.Memory;
|
||||
|
||||
using MongoDB.Entities;
|
||||
|
||||
using PrivaPub.Federation.Objects;
|
||||
using PrivaPub.Infrastructure.Geo;
|
||||
using PrivaPub.Models.Jobs;
|
||||
|
||||
namespace PrivaPub.Domain.Statistics
|
||||
{
|
||||
// Where a server was before it went behind a CDN, as its weekly snapshots last saw it.
|
||||
public sealed record PlaceBeforeCdn(string Week, string Country, string City, double? Latitude, double? Longitude);
|
||||
|
||||
// A CDN as a group of servers: those behind it now, and week by week how many were behind it, who joined and who left.
|
||||
public sealed record CdnGroup(string Key, string Name, string Domain, int Servers, int Ranges, DateTime? RangesUpdatedAt, List<string> DetectedBy);
|
||||
|
||||
public sealed record CdnWeek(string Week, int Servers, List<string> Joined, List<string> Left);
|
||||
|
||||
public sealed record CdnHistory(CdnGroup Group, List<string> Hosts, List<CdnWeek> Weeks);
|
||||
|
||||
public interface IServerPlaces
|
||||
{
|
||||
Task<Dictionary<string, PlaceBeforeCdn>> BeforeCdn(IEnumerable<string> hosts, CancellationToken token);
|
||||
Task<List<RemoteInstanceSnapshot>> History(string host, int weeks, CancellationToken token);
|
||||
Task<List<CdnGroup>> Cdns(CancellationToken token);
|
||||
Task<CdnHistory> Cdn(string key, int weeks, CancellationToken token);
|
||||
}
|
||||
|
||||
// Servers through time (owner decision 2026-10-04): each server's weekly snapshots (place, CDN, size), and the servers
|
||||
// grouped by the CDN in front of them, named by the CDN's own domain. Read from the weekly snapshots InstanceDescriber
|
||||
// writes; the CDN pages are kept for half an hour, because a large CDN reads many snapshots.
|
||||
public class ServerPlaces : IServerPlaces
|
||||
{
|
||||
public const int MaxHosts = 500;
|
||||
const int MaxChanges = 50;
|
||||
static readonly TimeSpan CacheFor = TimeSpan.FromMinutes(30);
|
||||
|
||||
readonly ICdnRanges _ranges;
|
||||
readonly IMemoryCache _cache;
|
||||
|
||||
public ServerPlaces(ICdnRanges ranges, IMemoryCache cache)
|
||||
{
|
||||
_ranges = ranges;
|
||||
_cache = cache;
|
||||
}
|
||||
|
||||
// the CDN key a server's place belongs to: the CDN's domain, or its name for a CDN that has none
|
||||
public static string KeyOf(InstanceGeo geo) =>
|
||||
geo?.Cdn == default ? default : geo.CdnDomain ?? CdnCatalog.ByKey(geo.Cdn)?.Key ?? geo.Cdn.ToLowerInvariant();
|
||||
|
||||
static string KeyOf(RemoteInstanceSnapshot snapshot) =>
|
||||
snapshot.Cdn == default ? default : snapshot.CdnDomain ?? CdnCatalog.ByKey(snapshot.Cdn)?.Key ?? snapshot.Cdn.ToLowerInvariant();
|
||||
|
||||
public async Task<Dictionary<string, PlaceBeforeCdn>> BeforeCdn(IEnumerable<string> hosts, CancellationToken token)
|
||||
{
|
||||
var wanted = hosts.Where(h => h != default).Distinct().ToList();
|
||||
if (wanted.Count == 0)
|
||||
return new();
|
||||
var located = await DB.Default.Find<RemoteInstanceSnapshot>()
|
||||
.Match(s => wanted.Contains(s.Host) && s.Cdn == null && (s.Country != null || s.Latitude != null))
|
||||
.Sort(s => s.Week, Order.Descending)
|
||||
.ExecuteAsync(token);
|
||||
return located.GroupBy(s => s.Host)
|
||||
.Select(g => g.First())
|
||||
.ToDictionary(s => s.Host, s => new PlaceBeforeCdn(s.Week, s.Country, s.City, s.Latitude, s.Longitude));
|
||||
}
|
||||
|
||||
public async Task<List<RemoteInstanceSnapshot>> History(string host, int weeks, CancellationToken token) =>
|
||||
await DB.Default.Find<RemoteInstanceSnapshot>()
|
||||
.Match(s => s.Host == host)
|
||||
.Sort(s => s.Week, Order.Descending)
|
||||
.Limit(weeks)
|
||||
.ExecuteAsync(token);
|
||||
|
||||
public async Task<List<CdnGroup>> Cdns(CancellationToken token)
|
||||
{
|
||||
var fronted = await Fronted(token);
|
||||
var status = _ranges.Status;
|
||||
var keys = CdnCatalog.All.Select(p => p.Key).Concat(fronted.Select(i => KeyOf(i.Geo))).Distinct(StringComparer.OrdinalIgnoreCase);
|
||||
return keys.Select(key => Group(key, fronted, status)).OrderByDescending(g => g.Servers).ThenBy(g => g.Name).ToList();
|
||||
}
|
||||
|
||||
public async Task<CdnHistory> Cdn(string key, int weeks, CancellationToken token)
|
||||
{
|
||||
key = key?.Trim().ToLowerInvariant();
|
||||
if (string.IsNullOrEmpty(key) || key.Length > 100)
|
||||
return default;
|
||||
return await _cache.GetOrCreateAsync($"cdn-history|{key}|{weeks}", async entry =>
|
||||
{
|
||||
entry.AbsoluteExpirationRelativeToNow = CacheFor;
|
||||
var fronted = await Fronted(token);
|
||||
var provider = CdnCatalog.ByKey(key);
|
||||
if (provider == default && fronted.All(i => KeyOf(i.Geo) != key))
|
||||
return default;
|
||||
var group = Group(provider?.Key ?? key, fronted, _ranges.Status);
|
||||
var hosts = fronted.Where(i => KeyOf(i.Geo) == group.Key).Select(i => i.Host).OrderBy(h => h).Take(MaxHosts).ToList();
|
||||
return new CdnHistory(group, hosts, await Weeks(group, weeks, token));
|
||||
});
|
||||
}
|
||||
|
||||
async Task<List<CdnWeek>> Weeks(CdnGroup group, int weeks, CancellationToken token)
|
||||
{
|
||||
var since = InstanceDescriber.Week(DateTime.UtcNow.AddDays(-7 * weeks));
|
||||
var everBehind = (await DB.Default.Find<RemoteInstanceSnapshot>()
|
||||
.Match(f => f.Gte(s => s.Week, since) & (f.Eq(s => s.CdnDomain, group.Key) | f.Eq(s => s.Cdn, group.Name)))
|
||||
.Project(s => new RemoteInstanceSnapshot { Host = s.Host })
|
||||
.ExecuteAsync(token))
|
||||
.Select(s => s.Host).Distinct().ToList();
|
||||
if (everBehind.Count == 0)
|
||||
return new();
|
||||
|
||||
var snapshots = await DB.Default.Find<RemoteInstanceSnapshot>()
|
||||
.Match(f => f.In(s => s.Host, everBehind) & f.Gte(s => s.Week, since))
|
||||
.Project(s => new RemoteInstanceSnapshot { Host = s.Host, Week = s.Week, Cdn = s.Cdn, CdnDomain = s.CdnDomain })
|
||||
.ExecuteAsync(token);
|
||||
|
||||
// a server is behind the CDN from the week a snapshot shows it there until one shows it elsewhere
|
||||
var behind = new Dictionary<string, bool>();
|
||||
var history = new List<CdnWeek>();
|
||||
foreach (var week in snapshots.GroupBy(s => s.Week).OrderBy(w => w.Key, StringComparer.Ordinal))
|
||||
{
|
||||
var joined = new List<string>();
|
||||
var left = new List<string>();
|
||||
foreach (var snapshot in week)
|
||||
{
|
||||
var now = KeyOf(snapshot) == group.Key;
|
||||
var before = behind.GetValueOrDefault(snapshot.Host);
|
||||
if (now && !before)
|
||||
joined.Add(snapshot.Host);
|
||||
else if (!now && before)
|
||||
left.Add(snapshot.Host);
|
||||
behind[snapshot.Host] = now;
|
||||
}
|
||||
history.Add(new CdnWeek(week.Key, behind.Count(b => b.Value), joined.OrderBy(h => h).Take(MaxChanges).ToList(),
|
||||
left.OrderBy(h => h).Take(MaxChanges).ToList()));
|
||||
}
|
||||
history.Reverse();
|
||||
return history;
|
||||
}
|
||||
|
||||
static async Task<List<RemoteInstance>> Fronted(CancellationToken token) =>
|
||||
await DB.Default.Find<RemoteInstance>()
|
||||
.Match(i => i.Geo.Cdn != null)
|
||||
.Project(i => new RemoteInstance { Host = i.Host, Geo = i.Geo })
|
||||
.ExecuteAsync(token);
|
||||
|
||||
static CdnGroup Group(string key, List<RemoteInstance> fronted, IReadOnlyDictionary<string, (int Ranges, DateTime FetchedAt)> status)
|
||||
{
|
||||
var provider = CdnCatalog.ByKey(key);
|
||||
var servers = fronted.Where(i => KeyOf(i.Geo) == key).ToList();
|
||||
var detectedBy = new List<string>();
|
||||
if (provider?.Lists.Length > 0)
|
||||
detectedBy.Add("ranges");
|
||||
if (provider != default)
|
||||
detectedBy.Add("headers");
|
||||
if (provider?.Asns.Length > 0)
|
||||
detectedBy.Add("asn");
|
||||
var ranges = status.TryGetValue(key, out var known) ? known : default;
|
||||
return new CdnGroup(key, provider?.Name ?? servers.Select(i => i.Geo.Cdn).FirstOrDefault() ?? key, provider?.Domain, servers.Count, ranges.Ranges,
|
||||
ranges.Ranges > 0 ? ranges.FetchedAt : null, detectedBy);
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user