CDNs found by themselves, and servers followed through time
PrivaPub now finds CDNs three ways, best first: the address ranges the CDNs publish (Cloudflare, Fastly, Amazon CloudFront, Bunny, Gcore, Imperva), downloaded daily by CdnUpdater and kept in CdnRangeSet; the CDN's fingerprint in the responses it already gets from a server (EdgeHintsHandler on the federation client); and the networks that carry only a CDN. The fixed ASN list is gone; ASNs shared with plain hosting (AWS, DataPacket) no longer hide a server. A server's Geo records the CDN, its domain and how it was found, and weekly snapshots now keep the city and coordinates too. Servers through time (ServerPlaces): /instances/:host/history lists a server's weekly snapshots, a CDN-fronted server's geo names the CDN's domain and where the server was before it (before_cdn), and /api/privapub/v1/cdns and /cdns/:domain group servers by CDN with week by week who joined and who left. Owner decisions recorded in ROADMAP. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01LsXgEaXee4GCU1hwYgPJXw
This commit is contained in:
1 parent
5d7edc4a4c
commit
436f7da464
26 files changed
+1159
-62
No files matched your search
@@ -0,0 +1,82 @@
|
||||
using Microsoft.AspNetCore.Authorization;
|
||||
using Microsoft.AspNetCore.Mvc;
|
||||
|
||||
using PrivaPub.Api.Mastodon.Infrastructure;
|
||||
using PrivaPub.Domain.Statistics;
|
||||
|
||||
namespace PrivaPub.Api.Mastodon.Controllers
|
||||
{
|
||||
// The CDNs in front of the servers this server knows, each named by its own domain (owner decision 2026-10-04): how many
|
||||
// servers are behind it, how it is recognised, and week by week who joined and who left.
|
||||
public class CdnsController : MastodonController
|
||||
{
|
||||
const int MaxWeeks = 260;
|
||||
|
||||
readonly IServerPlaces _places;
|
||||
|
||||
public CdnsController(IServerPlaces places) => _places = places;
|
||||
|
||||
[HttpGet("/api/privapub/v1/cdns"), Scope("read", requiresUser: false), AllowAnonymous]
|
||||
public async Task<IActionResult> List(CancellationToken token) => Json((await _places.Cdns(token)).Select(Summary).ToList());
|
||||
|
||||
[HttpGet("/api/privapub/v1/cdns/{key}"), Scope("read", requiresUser: false), AllowAnonymous]
|
||||
public async Task<IActionResult> Detail(string key, CancellationToken token)
|
||||
{
|
||||
var history = await _places.Cdn(key, Math.Clamp(Params.Int("weeks") ?? 52, 1, MaxWeeks), token);
|
||||
if (history == default)
|
||||
return NotFoundError();
|
||||
var detail = new CdnDetail
|
||||
{
|
||||
Hosts = history.Hosts,
|
||||
HostsTruncated = history.Group.Servers > history.Hosts.Count,
|
||||
Weeks = history.Weeks.Select(w => new CdnWeekEntry { Week = w.Week, Servers = w.Servers, Joined = w.Joined, Left = w.Left }).ToList()
|
||||
};
|
||||
Fill(detail, history.Group);
|
||||
return Json(detail);
|
||||
}
|
||||
|
||||
static CdnSummary Summary(CdnGroup group)
|
||||
{
|
||||
var summary = new CdnSummary();
|
||||
Fill(summary, group);
|
||||
return summary;
|
||||
}
|
||||
|
||||
static void Fill(CdnSummary summary, CdnGroup group)
|
||||
{
|
||||
summary.Key = group.Key;
|
||||
summary.Name = group.Name;
|
||||
summary.Domain = group.Domain;
|
||||
summary.Servers = group.Servers;
|
||||
summary.Ranges = group.Ranges;
|
||||
summary.RangesUpdatedAt = group.RangesUpdatedAt is { } at ? MastodonJson.Time(at) : default;
|
||||
summary.DetectedBy = group.DetectedBy;
|
||||
}
|
||||
|
||||
public class CdnSummary
|
||||
{
|
||||
public string Key { get; set; }
|
||||
public string Name { get; set; }
|
||||
public string Domain { get; set; }
|
||||
public int Servers { get; set; }
|
||||
public int Ranges { get; set; }
|
||||
public string RangesUpdatedAt { get; set; }
|
||||
public List<string> DetectedBy { get; set; } = new();
|
||||
}
|
||||
|
||||
public class CdnDetail : CdnSummary
|
||||
{
|
||||
public List<string> Hosts { get; set; } = new();
|
||||
public bool HostsTruncated { get; set; }
|
||||
public List<CdnWeekEntry> Weeks { get; set; } = new();
|
||||
}
|
||||
|
||||
public class CdnWeekEntry
|
||||
{
|
||||
public string Week { get; set; }
|
||||
public int Servers { get; set; }
|
||||
public List<string> Joined { get; set; } = new();
|
||||
public List<string> Left { get; set; } = new();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -10,6 +10,7 @@ using PrivaPub.Domain.Statistics;
|
||||
using PrivaPub.Federation.Actors;
|
||||
using PrivaPub.Federation.Objects;
|
||||
using PrivaPub.Infrastructure;
|
||||
using PrivaPub.Infrastructure.Geo;
|
||||
using PrivaPub.Models.Federation;
|
||||
using PrivaPub.Models.Jobs;
|
||||
using PrivaPub.Models.Post;
|
||||
@@ -26,15 +27,19 @@ namespace PrivaPub.Api.Mastodon.Controllers
|
||||
const int MaxHosts = 40;
|
||||
|
||||
readonly DbEntities _dbEntities;
|
||||
const int MaxWeeks = 260;
|
||||
|
||||
readonly ISelfLocation _self;
|
||||
readonly IServerPlaces _places;
|
||||
readonly ILocalActorService _localActors;
|
||||
readonly IOptionsMonitor<RegistrationOptions> _registrations;
|
||||
|
||||
public ProvenanceController(DbEntities dbEntities, ISelfLocation self, ILocalActorService localActors,
|
||||
public ProvenanceController(DbEntities dbEntities, ISelfLocation self, IServerPlaces places, ILocalActorService localActors,
|
||||
IOptionsMonitor<RegistrationOptions> registrations)
|
||||
{
|
||||
_dbEntities = dbEntities;
|
||||
_self = self;
|
||||
_places = places;
|
||||
_localActors = localActors;
|
||||
_registrations = registrations;
|
||||
}
|
||||
@@ -54,7 +59,8 @@ namespace PrivaPub.Api.Mastodon.Controllers
|
||||
?? await DB.Default.Find<ObjectRecord>().Match(r => r.ObjectURI == post.ObjectURI).ExecuteFirstAsync(token);
|
||||
var host = record?.Host ?? (Uri.TryCreate(post.ObjectURI, UriKind.Absolute, out var uri) ? uri.Host.ToLowerInvariant() : default);
|
||||
var instance = host == default ? default : await DB.Default.Find<RemoteInstance>().Match(i => i.Host == host).ExecuteFirstAsync(token);
|
||||
return Json(Describe(post, record, instance));
|
||||
var beforeCdn = instance?.Geo?.Cdn == default ? default : (await _places.BeforeCdn(new[] { instance.Host }, token)).GetValueOrDefault(instance.Host);
|
||||
return Json(Describe(post, record, instance, beforeCdn));
|
||||
}
|
||||
|
||||
[HttpGet("/api/privapub/v1/instances/{host}"), Scope("read", requiresUser: false), AllowAnonymous]
|
||||
@@ -68,19 +74,47 @@ namespace PrivaPub.Api.Mastodon.Controllers
|
||||
[HttpGet("/api/privapub/v1/instances"), Scope("read", requiresUser: false), AllowAnonymous]
|
||||
public async Task<IActionResult> Instances(CancellationToken token) => Json(await Describe(Params.List("host"), token));
|
||||
|
||||
//the server's weekly snapshots, newest first (weeks=, at most MaxWeeks): its place as shown publicly, its CDN, its size
|
||||
[HttpGet("/api/privapub/v1/instances/{host}/history"), Scope("read", requiresUser: false), AllowAnonymous]
|
||||
public async Task<IActionResult> History(string host, CancellationToken token)
|
||||
{
|
||||
host = host?.Trim().ToLowerInvariant();
|
||||
if (string.IsNullOrEmpty(host) || host.Length > 253 || !await DB.Default.Find<RemoteInstance>().Match(i => i.Host == host).ExecuteAnyAsync(token))
|
||||
return NotFoundError();
|
||||
var weeks = Math.Clamp(Params.Int("weeks") ?? 52, 1, MaxWeeks);
|
||||
return Json((await _places.History(host, weeks, token)).Select(s => new InstanceWeek
|
||||
{
|
||||
Week = s.Week,
|
||||
TakenAt = MastodonJson.Time(s.TakenAt),
|
||||
Reachable = s.Reachable,
|
||||
Software = s.Software,
|
||||
Version = s.SoftwareVersion,
|
||||
UsersTotal = s.UsersTotal,
|
||||
UsersActiveMonth = s.UsersActiveMonth,
|
||||
LocalPosts = s.LocalPosts,
|
||||
OpenRegistrations = s.OpenRegistrations,
|
||||
Geo = Located(PublicGeo.Project(new InstanceGeo
|
||||
{
|
||||
Country = s.Country, City = s.City, Latitude = s.Latitude, Longitude = s.Longitude, Asn = s.Asn,
|
||||
Cdn = s.Cdn, CdnDomain = s.CdnDomain, CdnSource = s.CdnSource, Ipv6 = s.Ipv6
|
||||
}), new InstanceGeo { Cdn = s.Cdn, CdnDomain = s.CdnDomain ?? CdnCatalog.ByKey(s.Cdn)?.Key, CdnSource = s.CdnSource })
|
||||
}).ToList());
|
||||
}
|
||||
|
||||
async Task<List<InstanceDescription>> Describe(IEnumerable<string> hosts, CancellationToken token)
|
||||
{
|
||||
var wanted = hosts.Select(h => h?.Trim().ToLowerInvariant()).Where(h => !string.IsNullOrEmpty(h) && h.Length <= 253)
|
||||
.Distinct().Take(MaxHosts).ToList();
|
||||
var selfHost = Uri.TryCreate(_localActors.BaseAddress, UriKind.Absolute, out var self) ? self.Host.ToLowerInvariant() : default;
|
||||
var remote = await DB.Default.Find<RemoteInstance>().Match(i => wanted.Contains(i.Host)).ExecuteAsync(token);
|
||||
var beforeCdn = await _places.BeforeCdn(remote.Where(i => i.Geo?.Cdn != default).Select(i => i.Host), token);
|
||||
var described = new List<InstanceDescription>();
|
||||
foreach (var host in wanted)
|
||||
{
|
||||
if (host == selfHost)
|
||||
described.Add(await DescribeSelf(host, token));
|
||||
else if (remote.FirstOrDefault(i => i.Host == host) is { } instance)
|
||||
described.Add(Describe(instance));
|
||||
described.Add(Describe(instance, beforeCdn.GetValueOrDefault(host)));
|
||||
}
|
||||
return described;
|
||||
}
|
||||
@@ -101,8 +135,9 @@ namespace PrivaPub.Api.Mastodon.Controllers
|
||||
};
|
||||
}
|
||||
|
||||
//the public projection of a server's place (PublicGeo)
|
||||
static InstanceLocation Located(PublicLocation location, InstanceGeo geo) => location == default ? default : new InstanceLocation
|
||||
//the public projection of a server's place (PublicGeo); for a CDN-fronted one, the CDN's domain, how it was found and
|
||||
//where the server was before it
|
||||
static InstanceLocation Located(PublicLocation location, InstanceGeo geo, PlaceBeforeCdn beforeCdn = default) => location == default ? default : new InstanceLocation
|
||||
{
|
||||
Precision = location.Cdn != default ? "cdn" : location.Latitude != default ? "city" : "country",
|
||||
Country = location.Country,
|
||||
@@ -111,11 +146,17 @@ namespace PrivaPub.Api.Mastodon.Controllers
|
||||
Longitude = location.Longitude,
|
||||
Network = location.Network,
|
||||
Cdn = location.Cdn,
|
||||
CdnDomain = location.Cdn != default ? ServerPlaces.KeyOf(geo) : default,
|
||||
CdnSource = location.Cdn != default ? geo?.CdnSource : default,
|
||||
BeforeCdn = location.Cdn == default || beforeCdn == default ? default : new InstancePlace
|
||||
{
|
||||
Week = beforeCdn.Week, Country = beforeCdn.Country, City = beforeCdn.City, Latitude = beforeCdn.Latitude, Longitude = beforeCdn.Longitude
|
||||
},
|
||||
Source = geo?.Source,
|
||||
LocatedAt = geo?.LocatedAt is { } at ? MastodonJson.Time(at) : default
|
||||
};
|
||||
|
||||
Provenance Describe(PostEntity post, ObjectRecord record, RemoteInstance instance)
|
||||
Provenance Describe(PostEntity post, ObjectRecord record, RemoteInstance instance, PlaceBeforeCdn beforeCdn)
|
||||
{
|
||||
var raw = record?.Raw == default ? default : JsonNode.Parse(record.Raw) as JsonObject;
|
||||
return new Provenance
|
||||
@@ -152,11 +193,11 @@ namespace PrivaPub.Api.Mastodon.Controllers
|
||||
RawHash = r.RawHash,
|
||||
Raw = r.Raw == default ? default : JsonNode.Parse(r.Raw)
|
||||
}).ToList() ?? new(),
|
||||
Instance = instance == default ? default : Describe(instance)
|
||||
Instance = instance == default ? default : Describe(instance, beforeCdn)
|
||||
};
|
||||
}
|
||||
|
||||
static InstanceDescription Describe(RemoteInstance instance) => new()
|
||||
static InstanceDescription Describe(RemoteInstance instance, PlaceBeforeCdn beforeCdn) => new()
|
||||
{
|
||||
Host = instance.Host,
|
||||
Software = instance.Software,
|
||||
@@ -174,7 +215,7 @@ namespace PrivaPub.Api.Mastodon.Controllers
|
||||
LastSuccessAt = instance.LastSuccessAt is { } success ? MastodonJson.Time(success) : default,
|
||||
LastFailureAt = instance.LastFailureAt is { } failure ? MastodonJson.Time(failure) : default
|
||||
},
|
||||
Geo = Located(PublicGeo.Project(instance.Geo), instance.Geo)
|
||||
Geo = Located(PublicGeo.Project(instance.Geo), instance.Geo, beforeCdn)
|
||||
};
|
||||
|
||||
public class Provenance
|
||||
@@ -251,10 +292,36 @@ namespace PrivaPub.Api.Mastodon.Controllers
|
||||
public double? Longitude { get; set; }
|
||||
public string Network { get; set; }
|
||||
public string Cdn { get; set; }
|
||||
public string CdnDomain { get; set; }
|
||||
public string CdnSource { get; set; }//"ranges", "headers" or "asn"
|
||||
public InstancePlace BeforeCdn { get; set; }//where a CDN-fronted server was located before the CDN, by its snapshots
|
||||
public string Source { get; set; }
|
||||
public string LocatedAt { get; set; }
|
||||
}
|
||||
|
||||
public class InstancePlace
|
||||
{
|
||||
public string Week { get; set; }
|
||||
public string Country { get; set; }
|
||||
public string City { get; set; }
|
||||
public double? Latitude { get; set; }
|
||||
public double? Longitude { get; set; }
|
||||
}
|
||||
|
||||
public class InstanceWeek
|
||||
{
|
||||
public string Week { get; set; }
|
||||
public string TakenAt { get; set; }
|
||||
public bool Reachable { get; set; }
|
||||
public string Software { get; set; }
|
||||
public string Version { get; set; }
|
||||
public long? UsersTotal { get; set; }
|
||||
public long? UsersActiveMonth { get; set; }
|
||||
public long? LocalPosts { get; set; }
|
||||
public bool? OpenRegistrations { get; set; }
|
||||
public InstanceLocation Geo { get; set; }
|
||||
}
|
||||
|
||||
public class InstanceDelivery
|
||||
{
|
||||
public int ConsecutiveFailures { get; set; }
|
||||
|
||||
@@ -24,6 +24,7 @@ namespace PrivaPub.Domain.Statistics
|
||||
static readonly TimeSpan Lifetime = TimeSpan.FromDays(1);
|
||||
|
||||
readonly IGeoLocator _geo;
|
||||
readonly ICdnRanges _cdnRanges;
|
||||
readonly IOptionsMonitor<StatisticsOptions> _options;
|
||||
readonly IOptionsMonitor<AppConfiguration> _app;
|
||||
readonly ILogger<SelfLocation> _logger;
|
||||
@@ -32,9 +33,11 @@ namespace PrivaPub.Domain.Statistics
|
||||
InstanceGeo _located;
|
||||
DateTime _locatedAt = DateTime.MinValue;
|
||||
|
||||
public SelfLocation(IGeoLocator geo, IOptionsMonitor<StatisticsOptions> options, IOptionsMonitor<AppConfiguration> app, ILogger<SelfLocation> logger)
|
||||
public SelfLocation(IGeoLocator geo, IOptionsMonitor<StatisticsOptions> options, IOptionsMonitor<AppConfiguration> app, ILogger<SelfLocation> logger,
|
||||
ICdnRanges cdnRanges = default)
|
||||
{
|
||||
_geo = geo;
|
||||
_cdnRanges = cdnRanges;
|
||||
_options = options;
|
||||
_app = app;
|
||||
_logger = logger;
|
||||
@@ -79,6 +82,7 @@ namespace PrivaPub.Domain.Statistics
|
||||
var addresses = await Dns.GetHostAddressesAsync(self.Host, token);
|
||||
var address = addresses.FirstOrDefault(a => a.AddressFamily == AddressFamily.InterNetwork) ?? addresses.FirstOrDefault();
|
||||
var fix = address == default ? default : _geo.Locate(address);
|
||||
var (cdn, cdnSource) = Federation.Objects.InstanceDescriber.Cdn(address, self.Host, fix?.Asn, _cdnRanges, default);
|
||||
return fix == default ? default : new InstanceGeo
|
||||
{
|
||||
Country = fix.Country,
|
||||
@@ -87,7 +91,9 @@ namespace PrivaPub.Domain.Statistics
|
||||
Longitude = fix.Longitude,
|
||||
Asn = fix.Asn,
|
||||
AsnOrg = fix.AsnOrg,
|
||||
Cdn = CdnNetworks.Of(fix.Asn),
|
||||
Cdn = cdn?.Name,
|
||||
CdnDomain = cdn?.Domain,
|
||||
CdnSource = cdnSource,
|
||||
Ipv6 = address.AddressFamily == AddressFamily.InterNetworkV6,
|
||||
Source = _geo.Source,
|
||||
LocatedAt = DateTime.UtcNow
|
||||
|
||||
@@ -0,0 +1,163 @@
|
||||
using Microsoft.Extensions.Caching.Memory;
|
||||
|
||||
using MongoDB.Entities;
|
||||
|
||||
using PrivaPub.Federation.Objects;
|
||||
using PrivaPub.Infrastructure.Geo;
|
||||
using PrivaPub.Models.Jobs;
|
||||
|
||||
namespace PrivaPub.Domain.Statistics
|
||||
{
|
||||
// Where a server was before it went behind a CDN, as its weekly snapshots last saw it.
|
||||
public sealed record PlaceBeforeCdn(string Week, string Country, string City, double? Latitude, double? Longitude);
|
||||
|
||||
// A CDN as a group of servers: those behind it now, and week by week how many were behind it, who joined and who left.
|
||||
public sealed record CdnGroup(string Key, string Name, string Domain, int Servers, int Ranges, DateTime? RangesUpdatedAt, List<string> DetectedBy);
|
||||
|
||||
public sealed record CdnWeek(string Week, int Servers, List<string> Joined, List<string> Left);
|
||||
|
||||
public sealed record CdnHistory(CdnGroup Group, List<string> Hosts, List<CdnWeek> Weeks);
|
||||
|
||||
public interface IServerPlaces
|
||||
{
|
||||
Task<Dictionary<string, PlaceBeforeCdn>> BeforeCdn(IEnumerable<string> hosts, CancellationToken token);
|
||||
Task<List<RemoteInstanceSnapshot>> History(string host, int weeks, CancellationToken token);
|
||||
Task<List<CdnGroup>> Cdns(CancellationToken token);
|
||||
Task<CdnHistory> Cdn(string key, int weeks, CancellationToken token);
|
||||
}
|
||||
|
||||
// Servers through time (owner decision 2026-10-04): each server's weekly snapshots (place, CDN, size), and the servers
|
||||
// grouped by the CDN in front of them, named by the CDN's own domain. Read from the weekly snapshots InstanceDescriber
|
||||
// writes; the CDN pages are kept for half an hour, because a large CDN reads many snapshots.
|
||||
public class ServerPlaces : IServerPlaces
|
||||
{
|
||||
public const int MaxHosts = 500;
|
||||
const int MaxChanges = 50;
|
||||
static readonly TimeSpan CacheFor = TimeSpan.FromMinutes(30);
|
||||
|
||||
readonly ICdnRanges _ranges;
|
||||
readonly IMemoryCache _cache;
|
||||
|
||||
public ServerPlaces(ICdnRanges ranges, IMemoryCache cache)
|
||||
{
|
||||
_ranges = ranges;
|
||||
_cache = cache;
|
||||
}
|
||||
|
||||
// the CDN key a server's place belongs to: the CDN's domain, or its name for a CDN that has none
|
||||
public static string KeyOf(InstanceGeo geo) =>
|
||||
geo?.Cdn == default ? default : geo.CdnDomain ?? CdnCatalog.ByKey(geo.Cdn)?.Key ?? geo.Cdn.ToLowerInvariant();
|
||||
|
||||
static string KeyOf(RemoteInstanceSnapshot snapshot) =>
|
||||
snapshot.Cdn == default ? default : snapshot.CdnDomain ?? CdnCatalog.ByKey(snapshot.Cdn)?.Key ?? snapshot.Cdn.ToLowerInvariant();
|
||||
|
||||
public async Task<Dictionary<string, PlaceBeforeCdn>> BeforeCdn(IEnumerable<string> hosts, CancellationToken token)
|
||||
{
|
||||
var wanted = hosts.Where(h => h != default).Distinct().ToList();
|
||||
if (wanted.Count == 0)
|
||||
return new();
|
||||
var located = await DB.Default.Find<RemoteInstanceSnapshot>()
|
||||
.Match(s => wanted.Contains(s.Host) && s.Cdn == null && (s.Country != null || s.Latitude != null))
|
||||
.Sort(s => s.Week, Order.Descending)
|
||||
.ExecuteAsync(token);
|
||||
return located.GroupBy(s => s.Host)
|
||||
.Select(g => g.First())
|
||||
.ToDictionary(s => s.Host, s => new PlaceBeforeCdn(s.Week, s.Country, s.City, s.Latitude, s.Longitude));
|
||||
}
|
||||
|
||||
public async Task<List<RemoteInstanceSnapshot>> History(string host, int weeks, CancellationToken token) =>
|
||||
await DB.Default.Find<RemoteInstanceSnapshot>()
|
||||
.Match(s => s.Host == host)
|
||||
.Sort(s => s.Week, Order.Descending)
|
||||
.Limit(weeks)
|
||||
.ExecuteAsync(token);
|
||||
|
||||
public async Task<List<CdnGroup>> Cdns(CancellationToken token)
|
||||
{
|
||||
var fronted = await Fronted(token);
|
||||
var status = _ranges.Status;
|
||||
var keys = CdnCatalog.All.Select(p => p.Key).Concat(fronted.Select(i => KeyOf(i.Geo))).Distinct(StringComparer.OrdinalIgnoreCase);
|
||||
return keys.Select(key => Group(key, fronted, status)).OrderByDescending(g => g.Servers).ThenBy(g => g.Name).ToList();
|
||||
}
|
||||
|
||||
public async Task<CdnHistory> Cdn(string key, int weeks, CancellationToken token)
|
||||
{
|
||||
key = key?.Trim().ToLowerInvariant();
|
||||
if (string.IsNullOrEmpty(key) || key.Length > 100)
|
||||
return default;
|
||||
return await _cache.GetOrCreateAsync($"cdn-history|{key}|{weeks}", async entry =>
|
||||
{
|
||||
entry.AbsoluteExpirationRelativeToNow = CacheFor;
|
||||
var fronted = await Fronted(token);
|
||||
var provider = CdnCatalog.ByKey(key);
|
||||
if (provider == default && fronted.All(i => KeyOf(i.Geo) != key))
|
||||
return default;
|
||||
var group = Group(provider?.Key ?? key, fronted, _ranges.Status);
|
||||
var hosts = fronted.Where(i => KeyOf(i.Geo) == group.Key).Select(i => i.Host).OrderBy(h => h).Take(MaxHosts).ToList();
|
||||
return new CdnHistory(group, hosts, await Weeks(group, weeks, token));
|
||||
});
|
||||
}
|
||||
|
||||
async Task<List<CdnWeek>> Weeks(CdnGroup group, int weeks, CancellationToken token)
|
||||
{
|
||||
var since = InstanceDescriber.Week(DateTime.UtcNow.AddDays(-7 * weeks));
|
||||
var everBehind = (await DB.Default.Find<RemoteInstanceSnapshot>()
|
||||
.Match(f => f.Gte(s => s.Week, since) & (f.Eq(s => s.CdnDomain, group.Key) | f.Eq(s => s.Cdn, group.Name)))
|
||||
.Project(s => new RemoteInstanceSnapshot { Host = s.Host })
|
||||
.ExecuteAsync(token))
|
||||
.Select(s => s.Host).Distinct().ToList();
|
||||
if (everBehind.Count == 0)
|
||||
return new();
|
||||
|
||||
var snapshots = await DB.Default.Find<RemoteInstanceSnapshot>()
|
||||
.Match(f => f.In(s => s.Host, everBehind) & f.Gte(s => s.Week, since))
|
||||
.Project(s => new RemoteInstanceSnapshot { Host = s.Host, Week = s.Week, Cdn = s.Cdn, CdnDomain = s.CdnDomain })
|
||||
.ExecuteAsync(token);
|
||||
|
||||
// a server is behind the CDN from the week a snapshot shows it there until one shows it elsewhere
|
||||
var behind = new Dictionary<string, bool>();
|
||||
var history = new List<CdnWeek>();
|
||||
foreach (var week in snapshots.GroupBy(s => s.Week).OrderBy(w => w.Key, StringComparer.Ordinal))
|
||||
{
|
||||
var joined = new List<string>();
|
||||
var left = new List<string>();
|
||||
foreach (var snapshot in week)
|
||||
{
|
||||
var now = KeyOf(snapshot) == group.Key;
|
||||
var before = behind.GetValueOrDefault(snapshot.Host);
|
||||
if (now && !before)
|
||||
joined.Add(snapshot.Host);
|
||||
else if (!now && before)
|
||||
left.Add(snapshot.Host);
|
||||
behind[snapshot.Host] = now;
|
||||
}
|
||||
history.Add(new CdnWeek(week.Key, behind.Count(b => b.Value), joined.OrderBy(h => h).Take(MaxChanges).ToList(),
|
||||
left.OrderBy(h => h).Take(MaxChanges).ToList()));
|
||||
}
|
||||
history.Reverse();
|
||||
return history;
|
||||
}
|
||||
|
||||
static async Task<List<RemoteInstance>> Fronted(CancellationToken token) =>
|
||||
await DB.Default.Find<RemoteInstance>()
|
||||
.Match(i => i.Geo.Cdn != null)
|
||||
.Project(i => new RemoteInstance { Host = i.Host, Geo = i.Geo })
|
||||
.ExecuteAsync(token);
|
||||
|
||||
static CdnGroup Group(string key, List<RemoteInstance> fronted, IReadOnlyDictionary<string, (int Ranges, DateTime FetchedAt)> status)
|
||||
{
|
||||
var provider = CdnCatalog.ByKey(key);
|
||||
var servers = fronted.Where(i => KeyOf(i.Geo) == key).ToList();
|
||||
var detectedBy = new List<string>();
|
||||
if (provider?.Lists.Length > 0)
|
||||
detectedBy.Add("ranges");
|
||||
if (provider != default)
|
||||
detectedBy.Add("headers");
|
||||
if (provider?.Asns.Length > 0)
|
||||
detectedBy.Add("asn");
|
||||
var ranges = status.TryGetValue(key, out var known) ? known : default;
|
||||
return new CdnGroup(key, provider?.Name ?? servers.Select(i => i.Geo.Cdn).FirstOrDefault() ?? key, provider?.Domain, servers.Count, ranges.Ranges,
|
||||
ranges.Ranges > 0 ? ranges.FetchedAt : null, detectedBy);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
using MongoDB.Entities;
|
||||
using MongoDB.Entities;
|
||||
|
||||
using PrivaPub.Infrastructure.Geo;
|
||||
using PrivaPub.Infrastructure.Http;
|
||||
@@ -18,12 +18,17 @@ namespace PrivaPub.Federation.Objects
|
||||
readonly IFederationHttp _http;
|
||||
readonly IGeoLocator _geo;
|
||||
readonly IConnectedAddresses _addresses;
|
||||
readonly ICdnRanges _cdnRanges;
|
||||
readonly IEdgeHints _edges;
|
||||
|
||||
public InstanceDescriber(IFederationHttp http, IGeoLocator geo = default, IConnectedAddresses addresses = default)
|
||||
public InstanceDescriber(IFederationHttp http, IGeoLocator geo = default, IConnectedAddresses addresses = default, ICdnRanges cdnRanges = default,
|
||||
IEdgeHints edges = default)
|
||||
{
|
||||
_http = http;
|
||||
_geo = geo;
|
||||
_addresses = addresses;
|
||||
_cdnRanges = cdnRanges;
|
||||
_edges = edges;
|
||||
}
|
||||
|
||||
public JobKind Kind => JobKind.DescribeInstance;
|
||||
@@ -145,35 +150,57 @@ namespace PrivaPub.Federation.Objects
|
||||
.Modify(s => s.RegistrationMode, instance?.RegistrationMode)
|
||||
.Modify(s => s.Languages, instance?.Languages)
|
||||
.Modify(s => s.Country, instance?.Geo?.Country)
|
||||
.Modify(s => s.City, instance?.Geo?.City)
|
||||
.Modify(s => s.Latitude, instance?.Geo?.Latitude)
|
||||
.Modify(s => s.Longitude, instance?.Geo?.Longitude)
|
||||
.Modify(s => s.Asn, instance?.Geo?.Asn)
|
||||
.Modify(s => s.Cdn, instance?.Geo?.Cdn)
|
||||
.Modify(s => s.CdnDomain, instance?.Geo?.CdnDomain)
|
||||
.Modify(s => s.CdnSource, instance?.Geo?.CdnSource)
|
||||
.Modify(s => s.Ipv6, instance?.Geo?.Ipv6)
|
||||
.Option(o => o.IsUpsert = true)
|
||||
.ExecuteAsync(token);
|
||||
return facts != default;
|
||||
}
|
||||
|
||||
// Where the address we reached sits, and the CDN in front of it, found by its published ranges, then by the
|
||||
// fingerprint in the responses we just got, then by a network that carries only a CDN.
|
||||
InstanceGeo Locate(string host)
|
||||
{
|
||||
var reached = _addresses?.Of(new Uri(Address($"https://{host}/")).Host);
|
||||
var reachedHost = new Uri(Address($"https://{host}/")).Host;
|
||||
var reached = _addresses?.Of(reachedHost);
|
||||
var fix = reached == default ? default : _geo?.Locate(reached);
|
||||
if (fix == default)
|
||||
var (cdn, cdnSource) = Cdn(reached, reachedHost, fix?.Asn, _cdnRanges, _edges);
|
||||
if (fix == default && cdn == default)
|
||||
return default;
|
||||
return new InstanceGeo
|
||||
{
|
||||
Country = fix.Country,
|
||||
City = fix.City,
|
||||
Latitude = fix.Latitude,
|
||||
Longitude = fix.Longitude,
|
||||
Asn = fix.Asn,
|
||||
AsnOrg = fix.AsnOrg,
|
||||
Cdn = CdnNetworks.Of(fix.Asn),
|
||||
Ipv6 = reached.AddressFamily == AddressFamily.InterNetworkV6,
|
||||
Source = _geo.Source,
|
||||
Country = fix?.Country,
|
||||
City = fix?.City,
|
||||
Latitude = fix?.Latitude,
|
||||
Longitude = fix?.Longitude,
|
||||
Asn = fix?.Asn,
|
||||
AsnOrg = fix?.AsnOrg,
|
||||
Cdn = cdn?.Name,
|
||||
CdnDomain = cdn?.Domain,
|
||||
CdnSource = cdnSource,
|
||||
Ipv6 = reached?.AddressFamily == AddressFamily.InterNetworkV6,
|
||||
Source = _geo?.Source,
|
||||
LocatedAt = DateTime.UtcNow
|
||||
};
|
||||
}
|
||||
|
||||
public static (CdnProvider Cdn, string Source) Cdn(System.Net.IPAddress address, string host, int? asn, ICdnRanges ranges, IEdgeHints edges)
|
||||
{
|
||||
if (ranges?.Of(address) is { } listed)
|
||||
return (listed, "ranges");
|
||||
if (edges?.Of(host) is { } hinted)
|
||||
return (hinted, "headers");
|
||||
if (CdnCatalog.OfAsn(asn) is { } network)
|
||||
return (network, "asn");
|
||||
return default;
|
||||
}
|
||||
|
||||
//where an https address is fetched from: itself, except in tests, which point it at a plain-http peer
|
||||
protected virtual string Address(string url) => url;
|
||||
}
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
using MongoDB.Bson;
|
||||
using MongoDB.Bson;
|
||||
using MongoDB.Driver;
|
||||
using MongoDB.Entities;
|
||||
|
||||
@@ -131,6 +131,9 @@ namespace PrivaPub.Infrastructure.Data
|
||||
await DB.Default.Index<RemoteInstanceSnapshot>().Key(s => s.Host, KeyType.Ascending).Key(s => s.Week, KeyType.Ascending)
|
||||
.Option(o => o.Unique = true).CreateAsync(token);
|
||||
await Plain<RemoteInstanceSnapshot>(token, s => s.Week);
|
||||
await Plain<RemoteInstanceSnapshot>(token, s => s.CdnDomain, s => s.Week);
|
||||
await Plain<RemoteInstanceSnapshot>(token, s => s.Cdn, s => s.Week);
|
||||
await Plain<CdnRangeSet>(token, s => s.Key);
|
||||
await Plain<RemoteInstance>(token, i => i.Seen);
|
||||
await Plain<RemoteInstance>(token, i => i.LastCrawledAt);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,156 @@
|
||||
using System.Net;
|
||||
using System.Text.Json;
|
||||
|
||||
namespace PrivaPub.Infrastructure.Geo
|
||||
{
|
||||
// How a published list of a CDN's addresses is written.
|
||||
public enum CdnListFormat
|
||||
{
|
||||
Lines,//one CIDR per line (Cloudflare)
|
||||
StringArray,//a JSON array of addresses or CIDRs (Bunny)
|
||||
AddressesJson,//{"addresses": [...], "ipv6_addresses"/"addresses_v6": [...]} (Fastly, Gcore)
|
||||
AwsCloudFront,//AWS ip-ranges.json, CLOUDFRONT entries only
|
||||
ImpervaJson//{"ipRanges": [...], "ipv6Ranges": [...]}
|
||||
}
|
||||
|
||||
public sealed record CdnList(string Url, CdnListFormat Format, string Method = "GET", string Body = default);
|
||||
|
||||
// A CDN, or any edge network whose addresses say nothing about where a server runs. Domain names it in statistics.
|
||||
public sealed record CdnProvider(string Name, string Domain, int[] Asns, CdnList[] Lists, Func<HttpResponseMessage, bool> Fingerprint)
|
||||
{
|
||||
public string Key => Domain ?? Name.ToLowerInvariant();
|
||||
}
|
||||
|
||||
// owner decision (2026-10-04): PrivaPub finds CDNs by itself, three ways, best first. The address ranges a CDN
|
||||
// publishes (downloaded daily by CdnUpdater), the CDN's fingerprint in the responses PrivaPub already gets from a server
|
||||
// (EdgeHintsHandler), and the networks (ASN) that carry nothing but a CDN. A network shared with plain hosting (AWS,
|
||||
// Google, Microsoft, DataPacket) is never listed by ASN: only ranges or fingerprints name those CDNs.
|
||||
public static class CdnCatalog
|
||||
{
|
||||
public static readonly IReadOnlyList<CdnProvider> All = new CdnProvider[]
|
||||
{
|
||||
new("Cloudflare", "cloudflare.com", new[] { 13335, 209242 },
|
||||
new[] { new CdnList("https://www.cloudflare.com/ips-v4", CdnListFormat.Lines), new CdnList("https://www.cloudflare.com/ips-v6", CdnListFormat.Lines) },
|
||||
r => Has(r, "cf-ray") || ServerIs(r, "cloudflare")),
|
||||
new("Fastly", "fastly.com", new[] { 54113 },
|
||||
new[] { new CdnList("https://api.fastly.com/public-ip-list", CdnListFormat.AddressesJson) },
|
||||
r => Has(r, "x-fastly-request-id") || Has(r, "fastly-restarts") || Value(r, "x-served-by")?.StartsWith("cache-", StringComparison.OrdinalIgnoreCase) == true),
|
||||
new("Amazon CloudFront", "cloudfront.net", Array.Empty<int>(),
|
||||
new[] { new CdnList("https://ip-ranges.amazonaws.com/ip-ranges.json", CdnListFormat.AwsCloudFront) },
|
||||
r => Has(r, "x-amz-cf-id") || Has(r, "x-amz-cf-pop") || Value(r, "via")?.Contains("cloudfront", StringComparison.OrdinalIgnoreCase) == true),
|
||||
new("Akamai", "akamai.com", new[] { 20940, 16625, 16702, 21342 }, Array.Empty<CdnList>(),
|
||||
r => Has(r, "akamai-grn") || Has(r, "x-akamai-transformed") || Has(r, "akamai-cache-status") || ServerStarts(r, "AkamaiGHost") || ServerStarts(r, "AkamaiNetStorage")),
|
||||
new("Bunny", "bunny.net", new[] { 200325 },
|
||||
new[] { new CdnList("https://api.bunny.net/system/edgeserverlist", CdnListFormat.StringArray), new CdnList("https://api.bunny.net/system/edgeserverlist/IPv6", CdnListFormat.StringArray) },
|
||||
r => ServerStarts(r, "BunnyCDN") || Has(r, "cdn-pullzone") || Has(r, "cdn-requestid")),
|
||||
new("Gcore", "gcore.com", Array.Empty<int>(),
|
||||
new[] { new CdnList("https://api.gcore.com/cdn/public-ip-list", CdnListFormat.AddressesJson) },
|
||||
r => ServerStarts(r, "gcore")),
|
||||
new("Imperva", "imperva.com", new[] { 19551 },
|
||||
new[] { new CdnList("https://my.imperva.com/api/integration/v1/ips", CdnListFormat.ImpervaJson, "POST", "resp_format=json") },
|
||||
r => Has(r, "x-iinfo") || Value(r, "x-cdn") is "Imperva" or "Incapsula"),
|
||||
new("CDN77", "cdn77.com", Array.Empty<int>(), Array.Empty<CdnList>(),
|
||||
r => Has(r, "x-77-nzt") || Has(r, "x-77-cache") || ServerStarts(r, "CDN77")),
|
||||
new("Edgio", "edg.io", new[] { 15133 }, Array.Empty<CdnList>(),
|
||||
r => ServerStarts(r, "ECAcc") || ServerStarts(r, "ECS (") || Has(r, "x-ec-custom-error")),
|
||||
new("Azure Front Door", "azure.microsoft.com", Array.Empty<int>(), Array.Empty<CdnList>(),
|
||||
r => Has(r, "x-azure-ref")),
|
||||
new("Google Cloud", "cloud.google.com", Array.Empty<int>(), Array.Empty<CdnList>(),
|
||||
r => Value(r, "via")?.Contains("google", StringComparison.OrdinalIgnoreCase) == true),
|
||||
new("Vercel", "vercel.com", Array.Empty<int>(), Array.Empty<CdnList>(),
|
||||
r => Has(r, "x-vercel-id") || ServerIs(r, "Vercel")),
|
||||
new("Netlify", "netlify.com", Array.Empty<int>(), Array.Empty<CdnList>(),
|
||||
r => Has(r, "x-nf-request-id") || ServerIs(r, "Netlify")),
|
||||
new("Sucuri", "sucuri.net", Array.Empty<int>(), Array.Empty<CdnList>(),
|
||||
r => Has(r, "x-sucuri-id") || ServerStarts(r, "Sucuri")),
|
||||
new("DDoS-Guard", "ddos-guard.net", Array.Empty<int>(), Array.Empty<CdnList>(),
|
||||
r => ServerIs(r, "ddos-guard"))
|
||||
};
|
||||
|
||||
public static CdnProvider ByKey(string key) =>
|
||||
key == default ? default : All.FirstOrDefault(p => string.Equals(p.Key, key, StringComparison.OrdinalIgnoreCase) || string.Equals(p.Name, key, StringComparison.OrdinalIgnoreCase));
|
||||
|
||||
public static CdnProvider OfAsn(int? asn) => asn is { } number ? All.FirstOrDefault(p => p.Asns.Contains(number)) : default;
|
||||
|
||||
// The CDN a response passed through: a known fingerprint, or a CDN that names itself in X-CDN.
|
||||
public static CdnProvider FromResponse(HttpResponseMessage response)
|
||||
{
|
||||
if (response == default)
|
||||
return default;
|
||||
var known = All.FirstOrDefault(p => p.Fingerprint(response));
|
||||
if (known != default)
|
||||
return known;
|
||||
var named = Value(response, "x-cdn")?.Trim();
|
||||
return named is { Length: > 1 and <= 40 } && named.All(c => char.IsLetterOrDigit(c) || c is ' ' or '-' or '.' or '_')
|
||||
? new CdnProvider(named, default, Array.Empty<int>(), Array.Empty<CdnList>(), _ => false)
|
||||
: default;
|
||||
}
|
||||
|
||||
// The networks a published list holds; an entry that is not an address or a network is an error, never skipped.
|
||||
public static List<IPNetwork> Parse(CdnListFormat format, string body)
|
||||
{
|
||||
IEnumerable<string> entries = format switch
|
||||
{
|
||||
CdnListFormat.Lines => body.Split('\n', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries).Where(l => !l.StartsWith('#')),
|
||||
CdnListFormat.StringArray => Root(body),
|
||||
CdnListFormat.AddressesJson => Json(body, "addresses", "ipv6_addresses", "addresses_v6"),
|
||||
CdnListFormat.ImpervaJson => Json(body, "ipRanges", "ipv6Ranges"),
|
||||
CdnListFormat.AwsCloudFront => CloudFront(body),
|
||||
_ => Enumerable.Empty<string>()
|
||||
};
|
||||
return entries.Select(Network).ToList();
|
||||
}
|
||||
|
||||
static IPNetwork Network(string entry)
|
||||
{
|
||||
if (IPNetwork.TryParse(entry, out var network))
|
||||
return network;
|
||||
if (IPAddress.TryParse(entry, out var address))
|
||||
return new IPNetwork(address, address.AddressFamily == System.Net.Sockets.AddressFamily.InterNetwork ? 32 : 128);
|
||||
throw new FormatException($"not an address or a network: {entry}");
|
||||
}
|
||||
|
||||
static List<string> Root(string body)
|
||||
{
|
||||
using var document = JsonDocument.Parse(body);
|
||||
return Strings(document.RootElement);
|
||||
}
|
||||
|
||||
static IEnumerable<string> Json(string body, params string[] properties)
|
||||
{
|
||||
using var document = JsonDocument.Parse(body);
|
||||
var entries = new List<string>();
|
||||
foreach (var property in properties)
|
||||
if (document.RootElement.TryGetProperty(property, out var array))
|
||||
entries.AddRange(Strings(array));
|
||||
return entries;
|
||||
}
|
||||
|
||||
static IEnumerable<string> CloudFront(string body)
|
||||
{
|
||||
using var document = JsonDocument.Parse(body);
|
||||
var entries = new List<string>();
|
||||
foreach (var (array, field) in new[] { ("prefixes", "ip_prefix"), ("ipv6_prefixes", "ipv6_prefix") })
|
||||
if (document.RootElement.TryGetProperty(array, out var prefixes))
|
||||
foreach (var prefix in prefixes.EnumerateArray())
|
||||
if (prefix.TryGetProperty("service", out var service) && service.GetString() == "CLOUDFRONT" && prefix.TryGetProperty(field, out var value))
|
||||
entries.Add(value.GetString());
|
||||
return entries;
|
||||
}
|
||||
|
||||
static List<string> Strings(JsonElement array) =>
|
||||
array.ValueKind == JsonValueKind.Array ? array.EnumerateArray().Select(e => e.GetString()).ToList() : throw new FormatException("not a list");
|
||||
|
||||
static bool Has(HttpResponseMessage response, string header) =>
|
||||
response.Headers.Contains(header) || response.Content?.Headers.Contains(header) == true;
|
||||
|
||||
static string Value(HttpResponseMessage response, string header) =>
|
||||
response.Headers.TryGetValues(header, out var values) || response.Content?.Headers.TryGetValues(header, out values) == true
|
||||
? string.Join(", ", values)
|
||||
: default;
|
||||
|
||||
static bool ServerIs(HttpResponseMessage response, string name) => string.Equals(Value(response, "server")?.Trim(), name, StringComparison.OrdinalIgnoreCase);
|
||||
|
||||
static bool ServerStarts(HttpResponseMessage response, string prefix) => Value(response, "server")?.TrimStart().StartsWith(prefix, StringComparison.OrdinalIgnoreCase) == true;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,65 @@
|
||||
using MongoDB.Entities;
|
||||
|
||||
using PrivaPub.Models.Jobs;
|
||||
|
||||
using System.Net;
|
||||
|
||||
namespace PrivaPub.Infrastructure.Geo
|
||||
{
|
||||
public interface ICdnRanges
|
||||
{
|
||||
// the CDN whose published ranges hold the address, or default
|
||||
CdnProvider Of(IPAddress address);
|
||||
void Replace(CdnRangeSet set);
|
||||
Task Load(CancellationToken token);
|
||||
// per CDN key: how many ranges are known and when they were downloaded
|
||||
IReadOnlyDictionary<string, (int Ranges, DateTime FetchedAt)> Status { get; }
|
||||
}
|
||||
|
||||
// The published CDN ranges in memory, rebuilt from the stored CdnRangeSets at start and replaced set by set as
|
||||
// CdnUpdater downloads them. A few thousand networks: a linear scan per describe is cheap, and describes are weekly.
|
||||
public class CdnRanges : ICdnRanges
|
||||
{
|
||||
readonly object _gate = new();
|
||||
Dictionary<string, (CdnProvider Provider, IPNetwork[] Networks, DateTime FetchedAt)> _sets = new(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
public IReadOnlyDictionary<string, (int Ranges, DateTime FetchedAt)> Status
|
||||
{
|
||||
get
|
||||
{
|
||||
lock (_gate)
|
||||
return _sets.ToDictionary(s => s.Key, s => (s.Value.Networks.Length, s.Value.FetchedAt), StringComparer.OrdinalIgnoreCase);
|
||||
}
|
||||
}
|
||||
|
||||
public CdnProvider Of(IPAddress address)
|
||||
{
|
||||
if (address == default)
|
||||
return default;
|
||||
if (address.IsIPv4MappedToIPv6)
|
||||
address = address.MapToIPv4();
|
||||
Dictionary<string, (CdnProvider Provider, IPNetwork[] Networks, DateTime FetchedAt)> sets;
|
||||
lock (_gate)
|
||||
sets = _sets;
|
||||
foreach (var (provider, networks, _) in sets.Values)
|
||||
foreach (var network in networks)
|
||||
if (network.BaseAddress.AddressFamily == address.AddressFamily && network.Contains(address))
|
||||
return provider;
|
||||
return default;
|
||||
}
|
||||
|
||||
public void Replace(CdnRangeSet set)
|
||||
{
|
||||
var provider = CdnCatalog.ByKey(set.Key) ?? new CdnProvider(set.Name, set.Key, Array.Empty<int>(), Array.Empty<CdnList>(), _ => false);
|
||||
var networks = set.Ranges.Select(r => IPNetwork.TryParse(r, out var n) ? n : (IPNetwork?)null).Where(n => n != default).Select(n => n!.Value).ToArray();
|
||||
lock (_gate)
|
||||
_sets = new(_sets, StringComparer.OrdinalIgnoreCase) { [set.Key] = (provider, networks, set.FetchedAt) };
|
||||
}
|
||||
|
||||
public async Task Load(CancellationToken token)
|
||||
{
|
||||
foreach (var set in await DB.Default.Find<CdnRangeSet>().ExecuteAsync(token))
|
||||
Replace(set);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,114 @@
|
||||
using Microsoft.Extensions.Options;
|
||||
|
||||
using MongoDB.Entities;
|
||||
|
||||
using PrivaPub.Infrastructure.Statistics;
|
||||
using PrivaPub.Models.Jobs;
|
||||
|
||||
using System.Text;
|
||||
|
||||
namespace PrivaPub.Infrastructure.Geo
|
||||
{
|
||||
// Keeps the CDNs' published address ranges current by itself (owner decision 2026-10-04): at start it loads the stored
|
||||
// ones, then once a day it downloads each list again. A list that fails to download or to parse, or comes back empty,
|
||||
// leaves the stored one in place and is tried the next day. Like GeoUpdater, plain HTTPS to fixed hosts, not federation.
|
||||
public sealed class CdnUpdater : BackgroundService
|
||||
{
|
||||
public const string ClientName = "cdn";
|
||||
static readonly TimeSpan FirstWait = TimeSpan.FromMinutes(1);
|
||||
static readonly TimeSpan Interval = TimeSpan.FromHours(24);
|
||||
const int MaxBytes = 16 * 1024 * 1024;
|
||||
const int MaxRanges = 200_000;
|
||||
|
||||
readonly IHttpClientFactory _http;
|
||||
readonly IOptionsMonitor<StatisticsOptions> _options;
|
||||
readonly ICdnRanges _ranges;
|
||||
readonly ILogger<CdnUpdater> _logger;
|
||||
|
||||
public CdnUpdater(IHttpClientFactory http, IOptionsMonitor<StatisticsOptions> options, ICdnRanges ranges, ILogger<CdnUpdater> logger)
|
||||
{
|
||||
_http = http;
|
||||
_options = options;
|
||||
_ranges = ranges;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
||||
{
|
||||
try
|
||||
{
|
||||
await _ranges.Load(stoppingToken);
|
||||
await Task.Delay(FirstWait, stoppingToken);
|
||||
while (!stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
if (_options.CurrentValue.Cdn.AutoUpdate)
|
||||
await Update(CdnCatalog.All, stoppingToken);
|
||||
await Task.Delay(Interval + TimeSpan.FromMinutes(Random.Shared.Next(60)), stoppingToken);
|
||||
}
|
||||
}
|
||||
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "{Service} stopped", nameof(CdnUpdater));
|
||||
}
|
||||
}
|
||||
|
||||
// The number of CDNs whose ranges were replaced.
|
||||
public async Task<int> Update(IEnumerable<CdnProvider> providers, CancellationToken token)
|
||||
{
|
||||
var replaced = 0;
|
||||
foreach (var provider in providers.Where(p => p.Lists.Length > 0))
|
||||
{
|
||||
try
|
||||
{
|
||||
var ranges = new List<string>();
|
||||
foreach (var list in provider.Lists)
|
||||
ranges.AddRange(CdnCatalog.Parse(list.Format, await Download(list, token)).Select(n => n.ToString()));
|
||||
ranges = ranges.Distinct().ToList();
|
||||
if (ranges.Count == 0 || ranges.Count > MaxRanges)
|
||||
throw new InvalidDataException($"{ranges.Count} ranges");
|
||||
|
||||
var set = await DB.Default.Find<CdnRangeSet>().Match(s => s.Key == provider.Key).ExecuteFirstAsync(token) ?? new CdnRangeSet { Key = provider.Key };
|
||||
set.Name = provider.Name;
|
||||
set.Ranges = ranges;
|
||||
set.Sources = provider.Lists.Select(l => l.Url).ToList();
|
||||
set.FetchedAt = DateTime.UtcNow;
|
||||
await DB.Default.SaveAsync(set, token);
|
||||
_ranges.Replace(set);
|
||||
replaced++;
|
||||
}
|
||||
catch (Exception ex) when (ex is HttpRequestException or InvalidDataException or FormatException or System.Text.Json.JsonException
|
||||
or TaskCanceledException && !token.IsCancellationRequested)
|
||||
{
|
||||
_logger.LogWarning(ex, "Could not update the published ranges of {Cdn}", provider.Name);
|
||||
}
|
||||
}
|
||||
return replaced;
|
||||
}
|
||||
|
||||
async Task<string> Download(CdnList list, CancellationToken token)
|
||||
{
|
||||
using var request = new HttpRequestMessage(list.Method == "POST" ? HttpMethod.Post : HttpMethod.Get, list.Url);
|
||||
request.Headers.Accept.ParseAdd("application/json, text/plain;q=0.9");
|
||||
if (list.Body != default)
|
||||
request.Content = new StringContent(list.Body, Encoding.UTF8, "application/x-www-form-urlencoded");
|
||||
using var response = await _http.CreateClient(ClientName).SendAsync(request, HttpCompletionOption.ResponseHeadersRead, token);
|
||||
response.EnsureSuccessStatusCode();
|
||||
if (response.Content.Headers.ContentLength > MaxBytes)
|
||||
throw new InvalidDataException($"more than {MaxBytes} bytes");
|
||||
await using var stream = await response.Content.ReadAsStreamAsync(token);
|
||||
using var buffer = new MemoryStream();
|
||||
var chunk = new byte[81920];
|
||||
int read;
|
||||
while ((read = await stream.ReadAsync(chunk, token)) > 0)
|
||||
{
|
||||
if (buffer.Length + read > MaxBytes)
|
||||
throw new InvalidDataException($"more than {MaxBytes} bytes");
|
||||
buffer.Write(chunk, 0, read);
|
||||
}
|
||||
return Encoding.UTF8.GetString(buffer.ToArray());
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
using MaxMind.Db;
|
||||
using MaxMind.Db;
|
||||
|
||||
using Microsoft.Extensions.Options;
|
||||
|
||||
@@ -165,23 +165,4 @@ namespace PrivaPub.Infrastructure.Geo
|
||||
_asn?.Dispose();
|
||||
}
|
||||
}
|
||||
|
||||
public static class CdnNetworks
|
||||
{
|
||||
static readonly Dictionary<int, string> Known = new()
|
||||
{
|
||||
[13335] = "Cloudflare",
|
||||
[209242] = "Cloudflare",
|
||||
[54113] = "Fastly",
|
||||
[20940] = "Akamai",
|
||||
[16625] = "Akamai",
|
||||
[16702] = "Akamai",
|
||||
[21342] = "Akamai",
|
||||
[200325] = "Bunny",
|
||||
[60068] = "CDN77",
|
||||
[15133] = "Edgio"
|
||||
};
|
||||
|
||||
public static string Of(int? asn) => asn is { } number && Known.TryGetValue(number, out var name) ? name : default;
|
||||
}
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
using MaxMind.Db;
|
||||
using MaxMind.Db;
|
||||
|
||||
using Microsoft.Extensions.Options;
|
||||
|
||||
@@ -11,8 +11,8 @@ namespace PrivaPub.Infrastructure.Geo
|
||||
{
|
||||
// Keeps the DB-IP Lite databases current by itself, so a deploy needs no timer and no root: once a day it checks
|
||||
// whether each database was built this month and, if not, fetches this month's (or, early in the month, last month's),
|
||||
// checks it opens as the right kind of database and swaps it in. This and SMTP are the server's only traffic that is not
|
||||
// federation, which is why it has its own client instead of IFederationHttp.
|
||||
// checks it opens as the right kind of database and swaps it in. This, CdnUpdater and SMTP are the server's only traffic
|
||||
// that is not federation, which is why each has its own client instead of IFederationHttp.
|
||||
public sealed class GeoUpdater : BackgroundService
|
||||
{
|
||||
public const string ClientName = "geo";
|
||||
|
||||
@@ -0,0 +1,55 @@
|
||||
using PrivaPub.Infrastructure.Geo;
|
||||
|
||||
using System.Collections.Concurrent;
|
||||
|
||||
namespace PrivaPub.Infrastructure.Http
|
||||
{
|
||||
public interface IEdgeHints
|
||||
{
|
||||
void Remember(string host, CdnProvider cdn);
|
||||
CdnProvider Of(string host);
|
||||
}
|
||||
|
||||
// The CDN each remote host's responses came through, as their headers tell it (CdnCatalog fingerprints), for the
|
||||
// weekly description of the server. Like ConnectedAddresses, memory only: what a host answers is never stored per
|
||||
// request.
|
||||
public class EdgeHints : IEdgeHints
|
||||
{
|
||||
const int MaxHosts = 50_000;
|
||||
|
||||
readonly ConcurrentDictionary<string, CdnProvider> _hints = new(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
public void Remember(string host, CdnProvider cdn)
|
||||
{
|
||||
if (string.IsNullOrEmpty(host))
|
||||
return;
|
||||
if (cdn == default)
|
||||
{
|
||||
_hints.TryRemove(host, out _);
|
||||
return;
|
||||
}
|
||||
if (_hints.Count >= MaxHosts)
|
||||
_hints.Clear();
|
||||
_hints[host] = cdn;
|
||||
}
|
||||
|
||||
public CdnProvider Of(string host) => host != default && _hints.TryGetValue(host, out var cdn) ? cdn : default;
|
||||
}
|
||||
|
||||
// On the federation client: reads each response's CDN fingerprint for its host. A response without one forgets the
|
||||
// host's hint, so a server that leaves its CDN is seen as itself again.
|
||||
public class EdgeHintsHandler : DelegatingHandler
|
||||
{
|
||||
readonly IEdgeHints _hints;
|
||||
|
||||
public EdgeHintsHandler(IEdgeHints hints) => _hints = hints;
|
||||
|
||||
protected override async Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
|
||||
{
|
||||
var response = await base.SendAsync(request, cancellationToken);
|
||||
if (request.RequestUri is { } uri && (int)response.StatusCode is >= 200 and < 500)
|
||||
_hints.Remember(uri.Host, CdnCatalog.FromResponse(response));
|
||||
return response;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -6,6 +6,13 @@
|
||||
public string GeoDirectory { get; set; } = "/var/lib/privapub/geo";
|
||||
public CrawlerOptions Crawler { get; set; } = new();
|
||||
public GeoOptions Geo { get; set; } = new();
|
||||
public CdnOptions Cdn { get; set; } = new();
|
||||
}
|
||||
|
||||
//owner decision (2026-10-04): the server downloads the CDNs' published address ranges daily (CdnUpdater)
|
||||
public class CdnOptions
|
||||
{
|
||||
public bool AutoUpdate { get; set; } = true;
|
||||
}
|
||||
|
||||
//owner decision (2026-10-04): the server keeps its DB-IP Lite databases current by itself
|
||||
|
||||
@@ -61,9 +61,11 @@ namespace PrivaPub.Middleware
|
||||
client.DefaultRequestHeaders.UserAgent.ParseAdd($"PrivaPub/{BuildInfo.Ref} (+{baseAddress}/)");
|
||||
})
|
||||
.ConfigurePrimaryHttpMessageHandler(provider =>
|
||||
SafeHttpHandlerFactory.Create(provider.GetRequiredService<IOptions<FederationOptions>>().Value, provider.GetRequiredService<IConnectedAddresses>()));
|
||||
SafeHttpHandlerFactory.Create(provider.GetRequiredService<IOptions<FederationOptions>>().Value, provider.GetRequiredService<IConnectedAddresses>()))
|
||||
.AddHttpMessageHandler(provider => new EdgeHintsHandler(provider.GetRequiredService<IEdgeHints>()));
|
||||
return service
|
||||
.AddSingleton<IConnectedAddresses, ConnectedAddresses>()
|
||||
.AddSingleton<IEdgeHints, EdgeHints>()
|
||||
.AddSingleton<IFederationHttp, FederationHttp>()
|
||||
.AddSingleton<IDomainBlocks, DomainBlocks>()
|
||||
.AddSingleton<IContentRenderer, ContentRenderer>()
|
||||
@@ -119,6 +121,13 @@ namespace PrivaPub.Middleware
|
||||
client.Timeout = TimeSpan.FromMinutes(15);
|
||||
client.DefaultRequestHeaders.UserAgent.ParseAdd($"PrivaPub/{BuildInfo.Ref} (+{baseAddress}/)");
|
||||
});
|
||||
// not federation either: the CDNs' published address ranges (CdnUpdater), plain HTTPS to the CDNs' own hosts
|
||||
service.AddHttpClient(CdnUpdater.ClientName, (provider, client) =>
|
||||
{
|
||||
var baseAddress = provider.GetRequiredService<IOptionsMonitor<AppConfiguration>>().CurrentValue.BackendBaseAddress?.TrimEnd('/');
|
||||
client.Timeout = TimeSpan.FromMinutes(2);
|
||||
client.DefaultRequestHeaders.UserAgent.ParseAdd($"PrivaPub/{BuildInfo.Ref} (+{baseAddress}/)");
|
||||
});
|
||||
return service
|
||||
.Configure<StatisticsOptions>(configuration.GetSection("Statistics"))
|
||||
.AddSingleton<InteractionSalts>()
|
||||
@@ -127,7 +136,10 @@ namespace PrivaPub.Middleware
|
||||
.AddHostedService(services => services.GetRequiredService<InteractionLedger>())
|
||||
.AddSingleton<IJobHandler, RollupJob>()
|
||||
.AddSingleton<IGeoLocator, DbIpLocator>()
|
||||
.AddSingleton<ICdnRanges, CdnRanges>()
|
||||
.AddHostedService<CdnUpdater>()
|
||||
.AddSingleton<Domain.Statistics.ISelfLocation, Domain.Statistics.SelfLocation>()
|
||||
.AddSingleton<Domain.Statistics.IServerPlaces, Domain.Statistics.ServerPlaces>()
|
||||
.AddHostedService<GeoUpdater>()
|
||||
.AddSingleton<Domain.Statistics.StatisticsQueries>()
|
||||
.AddSingleton<IJobHandler, Federation.Crawler.CrawlPlanner>()
|
||||
|
||||
@@ -0,0 +1,15 @@
|
||||
using MongoDB.Entities;
|
||||
|
||||
namespace PrivaPub.Models.Jobs
|
||||
{
|
||||
// The address ranges one CDN publishes, as CdnUpdater last downloaded them: kept, so a restart needs no download and a
|
||||
// failed download keeps the previous list.
|
||||
public class CdnRangeSet : Entity
|
||||
{
|
||||
public string Key { get; set; }//the CDN's domain (CdnProvider.Key)
|
||||
public string Name { get; set; }
|
||||
public List<string> Ranges { get; set; } = new();
|
||||
public List<string> Sources { get; set; } = new();
|
||||
public DateTime FetchedAt { get; set; }
|
||||
}
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
using MongoDB.Entities;
|
||||
using MongoDB.Entities;
|
||||
|
||||
namespace PrivaPub.Models.Jobs
|
||||
{
|
||||
@@ -57,6 +57,8 @@ namespace PrivaPub.Models.Jobs
|
||||
public int? Asn { get; set; }
|
||||
public string AsnOrg { get; set; }
|
||||
public string Cdn { get; set; }//the CDN in front of it, whose edge address says nothing about where it runs
|
||||
public string CdnDomain { get; set; }//the CDN's own domain, which groups its servers (CdnProvider.Key)
|
||||
public string CdnSource { get; set; }//how the CDN was found: "ranges", "headers" or "asn"
|
||||
public bool? Ipv6 { get; set; }
|
||||
public string Source { get; set; }
|
||||
public DateTime? LocatedAt { get; set; }
|
||||
@@ -79,8 +81,13 @@ namespace PrivaPub.Models.Jobs
|
||||
public string RegistrationMode { get; set; }
|
||||
public List<string> Languages { get; set; }
|
||||
public string Country { get; set; }
|
||||
public string City { get; set; }
|
||||
public double? Latitude { get; set; }
|
||||
public double? Longitude { get; set; }
|
||||
public int? Asn { get; set; }
|
||||
public string Cdn { get; set; }
|
||||
public string CdnDomain { get; set; }
|
||||
public string CdnSource { get; set; }
|
||||
public bool? Ipv6 { get; set; }
|
||||
}
|
||||
}
|
||||
@@ -14,7 +14,9 @@
|
||||
only counted, through a key that is destroyed at the end of each day.</p>
|
||||
<p>A server we exchange activities with is described once a week, from its public NodeInfo and, when it has one, its
|
||||
Mastodon instance API. Its location comes from the address we reached, looked up in an offline database, and is shown
|
||||
to the city; for a server behind a CDN, only the CDN is shown, since the address is the CDN's.</p>
|
||||
to the city; for a server behind a CDN, only the CDN is shown, since the address is the CDN's. CDNs are recognised from
|
||||
the address lists they publish and from their marks in the answers we get, and servers are grouped by their CDN week
|
||||
by week.</p>
|
||||
@if (Model.GeoSource is { } geo)
|
||||
{
|
||||
<p>Locations come from <a href="https://db-ip.com" rel="nofollow noopener">@geo</a>, licensed under
|
||||
|
||||
Reference in new issue
Block a user