CDNs found by themselves, and servers followed through time
Build / Build (push) Successful in 5m11s
Deploy / privapub.thepra.dev (push) Successful in 5m48s

PrivaPub now finds CDNs three ways, best first: the address ranges the
CDNs publish (Cloudflare, Fastly, Amazon CloudFront, Bunny, Gcore,
Imperva), downloaded daily by CdnUpdater and kept in CdnRangeSet; the
CDN's fingerprint in the responses it already gets from a server
(EdgeHintsHandler on the federation client); and the networks that carry
only a CDN. The fixed ASN list is gone; ASNs shared with plain hosting
(AWS, DataPacket) no longer hide a server. A server's Geo records the
CDN, its domain and how it was found, and weekly snapshots now keep the
city and coordinates too.

Servers through time (ServerPlaces): /instances/:host/history lists a
server's weekly snapshots, a CDN-fronted server's geo names the CDN's
domain and where the server was before it (before_cdn), and
/api/privapub/v1/cdns and /cdns/:domain group servers by CDN with week
by week who joined and who left. Owner decisions recorded in ROADMAP.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01LsXgEaXee4GCU1hwYgPJXw
This commit is contained in:
thepraandClaude Opus 5.5 committed 2026-10-04 11:33:39 +02:00
1 parent 5d7edc4a4c
commit 436f7da464
26 files changed
+1159 -62

No files matched your search

@@ -0,0 +1,82 @@
using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Mvc;
using PrivaPub.Api.Mastodon.Infrastructure;
using PrivaPub.Domain.Statistics;
namespace PrivaPub.Api.Mastodon.Controllers
{
// The CDNs in front of the servers this server knows, each named by its own domain (owner decision 2026-10-04): how many
// servers are behind it, how it is recognised, and week by week who joined and who left.
public class CdnsController : MastodonController
{
const int MaxWeeks = 260;
readonly IServerPlaces _places;
public CdnsController(IServerPlaces places) => _places = places;
[HttpGet("/api/privapub/v1/cdns"), Scope("read", requiresUser: false), AllowAnonymous]
public async Task<IActionResult> List(CancellationToken token) => Json((await _places.Cdns(token)).Select(Summary).ToList());
[HttpGet("/api/privapub/v1/cdns/{key}"), Scope("read", requiresUser: false), AllowAnonymous]
public async Task<IActionResult> Detail(string key, CancellationToken token)
{
var history = await _places.Cdn(key, Math.Clamp(Params.Int("weeks") ?? 52, 1, MaxWeeks), token);
if (history == default)
return NotFoundError();
var detail = new CdnDetail
{
Hosts = history.Hosts,
HostsTruncated = history.Group.Servers > history.Hosts.Count,
Weeks = history.Weeks.Select(w => new CdnWeekEntry { Week = w.Week, Servers = w.Servers, Joined = w.Joined, Left = w.Left }).ToList()
};
Fill(detail, history.Group);
return Json(detail);
}
static CdnSummary Summary(CdnGroup group)
{
var summary = new CdnSummary();
Fill(summary, group);
return summary;
}
static void Fill(CdnSummary summary, CdnGroup group)
{
summary.Key = group.Key;
summary.Name = group.Name;
summary.Domain = group.Domain;
summary.Servers = group.Servers;
summary.Ranges = group.Ranges;
summary.RangesUpdatedAt = group.RangesUpdatedAt is { } at ? MastodonJson.Time(at) : default;
summary.DetectedBy = group.DetectedBy;
}
public class CdnSummary
{
public string Key { get; set; }
public string Name { get; set; }
public string Domain { get; set; }
public int Servers { get; set; }
public int Ranges { get; set; }
public string RangesUpdatedAt { get; set; }
public List<string> DetectedBy { get; set; } = new();
}
public class CdnDetail : CdnSummary
{
public List<string> Hosts { get; set; } = new();
public bool HostsTruncated { get; set; }
public List<CdnWeekEntry> Weeks { get; set; } = new();
}
public class CdnWeekEntry
{
public string Week { get; set; }
public int Servers { get; set; }
public List<string> Joined { get; set; } = new();
public List<string> Left { get; set; } = new();
}
}
}
@@ -10,6 +10,7 @@ using PrivaPub.Domain.Statistics;
using PrivaPub.Federation.Actors;
using PrivaPub.Federation.Objects;
using PrivaPub.Infrastructure;
using PrivaPub.Infrastructure.Geo;
using PrivaPub.Models.Federation;
using PrivaPub.Models.Jobs;
using PrivaPub.Models.Post;
@@ -26,15 +27,19 @@ namespace PrivaPub.Api.Mastodon.Controllers
const int MaxHosts = 40;
readonly DbEntities _dbEntities;
const int MaxWeeks = 260;
readonly ISelfLocation _self;
readonly IServerPlaces _places;
readonly ILocalActorService _localActors;
readonly IOptionsMonitor<RegistrationOptions> _registrations;
public ProvenanceController(DbEntities dbEntities, ISelfLocation self, ILocalActorService localActors,
public ProvenanceController(DbEntities dbEntities, ISelfLocation self, IServerPlaces places, ILocalActorService localActors,
IOptionsMonitor<RegistrationOptions> registrations)
{
_dbEntities = dbEntities;
_self = self;
_places = places;
_localActors = localActors;
_registrations = registrations;
}
@@ -54,7 +59,8 @@ namespace PrivaPub.Api.Mastodon.Controllers
?? await DB.Default.Find<ObjectRecord>().Match(r => r.ObjectURI == post.ObjectURI).ExecuteFirstAsync(token);
var host = record?.Host ?? (Uri.TryCreate(post.ObjectURI, UriKind.Absolute, out var uri) ? uri.Host.ToLowerInvariant() : default);
var instance = host == default ? default : await DB.Default.Find<RemoteInstance>().Match(i => i.Host == host).ExecuteFirstAsync(token);
return Json(Describe(post, record, instance));
var beforeCdn = instance?.Geo?.Cdn == default ? default : (await _places.BeforeCdn(new[] { instance.Host }, token)).GetValueOrDefault(instance.Host);
return Json(Describe(post, record, instance, beforeCdn));
}
[HttpGet("/api/privapub/v1/instances/{host}"), Scope("read", requiresUser: false), AllowAnonymous]
@@ -68,19 +74,47 @@ namespace PrivaPub.Api.Mastodon.Controllers
[HttpGet("/api/privapub/v1/instances"), Scope("read", requiresUser: false), AllowAnonymous]
public async Task<IActionResult> Instances(CancellationToken token) => Json(await Describe(Params.List("host"), token));
//the server's weekly snapshots, newest first (weeks=, at most MaxWeeks): its place as shown publicly, its CDN, its size
[HttpGet("/api/privapub/v1/instances/{host}/history"), Scope("read", requiresUser: false), AllowAnonymous]
public async Task<IActionResult> History(string host, CancellationToken token)
{
host = host?.Trim().ToLowerInvariant();
if (string.IsNullOrEmpty(host) || host.Length > 253 || !await DB.Default.Find<RemoteInstance>().Match(i => i.Host == host).ExecuteAnyAsync(token))
return NotFoundError();
var weeks = Math.Clamp(Params.Int("weeks") ?? 52, 1, MaxWeeks);
return Json((await _places.History(host, weeks, token)).Select(s => new InstanceWeek
{
Week = s.Week,
TakenAt = MastodonJson.Time(s.TakenAt),
Reachable = s.Reachable,
Software = s.Software,
Version = s.SoftwareVersion,
UsersTotal = s.UsersTotal,
UsersActiveMonth = s.UsersActiveMonth,
LocalPosts = s.LocalPosts,
OpenRegistrations = s.OpenRegistrations,
Geo = Located(PublicGeo.Project(new InstanceGeo
{
Country = s.Country, City = s.City, Latitude = s.Latitude, Longitude = s.Longitude, Asn = s.Asn,
Cdn = s.Cdn, CdnDomain = s.CdnDomain, CdnSource = s.CdnSource, Ipv6 = s.Ipv6
}), new InstanceGeo { Cdn = s.Cdn, CdnDomain = s.CdnDomain ?? CdnCatalog.ByKey(s.Cdn)?.Key, CdnSource = s.CdnSource })
}).ToList());
}
async Task<List<InstanceDescription>> Describe(IEnumerable<string> hosts, CancellationToken token)
{
var wanted = hosts.Select(h => h?.Trim().ToLowerInvariant()).Where(h => !string.IsNullOrEmpty(h) && h.Length <= 253)
.Distinct().Take(MaxHosts).ToList();
var selfHost = Uri.TryCreate(_localActors.BaseAddress, UriKind.Absolute, out var self) ? self.Host.ToLowerInvariant() : default;
var remote = await DB.Default.Find<RemoteInstance>().Match(i => wanted.Contains(i.Host)).ExecuteAsync(token);
var beforeCdn = await _places.BeforeCdn(remote.Where(i => i.Geo?.Cdn != default).Select(i => i.Host), token);
var described = new List<InstanceDescription>();
foreach (var host in wanted)
{
if (host == selfHost)
described.Add(await DescribeSelf(host, token));
else if (remote.FirstOrDefault(i => i.Host == host) is { } instance)
described.Add(Describe(instance));
described.Add(Describe(instance, beforeCdn.GetValueOrDefault(host)));
}
return described;
}
@@ -101,8 +135,9 @@ namespace PrivaPub.Api.Mastodon.Controllers
};
}
//the public projection of a server's place (PublicGeo)
static InstanceLocation Located(PublicLocation location, InstanceGeo geo) => location == default ? default : new InstanceLocation
//the public projection of a server's place (PublicGeo); for a CDN-fronted one, the CDN's domain, how it was found and
//where the server was before it
static InstanceLocation Located(PublicLocation location, InstanceGeo geo, PlaceBeforeCdn beforeCdn = default) => location == default ? default : new InstanceLocation
{
Precision = location.Cdn != default ? "cdn" : location.Latitude != default ? "city" : "country",
Country = location.Country,
@@ -111,11 +146,17 @@ namespace PrivaPub.Api.Mastodon.Controllers
Longitude = location.Longitude,
Network = location.Network,
Cdn = location.Cdn,
CdnDomain = location.Cdn != default ? ServerPlaces.KeyOf(geo) : default,
CdnSource = location.Cdn != default ? geo?.CdnSource : default,
BeforeCdn = location.Cdn == default || beforeCdn == default ? default : new InstancePlace
{
Week = beforeCdn.Week, Country = beforeCdn.Country, City = beforeCdn.City, Latitude = beforeCdn.Latitude, Longitude = beforeCdn.Longitude
},
Source = geo?.Source,
LocatedAt = geo?.LocatedAt is { } at ? MastodonJson.Time(at) : default
};
Provenance Describe(PostEntity post, ObjectRecord record, RemoteInstance instance)
Provenance Describe(PostEntity post, ObjectRecord record, RemoteInstance instance, PlaceBeforeCdn beforeCdn)
{
var raw = record?.Raw == default ? default : JsonNode.Parse(record.Raw) as JsonObject;
return new Provenance
@@ -152,11 +193,11 @@ namespace PrivaPub.Api.Mastodon.Controllers
RawHash = r.RawHash,
Raw = r.Raw == default ? default : JsonNode.Parse(r.Raw)
}).ToList() ?? new(),
Instance = instance == default ? default : Describe(instance)
Instance = instance == default ? default : Describe(instance, beforeCdn)
};
}
static InstanceDescription Describe(RemoteInstance instance) => new()
static InstanceDescription Describe(RemoteInstance instance, PlaceBeforeCdn beforeCdn) => new()
{
Host = instance.Host,
Software = instance.Software,
@@ -174,7 +215,7 @@ namespace PrivaPub.Api.Mastodon.Controllers
LastSuccessAt = instance.LastSuccessAt is { } success ? MastodonJson.Time(success) : default,
LastFailureAt = instance.LastFailureAt is { } failure ? MastodonJson.Time(failure) : default
},
Geo = Located(PublicGeo.Project(instance.Geo), instance.Geo)
Geo = Located(PublicGeo.Project(instance.Geo), instance.Geo, beforeCdn)
};
public class Provenance
@@ -251,10 +292,36 @@ namespace PrivaPub.Api.Mastodon.Controllers
public double? Longitude { get; set; }
public string Network { get; set; }
public string Cdn { get; set; }
public string CdnDomain { get; set; }
public string CdnSource { get; set; }//"ranges", "headers" or "asn"
public InstancePlace BeforeCdn { get; set; }//where a CDN-fronted server was located before the CDN, by its snapshots
public string Source { get; set; }
public string LocatedAt { get; set; }
}
public class InstancePlace
{
public string Week { get; set; }
public string Country { get; set; }
public string City { get; set; }
public double? Latitude { get; set; }
public double? Longitude { get; set; }
}
public class InstanceWeek
{
public string Week { get; set; }
public string TakenAt { get; set; }
public bool Reachable { get; set; }
public string Software { get; set; }
public string Version { get; set; }
public long? UsersTotal { get; set; }
public long? UsersActiveMonth { get; set; }
public long? LocalPosts { get; set; }
public bool? OpenRegistrations { get; set; }
public InstanceLocation Geo { get; set; }
}
public class InstanceDelivery
{
public int ConsecutiveFailures { get; set; }
+8 -2
View File
@@ -24,6 +24,7 @@ namespace PrivaPub.Domain.Statistics
static readonly TimeSpan Lifetime = TimeSpan.FromDays(1);
readonly IGeoLocator _geo;
readonly ICdnRanges _cdnRanges;
readonly IOptionsMonitor<StatisticsOptions> _options;
readonly IOptionsMonitor<AppConfiguration> _app;
readonly ILogger<SelfLocation> _logger;
@@ -32,9 +33,11 @@ namespace PrivaPub.Domain.Statistics
InstanceGeo _located;
DateTime _locatedAt = DateTime.MinValue;
public SelfLocation(IGeoLocator geo, IOptionsMonitor<StatisticsOptions> options, IOptionsMonitor<AppConfiguration> app, ILogger<SelfLocation> logger)
public SelfLocation(IGeoLocator geo, IOptionsMonitor<StatisticsOptions> options, IOptionsMonitor<AppConfiguration> app, ILogger<SelfLocation> logger,
ICdnRanges cdnRanges = default)
{
_geo = geo;
_cdnRanges = cdnRanges;
_options = options;
_app = app;
_logger = logger;
@@ -79,6 +82,7 @@ namespace PrivaPub.Domain.Statistics
var addresses = await Dns.GetHostAddressesAsync(self.Host, token);
var address = addresses.FirstOrDefault(a => a.AddressFamily == AddressFamily.InterNetwork) ?? addresses.FirstOrDefault();
var fix = address == default ? default : _geo.Locate(address);
var (cdn, cdnSource) = Federation.Objects.InstanceDescriber.Cdn(address, self.Host, fix?.Asn, _cdnRanges, default);
return fix == default ? default : new InstanceGeo
{
Country = fix.Country,
@@ -87,7 +91,9 @@ namespace PrivaPub.Domain.Statistics
Longitude = fix.Longitude,
Asn = fix.Asn,
AsnOrg = fix.AsnOrg,
Cdn = CdnNetworks.Of(fix.Asn),
Cdn = cdn?.Name,
CdnDomain = cdn?.Domain,
CdnSource = cdnSource,
Ipv6 = address.AddressFamily == AddressFamily.InterNetworkV6,
Source = _geo.Source,
LocatedAt = DateTime.UtcNow
+163
View File
@@ -0,0 +1,163 @@
using Microsoft.Extensions.Caching.Memory;
using MongoDB.Entities;
using PrivaPub.Federation.Objects;
using PrivaPub.Infrastructure.Geo;
using PrivaPub.Models.Jobs;
namespace PrivaPub.Domain.Statistics
{
// Where a server was before it went behind a CDN, as its weekly snapshots last saw it.
public sealed record PlaceBeforeCdn(string Week, string Country, string City, double? Latitude, double? Longitude);
// A CDN as a group of servers: those behind it now, and week by week how many were behind it, who joined and who left.
public sealed record CdnGroup(string Key, string Name, string Domain, int Servers, int Ranges, DateTime? RangesUpdatedAt, List<string> DetectedBy);
public sealed record CdnWeek(string Week, int Servers, List<string> Joined, List<string> Left);
public sealed record CdnHistory(CdnGroup Group, List<string> Hosts, List<CdnWeek> Weeks);
public interface IServerPlaces
{
Task<Dictionary<string, PlaceBeforeCdn>> BeforeCdn(IEnumerable<string> hosts, CancellationToken token);
Task<List<RemoteInstanceSnapshot>> History(string host, int weeks, CancellationToken token);
Task<List<CdnGroup>> Cdns(CancellationToken token);
Task<CdnHistory> Cdn(string key, int weeks, CancellationToken token);
}
// Servers through time (owner decision 2026-10-04): each server's weekly snapshots (place, CDN, size), and the servers
// grouped by the CDN in front of them, named by the CDN's own domain. Read from the weekly snapshots InstanceDescriber
// writes; the CDN pages are kept for half an hour, because a large CDN reads many snapshots.
public class ServerPlaces : IServerPlaces
{
public const int MaxHosts = 500;
const int MaxChanges = 50;
static readonly TimeSpan CacheFor = TimeSpan.FromMinutes(30);
readonly ICdnRanges _ranges;
readonly IMemoryCache _cache;
public ServerPlaces(ICdnRanges ranges, IMemoryCache cache)
{
_ranges = ranges;
_cache = cache;
}
// the CDN key a server's place belongs to: the CDN's domain, or its name for a CDN that has none
public static string KeyOf(InstanceGeo geo) =>
geo?.Cdn == default ? default : geo.CdnDomain ?? CdnCatalog.ByKey(geo.Cdn)?.Key ?? geo.Cdn.ToLowerInvariant();
static string KeyOf(RemoteInstanceSnapshot snapshot) =>
snapshot.Cdn == default ? default : snapshot.CdnDomain ?? CdnCatalog.ByKey(snapshot.Cdn)?.Key ?? snapshot.Cdn.ToLowerInvariant();
public async Task<Dictionary<string, PlaceBeforeCdn>> BeforeCdn(IEnumerable<string> hosts, CancellationToken token)
{
var wanted = hosts.Where(h => h != default).Distinct().ToList();
if (wanted.Count == 0)
return new();
var located = await DB.Default.Find<RemoteInstanceSnapshot>()
.Match(s => wanted.Contains(s.Host) && s.Cdn == null && (s.Country != null || s.Latitude != null))
.Sort(s => s.Week, Order.Descending)
.ExecuteAsync(token);
return located.GroupBy(s => s.Host)
.Select(g => g.First())
.ToDictionary(s => s.Host, s => new PlaceBeforeCdn(s.Week, s.Country, s.City, s.Latitude, s.Longitude));
}
public async Task<List<RemoteInstanceSnapshot>> History(string host, int weeks, CancellationToken token) =>
await DB.Default.Find<RemoteInstanceSnapshot>()
.Match(s => s.Host == host)
.Sort(s => s.Week, Order.Descending)
.Limit(weeks)
.ExecuteAsync(token);
public async Task<List<CdnGroup>> Cdns(CancellationToken token)
{
var fronted = await Fronted(token);
var status = _ranges.Status;
var keys = CdnCatalog.All.Select(p => p.Key).Concat(fronted.Select(i => KeyOf(i.Geo))).Distinct(StringComparer.OrdinalIgnoreCase);
return keys.Select(key => Group(key, fronted, status)).OrderByDescending(g => g.Servers).ThenBy(g => g.Name).ToList();
}
public async Task<CdnHistory> Cdn(string key, int weeks, CancellationToken token)
{
key = key?.Trim().ToLowerInvariant();
if (string.IsNullOrEmpty(key) || key.Length > 100)
return default;
return await _cache.GetOrCreateAsync($"cdn-history|{key}|{weeks}", async entry =>
{
entry.AbsoluteExpirationRelativeToNow = CacheFor;
var fronted = await Fronted(token);
var provider = CdnCatalog.ByKey(key);
if (provider == default && fronted.All(i => KeyOf(i.Geo) != key))
return default;
var group = Group(provider?.Key ?? key, fronted, _ranges.Status);
var hosts = fronted.Where(i => KeyOf(i.Geo) == group.Key).Select(i => i.Host).OrderBy(h => h).Take(MaxHosts).ToList();
return new CdnHistory(group, hosts, await Weeks(group, weeks, token));
});
}
async Task<List<CdnWeek>> Weeks(CdnGroup group, int weeks, CancellationToken token)
{
var since = InstanceDescriber.Week(DateTime.UtcNow.AddDays(-7 * weeks));
var everBehind = (await DB.Default.Find<RemoteInstanceSnapshot>()
.Match(f => f.Gte(s => s.Week, since) & (f.Eq(s => s.CdnDomain, group.Key) | f.Eq(s => s.Cdn, group.Name)))
.Project(s => new RemoteInstanceSnapshot { Host = s.Host })
.ExecuteAsync(token))
.Select(s => s.Host).Distinct().ToList();
if (everBehind.Count == 0)
return new();
var snapshots = await DB.Default.Find<RemoteInstanceSnapshot>()
.Match(f => f.In(s => s.Host, everBehind) & f.Gte(s => s.Week, since))
.Project(s => new RemoteInstanceSnapshot { Host = s.Host, Week = s.Week, Cdn = s.Cdn, CdnDomain = s.CdnDomain })
.ExecuteAsync(token);
// a server is behind the CDN from the week a snapshot shows it there until one shows it elsewhere
var behind = new Dictionary<string, bool>();
var history = new List<CdnWeek>();
foreach (var week in snapshots.GroupBy(s => s.Week).OrderBy(w => w.Key, StringComparer.Ordinal))
{
var joined = new List<string>();
var left = new List<string>();
foreach (var snapshot in week)
{
var now = KeyOf(snapshot) == group.Key;
var before = behind.GetValueOrDefault(snapshot.Host);
if (now && !before)
joined.Add(snapshot.Host);
else if (!now && before)
left.Add(snapshot.Host);
behind[snapshot.Host] = now;
}
history.Add(new CdnWeek(week.Key, behind.Count(b => b.Value), joined.OrderBy(h => h).Take(MaxChanges).ToList(),
left.OrderBy(h => h).Take(MaxChanges).ToList()));
}
history.Reverse();
return history;
}
static async Task<List<RemoteInstance>> Fronted(CancellationToken token) =>
await DB.Default.Find<RemoteInstance>()
.Match(i => i.Geo.Cdn != null)
.Project(i => new RemoteInstance { Host = i.Host, Geo = i.Geo })
.ExecuteAsync(token);
static CdnGroup Group(string key, List<RemoteInstance> fronted, IReadOnlyDictionary<string, (int Ranges, DateTime FetchedAt)> status)
{
var provider = CdnCatalog.ByKey(key);
var servers = fronted.Where(i => KeyOf(i.Geo) == key).ToList();
var detectedBy = new List<string>();
if (provider?.Lists.Length > 0)
detectedBy.Add("ranges");
if (provider != default)
detectedBy.Add("headers");
if (provider?.Asns.Length > 0)
detectedBy.Add("asn");
var ranges = status.TryGetValue(key, out var known) ? known : default;
return new CdnGroup(key, provider?.Name ?? servers.Select(i => i.Geo.Cdn).FirstOrDefault() ?? key, provider?.Domain, servers.Count, ranges.Ranges,
ranges.Ranges > 0 ? ranges.FetchedAt : null, detectedBy);
}
}
}
@@ -1,4 +1,4 @@
using MongoDB.Entities;
using MongoDB.Entities;
using PrivaPub.Infrastructure.Geo;
using PrivaPub.Infrastructure.Http;
@@ -18,12 +18,17 @@ namespace PrivaPub.Federation.Objects
readonly IFederationHttp _http;
readonly IGeoLocator _geo;
readonly IConnectedAddresses _addresses;
readonly ICdnRanges _cdnRanges;
readonly IEdgeHints _edges;
public InstanceDescriber(IFederationHttp http, IGeoLocator geo = default, IConnectedAddresses addresses = default)
public InstanceDescriber(IFederationHttp http, IGeoLocator geo = default, IConnectedAddresses addresses = default, ICdnRanges cdnRanges = default,
IEdgeHints edges = default)
{
_http = http;
_geo = geo;
_addresses = addresses;
_cdnRanges = cdnRanges;
_edges = edges;
}
public JobKind Kind => JobKind.DescribeInstance;
@@ -145,35 +150,57 @@ namespace PrivaPub.Federation.Objects
.Modify(s => s.RegistrationMode, instance?.RegistrationMode)
.Modify(s => s.Languages, instance?.Languages)
.Modify(s => s.Country, instance?.Geo?.Country)
.Modify(s => s.City, instance?.Geo?.City)
.Modify(s => s.Latitude, instance?.Geo?.Latitude)
.Modify(s => s.Longitude, instance?.Geo?.Longitude)
.Modify(s => s.Asn, instance?.Geo?.Asn)
.Modify(s => s.Cdn, instance?.Geo?.Cdn)
.Modify(s => s.CdnDomain, instance?.Geo?.CdnDomain)
.Modify(s => s.CdnSource, instance?.Geo?.CdnSource)
.Modify(s => s.Ipv6, instance?.Geo?.Ipv6)
.Option(o => o.IsUpsert = true)
.ExecuteAsync(token);
return facts != default;
}
// Where the address we reached sits, and the CDN in front of it, found by its published ranges, then by the
// fingerprint in the responses we just got, then by a network that carries only a CDN.
InstanceGeo Locate(string host)
{
var reached = _addresses?.Of(new Uri(Address($"https://{host}/")).Host);
var reachedHost = new Uri(Address($"https://{host}/")).Host;
var reached = _addresses?.Of(reachedHost);
var fix = reached == default ? default : _geo?.Locate(reached);
if (fix == default)
var (cdn, cdnSource) = Cdn(reached, reachedHost, fix?.Asn, _cdnRanges, _edges);
if (fix == default && cdn == default)
return default;
return new InstanceGeo
{
Country = fix.Country,
City = fix.City,
Latitude = fix.Latitude,
Longitude = fix.Longitude,
Asn = fix.Asn,
AsnOrg = fix.AsnOrg,
Cdn = CdnNetworks.Of(fix.Asn),
Ipv6 = reached.AddressFamily == AddressFamily.InterNetworkV6,
Source = _geo.Source,
Country = fix?.Country,
City = fix?.City,
Latitude = fix?.Latitude,
Longitude = fix?.Longitude,
Asn = fix?.Asn,
AsnOrg = fix?.AsnOrg,
Cdn = cdn?.Name,
CdnDomain = cdn?.Domain,
CdnSource = cdnSource,
Ipv6 = reached?.AddressFamily == AddressFamily.InterNetworkV6,
Source = _geo?.Source,
LocatedAt = DateTime.UtcNow
};
}
public static (CdnProvider Cdn, string Source) Cdn(System.Net.IPAddress address, string host, int? asn, ICdnRanges ranges, IEdgeHints edges)
{
if (ranges?.Of(address) is { } listed)
return (listed, "ranges");
if (edges?.Of(host) is { } hinted)
return (hinted, "headers");
if (CdnCatalog.OfAsn(asn) is { } network)
return (network, "asn");
return default;
}
//where an https address is fetched from: itself, except in tests, which point it at a plain-http peer
protected virtual string Address(string url) => url;
}
+4 -1
View File
@@ -1,4 +1,4 @@
using MongoDB.Bson;
using MongoDB.Bson;
using MongoDB.Driver;
using MongoDB.Entities;
@@ -131,6 +131,9 @@ namespace PrivaPub.Infrastructure.Data
await DB.Default.Index<RemoteInstanceSnapshot>().Key(s => s.Host, KeyType.Ascending).Key(s => s.Week, KeyType.Ascending)
.Option(o => o.Unique = true).CreateAsync(token);
await Plain<RemoteInstanceSnapshot>(token, s => s.Week);
await Plain<RemoteInstanceSnapshot>(token, s => s.CdnDomain, s => s.Week);
await Plain<RemoteInstanceSnapshot>(token, s => s.Cdn, s => s.Week);
await Plain<CdnRangeSet>(token, s => s.Key);
await Plain<RemoteInstance>(token, i => i.Seen);
await Plain<RemoteInstance>(token, i => i.LastCrawledAt);
}
+156
View File
@@ -0,0 +1,156 @@
using System.Net;
using System.Text.Json;
namespace PrivaPub.Infrastructure.Geo
{
// How a published list of a CDN's addresses is written.
public enum CdnListFormat
{
Lines,//one CIDR per line (Cloudflare)
StringArray,//a JSON array of addresses or CIDRs (Bunny)
AddressesJson,//{"addresses": [...], "ipv6_addresses"/"addresses_v6": [...]} (Fastly, Gcore)
AwsCloudFront,//AWS ip-ranges.json, CLOUDFRONT entries only
ImpervaJson//{"ipRanges": [...], "ipv6Ranges": [...]}
}
public sealed record CdnList(string Url, CdnListFormat Format, string Method = "GET", string Body = default);
// A CDN, or any edge network whose addresses say nothing about where a server runs. Domain names it in statistics.
public sealed record CdnProvider(string Name, string Domain, int[] Asns, CdnList[] Lists, Func<HttpResponseMessage, bool> Fingerprint)
{
public string Key => Domain ?? Name.ToLowerInvariant();
}
// owner decision (2026-10-04): PrivaPub finds CDNs by itself, three ways, best first. The address ranges a CDN
// publishes (downloaded daily by CdnUpdater), the CDN's fingerprint in the responses PrivaPub already gets from a server
// (EdgeHintsHandler), and the networks (ASN) that carry nothing but a CDN. A network shared with plain hosting (AWS,
// Google, Microsoft, DataPacket) is never listed by ASN: only ranges or fingerprints name those CDNs.
public static class CdnCatalog
{
public static readonly IReadOnlyList<CdnProvider> All = new CdnProvider[]
{
new("Cloudflare", "cloudflare.com", new[] { 13335, 209242 },
new[] { new CdnList("https://www.cloudflare.com/ips-v4", CdnListFormat.Lines), new CdnList("https://www.cloudflare.com/ips-v6", CdnListFormat.Lines) },
r => Has(r, "cf-ray") || ServerIs(r, "cloudflare")),
new("Fastly", "fastly.com", new[] { 54113 },
new[] { new CdnList("https://api.fastly.com/public-ip-list", CdnListFormat.AddressesJson) },
r => Has(r, "x-fastly-request-id") || Has(r, "fastly-restarts") || Value(r, "x-served-by")?.StartsWith("cache-", StringComparison.OrdinalIgnoreCase) == true),
new("Amazon CloudFront", "cloudfront.net", Array.Empty<int>(),
new[] { new CdnList("https://ip-ranges.amazonaws.com/ip-ranges.json", CdnListFormat.AwsCloudFront) },
r => Has(r, "x-amz-cf-id") || Has(r, "x-amz-cf-pop") || Value(r, "via")?.Contains("cloudfront", StringComparison.OrdinalIgnoreCase) == true),
new("Akamai", "akamai.com", new[] { 20940, 16625, 16702, 21342 }, Array.Empty<CdnList>(),
r => Has(r, "akamai-grn") || Has(r, "x-akamai-transformed") || Has(r, "akamai-cache-status") || ServerStarts(r, "AkamaiGHost") || ServerStarts(r, "AkamaiNetStorage")),
new("Bunny", "bunny.net", new[] { 200325 },
new[] { new CdnList("https://api.bunny.net/system/edgeserverlist", CdnListFormat.StringArray), new CdnList("https://api.bunny.net/system/edgeserverlist/IPv6", CdnListFormat.StringArray) },
r => ServerStarts(r, "BunnyCDN") || Has(r, "cdn-pullzone") || Has(r, "cdn-requestid")),
new("Gcore", "gcore.com", Array.Empty<int>(),
new[] { new CdnList("https://api.gcore.com/cdn/public-ip-list", CdnListFormat.AddressesJson) },
r => ServerStarts(r, "gcore")),
new("Imperva", "imperva.com", new[] { 19551 },
new[] { new CdnList("https://my.imperva.com/api/integration/v1/ips", CdnListFormat.ImpervaJson, "POST", "resp_format=json") },
r => Has(r, "x-iinfo") || Value(r, "x-cdn") is "Imperva" or "Incapsula"),
new("CDN77", "cdn77.com", Array.Empty<int>(), Array.Empty<CdnList>(),
r => Has(r, "x-77-nzt") || Has(r, "x-77-cache") || ServerStarts(r, "CDN77")),
new("Edgio", "edg.io", new[] { 15133 }, Array.Empty<CdnList>(),
r => ServerStarts(r, "ECAcc") || ServerStarts(r, "ECS (") || Has(r, "x-ec-custom-error")),
new("Azure Front Door", "azure.microsoft.com", Array.Empty<int>(), Array.Empty<CdnList>(),
r => Has(r, "x-azure-ref")),
new("Google Cloud", "cloud.google.com", Array.Empty<int>(), Array.Empty<CdnList>(),
r => Value(r, "via")?.Contains("google", StringComparison.OrdinalIgnoreCase) == true),
new("Vercel", "vercel.com", Array.Empty<int>(), Array.Empty<CdnList>(),
r => Has(r, "x-vercel-id") || ServerIs(r, "Vercel")),
new("Netlify", "netlify.com", Array.Empty<int>(), Array.Empty<CdnList>(),
r => Has(r, "x-nf-request-id") || ServerIs(r, "Netlify")),
new("Sucuri", "sucuri.net", Array.Empty<int>(), Array.Empty<CdnList>(),
r => Has(r, "x-sucuri-id") || ServerStarts(r, "Sucuri")),
new("DDoS-Guard", "ddos-guard.net", Array.Empty<int>(), Array.Empty<CdnList>(),
r => ServerIs(r, "ddos-guard"))
};
public static CdnProvider ByKey(string key) =>
key == default ? default : All.FirstOrDefault(p => string.Equals(p.Key, key, StringComparison.OrdinalIgnoreCase) || string.Equals(p.Name, key, StringComparison.OrdinalIgnoreCase));
public static CdnProvider OfAsn(int? asn) => asn is { } number ? All.FirstOrDefault(p => p.Asns.Contains(number)) : default;
// The CDN a response passed through: a known fingerprint, or a CDN that names itself in X-CDN.
public static CdnProvider FromResponse(HttpResponseMessage response)
{
if (response == default)
return default;
var known = All.FirstOrDefault(p => p.Fingerprint(response));
if (known != default)
return known;
var named = Value(response, "x-cdn")?.Trim();
return named is { Length: > 1 and <= 40 } && named.All(c => char.IsLetterOrDigit(c) || c is ' ' or '-' or '.' or '_')
? new CdnProvider(named, default, Array.Empty<int>(), Array.Empty<CdnList>(), _ => false)
: default;
}
// The networks a published list holds; an entry that is not an address or a network is an error, never skipped.
public static List<IPNetwork> Parse(CdnListFormat format, string body)
{
IEnumerable<string> entries = format switch
{
CdnListFormat.Lines => body.Split('\n', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries).Where(l => !l.StartsWith('#')),
CdnListFormat.StringArray => Root(body),
CdnListFormat.AddressesJson => Json(body, "addresses", "ipv6_addresses", "addresses_v6"),
CdnListFormat.ImpervaJson => Json(body, "ipRanges", "ipv6Ranges"),
CdnListFormat.AwsCloudFront => CloudFront(body),
_ => Enumerable.Empty<string>()
};
return entries.Select(Network).ToList();
}
static IPNetwork Network(string entry)
{
if (IPNetwork.TryParse(entry, out var network))
return network;
if (IPAddress.TryParse(entry, out var address))
return new IPNetwork(address, address.AddressFamily == System.Net.Sockets.AddressFamily.InterNetwork ? 32 : 128);
throw new FormatException($"not an address or a network: {entry}");
}
static List<string> Root(string body)
{
using var document = JsonDocument.Parse(body);
return Strings(document.RootElement);
}
static IEnumerable<string> Json(string body, params string[] properties)
{
using var document = JsonDocument.Parse(body);
var entries = new List<string>();
foreach (var property in properties)
if (document.RootElement.TryGetProperty(property, out var array))
entries.AddRange(Strings(array));
return entries;
}
static IEnumerable<string> CloudFront(string body)
{
using var document = JsonDocument.Parse(body);
var entries = new List<string>();
foreach (var (array, field) in new[] { ("prefixes", "ip_prefix"), ("ipv6_prefixes", "ipv6_prefix") })
if (document.RootElement.TryGetProperty(array, out var prefixes))
foreach (var prefix in prefixes.EnumerateArray())
if (prefix.TryGetProperty("service", out var service) && service.GetString() == "CLOUDFRONT" && prefix.TryGetProperty(field, out var value))
entries.Add(value.GetString());
return entries;
}
static List<string> Strings(JsonElement array) =>
array.ValueKind == JsonValueKind.Array ? array.EnumerateArray().Select(e => e.GetString()).ToList() : throw new FormatException("not a list");
static bool Has(HttpResponseMessage response, string header) =>
response.Headers.Contains(header) || response.Content?.Headers.Contains(header) == true;
static string Value(HttpResponseMessage response, string header) =>
response.Headers.TryGetValues(header, out var values) || response.Content?.Headers.TryGetValues(header, out values) == true
? string.Join(", ", values)
: default;
static bool ServerIs(HttpResponseMessage response, string name) => string.Equals(Value(response, "server")?.Trim(), name, StringComparison.OrdinalIgnoreCase);
static bool ServerStarts(HttpResponseMessage response, string prefix) => Value(response, "server")?.TrimStart().StartsWith(prefix, StringComparison.OrdinalIgnoreCase) == true;
}
}
+65
View File
@@ -0,0 +1,65 @@
using MongoDB.Entities;
using PrivaPub.Models.Jobs;
using System.Net;
namespace PrivaPub.Infrastructure.Geo
{
public interface ICdnRanges
{
// the CDN whose published ranges hold the address, or default
CdnProvider Of(IPAddress address);
void Replace(CdnRangeSet set);
Task Load(CancellationToken token);
// per CDN key: how many ranges are known and when they were downloaded
IReadOnlyDictionary<string, (int Ranges, DateTime FetchedAt)> Status { get; }
}
// The published CDN ranges in memory, rebuilt from the stored CdnRangeSets at start and replaced set by set as
// CdnUpdater downloads them. A few thousand networks: a linear scan per describe is cheap, and describes are weekly.
public class CdnRanges : ICdnRanges
{
readonly object _gate = new();
Dictionary<string, (CdnProvider Provider, IPNetwork[] Networks, DateTime FetchedAt)> _sets = new(StringComparer.OrdinalIgnoreCase);
public IReadOnlyDictionary<string, (int Ranges, DateTime FetchedAt)> Status
{
get
{
lock (_gate)
return _sets.ToDictionary(s => s.Key, s => (s.Value.Networks.Length, s.Value.FetchedAt), StringComparer.OrdinalIgnoreCase);
}
}
public CdnProvider Of(IPAddress address)
{
if (address == default)
return default;
if (address.IsIPv4MappedToIPv6)
address = address.MapToIPv4();
Dictionary<string, (CdnProvider Provider, IPNetwork[] Networks, DateTime FetchedAt)> sets;
lock (_gate)
sets = _sets;
foreach (var (provider, networks, _) in sets.Values)
foreach (var network in networks)
if (network.BaseAddress.AddressFamily == address.AddressFamily && network.Contains(address))
return provider;
return default;
}
public void Replace(CdnRangeSet set)
{
var provider = CdnCatalog.ByKey(set.Key) ?? new CdnProvider(set.Name, set.Key, Array.Empty<int>(), Array.Empty<CdnList>(), _ => false);
var networks = set.Ranges.Select(r => IPNetwork.TryParse(r, out var n) ? n : (IPNetwork?)null).Where(n => n != default).Select(n => n!.Value).ToArray();
lock (_gate)
_sets = new(_sets, StringComparer.OrdinalIgnoreCase) { [set.Key] = (provider, networks, set.FetchedAt) };
}
public async Task Load(CancellationToken token)
{
foreach (var set in await DB.Default.Find<CdnRangeSet>().ExecuteAsync(token))
Replace(set);
}
}
}
+114
View File
@@ -0,0 +1,114 @@
using Microsoft.Extensions.Options;
using MongoDB.Entities;
using PrivaPub.Infrastructure.Statistics;
using PrivaPub.Models.Jobs;
using System.Text;
namespace PrivaPub.Infrastructure.Geo
{
// Keeps the CDNs' published address ranges current by itself (owner decision 2026-10-04): at start it loads the stored
// ones, then once a day it downloads each list again. A list that fails to download or to parse, or comes back empty,
// leaves the stored one in place and is tried the next day. Like GeoUpdater, plain HTTPS to fixed hosts, not federation.
public sealed class CdnUpdater : BackgroundService
{
public const string ClientName = "cdn";
static readonly TimeSpan FirstWait = TimeSpan.FromMinutes(1);
static readonly TimeSpan Interval = TimeSpan.FromHours(24);
const int MaxBytes = 16 * 1024 * 1024;
const int MaxRanges = 200_000;
readonly IHttpClientFactory _http;
readonly IOptionsMonitor<StatisticsOptions> _options;
readonly ICdnRanges _ranges;
readonly ILogger<CdnUpdater> _logger;
public CdnUpdater(IHttpClientFactory http, IOptionsMonitor<StatisticsOptions> options, ICdnRanges ranges, ILogger<CdnUpdater> logger)
{
_http = http;
_options = options;
_ranges = ranges;
_logger = logger;
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
try
{
await _ranges.Load(stoppingToken);
await Task.Delay(FirstWait, stoppingToken);
while (!stoppingToken.IsCancellationRequested)
{
if (_options.CurrentValue.Cdn.AutoUpdate)
await Update(CdnCatalog.All, stoppingToken);
await Task.Delay(Interval + TimeSpan.FromMinutes(Random.Shared.Next(60)), stoppingToken);
}
}
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
{
}
catch (Exception ex)
{
_logger.LogWarning(ex, "{Service} stopped", nameof(CdnUpdater));
}
}
// The number of CDNs whose ranges were replaced.
public async Task<int> Update(IEnumerable<CdnProvider> providers, CancellationToken token)
{
var replaced = 0;
foreach (var provider in providers.Where(p => p.Lists.Length > 0))
{
try
{
var ranges = new List<string>();
foreach (var list in provider.Lists)
ranges.AddRange(CdnCatalog.Parse(list.Format, await Download(list, token)).Select(n => n.ToString()));
ranges = ranges.Distinct().ToList();
if (ranges.Count == 0 || ranges.Count > MaxRanges)
throw new InvalidDataException($"{ranges.Count} ranges");
var set = await DB.Default.Find<CdnRangeSet>().Match(s => s.Key == provider.Key).ExecuteFirstAsync(token) ?? new CdnRangeSet { Key = provider.Key };
set.Name = provider.Name;
set.Ranges = ranges;
set.Sources = provider.Lists.Select(l => l.Url).ToList();
set.FetchedAt = DateTime.UtcNow;
await DB.Default.SaveAsync(set, token);
_ranges.Replace(set);
replaced++;
}
catch (Exception ex) when (ex is HttpRequestException or InvalidDataException or FormatException or System.Text.Json.JsonException
or TaskCanceledException && !token.IsCancellationRequested)
{
_logger.LogWarning(ex, "Could not update the published ranges of {Cdn}", provider.Name);
}
}
return replaced;
}
async Task<string> Download(CdnList list, CancellationToken token)
{
using var request = new HttpRequestMessage(list.Method == "POST" ? HttpMethod.Post : HttpMethod.Get, list.Url);
request.Headers.Accept.ParseAdd("application/json, text/plain;q=0.9");
if (list.Body != default)
request.Content = new StringContent(list.Body, Encoding.UTF8, "application/x-www-form-urlencoded");
using var response = await _http.CreateClient(ClientName).SendAsync(request, HttpCompletionOption.ResponseHeadersRead, token);
response.EnsureSuccessStatusCode();
if (response.Content.Headers.ContentLength > MaxBytes)
throw new InvalidDataException($"more than {MaxBytes} bytes");
await using var stream = await response.Content.ReadAsStreamAsync(token);
using var buffer = new MemoryStream();
var chunk = new byte[81920];
int read;
while ((read = await stream.ReadAsync(chunk, token)) > 0)
{
if (buffer.Length + read > MaxBytes)
throw new InvalidDataException($"more than {MaxBytes} bytes");
buffer.Write(chunk, 0, read);
}
return Encoding.UTF8.GetString(buffer.ToArray());
}
}
}
+1 -20
View File
@@ -1,4 +1,4 @@
using MaxMind.Db;
using MaxMind.Db;
using Microsoft.Extensions.Options;
@@ -165,23 +165,4 @@ namespace PrivaPub.Infrastructure.Geo
_asn?.Dispose();
}
}
public static class CdnNetworks
{
static readonly Dictionary<int, string> Known = new()
{
[13335] = "Cloudflare",
[209242] = "Cloudflare",
[54113] = "Fastly",
[20940] = "Akamai",
[16625] = "Akamai",
[16702] = "Akamai",
[21342] = "Akamai",
[200325] = "Bunny",
[60068] = "CDN77",
[15133] = "Edgio"
};
public static string Of(int? asn) => asn is { } number && Known.TryGetValue(number, out var name) ? name : default;
}
}
+3 -3
View File
@@ -1,4 +1,4 @@
using MaxMind.Db;
using MaxMind.Db;
using Microsoft.Extensions.Options;
@@ -11,8 +11,8 @@ namespace PrivaPub.Infrastructure.Geo
{
// Keeps the DB-IP Lite databases current by itself, so a deploy needs no timer and no root: once a day it checks
// whether each database was built this month and, if not, fetches this month's (or, early in the month, last month's),
// checks it opens as the right kind of database and swaps it in. This and SMTP are the server's only traffic that is not
// federation, which is why it has its own client instead of IFederationHttp.
// checks it opens as the right kind of database and swaps it in. This, CdnUpdater and SMTP are the server's only traffic
// that is not federation, which is why each has its own client instead of IFederationHttp.
public sealed class GeoUpdater : BackgroundService
{
public const string ClientName = "geo";
+55
View File
@@ -0,0 +1,55 @@
using PrivaPub.Infrastructure.Geo;
using System.Collections.Concurrent;
namespace PrivaPub.Infrastructure.Http
{
public interface IEdgeHints
{
void Remember(string host, CdnProvider cdn);
CdnProvider Of(string host);
}
// The CDN each remote host's responses came through, as their headers tell it (CdnCatalog fingerprints), for the
// weekly description of the server. Like ConnectedAddresses, memory only: what a host answers is never stored per
// request.
public class EdgeHints : IEdgeHints
{
const int MaxHosts = 50_000;
readonly ConcurrentDictionary<string, CdnProvider> _hints = new(StringComparer.OrdinalIgnoreCase);
public void Remember(string host, CdnProvider cdn)
{
if (string.IsNullOrEmpty(host))
return;
if (cdn == default)
{
_hints.TryRemove(host, out _);
return;
}
if (_hints.Count >= MaxHosts)
_hints.Clear();
_hints[host] = cdn;
}
public CdnProvider Of(string host) => host != default && _hints.TryGetValue(host, out var cdn) ? cdn : default;
}
// On the federation client: reads each response's CDN fingerprint for its host. A response without one forgets the
// host's hint, so a server that leaves its CDN is seen as itself again.
public class EdgeHintsHandler : DelegatingHandler
{
readonly IEdgeHints _hints;
public EdgeHintsHandler(IEdgeHints hints) => _hints = hints;
protected override async Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
{
var response = await base.SendAsync(request, cancellationToken);
if (request.RequestUri is { } uri && (int)response.StatusCode is >= 200 and < 500)
_hints.Remember(uri.Host, CdnCatalog.FromResponse(response));
return response;
}
}
}
@@ -6,6 +6,13 @@
public string GeoDirectory { get; set; } = "/var/lib/privapub/geo";
public CrawlerOptions Crawler { get; set; } = new();
public GeoOptions Geo { get; set; } = new();
public CdnOptions Cdn { get; set; } = new();
}
//owner decision (2026-10-04): the server downloads the CDNs' published address ranges daily (CdnUpdater)
public class CdnOptions
{
public bool AutoUpdate { get; set; } = true;
}
//owner decision (2026-10-04): the server keeps its DB-IP Lite databases current by itself
+13 -1
View File
@@ -61,9 +61,11 @@ namespace PrivaPub.Middleware
client.DefaultRequestHeaders.UserAgent.ParseAdd($"PrivaPub/{BuildInfo.Ref} (+{baseAddress}/)");
})
.ConfigurePrimaryHttpMessageHandler(provider =>
SafeHttpHandlerFactory.Create(provider.GetRequiredService<IOptions<FederationOptions>>().Value, provider.GetRequiredService<IConnectedAddresses>()));
SafeHttpHandlerFactory.Create(provider.GetRequiredService<IOptions<FederationOptions>>().Value, provider.GetRequiredService<IConnectedAddresses>()))
.AddHttpMessageHandler(provider => new EdgeHintsHandler(provider.GetRequiredService<IEdgeHints>()));
return service
.AddSingleton<IConnectedAddresses, ConnectedAddresses>()
.AddSingleton<IEdgeHints, EdgeHints>()
.AddSingleton<IFederationHttp, FederationHttp>()
.AddSingleton<IDomainBlocks, DomainBlocks>()
.AddSingleton<IContentRenderer, ContentRenderer>()
@@ -119,6 +121,13 @@ namespace PrivaPub.Middleware
client.Timeout = TimeSpan.FromMinutes(15);
client.DefaultRequestHeaders.UserAgent.ParseAdd($"PrivaPub/{BuildInfo.Ref} (+{baseAddress}/)");
});
// not federation either: the CDNs' published address ranges (CdnUpdater), plain HTTPS to the CDNs' own hosts
service.AddHttpClient(CdnUpdater.ClientName, (provider, client) =>
{
var baseAddress = provider.GetRequiredService<IOptionsMonitor<AppConfiguration>>().CurrentValue.BackendBaseAddress?.TrimEnd('/');
client.Timeout = TimeSpan.FromMinutes(2);
client.DefaultRequestHeaders.UserAgent.ParseAdd($"PrivaPub/{BuildInfo.Ref} (+{baseAddress}/)");
});
return service
.Configure<StatisticsOptions>(configuration.GetSection("Statistics"))
.AddSingleton<InteractionSalts>()
@@ -127,7 +136,10 @@ namespace PrivaPub.Middleware
.AddHostedService(services => services.GetRequiredService<InteractionLedger>())
.AddSingleton<IJobHandler, RollupJob>()
.AddSingleton<IGeoLocator, DbIpLocator>()
.AddSingleton<ICdnRanges, CdnRanges>()
.AddHostedService<CdnUpdater>()
.AddSingleton<Domain.Statistics.ISelfLocation, Domain.Statistics.SelfLocation>()
.AddSingleton<Domain.Statistics.IServerPlaces, Domain.Statistics.ServerPlaces>()
.AddHostedService<GeoUpdater>()
.AddSingleton<Domain.Statistics.StatisticsQueries>()
.AddSingleton<IJobHandler, Federation.Crawler.CrawlPlanner>()
+15
View File
@@ -0,0 +1,15 @@
using MongoDB.Entities;
namespace PrivaPub.Models.Jobs
{
// The address ranges one CDN publishes, as CdnUpdater last downloaded them: kept, so a restart needs no download and a
// failed download keeps the previous list.
public class CdnRangeSet : Entity
{
public string Key { get; set; }//the CDN's domain (CdnProvider.Key)
public string Name { get; set; }
public List<string> Ranges { get; set; } = new();
public List<string> Sources { get; set; } = new();
public DateTime FetchedAt { get; set; }
}
}
+8 -1
View File
@@ -1,4 +1,4 @@
using MongoDB.Entities;
using MongoDB.Entities;
namespace PrivaPub.Models.Jobs
{
@@ -57,6 +57,8 @@ namespace PrivaPub.Models.Jobs
public int? Asn { get; set; }
public string AsnOrg { get; set; }
public string Cdn { get; set; }//the CDN in front of it, whose edge address says nothing about where it runs
public string CdnDomain { get; set; }//the CDN's own domain, which groups its servers (CdnProvider.Key)
public string CdnSource { get; set; }//how the CDN was found: "ranges", "headers" or "asn"
public bool? Ipv6 { get; set; }
public string Source { get; set; }
public DateTime? LocatedAt { get; set; }
@@ -79,8 +81,13 @@ namespace PrivaPub.Models.Jobs
public string RegistrationMode { get; set; }
public List<string> Languages { get; set; }
public string Country { get; set; }
public string City { get; set; }
public double? Latitude { get; set; }
public double? Longitude { get; set; }
public int? Asn { get; set; }
public string Cdn { get; set; }
public string CdnDomain { get; set; }
public string CdnSource { get; set; }
public bool? Ipv6 { get; set; }
}
}
+3 -1
View File
@@ -14,7 +14,9 @@
only counted, through a key that is destroyed at the end of each day.</p>
<p>A server we exchange activities with is described once a week, from its public NodeInfo and, when it has one, its
Mastodon instance API. Its location comes from the address we reached, looked up in an offline database, and is shown
to the city; for a server behind a CDN, only the CDN is shown, since the address is the CDN's.</p>
to the city; for a server behind a CDN, only the CDN is shown, since the address is the CDN's. CDNs are recognised from
the address lists they publish and from their marks in the answers we get, and servers are grouped by their CDN week
by week.</p>
@if (Model.GeoSource is { } geo)
{
<p>Locations come from <a href="https://db-ip.com" rel="nofollow noopener">@geo</a>, licensed under