CDNs found by themselves, and servers followed through time
PrivaPub now finds CDNs three ways, best first: the address ranges the CDNs publish (Cloudflare, Fastly, Amazon CloudFront, Bunny, Gcore, Imperva), downloaded daily by CdnUpdater and kept in CdnRangeSet; the CDN's fingerprint in the responses it already gets from a server (EdgeHintsHandler on the federation client); and the networks that carry only a CDN. The fixed ASN list is gone; ASNs shared with plain hosting (AWS, DataPacket) no longer hide a server. A server's Geo records the CDN, its domain and how it was found, and weekly snapshots now keep the city and coordinates too. Servers through time (ServerPlaces): /instances/:host/history lists a server's weekly snapshots, a CDN-fronted server's geo names the CDN's domain and where the server was before it (before_cdn), and /api/privapub/v1/cdns and /cdns/:domain group servers by CDN with week by week who joined and who left. Owner decisions recorded in ROADMAP. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01LsXgEaXee4GCU1hwYgPJXw
This commit is contained in:
1 parent
5d7edc4a4c
commit
436f7da464
26 files changed
+1159
-62
No files matched your search
@@ -1,4 +1,4 @@
|
||||
using MongoDB.Bson;
|
||||
using MongoDB.Bson;
|
||||
using MongoDB.Driver;
|
||||
using MongoDB.Entities;
|
||||
|
||||
@@ -131,6 +131,9 @@ namespace PrivaPub.Infrastructure.Data
|
||||
await DB.Default.Index<RemoteInstanceSnapshot>().Key(s => s.Host, KeyType.Ascending).Key(s => s.Week, KeyType.Ascending)
|
||||
.Option(o => o.Unique = true).CreateAsync(token);
|
||||
await Plain<RemoteInstanceSnapshot>(token, s => s.Week);
|
||||
await Plain<RemoteInstanceSnapshot>(token, s => s.CdnDomain, s => s.Week);
|
||||
await Plain<RemoteInstanceSnapshot>(token, s => s.Cdn, s => s.Week);
|
||||
await Plain<CdnRangeSet>(token, s => s.Key);
|
||||
await Plain<RemoteInstance>(token, i => i.Seen);
|
||||
await Plain<RemoteInstance>(token, i => i.LastCrawledAt);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,156 @@
|
||||
using System.Net;
|
||||
using System.Text.Json;
|
||||
|
||||
namespace PrivaPub.Infrastructure.Geo
|
||||
{
|
||||
// How a published list of a CDN's addresses is written.
|
||||
public enum CdnListFormat
|
||||
{
|
||||
Lines,//one CIDR per line (Cloudflare)
|
||||
StringArray,//a JSON array of addresses or CIDRs (Bunny)
|
||||
AddressesJson,//{"addresses": [...], "ipv6_addresses"/"addresses_v6": [...]} (Fastly, Gcore)
|
||||
AwsCloudFront,//AWS ip-ranges.json, CLOUDFRONT entries only
|
||||
ImpervaJson//{"ipRanges": [...], "ipv6Ranges": [...]}
|
||||
}
|
||||
|
||||
public sealed record CdnList(string Url, CdnListFormat Format, string Method = "GET", string Body = default);
|
||||
|
||||
// A CDN, or any edge network whose addresses say nothing about where a server runs. Domain names it in statistics.
|
||||
public sealed record CdnProvider(string Name, string Domain, int[] Asns, CdnList[] Lists, Func<HttpResponseMessage, bool> Fingerprint)
|
||||
{
|
||||
public string Key => Domain ?? Name.ToLowerInvariant();
|
||||
}
|
||||
|
||||
// owner decision (2026-10-04): PrivaPub finds CDNs by itself, three ways, best first. The address ranges a CDN
|
||||
// publishes (downloaded daily by CdnUpdater), the CDN's fingerprint in the responses PrivaPub already gets from a server
|
||||
// (EdgeHintsHandler), and the networks (ASN) that carry nothing but a CDN. A network shared with plain hosting (AWS,
|
||||
// Google, Microsoft, DataPacket) is never listed by ASN: only ranges or fingerprints name those CDNs.
|
||||
public static class CdnCatalog
|
||||
{
|
||||
public static readonly IReadOnlyList<CdnProvider> All = new CdnProvider[]
|
||||
{
|
||||
new("Cloudflare", "cloudflare.com", new[] { 13335, 209242 },
|
||||
new[] { new CdnList("https://www.cloudflare.com/ips-v4", CdnListFormat.Lines), new CdnList("https://www.cloudflare.com/ips-v6", CdnListFormat.Lines) },
|
||||
r => Has(r, "cf-ray") || ServerIs(r, "cloudflare")),
|
||||
new("Fastly", "fastly.com", new[] { 54113 },
|
||||
new[] { new CdnList("https://api.fastly.com/public-ip-list", CdnListFormat.AddressesJson) },
|
||||
r => Has(r, "x-fastly-request-id") || Has(r, "fastly-restarts") || Value(r, "x-served-by")?.StartsWith("cache-", StringComparison.OrdinalIgnoreCase) == true),
|
||||
new("Amazon CloudFront", "cloudfront.net", Array.Empty<int>(),
|
||||
new[] { new CdnList("https://ip-ranges.amazonaws.com/ip-ranges.json", CdnListFormat.AwsCloudFront) },
|
||||
r => Has(r, "x-amz-cf-id") || Has(r, "x-amz-cf-pop") || Value(r, "via")?.Contains("cloudfront", StringComparison.OrdinalIgnoreCase) == true),
|
||||
new("Akamai", "akamai.com", new[] { 20940, 16625, 16702, 21342 }, Array.Empty<CdnList>(),
|
||||
r => Has(r, "akamai-grn") || Has(r, "x-akamai-transformed") || Has(r, "akamai-cache-status") || ServerStarts(r, "AkamaiGHost") || ServerStarts(r, "AkamaiNetStorage")),
|
||||
new("Bunny", "bunny.net", new[] { 200325 },
|
||||
new[] { new CdnList("https://api.bunny.net/system/edgeserverlist", CdnListFormat.StringArray), new CdnList("https://api.bunny.net/system/edgeserverlist/IPv6", CdnListFormat.StringArray) },
|
||||
r => ServerStarts(r, "BunnyCDN") || Has(r, "cdn-pullzone") || Has(r, "cdn-requestid")),
|
||||
new("Gcore", "gcore.com", Array.Empty<int>(),
|
||||
new[] { new CdnList("https://api.gcore.com/cdn/public-ip-list", CdnListFormat.AddressesJson) },
|
||||
r => ServerStarts(r, "gcore")),
|
||||
new("Imperva", "imperva.com", new[] { 19551 },
|
||||
new[] { new CdnList("https://my.imperva.com/api/integration/v1/ips", CdnListFormat.ImpervaJson, "POST", "resp_format=json") },
|
||||
r => Has(r, "x-iinfo") || Value(r, "x-cdn") is "Imperva" or "Incapsula"),
|
||||
new("CDN77", "cdn77.com", Array.Empty<int>(), Array.Empty<CdnList>(),
|
||||
r => Has(r, "x-77-nzt") || Has(r, "x-77-cache") || ServerStarts(r, "CDN77")),
|
||||
new("Edgio", "edg.io", new[] { 15133 }, Array.Empty<CdnList>(),
|
||||
r => ServerStarts(r, "ECAcc") || ServerStarts(r, "ECS (") || Has(r, "x-ec-custom-error")),
|
||||
new("Azure Front Door", "azure.microsoft.com", Array.Empty<int>(), Array.Empty<CdnList>(),
|
||||
r => Has(r, "x-azure-ref")),
|
||||
new("Google Cloud", "cloud.google.com", Array.Empty<int>(), Array.Empty<CdnList>(),
|
||||
r => Value(r, "via")?.Contains("google", StringComparison.OrdinalIgnoreCase) == true),
|
||||
new("Vercel", "vercel.com", Array.Empty<int>(), Array.Empty<CdnList>(),
|
||||
r => Has(r, "x-vercel-id") || ServerIs(r, "Vercel")),
|
||||
new("Netlify", "netlify.com", Array.Empty<int>(), Array.Empty<CdnList>(),
|
||||
r => Has(r, "x-nf-request-id") || ServerIs(r, "Netlify")),
|
||||
new("Sucuri", "sucuri.net", Array.Empty<int>(), Array.Empty<CdnList>(),
|
||||
r => Has(r, "x-sucuri-id") || ServerStarts(r, "Sucuri")),
|
||||
new("DDoS-Guard", "ddos-guard.net", Array.Empty<int>(), Array.Empty<CdnList>(),
|
||||
r => ServerIs(r, "ddos-guard"))
|
||||
};
|
||||
|
||||
public static CdnProvider ByKey(string key) =>
|
||||
key == default ? default : All.FirstOrDefault(p => string.Equals(p.Key, key, StringComparison.OrdinalIgnoreCase) || string.Equals(p.Name, key, StringComparison.OrdinalIgnoreCase));
|
||||
|
||||
public static CdnProvider OfAsn(int? asn) => asn is { } number ? All.FirstOrDefault(p => p.Asns.Contains(number)) : default;
|
||||
|
||||
// The CDN a response passed through: a known fingerprint, or a CDN that names itself in X-CDN.
|
||||
public static CdnProvider FromResponse(HttpResponseMessage response)
|
||||
{
|
||||
if (response == default)
|
||||
return default;
|
||||
var known = All.FirstOrDefault(p => p.Fingerprint(response));
|
||||
if (known != default)
|
||||
return known;
|
||||
var named = Value(response, "x-cdn")?.Trim();
|
||||
return named is { Length: > 1 and <= 40 } && named.All(c => char.IsLetterOrDigit(c) || c is ' ' or '-' or '.' or '_')
|
||||
? new CdnProvider(named, default, Array.Empty<int>(), Array.Empty<CdnList>(), _ => false)
|
||||
: default;
|
||||
}
|
||||
|
||||
// The networks a published list holds; an entry that is not an address or a network is an error, never skipped.
|
||||
public static List<IPNetwork> Parse(CdnListFormat format, string body)
|
||||
{
|
||||
IEnumerable<string> entries = format switch
|
||||
{
|
||||
CdnListFormat.Lines => body.Split('\n', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries).Where(l => !l.StartsWith('#')),
|
||||
CdnListFormat.StringArray => Root(body),
|
||||
CdnListFormat.AddressesJson => Json(body, "addresses", "ipv6_addresses", "addresses_v6"),
|
||||
CdnListFormat.ImpervaJson => Json(body, "ipRanges", "ipv6Ranges"),
|
||||
CdnListFormat.AwsCloudFront => CloudFront(body),
|
||||
_ => Enumerable.Empty<string>()
|
||||
};
|
||||
return entries.Select(Network).ToList();
|
||||
}
|
||||
|
||||
static IPNetwork Network(string entry)
|
||||
{
|
||||
if (IPNetwork.TryParse(entry, out var network))
|
||||
return network;
|
||||
if (IPAddress.TryParse(entry, out var address))
|
||||
return new IPNetwork(address, address.AddressFamily == System.Net.Sockets.AddressFamily.InterNetwork ? 32 : 128);
|
||||
throw new FormatException($"not an address or a network: {entry}");
|
||||
}
|
||||
|
||||
static List<string> Root(string body)
|
||||
{
|
||||
using var document = JsonDocument.Parse(body);
|
||||
return Strings(document.RootElement);
|
||||
}
|
||||
|
||||
static IEnumerable<string> Json(string body, params string[] properties)
|
||||
{
|
||||
using var document = JsonDocument.Parse(body);
|
||||
var entries = new List<string>();
|
||||
foreach (var property in properties)
|
||||
if (document.RootElement.TryGetProperty(property, out var array))
|
||||
entries.AddRange(Strings(array));
|
||||
return entries;
|
||||
}
|
||||
|
||||
static IEnumerable<string> CloudFront(string body)
|
||||
{
|
||||
using var document = JsonDocument.Parse(body);
|
||||
var entries = new List<string>();
|
||||
foreach (var (array, field) in new[] { ("prefixes", "ip_prefix"), ("ipv6_prefixes", "ipv6_prefix") })
|
||||
if (document.RootElement.TryGetProperty(array, out var prefixes))
|
||||
foreach (var prefix in prefixes.EnumerateArray())
|
||||
if (prefix.TryGetProperty("service", out var service) && service.GetString() == "CLOUDFRONT" && prefix.TryGetProperty(field, out var value))
|
||||
entries.Add(value.GetString());
|
||||
return entries;
|
||||
}
|
||||
|
||||
static List<string> Strings(JsonElement array) =>
|
||||
array.ValueKind == JsonValueKind.Array ? array.EnumerateArray().Select(e => e.GetString()).ToList() : throw new FormatException("not a list");
|
||||
|
||||
static bool Has(HttpResponseMessage response, string header) =>
|
||||
response.Headers.Contains(header) || response.Content?.Headers.Contains(header) == true;
|
||||
|
||||
static string Value(HttpResponseMessage response, string header) =>
|
||||
response.Headers.TryGetValues(header, out var values) || response.Content?.Headers.TryGetValues(header, out values) == true
|
||||
? string.Join(", ", values)
|
||||
: default;
|
||||
|
||||
static bool ServerIs(HttpResponseMessage response, string name) => string.Equals(Value(response, "server")?.Trim(), name, StringComparison.OrdinalIgnoreCase);
|
||||
|
||||
static bool ServerStarts(HttpResponseMessage response, string prefix) => Value(response, "server")?.TrimStart().StartsWith(prefix, StringComparison.OrdinalIgnoreCase) == true;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,65 @@
|
||||
using MongoDB.Entities;
|
||||
|
||||
using PrivaPub.Models.Jobs;
|
||||
|
||||
using System.Net;
|
||||
|
||||
namespace PrivaPub.Infrastructure.Geo
|
||||
{
|
||||
public interface ICdnRanges
|
||||
{
|
||||
// the CDN whose published ranges hold the address, or default
|
||||
CdnProvider Of(IPAddress address);
|
||||
void Replace(CdnRangeSet set);
|
||||
Task Load(CancellationToken token);
|
||||
// per CDN key: how many ranges are known and when they were downloaded
|
||||
IReadOnlyDictionary<string, (int Ranges, DateTime FetchedAt)> Status { get; }
|
||||
}
|
||||
|
||||
// The published CDN ranges in memory, rebuilt from the stored CdnRangeSets at start and replaced set by set as
|
||||
// CdnUpdater downloads them. A few thousand networks: a linear scan per describe is cheap, and describes are weekly.
|
||||
public class CdnRanges : ICdnRanges
|
||||
{
|
||||
readonly object _gate = new();
|
||||
Dictionary<string, (CdnProvider Provider, IPNetwork[] Networks, DateTime FetchedAt)> _sets = new(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
public IReadOnlyDictionary<string, (int Ranges, DateTime FetchedAt)> Status
|
||||
{
|
||||
get
|
||||
{
|
||||
lock (_gate)
|
||||
return _sets.ToDictionary(s => s.Key, s => (s.Value.Networks.Length, s.Value.FetchedAt), StringComparer.OrdinalIgnoreCase);
|
||||
}
|
||||
}
|
||||
|
||||
public CdnProvider Of(IPAddress address)
|
||||
{
|
||||
if (address == default)
|
||||
return default;
|
||||
if (address.IsIPv4MappedToIPv6)
|
||||
address = address.MapToIPv4();
|
||||
Dictionary<string, (CdnProvider Provider, IPNetwork[] Networks, DateTime FetchedAt)> sets;
|
||||
lock (_gate)
|
||||
sets = _sets;
|
||||
foreach (var (provider, networks, _) in sets.Values)
|
||||
foreach (var network in networks)
|
||||
if (network.BaseAddress.AddressFamily == address.AddressFamily && network.Contains(address))
|
||||
return provider;
|
||||
return default;
|
||||
}
|
||||
|
||||
public void Replace(CdnRangeSet set)
|
||||
{
|
||||
var provider = CdnCatalog.ByKey(set.Key) ?? new CdnProvider(set.Name, set.Key, Array.Empty<int>(), Array.Empty<CdnList>(), _ => false);
|
||||
var networks = set.Ranges.Select(r => IPNetwork.TryParse(r, out var n) ? n : (IPNetwork?)null).Where(n => n != default).Select(n => n!.Value).ToArray();
|
||||
lock (_gate)
|
||||
_sets = new(_sets, StringComparer.OrdinalIgnoreCase) { [set.Key] = (provider, networks, set.FetchedAt) };
|
||||
}
|
||||
|
||||
public async Task Load(CancellationToken token)
|
||||
{
|
||||
foreach (var set in await DB.Default.Find<CdnRangeSet>().ExecuteAsync(token))
|
||||
Replace(set);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,114 @@
|
||||
using Microsoft.Extensions.Options;
|
||||
|
||||
using MongoDB.Entities;
|
||||
|
||||
using PrivaPub.Infrastructure.Statistics;
|
||||
using PrivaPub.Models.Jobs;
|
||||
|
||||
using System.Text;
|
||||
|
||||
namespace PrivaPub.Infrastructure.Geo
|
||||
{
|
||||
// Keeps the CDNs' published address ranges current by itself (owner decision 2026-10-04): at start it loads the stored
|
||||
// ones, then once a day it downloads each list again. A list that fails to download or to parse, or comes back empty,
|
||||
// leaves the stored one in place and is tried the next day. Like GeoUpdater, plain HTTPS to fixed hosts, not federation.
|
||||
public sealed class CdnUpdater : BackgroundService
|
||||
{
|
||||
public const string ClientName = "cdn";
|
||||
static readonly TimeSpan FirstWait = TimeSpan.FromMinutes(1);
|
||||
static readonly TimeSpan Interval = TimeSpan.FromHours(24);
|
||||
const int MaxBytes = 16 * 1024 * 1024;
|
||||
const int MaxRanges = 200_000;
|
||||
|
||||
readonly IHttpClientFactory _http;
|
||||
readonly IOptionsMonitor<StatisticsOptions> _options;
|
||||
readonly ICdnRanges _ranges;
|
||||
readonly ILogger<CdnUpdater> _logger;
|
||||
|
||||
public CdnUpdater(IHttpClientFactory http, IOptionsMonitor<StatisticsOptions> options, ICdnRanges ranges, ILogger<CdnUpdater> logger)
|
||||
{
|
||||
_http = http;
|
||||
_options = options;
|
||||
_ranges = ranges;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
||||
{
|
||||
try
|
||||
{
|
||||
await _ranges.Load(stoppingToken);
|
||||
await Task.Delay(FirstWait, stoppingToken);
|
||||
while (!stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
if (_options.CurrentValue.Cdn.AutoUpdate)
|
||||
await Update(CdnCatalog.All, stoppingToken);
|
||||
await Task.Delay(Interval + TimeSpan.FromMinutes(Random.Shared.Next(60)), stoppingToken);
|
||||
}
|
||||
}
|
||||
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "{Service} stopped", nameof(CdnUpdater));
|
||||
}
|
||||
}
|
||||
|
||||
// The number of CDNs whose ranges were replaced.
|
||||
public async Task<int> Update(IEnumerable<CdnProvider> providers, CancellationToken token)
|
||||
{
|
||||
var replaced = 0;
|
||||
foreach (var provider in providers.Where(p => p.Lists.Length > 0))
|
||||
{
|
||||
try
|
||||
{
|
||||
var ranges = new List<string>();
|
||||
foreach (var list in provider.Lists)
|
||||
ranges.AddRange(CdnCatalog.Parse(list.Format, await Download(list, token)).Select(n => n.ToString()));
|
||||
ranges = ranges.Distinct().ToList();
|
||||
if (ranges.Count == 0 || ranges.Count > MaxRanges)
|
||||
throw new InvalidDataException($"{ranges.Count} ranges");
|
||||
|
||||
var set = await DB.Default.Find<CdnRangeSet>().Match(s => s.Key == provider.Key).ExecuteFirstAsync(token) ?? new CdnRangeSet { Key = provider.Key };
|
||||
set.Name = provider.Name;
|
||||
set.Ranges = ranges;
|
||||
set.Sources = provider.Lists.Select(l => l.Url).ToList();
|
||||
set.FetchedAt = DateTime.UtcNow;
|
||||
await DB.Default.SaveAsync(set, token);
|
||||
_ranges.Replace(set);
|
||||
replaced++;
|
||||
}
|
||||
catch (Exception ex) when (ex is HttpRequestException or InvalidDataException or FormatException or System.Text.Json.JsonException
|
||||
or TaskCanceledException && !token.IsCancellationRequested)
|
||||
{
|
||||
_logger.LogWarning(ex, "Could not update the published ranges of {Cdn}", provider.Name);
|
||||
}
|
||||
}
|
||||
return replaced;
|
||||
}
|
||||
|
||||
async Task<string> Download(CdnList list, CancellationToken token)
|
||||
{
|
||||
using var request = new HttpRequestMessage(list.Method == "POST" ? HttpMethod.Post : HttpMethod.Get, list.Url);
|
||||
request.Headers.Accept.ParseAdd("application/json, text/plain;q=0.9");
|
||||
if (list.Body != default)
|
||||
request.Content = new StringContent(list.Body, Encoding.UTF8, "application/x-www-form-urlencoded");
|
||||
using var response = await _http.CreateClient(ClientName).SendAsync(request, HttpCompletionOption.ResponseHeadersRead, token);
|
||||
response.EnsureSuccessStatusCode();
|
||||
if (response.Content.Headers.ContentLength > MaxBytes)
|
||||
throw new InvalidDataException($"more than {MaxBytes} bytes");
|
||||
await using var stream = await response.Content.ReadAsStreamAsync(token);
|
||||
using var buffer = new MemoryStream();
|
||||
var chunk = new byte[81920];
|
||||
int read;
|
||||
while ((read = await stream.ReadAsync(chunk, token)) > 0)
|
||||
{
|
||||
if (buffer.Length + read > MaxBytes)
|
||||
throw new InvalidDataException($"more than {MaxBytes} bytes");
|
||||
buffer.Write(chunk, 0, read);
|
||||
}
|
||||
return Encoding.UTF8.GetString(buffer.ToArray());
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
using MaxMind.Db;
|
||||
using MaxMind.Db;
|
||||
|
||||
using Microsoft.Extensions.Options;
|
||||
|
||||
@@ -165,23 +165,4 @@ namespace PrivaPub.Infrastructure.Geo
|
||||
_asn?.Dispose();
|
||||
}
|
||||
}
|
||||
|
||||
public static class CdnNetworks
|
||||
{
|
||||
static readonly Dictionary<int, string> Known = new()
|
||||
{
|
||||
[13335] = "Cloudflare",
|
||||
[209242] = "Cloudflare",
|
||||
[54113] = "Fastly",
|
||||
[20940] = "Akamai",
|
||||
[16625] = "Akamai",
|
||||
[16702] = "Akamai",
|
||||
[21342] = "Akamai",
|
||||
[200325] = "Bunny",
|
||||
[60068] = "CDN77",
|
||||
[15133] = "Edgio"
|
||||
};
|
||||
|
||||
public static string Of(int? asn) => asn is { } number && Known.TryGetValue(number, out var name) ? name : default;
|
||||
}
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
using MaxMind.Db;
|
||||
using MaxMind.Db;
|
||||
|
||||
using Microsoft.Extensions.Options;
|
||||
|
||||
@@ -11,8 +11,8 @@ namespace PrivaPub.Infrastructure.Geo
|
||||
{
|
||||
// Keeps the DB-IP Lite databases current by itself, so a deploy needs no timer and no root: once a day it checks
|
||||
// whether each database was built this month and, if not, fetches this month's (or, early in the month, last month's),
|
||||
// checks it opens as the right kind of database and swaps it in. This and SMTP are the server's only traffic that is not
|
||||
// federation, which is why it has its own client instead of IFederationHttp.
|
||||
// checks it opens as the right kind of database and swaps it in. This, CdnUpdater and SMTP are the server's only traffic
|
||||
// that is not federation, which is why each has its own client instead of IFederationHttp.
|
||||
public sealed class GeoUpdater : BackgroundService
|
||||
{
|
||||
public const string ClientName = "geo";
|
||||
|
||||
@@ -0,0 +1,55 @@
|
||||
using PrivaPub.Infrastructure.Geo;
|
||||
|
||||
using System.Collections.Concurrent;
|
||||
|
||||
namespace PrivaPub.Infrastructure.Http
|
||||
{
|
||||
public interface IEdgeHints
|
||||
{
|
||||
void Remember(string host, CdnProvider cdn);
|
||||
CdnProvider Of(string host);
|
||||
}
|
||||
|
||||
// The CDN each remote host's responses came through, as their headers tell it (CdnCatalog fingerprints), for the
|
||||
// weekly description of the server. Like ConnectedAddresses, memory only: what a host answers is never stored per
|
||||
// request.
|
||||
public class EdgeHints : IEdgeHints
|
||||
{
|
||||
const int MaxHosts = 50_000;
|
||||
|
||||
readonly ConcurrentDictionary<string, CdnProvider> _hints = new(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
public void Remember(string host, CdnProvider cdn)
|
||||
{
|
||||
if (string.IsNullOrEmpty(host))
|
||||
return;
|
||||
if (cdn == default)
|
||||
{
|
||||
_hints.TryRemove(host, out _);
|
||||
return;
|
||||
}
|
||||
if (_hints.Count >= MaxHosts)
|
||||
_hints.Clear();
|
||||
_hints[host] = cdn;
|
||||
}
|
||||
|
||||
public CdnProvider Of(string host) => host != default && _hints.TryGetValue(host, out var cdn) ? cdn : default;
|
||||
}
|
||||
|
||||
// On the federation client: reads each response's CDN fingerprint for its host. A response without one forgets the
|
||||
// host's hint, so a server that leaves its CDN is seen as itself again.
|
||||
public class EdgeHintsHandler : DelegatingHandler
|
||||
{
|
||||
readonly IEdgeHints _hints;
|
||||
|
||||
public EdgeHintsHandler(IEdgeHints hints) => _hints = hints;
|
||||
|
||||
protected override async Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
|
||||
{
|
||||
var response = await base.SendAsync(request, cancellationToken);
|
||||
if (request.RequestUri is { } uri && (int)response.StatusCode is >= 200 and < 500)
|
||||
_hints.Remember(uri.Host, CdnCatalog.FromResponse(response));
|
||||
return response;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -6,6 +6,13 @@
|
||||
public string GeoDirectory { get; set; } = "/var/lib/privapub/geo";
|
||||
public CrawlerOptions Crawler { get; set; } = new();
|
||||
public GeoOptions Geo { get; set; } = new();
|
||||
public CdnOptions Cdn { get; set; } = new();
|
||||
}
|
||||
|
||||
//owner decision (2026-10-04): the server downloads the CDNs' published address ranges daily (CdnUpdater)
|
||||
public class CdnOptions
|
||||
{
|
||||
public bool AutoUpdate { get; set; } = true;
|
||||
}
|
||||
|
||||
//owner decision (2026-10-04): the server keeps its DB-IP Lite databases current by itself
|
||||
|
||||
Reference in new issue
Block a user