thepra/SocialPub
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

CDNs found by themselves, and servers followed through time
Build / Build (push) Successful in 5m11s
Details
Deploy / privapub.thepra.dev (push) Successful in 5m48s
Details

Browse Source 
PrivaPub now finds CDNs three ways, best first: the address ranges the
CDNs publish (Cloudflare, Fastly, Amazon CloudFront, Bunny, Gcore,
Imperva), downloaded daily by CdnUpdater and kept in CdnRangeSet; the
CDN's fingerprint in the responses it already gets from a server
(EdgeHintsHandler on the federation client); and the networks that carry
only a CDN. The fixed ASN list is gone; ASNs shared with plain hosting
(AWS, DataPacket) no longer hide a server. A server's Geo records the
CDN, its domain and how it was found, and weekly snapshots now keep the
city and coordinates too.

Servers through time (ServerPlaces): /instances/:host/history lists a
server's weekly snapshots, a CDN-fronted server's geo names the CDN's
domain and where the server was before it (before_cdn), and
/api/privapub/v1/cdns and /cdns/:domain group servers by CDN with week
by week who joined and who left. Owner decisions recorded in ROADMAP.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01LsXgEaXee4GCU1hwYgPJXw
This commit is contained in:
thepraandClaude Opus 5.5 committed 2026-10-04 11:33:39 +02:00
1 parent 5d7edc4a4c
commit 436f7da464
26 files changed
+1159 -62
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

PrivaPub/Api/Mastodon/Controllers/ProvenanceController.cs
+76 -9
View File
@@ -10,6 +10,7 @@ using PrivaPub.Domain.Statistics;
using PrivaPub.Federation.Actors;
using PrivaPub.Federation.Objects;
using PrivaPub.Infrastructure;
using PrivaPub.Infrastructure.Geo;
using PrivaPub.Models.Federation;
using PrivaPub.Models.Jobs;
using PrivaPub.Models.Post;
@@ -26,15 +27,19 @@ namespace PrivaPub.Api.Mastodon.Controllers
const int MaxHosts = 40;
readonly DbEntities _dbEntities;
const int MaxWeeks = 260;
readonly ISelfLocation _self;
readonly IServerPlaces _places;
readonly ILocalActorService _localActors;
readonly IOptionsMonitor<RegistrationOptions> _registrations;
public ProvenanceController(DbEntities dbEntities, ISelfLocation self, ILocalActorService localActors,
public ProvenanceController(DbEntities dbEntities, ISelfLocation self, IServerPlaces places, ILocalActorService localActors,
IOptionsMonitor<RegistrationOptions> registrations)
{
_dbEntities = dbEntities;
_self = self;
_places = places;
_localActors = localActors;
_registrations = registrations;
}
@@ -54,7 +59,8 @@ namespace PrivaPub.Api.Mastodon.Controllers
?? await DB.Default.Find<ObjectRecord>().Match(r => r.ObjectURI == post.ObjectURI).ExecuteFirstAsync(token);
var host = record?.Host ?? (Uri.TryCreate(post.ObjectURI, UriKind.Absolute, out var uri) ? uri.Host.ToLowerInvariant() : default);
var instance = host == default ? default : await DB.Default.Find<RemoteInstance>().Match(i => i.Host == host).ExecuteFirstAsync(token);
return Json(Describe(post, record, instance));
var beforeCdn = instance?.Geo?.Cdn == default ? default : (await _places.BeforeCdn(new[] { instance.Host }, token)).GetValueOrDefault(instance.Host);
return Json(Describe(post, record, instance, beforeCdn));
}
[HttpGet("/api/privapub/v1/instances/{host}"), Scope("read", requiresUser: false), AllowAnonymous]
@@ -68,19 +74,47 @@ namespace PrivaPub.Api.Mastodon.Controllers
[HttpGet("/api/privapub/v1/instances"), Scope("read", requiresUser: false), AllowAnonymous]
public async Task<IActionResult> Instances(CancellationToken token) => Json(await Describe(Params.List("host"), token));
//the server's weekly snapshots, newest first (weeks=, at most MaxWeeks): its place as shown publicly, its CDN, its size
[HttpGet("/api/privapub/v1/instances/{host}/history"), Scope("read", requiresUser: false), AllowAnonymous]
public async Task<IActionResult> History(string host, CancellationToken token)
{
host = host?.Trim().ToLowerInvariant();
if (string.IsNullOrEmpty(host) || host.Length > 253 || !await DB.Default.Find<RemoteInstance>().Match(i => i.Host == host).ExecuteAnyAsync(token))
return NotFoundError();
var weeks = Math.Clamp(Params.Int("weeks") ?? 52, 1, MaxWeeks);
return Json((await _places.History(host, weeks, token)).Select(s => new InstanceWeek
{
Week = s.Week,
TakenAt = MastodonJson.Time(s.TakenAt),
Reachable = s.Reachable,
Software = s.Software,
Version = s.SoftwareVersion,
UsersTotal = s.UsersTotal,
UsersActiveMonth = s.UsersActiveMonth,
LocalPosts = s.LocalPosts,
OpenRegistrations = s.OpenRegistrations,
Geo = Located(PublicGeo.Project(new InstanceGeo
{
Country = s.Country, City = s.City, Latitude = s.Latitude, Longitude = s.Longitude, Asn = s.Asn,
Cdn = s.Cdn, CdnDomain = s.CdnDomain, CdnSource = s.CdnSource, Ipv6 = s.Ipv6
}), new InstanceGeo { Cdn = s.Cdn, CdnDomain = s.CdnDomain ?? CdnCatalog.ByKey(s.Cdn)?.Key, CdnSource = s.CdnSource })
}).ToList());
}
async Task<List<InstanceDescription>> Describe(IEnumerable<string> hosts, CancellationToken token)
{
var wanted = hosts.Select(h => h?.Trim().ToLowerInvariant()).Where(h => !string.IsNullOrEmpty(h) && h.Length <= 253)
.Distinct().Take(MaxHosts).ToList();
var selfHost = Uri.TryCreate(_localActors.BaseAddress, UriKind.Absolute, out var self) ? self.Host.ToLowerInvariant() : default;
var remote = await DB.Default.Find<RemoteInstance>().Match(i => wanted.Contains(i.Host)).ExecuteAsync(token);
var beforeCdn = await _places.BeforeCdn(remote.Where(i => i.Geo?.Cdn != default).Select(i => i.Host), token);
var described = new List<InstanceDescription>();
foreach (var host in wanted)
{
if (host == selfHost)
described.Add(await DescribeSelf(host, token));
else if (remote.FirstOrDefault(i => i.Host == host) is { } instance)
described.Add(Describe(instance));
described.Add(Describe(instance, beforeCdn.GetValueOrDefault(host)));
}
return described;
}
@@ -101,8 +135,9 @@ namespace PrivaPub.Api.Mastodon.Controllers
};
}
//the public projection of a server's place (PublicGeo)
static InstanceLocation Located(PublicLocation location, InstanceGeo geo) => location == default ? default : new InstanceLocation
//the public projection of a server's place (PublicGeo); for a CDN-fronted one, the CDN's domain, how it was found and
//where the server was before it
static InstanceLocation Located(PublicLocation location, InstanceGeo geo, PlaceBeforeCdn beforeCdn = default) => location == default ? default : new InstanceLocation
{
Precision = location.Cdn != default ? "cdn" : location.Latitude != default ? "city" : "country",
Country = location.Country,
@@ -111,11 +146,17 @@ namespace PrivaPub.Api.Mastodon.Controllers
Longitude = location.Longitude,
Network = location.Network,
Cdn = location.Cdn,
CdnDomain = location.Cdn != default ? ServerPlaces.KeyOf(geo) : default,
CdnSource = location.Cdn != default ? geo?.CdnSource : default,
BeforeCdn = location.Cdn == default || beforeCdn == default ? default : new InstancePlace
{
Week = beforeCdn.Week, Country = beforeCdn.Country, City = beforeCdn.City, Latitude = beforeCdn.Latitude, Longitude = beforeCdn.Longitude
},
Source = geo?.Source,
LocatedAt = geo?.LocatedAt is { } at ? MastodonJson.Time(at) : default
};
Provenance Describe(PostEntity post, ObjectRecord record, RemoteInstance instance)
Provenance Describe(PostEntity post, ObjectRecord record, RemoteInstance instance, PlaceBeforeCdn beforeCdn)
{
var raw = record?.Raw == default ? default : JsonNode.Parse(record.Raw) as JsonObject;
return new Provenance
@@ -152,11 +193,11 @@ namespace PrivaPub.Api.Mastodon.Controllers
RawHash = r.RawHash,
Raw = r.Raw == default ? default : JsonNode.Parse(r.Raw)
}).ToList() ?? new(),
Instance = instance == default ? default : Describe(instance)
Instance = instance == default ? default : Describe(instance, beforeCdn)
};
}
static InstanceDescription Describe(RemoteInstance instance) => new()
static InstanceDescription Describe(RemoteInstance instance, PlaceBeforeCdn beforeCdn) => new()
{
Host = instance.Host,
Software = instance.Software,
@@ -174,7 +215,7 @@ namespace PrivaPub.Api.Mastodon.Controllers
LastSuccessAt = instance.LastSuccessAt is { } success ? MastodonJson.Time(success) : default,
LastFailureAt = instance.LastFailureAt is { } failure ? MastodonJson.Time(failure) : default
},
Geo = Located(PublicGeo.Project(instance.Geo), instance.Geo)
Geo = Located(PublicGeo.Project(instance.Geo), instance.Geo, beforeCdn)
};
public class Provenance
@@ -251,10 +292,36 @@ namespace PrivaPub.Api.Mastodon.Controllers
public double? Longitude { get; set; }
public string Network { get; set; }
public string Cdn { get; set; }
public string CdnDomain { get; set; }
public string CdnSource { get; set; }//"ranges", "headers" or "asn"
public InstancePlace BeforeCdn { get; set; }//where a CDN-fronted server was located before the CDN, by its snapshots
public string Source { get; set; }
public string LocatedAt { get; set; }
}
public class InstancePlace
{
public string Week { get; set; }
public string Country { get; set; }
public string City { get; set; }
public double? Latitude { get; set; }
public double? Longitude { get; set; }
}
public class InstanceWeek
{
public string Week { get; set; }
public string TakenAt { get; set; }
public bool Reachable { get; set; }
public string Software { get; set; }
public string Version { get; set; }
public long? UsersTotal { get; set; }
public long? UsersActiveMonth { get; set; }
public long? LocalPosts { get; set; }
public bool? OpenRegistrations { get; set; }
public InstanceLocation Geo { get; set; }
}
public class InstanceDelivery
{
public int ConsecutiveFailures { get; set; }