M8: every server we touch is described, located and snapshotted weekly
- Touches: the ledger marks a server as touched when it sends us a verified activity, when
we exchange activities with it, or when we read its actors, keys, objects or WebFinger.
It upserts RemoteInstance.Seen, FirstSeenAt and LastSeenAt at most hourly per server, and
queues one DescribeInstance a week with the same dedupe key ObjectRecords uses. Suspended
servers and pages behind link previews are never described. Migration _010 marks the
servers already known as touched, with their dates.
- InstanceDescriber.Describe(host, crawled, allowed) reads:
- NodeInfo 2.2/2.1/2.0, now with its published user counts, posts, comments,
description, languages and schema version;
- for software with a Mastodon API, /api/v2/instance falling back to v1: title,
languages, registration mode, character limit, API version, source URL.
It never keeps a contact as a field; the raw document is kept for the admin only. It
locates the server from the address our connection reached (DB-IP Lite city and ASN, the
CDN named when fronted) and writes a RemoteInstanceSnapshot per ISO week, unreachable
weeks included. A crawled server is upserted as crawled only on insert, so it never
downgrades a touched one, and robots.txt can deny any path.
- PublicGeo.Project is the only public form of a location: a CDN-fronted server shows its
CDN only, a server reporting at least ten users shows its city, coordinates and network,
any other only its country.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01ELjqpznMFMNrJoJUj6K5p2
This commit is contained in:
thepraandClaude Opus 5.5 committed
1 parent
fc5bb9511f
commit
c8a305ae92
10 files changed
+757
-72
No files matched your search
@@ -1,88 +1,180 @@
|
||||
using MongoDB.Entities;
|
||||
|
||||
using PrivaPub.Infrastructure.Geo;
|
||||
using PrivaPub.Infrastructure.Http;
|
||||
using PrivaPub.Infrastructure.Jobs;
|
||||
using PrivaPub.Models.Jobs;
|
||||
|
||||
using System.Globalization;
|
||||
using System.Net.Sockets;
|
||||
using System.Text.Json;
|
||||
|
||||
namespace PrivaPub.Federation.Objects
|
||||
{
|
||||
public class InstanceDescriber : IJobHandler
|
||||
{
|
||||
const int MaxNodeInfoChars = 64 * 1024;
|
||||
static readonly string[] Schemas =
|
||||
{
|
||||
"http://nodeinfo.diaspora.software/ns/schema/2.1",
|
||||
"http://nodeinfo.diaspora.software/ns/schema/2.0"
|
||||
};
|
||||
const int MaxRawChars = 64 * 1024;
|
||||
|
||||
readonly IFederationHttp _http;
|
||||
readonly IGeoLocator _geo;
|
||||
readonly IConnectedAddresses _addresses;
|
||||
|
||||
public InstanceDescriber(IFederationHttp http) => _http = http;
|
||||
public InstanceDescriber(IFederationHttp http, IGeoLocator geo = default, IConnectedAddresses addresses = default)
|
||||
{
|
||||
_http = http;
|
||||
_geo = geo;
|
||||
_addresses = addresses;
|
||||
}
|
||||
|
||||
public JobKind Kind => JobKind.DescribeInstance;
|
||||
public int Concurrency => 1;
|
||||
public int MaxAttempts => 2;
|
||||
public int PerHostLimit => 1;
|
||||
|
||||
public static string Week(DateTime at) => $"{ISOWeek.GetYear(at)}-W{ISOWeek.GetWeekOfYear(at):00}";
|
||||
|
||||
public static string DedupeKey(string host, DateTime at) => $"describe|{host}|{ISOWeek.GetYear(at)}-{ISOWeek.GetWeekOfYear(at)}";
|
||||
|
||||
public async Task<JobOutcome> Handle(Job job, CancellationToken token)
|
||||
{
|
||||
var host = job.Payload;
|
||||
using var scope = HttpScope.For("nodeinfo");
|
||||
using var links = await _http.GetJson(Address($"https://{host}/.well-known/nodeinfo"), "application/json", sign: default, token);
|
||||
var href = links == default ? default : Schemas.Select(schema => LinkFor(links.Root, schema)).FirstOrDefault(link => link != default);
|
||||
if (href == default)
|
||||
return await Described(host, default, "no NodeInfo", token);
|
||||
await Describe(job.Payload, crawled: false, default, token);
|
||||
return JobOutcome.Done;
|
||||
}
|
||||
|
||||
using var nodeInfo = await _http.GetJson(Address(href), "application/json", sign: default, token);
|
||||
return nodeInfo == default ? await Described(host, default, "NodeInfo unreadable", token) : await Described(host, nodeInfo.Root, default, token);
|
||||
//NodeInfo, the Mastodon instance API where the software has one, and where the address we reached sits; unsigned,
|
||||
//because none of these are ActivityPub documents. `allowed` is the crawler's robots.txt.
|
||||
public async Task<bool> Describe(string host, bool crawled, Func<string, bool> allowed, CancellationToken token)
|
||||
{
|
||||
allowed ??= _ => true;
|
||||
NodeInfoFacts facts = default;
|
||||
string rawNodeInfo = default, error;
|
||||
using (HttpScope.For("nodeinfo"))
|
||||
{
|
||||
error = allowed("/.well-known/nodeinfo") ? "no NodeInfo" : "robots.txt disallows NodeInfo";
|
||||
using var links = allowed("/.well-known/nodeinfo")
|
||||
? await _http.GetJson(Address($"https://{host}/.well-known/nodeinfo"), "application/json", sign: default, token)
|
||||
: default;
|
||||
var (href, version) = links == default ? default : InstanceDocuments.NodeInfoLink(links.Root);
|
||||
if (href != default && allowed(new Uri(href).AbsolutePath))
|
||||
{
|
||||
using var nodeInfo = await _http.GetJson(Address(href), "application/json", sign: default, token);
|
||||
if (nodeInfo != default)
|
||||
{
|
||||
facts = InstanceDocuments.NodeInfo(nodeInfo.Root, version);
|
||||
rawNodeInfo = nodeInfo.Root.GetRawText();
|
||||
error = default;
|
||||
}
|
||||
else
|
||||
error = "NodeInfo unreadable";
|
||||
}
|
||||
}
|
||||
|
||||
InstanceApiFacts api = default;
|
||||
string apiKind = default, rawApi = default;
|
||||
if (InstanceDocuments.SpeaksMastodonApi(facts?.Software))
|
||||
using (HttpScope.For("instance-api"))
|
||||
foreach (var (path, kind) in new[] { ("/api/v2/instance", "mastodon-v2"), ("/api/v1/instance", "mastodon-v1") })
|
||||
{
|
||||
if (!allowed(path))
|
||||
continue;
|
||||
using var document = await _http.GetJson(Address($"https://{host}{path}"), "application/json", sign: default, token);
|
||||
if (document == default || document.Root.ValueKind != JsonValueKind.Object)
|
||||
continue;
|
||||
api = InstanceDocuments.InstanceApi(document.Root);
|
||||
apiKind = kind;
|
||||
rawApi = document.Root.GetRawText();
|
||||
break;
|
||||
}
|
||||
|
||||
var geo = Locate(host);
|
||||
var now = DateTime.UtcNow;
|
||||
var update = DB.Default.Update<RemoteInstance>()
|
||||
.Match(i => i.Host == host)
|
||||
.Modify(i => i.DescribedAt, now)
|
||||
.Modify(i => i.DescriptionError, error)
|
||||
.Option(o => o.IsUpsert = true);
|
||||
if (crawled)
|
||||
update = update
|
||||
.Modify(b => b.SetOnInsert(i => i.Seen, "crawled"))
|
||||
.Modify(b => b.SetOnInsert(i => i.FirstSeenAt, now));
|
||||
if (facts != default)
|
||||
update = update
|
||||
.Modify(i => i.Software, facts.Software)
|
||||
.Modify(i => i.SoftwareVersion, facts.SoftwareVersion)
|
||||
.Modify(i => i.NodeName, facts.NodeName)
|
||||
.Modify(i => i.NodeDescription, facts.NodeDescription)
|
||||
.Modify(i => i.NodeInfoVersion, facts.Version)
|
||||
.Modify(i => i.Protocols, facts.Protocols)
|
||||
.Modify(i => i.OpenRegistrations, facts.OpenRegistrations)
|
||||
.Modify(i => i.UsersTotal, facts.UsersTotal)
|
||||
.Modify(i => i.UsersActiveMonth, facts.UsersActiveMonth)
|
||||
.Modify(i => i.UsersActiveHalfyear, facts.UsersActiveHalfyear)
|
||||
.Modify(i => i.LocalPosts, facts.LocalPosts)
|
||||
.Modify(i => i.LocalComments, facts.LocalComments)
|
||||
.Modify(i => i.NodeInfo, rawNodeInfo.Length <= MaxRawChars ? rawNodeInfo : default);
|
||||
if (api != default)
|
||||
update = update
|
||||
.Modify(i => i.InstanceApi, apiKind)
|
||||
.Modify(i => i.InstanceApiRaw, rawApi.Length <= MaxRawChars ? rawApi : default)
|
||||
.Modify(i => i.Title, api.Title)
|
||||
.Modify(i => i.Languages, api.Languages ?? facts?.Languages)
|
||||
.Modify(i => i.RegistrationMode, api.RegistrationMode)
|
||||
.Modify(i => i.MaxPostChars, api.MaxPostChars)
|
||||
.Modify(i => i.MastodonApiVersion, api.ApiVersion)
|
||||
.Modify(i => i.SourceUrl, api.SourceUrl);
|
||||
else if (facts?.Languages != default)
|
||||
update = update.Modify(i => i.Languages, facts.Languages);
|
||||
if (geo != default)
|
||||
update = update.Modify(i => i.Geo, geo);
|
||||
await update.ExecuteAsync(token);
|
||||
var instance = await DB.Default.Find<RemoteInstance>().Match(i => i.Host == host).ExecuteFirstAsync(token);
|
||||
|
||||
var week = Week(now);
|
||||
await DB.Default.Update<RemoteInstanceSnapshot>()
|
||||
.Match(s => s.Host == host && s.Week == week)
|
||||
.Modify(s => s.TakenAt, now)
|
||||
.Modify(s => s.Seen, instance?.Seen)
|
||||
.Modify(s => s.Reachable, facts != default || api != default)
|
||||
.Modify(s => s.Software, instance?.Software)
|
||||
.Modify(s => s.SoftwareVersion, instance?.SoftwareVersion)
|
||||
.Modify(s => s.UsersTotal, instance?.UsersTotal)
|
||||
.Modify(s => s.UsersActiveMonth, instance?.UsersActiveMonth)
|
||||
.Modify(s => s.UsersActiveHalfyear, instance?.UsersActiveHalfyear)
|
||||
.Modify(s => s.LocalPosts, instance?.LocalPosts)
|
||||
.Modify(s => s.OpenRegistrations, instance?.OpenRegistrations)
|
||||
.Modify(s => s.RegistrationMode, instance?.RegistrationMode)
|
||||
.Modify(s => s.Languages, instance?.Languages)
|
||||
.Modify(s => s.Country, instance?.Geo?.Country)
|
||||
.Modify(s => s.Asn, instance?.Geo?.Asn)
|
||||
.Modify(s => s.Cdn, instance?.Geo?.Cdn)
|
||||
.Modify(s => s.Ipv6, instance?.Geo?.Ipv6)
|
||||
.Option(o => o.IsUpsert = true)
|
||||
.ExecuteAsync(token);
|
||||
return facts != default;
|
||||
}
|
||||
|
||||
InstanceGeo Locate(string host)
|
||||
{
|
||||
var reached = _addresses?.Of(new Uri(Address($"https://{host}/")).Host);
|
||||
var fix = reached == default ? default : _geo?.Locate(reached);
|
||||
if (fix == default)
|
||||
return default;
|
||||
return new InstanceGeo
|
||||
{
|
||||
Country = fix.Country,
|
||||
City = fix.City,
|
||||
Latitude = fix.Latitude,
|
||||
Longitude = fix.Longitude,
|
||||
Asn = fix.Asn,
|
||||
AsnOrg = fix.AsnOrg,
|
||||
Cdn = CdnNetworks.Of(fix.Asn),
|
||||
Ipv6 = reached.AddressFamily == AddressFamily.InterNetworkV6,
|
||||
Source = _geo.Source,
|
||||
LocatedAt = DateTime.UtcNow
|
||||
};
|
||||
}
|
||||
|
||||
//where an https address is fetched from: itself, except in tests, which point it at a plain-http peer
|
||||
protected virtual string Address(string url) => url;
|
||||
|
||||
static async Task<JobOutcome> Described(string host, JsonElement? nodeInfo, string error, CancellationToken token)
|
||||
{
|
||||
var update = DB.Default.Update<RemoteInstance>()
|
||||
.Match(i => i.Host == host)
|
||||
.Modify(i => i.DescribedAt, DateTime.UtcNow)
|
||||
.Modify(i => i.DescriptionError, error)
|
||||
.Option(o => o.IsUpsert = true);
|
||||
if (nodeInfo is { } root)
|
||||
{
|
||||
var raw = root.GetRawText();
|
||||
update = update
|
||||
.Modify(i => i.Software, Text(root, "software", "name"))
|
||||
.Modify(i => i.SoftwareVersion, Text(root, "software", "version"))
|
||||
.Modify(i => i.NodeName, Text(root, "metadata", "nodeName"))
|
||||
.Modify(i => i.Protocols, root.TryGetProperty("protocols", out var protocols) && protocols.ValueKind == JsonValueKind.Array
|
||||
? protocols.EnumerateArray().Where(p => p.ValueKind == JsonValueKind.String).Select(p => p.GetString()).Take(16).ToList()
|
||||
: new List<string>())
|
||||
.Modify(i => i.OpenRegistrations, root.TryGetProperty("openRegistrations", out var open) && open.ValueKind is JsonValueKind.True or JsonValueKind.False
|
||||
? open.GetBoolean()
|
||||
: default(bool?))
|
||||
.Modify(i => i.NodeInfo, raw.Length <= MaxNodeInfoChars ? raw : default);
|
||||
}
|
||||
await update.ExecuteAsync(token);
|
||||
return JobOutcome.Done;
|
||||
}
|
||||
|
||||
static string LinkFor(JsonElement root, string schema) =>
|
||||
root.TryGetProperty("links", out var links) && links.ValueKind == JsonValueKind.Array
|
||||
? links.EnumerateArray()
|
||||
.Where(l => l.ValueKind == JsonValueKind.Object && l.TryGetProperty("rel", out var rel) && rel.ValueKind == JsonValueKind.String && rel.GetString() == schema)
|
||||
.Select(l => l.TryGetProperty("href", out var href) && href.ValueKind == JsonValueKind.String ? href.GetString() : default)
|
||||
.FirstOrDefault(h => h != default && h.StartsWith("https://", StringComparison.OrdinalIgnoreCase))
|
||||
: default;
|
||||
|
||||
static string Text(JsonElement root, string parent, string name)
|
||||
{
|
||||
if (!root.TryGetProperty(parent, out var section) || section.ValueKind != JsonValueKind.Object
|
||||
|| !section.TryGetProperty(name, out var value) || value.ValueKind != JsonValueKind.String)
|
||||
return default;
|
||||
var text = value.GetString();
|
||||
return text.Length <= 200 ? text : text[..200];
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user