M8: every server we touch is described, located and snapshotted weekly

- Touches: the ledger marks a server as touched when it sends us a verified activity, when
  we exchange activities with it, or when we read its actors, keys, objects or WebFinger.
  It upserts RemoteInstance.Seen, FirstSeenAt and LastSeenAt at most hourly per server, and
  queues one DescribeInstance a week with the same dedupe key ObjectRecords uses. Suspended
  servers and pages behind link previews are never described. Migration _010 marks the
  servers already known as touched, with their dates.
- InstanceDescriber.Describe(host, crawled, allowed) reads:
  - NodeInfo 2.2/2.1/2.0, now with its published user counts, posts, comments,
    description, languages and schema version;
  - for software with a Mastodon API, /api/v2/instance falling back to v1: title,
    languages, registration mode, character limit, API version, source URL.

  It never keeps a contact as a field; the raw document is kept for the admin only. It
  locates the server from the address our connection reached (DB-IP Lite city and ASN, the
  CDN named when fronted) and writes a RemoteInstanceSnapshot per ISO week, unreachable
  weeks included. A crawled server is upserted as crawled only on insert, so it never
  downgrades a touched one, and robots.txt can deny any path.
- PublicGeo.Project is the only public form of a location: a CDN-fronted server shows its
  CDN only, a server reporting at least ten users shows its city, coordinates and network,
  any other only its country.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01ELjqpznMFMNrJoJUj6K5p2

PrivaPub/Infrastructure/Data/Indexes.cs

@@ -127,6 +127,11 @@ namespace PrivaPub.Infrastructure.Data
 			await DB.Default.Index<InstanceDay>().Key(d => d.Day, KeyType.Ascending).Key(d => d.Host, KeyType.Ascending).Option(o => o.Unique = true).CreateAsync(token);
 			await Plain<InstanceDay>(token, d => d.Host, d => d.Day);
 			await DB.Default.Index<ServerDay>().Key(d => d.Day, KeyType.Ascending).Option(o => o.Unique = true).CreateAsync(token);
+			await DB.Default.Index<RemoteInstanceSnapshot>().Key(s => s.Host, KeyType.Ascending).Key(s => s.Week, KeyType.Ascending)
+				.Option(o => o.Unique = true).CreateAsync(token);
+			await Plain<RemoteInstanceSnapshot>(token, s => s.Week);
+			await Plain<RemoteInstance>(token, i => i.Seen);
+			await Plain<RemoteInstance>(token, i => i.LastCrawledAt);
 		}
 
 		static async Task Unique<T>(System.Linq.Expressions.Expression<Func<T, object>> key, FilterDefinition<T> partial,

PrivaPub/Infrastructure/Data/Migrations/_010_remote_instances_were_touched.cs

@@ -0,0 +1,24 @@
+using MongoDB.Bson;
+using MongoDB.Entities;
+
+using PrivaPub.Models.Jobs;
+
+namespace PrivaPub.Infrastructure.Data.Migrations
+{
+	public class _010_remote_instances_were_touched : IMigration
+	{
+		public async Task UpgradeAsync()
+		{
+			foreach (var instance in await DB.Default.Find<RemoteInstance>().Match(i => i.Seen == null).ExecuteAsync())
+			{
+				var firstSeen = ObjectId.TryParse(instance.ID, out var id) ? id.CreationTime : DateTime.UtcNow;
+				var lastSeen = new[] { instance.LastSuccessAt, instance.LastFailureAt, instance.DescribedAt }.Max() ?? firstSeen;
+				await DB.Default.Update<RemoteInstance>().MatchID(instance.ID)
+					.Modify(i => i.Seen, "touched")
+					.Modify(i => i.FirstSeenAt, firstSeen)
+					.Modify(i => i.LastSeenAt, lastSeen)
+					.ExecuteAsync();
+			}
+		}
+	}
+}

PrivaPub/Infrastructure/Statistics/InteractionLedger.cs

@@ -3,6 +3,8 @@ using Microsoft.Extensions.Options;
 using MongoDB.Driver;
 using MongoDB.Entities;
 
+using PrivaPub.Federation.Moderation;
+using PrivaPub.Infrastructure.Jobs;
 using PrivaPub.Models.Jobs;
 using PrivaPub.Models.Statistics;
 
@@ -41,9 +43,16 @@ namespace PrivaPub.Infrastructure.Statistics
 			SingleWriter = false,
 			SingleReader = false
 		});
+		static readonly TimeSpan TouchInterval = TimeSpan.FromHours(1);
+		static readonly HashSet<string> TouchingPurposes = new(StringComparer.Ordinal) { "actor", "key", "object", "webfinger", "context" };
+
 		readonly InteractionSalts _salts;
 		readonly IOptionsMonitor<StatisticsOptions> _options;
 		readonly ILogger<InteractionLedger> _logger;
+		readonly IJobQueue _queue;
+		readonly IDomainBlocks _blocks;
+		readonly ConcurrentDictionary<string, byte> _touches = new(StringComparer.Ordinal);
+		readonly ConcurrentDictionary<string, DateTime> _touchedAt = new(StringComparer.Ordinal);
 		readonly SemaphoreSlim _flushing = new(1, 1);
 		readonly ConcurrentDictionary<(DateTime Day, string Host, string Key), long> _reads = new();
 		readonly ConcurrentDictionary<(DateTime Day, string Field, string Key), long> _server = new();
@@ -54,11 +63,14 @@ namespace PrivaPub.Infrastructure.Statistics
 		long _written;
 		long _failed;
 
-		public InteractionLedger(InteractionSalts salts, IOptionsMonitor<StatisticsOptions> options, ILogger<InteractionLedger> logger)
+		public InteractionLedger(InteractionSalts salts, IOptionsMonitor<StatisticsOptions> options, ILogger<InteractionLedger> logger,
+			IJobQueue queue = default, IDomainBlocks blocks = default)
 		{
 			_salts = salts;
 			_options = options;
 			_logger = logger;
+			_queue = queue;
+			_blocks = blocks;
 		}
 
 		public long Dropped => Interlocked.Read(ref _dropped);
@@ -85,6 +97,9 @@ namespace PrivaPub.Infrastructure.Statistics
 			var day = DateTime.UtcNow.Date;
 			host = Interactions.Host(host);
 			key = Key(key);
+			if (key.StartsWith("http:", StringComparison.Ordinal) && key.EndsWith(":ok", StringComparison.Ordinal)
+				&& TouchingPurposes.Contains(key.Split(':')[1]))
+				_touches[host] = 0;
 			_reads.AddOrUpdate((day, host, key), 1, (_, count) => count + 1);
 			if (bytes > 0)
 				_reads.AddOrUpdate((day, host, key + ":bytes"), bytes, (_, count) => count + bytes);
@@ -118,6 +133,7 @@ namespace PrivaPub.Infrastructure.Statistics
 					{
 					}
 					await FlushEvents(stoppingToken);
+					await FlushTouches(stoppingToken);
 					if (DateTime.UtcNow - countersAt >= CounterInterval)
 					{
 						countersAt = DateTime.UtcNow;
@@ -152,9 +168,56 @@ namespace PrivaPub.Infrastructure.Statistics
 		public async Task Flush(CancellationToken token)
 		{
 			await FlushEvents(token);
+			await FlushTouches(token);
 			await FlushCounters(token);
 		}
 
+		static bool Touches(InteractionEvent e, bool verified) => e.Channel switch
+		{
+			Interactions.Receive => verified && e.Outcome == Interactions.Queued,
+			Interactions.In or Interactions.Out => true,
+			Interactions.Http => e.Outcome == Interactions.Ok && TouchingPurposes.Contains(e.Purpose ?? string.Empty),
+			_ => false
+		};
+
+		//a server we exchanged something with is marked as seen, and described once a week
+		async Task FlushTouches(CancellationToken token)
+		{
+			await _flushing.WaitAsync(token);
+			try
+			{
+				await Touch(token);
+			}
+			finally
+			{
+				_flushing.Release();
+			}
+		}
+
+		async Task Touch(CancellationToken token)
+		{
+			foreach (var host in _touches.Keys.ToList())
+			{
+				_touches.TryRemove(host, out _);
+				var now = DateTime.UtcNow;
+				if (host == Interactions.Unknown || _touchedAt.TryGetValue(host, out var at) && now - at < TouchInterval)
+					continue;
+				_touchedAt[host] = now;
+				await DB.Default.Update<RemoteInstance>()
+					.Match(i => i.Host == host)
+					.Modify(i => i.Seen, "touched")
+					.Modify(i => i.LastSeenAt, now)
+					.Modify(b => b.SetOnInsert(i => i.FirstSeenAt, now))
+					.Option(o => o.IsUpsert = true)
+					.ExecuteAsync(token);
+				_knownHosts.Add(host);
+				if (_queue != default && _blocks?.IsSuspended(host) != true)
+					await _queue.Enqueue(JobKind.DescribeInstance, host, host, Federation.Objects.InstanceDescriber.DedupeKey(host, now), token);
+			}
+			if (_touchedAt.Count > 100_000)
+				_touchedAt.Clear();
+		}
+
 		async Task FlushEvents(CancellationToken token)
 		{
 			await _flushing.WaitAsync(token);
@@ -192,6 +255,8 @@ namespace PrivaPub.Infrastructure.Statistics
 				interaction.Host = Interactions.Unknown;
 			if (pending.ActorUri != default)
 				interaction.ActorHash = InteractionSalts.Hash(await _salts.For(interaction.At, token), pending.ActorUri);
+			if (interaction.Host != Interactions.Unknown && Touches(interaction, pending.ActorUri != default))
+				_touches[interaction.Host] = 0;
 			return interaction;
 		}