A remote thread brings the replies that never reached us

The context of a remote post showed only what PrivaPub happened to hold:
replies from servers nobody here follows were never seen, and only the
ancestors were ever fetched. Now a persona opening a public remote thread
queues FetchReplies for the post and its root, at most once an hour each.

The job reads the thread's own collection first (FEP-7888 `context`, which
Mastodon 4.5+ serves with every reply at any depth; posts or, as FEP-f228
allows, the activities that made them), and otherwise the post's `replies`
(PeerTube's `comments`) and the replies' own, two levels down. At most 5
pages and 100 posts a job, signed by the instance actor, never a persona;
each post is fetched from its own origin and stored through StoreContext,
so only public and unlisted ones are kept.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01LsXgEaXee4GCU1hwYgPJXw

PrivaPub/Api/Mastodon/Controllers/StatusesController.cs

@@ -203,6 +203,11 @@ namespace PrivaPub.Api.Mastodon.Controllers
 						frontier.Add(child.ID);
 					}
 			}
+			// a persona opening a public remote thread has its replies read from their servers, for the next look (once an hour)
+			if (MyId != default)
+				await _jobs.EnqueueMany(new[] { post, ancestors.FirstOrDefault() }
+					.Where(p => p is { IsFederatedCopy: true, Visibility: PostVisibility.Public or PostVisibility.Unlisted }).DistinctBy(p => p.ID)
+					.Select(p => Federation.Inbox.RepliesJobHandler.For(p, 1)), token);
 			return Json(new Context
 			{
 				Ancestors = await _mapper.Statuses(ancestors, MyId, token),

PrivaPub/Federation/Inbox/RemoteReplies.cs

@@ -0,0 +1,158 @@
+using System.Text.Json;
+using System.Text.Json.Nodes;
+
+using MongoDB.Entities;
+
+using PrivaPub.Federation.Actors;
+using PrivaPub.Infrastructure.Http;
+using PrivaPub.Infrastructure.Jobs;
+using PrivaPub.Models.Federation;
+using PrivaPub.Models.Jobs;
+using PrivaPub.Models.Post;
+using PrivaPub.StaticServices;
+
+using static PrivaPub.Federation.Objects.ActivityJson;
+
+using PostEntity = PrivaPub.Models.Post.Post;
+
+namespace PrivaPub.Federation.Inbox
+{
+	public sealed record RepliesPayload(string PostId, int Depth);
+
+	// What was said under a remote post on other servers, read when a persona opens its thread, so the thread shows more
+	// than what happened to reach us. The thread's own collection comes first (FEP-7888's `context`, which Mastodon 4.5+
+	// serves with every reply at any depth); otherwise the post's `replies` (PeerTube's `comments`), and the replies'
+	// own, two levels down. Read with the instance actor's signature, never a persona's, a few pages and at most
+	// MaxItems posts a job; each one is fetched from its own server and stored like any post fetched for a thread
+	// (RemotePosts.StoreContext: public and unlisted only, its author checked against its origin).
+	public class RepliesJobHandler : IJobHandler
+	{
+		public const int MaxPages = 5;
+		public const int MaxItems = 100;
+		public const int MaxDepth = 2;
+		public const int MaxBranches = 20;
+
+		static readonly HashSet<string> CollectionTypes = new(StringComparer.Ordinal)
+		{
+			"Collection", "OrderedCollection", "CollectionPage", "OrderedCollectionPage"
+		};
+
+		readonly DbEntities _dbEntities;
+		readonly IRemoteActorService _remoteActors;
+		readonly IRemotePosts _remotePosts;
+		readonly IJobQueue _queue;
+
+		public RepliesJobHandler(DbEntities dbEntities, IRemoteActorService remoteActors, IRemotePosts remotePosts, IJobQueue queue)
+		{
+			_dbEntities = dbEntities;
+			_remoteActors = remoteActors;
+			_remotePosts = remotePosts;
+			_queue = queue;
+		}
+
+		public JobKind Kind => JobKind.FetchReplies;
+		public int Concurrency => 2;
+		public int MaxAttempts => 2;
+		public int PerHostLimit => 1;
+
+		// a thread is read again at most once an hour
+		public static Job For(PostEntity post, int depth) => new()
+		{
+			Kind = JobKind.FetchReplies,
+			Payload = JsonSerializer.Serialize(new RepliesPayload(post.ID, depth)),
+			Host = new Uri(post.ObjectURI).Host,
+			DedupeKey = $"replies|{post.ID}|{DateTime.UtcNow:yyyyMMddHH}"
+		};
+
+		public async Task<JobOutcome> Handle(Job job, CancellationToken token)
+		{
+			var payload = JsonSerializer.Deserialize<RepliesPayload>(job.Payload);
+			var post = await _dbEntities.Posts.MatchID(payload.PostId).ExecuteFirstAsync(token);
+			if (post is not { IsFederatedCopy: true, Visibility: PostVisibility.Public or PostVisibility.Unlisted } || post.DeletedAt.HasValue
+				|| string.IsNullOrEmpty(post.ObjectURI))
+				return JobOutcome.Done;
+			if (await Document(post, token) is not { } document)
+				return JobOutcome.Done;
+
+			if (await Collection(document["context"], token) is { } thread)
+			{
+				await StoreAll(thread, token);
+				return JobOutcome.Done;
+			}
+			if (await Collection(document["replies"] ?? document["comments"], token) is not { } replies)
+				return JobOutcome.Done;
+			var answers = await StoreAll(replies, token);
+			if (payload.Depth < MaxDepth)
+				await _queue.EnqueueMany(answers.Where(a => a.IsFederatedCopy && a.AnsweringToPostId == post.ID).Take(MaxBranches)
+					.Select(a => For(a, payload.Depth + 1)), token);
+			return JobOutcome.Done;
+		}
+
+		// the post as it reached us, or as its server serves it now when we kept no copy
+		async Task<JsonObject> Document(PostEntity post, CancellationToken token)
+		{
+			var record = await DB.Default.Find<ObjectRecord>().Match(r => r.ObjectURI == post.ObjectURI && r.Raw != null && !r.RawTruncated)
+				.Project(r => new ObjectRecord { Raw = r.Raw }).ExecuteFirstAsync(token);
+			if (record != default)
+				return JsonNode.Parse(record.Raw) as JsonObject;
+			using var scope = HttpScope.For("replies");
+			using var fetched = await _remoteActors.FetchObject(post.ObjectURI, token);
+			return fetched == default ? default : JsonNode.Parse(fetched.Root.GetRawText()) as JsonObject;
+		}
+
+		// a collection as its server serves it now: what a post carries is read again by its id, since an embedded first
+		// page is as old as the post; a value that is no web address (Pleroma's `context` is a tag: URI), or no collection,
+		// gives nothing
+		async Task<JsonObject> Collection(JsonNode node, CancellationToken token)
+		{
+			var uri = Id(node);
+			if (uri != default && Uri.TryCreate(uri, UriKind.Absolute, out var parsed) && (parsed.Scheme == Uri.UriSchemeHttps || parsed.Scheme == Uri.UriSchemeHttp))
+			{
+				using var scope = HttpScope.For("replies");
+				using var fetched = await _remoteActors.FetchObject(uri, token);
+				node = fetched == default ? default : JsonNode.Parse(fetched.Root.GetRawText());
+			}
+			return node is JsonObject collection && Value(collection, "type") is { } type && CollectionTypes.Contains(type) ? collection : default;
+		}
+
+		// every post the collection lists, stored when we lack it: its pages in order, at most MaxPages and MaxItems
+		async Task<List<PostEntity>> StoreAll(JsonObject collection, CancellationToken token)
+		{
+			var posts = new List<PostEntity>();
+			var seen = 0;
+			var page = collection["orderedItems"] is JsonArray || collection["items"] is JsonArray ? collection : await Page(collection["first"], token);
+			for (var pages = 0; page != default && pages < MaxPages && seen < MaxItems; pages++)
+			{
+				foreach (var item in Items(page))
+				{
+					if (++seen > MaxItems)
+						break;
+					// a context may list the activities that made the thread (FEP-f228) rather than its posts
+					var id = Value(item, "type") is "Create" or "Update" ? Id(item["object"]) : Id(item);
+					if (id == default)
+						continue;
+					var post = await _dbEntities.Posts.Match(p => p.ObjectURI == id && !p.DeletedAt.HasValue).ExecuteFirstAsync(token)
+						?? await _remotePosts.StoreContext(id, 0, token);
+					if (post != default)
+						posts.Add(post);
+				}
+				page = page["next"] is { } next ? await Page(next, token) : default;
+			}
+			return posts;
+		}
+
+		async Task<JsonObject> Page(JsonNode node, CancellationToken token)
+		{
+			if (node is JsonValue && Id(node) is { } uri)
+			{
+				using var scope = HttpScope.For("replies");
+				using var fetched = await _remoteActors.FetchObject(uri, token);
+				node = fetched == default ? default : JsonNode.Parse(fetched.Root.GetRawText());
+			}
+			return node as JsonObject;
+		}
+
+		static IEnumerable<JsonNode> Items(JsonNode page) =>
+			(page["orderedItems"] as JsonArray ?? page["items"] as JsonArray ?? new JsonArray()).Where(i => i != default);
+	}
+}

PrivaPub/Infrastructure/Statistics/InteractionLedger.cs

@@ -44,7 +44,7 @@ namespace PrivaPub.Infrastructure.Statistics
 			SingleReader = false
 		});
 		static readonly TimeSpan TouchInterval = TimeSpan.FromHours(1);
-		static readonly HashSet<string> TouchingPurposes = new(StringComparer.Ordinal) { "actor", "key", "object", "webfinger", "context" };
+		static readonly HashSet<string> TouchingPurposes = new(StringComparer.Ordinal) { "actor", "key", "object", "webfinger", "context", "replies" };
 
 		readonly InteractionSalts _salts;
 		readonly IOptionsMonitor<StatisticsOptions> _options;

PrivaPub/Middleware/SocialPubConfigurations.cs

@@ -109,6 +109,7 @@ namespace PrivaPub.Middleware
 				.AddSingleton<InstanceDescriber>()
 				.AddSingleton<IJobHandler>(services => services.GetRequiredService<InstanceDescriber>())
 				.AddSingleton<IJobHandler, AncestorsJobHandler>()
+				.AddSingleton<IJobHandler, Federation.Inbox.RepliesJobHandler>()
 				.AddSingleton<IJobQueue, JobQueue>()
 				.AddSingleton<IHostCircuitBreaker, HostCircuitBreaker>()
 				.AddSingleton<IJobHandler, DeliveryJobHandler>()

PrivaPub/Models/Jobs/Job.cs

@@ -32,7 +32,8 @@ namespace PrivaPub.Models.Jobs
 		CrawlInstance,
 		SendRecovery,
 		CountAccount,
-		PublishScheduled
+		PublishScheduled,
+		FetchReplies
 	}
 
 	public enum JobState

PrivaPub/Models/Statistics/InteractionEvent.cs

@@ -11,7 +11,7 @@ namespace PrivaPub.Models.Statistics
 		public string Channel { get; set; }//recv | in | out | http | preview | crawl
 		public string Activity { get; set; }//Create, Follow, ...; "other" when it is no plain type name
 		public string Object { get; set; }//Note, Article, Person, ...
-		public string Purpose { get; set; }//http and crawl: actor, key, object, context, webfinger, nodeinfo, ...
+		public string Purpose { get; set; }//http and crawl: actor, key, object, context, replies, webfinger, nodeinfo, ...
 		public string Trigger { get; set; }//http: what caused the request (inbox, deliver, describe, request, ...)
 		public string Outcome { get; set; }//queued, refused, accepted, dropped, ok, failed, deferred, retry, dead
 		public string Reason { get; set; }