A remote thread brings the replies that never reached us
The context of a remote post showed only what PrivaPub happened to hold: replies from servers nobody here follows were never seen, and only the ancestors were ever fetched. Now a persona opening a public remote thread queues FetchReplies for the post and its root, at most once an hour each. The job reads the thread's own collection first (FEP-7888 `context`, which Mastodon 4.5+ serves with every reply at any depth; posts or, as FEP-f228 allows, the activities that made them), and otherwise the post's `replies` (PeerTube's `comments`) and the replies' own, two levels down. At most 5 pages and 100 posts a job, signed by the instance actor, never a persona; each post is fetched from its own origin and stored through StoreContext, so only public and unlisted ones are kept. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01LsXgEaXee4GCU1hwYgPJXw
This commit is contained in:
thepraandClaude Opus 5.5 committed
1 parent
bbeeda7268
commit
0695c08ac3
12 files changed
+344
-5
No files matched your search
@@ -111,6 +111,138 @@ namespace PrivaPub.Tests.Federation
|
||||
Assert.DoesNotContain(_harness.Peer.Requests, r => r.Path == new Uri(all[2]).AbsolutePath);
|
||||
}
|
||||
|
||||
JsonObject Served(JsonObject document)
|
||||
{
|
||||
_harness.Peer.Serve(new Uri(IdOf(document)).AbsolutePath, document.ToJsonString());
|
||||
return document;
|
||||
}
|
||||
|
||||
static JsonObject Reply(RemoteActor author, string text, JsonObject parent)
|
||||
{
|
||||
var note = PublicNote(author, $"<p>{text}</p>");
|
||||
note["inReplyTo"] = IdOf(parent);
|
||||
return note;
|
||||
}
|
||||
|
||||
static async Task<Post> Held(JsonObject note)
|
||||
{
|
||||
var post = new Post
|
||||
{
|
||||
ObjectURI = IdOf(note), ActorURI = note["attributedTo"]!.GetValue<string>(), IsFederatedCopy = true, Visibility = PostVisibility.Public,
|
||||
ContentHtml = note["content"]!.GetValue<string>()
|
||||
};
|
||||
await DB.Default.SaveAsync(post, TestContext.Current.CancellationToken);
|
||||
return post;
|
||||
}
|
||||
|
||||
// runs the FetchReplies job for the post, and those it queues for this peer's posts, until none is left; how many ran
|
||||
async Task<int> RunReplies(Post first)
|
||||
{
|
||||
var token = TestContext.Current.CancellationToken;
|
||||
var handler = new RepliesJobHandler(_harness.Db, _harness.Remote, _harness.RemotePosts, _harness.Queue);
|
||||
var origin = new Uri(first.ObjectURI).GetLeftPart(UriPartial.Authority);
|
||||
await _harness.Queue.EnqueueMany(new[] { RepliesJobHandler.For(first, 1) }, token);
|
||||
var ran = 0;
|
||||
while (true)
|
||||
{
|
||||
var mine = new List<Job>();
|
||||
foreach (var job in await DB.Default.Find<Job>().Match(j => j.Kind == JobKind.FetchReplies && j.State == JobState.Pending).ExecuteAsync(token))
|
||||
{
|
||||
var post = await DB.Default.Find<Post>().OneAsync(JsonSerializer.Deserialize<RepliesPayload>(job.Payload)!.PostId, token);
|
||||
if (post?.ObjectURI.StartsWith(origin) == true)
|
||||
mine.Add(job);
|
||||
}
|
||||
if (mine.Count == 0)
|
||||
return ran;
|
||||
foreach (var job in mine)
|
||||
{
|
||||
Assert.Equal(JobResult.Done, (await handler.Handle(job, token)).Result);
|
||||
await DB.Default.Update<Job>().MatchID(job.ID).Modify(j => j.State, JobState.Done).ExecuteAsync(token);
|
||||
ran++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static async Task<Dictionary<string, Post>> Stored(params JsonObject[] notes)
|
||||
{
|
||||
var uris = notes.Select(IdOf).ToList();
|
||||
return (await DB.Default.Find<Post>().Match(p => uris.Contains(p.ObjectURI)).ExecuteAsync(TestContext.Current.CancellationToken))
|
||||
.ToDictionary(p => p.ObjectURI);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_threads_replies_are_read_from_their_servers_two_levels_down_and_only_public_ones_kept()
|
||||
{
|
||||
var poster = new RemoteActor(_harness.Peer, "poster");
|
||||
var answerer = new RemoteActor(_harness.Peer, "answerer");
|
||||
var root = PublicNote(poster, "<p>the root</p>");
|
||||
// what a post carries is as old as the post: the collection is read again by its id
|
||||
root["replies"] = new JsonObject
|
||||
{
|
||||
["id"] = IdOf(root) + "/replies", ["type"] = "Collection", ["first"] = new JsonObject { ["type"] = "CollectionPage", ["items"] = new JsonArray() }
|
||||
};
|
||||
Served(root);
|
||||
var first = Reply(answerer, "first answer", root);
|
||||
var second = Served(Reply(poster, "second answer", root));
|
||||
var hidden = Reply(answerer, "for followers", root);
|
||||
hidden["to"] = new JsonArray(answerer.Id + "/followers");
|
||||
hidden["cc"] = new JsonArray();
|
||||
Served(hidden);
|
||||
var nested = Reply(poster, "nested", first);
|
||||
var deeper = Served(Reply(answerer, "too deep", nested));
|
||||
first["replies"] = IdOf(first) + "/replies";
|
||||
Served(first);
|
||||
nested["replies"] = IdOf(nested) + "/replies";
|
||||
Served(nested);
|
||||
|
||||
Served(new JsonObject { ["id"] = IdOf(root) + "/replies", ["type"] = "Collection", ["first"] = IdOf(root) + "/replies/1" });
|
||||
Served(new JsonObject
|
||||
{
|
||||
["id"] = IdOf(root) + "/replies/1", ["type"] = "CollectionPage", ["next"] = IdOf(root) + "/replies/2",
|
||||
["items"] = new JsonArray(first.DeepClone(), IdOf(second))
|
||||
});
|
||||
Served(new JsonObject { ["id"] = IdOf(root) + "/replies/2", ["type"] = "CollectionPage", ["items"] = new JsonArray(IdOf(hidden)) });
|
||||
Served(new JsonObject { ["id"] = IdOf(first) + "/replies", ["type"] = "OrderedCollection", ["orderedItems"] = new JsonArray(IdOf(nested)) });
|
||||
Served(new JsonObject { ["id"] = IdOf(nested) + "/replies", ["type"] = "OrderedCollection", ["orderedItems"] = new JsonArray(IdOf(deeper)) });
|
||||
var held = await Held(root);
|
||||
|
||||
Assert.Equal(3, await RunReplies(held));
|
||||
var stored = await Stored(first, second, hidden, nested, deeper);
|
||||
Assert.Equal(new[] { IdOf(first), IdOf(second), IdOf(nested) }.Order(), stored.Keys.Order());
|
||||
Assert.Equal(held.ID, stored[IdOf(first)].AnsweringToPostId);
|
||||
Assert.Equal(held.ID, stored[IdOf(second)].AnsweringToPostId);
|
||||
Assert.Equal(stored[IdOf(first)].ID, stored[IdOf(nested)].AnsweringToPostId);
|
||||
Assert.DoesNotContain(_harness.Peer.Requests, r => r.Path == new Uri(IdOf(nested) + "/replies").AbsolutePath);
|
||||
Assert.All(_harness.Peer.Requests.Where(r => r.Path.EndsWith("/replies")), r => Assert.False(string.IsNullOrEmpty(r.Signature)));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_threads_context_collection_brings_every_reply_at_once()
|
||||
{
|
||||
var poster = new RemoteActor(_harness.Peer, "poster");
|
||||
var answerer = new RemoteActor(_harness.Peer, "answerer");
|
||||
var root = PublicNote(poster, "<p>a thread</p>");
|
||||
var context = NewId(poster, "contexts");
|
||||
root["context"] = context;
|
||||
root["replies"] = NewId(poster, "never");
|
||||
Served(root);
|
||||
var answer = Served(Reply(answerer, "an answer", root));
|
||||
var deep = Served(Reply(poster, "an answer to it", answer));
|
||||
// FEP-f228 lets a context list the activities that made the thread
|
||||
Served(new JsonObject
|
||||
{
|
||||
["id"] = context, ["type"] = "OrderedCollection",
|
||||
["first"] = new JsonObject { ["type"] = "OrderedCollectionPage", ["orderedItems"] = new JsonArray(IdOf(root), Create(answerer, answer), IdOf(deep)) }
|
||||
});
|
||||
var held = await Held(root);
|
||||
|
||||
Assert.Equal(1, await RunReplies(held));
|
||||
var stored = await Stored(answer, deep);
|
||||
Assert.Equal(held.ID, stored[IdOf(answer)].AnsweringToPostId);
|
||||
Assert.Equal(stored[IdOf(answer)].ID, stored[IdOf(deep)].AnsweringToPostId);
|
||||
Assert.DoesNotContain(_harness.Peer.Requests, r => r.Path == new Uri(root["replies"]!.GetValue<string>()).AbsolutePath);
|
||||
}
|
||||
|
||||
async Task<(LocalActor Alice, RemoteActor Mallory, Post Poll)> LocalPoll()
|
||||
{
|
||||
var token = TestContext.Current.CancellationToken;
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
using MongoDB.Entities;
|
||||
|
||||
using PrivaPub.Models.Jobs;
|
||||
using PrivaPub.Models.User;
|
||||
using PrivaPub.Tests.Support;
|
||||
using PrivaPub.Tests.Support.Host;
|
||||
@@ -133,6 +134,36 @@ namespace PrivaPub.Tests.Http
|
||||
Assert.Equal(1, (await alice.Client.Get($"/api/v1/statuses/{parent.ID}")).Ok().Body.Number("replies_count"));
|
||||
}
|
||||
|
||||
// a remote thread opened by a persona has its replies read from its server (here its FEP-7888 context), at most hourly
|
||||
[Fact]
|
||||
public async Task Opening_a_remote_thread_reads_the_replies_that_never_reached_us()
|
||||
{
|
||||
var token = TestContext.Current.CancellationToken;
|
||||
var alice = await _host.Mastodon("alice");
|
||||
var bob = new RemoteActor(_peer, "bob");
|
||||
var carol = new RemoteActor(_peer, "carol");
|
||||
var context = $"{bob.Origin()}/contexts/{Guid.NewGuid():N}";
|
||||
var post = await _host.PublicPostFrom(bob, alice, shape: note => note["context"] = context);
|
||||
var answer = carol.Create("<p>said elsewhere</p>", new[] { MastodonHelpers.Public }, new[] { carol.Id + "/followers" },
|
||||
note => note["inReplyTo"] = post.ObjectURI)["object"]!;
|
||||
var answerId = answer["id"]!.GetValue<string>();
|
||||
_peer.Serve(new Uri(answerId).AbsolutePath, answer.ToJsonString());
|
||||
_peer.Serve(new Uri(context).AbsolutePath, new JsonObject
|
||||
{
|
||||
["id"] = context, ["type"] = "OrderedCollection", ["orderedItems"] = new JsonArray(post.ObjectURI, answerId)
|
||||
}.ToJsonString());
|
||||
async Task<int> Queued() => (await DB.Default.Find<Job>().Match(j => j.Kind == JobKind.FetchReplies && j.Payload.Contains(post.ID)).ExecuteAsync(token)).Count;
|
||||
|
||||
(await _host.Client().Get($"/api/v1/statuses/{post.ID}/context")).Ok();
|
||||
Assert.Equal(0, await Queued());
|
||||
Assert.Empty((await alice.Client.Get($"/api/v1/statuses/{post.ID}/context")).Ok().Body["descendants"]!.AsArray());
|
||||
Assert.Equal(1, await _host.Run(j => j.Kind == JobKind.FetchReplies && j.Payload.Contains(post.ID), token));
|
||||
|
||||
var descendants = (await alice.Client.Get($"/api/v1/statuses/{post.ID}/context")).Ok().Body["descendants"]!.AsArray();
|
||||
Assert.Equal(answerId, Assert.Single(descendants).Text("uri"));
|
||||
Assert.Equal(1, await Queued());
|
||||
}
|
||||
|
||||
// Akkoma marks a post with sensitive media "sensitive" and leaves its summary empty: the media is hidden, the words are
|
||||
// not, and no warning is made up for them (found by decePub's e2e tests on the town)
|
||||
[Fact]
|
||||
|
||||
Reference in new issue
Block a user