From b45321f28de6b6dbaae9e7fd3ff8111fce27aa52 Mon Sep 17 00:00:00 2001 From: thepra Date: Tue, 6 Oct 2026 02:15:49 +0200 Subject: [PATCH] Following an account brings its earlier posts to its profile Once a follow holds, OutboxBackfill reads the account's latest public posts from its outbox's first page (twenty at most, once a day) and keeps them as any fetched post: its profile shows them at once instead of only what it posts from then on. Homes still get only what arrives afterwards, as on Mastodon. Announces and other servers' objects in the outbox are left out. Checked live against Mastodon (scenarios/pins.sh, now 9 checks). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01LsXgEaXee4GCU1hwYgPJXw --- CLAUDE.md | 3 +- FEDERATION.md | 2 +- PrivaPub.Tests/Federation/BackfillTests.cs | 78 +++++++++++++++ PrivaPub.Tests/Support/Harness.cs | 2 +- PrivaPub.Tests/Support/RemoteActor.cs | 1 + PrivaPub/Federation/Actors/OutboxBackfill.cs | 94 +++++++++++++++++++ .../Inbox/Handlers/AcceptHandler.cs | 16 +++- .../Middleware/SocialPubConfigurations.cs | 1 + PrivaPub/Models/Jobs/Job.cs | 3 +- docs/ROADMAP.md | 3 +- tools/pasture/scenarios/pins.sh | 10 +- 11 files changed, 203 insertions(+), 10 deletions(-) create mode 100644 PrivaPub.Tests/Federation/BackfillTests.cs create mode 100644 PrivaPub/Federation/Actors/OutboxBackfill.cs diff --git a/CLAUDE.md b/CLAUDE.md index d776436..ef3a251 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -560,7 +560,8 @@ tools/pasture/run.sh down # removes e only as `ChatMessage`, which PrivaPub sends it, a first message too (invariant 17). `scenarios/lemmy19.sh`, 30 checks. - **Pins (`scenarios/pins.sh`, needs mastodon):** a Mastodon account pins and unpins while alice follows it, alice pins - and unpins while it follows her, and a fresh account's earlier pin shows once PrivaPub resolves it. 8 checks. + and unpins while it follows her, a fresh account's earlier pin shows once PrivaPub resolves it, and following it brings + its earlier posts (its outbox). 9 checks. - **Vernissage (1.43.0):** its image on SQLite in the `pasture-vernissage` volume, its queues on the shared Redis's database 0 (it federates nothing without them; every other database is taken), its seeded admin (admin / admin) logged in by the scenario. `scenarios/vernissage.sh`, 19 checks. diff --git a/FEDERATION.md b/FEDERATION.md index 3dedec4..46f5e96 100644 --- a/FEDERATION.md +++ b/FEDERATION.md @@ -143,7 +143,7 @@ Received: | Activity | Effect | |---|---| | `Follow` | follows an avatar or community; `Accept` is sent unless the community approves members by hand | -| `Accept{Follow}`, `Reject{Follow}` | completes or ends a follow an avatar requested | +| `Accept{Follow}`, `Reject{Follow}` | completes or ends a follow an avatar requested; once a follow holds, the account's latest public posts are read from its outbox's first page (twenty at most, once a day), so its profile is not empty; homes get only what arrives from then on | | `Accept{Join}`, `Reject{Join}` | from the event's server: a persona's participation in the event is accepted or refused | | `Undo{Follow, Like, Announce}` | reverses it | | `Create{Note, Article, Page, Question, Video, Audio, Event, ChatMessage, …}` | stored when a local avatar follows the author, is addressed or mentioned, when it replies to a local post, or when it is addressed to a community the author follows (a persona is addressed by its actor id or by its profile page, as Mbin addresses its private messages); a public parent is fetched to complete the thread | diff --git a/PrivaPub.Tests/Federation/BackfillTests.cs b/PrivaPub.Tests/Federation/BackfillTests.cs new file mode 100644 index 0000000..8bd5b59 --- /dev/null +++ b/PrivaPub.Tests/Federation/BackfillTests.cs @@ -0,0 +1,78 @@ +using MongoDB.Entities; + +using PrivaPub.Federation.Actors; +using PrivaPub.Models.Jobs; +using PrivaPub.Models.Post; +using PrivaPub.Models.Social; +using PrivaPub.Tests.Support; + +using System.Text.Json.Nodes; + +namespace PrivaPub.Tests.Federation +{ + // a follow that holds reads the account's latest public posts from its outbox, so its profile is not empty + [Trait("Category", "Integration")] + public sealed class BackfillTests : IAsyncLifetime + { + Harness _harness; + + public async ValueTask InitializeAsync() + { + Assert.SkipUnless(MongoFixture.Enabled, MongoFixture.Skip); + _harness = await Harness.Start(); + } + + public async ValueTask DisposeAsync() + { + if (_harness != default) + await _harness.DisposeAsync(); + } + + [Fact] + public async Task An_accepted_follow_reads_the_accounts_latest_public_posts_from_its_outbox() + { + var token = TestContext.Current.CancellationToken; + var (_, alice) = await _harness.Persona("alice"); + var bob = new RemoteActor(_harness.Peer, "bob"); + var mallory = new RemoteActor(_harness.Peer, "mallory", _harness.Peer.B); + var origin = new Uri(bob.Id).GetLeftPart(UriPartial.Authority); + JsonObject Note(string id, string text, RemoteActor author) => new() + { + ["id"] = id, ["type"] = "Note", ["attributedTo"] = author.Id, ["content"] = $"

{text}

", + ["to"] = new JsonArray("https://www.w3.org/ns/activitystreams#Public"), ["published"] = DateTime.UtcNow.ToString("O") + }; + var older = $"{origin}/notes/{Guid.NewGuid():N}"; + var newer = $"{origin}/notes/{Guid.NewGuid():N}"; + var notBobs = $"{new Uri(mallory.Id).GetLeftPart(UriPartial.Authority)}/notes/{Guid.NewGuid():N}"; + _harness.Peer.Serve(new Uri(older).AbsolutePath, Note(older, "an older post", bob).ToJsonString()); + _harness.Peer.Serve(new Uri(newer).AbsolutePath, Note(newer, "a newer post", bob).ToJsonString()); + _harness.Peer.Serve(new Uri(bob.Id).AbsolutePath + "/outbox", new JsonObject + { + ["id"] = bob.Id + "/outbox", ["type"] = "OrderedCollection", ["totalItems"] = 4, + ["orderedItems"] = new JsonArray( + new JsonObject { ["type"] = "Create", ["object"] = Note(newer, "a newer post", bob) }, + new JsonObject { ["type"] = "Announce", ["object"] = notBobs }, + new JsonObject { ["type"] = "Create", ["object"] = notBobs }, + new JsonObject { ["type"] = "Create", ["object"] = older }) + }.ToJsonString()); + var account = await _harness.Remote.GetActor(bob.Id, refresh: false, token); + var followId = alice.ActivityUri($"follow-{Guid.NewGuid():N}"); + await DB.Default.SaveAsync(new Following + { + AvatarId = alice.Id, TargetActorURI = bob.Id, TargetAccountId = account.ID, State = FollowState.Requested, FollowActivityURI = followId + }, token); + + await _harness.Deliver(bob, "/human-centipede", new JsonObject + { + ["id"] = $"{bob.Id}#accept-{Guid.NewGuid():N}", ["type"] = "Accept", ["actor"] = bob.Id, + ["object"] = new JsonObject { ["id"] = followId, ["type"] = "Follow", ["actor"] = alice.Uri, ["object"] = bob.Id } + }); + var job = await DB.Default.Find().Match(j => j.Kind == JobKind.BackfillOutbox && j.Payload == bob.Id).ExecuteSingleAsync(token); + await new OutboxBackfill(_harness.Db, _harness.Remote, _harness.RemotePosts).Handle(job, token); + + var held = (await DB.Default.Find().Match(p => p.ActorURI == bob.Id).ExecuteAsync(token)).Select(p => p.ObjectURI).ToHashSet(); + Assert.Equal(new HashSet { older, newer }, held); + Assert.False(await DB.Default.Find().Match(p => p.ObjectURI == notBobs).ExecuteAnyAsync(token)); + } + } +} diff --git a/PrivaPub.Tests/Support/Harness.cs b/PrivaPub.Tests/Support/Harness.cs index d1bcf67..4d7d2d4 100644 --- a/PrivaPub.Tests/Support/Harness.cs +++ b/PrivaPub.Tests/Support/Harness.cs @@ -62,7 +62,7 @@ namespace PrivaPub.Tests.Support Handlers = new IActivityHandler[] { new FollowHandler(Db, Local, Remote, Delivery), - new AcceptHandler(Db, Local, Quotes, Approvals, Participations, Relays), + new AcceptHandler(Db, Local, Quotes, Approvals, Participations, Relays, Queue), new RejectHandler(Db, Local, Quotes, Approvals, Participations, Relays), new UndoHandler(Db, Local, Reactions), new LikeHandler(Db, Reactions), diff --git a/PrivaPub.Tests/Support/RemoteActor.cs b/PrivaPub.Tests/Support/RemoteActor.cs index cfa5a93..6c5ff09 100644 --- a/PrivaPub.Tests/Support/RemoteActor.cs +++ b/PrivaPub.Tests/Support/RemoteActor.cs @@ -41,6 +41,7 @@ namespace PrivaPub.Tests.Support ["preferredUsername"] = Name, ["inbox"] = Id + "/inbox", ["followers"] = Id + "/followers", + ["outbox"] = Id + "/outbox", ["featured"] = Id + "/featured", ["publicKey"] = new JsonObject { diff --git a/PrivaPub/Federation/Actors/OutboxBackfill.cs b/PrivaPub/Federation/Actors/OutboxBackfill.cs new file mode 100644 index 0000000..6c7bbac --- /dev/null +++ b/PrivaPub/Federation/Actors/OutboxBackfill.cs @@ -0,0 +1,94 @@ +using MongoDB.Entities; + +using PrivaPub.Federation.Inbox; +using PrivaPub.Federation.Objects; +using PrivaPub.Infrastructure.Http; +using PrivaPub.Infrastructure.Jobs; +using PrivaPub.Models.Jobs; +using PrivaPub.StaticServices; + +using System.Text.Json; + +namespace PrivaPub.Federation.Actors +{ + // The latest public posts of an account a persona has just followed, read from the first page of its outbox (at most + // twenty, once a day) and kept as any fetched post: its profile shows them and threads find them at once, instead of + // only what it posts from then on. Homes still get only what arrives from now on, as on Mastodon. + public class OutboxBackfill : IJobHandler + { + public const int MaxPosts = 20; + + readonly DbEntities _dbEntities; + readonly IRemoteActorService _remoteActors; + readonly IRemotePosts _remotePosts; + + public OutboxBackfill(DbEntities dbEntities, IRemoteActorService remoteActors, IRemotePosts remotePosts) + { + _dbEntities = dbEntities; + _remoteActors = remoteActors; + _remotePosts = remotePosts; + } + + public JobKind Kind => JobKind.BackfillOutbox; + public int Concurrency => 1; + public int MaxAttempts => 2; + public int PerHostLimit => 1; + + public static string DedupeKey(string actorUri, DateTime day) => $"backfill|{actorUri}|{day:yyyyMMdd}"; + + public async Task Handle(Job job, CancellationToken token) + { + var actor = await _dbEntities.ForeignAvatars.Match(a => a.ActorURI == job.Payload && !a.DeletionAt.HasValue).ExecuteFirstAsync(token); + if (actor == default || string.IsNullOrEmpty(actor.OutboxURL) || !Origin.Same(actor.OutboxURL, actor.ActorURI)) + return JobOutcome.Done; + foreach (var objectUri in (await Items(actor, token)).Take(MaxPosts)) + await _remotePosts.StoreContext(objectUri, 0, token); + return JobOutcome.Done; + } + + // the posts the outbox's first page creates, newest first: Create items name or hold them; Announces are left out, + // being others' posts + async Task> Items(Models.User.ForeignAvatar actor, CancellationToken token) + { + using var scope = HttpScope.For("outbox"); + using var outbox = await _remoteActors.FetchObject(actor.OutboxURL, token); + if (outbox == default || outbox.Root.ValueKind != JsonValueKind.Object) + return new(); + var page = ItemsOf(outbox.Root, actor); + if (page.Count > 0 || !outbox.Root.TryGetProperty("first", out var first)) + return page; + if (first.ValueKind == JsonValueKind.Object) + return ItemsOf(first, actor); + var pageUri = first.ValueKind == JsonValueKind.String ? first.GetString() : default; + if (pageUri == default || !Origin.Same(pageUri, actor.ActorURI)) + return page; + using var fetched = await _remoteActors.FetchObject(pageUri, token); + return fetched == default || fetched.Root.ValueKind != JsonValueKind.Object ? page : ItemsOf(fetched.Root, actor); + } + + static List ItemsOf(JsonElement collection, Models.User.ForeignAvatar actor) + { + var posts = new List(); + foreach (var name in new[] { "orderedItems", "items" }) + { + if (!collection.TryGetProperty(name, out var list) || list.ValueKind != JsonValueKind.Array) + continue; + foreach (var item in list.EnumerateArray()) + { + var objectUri = item.ValueKind switch + { + JsonValueKind.Object when Text(item, "type") == "Create" && item.TryGetProperty("object", out var created) => + created.ValueKind == JsonValueKind.String ? created.GetString() : created.ValueKind == JsonValueKind.Object ? Text(created, "id") : default, + _ => default + }; + if (objectUri != default && Origin.Same(objectUri, actor.ActorURI)) + posts.Add(objectUri); + } + } + return posts; + } + + static string Text(JsonElement element, string property) => + element.TryGetProperty(property, out var value) && value.ValueKind == JsonValueKind.String ? value.GetString() : default; + } +} diff --git a/PrivaPub/Federation/Inbox/Handlers/AcceptHandler.cs b/PrivaPub/Federation/Inbox/Handlers/AcceptHandler.cs index 8d1e7ea..7ead988 100644 --- a/PrivaPub/Federation/Inbox/Handlers/AcceptHandler.cs +++ b/PrivaPub/Federation/Inbox/Handlers/AcceptHandler.cs @@ -20,10 +20,12 @@ namespace PrivaPub.Federation.Inbox.Handlers readonly IInteractionApprovals _approvals; readonly IParticipations _participations; readonly Relays.IRelays _relays; + readonly Infrastructure.Jobs.IJobQueue _queue; public AcceptHandler(DbEntities dbEntities, ILocalActorService localActors, IQuoteService quotes, IInteractionApprovals approvals = default, - IParticipations participations = default, Relays.IRelays relays = default) + IParticipations participations = default, Relays.IRelays relays = default, Infrastructure.Jobs.IJobQueue queue = default) { + _queue = queue; _relays = relays; _participations = participations; _dbEntities = dbEntities; @@ -67,8 +69,14 @@ namespace PrivaPub.Federation.Inbox.Handlers await Apply(following, token); } - protected virtual Task Apply(Following following, CancellationToken token) => - DB.Default.Update().MatchID(following.ID).Modify(f => f.State, FollowState.Accepted).ExecuteAsync(token); + // the follow holds: the account's latest public posts are read from its outbox (OutboxBackfill), once a day at most + protected virtual async Task Apply(Following following, CancellationToken token) + { + await DB.Default.Update().MatchID(following.ID).Modify(f => f.State, FollowState.Accepted).ExecuteAsync(token); + if (_queue != default) + await _queue.Enqueue(Models.Jobs.JobKind.BackfillOutbox, following.TargetActorURI, new Uri(following.TargetActorURI).Host, + Actors.OutboxBackfill.DedupeKey(following.TargetActorURI, DateTime.UtcNow), token); + } public static async Task FindFollowing(JsonNode follow, ForeignAvatar target, DbEntities dbEntities, ILocalActorService localActors, CancellationToken token) @@ -96,7 +104,7 @@ namespace PrivaPub.Federation.Inbox.Handlers { public RejectHandler(DbEntities dbEntities, ILocalActorService localActors, IQuoteService quotes, IInteractionApprovals approvals = default, IParticipations participations = default, Relays.IRelays relays = default) - : base(dbEntities, localActors, quotes, approvals, participations, relays) + : base(dbEntities, localActors, quotes, approvals, participations, relays, queue: default) { } diff --git a/PrivaPub/Middleware/SocialPubConfigurations.cs b/PrivaPub/Middleware/SocialPubConfigurations.cs index 09e73e0..3d2c4b5 100644 --- a/PrivaPub/Middleware/SocialPubConfigurations.cs +++ b/PrivaPub/Middleware/SocialPubConfigurations.cs @@ -111,6 +111,7 @@ namespace PrivaPub.Middleware .AddSingleton() .AddSingleton() .AddSingleton() + .AddSingleton() .AddSingleton() .AddSingleton() .AddSingleton() diff --git a/PrivaPub/Models/Jobs/Job.cs b/PrivaPub/Models/Jobs/Job.cs index 7fec6aa..28ac15a 100644 --- a/PrivaPub/Models/Jobs/Job.cs +++ b/PrivaPub/Models/Jobs/Job.cs @@ -33,7 +33,8 @@ namespace PrivaPub.Models.Jobs SendRecovery, CountAccount, PublishScheduled, - FetchReplies + FetchReplies, + BackfillOutbox } public enum JobState diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index 7b7e421..581005d 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -691,7 +691,8 @@ it, raw where it doesn't. events, user, notification, public, hashtag and list streams, each event mapped for its persona; a deletion reaches only the streams that showed the post), Web Push (gated: outbound traffic to push services), grouped notifications. Then advertise an honest version. -- **Backfill:** an author's outbox after following them. +- **Backfill:** an author's outbox after following them: **done 2026-10-05** (`OutboxBackfill`, the first page's + public posts, twenty at most, once a day; checked live against Mastodon). - **Long tail:** - MFM rendering data; - Misskey actor extras; diff --git a/tools/pasture/scenarios/pins.sh b/tools/pasture/scenarios/pins.sh index 9aa4594..0c04a85 100644 --- a/tools/pasture/scenarios/pins.sh +++ b/tools/pasture/scenarios/pins.sh @@ -1,6 +1,7 @@ # Pinned posts across servers (featured, FEP-e232's Add and Remove): a Mastodon account pins and unpins while alice # follows it; alice pins and unpins while it follows her; an account that pinned before PrivaPub ever saw it shows its -# pins once resolved (its featured collection, read with its counts). Needs the mastodon peer. +# pins once resolved (its featured collection, read with its counts), and following it brings its earlier posts (its +# outbox). Needs the mastodon peer. M=https://mastodon.test:6443 mcurl() { curl -sk --resolve mastodon.test:6443:127.0.0.1 "$@"; } . "$here/peers/mastodon.sh" @@ -52,3 +53,10 @@ mcurl -o /dev/null -X POST -H "Authorization: Bearer $ET" "$M/api/v1/statuses/$( early_on_p=$(curl -s -H "$PH" "$P/api/v2/search?q=$early@mastodon.test&resolve=true&type=accounts" | j "print(d['accounts'][0]['id'])") until_true 45 '[ "$(p_pinned $early_on_p)" = "$(echo "$e_post" | j "print(d[\"uri\"])")" ]' \ && ok "an account's earlier pin shows once it is resolved" || ko "the earlier pin never showed ($(p_pinned $early_on_p))" + +echo " posts made before" +# alice follows the fresh account: its outbox's latest public posts come to its profile here (OutboxBackfill) +e_other=$(mcurl -X POST -H "Authorization: Bearer $ET" "$M/api/v1/statuses" -d "status=written before anyone followed $run&visibility=public" | j "print(d['uri'])") +curl -s -o /dev/null -X POST -H "$PH" "$P/api/v1/accounts/$early_on_p/follow" +until_true 60 '[ "$(curl -s -H "$PH" "$P/api/v1/accounts/$early_on_p/statuses?limit=40" | j "print(any(s[\"uri\"] == \"$e_other\" for s in d))")" = "True" ]' \ + && ok "following an account brings its earlier posts to its profile" || ko "the account's earlier posts never came"