diff --git a/CLAUDE.md b/CLAUDE.md index d776436..ef3a251 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -560,7 +560,8 @@ tools/pasture/run.sh down # removes e only as `ChatMessage`, which PrivaPub sends it, a first message too (invariant 17). `scenarios/lemmy19.sh`, 30 checks. - **Pins (`scenarios/pins.sh`, needs mastodon):** a Mastodon account pins and unpins while alice follows it, alice pins - and unpins while it follows her, and a fresh account's earlier pin shows once PrivaPub resolves it. 8 checks. + and unpins while it follows her, a fresh account's earlier pin shows once PrivaPub resolves it, and following it brings + its earlier posts (its outbox). 9 checks. - **Vernissage (1.43.0):** its image on SQLite in the `pasture-vernissage` volume, its queues on the shared Redis's database 0 (it federates nothing without them; every other database is taken), its seeded admin (admin / admin) logged in by the scenario. `scenarios/vernissage.sh`, 19 checks. diff --git a/FEDERATION.md b/FEDERATION.md index 3dedec4..46f5e96 100644 --- a/FEDERATION.md +++ b/FEDERATION.md @@ -143,7 +143,7 @@ Received: | Activity | Effect | |---|---| | `Follow` | follows an avatar or community; `Accept` is sent unless the community approves members by hand | -| `Accept{Follow}`, `Reject{Follow}` | completes or ends a follow an avatar requested | +| `Accept{Follow}`, `Reject{Follow}` | completes or ends a follow an avatar requested; once a follow holds, the account's latest public posts are read from its outbox's first page (twenty at most, once a day), so its profile is not empty; homes get only what arrives from then on | | `Accept{Join}`, `Reject{Join}` | from the event's server: a persona's participation in the event is accepted or refused | | `Undo{Follow, Like, Announce}` | reverses it | | `Create{Note, Article, Page, Question, Video, Audio, Event, ChatMessage, …}` | stored when a local avatar follows the author, is addressed or mentioned, when it replies to a local post, or when it is addressed to a community the author follows (a persona is addressed by its actor id or by its profile page, as Mbin addresses its private messages); a public parent is fetched to complete the thread | diff --git a/PrivaPub.Tests/Federation/BackfillTests.cs b/PrivaPub.Tests/Federation/BackfillTests.cs new file mode 100644 index 0000000..8bd5b59 --- /dev/null +++ b/PrivaPub.Tests/Federation/BackfillTests.cs @@ -0,0 +1,78 @@ +using MongoDB.Entities; + +using PrivaPub.Federation.Actors; +using PrivaPub.Models.Jobs; +using PrivaPub.Models.Post; +using PrivaPub.Models.Social; +using PrivaPub.Tests.Support; + +using System.Text.Json.Nodes; + +namespace PrivaPub.Tests.Federation +{ + // a follow that holds reads the account's latest public posts from its outbox, so its profile is not empty + [Trait("Category", "Integration")] + public sealed class BackfillTests : IAsyncLifetime + { + Harness _harness; + + public async ValueTask InitializeAsync() + { + Assert.SkipUnless(MongoFixture.Enabled, MongoFixture.Skip); + _harness = await Harness.Start(); + } + + public async ValueTask DisposeAsync() + { + if (_harness != default) + await _harness.DisposeAsync(); + } + + [Fact] + public async Task An_accepted_follow_reads_the_accounts_latest_public_posts_from_its_outbox() + { + var token = TestContext.Current.CancellationToken; + var (_, alice) = await _harness.Persona("alice"); + var bob = new RemoteActor(_harness.Peer, "bob"); + var mallory = new RemoteActor(_harness.Peer, "mallory", _harness.Peer.B); + var origin = new Uri(bob.Id).GetLeftPart(UriPartial.Authority); + JsonObject Note(string id, string text, RemoteActor author) => new() + { + ["id"] = id, ["type"] = "Note", ["attributedTo"] = author.Id, ["content"] = $"

{text}

", + ["to"] = new JsonArray("https://www.w3.org/ns/activitystreams#Public"), ["published"] = DateTime.UtcNow.ToString("O") + }; + var older = $"{origin}/notes/{Guid.NewGuid():N}"; + var newer = $"{origin}/notes/{Guid.NewGuid():N}"; + var notBobs = $"{new Uri(mallory.Id).GetLeftPart(UriPartial.Authority)}/notes/{Guid.NewGuid():N}"; + _harness.Peer.Serve(new Uri(older).AbsolutePath, Note(older, "an older post", bob).ToJsonString()); + _harness.Peer.Serve(new Uri(newer).AbsolutePath, Note(newer, "a newer post", bob).ToJsonString()); + _harness.Peer.Serve(new Uri(bob.Id).AbsolutePath + "/outbox", new JsonObject + { + ["id"] = bob.Id + "/outbox", ["type"] = "OrderedCollection", ["totalItems"] = 4, + ["orderedItems"] = new JsonArray( + new JsonObject { ["type"] = "Create", ["object"] = Note(newer, "a newer post", bob) }, + new JsonObject { ["type"] = "Announce", ["object"] = notBobs }, + new JsonObject { ["type"] = "Create", ["object"] = notBobs }, + new JsonObject { ["type"] = "Create", ["object"] = older }) + }.ToJsonString()); + var account = await _harness.Remote.GetActor(bob.Id, refresh: false, token); + var followId = alice.ActivityUri($"follow-{Guid.NewGuid():N}"); + await DB.Default.SaveAsync(new Following + { + AvatarId = alice.Id, TargetActorURI = bob.Id, TargetAccountId = account.ID, State = FollowState.Requested, FollowActivityURI = followId + }, token); + + await _harness.Deliver(bob, "/human-centipede", new JsonObject + { + ["id"] = $"{bob.Id}#accept-{Guid.NewGuid():N}", ["type"] = "Accept", ["actor"] = bob.Id, + ["object"] = new JsonObject { ["id"] = followId, ["type"] = "Follow", ["actor"] = alice.Uri, ["object"] = bob.Id } + }); + var job = await DB.Default.Find().Match(j => j.Kind == JobKind.BackfillOutbox && j.Payload == bob.Id).ExecuteSingleAsync(token); + await new OutboxBackfill(_harness.Db, _harness.Remote, _harness.RemotePosts).Handle(job, token); + + var held = (await DB.Default.Find().Match(p => p.ActorURI == bob.Id).ExecuteAsync(token)).Select(p => p.ObjectURI).ToHashSet(); + Assert.Equal(new HashSet { older, newer }, held); + Assert.False(await DB.Default.Find().Match(p => p.ObjectURI == notBobs).ExecuteAnyAsync(token)); + } + } +} diff --git a/PrivaPub.Tests/Support/Harness.cs b/PrivaPub.Tests/Support/Harness.cs index d1bcf67..4d7d2d4 100644 --- a/PrivaPub.Tests/Support/Harness.cs +++ b/PrivaPub.Tests/Support/Harness.cs @@ -62,7 +62,7 @@ namespace PrivaPub.Tests.Support Handlers = new IActivityHandler[] { new FollowHandler(Db, Local, Remote, Delivery), - new AcceptHandler(Db, Local, Quotes, Approvals, Participations, Relays), + new AcceptHandler(Db, Local, Quotes, Approvals, Participations, Relays, Queue), new RejectHandler(Db, Local, Quotes, Approvals, Participations, Relays), new UndoHandler(Db, Local, Reactions), new LikeHandler(Db, Reactions), diff --git a/PrivaPub.Tests/Support/RemoteActor.cs b/PrivaPub.Tests/Support/RemoteActor.cs index cfa5a93..6c5ff09 100644 --- a/PrivaPub.Tests/Support/RemoteActor.cs +++ b/PrivaPub.Tests/Support/RemoteActor.cs @@ -41,6 +41,7 @@ namespace PrivaPub.Tests.Support ["preferredUsername"] = Name, ["inbox"] = Id + "/inbox", ["followers"] = Id + "/followers", + ["outbox"] = Id + "/outbox", ["featured"] = Id + "/featured", ["publicKey"] = new JsonObject { diff --git a/PrivaPub/Federation/Actors/OutboxBackfill.cs b/PrivaPub/Federation/Actors/OutboxBackfill.cs new file mode 100644 index 0000000..6c7bbac --- /dev/null +++ b/PrivaPub/Federation/Actors/OutboxBackfill.cs @@ -0,0 +1,94 @@ +using MongoDB.Entities; + +using PrivaPub.Federation.Inbox; +using PrivaPub.Federation.Objects; +using PrivaPub.Infrastructure.Http; +using PrivaPub.Infrastructure.Jobs; +using PrivaPub.Models.Jobs; +using PrivaPub.StaticServices; + +using System.Text.Json; + +namespace PrivaPub.Federation.Actors +{ + // The latest public posts of an account a persona has just followed, read from the first page of its outbox (at most + // twenty, once a day) and kept as any fetched post: its profile shows them and threads find them at once, instead of + // only what it posts from then on. Homes still get only what arrives from now on, as on Mastodon. + public class OutboxBackfill : IJobHandler + { + public const int MaxPosts = 20; + + readonly DbEntities _dbEntities; + readonly IRemoteActorService _remoteActors; + readonly IRemotePosts _remotePosts; + + public OutboxBackfill(DbEntities dbEntities, IRemoteActorService remoteActors, IRemotePosts remotePosts) + { + _dbEntities = dbEntities; + _remoteActors = remoteActors; + _remotePosts = remotePosts; + } + + public JobKind Kind => JobKind.BackfillOutbox; + public int Concurrency => 1; + public int MaxAttempts => 2; + public int PerHostLimit => 1; + + public static string DedupeKey(string actorUri, DateTime day) => $"backfill|{actorUri}|{day:yyyyMMdd}"; + + public async Task Handle(Job job, CancellationToken token) + { + var actor = await _dbEntities.ForeignAvatars.Match(a => a.ActorURI == job.Payload && !a.DeletionAt.HasValue).ExecuteFirstAsync(token); + if (actor == default || string.IsNullOrEmpty(actor.OutboxURL) || !Origin.Same(actor.OutboxURL, actor.ActorURI)) + return JobOutcome.Done; + foreach (var objectUri in (await Items(actor, token)).Take(MaxPosts)) + await _remotePosts.StoreContext(objectUri, 0, token); + return JobOutcome.Done; + } + + // the posts the outbox's first page creates, newest first: Create items name or hold them; Announces are left out, + // being others' posts + async Task> Items(Models.User.ForeignAvatar actor, CancellationToken token) + { + using var scope = HttpScope.For("outbox"); + using var outbox = await _remoteActors.FetchObject(actor.OutboxURL, token); + if (outbox == default || outbox.Root.ValueKind != JsonValueKind.Object) + return new(); + var page = ItemsOf(outbox.Root, actor); + if (page.Count > 0 || !outbox.Root.TryGetProperty("first", out var first)) + return page; + if (first.ValueKind == JsonValueKind.Object) + return ItemsOf(first, actor); + var pageUri = first.ValueKind == JsonValueKind.String ? first.GetString() : default; + if (pageUri == default || !Origin.Same(pageUri, actor.ActorURI)) + return page; + using var fetched = await _remoteActors.FetchObject(pageUri, token); + return fetched == default || fetched.Root.ValueKind != JsonValueKind.Object ? page : ItemsOf(fetched.Root, actor); + } + + static List ItemsOf(JsonElement collection, Models.User.ForeignAvatar actor) + { + var posts = new List(); + foreach (var name in new[] { "orderedItems", "items" }) + { + if (!collection.TryGetProperty(name, out var list) || list.ValueKind != JsonValueKind.Array) + continue; + foreach (var item in list.EnumerateArray()) + { + var objectUri = item.ValueKind switch + { + JsonValueKind.Object when Text(item, "type") == "Create" && item.TryGetProperty("object", out var created) => + created.ValueKind == JsonValueKind.String ? created.GetString() : created.ValueKind == JsonValueKind.Object ? Text(created, "id") : default, + _ => default + }; + if (objectUri != default && Origin.Same(objectUri, actor.ActorURI)) + posts.Add(objectUri); + } + } + return posts; + } + + static string Text(JsonElement element, string property) => + element.TryGetProperty(property, out var value) && value.ValueKind == JsonValueKind.String ? value.GetString() : default; + } +} diff --git a/PrivaPub/Federation/Inbox/Handlers/AcceptHandler.cs b/PrivaPub/Federation/Inbox/Handlers/AcceptHandler.cs index 8d1e7ea..7ead988 100644 --- a/PrivaPub/Federation/Inbox/Handlers/AcceptHandler.cs +++ b/PrivaPub/Federation/Inbox/Handlers/AcceptHandler.cs @@ -20,10 +20,12 @@ namespace PrivaPub.Federation.Inbox.Handlers readonly IInteractionApprovals _approvals; readonly IParticipations _participations; readonly Relays.IRelays _relays; + readonly Infrastructure.Jobs.IJobQueue _queue; public AcceptHandler(DbEntities dbEntities, ILocalActorService localActors, IQuoteService quotes, IInteractionApprovals approvals = default, - IParticipations participations = default, Relays.IRelays relays = default) + IParticipations participations = default, Relays.IRelays relays = default, Infrastructure.Jobs.IJobQueue queue = default) { + _queue = queue; _relays = relays; _participations = participations; _dbEntities = dbEntities; @@ -67,8 +69,14 @@ namespace PrivaPub.Federation.Inbox.Handlers await Apply(following, token); } - protected virtual Task Apply(Following following, CancellationToken token) => - DB.Default.Update().MatchID(following.ID).Modify(f => f.State, FollowState.Accepted).ExecuteAsync(token); + // the follow holds: the account's latest public posts are read from its outbox (OutboxBackfill), once a day at most + protected virtual async Task Apply(Following following, CancellationToken token) + { + await DB.Default.Update().MatchID(following.ID).Modify(f => f.State, FollowState.Accepted).ExecuteAsync(token); + if (_queue != default) + await _queue.Enqueue(Models.Jobs.JobKind.BackfillOutbox, following.TargetActorURI, new Uri(following.TargetActorURI).Host, + Actors.OutboxBackfill.DedupeKey(following.TargetActorURI, DateTime.UtcNow), token); + } public static async Task FindFollowing(JsonNode follow, ForeignAvatar target, DbEntities dbEntities, ILocalActorService localActors, CancellationToken token) @@ -96,7 +104,7 @@ namespace PrivaPub.Federation.Inbox.Handlers { public RejectHandler(DbEntities dbEntities, ILocalActorService localActors, IQuoteService quotes, IInteractionApprovals approvals = default, IParticipations participations = default, Relays.IRelays relays = default) - : base(dbEntities, localActors, quotes, approvals, participations, relays) + : base(dbEntities, localActors, quotes, approvals, participations, relays, queue: default) { } diff --git a/PrivaPub/Middleware/SocialPubConfigurations.cs b/PrivaPub/Middleware/SocialPubConfigurations.cs index 09e73e0..3d2c4b5 100644 --- a/PrivaPub/Middleware/SocialPubConfigurations.cs +++ b/PrivaPub/Middleware/SocialPubConfigurations.cs @@ -111,6 +111,7 @@ namespace PrivaPub.Middleware .AddSingleton() .AddSingleton() .AddSingleton() + .AddSingleton() .AddSingleton() .AddSingleton() .AddSingleton() diff --git a/PrivaPub/Models/Jobs/Job.cs b/PrivaPub/Models/Jobs/Job.cs index 7fec6aa..28ac15a 100644 --- a/PrivaPub/Models/Jobs/Job.cs +++ b/PrivaPub/Models/Jobs/Job.cs @@ -33,7 +33,8 @@ namespace PrivaPub.Models.Jobs SendRecovery, CountAccount, PublishScheduled, - FetchReplies + FetchReplies, + BackfillOutbox } public enum JobState diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index 7b7e421..581005d 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -691,7 +691,8 @@ it, raw where it doesn't. events, user, notification, public, hashtag and list streams, each event mapped for its persona; a deletion reaches only the streams that showed the post), Web Push (gated: outbound traffic to push services), grouped notifications. Then advertise an honest version. -- **Backfill:** an author's outbox after following them. +- **Backfill:** an author's outbox after following them: **done 2026-10-05** (`OutboxBackfill`, the first page's + public posts, twenty at most, once a day; checked live against Mastodon). - **Long tail:** - MFM rendering data; - Misskey actor extras; diff --git a/tools/pasture/scenarios/pins.sh b/tools/pasture/scenarios/pins.sh index 9aa4594..0c04a85 100644 --- a/tools/pasture/scenarios/pins.sh +++ b/tools/pasture/scenarios/pins.sh @@ -1,6 +1,7 @@ # Pinned posts across servers (featured, FEP-e232's Add and Remove): a Mastodon account pins and unpins while alice # follows it; alice pins and unpins while it follows her; an account that pinned before PrivaPub ever saw it shows its -# pins once resolved (its featured collection, read with its counts). Needs the mastodon peer. +# pins once resolved (its featured collection, read with its counts), and following it brings its earlier posts (its +# outbox). Needs the mastodon peer. M=https://mastodon.test:6443 mcurl() { curl -sk --resolve mastodon.test:6443:127.0.0.1 "$@"; } . "$here/peers/mastodon.sh" @@ -52,3 +53,10 @@ mcurl -o /dev/null -X POST -H "Authorization: Bearer $ET" "$M/api/v1/statuses/$( early_on_p=$(curl -s -H "$PH" "$P/api/v2/search?q=$early@mastodon.test&resolve=true&type=accounts" | j "print(d['accounts'][0]['id'])") until_true 45 '[ "$(p_pinned $early_on_p)" = "$(echo "$e_post" | j "print(d[\"uri\"])")" ]' \ && ok "an account's earlier pin shows once it is resolved" || ko "the earlier pin never showed ($(p_pinned $early_on_p))" + +echo " posts made before" +# alice follows the fresh account: its outbox's latest public posts come to its profile here (OutboxBackfill) +e_other=$(mcurl -X POST -H "Authorization: Bearer $ET" "$M/api/v1/statuses" -d "status=written before anyone followed $run&visibility=public" | j "print(d['uri'])") +curl -s -o /dev/null -X POST -H "$PH" "$P/api/v1/accounts/$early_on_p/follow" +until_true 60 '[ "$(curl -s -H "$PH" "$P/api/v1/accounts/$early_on_p/statuses?limit=40" | j "print(any(s[\"uri\"] == \"$e_other\" for s in d))")" = "True" ]' \ + && ok "following an account brings its earlier posts to its profile" || ko "the account's earlier posts never came"