diff --git a/CLAUDE.md b/CLAUDE.md index fa428e5..34dd754 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -561,6 +561,11 @@ tools/pasture/run.sh down # removes e checks. - **Pins (`scenarios/pins.sh`, needs mastodon):** a Mastodon account pins and unpins while alice follows it, alice pins and unpins while it follows her, and a fresh account's earlier pin shows once PrivaPub resolves it. 8 checks. +- **Relays (`peers/relay.sh` Activity-Relay 2.0.9 as `relay.test`, on the shared Redis's database 13; + `peers/aoderelay.sh` aode-relay 0.3.129 as `aoderelay.test`):** `appsettings.Pasture.json` names both in + `Federation:Relays`, so PrivaPub subscribes a minute after it starts. `scenarios/relay.sh` has Mastodon subscribe to + each in turn, checks that a post of an account nobody here follows reaches the federated timeline (forwarded by one, + announced by the other), and has Mastodon leave again, so the town sees no relayed posts. 13 checks. - **Smithereen (1.0.3):** its image on the shared MySQL (database `smithereen`, its schema from the image's commit), with imgproxy and a file server behind Caddy as `smithereen.test` (`/i` and `/s`), trusting the CA through a JDK store with it added (`JAVA_TOOL_OPTIONS`). MySQL takes its stored functions only with diff --git a/FEDERATION.md b/FEDERATION.md index c9d14d1..2aa9a4f 100644 --- a/FEDERATION.md +++ b/FEDERATION.md @@ -33,6 +33,7 @@ and every run starting clean, with signed fetches required (as privapub.thepra.d - **Mbin 1.10.1** - **NodeBB 4.16.1** - **Smithereen 1.0.3** +- **Activity-Relay 2.0.9** and **aode-relay 0.3.129**, as relays PrivaPub reads from - in the town only (a seeded community checked server by server): **Hollo 0.9.19**, **Iceshrimp.NET 2026.1.2-beta**, **Pleroma 2.10.2** @@ -250,6 +251,16 @@ The page's OpenGraph and Twitter tags give the title, description and image; the days and shared by every account on the server. Images are served to clients only through PrivaPub's media proxy. `Federation:FetchLinkPreviews=false` turns page fetching off. +## Relays + +PrivaPub reads from the relays `Federation:Relays` names (by their actor's address, or their inbox's), none by default. +Its instance actor follows `Public` at each, as Mastodon subscribes (`Federation/Relays/Relays.cs`, a minute after start +and every six hours: an unanswered or refused subscription is asked again a day later, and a relay no longer named gets +the `Undo`). What an accepted relay passes on comes to the federated timeline, never to anyone's home: a public post it +forwards as its author sent it (Activity-Relay), read again from its origin like any forwarded post, and a post it +announces (aode-relay), kept as its author's and never as the relay's boost. Nothing else is taken from a relay, and +nothing of a persona's is sent to one: sending public posts to relays waits for the owner. + ## Server descriptions and the crawler PrivaPub keeps statistics about servers, never about accounts (see `/stargazing` on the server). diff --git a/PrivaPub.Tests/Federation/RelayTests.cs b/PrivaPub.Tests/Federation/RelayTests.cs new file mode 100644 index 0000000..d3bfc39 --- /dev/null +++ b/PrivaPub.Tests/Federation/RelayTests.cs @@ -0,0 +1,125 @@ +using MongoDB.Entities; + +using PrivaPub.Models.Federation; +using PrivaPub.Models.Post; +using PrivaPub.Tests.Support; + +using System.Text.Json.Nodes; + +namespace PrivaPub.Tests.Federation +{ + // relays the configuration names: the instance actor follows Public there, and the public posts the relay passes on, + // forwarded or announced, come to the federated timeline as their authors' posts; nothing else is taken from it + [Trait("Category", "Integration")] + [Xunit.Collection(nameof(Exclusive))] + public sealed class RelayTests : IAsyncLifetime + { + Harness _harness; + + public async ValueTask InitializeAsync() + { + Assert.SkipUnless(MongoFixture.Enabled, MongoFixture.Skip); + _harness = await Harness.Start(); + } + + public async ValueTask DisposeAsync() + { + if (_harness != default) + { + _harness.RelayOptions.Relays.Clear(); + await _harness.Relays.Reconcile(TestContext.Current.CancellationToken); + await _harness.DisposeAsync(); + } + } + + static CancellationToken Token => TestContext.Current.CancellationToken; + static string Origin(RemoteActor actor) => new Uri(actor.Id).GetLeftPart(UriPartial.Authority); + + string Note(RemoteActor author, string text, string to = "https://www.w3.org/ns/activitystreams#Public") + { + var path = $"/notes/{Guid.NewGuid():N}"; + _harness.Peer.Serve(path, new JsonObject + { + ["id"] = Origin(author) + path, ["type"] = "Note", ["attributedTo"] = author.Id, ["content"] = $"

{text}

", + ["to"] = new JsonArray(to), ["published"] = DateTime.UtcNow.ToString("O") + }.ToJsonString()); + return Origin(author) + path; + } + + JsonObject Create(RemoteActor author, string noteId) => new() + { + ["id"] = noteId + "/activity", ["type"] = "Create", ["actor"] = author.Id, + ["to"] = new JsonArray("https://www.w3.org/ns/activitystreams#Public"), + ["object"] = new JsonObject { ["id"] = noteId, ["type"] = "Note", ["attributedTo"] = author.Id } + }; + + // a relay the instance actor subscribes to, which accepts + async Task Subscribed() + { + var relay = new RemoteActor(_harness.Peer, "relay", type: "Service"); + _harness.RelayOptions.Relays.Add(relay.Id); + await _harness.Relays.Reconcile(Token); + var follow = Assert.Single(await _harness.Outgoing(relay.Id + "/inbox"), a => a["type"]!.GetValue() == "Follow"); + await _harness.Deliver(relay, "/human-centipede", new JsonObject + { + ["id"] = $"{relay.Id}#accept-{Guid.NewGuid():N}", ["type"] = "Accept", ["actor"] = relay.Id, ["object"] = follow.DeepClone() + }); + return relay; + } + + [Fact] + public async Task The_instance_actor_follows_public_at_a_configured_relay_and_takes_its_answer() + { + var relay = await Subscribed(); + + var follow = Assert.Single(await _harness.Outgoing(relay.Id + "/inbox")); + Assert.Equal(("Follow", "https://www.w3.org/ns/activitystreams#Public"), (follow["type"]!.GetValue(), follow["object"]!.GetValue())); + Assert.Equal((await _harness.Local.GetInstanceActor(Token)).Uri, follow["actor"]!.GetValue()); + Assert.Equal(RelayState.Accepted, (await DB.Default.Find().Match(s => s.ActorURI == relay.Id).ExecuteSingleAsync(Token)).State); + Assert.True(await _harness.Relays.Passes(relay.Id, Token)); + + _harness.RelayOptions.Relays.Clear(); + await _harness.Relays.Reconcile(Token); + + Assert.Contains(await _harness.Outgoing(relay.Id + "/inbox"), a => a["type"]!.GetValue() == "Undo"); + Assert.False(await DB.Default.Find().Match(s => s.ActorURI == relay.Id).ExecuteAnyAsync(Token)); + } + + [Fact] + public async Task A_public_post_a_subscribed_relay_forwards_is_kept_and_one_from_anyone_else_is_not() + { + var relay = await Subscribed(); + var stranger = new RemoteActor(_harness.Peer, "stranger", type: "Service"); + var author = new RemoteActor(_harness.Peer, "author", _harness.Peer.B); + var relayed = Note(author, "through the relay"); + var forwarded = Note(author, "through a stranger"); + var followersOnly = Note(author, "for followers", author.Id + "/followers"); + + await _harness.Deliver(relay, "/human-centipede", Create(author, relayed)); + await _harness.Deliver(stranger, "/human-centipede", Create(author, forwarded)); + await _harness.Deliver(relay, "/human-centipede", Create(author, followersOnly)); + + var kept = await DB.Default.Find().Match(p => p.ActorURI == author.Id).ExecuteAsync(Token); + Assert.Equal(new[] { relayed }, kept.Select(p => p.ObjectURI)); + Assert.Equal(PostVisibility.Public, kept[0].Visibility); + } + + [Fact] + public async Task A_post_a_relay_announces_is_its_authors_never_the_relays_boost() + { + var relay = await Subscribed(); + var author = new RemoteActor(_harness.Peer, "author", _harness.Peer.B); + var noteId = Note(author, "announced by the relay"); + + await _harness.Deliver(relay, "/human-centipede", new JsonObject + { + ["id"] = $"{relay.Id}/activities/{Guid.NewGuid():N}", ["type"] = "Announce", ["actor"] = relay.Id, + ["to"] = new JsonArray(relay.Id + "/followers"), ["object"] = noteId + }); + + var post = await DB.Default.Find().Match(p => p.ObjectURI == noteId).ExecuteSingleAsync(Token); + Assert.Equal(author.Id, post.ActorURI); + Assert.False(await DB.Default.Find().Match(p => p.ReblogOfPostId == post.ID).ExecuteAnyAsync(Token)); + } + } +} diff --git a/PrivaPub.Tests/Support/Harness.cs b/PrivaPub.Tests/Support/Harness.cs index 2647ce5..d1bcf67 100644 --- a/PrivaPub.Tests/Support/Harness.cs +++ b/PrivaPub.Tests/Support/Harness.cs @@ -14,6 +14,7 @@ using PrivaPub.Federation.Actors; using PrivaPub.Federation.Inbox; using PrivaPub.Federation.Inbox.Handlers; using PrivaPub.Federation.Outbox; +using PrivaPub.Federation.Relays; using PrivaPub.Infrastructure.Jobs; using PrivaPub.Models; using PrivaPub.Federation.Objects; @@ -57,19 +58,20 @@ namespace PrivaPub.Tests.Support Follows = new FollowService(Db, Local, Remote, Delivery, new KeyLocalizer(), NullLogger.Instance); Relationships = new RelationshipService(Db, Follows, Delivery); Featured = new FeaturedPosts(Db, Remote, RemotePosts); + Relays = new Relays(Microsoft.Extensions.Options.Options.Create(RelayOptions), Remote, Local, Delivery); Handlers = new IActivityHandler[] { new FollowHandler(Db, Local, Remote, Delivery), - new AcceptHandler(Db, Local, Quotes, Approvals, Participations), - new RejectHandler(Db, Local, Quotes, Approvals, Participations), + new AcceptHandler(Db, Local, Quotes, Approvals, Participations, Relays), + new RejectHandler(Db, Local, Quotes, Approvals, Participations, Relays), new UndoHandler(Db, Local, Reactions), new LikeHandler(Db, Reactions), new EmojiReactHandler(Db, Reactions), new QuoteRequestHandler(Quotes), new DislikeHandler(Db), new JoinHandler(Db, Local, Delivery), - new AnnounceHandler(Db, Local, RemotePosts, Fanout, Remote, Records, Quotes), - new CreateHandler(Db, Local, Remote, Delivery, new NoBlocks(), Fanout, RemotePosts, Groups, Records, Polls, new NoPreviews(), Quotes, Approvals), + new AnnounceHandler(Db, Local, RemotePosts, Fanout, Remote, Records, Quotes, relays: Relays), + new CreateHandler(Db, Local, Remote, Delivery, new NoBlocks(), Fanout, RemotePosts, Groups, Records, Polls, new NoPreviews(), Quotes, Approvals, Relays), new DeleteHandler(Db, Local, Remote, Delivery, Groups, Quotes), new UpdateHandler(Db, Local, Remote, Groups, Records, Quotes, Delivery), new FlagHandler(Db, Local), @@ -118,6 +120,9 @@ namespace PrivaPub.Tests.Support public TimelineService Timelines { get; } public RelationshipService Relationships { get; } public FeaturedPosts Featured { get; } + public Relays Relays { get; } + // the relays the harness's instance actor subscribes to: a test sets them before Relays.Reconcile + public PrivaPub.Infrastructure.Http.FederationOptions RelayOptions { get; } = new(); public ReportService Reports { get; } public async Task FollowedBy(LocalActor local, RemoteActor follower) => diff --git a/PrivaPub/Federation/Inbox/Arrival.cs b/PrivaPub/Federation/Inbox/Arrival.cs index e4e6354..c1ddbe9 100644 --- a/PrivaPub/Federation/Inbox/Arrival.cs +++ b/PrivaPub/Federation/Inbox/Arrival.cs @@ -5,9 +5,10 @@ using PrivaPub.Models.Post; namespace PrivaPub.Federation.Inbox { - // Raw: the activity as it came, when its own author sent it (ReplyRelay passes it on as it is) + // Raw: the activity as it came, when its own author sent it (ReplyRelay passes it on as it is); ForwardedBy: the server + // that passed it on instead (a relay, a thread's server) public sealed record Arrival(string ActivityId, string ActivityType, string ActorURI, string Inbox, string KeyId, string Algorithm, - IReadOnlyList SignedHeaders, DateTime ReceivedAt, string Context = default, string Raw = default) + IReadOnlyList SignedHeaders, DateTime ReceivedAt, string Context = default, string Raw = default, string ForwardedBy = default) { static readonly AsyncLocal current = new(); diff --git a/PrivaPub/Federation/Inbox/Handlers/AcceptHandler.cs b/PrivaPub/Federation/Inbox/Handlers/AcceptHandler.cs index 5e0ae88..8d1e7ea 100644 --- a/PrivaPub/Federation/Inbox/Handlers/AcceptHandler.cs +++ b/PrivaPub/Federation/Inbox/Handlers/AcceptHandler.cs @@ -19,10 +19,12 @@ namespace PrivaPub.Federation.Inbox.Handlers readonly IQuoteService _quotes; readonly IInteractionApprovals _approvals; readonly IParticipations _participations; + readonly Relays.IRelays _relays; public AcceptHandler(DbEntities dbEntities, ILocalActorService localActors, IQuoteService quotes, IInteractionApprovals approvals = default, - IParticipations participations = default) + IParticipations participations = default, Relays.IRelays relays = default) { + _relays = relays; _participations = participations; _dbEntities = dbEntities; _localActors = localActors; @@ -49,6 +51,11 @@ namespace PrivaPub.Federation.Inbox.Handlers Arrival.Accept("participation-answer"); return; } + if (_relays != default && await _relays.Answered(activity, actor, accepted: Type == "Accept", token)) + { + Arrival.Accept("relay-answer"); + return; + } var following = await FindFollowing(activity["object"], actor, _dbEntities, _localActors, token); if (following == default) { @@ -88,8 +95,8 @@ namespace PrivaPub.Federation.Inbox.Handlers public class RejectHandler : AcceptHandler { public RejectHandler(DbEntities dbEntities, ILocalActorService localActors, IQuoteService quotes, IInteractionApprovals approvals = default, - IParticipations participations = default) - : base(dbEntities, localActors, quotes, approvals, participations) + IParticipations participations = default, Relays.IRelays relays = default) + : base(dbEntities, localActors, quotes, approvals, participations, relays) { } diff --git a/PrivaPub/Federation/Inbox/Handlers/AnnounceHandler.cs b/PrivaPub/Federation/Inbox/Handlers/AnnounceHandler.cs index e7041c7..da64043 100644 --- a/PrivaPub/Federation/Inbox/Handlers/AnnounceHandler.cs +++ b/PrivaPub/Federation/Inbox/Handlers/AnnounceHandler.cs @@ -38,9 +38,12 @@ namespace PrivaPub.Federation.Inbox.Handlers // handler is one of them; tests set it public IEnumerable Relays { get; set; } + readonly PrivaPub.Federation.Relays.IRelays _relayService; + public AnnounceHandler(DbEntities dbEntities, ILocalActorService localActors, IRemotePosts remotePosts, IFanout fanout, IRemoteActorService remoteActors, - IObjectRecords records, IQuoteService quotes, IServiceProvider services = default) + IObjectRecords records, IQuoteService quotes, IServiceProvider services = default, PrivaPub.Federation.Relays.IRelays relays = default) { + _relayService = relays; _services = services; _records = records; _quotes = quotes; @@ -74,6 +77,11 @@ namespace PrivaPub.Federation.Inbox.Handlers return; } + if (_relayService != default && await _relayService.Passes(actor.ActorURI, token)) + { + await FromRelay(objectUri, token); + return; + } var isLocal = objectUri.StartsWith(_localActors.BaseAddress + "/", StringComparison.OrdinalIgnoreCase); var original = await _dbEntities.Posts.Match(p => p.ObjectURI == objectUri && !p.DeletedAt.HasValue && p.ReblogOfPostId == null).ExecuteFirstAsync(token); var followed = await _dbEntities.Followings.Match(f => f.TargetActorURI == actor.ActorURI && f.State == FollowState.Accepted).ExecuteAnyAsync(token); @@ -139,6 +147,27 @@ namespace PrivaPub.Federation.Inbox.Handlers await _fanout.Distribute(reblog, token); } + // a post a relay we subscribe to announces: kept as its author's, for the federated timeline, never as the relay's + // boost (Mastodon unwraps them the same way) + async Task FromRelay(string objectUri, CancellationToken token) + { + var held = await _dbEntities.Posts.Match(p => p.ObjectURI == objectUri).ExecuteFirstAsync(token); + var post = held ?? await _remotePosts.StoreContext(objectUri, 0, token); + if (post == default) + { + Arrival.Drop("fetch-failed"); + return; + } + Arrival.About(post.ObjectType ?? "Note", post.Visibility, post.CreationDate); + if (held != default) + { + Arrival.Drop("duplicate"); + return; + } + Arrival.Accept("relayed"); + await _fanout.Distribute(post, token); + } + async Task GroupActivity(JsonNode inner, ForeignAvatar group, CancellationToken token) { Arrival.About(Value(inner, "type")); diff --git a/PrivaPub/Federation/Inbox/Handlers/CreateHandler.cs b/PrivaPub/Federation/Inbox/Handlers/CreateHandler.cs index 259695c..5bf73ac 100644 --- a/PrivaPub/Federation/Inbox/Handlers/CreateHandler.cs +++ b/PrivaPub/Federation/Inbox/Handlers/CreateHandler.cs @@ -41,11 +41,13 @@ namespace PrivaPub.Federation.Inbox.Handlers readonly ILinkPreviews _previews; readonly IQuoteService _quotes; readonly IInteractionApprovals _approvals; + readonly Relays.IRelays _relays; public CreateHandler(DbEntities dbEntities, ILocalActorService localActors, IRemoteActorService remoteActors, IDeliveryService delivery, IDomainBlocks domainBlocks, IFanout fanout, IRemotePosts remotePosts, IGroupDistributor groups, IObjectRecords records, IPollService polls, ILinkPreviews previews, IQuoteService quotes, - IInteractionApprovals approvals = default) + IInteractionApprovals approvals = default, Relays.IRelays relays = default) { + _relays = relays; _approvals = approvals; _quotes = quotes; _previews = previews; @@ -183,8 +185,12 @@ namespace PrivaPub.Federation.Inbox.Handlers var inFollowedGroup = !followed && visibility is PostVisibility.Public or PostVisibility.Unlisted && note.Audience != default && Origin.Same(note.Audience, author.ActorURI) && Origin.Same(note.Id, note.Audience) && await _dbEntities.Followings.Match(f => f.TargetActorURI == note.Audience && f.State == FollowState.Accepted).ExecuteAnyAsync(token); + // a public post a relay we subscribe to passes on, for the federated timeline (read again from its origin, as any + // forwarded post) + var relayed = !followed && visibility == PostVisibility.Public && _relays != default + && await _relays.Passes(Arrival.Current?.ForwardedBy, token); if (visibility == PostVisibility.Direct ? persons.Count == 0 - : group == default && persons.Count == 0 && !repliesToLocal && !followed && !repliesToFollowed && !inFollowedGroup) + : group == default && persons.Count == 0 && !repliesToLocal && !followed && !repliesToFollowed && !inFollowedGroup && !relayed) { Arrival.Drop("not-addressed"); return; diff --git a/PrivaPub/Federation/Inbox/InboxProcessor.cs b/PrivaPub/Federation/Inbox/InboxProcessor.cs index 0d1789a..5c4cc64 100644 --- a/PrivaPub/Federation/Inbox/InboxProcessor.cs +++ b/PrivaPub/Federation/Inbox/InboxProcessor.cs @@ -78,7 +78,7 @@ namespace PrivaPub.Federation.Inbox var arrival = new Arrival(Id(activity), type, actor.ActorURI, payload.Inbox, payload.KeyId, payload.Algorithm, payload.SignedHeaders ?? Array.Empty(), payload.ReceivedAt ?? job.CreatedAt, activity["@context"]?.ToJsonString(), - payload.ForwardedBy == default ? payload.Activity : default); + payload.ForwardedBy == default ? payload.Activity : default, payload.ForwardedBy); Arrival.Current = arrival; try { diff --git a/PrivaPub/Federation/Relays/Relays.cs b/PrivaPub/Federation/Relays/Relays.cs new file mode 100644 index 0000000..6f59447 --- /dev/null +++ b/PrivaPub/Federation/Relays/Relays.cs @@ -0,0 +1,163 @@ +using Microsoft.Extensions.Options; + +using MongoDB.Entities; + +using PrivaPub.Federation.Actors; +using PrivaPub.Federation.Objects; +using PrivaPub.Federation.Outbox; +using PrivaPub.Federation.Rendering; +using PrivaPub.Infrastructure.Http; +using PrivaPub.Models.Federation; +using PrivaPub.Models.User; + +using System.Text.Json.Nodes; + +using static PrivaPub.Federation.Objects.ActivityJson; + +namespace PrivaPub.Federation.Relays +{ + public interface IRelays + { + Task Reconcile(CancellationToken token); + Task Answered(JsonNode activity, ForeignAvatar actor, bool accepted, CancellationToken token); + Task Passes(string actorUri, CancellationToken token); + } + + // Relays (Activity-Relay, aode-relay, pub-relay): the instance actor follows Public at each relay the configuration + // names, as Mastodon subscribes, and takes what they pass on: public posts, forwarded as their authors sent them (read + // again from their origin, as any forwarded post) or announced by the relay. Nothing of a persona's is sent to a relay. + public class Relays : IRelays + { + static readonly TimeSpan AskAgain = TimeSpan.FromDays(1); + + readonly IOptions _options; + readonly IRemoteActorService _remoteActors; + readonly ILocalActorService _localActors; + readonly IDeliveryService _delivery; + volatile HashSet _accepted; + + public Relays(IOptions options, IRemoteActorService remoteActors, ILocalActorService localActors, IDeliveryService delivery) + { + _options = options; + _remoteActors = remoteActors; + _localActors = localActors; + _delivery = delivery; + } + + // subscribes to the relays the configuration names (again a day after an unanswered or refused request), and + // unsubscribes from those it no longer names + public async Task Reconcile(CancellationToken token) + { + var configured = (_options.Value.Relays ?? new()).Where(r => !string.IsNullOrWhiteSpace(r)).Select(r => r.Trim()).Distinct().ToList(); + var known = await DB.Default.Find().ExecuteAsync(token); + if (configured.Count == 0 && known.Count == 0) + return; + var instance = await _localActors.GetInstanceActor(token); + foreach (var gone in known.Where(k => !configured.Contains(k.Configured))) + { + if (gone.State != RelayState.Rejected && !string.IsNullOrEmpty(gone.InboxURL)) + await _delivery.Enqueue(instance, new[] { gone.InboxURL }, Undo(instance, gone.FollowActivityURI), token); + await DB.Default.DeleteAsync(gone.ID); + } + foreach (var address in configured) + { + var subscription = known.FirstOrDefault(k => k.Configured == address); + if (subscription is { State: RelayState.Accepted } || subscription != default && DateTime.UtcNow - subscription.RequestedAt < AskAgain) + continue; + var relay = await Resolve(address, token); + if (relay == default || string.IsNullOrEmpty(relay.InboxURL)) + continue; + subscription ??= new RelaySubscription { Configured = address }; + subscription.ActorURI = relay.ActorURI; + subscription.InboxURL = relay.InboxURL; + subscription.FollowActivityURI = instance.ActivityUri($"relay-{Guid.NewGuid():N}"); + subscription.State = RelayState.Pending; + subscription.RequestedAt = DateTime.UtcNow; + subscription.AnsweredAt = default; + await DB.Default.SaveAsync(subscription, token); + await _delivery.Enqueue(instance, new[] { relay.InboxURL }, Follow(instance, subscription.FollowActivityURI), token); + } + _accepted = default; + } + + // the relay's Accept or Reject of the subscription, by the Follow it answers + public async Task Answered(JsonNode activity, ForeignAvatar actor, bool accepted, CancellationToken token) + { + var followId = Id(activity["object"]); + if (followId == default) + return false; + var subscription = await DB.Default.Find().Match(s => s.FollowActivityURI == followId && s.ActorURI == actor.ActorURI) + .ExecuteFirstAsync(token); + if (subscription == default) + return false; + await DB.Default.Update().MatchID(subscription.ID) + .Modify(s => s.State, accepted ? RelayState.Accepted : RelayState.Rejected) + .Modify(s => s.AnsweredAt, DateTime.UtcNow) + .ExecuteAsync(token); + _accepted = default; + return true; + } + + // whether the actor is a relay that accepted our subscription + public async Task Passes(string actorUri, CancellationToken token) + { + if (string.IsNullOrEmpty(actorUri)) + return false; + var accepted = _accepted ??= (await DB.Default.Find().Match(s => s.State == RelayState.Accepted).ExecuteAsync(token)) + .Select(s => s.ActorURI).ToHashSet(StringComparer.Ordinal); + return accepted.Contains(actorUri); + } + + // the relay's actor, from its address or, for a relay named by its inbox, from the /actor beside it + async Task Resolve(string address, CancellationToken token) + { + var actor = await _remoteActors.GetActor(address, refresh: false, token); + if (actor != default || !address.EndsWith("/inbox", StringComparison.Ordinal)) + return actor; + return await _remoteActors.GetActor(address[..^"/inbox".Length] + "/actor", refresh: false, token); + } + + static JsonObject Follow(LocalActor instance, string id) => new() + { + ["@context"] = ActivityPubRenderer.Context(), + ["id"] = id, + ["type"] = "Follow", + ["actor"] = instance.Uri, + ["object"] = Addressing.Public + }; + + static JsonObject Undo(LocalActor instance, string followId) => new() + { + ["@context"] = ActivityPubRenderer.Context(), + ["id"] = instance.ActivityUri($"relay-undo-{Guid.NewGuid():N}"), + ["type"] = "Undo", + ["actor"] = instance.Uri, + ["object"] = new JsonObject { ["id"] = followId, ["type"] = "Follow", ["actor"] = instance.Uri, ["object"] = Addressing.Public } + }; + } + + // subscribes to the configured relays a minute after start, then every six hours + public sealed class RelaySubscriber(IServiceScopeFactory scopes, ILogger logger) : BackgroundService + { + static readonly TimeSpan Every = TimeSpan.FromHours(6); + + protected override async Task ExecuteAsync(CancellationToken token) + { + await Task.Delay(TimeSpan.FromMinutes(1), token); + using var timer = new PeriodicTimer(Every); + do + { + try + { + using var scope = scopes.CreateScope(); + await scope.ServiceProvider.GetRequiredService().Reconcile(token); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + logger.LogWarning(ex, "Relay subscriptions could not be brought up to date"); + } + } + while (await timer.WaitForNextTickAsync(token)); + } + } +} diff --git a/PrivaPub/Infrastructure/Data/Indexes.cs b/PrivaPub/Infrastructure/Data/Indexes.cs index 5e674bc..9baf353 100644 --- a/PrivaPub/Infrastructure/Data/Indexes.cs +++ b/PrivaPub/Infrastructure/Data/Indexes.cs @@ -122,6 +122,7 @@ namespace PrivaPub.Infrastructure.Data (() => DB.Default.Index().Key(b => b.AvatarId, KeyType.Ascending).Key(b => b.Domain, KeyType.Ascending).Option(o => o.Unique = true).CreateAsync(token), "domain block"), (() => DB.Default.Index().Key(b => b.AvatarId, KeyType.Ascending).Key(b => b.PostId, KeyType.Ascending).Option(o => o.Unique = true).CreateAsync(token), "bookmark"), (() => DB.Default.Index().Key(p => p.AvatarId, KeyType.Ascending).Key(p => p.PostId, KeyType.Ascending).Option(o => o.Unique = true).CreateAsync(token), "pin"), + (() => DB.Default.Index().Key(r => r.Configured, KeyType.Ascending).Option(o => o.Unique = true).CreateAsync(token), "relay"), (() => DB.Default.Index().Key(f => f.ActorURI, KeyType.Ascending).Key(f => f.ObjectURI, KeyType.Ascending).Option(o => o.Unique = true).CreateAsync(token), "remote-featured") }) await pair.Item1(); diff --git a/PrivaPub/Infrastructure/Http/FederationOptions.cs b/PrivaPub/Infrastructure/Http/FederationOptions.cs index 7ed10ec..1db8222 100644 --- a/PrivaPub/Infrastructure/Http/FederationOptions.cs +++ b/PrivaPub/Infrastructure/Http/FederationOptions.cs @@ -11,5 +11,8 @@ namespace PrivaPub.Infrastructure.Http // At two the inbox kept up with about 110 activities a second (tools/pasture/load.sh) public int InboxConcurrency { get; set; } = 8; public int DeliveryConcurrency { get; set; } = 8; + // relays the instance actor subscribes to, by their actor's (or inbox's) address: their public posts come to the + // federated timeline, and nothing is sent to them but the subscription + public List Relays { get; set; } = new(); } } diff --git a/PrivaPub/Middleware/SocialPubConfigurations.cs b/PrivaPub/Middleware/SocialPubConfigurations.cs index 5cb150e..09e73e0 100644 --- a/PrivaPub/Middleware/SocialPubConfigurations.cs +++ b/PrivaPub/Middleware/SocialPubConfigurations.cs @@ -121,7 +121,9 @@ namespace PrivaPub.Middleware .AddSingleton() .AddSingleton() .AddHostedService() - .AddHostedService(); + .AddHostedService() + .AddSingleton() + .AddHostedService(); } public static IServiceCollection PrivaPubStatisticsConfiguration(this IServiceCollection service, IConfiguration configuration) { diff --git a/PrivaPub/Models/Federation/RelaySubscription.cs b/PrivaPub/Models/Federation/RelaySubscription.cs new file mode 100644 index 0000000..2c30efb --- /dev/null +++ b/PrivaPub/Models/Federation/RelaySubscription.cs @@ -0,0 +1,24 @@ +using MongoDB.Entities; + +namespace PrivaPub.Models.Federation +{ + // a relay the instance actor subscribes to (Federation:Relays): it passes on the public posts its other subscribers + // send it. PrivaPub only reads from it, and sends it nothing of its personas. + public class RelaySubscription : Entity + { + public string Configured { get; set; }//as the configuration names it + public string ActorURI { get; set; } + public string InboxURL { get; set; } + public string FollowActivityURI { get; set; } + public RelayState State { get; set; } = RelayState.Pending; + public DateTime RequestedAt { get; set; } = DateTime.UtcNow; + public DateTime? AnsweredAt { get; set; } + } + + public enum RelayState + { + Pending, + Accepted, + Rejected + } +} diff --git a/docs/INTEROP-BACKLOG.md b/docs/INTEROP-BACKLOG.md index 8dacd82..e3de685 100644 --- a/docs/INTEROP-BACKLOG.md +++ b/docs/INTEROP-BACKLOG.md @@ -1,17 +1,12 @@ # Interop backlog -Generated by `tools/pasture/town.sh backlog --write` from `village-20261005-111857` on 2026-10-05. Do not edit by hand: fix the cause, or record a known gap in `tools/pasture/town/gaps.json`, and generate it again. +Generated by `tools/pasture/town.sh backlog --write` from `village-20261005-220341` on 2026-10-05. Do not edit by hand: fix the cause, or record a known gap in `tools/pasture/town/gaps.json`, and generate it again. -Checks: 2558 pass, 6 fail, 1 known gaps, 0 known gaps now passing. +Checks: 2574 pass, 0 fail, 1 known gaps, 0 known gaps now passing. ## New failures -- **`count.boost.public|*|misskey|control`**: 1 failing. Example: p19 expected `>=1`, got `0`. -- **`count.like.community|*|mastodon|control`**: 1 failing. Example: p161 expected `>=1`, got `0`. -- **`count.boost.public|*|sharkey|control`**: 1 failing. Example: p175 expected `>=1`, got `0`. -- **`edit.text|privapub|gts|out`**: 1 failing. Example: p4 expected `(edited)`, got `l market harvest rain quiet tide garden festival bridge (p4)`. -- **`client.render.image.followers|privapub|decepub|local`**: 1 failing. Example: p15 expected `None`, got `picture 'picture 1 of p15' did not load`. -- **`client.render.image.public|privapub|decepub|local`**: 1 failing. Example: p33 expected `None`, got `picture 'picture 1 of p33' did not load`. +None. ## Known gaps still failing diff --git a/docs/INTEROP.md b/docs/INTEROP.md index 37d5e64..d8726c7 100644 --- a/docs/INTEROP.md +++ b/docs/INTEROP.md @@ -753,6 +753,20 @@ PeerTube's own instance account announces each new video too, which we drop: nob follow only PeerTube-like channels and accounts, never a persona. It checks the `Host` header against its own name, without a port, before it gives out its OAuth client. +### Relays: Activity-Relay 2.0.9 and aode-relay 0.3.129 + +- **Activity-Relay** takes a subscription as a `Follow` of `Public` from an actor with a shared inbox (Mastodon's way) + and passes every public activity a subscriber sends on to the others as it came, signed with the relay's key; only a + follower whose actor ends in `/relay` (LitePub's way) gets an `Announce` instead. +- **aode-relay** takes either kind and announces the post from its own actor. +- A forwarded post carries its author's LD signature when Mastodon wrote it; PrivaPub does not verify LD signatures, + so it reads every relayed post again from its origin (one signed request each). Verifying them, or FEP-8b32 proofs, + would save that request. +- **Pasture evidence (2026-10-05, `tools/pasture/scenarios/relay.sh`):** 13 checks pass: PrivaPub's instance actor + subscribes to each relay and takes its Accept; Mastodon subscribes; a public post of a Mastodon account nobody here + follows reaches the federated timeline, forwarded by Activity-Relay and announced by aode-relay (as its author's, + never as the relay's boost), and nobody's home; nothing of a persona's goes to a relay. + ### Smithereen 1.0.3 - **Walls:** a post is a `Note`; one written on someone else's wall carries the wall (`sm:wall`, FEP-400e) as its diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index 664ff0c..7b7e421 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -682,7 +682,9 @@ it, raw where it doesn't. - FEP-8b32 proof verification; - `hs2019` with SHA-512. - **Discovery:** - - a relay client for both relay styles; + - a relay client for both relay styles: **reading done 2026-10-05** (`Federation:Relays`; forwarded posts read again + from their origin, announces unwrapped; checked live against Activity-Relay and aode-relay); sending public posts to + relays waits for the owner; - instance actor discovery (FEP-d556, FEP-2677); - `implements` (FEP-844e). - **Mastodon API:** ~~streaming WebSocket~~ (done 2026-10-05: `/api/v1/streaming` as a WebSocket and as server-sent diff --git a/tools/pasture/Caddyfile b/tools/pasture/Caddyfile index 16eac0b..b0e746f 100644 --- a/tools/pasture/Caddyfile +++ b/tools/pasture/Caddyfile @@ -108,6 +108,16 @@ nodebb.test { reverse_proxy pasture-nodebb:4567 } +aoderelay.test { + tls internal + reverse_proxy pasture-aoderelay:8080 +} + +relay.test { + tls internal + reverse_proxy pasture-relay-server:8080 +} + smithereen.test { tls internal handle /i/* { diff --git a/tools/pasture/appsettings.Pasture.json b/tools/pasture/appsettings.Pasture.json index 51defee..2b553e8 100644 --- a/tools/pasture/appsettings.Pasture.json +++ b/tools/pasture/appsettings.Pasture.json @@ -22,7 +22,8 @@ "Federation": { "AllowPrivateNetworks": true, "AllowPlainHttp": true, - "AcceptAnyCertificate": true + "AcceptAnyCertificate": true, + "Relays": [ "https://relay.test/actor", "https://aoderelay.test/actor" ] }, "Media": { "Root": "/tmp/privapub-media" }, "RateLimits": { "AccountsPerMinute": 1000 }, diff --git a/tools/pasture/peers/aoderelay.sh b/tools/pasture/peers/aoderelay.sh new file mode 100644 index 0000000..241383b --- /dev/null +++ b/tools/pasture/peers/aoderelay.sh @@ -0,0 +1,21 @@ +# aode-relay 0.3.129 (asonix): a relay that takes both kinds of subscription (Public, as Mastodon follows, or its actor, +# as LitePub does) and passes posts on as its own Announce, as aoderelay.test. Unrestricted, so anyone may subscribe; +# its state is in its own sled database. Its TLS client reads the system's roots, over which the pasture's bundle goes. +AODERELAY_IMAGE=${AODERELAY_IMAGE:-docker.io/asonix/relay:0.3.129} + +aoderelay_up() { + podman volume exists pasture-aoderelay || podman volume create --label pasture=1 pasture-aoderelay >/dev/null + podman run -d --replace --name pasture-aoderelay --label pasture=1 --network $net -v pasture-aoderelay:/var/lib/aode-relay:U \ + -v "$ca/bundle.pem:/etc/ssl/certs/ca-certificates.crt:z,ro" \ + -e HOSTNAME=aoderelay.test -e ADDR=0.0.0.0 -e PORT=8080 -e HTTPS=true -e RESTRICTED_MODE=false -e VALIDATE_SIGNATURES=true \ + -e SLED_PATH=/var/lib/aode-relay/sled -e API_TOKEN=pasture-aoderelay-token -e PRETTY_LOG=false \ + $AODERELAY_IMAGE >/dev/null + for _ in $(seq 1 60); do + site aoderelay.test -s -o /dev/null -w '%{http_code}' https://aoderelay.test:6443/actor 2>/dev/null | grep -q 200 && break + sleep 1 + done + echo "aoderelay: https://aoderelay.test:6443" +} + +# aoderelay_connected: the hosts subscribed to it, from its admin API +aoderelay_connected() { site aoderelay.test -s https://aoderelay.test:6443/api/v1/admin/connected -H 'X-Api-Token: pasture-aoderelay-token' | j "print('\n'.join(d.get('connected_actors', d) if isinstance(d, dict) else d))"; } diff --git a/tools/pasture/peers/relay.sh b/tools/pasture/peers/relay.sh new file mode 100644 index 0000000..c721785 --- /dev/null +++ b/tools/pasture/peers/relay.sh @@ -0,0 +1,37 @@ +# Activity-Relay 2.0.9 (yukimochi): a Mastodon-style relay as relay.test. A server subscribes by following Public from an +# actor with a shared inbox; the relay then passes on, signed with its own key, every public activity another subscriber +# sends it (a LitePub-style follower, whose actor ends in /relay, gets an Announce instead). Its state is in the shared +# Redis (database 13); its actor key is made once. Go trusts the bundle the pasture mounts over the system one. +RELAY_IMAGE=${RELAY_IMAGE:-docker.io/yukimochi/activity-relay:v2.0.9} +. "$here/peers/shared.sh" + +relay_up() { + shared_redis_up + local st="$here/.state/relay" + mkdir -p "$st" + [ -s "$st/actor.pem" ] || openssl genrsa -traditional -out "$st/actor.pem" 2048 2>/dev/null + chmod 644 "$st/actor.pem" + cat > "$st/config.yml" <<-EOF + ACTOR_PEM: /var/lib/relay/actor.pem + REDIS_URL: redis://redis:6379/13 + RELAY_BIND: 0.0.0.0:8080 + RELAY_DOMAIN: relay.test + RELAY_SERVICENAME: Pasture Relay + JOB_CONCURRENCY: 10 + RELAY_ICON: https://relay.test/icon.png + RELAY_IMAGE: https://relay.test/image.png + EOF + for role in server worker; do + podman run -d --replace --name pasture-relay-$role --label pasture=1 --network $net \ + -v "$st:/var/lib/relay:z,ro" -v "$ca/bundle.pem:/etc/ssl/certs/ca-certificates.crt:z,ro" \ + $RELAY_IMAGE relay --config /var/lib/relay/config.yml $role >/dev/null + done + for _ in $(seq 1 60); do + site relay.test -s -o /dev/null -w '%{http_code}' https://relay.test:6443/actor 2>/dev/null | grep -q 200 && break + sleep 1 + done + echo "relay: https://relay.test:6443" +} + +# relay_subscribers: the hosts subscribed to the relay, from its Redis +relay_subscribers() { podman exec pasture-redis redis-cli -n 13 --raw keys 'relay:subscription:*' | sed 's/relay:subscription://'; } diff --git a/tools/pasture/scenarios/relay.sh b/tools/pasture/scenarios/relay.sh new file mode 100644 index 0000000..db77aed --- /dev/null +++ b/tools/pasture/scenarios/relay.sh @@ -0,0 +1,56 @@ +# Relays as PrivaPub reads them (Federation:Relays in appsettings.Pasture.json): Activity-Relay, which forwards what its +# subscribers send, and aode-relay, which announces it. For each, the instance actor subscribes, Mastodon subscribes too, +# and a public post of a Mastodon account nobody here follows reaches PrivaPub's federated timeline through the relay; +# nothing of a persona's goes to a relay. Mastodon leaves each relay after, so the town sees no relayed posts. Needs the +# relay, aoderelay and mastodon peers. +M=https://mastodon.test:6443 +mcurl() { curl -sk --resolve mastodon.test:6443:127.0.0.1 "$@"; } +. "$here/peers/mastodon.sh" +. "$here/peers/relay.sh" +. "$here/peers/aoderelay.sh" +p_relay_state() { podman exec pasture-mongo mongosh --quiet PrivaPub --eval 'var s=db.RelaySubscription.findOne({ActorURI:"https://'$1'/actor"}); print(s ? s.State : "none")'; } +m_relay() { podman exec pasture-mastodon bin/rails runner 'r = Relay.find_or_create_by!(inbox_url: "https://'$1'/inbox"); r.enable! unless r.accepted? || r.pending?' >/dev/null 2>&1; } +m_unrelay() { podman exec pasture-mastodon bin/rails runner 'Relay.where(inbox_url: "https://'$1'/inbox").each { |r| r.disable!; r.destroy }' >/dev/null 2>&1; } +p_public_has() { [ "$(curl -s -H "$PH" "$P/api/v1/timelines/public?remote=true&limit=40" | j "print(any(s['uri'] == '$1' for s in d))")" = "True" ]; } +# relayed_post : a public post of a fresh Mastodon account nobody here follows, by its uri +relayed_post() { + local who="$1$run" + mastodon_user "$who" + mcurl -X POST -H "Authorization: Bearer $(mastodon_token "$who")" "$M/api/v1/statuses" -d "status=a post only the $1 brings $run&visibility=public" | j "print(d['uri'])" +} + +echo "relay" +PT=$(privapub_token alice_relay) +PH="Authorization: Bearer $PT" +[ -n "$PT" ] && ok "PrivaPub token for alice_relay" || { ko "PrivaPub token for alice_relay"; return 1; } +run=$(date +%s) + +echo " Activity-Relay" +# PrivaPub subscribes a minute after it starts (and again six hours later while unanswered) +until_true 60 'relay_subscribers | grep -qx privapub.test' && ok "PrivaPub's instance actor subscribes to Activity-Relay" || ko "PrivaPub never subscribed ($(relay_subscribers | tr '\n' ' '))" +# (1 = Accepted) +until_true 30 '[ "$(p_relay_state relay.test)" = "1" ]' && ok "and takes its Accept" || ko "PrivaPub's subscription is $(p_relay_state relay.test)" +m_relay relay.test +until_true 45 'relay_subscribers | grep -qx mastodon.test' && ok "Mastodon subscribes too" || ko "Mastodon never subscribed" +s_uri=$(relayed_post relay) +until_true 60 'p_public_has "$s_uri"' && ok "a public post of an account nobody here follows reaches the federated timeline, forwarded" || ko "the forwarded post never arrived" +[ "$(curl -s -H "$PH" "$P/api/v1/timelines/home?limit=40" | j "print(any(s['uri'] == '$s_uri' for s in d))")" = "False" ] \ + && ok "and no one's home" || ko "the relayed post landed in alice's home" +p_post=$(curl -s -X POST -H "$PH" "$P/api/v1/statuses" -d "status=a PrivaPub post the relay never sees $run&visibility=public" | j "print(d['uri'])") +sleep 5 +[ "$(podman exec pasture-mongo mongosh --quiet PrivaPub --eval 'print(db.Job.countDocuments({Host:"relay.test", Payload:/Create/}))')" = "0" ] \ + && ok "nothing of a persona's goes to the relay" || ko "PrivaPub sent the relay a post" + +m_unrelay relay.test +until_true 45 '! relay_subscribers | grep -qx mastodon.test' && ok "Mastodon leaves Activity-Relay" || ko "Mastodon is still subscribed to Activity-Relay" + +echo " aode-relay" +until_true 30 '[ "$(p_relay_state aoderelay.test)" = "1" ]' && ok "PrivaPub's instance actor subscribes to aode-relay, which accepts" || ko "PrivaPub's aode-relay subscription is $(p_relay_state aoderelay.test)" +m_relay aoderelay.test +until_true 45 'aoderelay_connected | grep -q mastodon.test' && ok "Mastodon subscribes too" || ko "Mastodon never subscribed to aode-relay ($(aoderelay_connected | tr '\n' ' '))" +a_uri=$(relayed_post aoderelay) +until_true 60 'p_public_has "$a_uri"' && ok "a post aode-relay announces reaches the federated timeline as its author's" || ko "the announced post never arrived" +[ "$(curl -s -H "$PH" "$P/api/v1/timelines/public?remote=true&limit=40" | j "print(any((s.get('reblog') or {}).get('uri') == '$a_uri' for s in d))")" = "False" ] \ + && ok "never as the relay's boost" || ko "the relay's announce shows as a boost" +m_unrelay aoderelay.test +until_true 45 '! aoderelay_connected | grep -q mastodon.test' && ok "Mastodon leaves aode-relay" || ko "Mastodon is still subscribed to aode-relay"