diff --git a/CLAUDE.md b/CLAUDE.md index c53abee..4238e24 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -87,7 +87,8 @@ PrivaPub/ ASP.NET Core Web API, net10.0 RequestSignature (whichever a request carries) Inbox/ InboxReceiver (verify, queue, 202) → InboxProcessor (job) → Handlers/{Follow,Accept,Reject, Undo,Create,Update,Delete,Like,Announce}; RemotePosts (build, fetch parents, FetchAncestors); - RemoteReplies (FetchReplies: a thread's `context`, else `replies` two levels down) + RemoteReplies (FetchReplies: a thread's `context`, else `replies` two levels down); + Forwarded (what a thread's server passes on, believed as far as the origin vouches) Outbox/ OutboxPublisher (who a post goes to), DeliveryService (queues jobs) + DeliveryJobHandler Rendering/ ActivityPubRenderer (Mastodon @context, actors, notes, collections) Domain/ @@ -168,7 +169,12 @@ group www-data and reaches the private mongod; `sudo -u www-data` works too. 4. **Inbound inboxes verify everything before acting:** - the signature covers `(request-target)`, `host`, `digest` and `date` or `(created)`; - the Digest matches the body and the date is at most an hour old and fifteen minutes ahead; - - the signature verifies against the key owner's key, and the activity's `actor` is the key owner; + - the signature verifies against the key owner's key, and the activity's `actor` is the key owner, or else it was + forwarded (`Forwarded`: a thread's server passing on what happens in it, as Mastodon and Friendica do). A forwarded + activity proves nothing about its author: answered 202, a Create or Update is taken as its object reads at the + actor's origin now, a Delete of a public or unlisted copy once that origin answers 404 or 410 + (`RemoteActorService.IsGone`), anything else let go. Forwarded copies have their own dedupe key, so a copy that + failed never hides the author's own delivery; - the activity's `id`, and any object it creates, updates or deletes, is on the actor's origin; a cross-origin object is refetched from its own origin. 5. **Status codes:** diff --git a/FEDERATION.md b/FEDERATION.md index 56d987f..1c359ce 100644 --- a/FEDERATION.md +++ b/FEDERATION.md @@ -224,6 +224,11 @@ Posts with a location (shown to nearby users of this server) never leave the ser - **Origins.** An actor document is accepted only from the address it names as its `id`. A key only if its actor lists it with `owner` set to the actor and on the actor's origin. An activity's `id`, and any object it creates, updates or deletes, must be on its actor's origin. An embedded object from another origin is fetched from that origin. +- **Forwarded activities** (ActivityPub's inbox forwarding: a thread's server passing on the replies in it, signed with + its own key) are answered 202 and believed only as far as their origin vouches for them: a created or edited post + is read again from its author's server, a deletion of a public or unlisted post is applied once that server answers + 404 or 410, and anything else is dropped. LD signatures are not verified. A reply forwarded this way is kept when + someone here follows the author of the post it answers, as Mastodon does. - **Follow requests** still unanswered are sent again after 15 minutes, an hour, 6 hours, a day, two and four days, the same activity each time: a server can take a Follow and lose its answer (Lemmy 1.0 sends nothing it queued for a server before it started sending there), and one that holds the follow already answers the copy. diff --git a/PrivaPub.Tests/Federation/ForwardedTests.cs b/PrivaPub.Tests/Federation/ForwardedTests.cs new file mode 100644 index 0000000..a5132c2 --- /dev/null +++ b/PrivaPub.Tests/Federation/ForwardedTests.cs @@ -0,0 +1,167 @@ +using MongoDB.Entities; + +using PrivaPub.Models.Post; +using PrivaPub.Tests.Support; + +using System.Text.Json.Nodes; + +using static PrivaPub.Tests.Support.FederatedSeeds; + +namespace PrivaPub.Tests.Federation +{ + // Inbox forwarding: a thread's server passes on the activities in it, signed with its own key (Mastodon the replies to + // its accounts' posts and their deletions, Friendica everything in its threads). They used to be refused with 401. + [Trait("Category", "Integration")] + public sealed class ForwardedTests : IAsyncLifetime + { + Harness _harness; + + public async ValueTask InitializeAsync() + { + Assert.SkipUnless(MongoFixture.Enabled, MongoFixture.Skip); + _harness = await Harness.Start(); + } + + public async ValueTask DisposeAsync() + { + if (_harness != default) + await _harness.DisposeAsync(); + } + + PrivaPub.Models.Statistics.InteractionEvent Processed(string activity) => _harness.Ledger.Of("in").Last(e => e.Activity == activity); + + PrivaPub.Models.Statistics.InteractionEvent Received(string activity) => _harness.Ledger.Of("recv").Last(e => e.Activity == activity); + + Task Stored(string objectUri) => + DB.Default.Find().Match(p => p.ObjectURI == objectUri).ExecuteFirstAsync(TestContext.Current.CancellationToken); + + // alice follows the thread's author (on its own server), whose post she holds; bob, elsewhere, replies to it + async Task<(RemoteActor Owner, RemoteActor Bob, JsonObject Reply)> Thread() + { + var (_, alice) = await _harness.Persona("alice"); + var owner = new RemoteActor(_harness.Peer, "owner", _harness.Peer.B); + var bob = new RemoteActor(_harness.Peer, "bob"); + await Follows(alice.Id, owner); + var post = PublicNote(owner, "

the thread

"); + await _harness.Deliver(owner, "/human-centipede", Create(owner, post)); + Assert.NotNull(await Stored(IdOf(post))); + + var reply = PublicNote(bob, "

bob's reply as he wrote it

", owner.Id); + reply["inReplyTo"] = IdOf(post); + _harness.Peer.Serve(new Uri(IdOf(reply)).AbsolutePath, reply.ToJsonString()); + return (owner, bob, reply); + } + + [Fact] + public async Task A_reply_the_threads_server_forwards_is_kept_as_its_author_wrote_it() + { + var (owner, bob, reply) = await Thread(); + var claimed = (JsonObject)reply.DeepClone(); + claimed["content"] = "

what the forwarder says bob wrote

"; + + var result = await _harness.Deliver(owner, "/human-centipede", Create(bob, claimed)); + + Assert.Equal((202, "forwarded"), (result.StatusCode, result.Reason)); + var stored = await Stored(IdOf(reply)); + Assert.NotNull(stored); + Assert.Equal(bob.Id, stored.ActorURI); + Assert.Contains("as he wrote it", stored.ContentHtml); + Assert.Equal(("accepted", "stored"), (Processed("Create").Outcome, Processed("Create").Reason)); + Assert.True((await DB.Default.Find().Match(r => r.ObjectURI == IdOf(reply)).ExecuteFirstAsync(TestContext.Current.CancellationToken)).Refetched); + } + + [Fact] + public async Task A_forwarded_edit_applies_the_post_as_its_origin_has_it_now() + { + var (owner, bob, reply) = await Thread(); + await _harness.Deliver(owner, "/human-centipede", Create(bob, reply)); + var edited = (JsonObject)reply.DeepClone(); + edited["content"] = "

bob's reply, edited

"; + edited["updated"] = DateTime.UtcNow.AddMinutes(1).ToString("O"); + _harness.Peer.Serve(new Uri(IdOf(reply)).AbsolutePath, edited.ToJsonString()); + var claimed = (JsonObject)edited.DeepClone(); + claimed["content"] = "

an edit bob never made

"; + + await _harness.Deliver(owner, "/human-centipede", Activity(bob, "Update", claimed)); + + Assert.Contains("bob's reply, edited", (await Stored(IdOf(reply))).ContentHtml); + } + + [Fact] + public async Task A_forwarded_deletion_waits_for_the_origin_to_say_the_post_is_gone() + { + var (owner, bob, reply) = await Thread(); + await _harness.Deliver(owner, "/human-centipede", Create(bob, reply)); + + await _harness.Deliver(owner, "/human-centipede", Activity(bob, "Delete", JsonValue.Create(IdOf(reply))!)); + Assert.Equal(("dropped", "forwarded-not-gone"), (Processed("Delete").Outcome, Processed("Delete").Reason)); + Assert.NotNull(await Stored(IdOf(reply))); + + _harness.Peer.Answer(new Uri(IdOf(reply)).AbsolutePath, 410); + await _harness.Deliver(owner, "/human-centipede", Activity(bob, "Delete", JsonValue.Create(IdOf(reply))!)); + Assert.Null(await Stored(IdOf(reply))); + } + + // the origin answers 404 for a followers-only post to our instance actor too: only its author's server says it is gone + [Fact] + public async Task A_forwarded_deletion_of_a_followers_only_post_is_left_to_its_author() + { + var (owner, bob, reply) = await Thread(); + await _harness.Deliver(owner, "/human-centipede", Create(bob, reply)); + await DB.Default.Update().Match(p => p.ObjectURI == IdOf(reply)).Modify(p => p.Visibility, PostVisibility.FollowersOnly).ExecuteAsync(TestContext.Current.CancellationToken); + _harness.Peer.Answer(new Uri(IdOf(reply)).AbsolutePath, 404); + + await _harness.Deliver(owner, "/human-centipede", Activity(bob, "Delete", JsonValue.Create(IdOf(reply))!)); + + Assert.Equal(("dropped", "forwarded-private"), (Processed("Delete").Outcome, Processed("Delete").Reason)); + Assert.NotNull(await Stored(IdOf(reply))); + } + + // a forwarded copy is kept apart from the author's own delivery, which carries what the copy could not + [Fact] + public async Task The_authors_own_delivery_is_still_taken_after_a_forwarded_copy_that_failed() + { + var (owner, bob, reply) = await Thread(); + _harness.Peer.Answer(new Uri(IdOf(reply)).AbsolutePath, 404); + var create = Create(bob, reply); + + await _harness.Deliver(owner, "/human-centipede", create); + Assert.Null(await Stored(IdOf(reply))); + + var direct = await _harness.Deliver(bob, "/human-centipede", create); + Assert.Equal("queued", direct.Reason); + Assert.NotNull(await Stored(IdOf(reply))); + } + + // what cannot be checked against its origin (a vote, a follow, someone else's post) is let go, answered 202: a 401 + // tells the forwarder its signature failed + [Fact] + public async Task What_a_forwarder_cannot_vouch_for_is_let_go_without_an_error() + { + var (owner, bob, reply) = await Thread(); + var like = new JsonObject { ["id"] = NewId(bob, "likes"), ["type"] = "Like", ["actor"] = bob.Id, ["object"] = IdOf(reply) }; + + var result = await _harness.Deliver(owner, "/human-centipede", like); + Assert.Equal((202, "forwarded-ignored"), (result.StatusCode, result.Reason)); + Assert.Equal("dropped", Received("Like").Outcome); + + var onTheForwardersServer = PublicNote(owner, "

a post of the forwarder's

"); + onTheForwardersServer["attributedTo"] = bob.Id; + result = await _harness.Deliver(owner, "/human-centipede", Create(bob, onTheForwardersServer)); + Assert.Equal((202, "forwarded-ignored"), (result.StatusCode, result.Reason)); + Assert.Null(await Stored(IdOf(onTheForwardersServer))); + + // our own, coming back from a thread on another server + var (_, carol) = await _harness.Persona("carol"); + var ours = new JsonObject + { + ["id"] = carol.Uri + "/grunts/create-" + Guid.NewGuid().ToString("N"), + ["type"] = "Create", + ["actor"] = carol.Uri, + ["object"] = new JsonObject { ["id"] = carol.Uri + "/scribbles/" + Guid.NewGuid().ToString("N"), ["type"] = "Note", ["attributedTo"] = carol.Uri } + }; + result = await _harness.Deliver(owner, "/human-centipede", ours); + Assert.Equal((202, "forwarded-ignored"), (result.StatusCode, result.Reason)); + } + } +} diff --git a/PrivaPub.Tests/Support/Harness.cs b/PrivaPub.Tests/Support/Harness.cs index 98af285..9ffd7bf 100644 --- a/PrivaPub.Tests/Support/Harness.cs +++ b/PrivaPub.Tests/Support/Harness.cs @@ -135,7 +135,8 @@ namespace PrivaPub.Tests.Support public async Task Deliver(RemoteActor sender, string path, JsonNode activity) { var result = await Receiver.Receive(sender.Post(Host, path, activity), default, CancellationToken.None); - var dedupe = "inbox|" + (activity is JsonObject ? activity["id"]?.GetValue() : default); + var id = activity is JsonObject ? activity["id"]?.GetValue() : default; + var dedupe = (result.Reason == "forwarded" ? "inbox|forwarded|" : "inbox|") + id; var job = await DB.Default.Find().Match(j => j.DedupeKey == dedupe).ExecuteFirstAsync(); if (job != default) Assert.Equal(JobResult.Done, (await Processor.Handle(job, CancellationToken.None)).Result); diff --git a/PrivaPub/Federation/Actors/RemoteActorService.cs b/PrivaPub/Federation/Actors/RemoteActorService.cs index 065f835..547f60c 100644 --- a/PrivaPub/Federation/Actors/RemoteActorService.cs +++ b/PrivaPub/Federation/Actors/RemoteActorService.cs @@ -20,6 +20,7 @@ namespace PrivaPub.Federation.Actors Task GetActor(string actorUri, bool refresh, CancellationToken token); Task GetActorByKeyId(string keyId, bool refresh, CancellationToken token); bool KeyTemporarilyUnavailable(string keyId) => false; + Task IsGone(string uri, CancellationToken token) => Task.FromResult(false); Task ResolveHandle(string handle, CancellationToken token); } @@ -138,6 +139,18 @@ namespace PrivaPub.Federation.Actors public bool KeyTemporarilyUnavailable(string keyId) => Origin.Of(keyId) != default && _http.FailedTemporarily(StripFragment(keyId)); + // whether an object's origin says it is gone: 404 or 410 to our instance actor, or a Tombstone in its place + public async Task IsGone(string uri, CancellationToken token) + { + if (Origin.Of(uri) == default) + return false; + using var scope = HttpScope.Default("object"); + var signer = await _localActors.GetInstanceActor(token); + var (status, fetched) = await _http.GetJsonStatus(uri, Accept, request => HttpSignatures.Sign(request, signer, body: null), token); + using (fetched) + return status is StatusCodes.Status404NotFound or StatusCodes.Status410Gone || fetched != default && Text(fetched.Root, "type") == "Tombstone"; + } + public async Task ResolveHandle(string handle, CancellationToken token) { var parts = handle?.TrimStart('@').Split('@'); diff --git a/PrivaPub/Federation/Inbox/Forwarded.cs b/PrivaPub/Federation/Inbox/Forwarded.cs new file mode 100644 index 0000000..448b506 --- /dev/null +++ b/PrivaPub/Federation/Inbox/Forwarded.cs @@ -0,0 +1,71 @@ +using MongoDB.Entities; + +using PrivaPub.Federation.Actors; +using PrivaPub.Federation.Objects; +using PrivaPub.Models.Post; + +using System.Text.Json.Nodes; + +using static PrivaPub.Federation.Objects.ActivityJson; + +using PostEntity = PrivaPub.Models.Post.Post; + +namespace PrivaPub.Federation.Inbox +{ + // Inbox forwarding (ActivityPub 7.1.2): a server passes on an activity about a thread it holds to the thread's followers, + // signed with its own key. Mastodon forwards the replies to its accounts' posts, and their deletions; Friendica every + // activity in its threads. The signature proves who passed it on, never who wrote it, so nothing in it is believed: a + // Create or an Update is taken as its object reads at the actor's origin now, a Delete once that origin says the object + // is gone, and anything else is let go (a vote or a follow cannot be checked against its origin). An LD signature + // (RsaSignature2017) would prove the author, but needs JSON-LD; integrity proofs (FEP-8b32) will. + public static class Forwarded + { + // what may be taken from a forwarder: a post of the activity's actor, created, edited or deleted + public static bool Takeable(string type, JsonNode activity, string actorUri) + { + var objectUri = Id(activity["object"]); + return type is "Create" or "Update" or "Delete" && objectUri != default && objectUri != actorUri && Origin.Same(objectUri, actorUri); + } + + // the activity as its origin vouches for it, or why it is dropped + public static async Task<(JsonNode Activity, string Drop)> Confirm(JsonNode activity, string type, string actorUri, + IRemoteActorService remoteActors, CancellationToken token) + { + var objectUri = Id(activity["object"]); + var trusted = new JsonObject + { + ["id"] = Id(activity), + ["type"] = type, + ["actor"] = actorUri + }; + if (type == "Delete") + { + // only a copy anyone may read: the origin answers 404 for a followers-only post to our instance actor too, + // and the author's own server tells its recipients of a deletion + var held = await DB.Default.Find().Match(p => p.ObjectURI == objectUri && p.ActorURI == actorUri).ExecuteFirstAsync(token); + if (held != default && held.Visibility is not (PostVisibility.Public or PostVisibility.Unlisted)) + return (default, "forwarded-private"); + if (!await remoteActors.IsGone(objectUri, token)) + return (default, "forwarded-not-gone"); + trusted["object"] = objectUri; + return (trusted, default); + } + + if (type == "Create") + { + // the Create handler reads it again from its origin, and records that it did + trusted["object"] = objectUri; + return (trusted, default); + } + + using var fetched = await remoteActors.FetchObject(objectUri, token); + if (fetched == default) + return (default, "fetch-failed"); + var node = JsonNode.Parse(fetched.Root.GetRawText()); + if (!Origin.Same(Id(node), actorUri)) + return (default, "cross-origin"); + trusted["object"] = node; + return (trusted, default); + } + } +} diff --git a/PrivaPub/Federation/Inbox/Handlers/CreateHandler.cs b/PrivaPub/Federation/Inbox/Handlers/CreateHandler.cs index 84338a1..0ca7687 100644 --- a/PrivaPub/Federation/Inbox/Handlers/CreateHandler.cs +++ b/PrivaPub/Federation/Inbox/Handlers/CreateHandler.cs @@ -145,7 +145,10 @@ namespace PrivaPub.Federation.Inbox.Handlers var followed = visibility != PostVisibility.Direct && await _dbEntities.Followings .Match(f => f.TargetActorURI == author.ActorURI && f.State == FollowState.Accepted) .ExecuteAnyAsync(token); - if (visibility == PostVisibility.Direct ? persons.Count == 0 : group == default && persons.Count == 0 && !repliesToLocal && !followed) + // a reply in the thread of someone followed here, as Mastodon keeps them: what the thread's server forwards + var repliesToFollowed = !followed && parent is { IsFederatedCopy: true } && visibility is PostVisibility.Public or PostVisibility.Unlisted + && await _dbEntities.Followings.Match(f => f.TargetActorURI == parent.ActorURI && f.State == FollowState.Accepted).ExecuteAnyAsync(token); + if (visibility == PostVisibility.Direct ? persons.Count == 0 : group == default && persons.Count == 0 && !repliesToLocal && !followed && !repliesToFollowed) { Arrival.Drop("not-addressed"); return; diff --git a/PrivaPub/Federation/Inbox/InboxProcessor.cs b/PrivaPub/Federation/Inbox/InboxProcessor.cs index 21185b2..8c435e4 100644 --- a/PrivaPub/Federation/Inbox/InboxProcessor.cs +++ b/PrivaPub/Federation/Inbox/InboxProcessor.cs @@ -61,6 +61,16 @@ namespace PrivaPub.Federation.Inbox Record(job, payload, activity, type, started, new ArrivalVerdict(), Interactions.Deferred, "actor-unavailable"); return JobOutcome.Retry("the actor could not be loaded"); } + if (payload.ForwardedBy != default) + { + var (confirmed, drop) = await Forwarded.Confirm(activity, type, actor.ActorURI, _remoteActors, token); + if (drop != default) + { + Record(job, payload, activity, type, started, new ArrivalVerdict(), Interactions.Dropped, drop); + return JobOutcome.Done; + } + activity = confirmed; + } var arrival = new Arrival(Id(activity), type, actor.ActorURI, payload.Inbox, payload.KeyId, payload.Algorithm, payload.SignedHeaders ?? Array.Empty(), payload.ReceivedAt ?? job.CreatedAt, activity["@context"]?.ToJsonString()); diff --git a/PrivaPub/Federation/Inbox/InboxReceiver.cs b/PrivaPub/Federation/Inbox/InboxReceiver.cs index 6c94adf..a26a35a 100644 --- a/PrivaPub/Federation/Inbox/InboxReceiver.cs +++ b/PrivaPub/Federation/Inbox/InboxReceiver.cs @@ -18,8 +18,9 @@ namespace PrivaPub.Federation.Inbox { public sealed record InboxResult(int StatusCode, string Error = default, int? RetryAfterSeconds = default, string Reason = default); + // ForwardedBy: the server that passed the activity on, signing with its own key (Forwarded) public sealed record InboxPayload(string ActorURI, string Activity, string Inbox = default, string KeyId = default, string Algorithm = default, - string[] SignedHeaders = default, DateTime? ReceivedAt = default); + string[] SignedHeaders = default, DateTime? ReceivedAt = default, string ForwardedBy = default); public interface IInboxReceiver { @@ -100,7 +101,8 @@ namespace PrivaPub.Federation.Inbox Activity = receipt.Type, Object = receipt.ObjectType, Status = result.StatusCode, - Outcome = result.StatusCode == StatusCodes.Status202Accepted ? Interactions.Queued : Interactions.Refused, + Outcome = result.StatusCode != StatusCodes.Status202Accepted ? Interactions.Refused + : result.Reason == "forwarded-ignored" ? Interactions.Dropped : Interactions.Queued, Reason = result.Reason, LatencyMs = latencyMs, Bytes = receipt.Bytes, @@ -171,9 +173,13 @@ namespace PrivaPub.Federation.Inbox return new(StatusCodes.Status401Unauthorized, "the signature does not verify", Reason: "signature-invalid"); } - if (!string.Equals(keyOwner.ActorURI, actorUri, StringComparison.Ordinal)) - return new(StatusCodes.Status401Unauthorized, "the activity's actor is not the key's owner", Reason: "actor-not-key-owner"); - receipt.VerifiedActor = actorUri; + // signed by someone else: passed on by a server that holds the thread, its signature good (a 401 would tell it + // otherwise, and make a double-knocking sender try its other scheme) + receipt.VerifiedActor = keyOwner.ActorURI; + var forwardedBy = string.Equals(keyOwner.ActorURI, actorUri, StringComparison.Ordinal) ? default : keyOwner.ActorURI; + // (ours come back too, Friendica forwarding our comments in its threads: we know them already) + if (forwardedBy != default && (!Forwarded.Takeable(type, activity, actorUri) || Origin.Same(actorUri, _localActors.BaseAddress))) + return new(StatusCodes.Status202Accepted, Reason: "forwarded-ignored"); var shapeProblem = await ShapeProblem(type, activity, actorUri, token); if (shapeProblem != default) @@ -181,11 +187,14 @@ namespace PrivaPub.Federation.Inbox var activityId = Id(activity); var payload = new InboxPayload(actorUri, activity.ToJsonString(), recipient == default ? "shared" : "personal", signature.KeyId, - signature.Algorithm, signature.Covered, DateTime.UtcNow); + signature.Algorithm, signature.Covered, DateTime.UtcNow, forwardedBy); + // a forwarded copy never stands in for the author's own delivery, which may carry what the copy could not + // (a followers-only reply our instance actor cannot read): each is kept once, apart + var dedupe = activityId == default ? default : (forwardedBy == default ? "inbox|" : "inbox|forwarded|") + activityId; var queued = await _queue.Enqueue(JobKind.ProcessInbox, JsonSerializer.Serialize(payload), - new Uri(actorUri).Host.ToLowerInvariant(), activityId == default ? default : "inbox|" + activityId, token); + new Uri(actorUri).Host.ToLowerInvariant(), dedupe, token); _logger.LogInformation("Inbox {Recipient}: {Type} from {Actor} queued", recipient?.Handle ?? "shared", type, actorUri); - return new(StatusCodes.Status202Accepted, Reason: queued ? "queued" : "duplicate"); + return new(StatusCodes.Status202Accepted, Reason: !queued ? "duplicate" : forwardedBy != default ? "forwarded" : "queued"); } static string HostOf(string uri) => Uri.TryCreate(uri, UriKind.Absolute, out var parsed) ? parsed.Host.ToLowerInvariant() : default; diff --git a/PrivaPub/Infrastructure/Http/FederationHttp.cs b/PrivaPub/Infrastructure/Http/FederationHttp.cs index b6a6e9f..1aa03e8 100644 --- a/PrivaPub/Infrastructure/Http/FederationHttp.cs +++ b/PrivaPub/Infrastructure/Http/FederationHttp.cs @@ -23,6 +23,7 @@ namespace PrivaPub.Infrastructure.Http { bool IsAllowed(Uri target); Task GetJson(string url, string accept, Action sign, CancellationToken token); + Task<(int Status, FetchedJson Json)> GetJsonStatus(string url, string accept, Action sign, CancellationToken token); bool FailedTemporarily(string url); Task<(Uri FinalUri, string Html)> GetPage(string url, CancellationToken token); Task OpenMedia(string url, System.Net.Http.Headers.RangeHeaderValue range, CancellationToken token); @@ -295,6 +296,24 @@ namespace PrivaPub.Infrastructure.Http } } + // the document and the status its origin answered with (a refusal remembered from the last five minutes keeps its + // status; 0 when it never answered) + public async Task<(int Status, FetchedJson Json)> GetJsonStatus(string url, string accept, Action sign, CancellationToken token) + { + if (Uri.TryCreate(url, UriKind.Absolute, out var target) && _cache.TryGetValue(NegativeKey(target), out Refusal remembered)) + return (remembered.Status, default); + var exchange = new Exchange(url, HttpScope.Purpose ?? "object"); + try + { + var json = await GetJson(url, accept, sign, exchange, token); + return (exchange.Status ?? 0, json); + } + finally + { + Record(exchange); + } + } + async Task GetJson(string url, string accept, Action sign, Exchange exchange, CancellationToken token) { if (!Uri.TryCreate(url, UriKind.Absolute, out var target) || !IsAllowed(target)) @@ -367,7 +386,7 @@ namespace PrivaPub.Infrastructure.Http } public bool FailedTemporarily(string url) => - Uri.TryCreate(url, UriKind.Absolute, out var target) && _cache.TryGetValue(NegativeKey(target), out bool transient) && transient; + Uri.TryCreate(url, UriKind.Absolute, out var target) && _cache.TryGetValue(NegativeKey(target), out Refusal refusal) && refusal.Transient; public async Task<(byte[] Bytes, string ContentType)> GetMedia(string url, long maxBytes, CancellationToken token) { @@ -610,13 +629,15 @@ namespace PrivaPub.Infrastructure.Http static string NegativeKey(Uri target) => "federation-http:refused:" + target.AbsoluteUri; + sealed record Refusal(bool Transient, int Status); + FetchedJson Refuse(string negativeKey, string url, string reason, Exchange exchange, string code, bool transient = false) { if (transient) exchange.Failed(code); else exchange.Refused(code); - _cache.Set(negativeKey, transient, NegativeCacheLifetime); + _cache.Set(negativeKey, new Refusal(transient, exchange.Status ?? 0), NegativeCacheLifetime); _logger.LogInformation("GET {Url} refused: {Reason}", url, reason); return default; } diff --git a/docs/INTEROP.md b/docs/INTEROP.md index 062339f..9aba335 100644 --- a/docs/INTEROP.md +++ b/docs/INTEROP.md @@ -151,12 +151,14 @@ Priorities, used throughout: | Publish `context` and a paged `replies` | P2 | — | | `FeatureRequest`: send `Reject` (or implement FEP-7aa9) | P3 | — | -**Pasture evidence (2026-10-05, Mastodon v4.7.3, `tools/pasture/scenarios/mastodon.sh`):** 56 checks pass, and the +**Pasture evidence (2026-10-05, Mastodon v4.7.3, `tools/pasture/scenarios/mastodon.sh`):** 57 checks pass, and the scenario can be run again on the same pasture (it undoes the lock and the block it leaves). They cover: - discovery, follows and locked follows both ways; - public, CW and followers-only posts (the last answering 404 unsigned); - replies threading both ways; -- a thread completed from Mastodon's FEP-7888 `context`: a reply by an account nobody here follows, never delivered, +- an outsider's reply to an account a persona follows, which Mastodon delivers to that account's followers, kept (it + was dropped as unaddressed until 2026-10-05); +- a thread completed from Mastodon's FEP-7888 `context`: the outsider's answer to their own reply, never delivered, joins the thread once a persona opens it; - likes, boosts and their undos both ways, with counts; - DMs both ways; @@ -594,6 +596,7 @@ What it showed: - **P1:** W2 for NodeBB Articles (`privapub.excerpt`). - **P2:** Groups without `followers`; Announces from an Application; context Move/Remove; paged `context` with ETag; Friendica's thread-Follow. + - ~~**P1:** forwarded activities refused with 401~~ done 2026-10-05 (`Forwarded`). ### PeerTube 8.3.1 (2026-09-28) diff --git a/tools/pasture/scenarios/mastodon.sh b/tools/pasture/scenarios/mastodon.sh index c529fc3..ea90c0a 100644 --- a/tools/pasture/scenarios/mastodon.sh +++ b/tools/pasture/scenarios/mastodon.sh @@ -171,18 +171,22 @@ alice_on_m_o=$(mcurl -H "Authorization: Bearer $OT" "$M/api/v2/search?q=@alice_m unserved "$circle_uri" && ok "the circle post is not served unsigned" || ko "circle post served unsigned" echo " threads" -# a reply from an account nobody here follows is never delivered to PrivaPub; opening the thread has PrivaPub read -# the thread's context collection (FEP-7888) and bring it in +# Mastodon sends a reply to the followers of the account it answers, so an outsider's reply to someone alice follows +# reaches PrivaPub, and is kept (as Mastodon keeps them); the outsider's answer to their own reply goes only to the +# outsider's followers, none here, and opening the thread has PrivaPub read its context collection (FEP-7888) for it t_root=$(mcurl -X POST -H "$MH" "$M/api/v1/statuses" -d "status=a thread for elsewhere $circle_name&visibility=public" | j "print(d['id'])") until_true 30 'curl -s -H "$AH" "$P/api/v1/timelines/home" | grep -q "a thread for elsewhere $circle_name"' || true t_root_on_p=$(curl -s -H "$AH" "$P/api/v1/timelines/home" | j "print(next(s['id'] for s in d if 'a thread for elsewhere $circle_name' in s['content']))") -mcurl -o /dev/null -X POST -H "Authorization: Bearer $OT" "$M/api/v1/statuses" -d "status=said where PrivaPub does not listen $circle_name&in_reply_to_id=$t_root&visibility=public" +t_reply=$(mcurl -X POST -H "Authorization: Bearer $OT" "$M/api/v1/statuses" -d "status=an outsider answers $circle_name&in_reply_to_id=$t_root&visibility=public" | j "print(d['id'])") +until_true 30 '[ "$(podman exec pasture-mongo mongosh --quiet PrivaPub --eval "print(db.Post.countDocuments({Text:/an outsider answers $circle_name/}))")" = "1" ]' \ + && ok "an outsider's reply in a followed account's thread is kept" || ko "the outsider's reply Mastodon delivered was dropped" +mcurl -o /dev/null -X POST -H "Authorization: Bearer $OT" "$M/api/v1/statuses" -d "status=said where PrivaPub does not listen $circle_name&in_reply_to_id=$t_reply&visibility=public" sleep 5 [ "$(podman exec pasture-mongo mongosh --quiet PrivaPub --eval "print(db.Post.countDocuments({Text:/said where PrivaPub does not listen $circle_name/}))")" = "0" ] \ - && ok "an outsider's reply is not delivered to PrivaPub" || ko "the outsider's reply was delivered (the check below proves nothing)" + && ok "the outsider's answer to their own reply is not delivered to PrivaPub" || ko "the outsider's own answer was delivered (the check below proves nothing)" curl -s -o /dev/null -H "$AH" "$P/api/v1/statuses/$t_root_on_p/context" until_true 30 'curl -s -H "$AH" "$P/api/v1/statuses/$t_root_on_p/context" | grep -q "said where PrivaPub does not listen $circle_name"' \ - && ok "opening the thread brings the outsider's reply from Mastodon's context" || ko "the thread never got the outsider's reply" + && ok "opening the thread brings the outsider's answer from Mastodon's context" || ko "the thread never got the outsider's answer" echo " reports" curl -s -o /dev/null -X POST -H "$AH" "$P/api/v1/reports" -d "account_id=$masto_on_p&comment=pasture-report-$circle_name&forward=true"