From 81de470f646457c534715533d3c20c0896fabb30 Mon Sep 17 00:00:00 2001 From: thepra Date: Thu, 1 Oct 2026 18:25:18 +0200 Subject: [PATCH] P5 done: a key we cannot fetch for now gets 503, and follow/like/block ids stay private on purpose - When a sender's key cannot be fetched because its server timed out or answered 5xx, the inbox answers 503 with Retry-After: 300 instead of 401, so Mastodon 4.7 retries rather than switching to RFC 9421 signatures we do not verify yet. The fetcher's failure cache now remembers whether a failure was temporary. - Follow, Like, Block, Accept, Reject and Undo ids are deliberately not dereferenceable: serving them would publish who follows, likes and blocks whom. They are always sent with their object embedded. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012CzABvBkbcFqoHdmi8b9WB --- FEDERATION.md | 5 +++++ .../Federation/InboundRoutingTests.cs | 15 +++++++++++++++ .../Federation/Actors/RemoteActorService.cs | 3 +++ .../Controllers/PeasantsController.cs | 2 ++ PrivaPub/Federation/Inbox/InboxReceiver.cs | 6 +++++- .../Infrastructure/Http/FederationHttp.cs | 19 ++++++++++++++----- docs/ROADMAP.md | 10 +++++++--- 7 files changed, 51 insertions(+), 9 deletions(-) diff --git a/FEDERATION.md b/FEDERATION.md index 439d408..229965b 100644 --- a/FEDERATION.md +++ b/FEDERATION.md @@ -132,6 +132,11 @@ Posts with a location (shown to nearby users of this server) never leave the ser - **Fetching.** All fetches are signed by the instance actor. They go only to public addresses, follow at most three redirects and read at most 1 MB. - **HTML.** Received HTML is sanitised to Mastodon's allowlist. +- **Keys we cannot fetch for now.** When a sender's key cannot be fetched because its server timed out or answered + 5xx, the inbox answers 503 with `Retry-After: 300` rather than 401. +- **Activity ids.** `Create` and `Announce` ids dereference. `Follow`, `Like`, `Block` and the `Accept`, `Reject` and + `Undo` that answer them do not, because serving them would reveal who follows, likes and blocks whom; they are always + sent with their object embedded. - **Delivery.** Failed deliveries are retried with Mastodon's backoff (16 attempts). A host that keeps failing is paused, starting at an hour and growing to a week. diff --git a/PrivaPub.Tests/Federation/InboundRoutingTests.cs b/PrivaPub.Tests/Federation/InboundRoutingTests.cs index 492036f..27a8556 100644 --- a/PrivaPub.Tests/Federation/InboundRoutingTests.cs +++ b/PrivaPub.Tests/Federation/InboundRoutingTests.cs @@ -106,6 +106,21 @@ namespace PrivaPub.Tests.Federation Assert.NotNull(message.ConversationId); } + [Fact] + public async Task An_unreachable_key_asks_the_sender_to_retry_instead_of_refusing() + { + var (_, alice) = await _harness.Persona("alice"); + var offline = new RemoteActor(_harness.Peer, "offline", origin: "http://127.0.0.1:9"); + + var result = await _harness.Deliver(offline, "/human-centipede", new JsonObject + { + ["id"] = NewId(offline, "follows"), ["type"] = "Follow", ["actor"] = offline.Id, ["object"] = alice.Uri + }); + + Assert.Equal(503, result.StatusCode); + Assert.Equal(300, result.RetryAfterSeconds); + } + [Fact] public async Task A_join_of_a_local_post_is_answered_with_ignore() { diff --git a/PrivaPub/Federation/Actors/RemoteActorService.cs b/PrivaPub/Federation/Actors/RemoteActorService.cs index 3be98dc..d33dce3 100644 --- a/PrivaPub/Federation/Actors/RemoteActorService.cs +++ b/PrivaPub/Federation/Actors/RemoteActorService.cs @@ -19,6 +19,7 @@ namespace PrivaPub.Federation.Actors Task FetchObject(string uri, CancellationToken token); Task GetActor(string actorUri, bool refresh, CancellationToken token); Task GetActorByKeyId(string keyId, bool refresh, CancellationToken token); + bool KeyTemporarilyUnavailable(string keyId) => false; Task ResolveHandle(string handle, CancellationToken token); } @@ -120,6 +121,8 @@ namespace PrivaPub.Federation.Actors return await Upsert(actor, key, token); } + public bool KeyTemporarilyUnavailable(string keyId) => Origin.Of(keyId) != default && _http.FailedTemporarily(StripFragment(keyId)); + public async Task ResolveHandle(string handle, CancellationToken token) { var parts = handle?.TrimStart('@').Split('@'); diff --git a/PrivaPub/Federation/Controllers/PeasantsController.cs b/PrivaPub/Federation/Controllers/PeasantsController.cs index af6ae89..04a51f2 100644 --- a/PrivaPub/Federation/Controllers/PeasantsController.cs +++ b/PrivaPub/Federation/Controllers/PeasantsController.cs @@ -333,6 +333,8 @@ namespace PrivaPub.Federation.Controllers { if (result.Error != default) _logger.LogInformation("Inbox refused with {Status}: {Error}", result.StatusCode, result.Error); + if (result.RetryAfterSeconds is { } seconds) + Response.Headers.RetryAfter = seconds.ToString(System.Globalization.CultureInfo.InvariantCulture); return result.Error == default ? StatusCode(result.StatusCode) : StatusCode(result.StatusCode, result.Error); } diff --git a/PrivaPub/Federation/Inbox/InboxReceiver.cs b/PrivaPub/Federation/Inbox/InboxReceiver.cs index aad8992..b64321a 100644 --- a/PrivaPub/Federation/Inbox/InboxReceiver.cs +++ b/PrivaPub/Federation/Inbox/InboxReceiver.cs @@ -13,7 +13,7 @@ using static PrivaPub.Federation.Objects.ActivityJson; namespace PrivaPub.Federation.Inbox { - public sealed record InboxResult(int StatusCode, string Error = default); + public sealed record InboxResult(int StatusCode, string Error = default, int? RetryAfterSeconds = default); public sealed record InboxPayload(string ActorURI, string Activity, string Inbox = default, string KeyId = default, string Algorithm = default, string[] SignedHeaders = default, DateTime? ReceivedAt = default); @@ -25,6 +25,8 @@ namespace PrivaPub.Federation.Inbox public class InboxReceiver : IInboxReceiver { + const int KeyRetrySeconds = 300; + const int MaxBodyBytes = 1024 * 1024; readonly ILocalActorService _localActors; @@ -88,6 +90,8 @@ namespace PrivaPub.Federation.Inbox if (keyOwner == default || !HttpSignatures.Verify(keyOwner.PublicKey, signingString, parameters.Signature)) { keyOwner = await _remoteActors.GetActorByKeyId(parameters.KeyId, refresh: true, token); + if (keyOwner == default && _remoteActors.KeyTemporarilyUnavailable(parameters.KeyId)) + return new(StatusCodes.Status503ServiceUnavailable, "the signing key could not be fetched; try again later", KeyRetrySeconds); if (keyOwner == default || !HttpSignatures.Verify(keyOwner.PublicKey, signingString, parameters.Signature)) return new(StatusCodes.Status401Unauthorized, "the signature does not verify"); } diff --git a/PrivaPub/Infrastructure/Http/FederationHttp.cs b/PrivaPub/Infrastructure/Http/FederationHttp.cs index 7f3b90e..76d23a6 100644 --- a/PrivaPub/Infrastructure/Http/FederationHttp.cs +++ b/PrivaPub/Infrastructure/Http/FederationHttp.cs @@ -21,6 +21,7 @@ namespace PrivaPub.Infrastructure.Http { bool IsAllowed(Uri target); Task GetJson(string url, string accept, Action sign, CancellationToken token); + bool FailedTemporarily(string url); Task Send(HttpRequestMessage request, CancellationToken token); Task<(byte[] Bytes, string ContentType)> GetMedia(string url, long maxBytes, CancellationToken token); } @@ -107,7 +108,8 @@ namespace PrivaPub.Infrastructure.Http } if (!response.IsSuccessStatusCode) - return Refuse(negativeKey, url, $"status {(int)response.StatusCode}"); + return Refuse(negativeKey, url, $"status {(int)response.StatusCode}", + transient: (int)response.StatusCode is >= 500 or 429 or 408); var mediaType = response.Content.Headers.ContentType?.MediaType; if (mediaType == default || !JsonMediaTypes.Contains(mediaType, StringComparer.OrdinalIgnoreCase)) @@ -125,14 +127,21 @@ namespace PrivaPub.Infrastructure.Http } catch (OperationCanceledException) when (!token.IsCancellationRequested) { - return Refuse(negativeKey, url, "a timeout"); + return Refuse(negativeKey, url, "a timeout", transient: true); } - catch (Exception ex) when (ex is HttpRequestException or JsonException or BlockedDestinationException) + catch (HttpRequestException ex) + { + return Refuse(negativeKey, url, ex.Message, transient: true); + } + catch (Exception ex) when (ex is JsonException or BlockedDestinationException) { return Refuse(negativeKey, url, ex.Message); } } + public bool FailedTemporarily(string url) => + Uri.TryCreate(url, UriKind.Absolute, out var target) && _cache.TryGetValue(NegativeKey(target), out bool transient) && transient; + public async Task<(byte[] Bytes, string ContentType)> GetMedia(string url, long maxBytes, CancellationToken token) { if (!Uri.TryCreate(url, UriKind.Absolute, out var target) || !IsAllowed(target)) @@ -200,9 +209,9 @@ namespace PrivaPub.Infrastructure.Http static string NegativeKey(Uri target) => "federation-http:refused:" + target.AbsoluteUri; - FetchedJson Refuse(string negativeKey, string url, string reason) + FetchedJson Refuse(string negativeKey, string url, string reason, bool transient = false) { - _cache.Set(negativeKey, true, NegativeCacheLifetime); + _cache.Set(negativeKey, transient, NegativeCacheLifetime); _logger.LogInformation("GET {Url} refused: {Reason}", url, reason); return default; } diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index ef8b27a..f054422 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -10,7 +10,7 @@ Written 2026-10-01 from the original 2023 code, the decePubClient UI, a federati - [x] P2 Mastodon client API: v1.4.0, deployed 2026-10-01; OAuth and the anonymous API verified on production, the signed-in API verified locally (no real-client login on production yet) - [x] P3 Social features: v1.5.0, deployed 2026-10-01; media, proxy, blocks, mutes, bookmarks, pins and reports verified by tests and locally (no upload on production yet) - [x] P4 Groups and privacy features: v1.6.0, deployed 2026-10-01; communities, circles and local-only located posts verified by tests. v1.6.1 adds the pasture (`tools/pasture/`): live interop with GoToSocial 0.22.1 passes all 25 checks, three runs in a row. Lemmy and a live Mastodon circle member are not run yet; the pasture has GoToSocial only -- [ ] P5 Lose nothing: wire tolerance, full objects, raw capture (see `docs/INTEROP.md`) +- [x] P5 Lose nothing: v1.7.0 to v1.9.1, deployed 2026-10-01; parsing, typed details, provenance, downvotes, tombstones, federated blocks, all checked live against GoToSocial. Book reviews and forum threads keep their raw form only (typed in P7 and P8) - [ ] P6 Emoji, polls, quotes, reactions, cards, players - [ ] P7 Threads, communities, moderation, the social graph - [ ] P8 Signatures, discovery, the long tail @@ -414,7 +414,7 @@ it, raw where it doesn't. - Persona usernames match Mastodon's and Misskey's pattern (groups already did). - Every term we emit is defined in our JSON-LD context. - `Vary: Accept`. - - Still open: every activity id dereferences. + - Not done on purpose: see "Deliberately not done" below. - **Owner decisions that were small** (done in v1.7.0): blocks federate (`Block`, `Undo{Block}`); a persona's and a group's `published` is a random day up to two weeks before its creation (migration `_007`). - **Parsing every shape** (done in v1.8.0, `Federation/Objects/ObjectShapes.cs`): @@ -441,7 +441,11 @@ it, raw where it doesn't. - A deleted object's id is kept for 90 days (W9), and its `ObjectRecord` is removed with the post. - **Typed details reach clients** (done in v1.9.0) as `Status.privapub`. - **Delivery:** a 503 with `Retry-After` is waited out like a 429 and no longer counts against the host (done in v1.9.0). - Still open: answering 503 ourselves when a key fetch fails temporarily, and making every activity id dereference. + We answer 503 with `Retry-After` ourselves when a sender's key cannot be fetched for a temporary reason (v1.9.1), + so Mastodon 4.7 retries instead of switching to RFC 9421. +- **Deliberately not done: dereferenceable `Follow`, `Like`, `Block`, `Accept`, `Reject` and `Undo` ids.** Serving them + would publish who follows, likes and blocks whom, which our collections deliberately hide. Every one of them is sent + with its object embedded, which is all Misskey needs. `Create` and `Announce` ids do dereference. #### P6 What people see: emoji, polls, quotes, reactions, cards, players - **Custom emoji** on posts, names, fields and poll options, proxied.