diff --git a/PrivaPub.Tests/Federation/JobHandlerTests.cs b/PrivaPub.Tests/Federation/JobHandlerTests.cs index 40e248b..758846d 100644 --- a/PrivaPub.Tests/Federation/JobHandlerTests.cs +++ b/PrivaPub.Tests/Federation/JobHandlerTests.cs @@ -469,7 +469,7 @@ namespace PrivaPub.Tests.Federation await DB.Default.DeleteAsync(i => PeerHosts.Contains(i.Host)); } - async Task Attempt(LocalActor signer, string inbox) + async Task Attempt(LocalActor signer, string inbox, int attempts = 1) { var token = TestContext.Current.CancellationToken; var id = $"{Harness.Base}/a/{Guid.NewGuid():N}"; @@ -479,7 +479,7 @@ namespace PrivaPub.Tests.Federation ["object"] = new JsonObject { ["type"] = "Note", ["content"] = "hi" } }, token); var job = await DB.Default.Find().Match(j => j.DedupeKey == $"{id}|{inbox}").ExecuteSingleAsync(token); - job.Attempts = 1; + job.Attempts = attempts; var handler = new DeliveryJobHandler(_harness.Local, Peer.Http(), new HostCircuitBreaker(new MemoryCache(new MemoryCacheOptions())), NullLogger.Instance); return await handler.Handle(job, token); @@ -533,5 +533,21 @@ namespace PrivaPub.Tests.Federation Assert.Equal(JobResult.Retry, broken.Result); Assert.StartsWith("500", broken.Error); } + + // Mastodon's 422 for two first contacts racing to create one account, and its 409 for a held lock, are retried a + // few times; the third refusal is final like any other + [Fact] + public async Task Conflicts_and_unprocessable_answers_are_retried_twice_then_dead() + { + var (_, alice) = await _harness.Persona("alice"); + _harness.Peer.Answer("/racing/inbox", 422); + _harness.Peer.Answer("/locked/inbox", 409); + + Assert.Equal(JobResult.Retry, (await Attempt(alice, _harness.Peer.A + "/racing/inbox")).Result); + Assert.Equal(JobResult.Retry, (await Attempt(alice, _harness.Peer.A + "/locked/inbox", attempts: 2)).Result); + var final = await Attempt(alice, _harness.Peer.A + "/racing/inbox", attempts: 3); + Assert.Equal(JobResult.Dead, final.Result); + Assert.StartsWith("422", final.Error); + } } } diff --git a/PrivaPub/Federation/Outbox/DeliveryService.cs b/PrivaPub/Federation/Outbox/DeliveryService.cs index 1e57f52..c29ebcc 100644 --- a/PrivaPub/Federation/Outbox/DeliveryService.cs +++ b/PrivaPub/Federation/Outbox/DeliveryService.cs @@ -216,6 +216,11 @@ namespace PrivaPub.Federation.Outbox ? JobOutcome.Defer(DateTime.UtcNow + Min(retryAfter.Value, TimeSpan.FromHours(6)), $"{status}") : JobOutcome.Retry($"{status}"); } + // Mastodon answers 422 when two first contacts from one actor race to create its account (RecordInvalid on the + // unique uri) and 409 while another worker holds its lock: a Follow refused that way was lost for good, and the + // persona waited on "requested" forever (found by the town). Both get a few tries before they count as refusals. + if (status is 409 or 422 && job.Attempts < TransientRefusalAttempts) + return JobOutcome.Retry($"{status} {response.ReasonPhrase}"); if (status is >= 400 and < 500 && status != 408) { await _breaker.Succeeded(job.Host, token); @@ -239,6 +244,8 @@ namespace PrivaPub.Federation.Outbox } } + const int TransientRefusalAttempts = 3; + static TimeSpan Min(TimeSpan a, TimeSpan b) => a < b ? a : b; } }