Deliveries refused with 409 or 422 get three tries

Mastodon answers 422 when two first contacts from one actor race to
create its account (ActiveRecord::RecordInvalid on the unique uri), and
409 while another worker holds its lock. A persona that followed two
Mastodon accounts at once had one Follow refused that way; the job died
on its first attempt and the persona waited on "requested" forever.
Both answers are now retried twice on the usual backoff before they
count as refusals.

Found by the town (a village of 23 accounts on seven servers).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01LsXgEaXee4GCU1hwYgPJXw

PrivaPub.Tests/Federation/JobHandlerTests.cs

@@ -469,7 +469,7 @@ namespace PrivaPub.Tests.Federation
 			await DB.Default.DeleteAsync<RemoteInstance>(i => PeerHosts.Contains(i.Host));
 		}
 
-		async Task<JobOutcome> Attempt(LocalActor signer, string inbox)
+		async Task<JobOutcome> Attempt(LocalActor signer, string inbox, int attempts = 1)
 		{
 			var token = TestContext.Current.CancellationToken;
 			var id = $"{Harness.Base}/a/{Guid.NewGuid():N}";
@@ -479,7 +479,7 @@ namespace PrivaPub.Tests.Federation
 				["object"] = new JsonObject { ["type"] = "Note", ["content"] = "hi" }
 			}, token);
 			var job = await DB.Default.Find<Job>().Match(j => j.DedupeKey == $"{id}|{inbox}").ExecuteSingleAsync(token);
-			job.Attempts = 1;
+			job.Attempts = attempts;
 			var handler = new DeliveryJobHandler(_harness.Local, Peer.Http(), new HostCircuitBreaker(new MemoryCache(new MemoryCacheOptions())),
 				NullLogger<DeliveryJobHandler>.Instance);
 			return await handler.Handle(job, token);
@@ -533,5 +533,21 @@ namespace PrivaPub.Tests.Federation
 			Assert.Equal(JobResult.Retry, broken.Result);
 			Assert.StartsWith("500", broken.Error);
 		}
+
+		// Mastodon's 422 for two first contacts racing to create one account, and its 409 for a held lock, are retried a
+		// few times; the third refusal is final like any other
+		[Fact]
+		public async Task Conflicts_and_unprocessable_answers_are_retried_twice_then_dead()
+		{
+			var (_, alice) = await _harness.Persona("alice");
+			_harness.Peer.Answer("/racing/inbox", 422);
+			_harness.Peer.Answer("/locked/inbox", 409);
+
+			Assert.Equal(JobResult.Retry, (await Attempt(alice, _harness.Peer.A + "/racing/inbox")).Result);
+			Assert.Equal(JobResult.Retry, (await Attempt(alice, _harness.Peer.A + "/locked/inbox", attempts: 2)).Result);
+			var final = await Attempt(alice, _harness.Peer.A + "/racing/inbox", attempts: 3);
+			Assert.Equal(JobResult.Dead, final.Result);
+			Assert.StartsWith("422", final.Error);
+		}
 	}
 }

PrivaPub/Federation/Outbox/DeliveryService.cs

@@ -216,6 +216,11 @@ namespace PrivaPub.Federation.Outbox
 						? JobOutcome.Defer(DateTime.UtcNow + Min(retryAfter.Value, TimeSpan.FromHours(6)), $"{status}")
 						: JobOutcome.Retry($"{status}");
 				}
+				// Mastodon answers 422 when two first contacts from one actor race to create its account (RecordInvalid on the
+				// unique uri) and 409 while another worker holds its lock: a Follow refused that way was lost for good, and the
+				// persona waited on "requested" forever (found by the town). Both get a few tries before they count as refusals.
+				if (status is 409 or 422 && job.Attempts < TransientRefusalAttempts)
+					return JobOutcome.Retry($"{status} {response.ReasonPhrase}");
 				if (status is >= 400 and < 500 && status != 408)
 				{
 					await _breaker.Succeeded(job.Host, token);
@@ -239,6 +244,8 @@ namespace PrivaPub.Federation.Outbox
 			}
 		}
 
+		const int TransientRefusalAttempts = 3;
+
 		static TimeSpan Min(TimeSpan a, TimeSpan b) => a < b ? a : b;
 	}
 }