Deliveries run on a Mongo job queue with leases, backoff and per-host limits

The single serial DeliveryWorker is replaced by Infrastructure/Jobs:
- Job rows are leased with one FindOneAndUpdate (oldest RunAt first, a
  two-minute lease) and a reaper returns expired leases every 30 s;
- enqueueing wakes the workers, which otherwise poll every five seconds;
- delivery runs eight at a time with at most two per host, so a slow or
  dead server holds two slots, not the queue;
- a failure waits n^4 + 15 + jitter seconds (Mastodon's curve) for up to
  16 attempts; a 4xx other than 408/429 is final, a 429 honours
  Retry-After;
- RemoteInstance is a per-host circuit breaker: ten consecutive failures
  quarantine a host for an hour, doubling to a week, and its jobs wait
  without spending attempts;
- a delivery is queued once per activity and inbox (unique DedupeKey), and
  finished jobs expire after seven days (TTL on FinishedAt).

Migration _004 moves pending Delivery rows into jobs and marks them
abandoned, so a rollback to the old worker cannot send them twice.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012CzABvBkbcFqoHdmi8b9WB

PrivaPub.Tests/Federation/InboxScenarioTests.cs

@@ -6,6 +6,7 @@ using MongoDB.Entities;
 using PrivaPub.Federation.Actors;
 using PrivaPub.Federation.Inbox;
 using PrivaPub.Federation.Outbox;
+using PrivaPub.Infrastructure.Jobs;
 using PrivaPub.Models;
 using PrivaPub.Models.Group;
 using PrivaPub.Models.Post;
@@ -36,7 +37,7 @@ namespace PrivaPub.Tests.Federation
 			var cache = new MemoryCache(new MemoryCacheOptions());
 			_local = new LocalActorService(new DbEntities(), new StaticOptions<AppConfiguration>(new AppConfiguration { BackendBaseAddress = Base }));
 			var remote = new RemoteActorService(Peer.Http(cache), _local, cache, new DbEntities());
-			_inbox = new InboxService(new DbEntities(), _local, remote, new DeliveryService(new DbEntities()), NullLogger<InboxService>.Instance);
+			_inbox = new InboxService(new DbEntities(), _local, remote, new DeliveryService(new DbEntities(), new JobQueue()), NullLogger<InboxService>.Instance);
 		}
 
 		public async ValueTask DisposeAsync()

PrivaPub.Tests/Infrastructure/IndexTests.cs

@@ -20,6 +20,8 @@ namespace PrivaPub.Tests.Infrastructure
 		{
 			Assert.SkipUnless(MongoFixture.Enabled, MongoFixture.Skip);
 			var token = TestContext.Current.CancellationToken;
+			await DB.Default.Index<ForeignAvatar>().DropAllAsync(token);
+			await DB.Default.Index<Follower>().DropAllAsync(token);
 			var actorUri = $"https://r.example/users/{Guid.NewGuid():N}";
 			var localId = Guid.NewGuid().ToString("N")[..24];
 			await DB.Default.SaveAsync(new[]
@@ -48,7 +50,6 @@ namespace PrivaPub.Tests.Infrastructure
 		{
 			Assert.SkipUnless(MongoFixture.Enabled, MongoFixture.Skip);
 			var token = TestContext.Current.CancellationToken;
-			await Indexes.Create(token);
 			var local = new LocalActorService(new DbEntities(), new StaticOptions<AppConfiguration>(new AppConfiguration { BackendBaseAddress = "https://privapub.test" }));
 			var name = $"n{Guid.NewGuid():N}"[..20];
 

PrivaPub.Tests/Infrastructure/JobQueueTests.cs

@@ -0,0 +1,145 @@
+using Microsoft.Extensions.Caching.Memory;
+using Microsoft.Extensions.Logging.Abstractions;
+
+using MongoDB.Entities;
+
+using PrivaPub.Federation.Actors;
+using PrivaPub.Federation.Outbox;
+using PrivaPub.Infrastructure.Jobs;
+using PrivaPub.Models;
+using PrivaPub.Models.Jobs;
+using PrivaPub.Models.User;
+using PrivaPub.StaticServices;
+using PrivaPub.Tests.Support;
+
+using System.Text.Json.Nodes;
+
+namespace PrivaPub.Tests.Infrastructure
+{
+	public class BackoffTests
+	{
+		[Fact]
+		public void Grows_like_mastodons()
+		{
+			Assert.InRange(Backoff.After(1).TotalSeconds, 16, 16 + 9 * 2);
+			Assert.InRange(Backoff.After(10).TotalSeconds, 10015, 10015 + 9 * 11);
+		}
+
+		[Fact]
+		public void Quarantines_a_host_only_past_the_threshold()
+		{
+			Assert.Equal(TimeSpan.Zero, Backoff.HostQuarantine(9, 10));
+			Assert.Equal(TimeSpan.FromHours(1), Backoff.HostQuarantine(10, 10));
+			Assert.Equal(TimeSpan.FromHours(4), Backoff.HostQuarantine(12, 10));
+			Assert.Equal(TimeSpan.FromDays(7), Backoff.HostQuarantine(40, 10));
+		}
+	}
+
+	[Trait("Category", "Integration")]
+	public sealed class JobQueueTests : IAsyncLifetime
+	{
+		Peer _peer;
+
+		public async ValueTask InitializeAsync()
+		{
+			Assert.SkipUnless(MongoFixture.Enabled, MongoFixture.Skip);
+			_peer = await Peer.Start();
+		}
+
+		public async ValueTask DisposeAsync()
+		{
+			if (_peer != default)
+				await _peer.DisposeAsync();
+		}
+
+		static Job NewJob(string host = default, string dedupe = default) => new()
+		{
+			Kind = (JobKind)99,
+			Host = host,
+			DedupeKey = dedupe,
+			Payload = "{}"
+		};
+
+		[Fact]
+		public async Task A_dedupe_key_is_queued_once()
+		{
+			var queue = new JobQueue();
+			var key = Guid.NewGuid().ToString("N");
+
+			var inserted = await queue.EnqueueMany(new[] { NewJob(dedupe: key), NewJob(dedupe: key) }, TestContext.Current.CancellationToken);
+
+			Assert.Equal(1, inserted);
+		}
+
+		[Fact]
+		public async Task A_busy_host_is_skipped_and_a_failed_job_backs_off()
+		{
+			var token = TestContext.Current.CancellationToken;
+			var queue = new JobQueue();
+			var kind = (JobKind)(100 + Random.Shared.Next(1000));
+			var busy = $"busy{Guid.NewGuid():N}.example";
+			var free = $"free{Guid.NewGuid():N}.example";
+			await queue.EnqueueMany(new[] { new Job { Kind = kind, Host = busy }, new Job { Kind = kind, Host = free } }, token);
+
+			var leased = await queue.Lease(kind, new[] { busy }, token);
+			Assert.Equal(free, leased.Host);
+			Assert.Equal(1, leased.Attempts);
+
+			await queue.Finish(leased, JobOutcome.Retry("boom"), maxAttempts: 3, token);
+			var after = await DB.Default.Find<Job>().OneAsync(leased.ID, token);
+			Assert.Equal(JobState.Pending, after.State);
+			Assert.True(after.RunAt > DateTime.UtcNow.AddSeconds(10));
+
+			leased.Attempts = 3;
+			await queue.Finish(leased, JobOutcome.Retry("boom"), maxAttempts: 3, token);
+			Assert.Equal(JobState.Dead, (await DB.Default.Find<Job>().OneAsync(leased.ID, token)).State);
+		}
+
+		[Fact]
+		public async Task An_expired_lease_returns_to_the_queue()
+		{
+			var token = TestContext.Current.CancellationToken;
+			var job = new Job { Kind = (JobKind)98, State = JobState.Running, LeasedUntil = DateTime.UtcNow.AddMinutes(-1) };
+			await DB.Default.SaveAsync(job, token);
+
+			await new JobQueue().Reap(token);
+
+			Assert.Equal(JobState.Pending, (await DB.Default.Find<Job>().OneAsync(job.ID, token)).State);
+		}
+
+		[Fact]
+		public async Task A_dead_host_does_not_hold_up_deliveries_to_live_ones()
+		{
+			var token = TestContext.Current.CancellationToken;
+			var (privateKey, publicKey) = Keys.NewKeyPair();
+			var avatar = new Avatar { UserName = $"sender{Guid.NewGuid():N}"[..20], PrivateKey = privateKey, PublicKey = publicKey };
+			await DB.Default.SaveAsync(avatar, token);
+			var local = new LocalActorService(new DbEntities(), new StaticOptions<AppConfiguration>(new AppConfiguration { BackendBaseAddress = "https://privapub.test" }));
+			var sender = local.FromAvatar(avatar);
+			var queue = new JobQueue();
+			var delivery = new DeliveryService(new DbEntities(), queue);
+			_peer.Answer("/dead/inbox", 503, TimeSpan.FromMilliseconds(300));
+			_peer.Answer("/live/inbox", 202);
+
+			var deadInboxes = Enumerable.Range(0, 60).Select(i => $"{_peer.B}/dead/inbox?{i}");
+			await delivery.Enqueue(sender, deadInboxes, new JsonObject { ["id"] = $"https://privapub.test/a/{Guid.NewGuid():N}", ["type"] = "Create" }, token);
+			await delivery.Enqueue(sender, new[] { $"{_peer.A}/live/inbox" }, new JsonObject { ["id"] = $"https://privapub.test/a/{Guid.NewGuid():N}", ["type"] = "Create" }, token);
+
+			var handler = new DeliveryJobHandler(local, Peer.Http(), new HostCircuitBreaker(new MemoryCache(new MemoryCacheOptions())), NullLogger<DeliveryJobHandler>.Instance);
+			using var worker = new JobWorker(queue, new IJobHandler[] { handler }, NullLogger<JobWorker>.Instance);
+			await worker.StartAsync(token);
+			var deadline = DateTime.UtcNow.AddSeconds(5);
+			while (DateTime.UtcNow < deadline && !_peer.Requests.Any(r => r.Path == "/live/inbox"))
+				await Task.Delay(50, token);
+			var live = Assert.Single(_peer.Requests, r => r.Path == "/live/inbox");
+			Assert.True(_peer.Requests.Count(r => r.Path == "/dead/inbox") < 10);
+			while (DateTime.UtcNow < deadline && !await DB.Default.Find<RemoteInstance>().Match(i => i.Host == "localhost").ExecuteAnyAsync(token))
+				await Task.Delay(50, token);
+			await worker.StopAsync(token);
+
+			Assert.Contains($"keyId=\"{sender.KeyId}\"", live.Signature);
+			var instance = await DB.Default.Find<RemoteInstance>().Match(i => i.Host == "localhost").ExecuteFirstAsync(token);
+			Assert.True(instance.ConsecutiveFailures > 0);
+		}
+	}
+}

PrivaPub.Tests/Support/MongoFixture.cs

@@ -32,6 +32,7 @@ namespace PrivaPub.Tests.Support
 			var connection = Environment.GetEnvironmentVariable("PRIVAPUB_TEST_MONGO") ?? "mongodb://127.0.0.1:27017";
 			await DB.InitAsync(Database, MongoClientSettings.FromConnectionString(connection));
 			EntityMaps.Warm();
+			await Indexes.Create();
 		}
 
 		public async ValueTask DisposeAsync()

PrivaPub.Tests/Support/Peer.cs

@@ -16,6 +16,7 @@ namespace PrivaPub.Tests.Support
 	{
 		readonly WebApplication _app;
 		readonly ConcurrentDictionary<string, string> _documents = new();
+		readonly ConcurrentDictionary<string, (int Status, TimeSpan Delay)> _answers = new();
 
 		public int Port { get; }
 		public string A => $"http://127.0.0.1:{Port}";
@@ -38,6 +39,13 @@ namespace PrivaPub.Tests.Support
 			{
 				peer.Requests.Enqueue(new(context.Request.Method, context.Request.Path, context.Request.Headers["Signature"].ToString()));
 				var key = context.Request.Path.Value;
+				if (peer._answers.TryGetValue(key, out var answer))
+				{
+					if (answer.Delay > TimeSpan.Zero)
+						await Task.Delay(answer.Delay);
+					context.Response.StatusCode = answer.Status;
+					return;
+				}
 				if (!peer._documents.TryGetValue(key, out var document))
 				{
 					context.Response.StatusCode = StatusCodes.Status404NotFound;
@@ -53,6 +61,8 @@ namespace PrivaPub.Tests.Support
 
 		public void Serve(string path, string json) => _documents[path] = json;
 
+		public void Answer(string path, int status, TimeSpan delay = default) => _answers[path] = (status, delay);
+
 		public static FederationHttp Http(IMemoryCache cache = default)
 		{
 			var options = new FederationOptions { AllowPrivateNetworks = true, AllowPlainHttp = true };