Deliveries run on a Mongo job queue with leases, backoff and per-host limits

The single serial DeliveryWorker is replaced by Infrastructure/Jobs:
- Job rows are leased with one FindOneAndUpdate (oldest RunAt first, a
  two-minute lease) and a reaper returns expired leases every 30 s;
- enqueueing wakes the workers, which otherwise poll every five seconds;
- delivery runs eight at a time with at most two per host, so a slow or
  dead server holds two slots, not the queue;
- a failure waits n^4 + 15 + jitter seconds (Mastodon's curve) for up to
  16 attempts; a 4xx other than 408/429 is final, a 429 honours
  Retry-After;
- RemoteInstance is a per-host circuit breaker: ten consecutive failures
  quarantine a host for an hour, doubling to a week, and its jobs wait
  without spending attempts;
- a delivery is queued once per activity and inbox (unique DedupeKey), and
  finished jobs expire after seven days (TTL on FinishedAt).

Migration _004 moves pending Delivery rows into jobs and marks them
abandoned, so a rollback to the old worker cannot send them twice.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012CzABvBkbcFqoHdmi8b9WB

PrivaPub/Infrastructure/Data/Migrations/_004_pending_deliveries_become_jobs.cs

@@ -0,0 +1,43 @@
+using MongoDB.Entities;
+
+using PrivaPub.Federation.Outbox;
+using PrivaPub.Models.Federation;
+using PrivaPub.Models.Jobs;
+
+using System.Text.Json;
+using System.Text.Json.Nodes;
+
+namespace PrivaPub.Infrastructure.Data.Migrations
+{
+	public class _004_pending_deliveries_become_jobs : IMigration
+	{
+		public async Task UpgradeAsync()
+		{
+			var pending = await DB.Default.Find<Delivery>()
+				.Match(d => !d.DeliveredAt.HasValue && !d.AbandonedAt.HasValue)
+				.ExecuteAsync();
+			foreach (var delivery in pending)
+			{
+				if (Uri.TryCreate(delivery.InboxURL, UriKind.Absolute, out var inbox))
+				{
+					var activityId = JsonNode.Parse(delivery.Body)?["id"]?.GetValue<string>();
+					var job = new Job
+					{
+						Kind = JobKind.Deliver,
+						Host = inbox.Host.ToLowerInvariant(),
+						DedupeKey = activityId == default ? default : $"{activityId}|{delivery.InboxURL}",
+						Attempts = delivery.Attempts,
+						RunAt = delivery.NextAttemptAt,
+						Payload = JsonSerializer.Serialize(new DeliveryPayload(delivery.SignerId, delivery.SignerKind, delivery.InboxURL, delivery.Body))
+					};
+					if (!await DB.Default.Find<Job>().Match(j => j.DedupeKey == job.DedupeKey && job.DedupeKey != null).ExecuteAnyAsync())
+						await DB.Default.SaveAsync(job);
+				}
+				await DB.Default.Update<Delivery>().MatchID(delivery.ID)
+					.Modify(d => d.AbandonedAt, DateTime.UtcNow)
+					.Modify(d => d.LastError, "moved to the job queue")
+					.ExecuteAsync();
+			}
+		}
+	}
+}