Deliveries run on a Mongo job queue with leases, backoff and per-host limits

The single serial DeliveryWorker is replaced by Infrastructure/Jobs:
- Job rows are leased with one FindOneAndUpdate (oldest RunAt first, a
  two-minute lease) and a reaper returns expired leases every 30 s;
- enqueueing wakes the workers, which otherwise poll every five seconds;
- delivery runs eight at a time with at most two per host, so a slow or
  dead server holds two slots, not the queue;
- a failure waits n^4 + 15 + jitter seconds (Mastodon's curve) for up to
  16 attempts; a 4xx other than 408/429 is final, a 429 honours
  Retry-After;
- RemoteInstance is a per-host circuit breaker: ten consecutive failures
  quarantine a host for an hour, doubling to a week, and its jobs wait
  without spending attempts;
- a delivery is queued once per activity and inbox (unique DedupeKey), and
  finished jobs expire after seven days (TTL on FinishedAt).

Migration _004 moves pending Delivery rows into jobs and marks them
abandoned, so a rollback to the old worker cannot send them twice.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012CzABvBkbcFqoHdmi8b9WB

PrivaPub/Infrastructure/Jobs/HostCircuitBreaker.cs

@@ -0,0 +1,76 @@
+using Microsoft.Extensions.Caching.Memory;
+
+using MongoDB.Entities;
+
+using PrivaPub.Models.Jobs;
+
+namespace PrivaPub.Infrastructure.Jobs
+{
+	public interface IHostCircuitBreaker
+	{
+		Task<DateTime?> UnavailableUntil(string host, CancellationToken token);
+		Task Succeeded(string host, CancellationToken token);
+		Task Failed(string host, string error, CancellationToken token);
+	}
+
+	public class HostCircuitBreaker : IHostCircuitBreaker
+	{
+		public const int Threshold = 10;
+		static readonly TimeSpan CacheLifetime = TimeSpan.FromSeconds(30);
+
+		readonly IMemoryCache _cache;
+
+		public HostCircuitBreaker(IMemoryCache cache)
+		{
+			_cache = cache;
+		}
+
+		public async Task<DateTime?> UnavailableUntil(string host, CancellationToken token)
+		{
+			var instance = await Instance(host, token);
+			return instance?.UnavailableUntil > DateTime.UtcNow ? instance.UnavailableUntil : default;
+		}
+
+		public async Task Succeeded(string host, CancellationToken token)
+		{
+			var instance = await Instance(host, token);
+			if (instance is not { ConsecutiveFailures: > 0 } && instance?.LastSuccessAt > DateTime.UtcNow.AddHours(-1))
+				return;
+			await DB.Default.Update<RemoteInstance>()
+				.Match(i => i.Host == host)
+				.Modify(i => i.ConsecutiveFailures, 0)
+				.Modify(i => i.UnavailableUntil, null)
+				.Modify(i => i.LastSuccessAt, DateTime.UtcNow)
+				.Option(o => o.IsUpsert = true)
+				.ExecuteAsync(token);
+			_cache.Remove(Key(host));
+		}
+
+		public async Task Failed(string host, string error, CancellationToken token)
+		{
+			var now = DateTime.UtcNow;
+			var instance = await DB.Default.UpdateAndGet<RemoteInstance>()
+				.Match(i => i.Host == host)
+				.Modify(b => b.Inc(i => i.ConsecutiveFailures, 1))
+				.Modify(i => i.LastFailureAt, now)
+				.Modify(i => i.LastError, error)
+				.Option(o => o.IsUpsert = true)
+				.ExecuteAsync(token);
+			var quarantine = Backoff.HostQuarantine(instance.ConsecutiveFailures, Threshold);
+			if (quarantine > TimeSpan.Zero)
+				await DB.Default.Update<RemoteInstance>().MatchID(instance.ID)
+					.Modify(i => i.UnavailableUntil, now + quarantine)
+					.ExecuteAsync(token);
+			_cache.Remove(Key(host));
+		}
+
+		async Task<RemoteInstance> Instance(string host, CancellationToken token) =>
+			await _cache.GetOrCreateAsync(Key(host), async entry =>
+			{
+				entry.AbsoluteExpirationRelativeToNow = CacheLifetime;
+				return await DB.Default.Find<RemoteInstance>().Match(i => i.Host == host).ExecuteFirstAsync(token);
+			});
+
+		static string Key(string host) => "remote-instance:" + host;
+	}
+}