Deliveries run on a Mongo job queue with leases, backoff and per-host limits

The single serial DeliveryWorker is replaced by Infrastructure/Jobs:
- Job rows are leased with one FindOneAndUpdate (oldest RunAt first, a
  two-minute lease) and a reaper returns expired leases every 30 s;
- enqueueing wakes the workers, which otherwise poll every five seconds;
- delivery runs eight at a time with at most two per host, so a slow or
  dead server holds two slots, not the queue;
- a failure waits n^4 + 15 + jitter seconds (Mastodon's curve) for up to
  16 attempts; a 4xx other than 408/429 is final, a 429 honours
  Retry-After;
- RemoteInstance is a per-host circuit breaker: ten consecutive failures
  quarantine a host for an hour, doubling to a week, and its jobs wait
  without spending attempts;
- a delivery is queued once per activity and inbox (unique DedupeKey), and
  finished jobs expire after seven days (TTL on FinishedAt).

Migration _004 moves pending Delivery rows into jobs and marks them
abandoned, so a rollback to the old worker cannot send them twice.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012CzABvBkbcFqoHdmi8b9WB

PrivaPub/Infrastructure/Jobs/Backoff.cs

@@ -0,0 +1,13 @@
+namespace PrivaPub.Infrastructure.Jobs
+{
+	public static class Backoff
+	{
+		public static TimeSpan After(int attempt) =>
+			TimeSpan.FromSeconds(Math.Pow(attempt, 4) + 15 + Random.Shared.Next(10) * (attempt + 1));
+
+		public static TimeSpan HostQuarantine(int consecutiveFailures, int threshold) =>
+			consecutiveFailures < threshold
+				? TimeSpan.Zero
+				: TimeSpan.FromHours(Math.Min(Math.Pow(2, consecutiveFailures - threshold), 24 * 7));
+	}
+}

PrivaPub/Infrastructure/Jobs/HostCircuitBreaker.cs

@@ -0,0 +1,76 @@
+using Microsoft.Extensions.Caching.Memory;
+
+using MongoDB.Entities;
+
+using PrivaPub.Models.Jobs;
+
+namespace PrivaPub.Infrastructure.Jobs
+{
+	public interface IHostCircuitBreaker
+	{
+		Task<DateTime?> UnavailableUntil(string host, CancellationToken token);
+		Task Succeeded(string host, CancellationToken token);
+		Task Failed(string host, string error, CancellationToken token);
+	}
+
+	public class HostCircuitBreaker : IHostCircuitBreaker
+	{
+		public const int Threshold = 10;
+		static readonly TimeSpan CacheLifetime = TimeSpan.FromSeconds(30);
+
+		readonly IMemoryCache _cache;
+
+		public HostCircuitBreaker(IMemoryCache cache)
+		{
+			_cache = cache;
+		}
+
+		public async Task<DateTime?> UnavailableUntil(string host, CancellationToken token)
+		{
+			var instance = await Instance(host, token);
+			return instance?.UnavailableUntil > DateTime.UtcNow ? instance.UnavailableUntil : default;
+		}
+
+		public async Task Succeeded(string host, CancellationToken token)
+		{
+			var instance = await Instance(host, token);
+			if (instance is not { ConsecutiveFailures: > 0 } && instance?.LastSuccessAt > DateTime.UtcNow.AddHours(-1))
+				return;
+			await DB.Default.Update<RemoteInstance>()
+				.Match(i => i.Host == host)
+				.Modify(i => i.ConsecutiveFailures, 0)
+				.Modify(i => i.UnavailableUntil, null)
+				.Modify(i => i.LastSuccessAt, DateTime.UtcNow)
+				.Option(o => o.IsUpsert = true)
+				.ExecuteAsync(token);
+			_cache.Remove(Key(host));
+		}
+
+		public async Task Failed(string host, string error, CancellationToken token)
+		{
+			var now = DateTime.UtcNow;
+			var instance = await DB.Default.UpdateAndGet<RemoteInstance>()
+				.Match(i => i.Host == host)
+				.Modify(b => b.Inc(i => i.ConsecutiveFailures, 1))
+				.Modify(i => i.LastFailureAt, now)
+				.Modify(i => i.LastError, error)
+				.Option(o => o.IsUpsert = true)
+				.ExecuteAsync(token);
+			var quarantine = Backoff.HostQuarantine(instance.ConsecutiveFailures, Threshold);
+			if (quarantine > TimeSpan.Zero)
+				await DB.Default.Update<RemoteInstance>().MatchID(instance.ID)
+					.Modify(i => i.UnavailableUntil, now + quarantine)
+					.ExecuteAsync(token);
+			_cache.Remove(Key(host));
+		}
+
+		async Task<RemoteInstance> Instance(string host, CancellationToken token) =>
+			await _cache.GetOrCreateAsync(Key(host), async entry =>
+			{
+				entry.AbsoluteExpirationRelativeToNow = CacheLifetime;
+				return await DB.Default.Find<RemoteInstance>().Match(i => i.Host == host).ExecuteFirstAsync(token);
+			});
+
+		static string Key(string host) => "remote-instance:" + host;
+	}
+}

PrivaPub/Infrastructure/Jobs/JobQueue.cs

@@ -0,0 +1,142 @@
+using MongoDB.Driver;
+using MongoDB.Entities;
+
+using PrivaPub.Models.Jobs;
+
+using System.Collections.Concurrent;
+
+namespace PrivaPub.Infrastructure.Jobs
+{
+	public sealed record JobOutcome(JobResult Result, string Error = default, DateTime? RetryAt = default)
+	{
+		public static readonly JobOutcome Done = new(JobResult.Done);
+		public static JobOutcome Retry(string error) => new(JobResult.Retry, error);
+		public static JobOutcome Dead(string error) => new(JobResult.Dead, error);
+		public static JobOutcome Defer(DateTime until, string error) => new(JobResult.Defer, error, until);
+	}
+
+	public enum JobResult
+	{
+		Done,
+		Retry,
+		Dead,
+		Defer
+	}
+
+	public interface IJobQueue
+	{
+		Task<bool> Enqueue(JobKind kind, string payload, string host, string dedupeKey, CancellationToken token);
+		Task<int> EnqueueMany(IEnumerable<Job> jobs, CancellationToken token);
+		Task<Job> Lease(JobKind kind, IReadOnlyCollection<string> busyHosts, CancellationToken token);
+		Task Finish(Job job, JobOutcome outcome, int maxAttempts, CancellationToken token);
+		Task<long> Reap(CancellationToken token);
+		Task WaitForWork(JobKind kind, TimeSpan poll, CancellationToken token);
+	}
+
+	public class JobQueue : IJobQueue
+	{
+		public static readonly TimeSpan LeaseTime = TimeSpan.FromMinutes(2);
+
+		readonly string _owner = $"{Environment.MachineName}:{Environment.ProcessId}";
+		readonly ConcurrentDictionary<JobKind, SemaphoreSlim> _signals = new();
+
+		public async Task<bool> Enqueue(JobKind kind, string payload, string host, string dedupeKey, CancellationToken token) =>
+			await EnqueueMany(new[] { new Job { Kind = kind, Payload = payload, Host = host, DedupeKey = dedupeKey } }, token) == 1;
+
+		public async Task<int> EnqueueMany(IEnumerable<Job> jobs, CancellationToken token)
+		{
+			var inserted = 0;
+			foreach (var job in jobs)
+			{
+				try
+				{
+					await DB.Default.SaveAsync(job, token);
+					inserted++;
+					Signal(job.Kind);
+				}
+				catch (MongoWriteException ex) when (ex.WriteError?.Category == ServerErrorCategory.DuplicateKey)
+				{
+				}
+			}
+			return inserted;
+		}
+
+		public async Task<Job> Lease(JobKind kind, IReadOnlyCollection<string> busyHosts, CancellationToken token)
+		{
+			var now = DateTime.UtcNow;
+			var busy = busyHosts?.ToList() ?? new List<string>();
+			return await DB.Default.UpdateAndGet<Job>()
+				.Match(j => j.Kind == kind && j.State == JobState.Pending && j.RunAt <= now && !busy.Contains(j.Host))
+				.Modify(j => j.State, JobState.Running)
+				.Modify(j => j.LeasedUntil, now + LeaseTime)
+				.Modify(j => j.LeaseOwner, _owner)
+				.Modify(b => b.Inc(j => j.Attempts, 1))
+				.Option(o => o.Sort = Builders<Job>.Sort.Ascending(j => j.RunAt))
+				.ExecuteAsync(token);
+		}
+
+		public async Task Finish(Job job, JobOutcome outcome, int maxAttempts, CancellationToken token)
+		{
+			var now = DateTime.UtcNow;
+			var update = DB.Default.Update<Job>().MatchID(job.ID)
+				.Modify(j => j.LeasedUntil, null)
+				.Modify(j => j.LeaseOwner, null)
+				.Modify(j => j.LastError, outcome.Error);
+			switch (outcome.Result)
+			{
+				case JobResult.Done:
+					update.Modify(j => j.State, JobState.Done).Modify(j => j.FinishedAt, now);
+					break;
+				case JobResult.Retry when job.Attempts < maxAttempts:
+					update.Modify(j => j.State, JobState.Pending).Modify(j => j.RunAt, now + Backoff.After(job.Attempts));
+					break;
+				case JobResult.Defer:
+					update.Modify(j => j.State, JobState.Pending)
+						.Modify(j => j.RunAt, outcome.RetryAt ?? now + Backoff.After(job.Attempts))
+						.Modify(b => b.Inc(j => j.Attempts, -1));
+					break;
+				default:
+					update.Modify(j => j.State, JobState.Dead).Modify(j => j.FinishedAt, now);
+					break;
+			}
+			await update.ExecuteAsync(token);
+		}
+
+		public async Task<long> Reap(CancellationToken token)
+		{
+			var now = DateTime.UtcNow;
+			var result = await DB.Default.Update<Job>()
+				.Match(j => j.State == JobState.Running && j.LeasedUntil < now)
+				.Modify(j => j.State, JobState.Pending)
+				.Modify(j => j.RunAt, now)
+				.Modify(j => j.LeasedUntil, null)
+				.Modify(j => j.LeaseOwner, null)
+				.ExecuteAsync(token);
+			return result.ModifiedCount;
+		}
+
+		public async Task WaitForWork(JobKind kind, TimeSpan poll, CancellationToken token)
+		{
+			try
+			{
+				await SignalFor(kind).WaitAsync(poll, token);
+			}
+			catch (OperationCanceledException) when (token.IsCancellationRequested)
+			{
+			}
+		}
+
+		void Signal(JobKind kind)
+		{
+			try
+			{
+				SignalFor(kind).Release();
+			}
+			catch (SemaphoreFullException)
+			{
+			}
+		}
+
+		SemaphoreSlim SignalFor(JobKind kind) => _signals.GetOrAdd(kind, _ => new SemaphoreSlim(0, 64));
+	}
+}

PrivaPub/Infrastructure/Jobs/JobWorker.cs

@@ -0,0 +1,137 @@
+using PrivaPub.Models.Jobs;
+
+using System.Collections.Concurrent;
+
+namespace PrivaPub.Infrastructure.Jobs
+{
+	public interface IJobHandler
+	{
+		JobKind Kind { get; }
+		int Concurrency { get; }
+		int MaxAttempts { get; }
+		int PerHostLimit { get; }
+		Task<JobOutcome> Handle(Job job, CancellationToken token);
+	}
+
+	public class JobWorker : BackgroundService
+	{
+		static readonly TimeSpan Poll = TimeSpan.FromSeconds(5);
+		static readonly TimeSpan ReapInterval = TimeSpan.FromSeconds(30);
+
+		readonly IJobQueue _queue;
+		readonly IEnumerable<IJobHandler> _handlers;
+		readonly ILogger<JobWorker> _logger;
+
+		public JobWorker(IJobQueue queue, IEnumerable<IJobHandler> handlers, ILogger<JobWorker> logger)
+		{
+			_queue = queue;
+			_handlers = handlers;
+			_logger = logger;
+		}
+
+		protected override async Task ExecuteAsync(CancellationToken stoppingToken)
+		{
+			var loops = _handlers.SelectMany(handler =>
+			{
+				var inFlight = new ConcurrentDictionary<string, int>(StringComparer.OrdinalIgnoreCase);
+				return Enumerable.Range(0, handler.Concurrency).Select(_ => Run(handler, inFlight, stoppingToken));
+			}).Append(Reap(stoppingToken));
+			await Task.WhenAll(loops);
+		}
+
+		async Task Run(IJobHandler handler, ConcurrentDictionary<string, int> inFlight, CancellationToken stoppingToken)
+		{
+			while (!stoppingToken.IsCancellationRequested)
+			{
+				Job job;
+				try
+				{
+					var busy = inFlight.Where(h => h.Value >= handler.PerHostLimit).Select(h => h.Key).ToList();
+					job = await _queue.Lease(handler.Kind, busy, stoppingToken);
+				}
+				catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
+				{
+					return;
+				}
+				catch (Exception ex)
+				{
+					_logger.LogError(ex, "{Worker} could not lease a {Kind} job", nameof(JobWorker), handler.Kind);
+					await Delay(Poll, stoppingToken);
+					continue;
+				}
+
+				if (job == default)
+				{
+					await _queue.WaitForWork(handler.Kind, Poll, stoppingToken);
+					continue;
+				}
+
+				var host = job.Host ?? string.Empty;
+				inFlight.AddOrUpdate(host, 1, (_, count) => count + 1);
+				try
+				{
+					JobOutcome outcome;
+					try
+					{
+						outcome = await handler.Handle(job, stoppingToken);
+					}
+					catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
+					{
+						return;
+					}
+					catch (Exception ex)
+					{
+						_logger.LogError(ex, "{Kind} job {Id} threw", handler.Kind, job.ID);
+						outcome = JobOutcome.Retry(ex.GetType().Name);
+					}
+
+					if (outcome.Result != JobResult.Done && job.Attempts >= handler.MaxAttempts && outcome.Result == JobResult.Retry)
+						_logger.LogWarning("{Kind} job {Id} for {Host} is dead after {Attempts} attempts: {Error}",
+							handler.Kind, job.ID, job.Host, job.Attempts, outcome.Error);
+					await _queue.Finish(job, outcome, handler.MaxAttempts, CancellationToken.None);
+				}
+				catch (Exception ex)
+				{
+					_logger.LogError(ex, "{Kind} job {Id} could not be finished; its lease will expire", handler.Kind, job.ID);
+				}
+				finally
+				{
+					inFlight.AddOrUpdate(host, 0, (_, count) => count - 1);
+				}
+			}
+		}
+
+		async Task Reap(CancellationToken stoppingToken)
+		{
+			while (!stoppingToken.IsCancellationRequested)
+			{
+				try
+				{
+					var reaped = await _queue.Reap(stoppingToken);
+					if (reaped > 0)
+						_logger.LogWarning("{Worker} returned {Count} expired leases to the queue", nameof(JobWorker), reaped);
+				}
+				catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
+				{
+					return;
+				}
+				catch (Exception ex)
+				{
+					_logger.LogError(ex, "{Worker} reaper failed", nameof(JobWorker));
+				}
+				await Delay(ReapInterval, stoppingToken);
+			}
+		}
+
+		static async Task Delay(TimeSpan delay, CancellationToken token)
+		{
+			try
+			{
+				await Task.Delay(delay, token);
+			}
+			catch (OperationCanceledException)
+			{
+			}
+		}
+	}
+}