Deliveries run on a Mongo job queue with leases, backoff and per-host limits

The single serial DeliveryWorker is replaced by Infrastructure/Jobs:
- Job rows are leased with one FindOneAndUpdate (oldest RunAt first, a
  two-minute lease) and a reaper returns expired leases every 30 s;
- enqueueing wakes the workers, which otherwise poll every five seconds;
- delivery runs eight at a time with at most two per host, so a slow or
  dead server holds two slots, not the queue;
- a failure waits n^4 + 15 + jitter seconds (Mastodon's curve) for up to
  16 attempts; a 4xx other than 408/429 is final, a 429 honours
  Retry-After;
- RemoteInstance is a per-host circuit breaker: ten consecutive failures
  quarantine a host for an hour, doubling to a week, and its jobs wait
  without spending attempts;
- a delivery is queued once per activity and inbox (unique DedupeKey), and
  finished jobs expire after seven days (TTL on FinishedAt).

Migration _004 moves pending Delivery rows into jobs and marks them
abandoned, so a rollback to the old worker cannot send them twice.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012CzABvBkbcFqoHdmi8b9WB
This commit is contained in:
thepraandClaude Opus 5.5 committed 2026-10-01 11:09:35 +02:00
1 parent 934b6fe687
commit d1a91c40c4
15 files changed
+704 -99

No files matched your search

+10
View File
@@ -16,6 +16,7 @@ namespace PrivaPub.Tests.Support
{
readonly WebApplication _app;
readonly ConcurrentDictionary<string, string> _documents = new();
readonly ConcurrentDictionary<string, (int Status, TimeSpan Delay)> _answers = new();
public int Port { get; }
public string A => $"http://127.0.0.1:{Port}";
@@ -38,6 +39,13 @@ namespace PrivaPub.Tests.Support
{
peer.Requests.Enqueue(new(context.Request.Method, context.Request.Path, context.Request.Headers["Signature"].ToString()));
var key = context.Request.Path.Value;
if (peer._answers.TryGetValue(key, out var answer))
{
if (answer.Delay > TimeSpan.Zero)
await Task.Delay(answer.Delay);
context.Response.StatusCode = answer.Status;
return;
}
if (!peer._documents.TryGetValue(key, out var document))
{
context.Response.StatusCode = StatusCodes.Status404NotFound;
@@ -53,6 +61,8 @@ namespace PrivaPub.Tests.Support
public void Serve(string path, string json) => _documents[path] = json;
public void Answer(string path, int status, TimeSpan delay = default) => _answers[path] = (status, delay);
public static FederationHttp Http(IMemoryCache cache = default)
{
var options = new FederationOptions { AllowPrivateNetworks = true, AllowPlainHttp = true };