Deliveries run on a Mongo job queue with leases, backoff and per-host limits
The single serial DeliveryWorker is replaced by Infrastructure/Jobs: - Job rows are leased with one FindOneAndUpdate (oldest RunAt first, a two-minute lease) and a reaper returns expired leases every 30 s; - enqueueing wakes the workers, which otherwise poll every five seconds; - delivery runs eight at a time with at most two per host, so a slow or dead server holds two slots, not the queue; - a failure waits n^4 + 15 + jitter seconds (Mastodon's curve) for up to 16 attempts; a 4xx other than 408/429 is final, a 429 honours Retry-After; - RemoteInstance is a per-host circuit breaker: ten consecutive failures quarantine a host for an hour, doubling to a week, and its jobs wait without spending attempts; - a delivery is queued once per activity and inbox (unique DedupeKey), and finished jobs expire after seven days (TTL on FinishedAt). Migration _004 moves pending Delivery rows into jobs and marks them abandoned, so a rollback to the old worker cannot send them twice. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_012CzABvBkbcFqoHdmi8b9WB
This commit is contained in:
1 parent
934b6fe687
commit
d1a91c40c4
15 files changed
+704
-99
No files matched your search
@@ -32,6 +32,7 @@ namespace PrivaPub.Tests.Support
|
||||
var connection = Environment.GetEnvironmentVariable("PRIVAPUB_TEST_MONGO") ?? "mongodb://127.0.0.1:27017";
|
||||
await DB.InitAsync(Database, MongoClientSettings.FromConnectionString(connection));
|
||||
EntityMaps.Warm();
|
||||
await Indexes.Create();
|
||||
}
|
||||
|
||||
public async ValueTask DisposeAsync()
|
||||
|
||||
@@ -16,6 +16,7 @@ namespace PrivaPub.Tests.Support
|
||||
{
|
||||
readonly WebApplication _app;
|
||||
readonly ConcurrentDictionary<string, string> _documents = new();
|
||||
readonly ConcurrentDictionary<string, (int Status, TimeSpan Delay)> _answers = new();
|
||||
|
||||
public int Port { get; }
|
||||
public string A => $"http://127.0.0.1:{Port}";
|
||||
@@ -38,6 +39,13 @@ namespace PrivaPub.Tests.Support
|
||||
{
|
||||
peer.Requests.Enqueue(new(context.Request.Method, context.Request.Path, context.Request.Headers["Signature"].ToString()));
|
||||
var key = context.Request.Path.Value;
|
||||
if (peer._answers.TryGetValue(key, out var answer))
|
||||
{
|
||||
if (answer.Delay > TimeSpan.Zero)
|
||||
await Task.Delay(answer.Delay);
|
||||
context.Response.StatusCode = answer.Status;
|
||||
return;
|
||||
}
|
||||
if (!peer._documents.TryGetValue(key, out var document))
|
||||
{
|
||||
context.Response.StatusCode = StatusCodes.Status404NotFound;
|
||||
@@ -53,6 +61,8 @@ namespace PrivaPub.Tests.Support
|
||||
|
||||
public void Serve(string path, string json) => _documents[path] = json;
|
||||
|
||||
public void Answer(string path, int status, TimeSpan delay = default) => _answers[path] = (status, delay);
|
||||
|
||||
public static FederationHttp Http(IMemoryCache cache = default)
|
||||
{
|
||||
var options = new FederationOptions { AllowPrivateNetworks = true, AllowPlainHttp = true };
|
||||
|
||||
Reference in new issue
Block a user