T1: tests stop sharing state they don't own
- JobQueue takes an optional scope, so a test's worker leases and reaps only its own jobs. - The dead-host delivery test runs alone (Exclusive), on its own jobs, and cleans up the breaker rows it trips; the breaker has tests of its own on unique hosts. - Index and migration tests run alone: they drop indexes and rewrite every post. - DomainBlocks.Load replaces reflection and a database-wide block in tests. - Harness.Outgoing sees only deliveries queued since the harness started: Peer ports are reused within a run, which made the circle test flaky. - Two pure-logic tests leave Mongo-gated classes, so CI runs them. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01ELjqpznMFMNrJoJUj6K5p2
This commit is contained in:
1 parent
7c6fe80f1b
commit
00b2685cf4
14 files changed
+280
-101
No files matched your search
@@ -316,7 +316,7 @@ cd /var/www/privapub.thepra.dev && sudo -u www-data ASPNETCORE_ENVIRONMENT=Produ
|
|||||||
stored as an edit date on every remote post. Write `(T?)null`. It has shipped four times (`MastodonParams.Bool/Int`,
|
stored as an edit date on every remote post. Write `(T?)null`. It has shipped four times (`MastodonParams.Bool/Int`,
|
||||||
`NoteParser.Int`, `NoteParser.Time`).
|
`NoteParser.Int`, `NoteParser.Time`).
|
||||||
- ActivityPub output is built with `System.Text.Json.Nodes` in `ActivityPubRenderer`, not typed models. Inbound
|
- ActivityPub output is built with `System.Text.Json.Nodes` in `ActivityPubRenderer`, not typed models. Inbound
|
||||||
documents are read through `InboxService.Id`/`Value`, which handle string, object and array.
|
documents are read through `ActivityJson.Id`/`Value`, which handle string, object and array.
|
||||||
- Libraries chosen for the roadmap: HtmlSanitizer, Markdig (`DisableHtml`), NSign (RFC 9421 inbound), OpenIddict +
|
- Libraries chosen for the roadmap: HtmlSanitizer, Markdig (`DisableHtml`), NSign (RFC 9421 inbound), OpenIddict +
|
||||||
OpenIddict.MongoDb, NetVips, Blurhash.Core, FFMpegCore. No ImageSharp (licence key enforced), no MassTransit.
|
OpenIddict.MongoDb, NetVips, Blurhash.Core, FFMpegCore. No ImageSharp (licence key enforced), no MassTransit.
|
||||||
|
|
||||||
@@ -327,7 +327,16 @@ without `PRIVAPUB_TEST_MONGOD=1`. CI runs the unit tests only (the box's mongods
|
|||||||
|
|
||||||
- `Support/Peer` is an in-process HTTP server answering on two origins (`127.0.0.1` and `localhost`), so origin rules
|
- `Support/Peer` is an in-process HTTP server answering on two origins (`127.0.0.1` and `localhost`), so origin rules
|
||||||
can be tested; `Support/RemoteActor` signs real deliveries with its own key.
|
can be tested; `Support/RemoteActor` signs real deliveries with its own key.
|
||||||
- Inbox scenarios go through `InboxService.Receive` with a signed request, not through the private handlers.
|
- Inbox scenarios go through `InboxReceiver.Receive` with a signed request, not through the private handlers.
|
||||||
|
- All test classes share one database and run in parallel, so a test touches only rows it made:
|
||||||
|
- random names and GUIDs;
|
||||||
|
- `Harness.Outgoing` sees only deliveries queued since that harness started, because Peer ports are reused;
|
||||||
|
- a worker gets a scoped `new JobQueue(j => ...)` so it never leases another test's jobs;
|
||||||
|
- domain blocks are set with `DomainBlocks.Load`, never written to the database.
|
||||||
|
|
||||||
|
A test that must change something database-wide (drop indexes, run a migration over every post, let deliveries to
|
||||||
|
`localhost` fail and trip its breaker) goes in `[Xunit.Collection(nameof(Exclusive))]`, which runs alone, and
|
||||||
|
cleans up after itself. Pure logic belongs in an unconditional unit test, not in a Mongo-gated class.
|
||||||
|
|
||||||
Beyond the tests, verify by building, running locally, and exercising:
|
Beyond the tests, verify by building, running locally, and exercising:
|
||||||
- the client API (sign up, create an avatar, a group, a post);
|
- the client API (sign up, create an avatar, a group, a post);
|
||||||
@@ -337,7 +346,7 @@ Interop is checked against real servers, starting with the workstation's own pas
|
|||||||
|
|
||||||
```bash
|
```bash
|
||||||
DOTNET=~/.dotnet/dotnet tools/pasture/run.sh up # podman: PrivaPub + the latest GoToSocial + Mongo, behind Caddy
|
DOTNET=~/.dotnet/dotnet tools/pasture/run.sh up # podman: PrivaPub + the latest GoToSocial + Mongo, behind Caddy
|
||||||
tools/pasture/interop.sh # 25 checks, each side driven through its own Mastodon API
|
tools/pasture/interop.sh # 33 checks, each side driven through its own Mastodon API
|
||||||
tools/pasture/run.sh down
|
tools/pasture/run.sh down
|
||||||
```
|
```
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,37 @@
|
|||||||
|
using Microsoft.Extensions.Logging.Abstractions;
|
||||||
|
|
||||||
|
using PrivaPub.Federation.Moderation;
|
||||||
|
using PrivaPub.Models.Federation;
|
||||||
|
|
||||||
|
namespace PrivaPub.Tests.Federation
|
||||||
|
{
|
||||||
|
public class DomainBlocksTests
|
||||||
|
{
|
||||||
|
[Fact]
|
||||||
|
public void A_block_covers_subdomains_but_not_lookalikes()
|
||||||
|
{
|
||||||
|
var blocks = new DomainBlocks(NullLogger<DomainBlocks>.Instance);
|
||||||
|
blocks.Load(new[] { new DomainBlock { Domain = "evil.example", Severity = DomainBlockSeverity.Silence } });
|
||||||
|
|
||||||
|
Assert.NotNull(blocks.Find("evil.example"));
|
||||||
|
Assert.NotNull(blocks.Find("A.Evil.Example."));
|
||||||
|
Assert.Null(blocks.Find("notevil.example"));
|
||||||
|
Assert.False(blocks.IsSuspended("evil.example"));
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public void A_suspension_is_found_from_any_subdomain()
|
||||||
|
{
|
||||||
|
var blocks = new DomainBlocks(NullLogger<DomainBlocks>.Instance);
|
||||||
|
blocks.Load(new[]
|
||||||
|
{
|
||||||
|
new DomainBlock { Domain = " Spam.Example. ", Severity = DomainBlockSeverity.Suspend },
|
||||||
|
new DomainBlock { Domain = "", Severity = DomainBlockSeverity.Suspend }
|
||||||
|
});
|
||||||
|
|
||||||
|
Assert.True(blocks.IsSuspended("deep.inside.spam.example"));
|
||||||
|
Assert.False(blocks.IsSuspended("example"));
|
||||||
|
Assert.Null(blocks.Find(default));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -269,10 +269,7 @@ namespace PrivaPub.Tests.Federation
|
|||||||
var token = TestContext.Current.CancellationToken;
|
var token = TestContext.Current.CancellationToken;
|
||||||
var alice = await LocalAvatar("alice");
|
var alice = await LocalAvatar("alice");
|
||||||
var bob = new RemoteActor(_peer, "bob", _peer.B);
|
var bob = new RemoteActor(_peer, "bob", _peer.B);
|
||||||
await DB.Default.SaveAsync(new DomainBlock { Domain = "localhost", Severity = DomainBlockSeverity.Suspend }, token);
|
_blocks.Load(new[] { new DomainBlock { Domain = "localhost", Severity = DomainBlockSeverity.Suspend } });
|
||||||
try
|
|
||||||
{
|
|
||||||
await _blocks.Reload(token);
|
|
||||||
var before = _peer.Requests.Count;
|
var before = _peer.Requests.Count;
|
||||||
|
|
||||||
var result = await Deliver(bob, $"/peasants/{alice.UserName}/mouth", DirectCreate(bob, alice.Uri));
|
var result = await Deliver(bob, $"/peasants/{alice.UserName}/mouth", DirectCreate(bob, alice.Uri));
|
||||||
@@ -281,27 +278,6 @@ namespace PrivaPub.Tests.Federation
|
|||||||
Assert.Equal(before, _peer.Requests.Count);
|
Assert.Equal(before, _peer.Requests.Count);
|
||||||
Assert.False(await DB.Default.Find<Post>().Match(p => p.ActorURI == bob.Id).ExecuteAnyAsync(token));
|
Assert.False(await DB.Default.Find<Post>().Match(p => p.ActorURI == bob.Id).ExecuteAnyAsync(token));
|
||||||
}
|
}
|
||||||
finally
|
|
||||||
{
|
|
||||||
await DB.Default.DeleteAsync<DomainBlock>(b => b.Domain == "localhost");
|
|
||||||
await _blocks.Reload(token);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
[Fact]
|
|
||||||
public void A_block_covers_subdomains_but_not_lookalikes()
|
|
||||||
{
|
|
||||||
var blocks = new DomainBlocks(NullLogger<DomainBlocks>.Instance);
|
|
||||||
typeof(DomainBlocks).GetField("_blocks", System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Instance)!
|
|
||||||
.SetValue(blocks, new Dictionary<string, DomainBlock> { ["evil.example"] = new() { Domain = "evil.example", Severity = DomainBlockSeverity.Silence } });
|
|
||||||
typeof(DomainBlocks).GetField("_loadedAt", System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Instance)!
|
|
||||||
.SetValue(blocks, DateTime.UtcNow);
|
|
||||||
|
|
||||||
Assert.NotNull(blocks.Find("evil.example"));
|
|
||||||
Assert.NotNull(blocks.Find("A.Evil.Example."));
|
|
||||||
Assert.Null(blocks.Find("notevil.example"));
|
|
||||||
Assert.False(blocks.IsSuspended("evil.example"));
|
|
||||||
}
|
|
||||||
|
|
||||||
[Fact]
|
[Fact]
|
||||||
public async Task A_bad_signature_is_a_401()
|
public async Task A_bad_signature_is_a_401()
|
||||||
|
|||||||
@@ -0,0 +1,36 @@
|
|||||||
|
using PrivaPub.Federation.Objects;
|
||||||
|
|
||||||
|
using System.Text.Json.Nodes;
|
||||||
|
|
||||||
|
namespace PrivaPub.Tests.Federation
|
||||||
|
{
|
||||||
|
public class ObjectRecordsTests
|
||||||
|
{
|
||||||
|
[Fact]
|
||||||
|
public void An_object_larger_than_the_cap_keeps_only_its_hash()
|
||||||
|
{
|
||||||
|
var big = new JsonObject { ["content"] = new string('x', ObjectRecords.MaxRawBytes + 1) };
|
||||||
|
|
||||||
|
var (text, hash, bytes, truncated) = ObjectRecords.Capture(big);
|
||||||
|
|
||||||
|
Assert.Null(text);
|
||||||
|
Assert.True(truncated);
|
||||||
|
Assert.True(bytes > ObjectRecords.MaxRawBytes);
|
||||||
|
Assert.Equal(64, hash.Length);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public void A_small_object_is_kept_whole_with_its_hash()
|
||||||
|
{
|
||||||
|
var small = new JsonObject { ["type"] = "Note", ["content"] = "hi" };
|
||||||
|
|
||||||
|
var (text, hash, bytes, truncated) = ObjectRecords.Capture(small);
|
||||||
|
|
||||||
|
Assert.Equal(small.ToJsonString(), text);
|
||||||
|
Assert.False(truncated);
|
||||||
|
Assert.Equal(text.Length, bytes);
|
||||||
|
Assert.Equal(64, hash.Length);
|
||||||
|
Assert.Equal(default, ObjectRecords.Capture(default));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -68,18 +68,5 @@ namespace PrivaPub.Tests.Federation
|
|||||||
Assert.Equal("<p>hi again</p>", JsonNode.Parse(Assert.Single(record.Revisions).Raw)!["content"]!.GetValue<string>());
|
Assert.Equal("<p>hi again</p>", JsonNode.Parse(Assert.Single(record.Revisions).Raw)!["content"]!.GetValue<string>());
|
||||||
Assert.True(await DB.Default.Find<Job>().Match(j => j.Kind == JobKind.DescribeInstance && j.Payload == record.Host).ExecuteAnyAsync(token));
|
Assert.True(await DB.Default.Find<Job>().Match(j => j.Kind == JobKind.DescribeInstance && j.Payload == record.Host).ExecuteAnyAsync(token));
|
||||||
}
|
}
|
||||||
|
|
||||||
[Fact]
|
|
||||||
public void An_object_larger_than_the_cap_keeps_only_its_hash()
|
|
||||||
{
|
|
||||||
var big = new JsonObject { ["content"] = new string('x', ObjectRecords.MaxRawBytes + 1) };
|
|
||||||
|
|
||||||
var (text, hash, bytes, truncated) = ObjectRecords.Capture(big);
|
|
||||||
|
|
||||||
Assert.Null(text);
|
|
||||||
Assert.True(truncated);
|
|
||||||
Assert.True(bytes > ObjectRecords.MaxRawBytes);
|
|
||||||
Assert.Equal(64, hash.Length);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -0,0 +1,95 @@
|
|||||||
|
using Microsoft.Extensions.Caching.Memory;
|
||||||
|
using Microsoft.Extensions.Logging.Abstractions;
|
||||||
|
|
||||||
|
using MongoDB.Entities;
|
||||||
|
|
||||||
|
using PrivaPub.Federation.Actors;
|
||||||
|
using PrivaPub.Federation.Outbox;
|
||||||
|
using PrivaPub.Infrastructure.Jobs;
|
||||||
|
using PrivaPub.Models;
|
||||||
|
using PrivaPub.Models.Jobs;
|
||||||
|
using PrivaPub.Models.User;
|
||||||
|
using PrivaPub.StaticServices;
|
||||||
|
using PrivaPub.Tests.Support;
|
||||||
|
|
||||||
|
using System.Text.Json.Nodes;
|
||||||
|
|
||||||
|
namespace PrivaPub.Tests.Infrastructure
|
||||||
|
{
|
||||||
|
[Xunit.Collection(nameof(Exclusive))]
|
||||||
|
[Trait("Category", "Integration")]
|
||||||
|
public sealed class DeliveryWorkerTests : IAsyncLifetime
|
||||||
|
{
|
||||||
|
static readonly string[] PeerHosts = { "localhost", "127.0.0.1" };
|
||||||
|
|
||||||
|
Peer _peer;
|
||||||
|
|
||||||
|
public async ValueTask InitializeAsync()
|
||||||
|
{
|
||||||
|
Assert.SkipUnless(MongoFixture.Enabled, MongoFixture.Skip);
|
||||||
|
_peer = await Peer.Start();
|
||||||
|
await DB.Default.DeleteAsync<RemoteInstance>(i => PeerHosts.Contains(i.Host));
|
||||||
|
}
|
||||||
|
|
||||||
|
public async ValueTask DisposeAsync()
|
||||||
|
{
|
||||||
|
if (_peer == default)
|
||||||
|
return;
|
||||||
|
await _peer.DisposeAsync();
|
||||||
|
await DB.Default.DeleteAsync<RemoteInstance>(i => PeerHosts.Contains(i.Host));
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task A_dead_host_does_not_hold_up_deliveries_to_live_ones()
|
||||||
|
{
|
||||||
|
var token = TestContext.Current.CancellationToken;
|
||||||
|
var (privateKey, publicKey) = Keys.NewKeyPair();
|
||||||
|
var avatar = new Avatar { UserName = $"sender{Guid.NewGuid():N}"[..20], PrivateKey = privateKey, PublicKey = publicKey };
|
||||||
|
await DB.Default.SaveAsync(avatar, token);
|
||||||
|
var local = new LocalActorService(new DbEntities(), new StaticOptions<AppConfiguration>(new AppConfiguration { BackendBaseAddress = "https://privapub.test" }));
|
||||||
|
var sender = local.FromAvatar(avatar);
|
||||||
|
var run = $"https://privapub.test/a/{Guid.NewGuid():N}/";
|
||||||
|
var queue = new JobQueue(j => j.DedupeKey.StartsWith(run));
|
||||||
|
var delivery = new DeliveryService(new DbEntities(), queue);
|
||||||
|
_peer.Answer("/dead/inbox", 503, TimeSpan.FromMilliseconds(300));
|
||||||
|
_peer.Answer("/live/inbox", 202);
|
||||||
|
|
||||||
|
var deadInboxes = Enumerable.Range(0, 60).Select(i => $"{_peer.B}/dead/inbox?{i}");
|
||||||
|
await delivery.Enqueue(sender, deadInboxes, new JsonObject { ["id"] = run + "dead", ["type"] = "Create" }, token);
|
||||||
|
await delivery.Enqueue(sender, new[] { $"{_peer.A}/live/inbox" }, new JsonObject { ["id"] = run + "live", ["type"] = "Create" }, token);
|
||||||
|
|
||||||
|
var handler = new DeliveryJobHandler(local, Peer.Http(), new HostCircuitBreaker(new MemoryCache(new MemoryCacheOptions())), NullLogger<DeliveryJobHandler>.Instance);
|
||||||
|
using var worker = new JobWorker(queue, new IJobHandler[] { handler }, NullLogger<JobWorker>.Instance);
|
||||||
|
await worker.StartAsync(token);
|
||||||
|
try
|
||||||
|
{
|
||||||
|
await Until(() => Task.FromResult(_peer.Requests.Any(r => r.Path == "/live/inbox")), token);
|
||||||
|
var deadSoFar = _peer.Requests.Count(r => r.Path == "/dead/inbox");
|
||||||
|
var live = Assert.Single(_peer.Requests, r => r.Path == "/live/inbox");
|
||||||
|
Assert.Contains($"keyId=\"{sender.KeyId}\"", live.Signature);
|
||||||
|
Assert.True(deadSoFar < 30, $"{deadSoFar} dead deliveries were tried before the live one");
|
||||||
|
|
||||||
|
await Until(() => DB.Default.Find<Job>().Match(j => j.DedupeKey.StartsWith(run + "dead") && j.State == JobState.Pending && j.Attempts > 0).ExecuteAnyAsync(token), token);
|
||||||
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
await worker.StopAsync(CancellationToken.None);
|
||||||
|
}
|
||||||
|
|
||||||
|
var retried = await DB.Default.Find<Job>().Match(j => j.DedupeKey.StartsWith(run + "dead") && j.Attempts > 0 && j.State == JobState.Pending).ExecuteFirstAsync(token);
|
||||||
|
Assert.Equal("503", retried.LastError?.Split(' ')[0]);
|
||||||
|
Assert.True(retried.RunAt > DateTime.UtcNow);
|
||||||
|
Assert.True((await DB.Default.Find<RemoteInstance>().Match(i => i.Host == "localhost").ExecuteFirstAsync(token))?.ConsecutiveFailures > 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
static async Task Until(Func<Task<bool>> condition, CancellationToken token)
|
||||||
|
{
|
||||||
|
var deadline = DateTime.UtcNow.AddSeconds(30);
|
||||||
|
while (!await condition())
|
||||||
|
{
|
||||||
|
Assert.True(DateTime.UtcNow < deadline, "timed out");
|
||||||
|
await Task.Delay(50, token);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,67 @@
|
|||||||
|
using Microsoft.Extensions.Caching.Memory;
|
||||||
|
|
||||||
|
using MongoDB.Entities;
|
||||||
|
|
||||||
|
using PrivaPub.Infrastructure.Jobs;
|
||||||
|
using PrivaPub.Models.Jobs;
|
||||||
|
using PrivaPub.Tests.Support;
|
||||||
|
|
||||||
|
namespace PrivaPub.Tests.Infrastructure
|
||||||
|
{
|
||||||
|
[Trait("Category", "Integration")]
|
||||||
|
public sealed class HostCircuitBreakerTests : IAsyncLifetime
|
||||||
|
{
|
||||||
|
readonly string _host = $"breaker{Guid.NewGuid():N}.example";
|
||||||
|
|
||||||
|
public ValueTask InitializeAsync()
|
||||||
|
{
|
||||||
|
Assert.SkipUnless(MongoFixture.Enabled, MongoFixture.Skip);
|
||||||
|
return ValueTask.CompletedTask;
|
||||||
|
}
|
||||||
|
|
||||||
|
public async ValueTask DisposeAsync()
|
||||||
|
{
|
||||||
|
if (MongoFixture.Enabled)
|
||||||
|
await DB.Default.DeleteAsync<RemoteInstance>(i => i.Host == _host);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task A_host_is_quarantined_only_past_the_threshold_and_freed_by_a_success()
|
||||||
|
{
|
||||||
|
var token = TestContext.Current.CancellationToken;
|
||||||
|
var breaker = new HostCircuitBreaker(new MemoryCache(new MemoryCacheOptions()));
|
||||||
|
|
||||||
|
for (var i = 1; i < HostCircuitBreaker.Threshold; i++)
|
||||||
|
await breaker.Failed(_host, "503", token);
|
||||||
|
Assert.Null(await breaker.UnavailableUntil(_host, token));
|
||||||
|
|
||||||
|
await breaker.Failed(_host, "503", token);
|
||||||
|
var until = await breaker.UnavailableUntil(_host, token);
|
||||||
|
Assert.NotNull(until);
|
||||||
|
Assert.InRange(until.Value, DateTime.UtcNow.AddMinutes(55), DateTime.UtcNow.AddMinutes(65));
|
||||||
|
var instance = await DB.Default.Find<RemoteInstance>().Match(i => i.Host == _host).ExecuteFirstAsync(token);
|
||||||
|
Assert.Equal(HostCircuitBreaker.Threshold, instance.ConsecutiveFailures);
|
||||||
|
Assert.Equal("503", instance.LastError);
|
||||||
|
|
||||||
|
await breaker.Succeeded(_host, token);
|
||||||
|
Assert.Null(await breaker.UnavailableUntil(_host, token));
|
||||||
|
instance = await DB.Default.Find<RemoteInstance>().Match(i => i.Host == _host).ExecuteFirstAsync(token);
|
||||||
|
Assert.Equal(0, instance.ConsecutiveFailures);
|
||||||
|
Assert.NotNull(instance.LastSuccessAt);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task A_healthy_host_is_written_at_most_once_an_hour()
|
||||||
|
{
|
||||||
|
var token = TestContext.Current.CancellationToken;
|
||||||
|
var breaker = new HostCircuitBreaker(new MemoryCache(new MemoryCacheOptions()));
|
||||||
|
|
||||||
|
await breaker.Succeeded(_host, token);
|
||||||
|
var first = (await DB.Default.Find<RemoteInstance>().Match(i => i.Host == _host).ExecuteFirstAsync(token)).LastSuccessAt;
|
||||||
|
await breaker.Succeeded(_host, token);
|
||||||
|
var second = (await DB.Default.Find<RemoteInstance>().Match(i => i.Host == _host).ExecuteFirstAsync(token)).LastSuccessAt;
|
||||||
|
|
||||||
|
Assert.Equal(first, second);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -12,6 +12,7 @@ using PrivaPub.Tests.Support;
|
|||||||
|
|
||||||
namespace PrivaPub.Tests.Infrastructure
|
namespace PrivaPub.Tests.Infrastructure
|
||||||
{
|
{
|
||||||
|
[Xunit.Collection(nameof(Exclusive))]
|
||||||
[Trait("Category", "Integration")]
|
[Trait("Category", "Integration")]
|
||||||
public class IndexTests
|
public class IndexTests
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -1,19 +1,9 @@
|
|||||||
using Microsoft.Extensions.Caching.Memory;
|
|
||||||
using Microsoft.Extensions.Logging.Abstractions;
|
|
||||||
|
|
||||||
using MongoDB.Entities;
|
using MongoDB.Entities;
|
||||||
|
|
||||||
using PrivaPub.Federation.Actors;
|
|
||||||
using PrivaPub.Federation.Outbox;
|
|
||||||
using PrivaPub.Infrastructure.Jobs;
|
using PrivaPub.Infrastructure.Jobs;
|
||||||
using PrivaPub.Models;
|
|
||||||
using PrivaPub.Models.Jobs;
|
using PrivaPub.Models.Jobs;
|
||||||
using PrivaPub.Models.User;
|
|
||||||
using PrivaPub.StaticServices;
|
|
||||||
using PrivaPub.Tests.Support;
|
using PrivaPub.Tests.Support;
|
||||||
|
|
||||||
using System.Text.Json.Nodes;
|
|
||||||
|
|
||||||
namespace PrivaPub.Tests.Infrastructure
|
namespace PrivaPub.Tests.Infrastructure
|
||||||
{
|
{
|
||||||
public class BackoffTests
|
public class BackoffTests
|
||||||
@@ -38,19 +28,13 @@ namespace PrivaPub.Tests.Infrastructure
|
|||||||
[Trait("Category", "Integration")]
|
[Trait("Category", "Integration")]
|
||||||
public sealed class JobQueueTests : IAsyncLifetime
|
public sealed class JobQueueTests : IAsyncLifetime
|
||||||
{
|
{
|
||||||
Peer _peer;
|
public ValueTask InitializeAsync()
|
||||||
|
|
||||||
public async ValueTask InitializeAsync()
|
|
||||||
{
|
{
|
||||||
Assert.SkipUnless(MongoFixture.Enabled, MongoFixture.Skip);
|
Assert.SkipUnless(MongoFixture.Enabled, MongoFixture.Skip);
|
||||||
_peer = await Peer.Start();
|
return ValueTask.CompletedTask;
|
||||||
}
|
}
|
||||||
|
|
||||||
public async ValueTask DisposeAsync()
|
public ValueTask DisposeAsync() => ValueTask.CompletedTask;
|
||||||
{
|
|
||||||
if (_peer != default)
|
|
||||||
await _peer.DisposeAsync();
|
|
||||||
}
|
|
||||||
|
|
||||||
static Job NewJob(string host = default, string dedupe = default) => new()
|
static Job NewJob(string host = default, string dedupe = default) => new()
|
||||||
{
|
{
|
||||||
@@ -102,44 +86,9 @@ namespace PrivaPub.Tests.Infrastructure
|
|||||||
var job = new Job { Kind = (JobKind)98, State = JobState.Running, LeasedUntil = DateTime.UtcNow.AddMinutes(-1) };
|
var job = new Job { Kind = (JobKind)98, State = JobState.Running, LeasedUntil = DateTime.UtcNow.AddMinutes(-1) };
|
||||||
await DB.Default.SaveAsync(job, token);
|
await DB.Default.SaveAsync(job, token);
|
||||||
|
|
||||||
await new JobQueue().Reap(token);
|
await new JobQueue(j => j.ID == job.ID).Reap(token);
|
||||||
|
|
||||||
Assert.Equal(JobState.Pending, (await DB.Default.Find<Job>().OneAsync(job.ID, token)).State);
|
Assert.Equal(JobState.Pending, (await DB.Default.Find<Job>().OneAsync(job.ID, token)).State);
|
||||||
}
|
}
|
||||||
|
|
||||||
[Fact]
|
|
||||||
public async Task A_dead_host_does_not_hold_up_deliveries_to_live_ones()
|
|
||||||
{
|
|
||||||
var token = TestContext.Current.CancellationToken;
|
|
||||||
var (privateKey, publicKey) = Keys.NewKeyPair();
|
|
||||||
var avatar = new Avatar { UserName = $"sender{Guid.NewGuid():N}"[..20], PrivateKey = privateKey, PublicKey = publicKey };
|
|
||||||
await DB.Default.SaveAsync(avatar, token);
|
|
||||||
var local = new LocalActorService(new DbEntities(), new StaticOptions<AppConfiguration>(new AppConfiguration { BackendBaseAddress = "https://privapub.test" }));
|
|
||||||
var sender = local.FromAvatar(avatar);
|
|
||||||
var queue = new JobQueue();
|
|
||||||
var delivery = new DeliveryService(new DbEntities(), queue);
|
|
||||||
_peer.Answer("/dead/inbox", 503, TimeSpan.FromMilliseconds(300));
|
|
||||||
_peer.Answer("/live/inbox", 202);
|
|
||||||
|
|
||||||
var deadInboxes = Enumerable.Range(0, 60).Select(i => $"{_peer.B}/dead/inbox?{i}");
|
|
||||||
await delivery.Enqueue(sender, deadInboxes, new JsonObject { ["id"] = $"https://privapub.test/a/{Guid.NewGuid():N}", ["type"] = "Create" }, token);
|
|
||||||
await delivery.Enqueue(sender, new[] { $"{_peer.A}/live/inbox" }, new JsonObject { ["id"] = $"https://privapub.test/a/{Guid.NewGuid():N}", ["type"] = "Create" }, token);
|
|
||||||
|
|
||||||
var handler = new DeliveryJobHandler(local, Peer.Http(), new HostCircuitBreaker(new MemoryCache(new MemoryCacheOptions())), NullLogger<DeliveryJobHandler>.Instance);
|
|
||||||
using var worker = new JobWorker(queue, new IJobHandler[] { handler }, NullLogger<JobWorker>.Instance);
|
|
||||||
await worker.StartAsync(token);
|
|
||||||
var deadline = DateTime.UtcNow.AddSeconds(5);
|
|
||||||
while (DateTime.UtcNow < deadline && !_peer.Requests.Any(r => r.Path == "/live/inbox"))
|
|
||||||
await Task.Delay(50, token);
|
|
||||||
var live = Assert.Single(_peer.Requests, r => r.Path == "/live/inbox");
|
|
||||||
Assert.True(_peer.Requests.Count(r => r.Path == "/dead/inbox") < 30);
|
|
||||||
while (DateTime.UtcNow < deadline && !await DB.Default.Find<RemoteInstance>().Match(i => i.Host == "localhost").ExecuteAnyAsync(token))
|
|
||||||
await Task.Delay(50, token);
|
|
||||||
await worker.StopAsync(token);
|
|
||||||
|
|
||||||
Assert.Contains($"keyId=\"{sender.KeyId}\"", live.Signature);
|
|
||||||
var instance = await DB.Default.Find<RemoteInstance>().Match(i => i.Host == "localhost").ExecuteFirstAsync(token);
|
|
||||||
Assert.True(instance.ConsecutiveFailures > 0);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -6,6 +6,7 @@ using PrivaPub.Tests.Support;
|
|||||||
|
|
||||||
namespace PrivaPub.Tests.Infrastructure
|
namespace PrivaPub.Tests.Infrastructure
|
||||||
{
|
{
|
||||||
|
[Xunit.Collection(nameof(Exclusive))]
|
||||||
[Trait("Category", "Integration")]
|
[Trait("Category", "Integration")]
|
||||||
public class MigrationTests
|
public class MigrationTests
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -0,0 +1,7 @@
|
|||||||
|
namespace PrivaPub.Tests.Support
|
||||||
|
{
|
||||||
|
[CollectionDefinition(nameof(Exclusive), DisableParallelization = true)]
|
||||||
|
public sealed class Exclusive
|
||||||
|
{
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -34,6 +34,8 @@ namespace PrivaPub.Tests.Support
|
|||||||
public const string Host = "privapub.test";
|
public const string Host = "privapub.test";
|
||||||
public const string Base = "https://" + Host;
|
public const string Base = "https://" + Host;
|
||||||
|
|
||||||
|
readonly DateTime _started = Millisecond(DateTime.UtcNow);
|
||||||
|
|
||||||
Harness(Peer peer)
|
Harness(Peer peer)
|
||||||
{
|
{
|
||||||
Peer = peer;
|
Peer = peer;
|
||||||
@@ -136,13 +138,15 @@ namespace PrivaPub.Tests.Support
|
|||||||
}
|
}
|
||||||
|
|
||||||
public async Task<List<JsonObject>> Outgoing(string inbox) =>
|
public async Task<List<JsonObject>> Outgoing(string inbox) =>
|
||||||
(await DB.Default.Find<Job>().Match(j => j.Kind == JobKind.Deliver).ExecuteAsync())
|
(await DB.Default.Find<Job>().Match(j => j.Kind == JobKind.Deliver && j.CreatedAt >= _started).ExecuteAsync())
|
||||||
.Select(j => JsonSerializer.Deserialize<DeliveryPayload>(j.Payload))
|
.Select(j => JsonSerializer.Deserialize<DeliveryPayload>(j.Payload))
|
||||||
.Where(p => p.Inbox == inbox)
|
.Where(p => p.Inbox == inbox)
|
||||||
.Select(p => JsonNode.Parse(p.Body)!.AsObject())
|
.Select(p => JsonNode.Parse(p.Body)!.AsObject())
|
||||||
.ToList();
|
.ToList();
|
||||||
|
|
||||||
public async ValueTask DisposeAsync() => await Peer.DisposeAsync();
|
public async ValueTask DisposeAsync() => await Peer.DisposeAsync();
|
||||||
|
|
||||||
|
static DateTime Millisecond(DateTime time) => new(time.Ticks - time.Ticks % TimeSpan.TicksPerMillisecond, DateTimeKind.Utc);
|
||||||
}
|
}
|
||||||
|
|
||||||
public sealed class KeyLocalizer<T> : IStringLocalizer<T>
|
public sealed class KeyLocalizer<T> : IStringLocalizer<T>
|
||||||
|
|||||||
@@ -44,9 +44,11 @@ namespace PrivaPub.Federation.Moderation
|
|||||||
|
|
||||||
public bool IsSuspended(string host) => Find(host)?.Severity == DomainBlockSeverity.Suspend;
|
public bool IsSuspended(string host) => Find(host)?.Severity == DomainBlockSeverity.Suspend;
|
||||||
|
|
||||||
public async Task Reload(CancellationToken token)
|
public async Task Reload(CancellationToken token) =>
|
||||||
|
Load(await DB.Default.Find<DomainBlock>().ExecuteAsync(token));
|
||||||
|
|
||||||
|
public void Load(IEnumerable<DomainBlock> blocks)
|
||||||
{
|
{
|
||||||
var blocks = await DB.Default.Find<DomainBlock>().ExecuteAsync(token);
|
|
||||||
_blocks = blocks.Where(b => !string.IsNullOrEmpty(b.Domain))
|
_blocks = blocks.Where(b => !string.IsNullOrEmpty(b.Domain))
|
||||||
.GroupBy(b => Normalise(b.Domain))
|
.GroupBy(b => Normalise(b.Domain))
|
||||||
.ToDictionary(g => g.Key, g => g.First());
|
.ToDictionary(g => g.Key, g => g.First());
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ using MongoDB.Entities;
|
|||||||
using PrivaPub.Models.Jobs;
|
using PrivaPub.Models.Jobs;
|
||||||
|
|
||||||
using System.Collections.Concurrent;
|
using System.Collections.Concurrent;
|
||||||
|
using System.Linq.Expressions;
|
||||||
|
|
||||||
namespace PrivaPub.Infrastructure.Jobs
|
namespace PrivaPub.Infrastructure.Jobs
|
||||||
{
|
{
|
||||||
@@ -39,6 +40,13 @@ namespace PrivaPub.Infrastructure.Jobs
|
|||||||
|
|
||||||
readonly string _owner = $"{Environment.MachineName}:{Environment.ProcessId}";
|
readonly string _owner = $"{Environment.MachineName}:{Environment.ProcessId}";
|
||||||
readonly ConcurrentDictionary<JobKind, SemaphoreSlim> _signals = new();
|
readonly ConcurrentDictionary<JobKind, SemaphoreSlim> _signals = new();
|
||||||
|
readonly FilterDefinition<Job> _scope = Builders<Job>.Filter.Empty;
|
||||||
|
|
||||||
|
public JobQueue()
|
||||||
|
{
|
||||||
|
}
|
||||||
|
|
||||||
|
public JobQueue(Expression<Func<Job, bool>> scope) => _scope = Builders<Job>.Filter.Where(scope);
|
||||||
|
|
||||||
public async Task<bool> Enqueue(JobKind kind, string payload, string host, string dedupeKey, CancellationToken token) =>
|
public async Task<bool> Enqueue(JobKind kind, string payload, string host, string dedupeKey, CancellationToken token) =>
|
||||||
await EnqueueMany(new[] { new Job { Kind = kind, Payload = payload, Host = host, DedupeKey = dedupeKey } }, token) == 1;
|
await EnqueueMany(new[] { new Job { Kind = kind, Payload = payload, Host = host, DedupeKey = dedupeKey } }, token) == 1;
|
||||||
@@ -66,7 +74,7 @@ namespace PrivaPub.Infrastructure.Jobs
|
|||||||
var now = DateTime.UtcNow;
|
var now = DateTime.UtcNow;
|
||||||
var busy = busyHosts?.ToList() ?? new List<string>();
|
var busy = busyHosts?.ToList() ?? new List<string>();
|
||||||
return await DB.Default.UpdateAndGet<Job>()
|
return await DB.Default.UpdateAndGet<Job>()
|
||||||
.Match(j => j.Kind == kind && j.State == JobState.Pending && j.RunAt <= now && !busy.Contains(j.Host))
|
.Match(f => f.Where(j => j.Kind == kind && j.State == JobState.Pending && j.RunAt <= now && !busy.Contains(j.Host)) & _scope)
|
||||||
.Modify(j => j.State, JobState.Running)
|
.Modify(j => j.State, JobState.Running)
|
||||||
.Modify(j => j.LeasedUntil, now + LeaseTime)
|
.Modify(j => j.LeasedUntil, now + LeaseTime)
|
||||||
.Modify(j => j.LeaseOwner, _owner)
|
.Modify(j => j.LeaseOwner, _owner)
|
||||||
@@ -106,7 +114,7 @@ namespace PrivaPub.Infrastructure.Jobs
|
|||||||
{
|
{
|
||||||
var now = DateTime.UtcNow;
|
var now = DateTime.UtcNow;
|
||||||
var result = await DB.Default.Update<Job>()
|
var result = await DB.Default.Update<Job>()
|
||||||
.Match(j => j.State == JobState.Running && j.LeasedUntil < now)
|
.Match(f => f.Where(j => j.State == JobState.Running && j.LeasedUntil < now) & _scope)
|
||||||
.Modify(j => j.State, JobState.Pending)
|
.Modify(j => j.State, JobState.Pending)
|
||||||
.Modify(j => j.RunAt, now)
|
.Modify(j => j.RunAt, now)
|
||||||
.Modify(j => j.LeasedUntil, null)
|
.Modify(j => j.LeasedUntil, null)
|
||||||
|
|||||||
Reference in new issue
Block a user