From 2645dea26f5fb3779b3232efe851be1a20a8edbd Mon Sep 17 00:00:00 2001 From: thepra Date: Sat, 3 Oct 2026 11:51:19 +0200 Subject: [PATCH] T8: inbox gaps, jobs, migrations and pages; a deleted remote account's posts are hidden Owner decision (2026-10-03, "A remote account deletes itself"): its posts are kept but hidden everywhere. - Post.AuthorGone (additive bool). DeleteHandler's actor-delete branch sets it on every post whose ActorURI is the actor (one update-many), besides dropping its follows and timeline rows as before. RemotePosts.Build sets it on a post stored later for an account already marked Deleted. - One rule in VisibilityPolicy: IsShown (not deleted, author not gone), IsPublic and CanSee exclude AuthorGone, plus Shown(post) for loaded posts. - Lookups by id answer 404 through CanSee (statuses/:id and every sub-route, context, bookmarks, favourites, polls, reactions, search); provenance, account statuses, home/public/tag timelines, notifications, conversations, reblogged_by, the clientapi home and post/DM lists, a community's outbox and our Announces filter on IsShown or IsPublic; the Mastodon mapper never renders a hidden post or a boost of one. Tests (30 new): - AuthorGoneTests: the rule, the handler (posts kept, boosts included, follows and rows gone), a post fetched after the delete, and 20 Mastodon/ActivityPub lookups over HTTP seen before and hidden after. - InboxGapTests: actor Update refresh (name, sanitised summary, key rotation in place and to a new key id) even with an older `updated`; Undo{Follow} by activity id and by object; Reject of our QuoteRequest (and a stranger's ignored); group-wrapped Announce{Like} and Announce{Undo{Like}}; a locked persona's pending follow, FollowRequest notification, and Decide accepting and rejecting with the original Follow. - JobHandlerTests: AncestorsJobHandler up to its depth limit; PollRefreshJob and PollCloseJob (local and remote polls); InstanceDescriber from a peer's NodeInfo and the weekly dedupe through ObjectRecords; LinkPreviews for public posts only; DeliveryJobHandler outcomes (2xx, 404/410, 429/503 with Retry-After in seconds and as a date, 5xx) and a signature and Digest the peer can verify; MediaJanitor.Sweep; OAuthPruner.Prune. - MigrationTests: _003, _004, _006 and _007 on seeded rows. - PublicPagesTests: /@user and /@user/{id} (visibility, junk ids, exact CSP, Referrer-Policy and nosniff), circle 404, community page, the instance actor, ActivityPub redirects, and markup escaped in posts, titles and bios. Production changes besides the rule: - LinkPreviews.Handle re-checks that a post is still shown and public (the rule Wanted applies) before fetching anything; before, only enqueueing checked it. - The legacy /clientapi post and DM lists no longer return soft-deleted posts. - MediaJanitor.Sweep and OAuthPruner.Prune are the loop bodies, now public and tested. - InstanceDescriber.Address: a protected virtual identity seam so a test can point the https NodeInfo addresses at a plain-http peer; production behaviour unchanged. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01ELjqpznMFMNrJoJUj6K5p2 --- PrivaPub.Tests/Federation/AuthorGoneTests.cs | 277 +++++++++ PrivaPub.Tests/Federation/InboxGapTests.cs | 232 ++++++++ PrivaPub.Tests/Federation/JobHandlerTests.cs | 537 ++++++++++++++++++ .../Infrastructure/MigrationTests.cs | 141 +++++ PrivaPub.Tests/Support/FederatedSeeds.cs | 55 ++ PrivaPub.Tests/Web/PublicPagesTests.cs | 222 ++++++++ PrivaPub/Api/Mastodon/Auth/OAuthPruner.cs | 14 +- .../Controllers/AccountsController.cs | 6 +- .../Controllers/ProvenanceController.cs | 4 +- .../Controllers/StatusesController.cs | 2 +- .../Controllers/TimelinesController.cs | 12 +- .../Api/Mastodon/Mappers/MastodonMapper.cs | 6 +- PrivaPub/Domain/Content/LinkPreviews.cs | 10 +- PrivaPub/Domain/Media/MediaProxy.cs | 14 +- PrivaPub/Domain/Privacy/VisibilityPolicy.cs | 12 +- PrivaPub/Domain/Timelines/TimelineService.cs | 3 +- .../Controllers/PeasantsController.cs | 2 +- .../Inbox/Handlers/DeleteHandler.cs | 1 + PrivaPub/Federation/Inbox/RemotePosts.cs | 1 + .../Federation/Objects/InstanceDescriber.cs | 7 +- PrivaPub/Models/Post/Post.cs | 1 + PrivaPub/Services/PostsService.cs | 4 +- 22 files changed, 1529 insertions(+), 34 deletions(-) create mode 100644 PrivaPub.Tests/Federation/AuthorGoneTests.cs create mode 100644 PrivaPub.Tests/Federation/InboxGapTests.cs create mode 100644 PrivaPub.Tests/Federation/JobHandlerTests.cs create mode 100644 PrivaPub.Tests/Support/FederatedSeeds.cs create mode 100644 PrivaPub.Tests/Web/PublicPagesTests.cs diff --git a/PrivaPub.Tests/Federation/AuthorGoneTests.cs b/PrivaPub.Tests/Federation/AuthorGoneTests.cs new file mode 100644 index 0000000..06f6da2 --- /dev/null +++ b/PrivaPub.Tests/Federation/AuthorGoneTests.cs @@ -0,0 +1,277 @@ +using MongoDB.Entities; + +using PrivaPub.Api.Mastodon.Mappers; +using PrivaPub.Domain.Privacy; +using PrivaPub.Federation.Objects; +using PrivaPub.Models.Federation; +using PrivaPub.Models.Group; +using PrivaPub.Models.Post; +using PrivaPub.Models.Social; +using PrivaPub.Models.User; +using PrivaPub.Tests.Support; +using PrivaPub.Tests.Support.Host; + +using System.Net; +using System.Text.Json.Nodes; + +using static PrivaPub.Tests.Support.FederatedSeeds; + +using GroupEntity = PrivaPub.Models.Group.Group; + +namespace PrivaPub.Tests.Federation +{ + public class AuthorGoneRuleTests + { + [Fact] + public async Task A_post_whose_author_is_gone_is_neither_shown_nor_public_nor_seen_by_anyone() + { + var gone = new Post { Visibility = PostVisibility.Public, AuthorGone = true, GroupUserId = "alice" }; + + Assert.False(VisibilityPolicy.Shown(gone)); + Assert.False(VisibilityPolicy.IsPublic.Compile()(gone)); + Assert.False(await VisibilityPolicy.CanSee(gone, default, TestContext.Current.CancellationToken)); + Assert.False(await VisibilityPolicy.CanSee(gone, "alice", TestContext.Current.CancellationToken)); + Assert.True(VisibilityPolicy.Shown(new Post())); + Assert.False(VisibilityPolicy.Shown(new Post { DeletedAt = DateTime.UtcNow })); + Assert.False(VisibilityPolicy.Shown(default)); + } + } + + [Trait("Category", "Integration")] + public sealed class AuthorGoneTests : IAsyncLifetime + { + Harness _harness; + + public async ValueTask InitializeAsync() + { + Assert.SkipUnless(MongoFixture.Enabled, MongoFixture.Skip); + _harness = await Harness.Start(); + } + + public async ValueTask DisposeAsync() + { + if (_harness != default) + await _harness.DisposeAsync(); + } + + async Task Delivered(RemoteActor author, JsonObject note) + { + await _harness.Deliver(author, "/human-centipede", Create(author, note)); + return await DB.Default.Find().Match(p => p.ObjectURI == IdOf(note)).ExecuteFirstAsync(TestContext.Current.CancellationToken); + } + + [Fact] + public async Task Deleting_itself_keeps_every_post_of_the_account_but_hides_them_and_drops_its_follows_and_rows() + { + var token = TestContext.Current.CancellationToken; + var (root, alice) = await _harness.Persona("alice"); + var ghost = new RemoteActor(_harness.Peer, "ghost"); + var friend = new RemoteActor(_harness.Peer, "friend"); + await Follows(alice.Id, ghost); + await Follows(alice.Id, friend); + await _harness.FollowedBy(alice, ghost); + var post = await Delivered(ghost, PublicNote(ghost, "

soon gone

")); + var friends = await Delivered(friend, PublicNote(friend, "

still here

")); + await _harness.Deliver(ghost, "/human-centipede", new JsonObject + { + ["id"] = NewId(ghost, "announces"), ["type"] = "Announce", ["actor"] = ghost.Id, ["object"] = friends.ObjectURI, + ["to"] = new JsonArray(Addressing.Public) + }); + var ghostAccount = await DB.Default.Find().Match(f => f.ActorURI == ghost.Id).ExecuteSingleAsync(token); + Assert.True(await DB.Default.Find().Match(e => e.AvatarId == alice.Id && e.AuthorAccountId == ghostAccount.ID).ExecuteAnyAsync(token)); + + Assert.Equal(202, (await _harness.Deliver(ghost, "/human-centipede", Activity(ghost, "Delete", JsonValue.Create(ghost.Id)!))).StatusCode); + + var kept = await DB.Default.Find().Match(p => p.ActorURI == ghost.Id).ExecuteAsync(token); + Assert.Equal(2, kept.Count); + Assert.All(kept, p => Assert.True(p.AuthorGone)); + Assert.Contains(kept, p => p.ID == post.ID && p.ContentHtml == "

soon gone

"); + Assert.Contains(kept, p => p.ReblogOfPostId == friends.ID); + Assert.False((await DB.Default.Find().OneAsync(friends.ID, token)).AuthorGone); + Assert.All(kept, p => Assert.False(VisibilityPolicy.Shown(p))); + foreach (var hidden in kept) + Assert.False(await VisibilityPolicy.CanSee(hidden, alice.Id, token)); + Assert.False(await DB.Default.Find().Match(p => p.ActorURI == ghost.Id).Match(VisibilityPolicy.IsPublic).ExecuteAnyAsync(token)); + + Assert.Equal(AvatarAccountState.Deleted, (await DB.Default.Find().OneAsync(ghostAccount.ID, token)).AccountState); + Assert.False(await DB.Default.Find().Match(f => f.ActorURI == ghost.Id).ExecuteAnyAsync(token)); + Assert.False(await DB.Default.Find().Match(f => f.TargetActorURI == ghost.Id).ExecuteAnyAsync(token)); + Assert.False(await DB.Default.Find().Match(e => e.AuthorAccountId == ghostAccount.ID).ExecuteAnyAsync(token)); + + var mapper = new MastodonMapper(_harness.Db, _harness.Local); + Assert.Empty(await mapper.Statuses(kept, alice.Id, token)); + var home = (List)(await _harness.Timelines.Home(root, alice.Id, default, 40, token)).Data; + Assert.DoesNotContain(home, v => v.AuthorActorURI == ghost.Id); + Assert.Contains(home, v => v.Id == friends.ID); + } + + [Fact] + public async Task A_post_by_an_account_already_gone_is_hidden_from_the_moment_it_is_stored() + { + var token = TestContext.Current.CancellationToken; + var (_, alice) = await _harness.Persona("alice"); + var ghost = new RemoteActor(_harness.Peer, "ghost"); + await _harness.FollowedBy(alice, ghost); + await _harness.Deliver(ghost, "/human-centipede", Activity(ghost, "Delete", JsonValue.Create(ghost.Id)!)); + var late = PublicNote(ghost, "

from beyond

"); + _harness.Peer.Serve(new Uri(IdOf(late)).AbsolutePath, late.ToJsonString()); + + var stored = await _harness.RemotePosts.StoreContext(IdOf(late), 0, token); + + Assert.NotNull(stored); + Assert.True(stored.AuthorGone); + Assert.False(await VisibilityPolicy.CanSee(stored, alice.Id, token)); + } + } + + [Trait("Category", "Integration")] + public sealed class AuthorGoneOverHttpTests : IAsyncLifetime + { + PrivaPubHost _host; + Peer _peer; + + public async ValueTask InitializeAsync() + { + Assert.SkipUnless(MongoFixture.Enabled, MongoFixture.Skip); + _host = await PrivaPubHost.Shared(); + _peer = await Peer.Start(); + } + + public async ValueTask DisposeAsync() + { + if (_peer != default) + await _peer.DisposeAsync(); + } + + static async Task<(HttpStatusCode Status, JsonNode Body)> Get(HttpClient client, string path) + { + var response = await client.GetAsync(path, TestContext.Current.CancellationToken); + var text = await response.Content.ReadAsStringAsync(TestContext.Current.CancellationToken); + return (response.StatusCode, string.IsNullOrEmpty(text) || response.Content.Headers.ContentType?.MediaType?.Contains("json") != true ? default : JsonNode.Parse(text)); + } + + static async Task> Ids(HttpClient client, string path) + { + var (status, body) = await Get(client, path); + Assert.Equal(HttpStatusCode.OK, status); + return body!.AsArray().Select(s => IdOf(s!)).ToList(); + } + + static async Task Found(HttpClient client, string path) => (await Get(client, path)).Status == HttpStatusCode.OK; + + sealed record Seen(bool Status, bool Context, bool FavouritedBy, bool RebloggedBy, bool History, bool Many, bool Boost, bool InAncestors, + bool BoostersListed, bool OnProfile, bool OnTag, bool OnPublic, bool OnHome, bool Notified, bool InConversation, bool Searched, + bool Favourited, bool Bookmarked, bool Provenance, bool InCommunityOutbox); + + [Fact] + public async Task Every_lookup_treats_a_post_whose_author_deleted_itself_as_not_found() + { + var token = TestContext.Current.CancellationToken; + var persona = await _host.Persona(await _host.SignUp(), "reader"); + var bearer = await _host.MastodonToken(persona); + using var client = _host.As(bearer); + var (privateKey, publicKey) = PrivaPub.Federation.Actors.Keys.NewKeyPair(); + var community = new GroupEntity + { + UserName = $"commons{Guid.NewGuid():N}"[..20], Kind = GroupKind.Community, PostingPolicy = PostingPolicy.Anyone, + PrivateKey = privateKey, PublicKey = publicKey, Members = new() { new GroupMember { AvatarId = persona.Id, Role = GroupRole.Owner } } + }; + await DB.Default.SaveAsync(community, token); + var personaUri = $"{PrivaPubHost.Base}/peasants/{persona.UserName}"; + var communityUri = $"{PrivaPubHost.Base}/peasants/{community.UserName}"; + var ghost = new RemoteActor(_peer, "ghost"); + var friend = new RemoteActor(_peer, "friend"); + await Follows(persona.Id, ghost); + await Follows(persona.Id, friend); + var tag = $"gone{Guid.NewGuid():N}"[..16]; + + var note = PublicNote(ghost, $"

hello #{tag}

", personaUri, communityUri); + note["tag"] = new JsonArray( + new JsonObject { ["type"] = "Mention", ["href"] = personaUri, ["name"] = "@reader" }, + new JsonObject { ["type"] = "Hashtag", ["name"] = "#" + tag, ["href"] = $"{Origin(ghost)}/tags/{tag}" }); + Assert.Equal(HttpStatusCode.Accepted, await DeliverSigned(_host, ghost, Create(ghost, note))); + var friendsNote = PublicNote(friend, "

a friend's post

"); + await DeliverSigned(_host, friend, Create(friend, friendsNote)); + var reply = PublicNote(friend, "

a reply

"); + reply["inReplyTo"] = IdOf(note); + await DeliverSigned(_host, friend, Create(friend, reply)); + await DeliverSigned(_host, friend, new JsonObject + { + ["id"] = NewId(friend, "announces"), ["type"] = "Announce", ["actor"] = friend.Id, ["object"] = IdOf(note), ["to"] = new JsonArray(Addressing.Public) + }); + var ghostBoostId = NewId(ghost, "announces"); + await DeliverSigned(_host, ghost, new JsonObject + { + ["id"] = ghostBoostId, ["type"] = "Announce", ["actor"] = ghost.Id, ["object"] = IdOf(friendsNote), ["to"] = new JsonArray(Addressing.Public) + }); + var dm = new JsonObject + { + ["id"] = NewId(ghost, "notes"), ["type"] = "Note", ["attributedTo"] = ghost.Id, ["content"] = "

psst

", ["to"] = new JsonArray(personaUri), + ["tag"] = new JsonArray(new JsonObject { ["type"] = "Mention", ["href"] = personaUri, ["name"] = "@reader" }) + }; + await DeliverSigned(_host, ghost, Create(ghost, dm)); + + var post = await DB.Default.Find().Match(p => p.ObjectURI == IdOf(note)).ExecuteSingleAsync(token); + var friends = await DB.Default.Find().Match(p => p.ObjectURI == IdOf(friendsNote)).ExecuteSingleAsync(token); + var replyPost = await DB.Default.Find().Match(p => p.ObjectURI == IdOf(reply)).ExecuteSingleAsync(token); + var ghostBoost = await DB.Default.Find().Match(p => p.ObjectURI == ghostBoostId).ExecuteSingleAsync(token); + var dmPost = await DB.Default.Find().Match(p => p.ObjectURI == IdOf(dm)).ExecuteSingleAsync(token); + var ghostAccount = await DB.Default.Find().Match(f => f.ActorURI == ghost.Id).ExecuteSingleAsync(token); + Assert.Equal(community.ID, post.GroupId); + //search looks up https addresses only, which the peer does not have + var searchable = new Post + { + ObjectURI = $"https://ghost{Guid.NewGuid():N}.example/notes/1", ActorURI = ghost.Id, AuthorAccountId = ghostAccount.ID, IsFederatedCopy = true, + Visibility = PostVisibility.Public, ContentHtml = "

found by its address

" + }; + await DB.Default.SaveAsync(searchable, token); + Assert.Equal(HttpStatusCode.OK, (await client.PostAsync($"/api/v1/statuses/{post.ID}/favourite", default, token)).StatusCode); + Assert.Equal(HttpStatusCode.OK, (await client.PostAsync($"/api/v1/statuses/{post.ID}/bookmark", default, token)).StatusCode); + + async Task Look() + { + var home = (await Get(client, "/api/v1/timelines/home?limit=40")).Body!.AsArray(); + var notifications = (await Get(client, "/api/v1/notifications?limit=30")).Body!.AsArray(); + var conversations = (await Get(client, "/api/v1/conversations")).Body!.AsArray(); + var search = (await Get(client, $"/api/v2/search?type=statuses&q={Uri.EscapeDataString(searchable.ObjectURI)}")).Body!; + using var anonymous = _host.Client(); + using var outboxRequest = new HttpRequestMessage(HttpMethod.Get, $"/peasants/{community.UserName}/anus?page=true"); + outboxRequest.Headers.Accept.ParseAdd("application/activity+json"); + var outbox = JsonNode.Parse(await (await anonymous.SendAsync(outboxRequest, token)).Content.ReadAsStringAsync(token))!; + return new Seen( + await Found(client, $"/api/v1/statuses/{post.ID}"), + await Found(client, $"/api/v1/statuses/{post.ID}/context"), + await Found(client, $"/api/v1/statuses/{post.ID}/favourited_by"), + await Found(client, $"/api/v1/statuses/{post.ID}/reblogged_by"), + await Found(client, $"/api/v1/statuses/{post.ID}/history"), + (await Ids(client, $"/api/v1/statuses?id[]={post.ID}")).Contains(post.ID), + await Found(client, $"/api/v1/statuses/{ghostBoost.ID}"), + (await Get(client, $"/api/v1/statuses/{replyPost.ID}/context")).Body!["ancestors"]!.AsArray().Any(a => IdOf(a!) == post.ID), + (await Ids(client, $"/api/v1/statuses/{friends.ID}/reblogged_by")).Contains(ghostAccount.ID), + (await Ids(client, $"/api/v1/accounts/{ghostAccount.ID}/statuses")).Count > 0, + (await Ids(client, $"/api/v1/timelines/tag/{tag}")).Contains(post.ID), + (await Ids(client, $"/api/v1/timelines/public?remote=true&limit=1&max_id={IdAbove(post.ID)}")).Contains(post.ID), + home.Any(s => IdOf(s!) == post.ID || IdOf(s!) == ghostBoost.ID || s!["reblog"] is JsonObject inner && IdOf(inner) == post.ID), + notifications.Any(n => n!["status"] is JsonObject status && IdOf(status) == post.ID), + conversations.Any(c => c!["last_status"] is JsonObject last && IdOf(last) == dmPost.ID), + search["statuses"]!.AsArray().Count > 0, + (await Ids(client, "/api/v1/favourites")).Contains(post.ID), + (await Ids(client, "/api/v1/bookmarks")).Contains(post.ID), + await Found(client, $"/api/privapub/v1/statuses/{post.ID}/provenance"), + outbox["orderedItems"]!.AsArray().Any(a => a!["object"]?.GetValue() == post.ObjectURI)); + } + + var before = await Look(); + Assert.Equal(HttpStatusCode.Accepted, await DeliverSigned(_host, ghost, Activity(ghost, "Delete", JsonValue.Create(ghost.Id)!))); + var after = await Look(); + + var everything = new Seen(true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true); + var nothing = new Seen(false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false); + Assert.Equal(everything, before); + Assert.Equal(nothing, after); + Assert.True(await DB.Default.Find().Match(p => p.ID == post.ID && p.AuthorGone && p.DeletedAt == null).ExecuteAnyAsync(token)); + Assert.True(await Found(client, $"/api/v1/statuses/{friends.ID}")); + Assert.True(await Found(client, $"/api/v1/statuses/{replyPost.ID}")); + } + } +} diff --git a/PrivaPub.Tests/Federation/InboxGapTests.cs b/PrivaPub.Tests/Federation/InboxGapTests.cs new file mode 100644 index 0000000..0225896 --- /dev/null +++ b/PrivaPub.Tests/Federation/InboxGapTests.cs @@ -0,0 +1,232 @@ +using MongoDB.Entities; + +using PrivaPub.ClientModels.Social; +using PrivaPub.Domain.Statuses; +using PrivaPub.Federation.Actors; +using PrivaPub.Federation.Objects; +using PrivaPub.Models.Federation; +using PrivaPub.Models.Post; +using PrivaPub.Models.Social; +using PrivaPub.Models.User; +using PrivaPub.Tests.Support; + +using System.Security.Cryptography; +using System.Text.Json.Nodes; + +using static PrivaPub.Tests.Support.FederatedSeeds; + +namespace PrivaPub.Tests.Federation +{ + [Trait("Category", "Integration")] + public sealed class InboxGapTests : IAsyncLifetime + { + Harness _harness; + + public async ValueTask InitializeAsync() + { + Assert.SkipUnless(MongoFixture.Enabled, MongoFixture.Skip); + _harness = await Harness.Start(); + } + + public async ValueTask DisposeAsync() + { + if (_harness != default) + await _harness.DisposeAsync(); + } + + static JsonObject Follow(RemoteActor follower, string target) => + new() { ["id"] = NewId(follower, "follows"), ["type"] = "Follow", ["actor"] = follower.Id, ["object"] = target }; + + Task Stored(RemoteActor actor) => + DB.Default.Find().Match(f => f.ActorURI == actor.Id).ExecuteSingleAsync(TestContext.Current.CancellationToken); + + PrivaPub.Models.Statistics.InteractionEvent Processed(string activity) => _harness.Ledger.Of("in").Last(e => e.Activity == activity); + + [Fact] + public async Task An_actor_update_refreshes_the_account_from_its_origin_even_when_its_updated_is_older() + { + var token = TestContext.Current.CancellationToken; + var (_, alice) = await _harness.Persona("alice"); + var bob = new RemoteActor(_harness.Peer, "bob"); + await _harness.Deliver(bob, "/human-centipede", Follow(bob, alice.Uri)); + var before = await Stored(bob); + using var rotated = RSA.Create(2048); + var document = bob.Document(); + document["name"] = "Bob Renamed"; + document["summary"] = "

a new bio

"; + document["publicKey"]!["publicKeyPem"] = rotated.ExportSubjectPublicKeyInfoPem(); + _harness.Peer.Serve(new Uri(bob.Id).AbsolutePath, document.ToJsonString()); + var embedded = (JsonObject)document.DeepClone(); + embedded["name"] = "What the activity claims"; + embedded["updated"] = "2001-01-01T00:00:00Z"; + + var result = await _harness.Deliver(bob, "/human-centipede", Activity(bob, "Update", embedded)); + + var after = await Stored(bob); + Assert.Equal(202, result.StatusCode); + Assert.Null(before.Name); + Assert.Equal(before.ID, after.ID); + Assert.Equal("Bob Renamed", after.Name); + Assert.Equal("

a new bio

", after.Biography); + Assert.Equal(bob.KeyId, after.PublicKeyId); + Assert.NotEqual(before.PublicKey, after.PublicKey); + Assert.Equal(rotated.ExportSubjectPublicKeyInfoPem(), after.PublicKey); + Assert.Equal(("accepted", "actor-refresh"), (Processed("Update").Outcome, Processed("Update").Reason)); + } + + [Fact] + public async Task A_key_moved_to_a_new_id_replaces_the_old_one() + { + var token = TestContext.Current.CancellationToken; + var (_, alice) = await _harness.Persona("alice"); + var bob = new RemoteActor(_harness.Peer, "bob"); + await _harness.Deliver(bob, "/human-centipede", Follow(bob, alice.Uri)); + using var rotated = RSA.Create(2048); + var document = bob.Document(); + document["publicKey"] = new JsonObject { ["id"] = bob.Id + "#key-2", ["owner"] = bob.Id, ["publicKeyPem"] = rotated.ExportSubjectPublicKeyInfoPem() }; + _harness.Peer.Serve(new Uri(bob.Id).AbsolutePath, document.ToJsonString()); + + await _harness.Deliver(bob, "/human-centipede", Activity(bob, "Update", JsonValue.Create(bob.Id)!)); + + var after = await Stored(bob); + Assert.Equal(bob.Id + "#key-2", after.PublicKeyId); + Assert.Equal(rotated.ExportSubjectPublicKeyInfoPem(), after.PublicKey); + } + + [Fact] + public async Task Undoing_a_follow_removes_the_follower_by_the_activity_id_or_by_its_object() + { + var token = TestContext.Current.CancellationToken; + var (_, alice) = await _harness.Persona("alice"); + var bob = new RemoteActor(_harness.Peer, "bob"); + var carol = new RemoteActor(_harness.Peer, "carol"); + var bobFollows = Follow(bob, alice.Uri); + await _harness.Deliver(bob, "/human-centipede", bobFollows); + await _harness.Deliver(carol, "/human-centipede", Follow(carol, alice.Uri)); + Assert.Equal(2, await DB.Default.CountAsync(f => f.LocalActorId == alice.Id, token)); + + await _harness.Deliver(bob, "/human-centipede", Activity(bob, "Undo", JsonValue.Create(IdOf(bobFollows))!)); + Assert.False(await DB.Default.Find().Match(f => f.LocalActorId == alice.Id && f.ActorURI == bob.Id).ExecuteAnyAsync(token)); + Assert.Equal(("accepted", "undone"), (Processed("Undo").Outcome, Processed("Undo").Reason)); + + var forgotten = Follow(carol, alice.Uri); + forgotten["id"] = NewId(carol, "follows"); + await _harness.Deliver(carol, "/human-centipede", Activity(carol, "Undo", forgotten)); + Assert.False(await DB.Default.Find().Match(f => f.LocalActorId == alice.Id).ExecuteAnyAsync(token)); + + await _harness.Deliver(carol, "/human-centipede", Activity(carol, "Undo", forgotten)); + Assert.Equal(("dropped", "unknown-object"), (Processed("Undo").Outcome, Processed("Undo").Reason)); + } + + [Fact] + public async Task A_rejected_quote_request_marks_the_quote_rejected_and_only_the_quoted_author_can_reject_it() + { + var token = TestContext.Current.CancellationToken; + var (_, alice) = await _harness.Persona("alice"); + var ann = new RemoteActor(_harness.Peer, "ann"); + var mallory = new RemoteActor(_harness.Peer, "mallory"); + var note = PublicNote(ann, "

ask me first

", alice.Uri); + note["tag"] = new JsonArray(new JsonObject { ["type"] = "Mention", ["href"] = alice.Uri, ["name"] = "@alice" }); + note["interactionPolicy"] = new JsonObject { ["canQuote"] = new JsonObject { ["manualApproval"] = new JsonArray(Addressing.Public) } }; + await _harness.Deliver(ann, "/human-centipede", Create(ann, note)); + var original = await DB.Default.Find().Match(p => p.ObjectURI == IdOf(note)).ExecuteSingleAsync(token); + var outcome = await _harness.Statuses.Publish(alice, new StatusDraft { Text = "may I?", QuotedStatusId = original.ID }, token); + Assert.Equal(QuoteState.Pending, outcome.Post.QuoteState); + var request = Assert.Single(await _harness.Outgoing(ann.Id + "/inbox"), a => a["type"]!.GetValue() == "QuoteRequest"); + + await _harness.Deliver(mallory, "/human-centipede", Activity(mallory, "Reject", JsonValue.Create(IdOf(request))!)); + Assert.Equal(QuoteState.Pending, (await DB.Default.Find().OneAsync(outcome.Post.ID, token)).QuoteState); + + await _harness.Deliver(ann, "/human-centipede", Activity(ann, "Reject", request)); + + var quoting = await DB.Default.Find().OneAsync(outcome.Post.ID, token); + Assert.Equal(QuoteState.Rejected, quoting.QuoteState); + Assert.Null(quoting.QuoteAuthorizationURI); + Assert.Equal(0, (await DB.Default.Find().OneAsync(original.ID, token)).QuotesCount); + Assert.Equal(("accepted", "quote-answer"), (Processed("Reject").Outcome, Processed("Reject").Reason)); + } + + [Fact] + public async Task A_followed_community_announcing_a_vote_or_its_undo_is_recorded_and_changes_nothing_yet() + { + var token = TestContext.Current.CancellationToken; + var (root, alice) = await _harness.Persona("alice"); + var community = new RemoteActor(_harness.Peer, "cats", type: "Group"); + var stranger = new RemoteActor(_harness.Peer, "dogs", type: "Group"); + var poster = new RemoteActor(_harness.Peer, "poster"); + var voter = new RemoteActor(_harness.Peer, "voter"); + await _harness.Follows.Follow(root, new FollowForm { AvatarId = alice.Id, Target = community.Id }, token); + await DB.Default.Update().Match(f => f.AvatarId == alice.Id).Modify(f => f.State, FollowState.Accepted).ExecuteAsync(token); + var page = PublicNote(poster, "

a post

", community.Id); + page["type"] = "Page"; + page["name"] = "a title"; + page["audience"] = community.Id; + _harness.Peer.Serve(new Uri(IdOf(page)).AbsolutePath, page.ToJsonString()); + await _harness.Deliver(community, "/human-centipede", Activity(community, "Announce", Create(poster, page))); + var post = await DB.Default.Find().Match(p => p.ObjectURI == IdOf(page)).ExecuteSingleAsync(token); + var like = new JsonObject { ["id"] = NewId(voter, "likes"), ["type"] = "Like", ["actor"] = voter.Id, ["object"] = post.ObjectURI, ["audience"] = community.Id }; + + await _harness.Deliver(community, "/human-centipede", Activity(community, "Announce", like)); + var liked = Processed("Announce"); + await _harness.Deliver(community, "/human-centipede", Activity(community, "Announce", Activity(voter, "Undo", like))); + var unliked = Processed("Announce"); + await _harness.Deliver(stranger, "/human-centipede", Activity(stranger, "Announce", like)); + var unfollowed = Processed("Announce"); + + //votes relayed by a community are a documented gap (docs/INTEROP.md, Lemmy: "Announces of activities other than Create") + Assert.Equal(("dropped", "unsupported", "Like"), (liked.Outcome, liked.Reason, liked.Object)); + Assert.Equal(("dropped", "unsupported", "Undo"), (unliked.Outcome, unliked.Reason, unliked.Object)); + Assert.Equal(("dropped", "not-followed"), (unfollowed.Outcome, unfollowed.Reason)); + var after = await DB.Default.Find().OneAsync(post.ID, token); + Assert.Equal(0, after.FavouritesCount); + Assert.False(await DB.Default.Find().Match(f => f.PostId == post.ID).ExecuteAnyAsync(token)); + } + + [Fact] + public async Task A_locked_persona_holds_a_follow_until_it_decides_and_answers_with_the_original_follow() + { + var token = TestContext.Current.CancellationToken; + var (_, open) = await _harness.Persona("alice"); + await DB.Default.Update().MatchID(open.Id).Modify(a => a.Settings.IsLocked, true).ExecuteAsync(token); + var alice = await _harness.Local.FindById(LocalActorKind.Person, open.Id, token); + var bob = new RemoteActor(_harness.Peer, "bob"); + var carol = new RemoteActor(_harness.Peer, "carol"); + var bobFollows = Follow(bob, alice.Uri); + var carolFollows = Follow(carol, alice.Uri); + Assert.True(alice.ManuallyApprovesFollowers); + + await _harness.Deliver(bob, "/human-centipede", bobFollows); + await _harness.Deliver(carol, "/human-centipede", carolFollows); + + var pending = await DB.Default.Find().Match(f => f.LocalActorId == alice.Id).ExecuteAsync(token); + Assert.Equal(2, pending.Count); + Assert.All(pending, f => Assert.False(f.IsAccepted)); + Assert.Equal(("accepted", "pending"), (Processed("Follow").Outcome, Processed("Follow").Reason)); + var requests = await DB.Default.Find().Match(n => n.AvatarId == alice.Id).ExecuteAsync(token); + Assert.Equal(2, requests.Count); + Assert.All(requests, n => Assert.Equal(NotificationType.FollowRequest, n.Type)); + Assert.Empty(await _harness.Outgoing(bob.Id + "/inbox")); + Assert.Empty(await _harness.Delivery.FollowerInboxes(alice, token)); + + var bobAccount = await Stored(bob); + var carolAccount = await Stored(carol); + Assert.True(await _harness.Follows.Decide(alice, bobAccount.ID, accept: true, token)); + Assert.True(await _harness.Follows.Decide(alice, carolAccount.ID, accept: false, token)); + Assert.False(await _harness.Follows.Decide(alice, carolAccount.ID, accept: true, token)); + + var accept = Assert.Single(await _harness.Outgoing(bob.Id + "/inbox")); + Assert.Equal("Accept", accept["type"]!.GetValue()); + Assert.Equal(alice.Uri, accept["actor"]!.GetValue()); + Assert.Equal(IdOf(bobFollows), IdOf(accept["object"]!)); + Assert.Equal(bob.Id, accept["object"]!["actor"]!.GetValue()); + var reject = Assert.Single(await _harness.Outgoing(carol.Id + "/inbox")); + Assert.Equal("Reject", reject["type"]!.GetValue()); + Assert.Equal(IdOf(carolFollows), IdOf(reject["object"]!)); + Assert.True((await DB.Default.Find().Match(f => f.LocalActorId == alice.Id && f.ActorURI == bob.Id).ExecuteSingleAsync(token)).IsAccepted); + Assert.False(await DB.Default.Find().Match(f => f.LocalActorId == alice.Id && f.ActorURI == carol.Id).ExecuteAnyAsync(token)); + Assert.Contains(await DB.Default.Find().Match(n => n.AvatarId == alice.Id && n.Type == NotificationType.Follow).ExecuteAsync(token), + n => n.FromAccountId == bobAccount.ID); + Assert.Equal(new[] { bob.Id + "/inbox" }, await _harness.Delivery.FollowerInboxes(alice, token)); + } + } +} diff --git a/PrivaPub.Tests/Federation/JobHandlerTests.cs b/PrivaPub.Tests/Federation/JobHandlerTests.cs new file mode 100644 index 0000000..40e248b --- /dev/null +++ b/PrivaPub.Tests/Federation/JobHandlerTests.cs @@ -0,0 +1,537 @@ +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Hosting; +using Microsoft.AspNetCore.Http; +using Microsoft.Extensions.Caching.Memory; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging.Abstractions; + +using MongoDB.Entities; + +using OpenIddict.Abstractions; + +using PrivaPub.Api.Mastodon.Auth; +using PrivaPub.Domain.Content; +using PrivaPub.Domain.Media; +using PrivaPub.Domain.Statuses; +using PrivaPub.Federation.Actors; +using PrivaPub.Federation.Inbox; +using PrivaPub.Federation.Objects; +using PrivaPub.Federation.Outbox; +using PrivaPub.Federation.Signing; +using PrivaPub.Infrastructure.Http; +using PrivaPub.Infrastructure.Jobs; +using PrivaPub.Models.Federation; +using PrivaPub.Models.Jobs; +using PrivaPub.Models.Media; +using PrivaPub.Models.Post; +using PrivaPub.Models.Social; +using PrivaPub.Tests.Support; +using PrivaPub.Tests.Support.Host; + +using System.Security.Cryptography; +using System.Text; +using System.Text.Json; +using System.Text.Json.Nodes; + +using static PrivaPub.Tests.Support.FederatedSeeds; + +namespace PrivaPub.Tests.Federation +{ + [Trait("Category", "Integration")] + public sealed class JobHandlerTests : IAsyncLifetime + { + Harness _harness; + + public async ValueTask InitializeAsync() + { + Assert.SkipUnless(MongoFixture.Enabled, MongoFixture.Skip); + _harness = await Harness.Start(); + } + + public async ValueTask DisposeAsync() + { + if (_harness != default) + await _harness.DisposeAsync(); + } + + sealed class PeerDescriber : InstanceDescriber + { + readonly string _peer; + + public PeerDescriber(IFederationHttp http, string peer) : base(http) => _peer = peer; + + protected override string Address(string url) => _peer + new Uri(url).PathAndQuery; + } + + [Fact] + public async Task Ancestors_are_fetched_one_job_at_a_time_and_stop_at_the_depth_limit() + { + var token = TestContext.Current.CancellationToken; + var poster = new RemoteActor(_harness.Peer, "poster"); + var chain = new List(); + for (var i = 0; i < RemotePosts.MaxDepth + 3; i++) + { + var note = PublicNote(poster, $"

number {i}

"); + if (i > 0) + note["inReplyTo"] = IdOf(chain[^1]); + _harness.Peer.Serve(new Uri(IdOf(note)).AbsolutePath, note.ToJsonString()); + chain.Add(note); + } + var leaf = new Post + { + ObjectURI = NewId(poster, "notes"), ActorURI = poster.Id, IsFederatedCopy = true, Visibility = PostVisibility.Public, + InReplyToURI = IdOf(chain[^1]), ContentHtml = "

the leaf

" + }; + await DB.Default.SaveAsync(leaf, token); + await _harness.Queue.Enqueue(JobKind.FetchAncestors, JsonSerializer.Serialize(new AncestorsPayload(leaf.ID, 1)), "127.0.0.1", "ancestors|" + leaf.ID, token); + var handler = new AncestorsJobHandler(_harness.Db, _harness.RemotePosts); + + var keys = new List { "ancestors|" + leaf.ID }; + var ran = 0; + while (await DB.Default.Find().Match(j => j.Kind == JobKind.FetchAncestors && j.State == JobState.Pending && keys.Contains(j.DedupeKey)) + .ExecuteFirstAsync(token) is { } job) + { + Assert.Equal(JobResult.Done, (await handler.Handle(job, token)).Result); + await DB.Default.Update().MatchID(job.ID).Modify(j => j.State, JobState.Done).ExecuteAsync(token); + ran++; + var uris = chain.Select(IdOf).ToList(); + keys = (await DB.Default.Find().Match(p => uris.Contains(p.ObjectURI)).ExecuteAsync(token)).Select(p => "ancestors|" + p.ID).Append("ancestors|" + leaf.ID).ToList(); + Assert.True(ran <= RemotePosts.MaxDepth, "the chain did not stop"); + } + + var all = chain.Select(IdOf).ToList(); + var stored = (await DB.Default.Find().Match(p => all.Contains(p.ObjectURI)).ExecuteAsync(token)).ToDictionary(p => p.ObjectURI); + Assert.Equal(RemotePosts.MaxDepth, ran); + Assert.Equal(all.Skip(3), stored.Keys.OrderBy(all.IndexOf)); + Assert.Equal(stored[all[^1]].ID, (await DB.Default.Find().OneAsync(leaf.ID, token)).AnsweringToPostId); + for (var i = 4; i < all.Count; i++) + Assert.Equal(stored[all[i - 1]].ID, stored[all[i]].AnsweringToPostId); + Assert.Null(stored[all[3]].AnsweringToPostId); + Assert.Equal(all[2], stored[all[3]].InReplyToURI); + Assert.DoesNotContain(_harness.Peer.Requests, r => r.Path == new Uri(all[2]).AbsolutePath); + } + + async Task<(LocalActor Alice, RemoteActor Mallory, Post Poll)> LocalPoll() + { + var token = TestContext.Current.CancellationToken; + var (_, alice) = await _harness.Persona("alice"); + var mallory = new RemoteActor(_harness.Peer, "mallory"); + await _harness.Deliver(mallory, "/human-centipede", new JsonObject + { + ["id"] = NewId(mallory, "follows"), ["type"] = "Follow", ["actor"] = mallory.Id, ["object"] = alice.Uri + }); + var outcome = await _harness.Statuses.Publish(alice, new StatusDraft { Text = "which?", Poll = new PollDraft(new[] { "tea", "coffee" }, 3600, false, false) }, token); + Assert.True(outcome.Ok); + return (alice, mallory, outcome.Post); + } + + async Task> Updates(RemoteActor to) => + (await _harness.Outgoing(to.Id + "/inbox")).Where(a => a["type"]!.GetValue() == "Update").ToList(); + + static int[] Counts(JsonNode question) => + question["oneOf"]!.AsArray().Select(o => o!["replies"]!["totalItems"]!.GetValue()).ToArray(); + + [Fact] + public async Task A_poll_refresh_sends_the_new_counts_to_followers_as_an_update() + { + var token = TestContext.Current.CancellationToken; + var (_, mallory, poll) = await LocalPoll(); + var (_, bob) = await _harness.Persona("bob"); + Assert.True((await _harness.Polls.Vote(bob, poll, new[] { 1 }, token)).Ok); + var job = await DB.Default.Find().Match(j => j.Kind == JobKind.PollRefresh && j.Payload == poll.ID).ExecuteSingleAsync(token); + Assert.True(job.RunAt > DateTime.UtcNow); + Assert.Empty(await Updates(mallory)); + + Assert.Equal(JobResult.Done, (await new PollRefreshJob(_harness.Db, _harness.Local, _harness.Outbox).Handle(job, token)).Result); + + var question = Assert.Single(await Updates(mallory))["object"]!; + Assert.Equal("Question", question["type"]!.GetValue()); + Assert.Equal(new[] { 0, 1 }, Counts(question)); + Assert.Null(question["closed"]); + Assert.Null((await DB.Default.Find().OneAsync(poll.ID, token)).Poll.ClosedAt); + } + + [Fact] + public async Task Closing_a_local_poll_tells_its_voters_and_sends_the_closed_question() + { + var token = TestContext.Current.CancellationToken; + var (alice, mallory, poll) = await LocalPoll(); + var (_, bob) = await _harness.Persona("bob"); + Assert.True((await _harness.Polls.Vote(bob, poll, new[] { 0 }, token)).Ok); + var job = await DB.Default.Find().Match(j => j.Kind == JobKind.PollClose && j.Payload == poll.ID).ExecuteSingleAsync(token); + + Assert.Equal(JobResult.Done, (await new PollCloseJob(_harness.Db, _harness.Local, _harness.Outbox).Handle(job, token)).Result); + + var closed = (await DB.Default.Find().OneAsync(poll.ID, token)).Poll.ClosedAt; + Assert.NotNull(closed); + var question = Assert.Single(await Updates(mallory))["object"]!; + Assert.NotNull(question["closed"]); + Assert.Equal(new[] { 1, 0 }, Counts(question)); + var told = await DB.Default.Find().Match(n => n.Type == NotificationType.Poll && n.PostId == poll.ID).ExecuteAsync(token); + Assert.Equal(new[] { alice.Id, bob.Id }.Order(), told.Select(n => n.AvatarId).Order()); + } + + [Fact] + public async Task Closing_a_remote_poll_tells_the_local_voters_and_sends_nothing() + { + var token = TestContext.Current.CancellationToken; + var (_, alice) = await _harness.Persona("alice"); + var mallory = new RemoteActor(_harness.Peer, "mallory"); + var question = PublicNote(mallory, "

pick

", alice.Uri); + question["type"] = "Question"; + question["tag"] = new JsonArray(new JsonObject { ["type"] = "Mention", ["href"] = alice.Uri, ["name"] = "@alice" }); + question["endTime"] = DateTime.UtcNow.AddMinutes(10).ToString("O"); + question["oneOf"] = new JsonArray( + new JsonObject { ["type"] = "Note", ["name"] = "red", ["replies"] = new JsonObject { ["type"] = "Collection", ["totalItems"] = 0 } }, + new JsonObject { ["type"] = "Note", ["name"] = "blue", ["replies"] = new JsonObject { ["type"] = "Collection", ["totalItems"] = 0 } }); + await _harness.Deliver(mallory, "/human-centipede", Create(mallory, question)); + var poll = await DB.Default.Find().Match(p => p.ObjectURI == IdOf(question)).ExecuteSingleAsync(token); + Assert.True((await _harness.Polls.Vote(alice, poll, new[] { 1 }, token)).Ok); + var job = await DB.Default.Find().Match(j => j.Kind == JobKind.PollClose && j.Payload == poll.ID).ExecuteSingleAsync(token); + + Assert.Equal(JobResult.Done, (await new PollCloseJob(_harness.Db, _harness.Local, _harness.Outbox).Handle(job, token)).Result); + + var told = Assert.Single(await DB.Default.Find().Match(n => n.Type == NotificationType.Poll && n.PostId == poll.ID).ExecuteAsync(token)); + Assert.Equal(alice.Id, told.AvatarId); + Assert.Equal(poll.AuthorAccountId, told.FromAccountId); + Assert.Empty(await Updates(mallory)); + Assert.Null((await DB.Default.Find().OneAsync(poll.ID, token)).Poll.ClosedAt); + } + + [Fact] + public async Task A_server_is_described_from_its_nodeinfo_and_at_most_once_a_week() + { + var token = TestContext.Current.CancellationToken; + var host = $"describe{Guid.NewGuid():N}.example"; + var peer = _harness.Peer; + peer.ServeText("/.well-known/nodeinfo", new JsonObject + { + ["links"] = new JsonArray( + new JsonObject { ["rel"] = "http://nodeinfo.diaspora.software/ns/schema/2.0", ["href"] = $"https://{host}/nodeinfo/2.0" }, + new JsonObject { ["rel"] = "http://nodeinfo.diaspora.software/ns/schema/2.1", ["href"] = $"https://{host}/nodeinfo/2.1" }) + }.ToJsonString(), "application/json"); + peer.ServeText("/nodeinfo/2.1", new JsonObject + { + ["version"] = "2.1", + ["software"] = new JsonObject { ["name"] = "gotosocial", ["version"] = "0.20.1" }, + ["protocols"] = new JsonArray("activitypub", 42), + ["openRegistrations"] = false, + ["usage"] = new JsonObject { ["users"] = new JsonObject { ["total"] = 3 } }, + ["metadata"] = new JsonObject { ["nodeName"] = "A small place" } + }.ToJsonString(), "application/json"); + var describer = new PeerDescriber(Peer.Http(), peer.A); + + async Task Record() + { + var note = NoteParser.Parse(new JsonObject + { + ["id"] = $"https://{host}/notes/{Guid.NewGuid():N}", ["type"] = "Note", ["attributedTo"] = $"https://{host}/users/x", + ["content"] = "

hi

", ["to"] = new JsonArray(Addressing.Public) + }); + var post = new Post { ObjectURI = note.Id, IsFederatedCopy = true }; + await DB.Default.SaveAsync(post, token); + await _harness.Records.Record(note, post, ObjectPath.Fetched, refetched: false, token); + } + Task> Jobs() => DB.Default.Find().Match(j => j.Kind == JobKind.DescribeInstance && j.Payload == host).ExecuteAsync(token); + + await Record(); + await Record(); + var job = Assert.Single(await Jobs()); + Assert.StartsWith($"describe|{host}|", job.DedupeKey); + + Assert.Equal(JobResult.Done, (await describer.Handle(job, token)).Result); + + var instance = await DB.Default.Find().Match(i => i.Host == host).ExecuteSingleAsync(token); + Assert.Equal(("gotosocial", "0.20.1", "A small place"), (instance.Software, instance.SoftwareVersion, instance.NodeName)); + Assert.Equal(new[] { "activitypub" }, instance.Protocols); + Assert.False(instance.OpenRegistrations); + Assert.Equal(3, JsonNode.Parse(instance.NodeInfo)!["usage"]!["users"]!["total"]!.GetValue()); + Assert.Null(instance.DescriptionError); + Assert.InRange(instance.DescribedAt!.Value, DateTime.UtcNow.AddMinutes(-1), DateTime.UtcNow.AddSeconds(1)); + Assert.DoesNotContain(peer.Requests, r => r.Path == "/nodeinfo/2.0"); + + await DB.Default.DeleteAsync(job.ID); + await Record(); + Assert.Empty(await Jobs()); + + await DB.Default.Update().MatchID(instance.ID).Modify(i => i.DescribedAt, DateTime.UtcNow.AddDays(-8)).ExecuteAsync(token); + await Record(); + Assert.Single(await Jobs()); + } + + [Fact] + public async Task A_server_without_usable_nodeinfo_is_described_as_such() + { + var token = TestContext.Current.CancellationToken; + var plain = $"plain{Guid.NewGuid():N}.example"; + var describer = new PeerDescriber(Peer.Http(), _harness.Peer.A + "/" + plain); + _harness.Peer.ServeText($"/{plain}/.well-known/nodeinfo", new JsonObject + { + ["links"] = new JsonArray(new JsonObject { ["rel"] = "http://nodeinfo.diaspora.software/ns/schema/2.1", ["href"] = $"http://{plain}/nodeinfo/2.1" }) + }.ToJsonString(), "application/json"); + var silent = $"silent{Guid.NewGuid():N}.example"; + + await describer.Handle(new Job { Kind = JobKind.DescribeInstance, Payload = plain }, token); + await new PeerDescriber(Peer.Http(), _harness.Peer.A + "/" + silent).Handle(new Job { Kind = JobKind.DescribeInstance, Payload = silent }, token); + + Assert.Equal("no NodeInfo", (await DB.Default.Find().Match(i => i.Host == plain).ExecuteSingleAsync(token)).DescriptionError); + Assert.Equal("no NodeInfo", (await DB.Default.Find().Match(i => i.Host == silent).ExecuteSingleAsync(token)).DescriptionError); + } + + LinkPreviews Previews() => new(_harness.Db, _harness.Local, Peer.Http(), _harness.Queue, + new StaticOptions(new FederationOptions { AllowPrivateNetworks = true, AllowPlainHttp = true })); + + async Task Linking(PostVisibility visibility, string path) + { + var post = new Post + { + ObjectURI = $"https://elsewhere.example/notes/{Guid.NewGuid():N}", IsFederatedCopy = true, Visibility = visibility, + ContentHtml = $"

read this

" + }; + await DB.Default.SaveAsync(post, TestContext.Current.CancellationToken); + _harness.Peer.ServeText(path, """ + Fallback + + + + + """, "text/html; charset=utf-8"); + return post; + } + + [Fact] + public async Task A_link_preview_is_built_from_open_graph_for_a_public_post_only() + { + var token = TestContext.Current.CancellationToken; + var publicPath = $"/articles/{Guid.NewGuid():N}"; + var privatePath = $"/articles/{Guid.NewGuid():N}"; + var open = await Linking(PostVisibility.Public, publicPath); + var followersOnly = await Linking(PostVisibility.FollowersOnly, privatePath); + + Assert.Equal(JobResult.Done, (await Previews().Handle(new Job { Kind = JobKind.FetchPreview, Payload = open.ID }, token)).Result); + Assert.Equal(JobResult.Done, (await Previews().Handle(new Job { Kind = JobKind.FetchPreview, Payload = followersOnly.ID }, token)).Result); + + var card = (await DB.Default.Find().OneAsync(open.ID, token)).Link; + Assert.Equal(_harness.Peer.A + publicPath, card.Href); + Assert.Equal(("A headline", "What it says", _harness.Peer.A + "/cover.jpg"), (card.Title, card.Description, card.ImageURL)); + Assert.True(await DB.Default.Find().Match(p => p.Url == _harness.Peer.A + publicPath).ExecuteAnyAsync(token)); + Assert.Null((await DB.Default.Find().OneAsync(followersOnly.ID, token)).Link); + Assert.DoesNotContain(_harness.Peer.Requests, r => r.Path == privatePath); + Assert.False(await DB.Default.Find().Match(p => p.Url == _harness.Peer.A + privatePath).ExecuteAnyAsync(token)); + } + + [Fact] + public async Task The_janitor_removes_stale_unattached_uploads_and_trims_the_proxy_cache_oldest_first() + { + var token = TestContext.Current.CancellationToken; + var root = Path.Combine(Path.GetTempPath(), $"privapub-janitor-{Guid.NewGuid():N}"); + var options = new StaticOptions(new MediaOptions { Root = root, ProxyCacheBytes = 800 }); + var media = new MediaService(options, _harness.Local, default, NullLogger.Instance); + var seeded = new List(); + try + { + MediaAttachment Upload(string name, DateTime created, string postId = default) + { + Directory.CreateDirectory(Path.Combine(root, "files")); + File.WriteAllBytes(Path.Combine(root, "files", name), new byte[] { 1, 2, 3 }); + File.WriteAllBytes(Path.Combine(root, "files", "small-" + name), new byte[] { 1 }); + return new MediaAttachment { FilePath = Path.Combine("files", name), PreviewPath = Path.Combine("files", "small-" + name), CreatedAt = created, PostId = postId }; + } + var stale = Upload("stale.webp", DateTime.UtcNow.AddDays(-2)); + var fresh = Upload("fresh.webp", DateTime.UtcNow.AddHours(-1)); + var attached = Upload("attached.webp", DateTime.UtcNow.AddDays(-3), "000000000000000000000001"); + await DB.Default.SaveAsync(new[] { stale, fresh, attached }, token); + seeded.AddRange(new[] { stale.ID, fresh.ID, attached.ID }); + Directory.CreateDirectory(media.ProxyRoot); + FileInfo Cached(string name, int bytes, int minutesAgo) + { + var path = Path.Combine(media.ProxyRoot, name); + File.WriteAllBytes(path, new byte[bytes]); + File.WriteAllText(path + ".type", "image/png"); + File.SetLastWriteTimeUtc(path, DateTime.UtcNow.AddMinutes(-minutesAgo)); + return new FileInfo(path); + } + var oldest = Cached("a", 600, 30); + var older = Cached("b", 400, 20); + var newest = Cached("c", 300, 10); + + await new MediaJanitor(media, options, NullLogger.Instance).Sweep(token); + + Assert.False(await DB.Default.Find().Match(m => m.ID == stale.ID).ExecuteAnyAsync(token)); + Assert.False(File.Exists(Path.Combine(root, stale.FilePath))); + Assert.False(File.Exists(Path.Combine(root, stale.PreviewPath))); + Assert.True(await DB.Default.Find().Match(m => m.ID == fresh.ID).ExecuteAnyAsync(token)); + Assert.True(await DB.Default.Find().Match(m => m.ID == attached.ID).ExecuteAnyAsync(token)); + Assert.True(File.Exists(Path.Combine(root, fresh.FilePath))); + Assert.False(File.Exists(oldest.FullName)); + Assert.False(File.Exists(oldest.FullName + ".type")); + Assert.True(File.Exists(older.FullName)); + Assert.True(File.Exists(older.FullName + ".type")); + Assert.True(File.Exists(newest.FullName)); + } + finally + { + await DB.Default.DeleteAsync(m => seeded.Contains(m.ID)); + foreach (var directory in new[] { root, media.ProxyRoot }.Where(Directory.Exists)) + Directory.Delete(directory, recursive: true); + } + } + } + + [Trait("Category", "Integration")] + public sealed class OAuthPrunerTests + { + [Fact] + public async Task Old_tokens_and_authorizations_that_are_no_longer_valid_are_pruned_and_the_rest_kept() + { + Assert.SkipUnless(MongoFixture.Enabled, MongoFixture.Skip); + var token = TestContext.Current.CancellationToken; + var host = await PrivaPubHost.Shared(); + var now = DateTimeOffset.UtcNow; + using var scope = host.Services.CreateScope(); + var tokens = scope.ServiceProvider.GetRequiredService(); + var authorizations = scope.ServiceProvider.GetRequiredService(); + var subject = $"pruned{Guid.NewGuid():N}"; + async Task Token(DateTimeOffset created, DateTimeOffset expires, string status) => + await tokens.GetIdAsync(await tokens.CreateAsync(new OpenIddictTokenDescriptor + { + CreationDate = created, ExpirationDate = expires, Status = status, Subject = subject, Type = OpenIddictConstants.TokenTypeHints.AccessToken + }, token), token); + async Task Authorization(DateTimeOffset created, string status) => + await authorizations.GetIdAsync(await authorizations.CreateAsync(new OpenIddictAuthorizationDescriptor + { + CreationDate = created, Status = status, Subject = subject, Type = OpenIddictConstants.AuthorizationTypes.Permanent + }, token), token); + var expired = await Token(now.AddDays(-30), now.AddDays(-29), OpenIddictConstants.Statuses.Valid); + var revoked = await Token(now.AddDays(-30), now.AddDays(1), OpenIddictConstants.Statuses.Revoked); + var current = await Token(now.AddMinutes(-5), now.AddHours(1), OpenIddictConstants.Statuses.Valid); + var recentlyRevoked = await Token(now.AddDays(-1), now.AddDays(-1).AddMinutes(1), OpenIddictConstants.Statuses.Revoked); + var oldRevoked = await Authorization(now.AddDays(-30), OpenIddictConstants.Statuses.Revoked); + var oldValid = await Authorization(now.AddDays(-30), OpenIddictConstants.Statuses.Valid); + + var (prunedTokens, prunedAuthorizations) = await new OAuthPruner(host.Services, NullLogger.Instance).Prune(now.AddDays(-14), token); + + Assert.True(prunedTokens >= 2, $"{prunedTokens} tokens pruned"); + Assert.True(prunedAuthorizations >= 1, $"{prunedAuthorizations} authorizations pruned"); + //a fresh scope: the managers cache what they created + using var after = host.Services.CreateScope(); + tokens = after.ServiceProvider.GetRequiredService(); + authorizations = after.ServiceProvider.GetRequiredService(); + Assert.Null(await tokens.FindByIdAsync(expired, token)); + Assert.Null(await tokens.FindByIdAsync(revoked, token)); + Assert.NotNull(await tokens.FindByIdAsync(current, token)); + Assert.NotNull(await tokens.FindByIdAsync(recentlyRevoked, token)); + Assert.Null(await authorizations.FindByIdAsync(oldRevoked, token)); + Assert.NotNull(await authorizations.FindByIdAsync(oldValid, token)); + } + } + + [Xunit.Collection(nameof(Exclusive))] + [Trait("Category", "Integration")] + public sealed class DeliveryOutcomeTests : IAsyncLifetime + { + static readonly string[] PeerHosts = { "localhost", "127.0.0.1" }; + + Harness _harness; + WebApplication _hinting; + string _hintingBase; + + public async ValueTask InitializeAsync() + { + Assert.SkipUnless(MongoFixture.Enabled, MongoFixture.Skip); + _harness = await Harness.Start(); + await DB.Default.DeleteAsync(i => PeerHosts.Contains(i.Host)); + var builder = WebApplication.CreateSlimBuilder(); + builder.WebHost.UseUrls("http://127.0.0.1:0"); + _hinting = builder.Build(); + //answers /{status}/{retry-after}: the peer cannot send a Retry-After + _hinting.Run(context => + { + var parts = context.Request.Path.Value!.Trim('/').Split('/'); + context.Response.StatusCode = int.Parse(parts[0]); + if (parts.Length > 1) + context.Response.Headers.RetryAfter = parts[1] == "date" + ? DateTimeOffset.UtcNow.AddMinutes(10).ToString("r") + : parts[1]; + return Task.CompletedTask; + }); + await _hinting.StartAsync(); + _hintingBase = _hinting.Urls.First(); + } + + public async ValueTask DisposeAsync() + { + if (_hinting != default) + await _hinting.DisposeAsync(); + if (_harness == default) + return; + await _harness.DisposeAsync(); + await DB.Default.DeleteAsync(i => PeerHosts.Contains(i.Host)); + } + + async Task Attempt(LocalActor signer, string inbox) + { + var token = TestContext.Current.CancellationToken; + var id = $"{Harness.Base}/a/{Guid.NewGuid():N}"; + await _harness.Delivery.Enqueue(signer, new[] { inbox }, new JsonObject + { + ["id"] = id, ["type"] = "Create", ["actor"] = signer.Uri, ["to"] = new JsonArray(Addressing.Public), + ["object"] = new JsonObject { ["type"] = "Note", ["content"] = "hi" } + }, token); + var job = await DB.Default.Find().Match(j => j.DedupeKey == $"{id}|{inbox}").ExecuteSingleAsync(token); + job.Attempts = 1; + var handler = new DeliveryJobHandler(_harness.Local, Peer.Http(), new HostCircuitBreaker(new MemoryCache(new MemoryCacheOptions())), + NullLogger.Instance); + return await handler.Handle(job, token); + } + + [Fact] + public async Task A_delivery_carries_a_valid_digest_and_a_signature_by_the_signer() + { + var (_, alice) = await _harness.Persona("alice"); + _harness.Peer.Answer("/ok/inbox", 202); + + var outcome = await Attempt(alice, _harness.Peer.A + "/ok/inbox"); + + Assert.Equal(JobResult.Done, outcome.Result); + var received = Assert.Single(_harness.Peer.Requests, r => r.Path == "/ok/inbox"); + Assert.Equal("POST", received.Method); + Assert.Equal(alice.Uri, JsonNode.Parse(received.Body)!["actor"]!.GetValue()); + Assert.Equal(HttpSignatures.Digest(Encoding.UTF8.GetBytes(received.Body)), received.Headers["Digest"]); + var signature = HttpSignatures.Parse(received.Signature); + Assert.Equal(alice.KeyId, signature.KeyId); + Assert.Equal(new[] { "(request-target)", "host", "date", "digest" }, signature.Headers); + var signingString = $"(request-target): post /ok/inbox\nhost: {received.Headers["Host"]}\ndate: {received.Headers["Date"]}\ndigest: {received.Headers["Digest"]}"; + using var key = RSA.Create(); + key.ImportFromPem(alice.PublicKeyPem); + Assert.True(key.VerifyData(Encoding.UTF8.GetBytes(signingString), signature.Signature, HashAlgorithmName.SHA256, RSASignaturePadding.Pkcs1)); + } + + [Fact] + public async Task Refusals_are_dead_hints_defer_and_failures_retry() + { + var (_, alice) = await _harness.Persona("alice"); + _harness.Peer.Answer("/gone/inbox", 410); + _harness.Peer.Answer("/missing/inbox", 404); + _harness.Peer.Answer("/broken/inbox", 500); + + var gone = await Attempt(alice, _harness.Peer.A + "/gone/inbox"); + var missing = await Attempt(alice, _harness.Peer.A + "/missing/inbox"); + var busy = await Attempt(alice, _hintingBase + "/429/120"); + var down = await Attempt(alice, _hintingBase + "/503/date"); + var unhinted = await Attempt(alice, _hintingBase + "/503"); + var broken = await Attempt(alice, _harness.Peer.A + "/broken/inbox"); + + Assert.Equal(JobResult.Dead, gone.Result); + Assert.StartsWith("410", gone.Error); + Assert.Equal(JobResult.Dead, missing.Result); + Assert.Equal(JobResult.Defer, busy.Result); + Assert.InRange(busy.RetryAt!.Value, DateTime.UtcNow.AddSeconds(100), DateTime.UtcNow.AddSeconds(125)); + Assert.Equal(JobResult.Defer, down.Result); + Assert.InRange(down.RetryAt!.Value, DateTime.UtcNow.AddMinutes(9), DateTime.UtcNow.AddMinutes(11)); + Assert.Equal(JobResult.Retry, unhinted.Result); + Assert.Equal(JobResult.Retry, broken.Result); + Assert.StartsWith("500", broken.Error); + } + } +} diff --git a/PrivaPub.Tests/Infrastructure/MigrationTests.cs b/PrivaPub.Tests/Infrastructure/MigrationTests.cs index 3f4adc4..93f2932 100644 --- a/PrivaPub.Tests/Infrastructure/MigrationTests.cs +++ b/PrivaPub.Tests/Infrastructure/MigrationTests.cs @@ -1,10 +1,21 @@ +using MongoDB.Bson; +using MongoDB.Driver; using MongoDB.Entities; +using PrivaPub.Federation.Outbox; using PrivaPub.Infrastructure.Data.Migrations; using PrivaPub.Models.Federation; +using PrivaPub.Models.Group; +using PrivaPub.Models.Jobs; using PrivaPub.Models.Post; +using PrivaPub.Models.User; using PrivaPub.Tests.Support; +using System.Text.Json; +using System.Text.Json.Nodes; + +using GroupEntity = PrivaPub.Models.Group.Group; + namespace PrivaPub.Tests.Infrastructure { [Xunit.Collection(nameof(Exclusive))] @@ -79,5 +90,135 @@ namespace PrivaPub.Tests.Infrastructure Assert.Contains("legacy-quote", afterFetched.Extensions); Assert.Empty(afterDelivered.Extensions); } + + [Fact] + public async Task Conversations_get_their_participants_key_groups_become_circles_and_their_posts_stay_home() + { + Assert.SkipUnless(MongoFixture.Enabled, MongoFixture.Skip); + var token = TestContext.Current.CancellationToken; + var members = new List { new() { AvatarId = "a1" }, new() { AvatarId = "https://r.example/u/b", IsForeign = true } }; + var conversation = new DmGroup { Members = members }; + var group = new GroupEntity { UserName = $"old{Guid.NewGuid():N}"[..20], Kind = GroupKind.Community }; + await DB.Default.SaveAsync(conversation, token); + await DB.Default.SaveAsync(group, token); + var localInGroup = new Post { GroupId = group.ID, Text = "inside", ObjectURI = $"https://privapub.test/{Guid.NewGuid():N}" }; + var remoteInGroup = new Post { GroupId = group.ID, IsFederatedCopy = true, Text = "visiting", ObjectURI = $"https://r.example/{Guid.NewGuid():N}" }; + var ungrouped = new Post { Text = "outside", ObjectURI = $"https://privapub.test/{Guid.NewGuid():N}" }; + await DB.Default.SaveAsync(new[] { localInGroup, remoteInGroup, ungrouped }, token); + //the migration turns every group into a circle and keeps every local group post home: put the others back afterwards + var communities = (await DB.Default.Find().Match(g => g.Kind != GroupKind.Circle && g.ID != group.ID).ExecuteAsync(token)).Select(g => g.ID).ToList(); + var federatedGroupPosts = (await DB.Default.Find().Match(p => p.GroupId != null && !p.IsFederatedCopy && !p.IsLocalOnly && p.ID != localInGroup.ID) + .ExecuteAsync(token)).Select(p => p.ID).ToList(); + try + { + await new _003_conversation_keys_and_circles().UpgradeAsync(); + + Assert.Equal(DmGroup.KeyOf(members), (await DB.Default.Find().OneAsync(conversation.ID, token)).ParticipantsKey); + Assert.Equal(GroupKind.Circle, (await DB.Default.Find().OneAsync(group.ID, token)).Kind); + Assert.True((await DB.Default.Find().OneAsync(localInGroup.ID, token)).IsLocalOnly); + Assert.False((await DB.Default.Find().OneAsync(remoteInGroup.ID, token)).IsLocalOnly); + Assert.False((await DB.Default.Find().OneAsync(ungrouped.ID, token)).IsLocalOnly); + } + finally + { + if (communities.Count > 0) + await DB.Default.Update().Match(g => communities.Contains(g.ID)).Modify(g => g.Kind, GroupKind.Community).ExecuteAsync(token); + if (federatedGroupPosts.Count > 0) + await DB.Default.Update().Match(p => federatedGroupPosts.Contains(p.ID)).Modify(p => p.IsLocalOnly, false).ExecuteAsync(token); + } + } + + [Fact] + public async Task Pending_deliveries_move_to_the_job_queue_once_and_the_rest_are_left_alone() + { + Assert.SkipUnless(MongoFixture.Enabled, MongoFixture.Skip); + var token = TestContext.Current.CancellationToken; + var host = $"r{Guid.NewGuid():N}.example"; + var activityId = $"https://privapub.test/a/{Guid.NewGuid():N}"; + var body = new JsonObject { ["id"] = activityId, ["type"] = "Create" }.ToJsonString(); + var nextAttempt = DateTime.UtcNow.AddMinutes(7); + Delivery Legacy(string inbox, DateTime? deliveredAt = default, DateTime? abandonedAt = default) => new() + { + SignerId = "000000000000000000000001", SignerKind = LocalActorKind.Person, InboxURL = inbox, Body = body, Attempts = 3, + NextAttemptAt = nextAttempt, DeliveredAt = deliveredAt, AbandonedAt = abandonedAt + }; + var pending = Legacy($"https://{host.ToUpperInvariant()}/inbox"); + var twice = Legacy($"https://{host}/shared"); + var already = Legacy($"https://{host}/shared"); + var broken = Legacy("not an address"); + var delivered = Legacy($"https://{host}/done", deliveredAt: DateTime.UtcNow.AddDays(-1)); + var abandoned = Legacy($"https://{host}/given-up", abandonedAt: DateTime.UtcNow.AddDays(-1)); + await DB.Default.SaveAsync(new[] { pending, twice, already, broken, delivered, abandoned }, token); + + await new _004_pending_deliveries_become_jobs().UpgradeAsync(); + + var jobs = await DB.Default.Find().Match(j => j.Kind == JobKind.Deliver && j.DedupeKey.StartsWith(activityId + "|")).ExecuteAsync(token); + Assert.Equal(2, jobs.Count); + var job = Assert.Single(jobs, j => j.DedupeKey == $"{activityId}|{pending.InboxURL}"); + Assert.Equal(host, job.Host); + Assert.Equal(3, job.Attempts); + Assert.InRange(job.RunAt, nextAttempt.AddSeconds(-1), nextAttempt.AddSeconds(1)); + var payload = JsonSerializer.Deserialize(job.Payload); + Assert.Equal((pending.SignerId, LocalActorKind.Person, pending.InboxURL, body), (payload.SignerId, payload.SignerKind, payload.Inbox, payload.Body)); + Assert.Contains(jobs, j => j.DedupeKey == $"{activityId}|{twice.InboxURL}"); + foreach (var moved in new[] { pending, twice, already, broken }) + { + var after = await DB.Default.Find().OneAsync(moved.ID, token); + Assert.NotNull(after.AbandonedAt); + Assert.Equal("moved to the job queue", after.LastError); + } + Assert.Null((await DB.Default.Find().OneAsync(delivered.ID, token)).AbandonedAt); + Assert.Null((await DB.Default.Find().OneAsync(abandoned.ID, token)).LastError); + } + + [Fact] + public async Task Year_one_edit_dates_become_no_edit_and_year_one_revisions_take_the_creation_date() + { + Assert.SkipUnless(MongoFixture.Enabled, MongoFixture.Skip); + var token = TestContext.Current.CancellationToken; + var created = new DateTime(2026, 9, 1, 12, 0, 0, DateTimeKind.Utc); + var edited = new DateTime(2026, 9, 2, 12, 0, 0, DateTimeKind.Utc); + var broken = new Post + { + IsFederatedCopy = true, ObjectURI = $"https://r.example/{Guid.NewGuid():N}", CreationDate = created, EditedAt = DateTime.MinValue, UpdateDate = DateTime.MinValue, + Revisions = new() { new PostRevision { ContentHtml = "

first

", EditedAt = DateTime.MinValue }, new PostRevision { ContentHtml = "

second

", EditedAt = edited } } + }; + var fine = new Post { IsFederatedCopy = true, ObjectURI = $"https://r.example/{Guid.NewGuid():N}", CreationDate = created, EditedAt = edited, UpdateDate = edited }; + await DB.Default.SaveAsync(new[] { broken, fine }, token); + + await new _006_unedited_remote_posts_lose_year_one().UpgradeAsync(); + + var repaired = await DB.Default.Find().OneAsync(broken.ID, token); + Assert.Null(repaired.EditedAt); + Assert.Equal(created, repaired.UpdateDate); + Assert.Equal(new[] { created, edited }, repaired.Revisions.Select(r => r.EditedAt)); + var untouched = await DB.Default.Find().OneAsync(fine.ID, token); + Assert.Equal((edited, edited), (untouched.EditedAt, untouched.UpdateDate)); + } + + [Fact] + public async Task Personas_and_groups_without_a_published_day_get_one_within_two_weeks_before_creation() + { + Assert.SkipUnless(MongoFixture.Enabled, MongoFixture.Skip); + var token = TestContext.Current.CancellationToken; + var created = new DateTime(2026, 6, 15, 18, 30, 0, DateTimeKind.Utc); + var avatar = new Avatar { UserName = $"aged{Guid.NewGuid():N}"[..20], CreatedAt = created }; + var group = new GroupEntity { UserName = $"aged{Guid.NewGuid():N}"[..20], CreationDate = created }; + var dated = new Avatar { UserName = $"dated{Guid.NewGuid():N}"[..20], CreatedAt = created, PublishedOn = new DateTime(2026, 6, 10, 0, 0, 0, DateTimeKind.Utc) }; + await DB.Default.SaveAsync(new[] { avatar, dated }, token); + await DB.Default.SaveAsync(group, token); + await DB.Default.Update().MatchID(avatar.ID).Modify(b => b.Unset(a => a.PublishedOn)).ExecuteAsync(token); + await DB.Default.Update().MatchID(group.ID).Modify(b => b.Unset(g => g.PublishedOn)).ExecuteAsync(token); + Assert.False(await DB.Default.Find().Match(new BsonDocument("_id", ObjectId.Parse(avatar.ID)).Add(nameof(Avatar.PublishedOn), new BsonDocument("$exists", true))).ExecuteAnyAsync(token)); + + await new _007_personas_publish_a_spread_day().UpgradeAsync(); + + foreach (var published in new[] { (await DB.Default.Find().OneAsync(avatar.ID, token)).PublishedOn, (await DB.Default.Find().OneAsync(group.ID, token)).PublishedOn }) + { + Assert.Equal(TimeSpan.Zero, published.TimeOfDay); + Assert.InRange(published, created.Date.AddDays(-13), created.Date); + } + Assert.Equal(new DateTime(2026, 6, 10, 0, 0, 0, DateTimeKind.Utc), (await DB.Default.Find().OneAsync(dated.ID, token)).PublishedOn); + } } } diff --git a/PrivaPub.Tests/Support/FederatedSeeds.cs b/PrivaPub.Tests/Support/FederatedSeeds.cs new file mode 100644 index 0000000..025bd5c --- /dev/null +++ b/PrivaPub.Tests/Support/FederatedSeeds.cs @@ -0,0 +1,55 @@ +using MongoDB.Entities; + +using PrivaPub.Federation.Objects; +using PrivaPub.Models.Social; +using PrivaPub.Tests.Support.Host; + +using System.Globalization; +using System.Net; +using System.Numerics; +using System.Text.Json.Nodes; + +namespace PrivaPub.Tests.Support +{ + public static class FederatedSeeds + { + public static string Origin(RemoteActor actor) => new Uri(actor.Id).GetLeftPart(UriPartial.Authority); + + public static string NewId(RemoteActor actor, string kind) => $"{Origin(actor)}/{kind}/{Guid.NewGuid():N}"; + + public static JsonObject PublicNote(RemoteActor author, string content = "

hello

", params string[] cc) => new() + { + ["id"] = NewId(author, "notes"), + ["type"] = "Note", + ["attributedTo"] = author.Id, + ["content"] = content, + ["published"] = DateTime.UtcNow.ToString("O"), + ["to"] = new JsonArray(Addressing.Public), + ["cc"] = new JsonArray(cc.Prepend(author.Id + "/followers").Select(c => (JsonNode)c).ToArray()) + }; + + public static JsonObject Create(RemoteActor author, JsonObject note) => + new() { ["id"] = NewId(author, "activities"), ["type"] = "Create", ["actor"] = author.Id, ["object"] = note.DeepClone() }; + + public static JsonObject Activity(RemoteActor actor, string type, JsonNode inner) => + new() { ["id"] = NewId(actor, "activities"), ["type"] = type, ["actor"] = actor.Id, ["object"] = inner.DeepClone() }; + + public static string IdOf(JsonNode node) => node["id"]!.GetValue(); + + //an accepted follow made here, without the round trip + public static Task Follows(string avatarId, RemoteActor target) => + DB.Default.SaveAsync(new Following { AvatarId = avatarId, TargetActorURI = target.Id, TargetInboxURL = target.Id + "/inbox", State = FollowState.Accepted }); + + public static async Task DeliverSigned(PrivaPubHost host, RemoteActor sender, JsonObject activity, string path = "/human-centipede") + { + using var client = host.Client(); + var response = await client.SendAsync(sender.SignedPost(path, activity), TestContext.Current.CancellationToken); + await host.RunInbox(IdOf(activity), TestContext.Current.CancellationToken); + return response.StatusCode; + } + + //the smallest id above this one, for a max_id that starts a page exactly at it + public static string IdAbove(string id) => + (BigInteger.Parse("0" + id, NumberStyles.HexNumber) + 1).ToString("x24")[^24..]; + } +} diff --git a/PrivaPub.Tests/Web/PublicPagesTests.cs b/PrivaPub.Tests/Web/PublicPagesTests.cs new file mode 100644 index 0000000..8824e46 --- /dev/null +++ b/PrivaPub.Tests/Web/PublicPagesTests.cs @@ -0,0 +1,222 @@ +using Microsoft.Extensions.DependencyInjection; + +using MongoDB.Entities; + +using PrivaPub.Domain.Statuses; +using PrivaPub.Federation.Actors; +using PrivaPub.Models.Federation; +using PrivaPub.Models.Group; +using PrivaPub.Models.Post; +using PrivaPub.Models.User; +using PrivaPub.Tests.Support; +using PrivaPub.Tests.Support.Host; + +using System.Net; + +using GroupEntity = PrivaPub.Models.Group.Group; + +namespace PrivaPub.Tests.Web +{ + [Trait("Category", "Integration")] + public sealed class PublicPagesTests : IAsyncLifetime + { + const string Csp = "default-src 'none'; style-src 'unsafe-inline'; img-src https: data:; base-uri 'none'; form-action 'none'; frame-ancestors 'none'"; + + PrivaPubHost _host; + + public async ValueTask InitializeAsync() + { + Assert.SkipUnless(MongoFixture.Enabled, MongoFixture.Skip); + _host = await PrivaPubHost.Shared(); + } + + public ValueTask DisposeAsync() => ValueTask.CompletedTask; + + async Task Author(string name = "author") + { + var persona = await _host.Persona(await _host.SignUp(), name); + return await _host.Get().FindById(LocalActorKind.Person, persona.Id, TestContext.Current.CancellationToken); + } + + async Task Publish(LocalActor author, string text, PostVisibility visibility = PostVisibility.Public, string groupId = default) + { + using var scope = _host.Services.CreateScope(); + var outcome = await scope.ServiceProvider.GetRequiredService() + .Publish(author, new StatusDraft { Text = text, Visibility = visibility, GroupId = groupId }, TestContext.Current.CancellationToken); + Assert.True(outcome.Ok, outcome.Error); + return outcome.Post; + } + + async Task Seeded(LocalActor author, string text, PostVisibility visibility) + { + var post = new Post { GroupUserId = author.Id, AuthorAccountId = author.Id, ActorURI = author.Uri, Text = text, Visibility = visibility }; + post.ID = (string)post.GenerateNewID(); + post.ObjectURI = author.PostUri(post.ID); + await DB.Default.SaveAsync(post, TestContext.Current.CancellationToken); + return post; + } + + async Task<(HttpResponseMessage Response, string Body)> Page(string path, string accept = "text/html") + { + using var client = _host.Client(); + using var request = new HttpRequestMessage(HttpMethod.Get, path); + request.Headers.Accept.ParseAdd(accept); + var response = await client.SendAsync(request, TestContext.Current.CancellationToken); + return (response, await response.Content.ReadAsStringAsync(TestContext.Current.CancellationToken)); + } + + static void Hardened(HttpResponseMessage response) + { + Assert.Equal(Csp, Assert.Single(response.Headers.GetValues("Content-Security-Policy"))); + Assert.Equal("no-referrer", Assert.Single(response.Headers.GetValues("Referrer-Policy"))); + Assert.Equal("nosniff", Assert.Single(response.Headers.GetValues("X-Content-Type-Options"))); + } + + [Fact] + public async Task A_profile_shows_public_and_unlisted_posts_only_behind_hardened_headers() + { + var token = TestContext.Current.CancellationToken; + var author = await Author(); + await Publish(author, "for everyone"); + await Publish(author, "unlisted but linkable", PostVisibility.Unlisted); + await Publish(author, "for followers", PostVisibility.FollowersOnly); + await Seeded(author, "for one person", PostVisibility.Direct); + await Seeded(author, "for the neighbourhood", PostVisibility.LocalGeo); + var removed = await Publish(author, "taken back"); + using (var scope = _host.Services.CreateScope()) + Assert.True((await scope.ServiceProvider.GetRequiredService().Remove(author, removed.ID, token)).Ok); + + var (response, body) = await Page($"/@{author.UserName}"); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + Hardened(response); + Assert.Contains("for everyone", body); + Assert.Contains("unlisted but linkable", body); + Assert.DoesNotContain("for followers", body); + Assert.DoesNotContain("for one person", body); + Assert.DoesNotContain("for the neighbourhood", body); + Assert.DoesNotContain("taken back", body); + Assert.Contains("", body); + Assert.Contains($"", body); + } + + [Fact] + public async Task A_post_page_serves_public_and_unlisted_posts_and_answers_404_for_everything_else() + { + var token = TestContext.Current.CancellationToken; + var author = await Author(); + var other = await Author("other"); + var open = await Publish(author, "an open post"); + var unlisted = await Publish(author, "an unlisted post", PostVisibility.Unlisted); + var followersOnly = await Publish(author, "a followers post", PostVisibility.FollowersOnly); + var direct = await Seeded(author, "a direct post", PostVisibility.Direct); + var located = await Seeded(author, "a located post", PostVisibility.LocalGeo); + var removed = await Publish(author, "a removed post"); + using (var scope = _host.Services.CreateScope()) + Assert.True((await scope.ServiceProvider.GetRequiredService().Remove(author, removed.ID, token)).Ok); + + var (shown, body) = await Page($"/@{author.UserName}/{open.ID}"); + Assert.Equal(HttpStatusCode.OK, shown.StatusCode); + Hardened(shown); + Assert.Contains("an open post", body); + Assert.Contains($"href=\"{author.PostUri(open.ID)}\"", body); + Assert.Equal(HttpStatusCode.OK, (await Page($"/@{author.UserName}/{unlisted.ID}")).Response.StatusCode); + + foreach (var path in new[] + { + $"/@{author.UserName}/{followersOnly.ID}", + $"/@{author.UserName}/{direct.ID}", + $"/@{author.UserName}/{located.ID}", + $"/@{author.UserName}/{removed.ID}", + $"/@{other.UserName}/{open.ID}", + $"/@{author.UserName}/not-a-post-id", + $"/@{author.UserName}/ffffffffffffffffffffffff", + $"/@nobody{Guid.NewGuid():N}"[..20] + $"/{open.ID}" + }) + { + var (response, missing) = await Page(path); + Assert.True(response.StatusCode == HttpStatusCode.NotFound, $"{path} answered {(int)response.StatusCode}"); + Assert.DoesNotContain("a followers post", missing); + Assert.DoesNotContain("a direct post", missing); + } + } + + [Fact] + public async Task A_circle_has_no_page_a_community_has_one_and_the_instance_actor_has_none() + { + var token = TestContext.Current.CancellationToken; + var member = await Author("member"); + GroupEntity Group(GroupKind kind) => new() + { + UserName = $"{kind}{Guid.NewGuid():N}"[..20].ToLowerInvariant(), Name = $"The {kind}", Kind = kind, PostingPolicy = PostingPolicy.Followers, + Members = new() { new GroupMember { AvatarId = member.Id, Role = GroupRole.Owner } } + }; + var circle = Group(GroupKind.Circle); + var community = Group(GroupKind.Community); + await DB.Default.SaveAsync(new[] { circle, community }, token); + var inCommunity = await Publish(member, "said in the community", groupId: community.ID); + var inCircle = await Seeded(member, "said in the circle", PostVisibility.Circle); + await DB.Default.Update().MatchID(inCircle.ID).Modify(p => p.GroupId, circle.ID).ExecuteAsync(token); + + var (circlePage, circleBody) = await Page($"/@{circle.UserName}"); + var (communityPage, communityBody) = await Page($"/@{community.UserName}"); + + Assert.Equal(HttpStatusCode.NotFound, circlePage.StatusCode); + Assert.DoesNotContain("The Circle", circleBody); + Assert.Equal(HttpStatusCode.NotFound, (await Page($"/@{circle.UserName}/{inCircle.ID}")).Response.StatusCode); + Assert.Equal(HttpStatusCode.NotFound, (await Page($"/@{circle.UserName}", "application/activity+json")).Response.StatusCode); + Assert.Equal(HttpStatusCode.OK, communityPage.StatusCode); + Hardened(communityPage); + Assert.Contains("The Community", communityBody); + Assert.Contains("said in the community", communityBody); + Assert.Contains($"href=\"{member.PostHtmlUrl(inCommunity.ID)}\"", communityBody); + Assert.Equal(HttpStatusCode.NotFound, (await Page("/@privapub")).Response.StatusCode); + } + + [Fact] + public async Task An_activitypub_request_for_a_page_is_sent_to_the_actor_or_the_object() + { + var author = await Author(); + var open = await Publish(author, "fetch me as json"); + var followersOnly = await Publish(author, "not for strangers", PostVisibility.FollowersOnly); + + var (profile, _) = await Page($"/@{author.UserName}", "application/activity+json"); + var (post, _) = await Page($"/@{author.UserName}/{open.ID}", "application/ld+json; profile=\"https://www.w3.org/ns/activitystreams\""); + var (hidden, _) = await Page($"/@{author.UserName}/{followersOnly.ID}", "application/activity+json"); + + Assert.Equal(HttpStatusCode.Redirect, profile.StatusCode); + Assert.Equal(author.Uri, profile.Headers.Location!.ToString()); + Assert.Equal(HttpStatusCode.Redirect, post.StatusCode); + Assert.Equal(author.PostUri(open.ID), post.Headers.Location!.ToString()); + Assert.Equal(HttpStatusCode.NotFound, hidden.StatusCode); + Assert.False(profile.Headers.Contains("Content-Security-Policy")); + } + + [Fact] + public async Task Markup_in_a_post_a_title_or_a_bio_is_escaped_never_run() + { + var token = TestContext.Current.CancellationToken; + var author = await Author(); + await DB.Default.Update().MatchID(author.Id).Modify(a => a.Biography, " hi").ExecuteAsync(token); + var rendered = await Publish(author, " **bold** "); + var legacy = await Seeded(author, "", PostVisibility.Public); + await DB.Default.Update().MatchID(legacy.ID).Modify(p => p.Title, "").ExecuteAsync(token); + + var (profile, profileBody) = await Page($"/@{author.UserName}"); + var (post, postBody) = await Page($"/@{author.UserName}/{rendered.ID}"); + var (old, oldBody) = await Page($"/@{author.UserName}/{legacy.ID}"); + + Assert.All(new[] { profile, post, old }, r => Assert.Equal(HttpStatusCode.OK, r.StatusCode)); + foreach (var body in new[] { profileBody, postBody, oldBody }) + { + Assert.DoesNotContain("bold", postBody); + Assert.Contains("<script>alert(legacy)</script>", oldBody); + Assert.Contains("<script>alert(title)</script>", oldBody); + } + } +} diff --git a/PrivaPub/Api/Mastodon/Auth/OAuthPruner.cs b/PrivaPub/Api/Mastodon/Auth/OAuthPruner.cs index b2b8e7f..88d1818 100644 --- a/PrivaPub/Api/Mastodon/Auth/OAuthPruner.cs +++ b/PrivaPub/Api/Mastodon/Auth/OAuthPruner.cs @@ -23,10 +23,7 @@ namespace PrivaPub.Api.Mastodon.Auth try { await Task.Delay(Interval, stoppingToken); - using var scope = _services.CreateScope(); - var threshold = DateTimeOffset.UtcNow - Threshold; - var tokens = await scope.ServiceProvider.GetRequiredService().PruneAsync(threshold, stoppingToken); - var authorizations = await scope.ServiceProvider.GetRequiredService().PruneAsync(threshold, stoppingToken); + var (tokens, authorizations) = await Prune(DateTimeOffset.UtcNow - Threshold, stoppingToken); if (tokens + authorizations > 0) _logger.LogInformation("{Service} removed {Tokens} tokens and {Authorizations} authorizations", nameof(OAuthPruner), tokens, authorizations); } @@ -40,5 +37,14 @@ namespace PrivaPub.Api.Mastodon.Auth } } } + + //one pass: tokens and authorizations created before the threshold that are no longer valid + public async Task<(long Tokens, long Authorizations)> Prune(DateTimeOffset threshold, CancellationToken token) + { + using var scope = _services.CreateScope(); + var tokens = await scope.ServiceProvider.GetRequiredService().PruneAsync(threshold, token); + var authorizations = await scope.ServiceProvider.GetRequiredService().PruneAsync(threshold, token); + return (tokens, authorizations); + } } } diff --git a/PrivaPub/Api/Mastodon/Controllers/AccountsController.cs b/PrivaPub/Api/Mastodon/Controllers/AccountsController.cs index 21aeba2..5d1118d 100644 --- a/PrivaPub/Api/Mastodon/Controllers/AccountsController.cs +++ b/PrivaPub/Api/Mastodon/Controllers/AccountsController.cs @@ -155,13 +155,13 @@ namespace PrivaPub.Api.Mastodon.Controllers if (local == default) return Json(Array.Empty()); var pinIds = (await DB.Default.Find().Match(p => p.AvatarId == local.Id).Sort(p => p.ID, Order.Descending).ExecuteAsync(token)).Select(p => p.PostId).ToList(); - var pinnedPosts = await _dbEntities.Posts.Match(p => pinIds.Contains(p.ID) && !p.DeletedAt.HasValue).Match(VisibilityPolicy.IsPublic).ExecuteAsync(token); + var pinnedPosts = await _dbEntities.Posts.Match(p => pinIds.Contains(p.ID)).Match(VisibilityPolicy.IsPublic).ExecuteAsync(token); return Json(await _mapper.Statuses(pinIds.Select(id => pinnedPosts.FirstOrDefault(p => p.ID == id)).Where(p => p != default).ToList(), MyId, token)); } var query = local != default - ? _dbEntities.Posts.Match(p => p.GroupUserId == local.Id && !p.IsFederatedCopy && !p.DeletedAt.HasValue) - : _dbEntities.Posts.Match(p => p.ActorURI == remote.ActorURI && p.IsFederatedCopy && !p.DeletedAt.HasValue); + ? _dbEntities.Posts.Match(p => p.GroupUserId == local.Id && !p.IsFederatedCopy).Match(VisibilityPolicy.IsShown) + : _dbEntities.Posts.Match(p => p.ActorURI == remote.ActorURI && p.IsFederatedCopy).Match(VisibilityPolicy.IsShown); var viewerFollows = MyId != default && local != default && await _dbEntities.Followings.Match(f => f.AvatarId == MyId && f.TargetAccountId == local.Id && f.State == FollowState.Accepted).ExecuteAnyAsync(token); if (MyId != id) diff --git a/PrivaPub/Api/Mastodon/Controllers/ProvenanceController.cs b/PrivaPub/Api/Mastodon/Controllers/ProvenanceController.cs index 9b3b185..7a0cdbf 100644 --- a/PrivaPub/Api/Mastodon/Controllers/ProvenanceController.cs +++ b/PrivaPub/Api/Mastodon/Controllers/ProvenanceController.cs @@ -26,9 +26,9 @@ namespace PrivaPub.Api.Mastodon.Controllers [HttpGet("/api/privapub/v1/statuses/{id}/provenance"), Scope("read:statuses", requiresUser: false), AllowAnonymous] public async Task Status(string id, CancellationToken token) { - var post = await _dbEntities.Posts.Match(p => p.ID == id && !p.DeletedAt.HasValue).ExecuteFirstAsync(token); + var post = await _dbEntities.Posts.Match(p => p.ID == id).Match(VisibilityPolicy.IsShown).ExecuteFirstAsync(token); if (post?.ReblogOfPostId != default) - post = await _dbEntities.Posts.Match(p => p.ID == post.ReblogOfPostId && !p.DeletedAt.HasValue).ExecuteFirstAsync(token); + post = await _dbEntities.Posts.Match(p => p.ID == post.ReblogOfPostId).Match(VisibilityPolicy.IsShown).ExecuteFirstAsync(token); if (post == default || post.Visibility == PostVisibility.LocalGeo || !await VisibilityPolicy.CanSee(post, MyId, token)) return NotFoundError(); if (!post.IsFederatedCopy) diff --git a/PrivaPub/Api/Mastodon/Controllers/StatusesController.cs b/PrivaPub/Api/Mastodon/Controllers/StatusesController.cs index 7fcc0b3..2b2778c 100644 --- a/PrivaPub/Api/Mastodon/Controllers/StatusesController.cs +++ b/PrivaPub/Api/Mastodon/Controllers/StatusesController.cs @@ -231,7 +231,7 @@ namespace PrivaPub.Api.Mastodon.Controllers { if (await Visible(id, token) == default) return NotFoundError(); - var reblogs = await _dbEntities.Posts.Match(p => p.ReblogOfPostId == id && !p.DeletedAt.HasValue).Sort(p => p.ID, Order.Descending).Limit(Limit(40, 80)).ExecuteAsync(token); + var reblogs = await _dbEntities.Posts.Match(p => p.ReblogOfPostId == id).Match(VisibilityPolicy.IsShown).Sort(p => p.ID, Order.Descending).Limit(Limit(40, 80)).ExecuteAsync(token); var accounts = await _mapper.Accounts(reblogs.Select(MastodonMapper.AuthorOf), token); return Json(reblogs.Select(r => accounts.GetValueOrDefault(MastodonMapper.AuthorOf(r))).Where(a => a != default).ToList()); } diff --git a/PrivaPub/Api/Mastodon/Controllers/TimelinesController.cs b/PrivaPub/Api/Mastodon/Controllers/TimelinesController.cs index e34649a..799b714 100644 --- a/PrivaPub/Api/Mastodon/Controllers/TimelinesController.cs +++ b/PrivaPub/Api/Mastodon/Controllers/TimelinesController.cs @@ -32,7 +32,7 @@ namespace PrivaPub.Api.Mastodon.Controllers { var entries = await Page.From(Params, Limit()).Fetch(_dbEntities.TimelineEntries.Match(e => e.AvatarId == MyId), e => e.PostId, token); var ids = entries.Select(e => e.PostId).ToList(); - var posts = (await _dbEntities.Posts.Match(p => ids.Contains(p.ID) && !p.DeletedAt.HasValue).ExecuteAsync(token)).ToDictionary(p => p.ID); + var posts = (await _dbEntities.Posts.Match(p => ids.Contains(p.ID)).Match(VisibilityPolicy.IsShown).ExecuteAsync(token)).ToDictionary(p => p.ID); var statuses = await _mapper.Statuses(ids.Where(posts.ContainsKey).Select(id => posts[id]).ToList(), MyId, token); Link("/api/v1/timelines/home", entries.LastOrDefault()?.PostId, entries.FirstOrDefault()?.PostId); return Json(statuses); @@ -41,7 +41,7 @@ namespace PrivaPub.Api.Mastodon.Controllers [HttpGet("/api/v1/timelines/public"), Scope("read:statuses", requiresUser: false), Microsoft.AspNetCore.Authorization.AllowAnonymous] public async Task Public(CancellationToken token) { - var query = _dbEntities.Posts.Match(p => p.Visibility == PostVisibility.Public && !p.DeletedAt.HasValue && p.ReblogOfPostId == null); + var query = _dbEntities.Posts.Match(p => p.Visibility == PostVisibility.Public && p.ReblogOfPostId == null).Match(VisibilityPolicy.IsPublic); if (Params.Bool("local") == true) query.Match(p => !p.IsFederatedCopy); if (Params.Bool("remote") == true) @@ -55,7 +55,7 @@ namespace PrivaPub.Api.Mastodon.Controllers public async Task Tag(string hashtag, CancellationToken token) { var tag = hashtag.TrimStart('#').ToLowerInvariant(); - var query = _dbEntities.Posts.Match(p => p.Tags.Contains(tag) && p.Visibility == PostVisibility.Public && !p.DeletedAt.HasValue && p.ReblogOfPostId == null); + var query = _dbEntities.Posts.Match(p => p.Tags.Contains(tag) && p.Visibility == PostVisibility.Public && p.ReblogOfPostId == null).Match(VisibilityPolicy.IsPublic); if (Params.Bool("local") == true) query.Match(p => !p.IsFederatedCopy); return await Respond(query, $"/api/v1/timelines/tag/{hashtag}", token); @@ -86,7 +86,7 @@ namespace PrivaPub.Api.Mastodon.Controllers var conversations = new List(); foreach (var group in groups) { - var last = await _dbEntities.Posts.Match(p => p.ConversationId == group.ID && !p.DeletedAt.HasValue).Sort(p => p.ID, Order.Descending).ExecuteFirstAsync(token); + var last = await _dbEntities.Posts.Match(p => p.ConversationId == group.ID).Match(VisibilityPolicy.IsShown).Sort(p => p.ID, Order.Descending).ExecuteFirstAsync(token); if (last == default) continue; var others = new List(); @@ -111,7 +111,7 @@ namespace PrivaPub.Api.Mastodon.Controllers var group = await _dbEntities.DmGroups.Match(g => g.ID == id && g.Members.Any(m => !m.IsForeign && m.AvatarId == MyId)).ExecuteFirstAsync(token); if (group == default) return NotFoundError(); - var last = await _dbEntities.Posts.Match(p => p.ConversationId == group.ID && !p.DeletedAt.HasValue).Sort(p => p.ID, Order.Descending).ExecuteFirstAsync(token); + var last = await _dbEntities.Posts.Match(p => p.ConversationId == group.ID).Match(VisibilityPolicy.IsShown).Sort(p => p.ID, Order.Descending).ExecuteFirstAsync(token); return Json(new Conversation { Id = group.ID, LastStatus = last == default ? default : await _mapper.Status(last, MyId, token) }); } @@ -225,7 +225,7 @@ namespace PrivaPub.Api.Mastodon.Controllers var postIds = notifications.Where(n => n.PostId != default).Select(n => n.PostId).Distinct().ToList(); var posts = postIds.Count == 0 ? new List() - : await _dbEntities.Posts.Match(p => postIds.Contains(p.ID) && !p.DeletedAt.HasValue).ExecuteAsync(token); + : await _dbEntities.Posts.Match(p => postIds.Contains(p.ID)).Match(VisibilityPolicy.IsShown).ExecuteAsync(token); var statuses = (await _mapper.Statuses(posts, MyId, token)).ToDictionary(s => s.Id); return notifications .Where(n => accounts.ContainsKey(n.FromAccountId ?? string.Empty) && (n.PostId == default || statuses.ContainsKey(n.PostId))) diff --git a/PrivaPub/Api/Mastodon/Mappers/MastodonMapper.cs b/PrivaPub/Api/Mastodon/Mappers/MastodonMapper.cs index cc61a5e..036526d 100644 --- a/PrivaPub/Api/Mastodon/Mappers/MastodonMapper.cs +++ b/PrivaPub/Api/Mastodon/Mappers/MastodonMapper.cs @@ -157,7 +157,7 @@ namespace PrivaPub.Api.Mastodon.Mappers var originalIds = posts.Where(p => p.ReblogOfPostId != default).Select(p => p.ReblogOfPostId).Distinct().ToList(); var originals = originalIds.Count == 0 ? new Dictionary() - : (await _dbEntities.Posts.Match(p => originalIds.Contains(p.ID) && !p.DeletedAt.HasValue).ExecuteAsync(token)).ToDictionary(p => p.ID); + : (await _dbEntities.Posts.Match(p => originalIds.Contains(p.ID)).Match(VisibilityPolicy.IsShown).ExecuteAsync(token)).ToDictionary(p => p.ID); var all = posts.Concat(originals.Values).ToList(); var mentionUris = all.SelectMany(p => p.Mentions).Where(m => !m.IsLocal).Select(m => m.ActorURI).Distinct().ToList(); @@ -245,7 +245,7 @@ namespace PrivaPub.Api.Mastodon.Mappers : await FollowedLocalAuthors(viewerId, localAuthors, token); var quotedIds = nested ? new List() : all.Where(p => p.QuoteState == QuoteState.Accepted && p.QuotedPostId != default).Select(p => p.QuotedPostId).Distinct().ToList(); var quotedPosts = new List(); - foreach (var quotedPost in quotedIds.Count == 0 ? new List() : await _dbEntities.Posts.Match(p => quotedIds.Contains(p.ID) && !p.DeletedAt.HasValue).ExecuteAsync(token)) + foreach (var quotedPost in quotedIds.Count == 0 ? new List() : await _dbEntities.Posts.Match(p => quotedIds.Contains(p.ID)).Match(VisibilityPolicy.IsShown).ExecuteAsync(token)) if (quotedPost.Visibility != PostVisibility.LocalGeo && await VisibilityPolicy.CanSee(quotedPost, viewerId, token)) quotedPosts.Add(quotedPost); var quotedStatuses = quotedPosts.Count == 0 @@ -270,7 +270,7 @@ namespace PrivaPub.Api.Mastodon.Mappers var mapped = new List(); foreach (var post in posts) { - if (hidden.Contains(post.ActorURI) || post.ReblogOfPostId != default && originals.TryGetValue(post.ReblogOfPostId, out var hiddenOriginal) + if (!VisibilityPolicy.Shown(post) || hidden.Contains(post.ActorURI) || post.ReblogOfPostId != default && originals.TryGetValue(post.ReblogOfPostId, out var hiddenOriginal) && hidden.Contains(hiddenOriginal.ActorURI)) continue; var status = Map(post); diff --git a/PrivaPub/Domain/Content/LinkPreviews.cs b/PrivaPub/Domain/Content/LinkPreviews.cs index 7ba765c..3fd9faf 100644 --- a/PrivaPub/Domain/Content/LinkPreviews.cs +++ b/PrivaPub/Domain/Content/LinkPreviews.cs @@ -6,6 +6,7 @@ using MongoDB.Entities; using PrivaPub.Infrastructure.Statistics; using PrivaPub.Models.Statistics; +using PrivaPub.Domain.Privacy; using PrivaPub.Federation.Actors; using PrivaPub.Federation.Objects; using PrivaPub.Infrastructure.Http; @@ -68,8 +69,7 @@ namespace PrivaPub.Domain.Content public async Task Wanted(PostEntity post, CancellationToken token) { - if (!_options.CurrentValue.FetchLinkPreviews || post.Visibility is not (PostVisibility.Public or PostVisibility.Unlisted) || post.IsLocalOnly - || post.ReblogOfPostId != default || post.Link?.Title != default || post.Link == default && post.Media.Count > 0) + if (!_options.CurrentValue.FetchLinkPreviews || !Previewable(post) || post.Link == default && post.Media.Count > 0) return; var url = post.Link?.Href ?? FirstLink(post, _localActors.BaseAddress); if (url == default) @@ -87,7 +87,7 @@ namespace PrivaPub.Domain.Content public async Task Handle(Job job, CancellationToken token) { var post = await _dbEntities.Posts.MatchID(job.Payload).ExecuteFirstAsync(token); - if (post == default || post.DeletedAt.HasValue || post.Link?.Title != default) + if (!VisibilityPolicy.Shown(post) || !Previewable(post)) return JobOutcome.Done; var url = post.Link?.Href ?? FirstLink(post, _localActors.BaseAddress); if (url == default) @@ -135,6 +135,10 @@ namespace PrivaPub.Domain.Content return preview; } + //only public posts (owner decision 1), checked again when the job runs; never a boost or a post whose card is complete + static bool Previewable(PostEntity post) => post.Visibility is (PostVisibility.Public or PostVisibility.Unlisted) && !post.IsLocalOnly + && post.ReblogOfPostId == default && post.Link?.Title == default; + public static LinkPreview Read(string url, Uri finalUri, string html) { var document = Parser.ParseDocument(html); diff --git a/PrivaPub/Domain/Media/MediaProxy.cs b/PrivaPub/Domain/Media/MediaProxy.cs index d3c8d96..a58040b 100644 --- a/PrivaPub/Domain/Media/MediaProxy.cs +++ b/PrivaPub/Domain/Media/MediaProxy.cs @@ -144,10 +144,7 @@ namespace PrivaPub.Domain.Media try { await Task.Delay(Interval, stoppingToken); - var cutoff = DateTime.UtcNow - UnattachedLifetime; - foreach (var stale in await DB.Default.Find().Match(m => m.PostId == null && m.CreatedAt < cutoff).Limit(500).ExecuteAsync(stoppingToken)) - await _media.Delete(stale); - TrimProxyCache(); + await Sweep(stoppingToken); } catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) { @@ -160,6 +157,15 @@ namespace PrivaPub.Domain.Media } } + //one pass: uploads left unattached for a day go, then the proxy cache is trimmed to its size, oldest first + public async Task Sweep(CancellationToken token) + { + var cutoff = DateTime.UtcNow - UnattachedLifetime; + foreach (var stale in await DB.Default.Find().Match(m => m.PostId == null && m.CreatedAt < cutoff).Limit(500).ExecuteAsync(token)) + await _media.Delete(stale); + TrimProxyCache(); + } + void TrimProxyCache() { var directory = new DirectoryInfo(_media.ProxyRoot); diff --git a/PrivaPub/Domain/Privacy/VisibilityPolicy.cs b/PrivaPub/Domain/Privacy/VisibilityPolicy.cs index 6846fbf..ecba0c7 100644 --- a/PrivaPub/Domain/Privacy/VisibilityPolicy.cs +++ b/PrivaPub/Domain/Privacy/VisibilityPolicy.cs @@ -11,14 +11,20 @@ namespace PrivaPub.Domain.Privacy { public static class VisibilityPolicy { - public static readonly Expression> IsPublic = p => - !p.DeletedAt.HasValue && (p.Visibility == PostVisibility.Public || p.Visibility == PostVisibility.Unlisted); + //neither deleted nor kept back after its remote author deleted their account (owner decision, 2026-10-03) + public static readonly Expression> IsShown = p => !p.DeletedAt.HasValue && !p.AuthorGone; + public static readonly Expression> IsPublic = p => + !p.DeletedAt.HasValue && !p.AuthorGone && (p.Visibility == PostVisibility.Public || p.Visibility == PostVisibility.Unlisted); + + static readonly Func IsShownCompiled = IsShown.Compile(); static readonly Func IsPublicCompiled = IsPublic.Compile(); + public static bool Shown(Post post) => post != default && IsShownCompiled(post); + public static async Task CanSee(Post post, string viewerAvatarId, CancellationToken token) { - if (post == default || post.DeletedAt.HasValue) + if (!Shown(post)) return false; if (IsPublicCompiled(post)) return true; diff --git a/PrivaPub/Domain/Timelines/TimelineService.cs b/PrivaPub/Domain/Timelines/TimelineService.cs index 642f95c..edcf6dc 100644 --- a/PrivaPub/Domain/Timelines/TimelineService.cs +++ b/PrivaPub/Domain/Timelines/TimelineService.cs @@ -5,6 +5,7 @@ using MongoDB.Entities; using PrivaPub.ClientModels; using PrivaPub.ClientModels.Post; using PrivaPub.ClientModels.Social; +using PrivaPub.Domain.Privacy; using PrivaPub.Models.Post; using PrivaPub.Models.Social; using PrivaPub.Resources; @@ -46,7 +47,7 @@ namespace PrivaPub.Domain.Timelines var ids = entries.Select(e => e.PostId).Concat(entries.Where(e => e.ReblogOfPostId != default).Select(e => e.ReblogOfPostId)).Distinct().ToList(); var posts = ids.Count == 0 ? new Dictionary() - : (await _dbEntities.Posts.Match(p => ids.Contains(p.ID) && !p.DeletedAt.HasValue).ExecuteAsync(token)).ToDictionary(p => p.ID); + : (await _dbEntities.Posts.Match(p => ids.Contains(p.ID)).Match(VisibilityPolicy.IsShown).ExecuteAsync(token)).ToDictionary(p => p.ID); result.Data = entries .Where(e => posts.ContainsKey(e.PostId) && (e.ReblogOfPostId == default || posts.ContainsKey(e.ReblogOfPostId))) diff --git a/PrivaPub/Federation/Controllers/PeasantsController.cs b/PrivaPub/Federation/Controllers/PeasantsController.cs index 430ed61..5cce633 100644 --- a/PrivaPub/Federation/Controllers/PeasantsController.cs +++ b/PrivaPub/Federation/Controllers/PeasantsController.cs @@ -314,7 +314,7 @@ namespace PrivaPub.Federation.Controllers async Task AnnounceFor(PostEntity reblog, CancellationToken token) { var original = await _dbEntities.Posts.MatchID(reblog.ReblogOfPostId).ExecuteFirstAsync(token); - if (original == default || original.DeletedAt.HasValue) + if (!VisibilityPolicy.Shown(original)) return default; return new JsonObject { diff --git a/PrivaPub/Federation/Inbox/Handlers/DeleteHandler.cs b/PrivaPub/Federation/Inbox/Handlers/DeleteHandler.cs index 01eb932..6e75dce 100644 --- a/PrivaPub/Federation/Inbox/Handlers/DeleteHandler.cs +++ b/PrivaPub/Federation/Inbox/Handlers/DeleteHandler.cs @@ -70,6 +70,7 @@ namespace PrivaPub.Federation.Inbox.Handlers } await DB.Default.DeleteAsync(f => f.TargetActorURI == actor.ActorURI); await DB.Default.DeleteAsync(e => e.AuthorAccountId == actor.ID); + await DB.Default.Update().Match(p => p.ActorURI == actor.ActorURI).Modify(p => p.AuthorGone, true).ExecuteAsync(token); return; } diff --git a/PrivaPub/Federation/Inbox/RemotePosts.cs b/PrivaPub/Federation/Inbox/RemotePosts.cs index 2e6bf6d..192075e 100644 --- a/PrivaPub/Federation/Inbox/RemotePosts.cs +++ b/PrivaPub/Federation/Inbox/RemotePosts.cs @@ -102,6 +102,7 @@ namespace PrivaPub.Federation.Inbox AnsweringToPostId = parent?.ID, InReplyToAccountId = parent?.AuthorAccountId ?? parent?.GroupUserId, IsFederatedCopy = true, + AuthorGone = author.AccountState == AvatarAccountState.Deleted, CreationDate = note.Published, UpdateDate = note.Updated ?? note.Published, EditedAt = note.Updated diff --git a/PrivaPub/Federation/Objects/InstanceDescriber.cs b/PrivaPub/Federation/Objects/InstanceDescriber.cs index a5d1bf4..8b0522f 100644 --- a/PrivaPub/Federation/Objects/InstanceDescriber.cs +++ b/PrivaPub/Federation/Objects/InstanceDescriber.cs @@ -30,15 +30,18 @@ namespace PrivaPub.Federation.Objects { var host = job.Payload; using var scope = HttpScope.For("nodeinfo"); - using var links = await _http.GetJson($"https://{host}/.well-known/nodeinfo", "application/json", sign: default, token); + using var links = await _http.GetJson(Address($"https://{host}/.well-known/nodeinfo"), "application/json", sign: default, token); var href = links == default ? default : Schemas.Select(schema => LinkFor(links.Root, schema)).FirstOrDefault(link => link != default); if (href == default) return await Described(host, default, "no NodeInfo", token); - using var nodeInfo = await _http.GetJson(href, "application/json", sign: default, token); + using var nodeInfo = await _http.GetJson(Address(href), "application/json", sign: default, token); return nodeInfo == default ? await Described(host, default, "NodeInfo unreadable", token) : await Described(host, nodeInfo.Root, default, token); } + //where an https address is fetched from: itself, except in tests, which point it at a plain-http peer + protected virtual string Address(string url) => url; + static async Task Described(string host, JsonElement? nodeInfo, string error, CancellationToken token) { var update = DB.Default.Update() diff --git a/PrivaPub/Models/Post/Post.cs b/PrivaPub/Models/Post/Post.cs index 823959f..e897d86 100644 --- a/PrivaPub/Models/Post/Post.cs +++ b/PrivaPub/Models/Post/Post.cs @@ -48,6 +48,7 @@ namespace PrivaPub.Models.Post public double? Distance { get; set; } public bool HasContentWarning { get; set; } = false; public bool IsLocalOnly { get; set; } + public bool AuthorGone { get; set; }//its remote author deleted their account: kept, but hidden everywhere (VisibilityPolicy) public bool IsFederatedCopy { get; set; } public string ObjectURI { get; set; }//the Note's id diff --git a/PrivaPub/Services/PostsService.cs b/PrivaPub/Services/PostsService.cs index b976796..89c96ef 100644 --- a/PrivaPub/Services/PostsService.cs +++ b/PrivaPub/Services/PostsService.cs @@ -15,6 +15,7 @@ using PrivaPub.StaticServices; using System.Text.Json.Nodes; using PostEntity = PrivaPub.Models.Post.Post; +using PrivaPub.Domain.Privacy; using PrivaPub.Domain.Statuses; using PrivaPub.Domain.Timelines; using PrivaPub.Federation.Actors; @@ -123,7 +124,7 @@ namespace PrivaPub.Services query.Match(p => p.GroupId == groupId); } - var posts = await query.Sort(p => p.CreationDate, Order.Descending).Limit(PageSize).ExecuteAsync(token); + var posts = await query.Match(VisibilityPolicy.IsShown).Sort(p => p.CreationDate, Order.Descending).Limit(PageSize).ExecuteAsync(token); result.Data = posts.Select(ToView).ToList(); return result; } @@ -183,6 +184,7 @@ namespace PrivaPub.Services var dms = await _dbEntities.Posts .Match(p => p.ConversationId == dmGroupId && p.Visibility == PostVisibility.Direct) + .Match(VisibilityPolicy.IsShown) .Sort(p => p.CreationDate, Order.Descending) .Limit(PageSize) .ExecuteAsync(token);