diff --git a/CLAUDE.md b/CLAUDE.md index eaf6c0f..b6647b5 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -272,13 +272,15 @@ cd /var/www/privapub.thepra.dev && sudo -u www-data ASPNETCORE_ENVIRONMENT=Produ activity, inbox, key and signed headers; received time; up to 10 later revisions). Write it right after the post's save: `CreateHandler` (delivered) and `RemotePosts.StoreContext` (fetched) do, and `UpdateHandler` appends a revision. Delivery details reach them through `Arrival.Current`, which `InboxProcessor` sets for the handler's - duration. The raw form lives outside `Post` so timelines never load it. Read through + duration. A fetched record has no signature, key or `@context` of the activity that caused the fetch (its activity + fields name that trigger), and every record stores its `Extensions` and `ContextNamespaces` for statistics. The raw form lives outside `Post` so timelines never load it. Read through `/api/privapub/v1/statuses/:id/provenance` and `/api/privapub/v1/instances/:host`. - **Remote objects are parsed for every shape in `ObjectShapes`:** `url`/`icon`/`image` as a value, an object or an array; Markdown `content`; missing `mediaType`s inferred; thumbnails from any of their five places; and the typed `Link`, `Video`, `Audio` and `Event` details. A Mastodon API card is built from those, never by fetching the linked page (that fetch is the owner's decision 1, for P6). -- **An `Update` is an edit only when its `updated` is newer than ours** (`UpdateHandler.IsEdit`). Otherwise it refreshes +- **An `Update` is an edit only when its `updated` is newer than ours** (`RemoteEdits.IsEdit`; plain and community-wrapped + Updates both go through `RemoteEdits.Apply`, and deletes through `RemoteDeletes.Remove`). Otherwise it refreshes the poll, video, audio and event details and nothing else, and leaves no revision: Mastodon and Misskey refresh poll counts with bare Updates. - **A poll vote is a `Note` with a `name`, an `inReplyTo` that is a poll we hold, and no content.** `CreateHandler` diff --git a/PrivaPub.Tests/Federation/ActorFeaturesTests.cs b/PrivaPub.Tests/Federation/ActorFeaturesTests.cs new file mode 100644 index 0000000..a276774 --- /dev/null +++ b/PrivaPub.Tests/Federation/ActorFeaturesTests.cs @@ -0,0 +1,50 @@ +using PrivaPub.Federation.Actors; + +using System.Security.Cryptography; +using System.Text.Json; +using System.Text.Json.Nodes; + +namespace PrivaPub.Tests.Federation +{ + public class ActorFeaturesTests + { + [Fact] + public void A_rich_actor_reports_the_extensions_it_uses() + { + using var rsa = RSA.Create(4096); + var actor = new JsonObject + { + ["id"] = "https://social.example/users/a", + ["type"] = "Person", + ["featured"] = "https://social.example/users/a/featured", + ["alsoKnownAs"] = new JsonArray("https://old.example/users/a"), + ["endpoints"] = new JsonObject { ["sharedInbox"] = "https://social.example/inbox" }, + ["assertionMethod"] = new JsonArray(), + ["attachment"] = new JsonArray(new JsonObject { ["type"] = "PropertyValue", ["name"] = "x", ["value"] = "y" }), + ["manuallyApprovesFollowers"] = true, + ["discoverable"] = false, + ["indexable"] = true, + ["isCat"] = true, + ["_misskey_summary"] = "hi", + ["publicKey"] = new JsonObject { ["id"] = "https://social.example/users/a#main-key", ["publicKeyPem"] = rsa.ExportSubjectPublicKeyInfoPem() } + }; + + var features = ActorFeatures.Detect(JsonDocument.Parse(actor.ToJsonString()).RootElement); + + Assert.Equal(new[] + { + "featured", "also-known-as", "shared-inbox", "fep-521a-assertion-method", "property-value", "misskey", "is-cat", + "indexable", "undiscoverable", "locked", "key:rsa-4096" + }, features); + } + + [Fact] + public void A_bare_actor_reports_nothing_and_junk_is_survived() + { + Assert.Empty(ActorFeatures.Detect(JsonDocument.Parse("{\"id\":\"https://a.example/u\",\"type\":\"Person\"}").RootElement)); + Assert.Equal(new[] { "public-key-array" }, + ActorFeatures.Detect(JsonDocument.Parse("{\"publicKey\":[{\"publicKeyPem\":\"not a key\"}]}").RootElement)); + Assert.Empty(ActorFeatures.Detect(JsonDocument.Parse("[1,2]").RootElement)); + } + } +} diff --git a/PrivaPub.Tests/Federation/CommunityEditsTests.cs b/PrivaPub.Tests/Federation/CommunityEditsTests.cs new file mode 100644 index 0000000..692bc59 --- /dev/null +++ b/PrivaPub.Tests/Federation/CommunityEditsTests.cs @@ -0,0 +1,131 @@ +using MongoDB.Entities; + +using PrivaPub.ClientModels.Social; +using PrivaPub.Federation.Objects; +using PrivaPub.Models.Federation; +using PrivaPub.Models.Post; +using PrivaPub.Models.Social; +using PrivaPub.Tests.Support; + +using System.Text.Json.Nodes; + +namespace PrivaPub.Tests.Federation +{ + [Trait("Category", "Integration")] + public sealed class CommunityEditsTests : IAsyncLifetime + { + Harness _harness; + + public async ValueTask InitializeAsync() + { + Assert.SkipUnless(MongoFixture.Enabled, MongoFixture.Skip); + _harness = await Harness.Start(); + } + + public async ValueTask DisposeAsync() + { + if (_harness != default) + await _harness.DisposeAsync(); + } + + static string Origin(RemoteActor actor) => new Uri(actor.Id).GetLeftPart(UriPartial.Authority); + + async Task<(RemoteActor Community, RemoteActor Poster, JsonObject Note, Post Post)> Announced() + { + var token = TestContext.Current.CancellationToken; + var (root, alice) = await _harness.Persona("alice"); + var community = new RemoteActor(_harness.Peer, "cats", type: "Group"); + var poster = new RemoteActor(_harness.Peer, "poster"); + await _harness.Follows.Follow(root, new FollowForm { AvatarId = alice.Id, Target = community.Id }, token); + await DB.Default.Update().Match(f => f.AvatarId == alice.Id).Modify(f => f.State, FollowState.Accepted).ExecuteAsync(token); + var note = new JsonObject + { + ["id"] = $"{Origin(poster)}/notes/{Guid.NewGuid():N}", + ["type"] = "Page", + ["name"] = "a title", + ["attributedTo"] = poster.Id, + ["content"] = "

first

", + ["to"] = new JsonArray(community.Id, Addressing.Public), + ["audience"] = community.Id, + ["published"] = DateTime.UtcNow.AddMinutes(-5).ToString("O") + }; + _harness.Peer.Serve(new Uri(note["id"]!.GetValue()).AbsolutePath, note.ToJsonString()); + await Announce(community, new JsonObject { ["id"] = $"{Origin(poster)}/create/{Guid.NewGuid():N}", ["type"] = "Create", ["actor"] = poster.Id, ["object"] = note.DeepClone() }); + var post = await DB.Default.Find().Match(p => p.ObjectURI == note["id"]!.GetValue()).ExecuteSingleAsync(token); + return (community, poster, note, post); + } + + Task Announce(RemoteActor community, JsonObject inner) => + _harness.Deliver(community, "/human-centipede", new JsonObject + { + ["id"] = $"{Origin(community)}/announce/{Guid.NewGuid():N}", + ["type"] = "Announce", + ["actor"] = community.Id, + ["to"] = new JsonArray(Addressing.Public), + ["object"] = inner + }); + + [Fact] + public async Task A_community_edit_is_an_edit_only_when_newer_and_keeps_its_history() + { + var token = TestContext.Current.CancellationToken; + var (community, poster, note, post) = await Announced(); + + var refresh = (JsonObject)note.DeepClone(); + _harness.Peer.Serve(new Uri(note["id"]!.GetValue()).AbsolutePath, refresh.ToJsonString()); + await Announce(community, new JsonObject { ["id"] = $"{Origin(poster)}/update/{Guid.NewGuid():N}", ["type"] = "Update", ["actor"] = poster.Id, ["object"] = note["id"]!.GetValue() }); + var afterRefresh = await DB.Default.Find().OneAsync(post.ID, token); + + var edit = (JsonObject)note.DeepClone(); + edit["content"] = "

second

"; + edit["updated"] = DateTime.UtcNow.ToString("O"); + _harness.Peer.Serve(new Uri(note["id"]!.GetValue()).AbsolutePath, edit.ToJsonString()); + await Announce(community, new JsonObject { ["id"] = $"{Origin(poster)}/update/{Guid.NewGuid():N}", ["type"] = "Update", ["actor"] = poster.Id, ["object"] = note["id"]!.GetValue() }); + var afterEdit = await DB.Default.Find().OneAsync(post.ID, token); + var record = await DB.Default.Find().Match(r => r.ObjectURI == post.ObjectURI).ExecuteSingleAsync(token); + + Assert.Empty(afterRefresh.Revisions); + Assert.Null(afterRefresh.EditedAt); + Assert.Equal("

second

", afterEdit.ContentHtml); + Assert.Equal("a title", afterEdit.Title); + Assert.Single(afterEdit.Revisions); + Assert.NotNull(afterEdit.EditedAt); + Assert.Equal(2, record.Revisions.Count); + Assert.Equal(new[] { "refresh", "edit" }, _harness.Ledger.Of("in").Where(e => e.Activity == "Announce" && e.Object == "Update").Select(e => e.Reason)); + } + + [Fact] + public async Task A_community_delete_leaves_a_tombstone_and_no_trace() + { + var token = TestContext.Current.CancellationToken; + var (community, poster, note, post) = await Announced(); + _harness.Peer.Serve(new Uri(note["id"]!.GetValue()).AbsolutePath, new JsonObject { ["id"] = note["id"]!.GetValue(), ["type"] = "Tombstone" }.ToJsonString()); + + await Announce(community, new JsonObject { ["id"] = $"{Origin(poster)}/delete/{Guid.NewGuid():N}", ["type"] = "Delete", ["actor"] = poster.Id, ["object"] = note["id"]!.GetValue() }); + + Assert.False(await DB.Default.Find().Match(p => p.ID == post.ID).ExecuteAnyAsync(token)); + Assert.False(await DB.Default.Find().Match(r => r.ObjectURI == post.ObjectURI).ExecuteAnyAsync(token)); + Assert.False(await DB.Default.Find().Match(e => e.PostId == post.ID).ExecuteAnyAsync(token)); + Assert.True(await DB.Default.Find().Match(d => d.ObjectURI == post.ObjectURI).ExecuteAnyAsync(token)); + } + + [Fact] + public async Task A_fetched_object_does_not_wear_the_signature_of_the_activity_that_caused_the_fetch() + { + var token = TestContext.Current.CancellationToken; + var (_, _, _, post) = await Announced(); + + var record = await DB.Default.Find().Match(r => r.ObjectURI == post.ObjectURI).ExecuteSingleAsync(token); + + Assert.Equal(ObjectPath.Fetched, record.Path); + Assert.Null(record.KeyId); + Assert.Null(record.SignatureScheme); + Assert.Empty(record.SignedHeaders); + Assert.Null(record.ActivityContext); + Assert.Equal("Announce", record.ActivityType); + Assert.InRange(record.ReceivedAt, DateTime.UtcNow.AddMinutes(-1), DateTime.UtcNow.AddSeconds(1)); + Assert.Contains("fep-1b12-audience", record.Extensions); + Assert.NotNull(record.ContextNamespaces); + } + } +} diff --git a/PrivaPub.Tests/Federation/RemoteActorServiceTests.cs b/PrivaPub.Tests/Federation/RemoteActorServiceTests.cs index 3a99c27..380c07c 100644 --- a/PrivaPub.Tests/Federation/RemoteActorServiceTests.cs +++ b/PrivaPub.Tests/Federation/RemoteActorServiceTests.cs @@ -32,6 +32,16 @@ namespace PrivaPub.Tests.Federation await _peer.DisposeAsync(); } + [Fact] + public async Task A_stored_actor_remembers_the_extensions_its_document_uses() + { + var bob = new RemoteActor(_peer, "bob"); + + var stored = await _service.GetActor(bob.Id, refresh: true, TestContext.Current.CancellationToken); + + Assert.Contains("key:rsa-2048", stored.Features); + } + static string Actor(string id, string keyId = default, string owner = default, string pem = default, string type = "Person") => new JsonObject { diff --git a/PrivaPub.Tests/Infrastructure/MigrationTests.cs b/PrivaPub.Tests/Infrastructure/MigrationTests.cs index 611a86c..3f4adc4 100644 --- a/PrivaPub.Tests/Infrastructure/MigrationTests.cs +++ b/PrivaPub.Tests/Infrastructure/MigrationTests.cs @@ -1,6 +1,7 @@ using MongoDB.Entities; using PrivaPub.Infrastructure.Data.Migrations; +using PrivaPub.Models.Federation; using PrivaPub.Models.Post; using PrivaPub.Tests.Support; @@ -46,5 +47,37 @@ namespace PrivaPub.Tests.Infrastructure Assert.Equal("conversation1", post.ConversationId); Assert.Equal("a1", post.AuthorAccountId); } + + [Fact] + public async Task Fetched_records_lose_the_signature_they_never_had_and_every_record_gets_its_extensions() + { + Assert.SkipUnless(MongoFixture.Enabled, MongoFixture.Skip); + var token = TestContext.Current.CancellationToken; + var fetched = new ObjectRecord + { + ObjectURI = $"https://r.example/{Guid.NewGuid():N}", Path = ObjectPath.Fetched, KeyId = "https://x.example/u#k", SignatureScheme = "draft-cavage", + Algorithm = "rsa-sha256", SignedHeaders = new() { "host" }, ActivityContext = "\"https://www.w3.org/ns/activitystreams\"", + Raw = "{\"type\":\"Note\",\"quoteUrl\":\"https://q.example/1\"}" + }; + var delivered = new ObjectRecord + { + ObjectURI = $"https://r.example/{Guid.NewGuid():N}", Path = ObjectPath.Delivered, KeyId = "https://x.example/u#k", SignatureScheme = "draft-cavage", + Raw = "not json" + }; + await DB.Default.SaveAsync(new[] { fetched, delivered }, token); + + await new _008_fetched_records_lose_the_trigger_signature().UpgradeAsync(); + await new _009_object_records_keep_their_extensions().UpgradeAsync(); + + var afterFetched = await DB.Default.Find().OneAsync(fetched.ID, token); + var afterDelivered = await DB.Default.Find().OneAsync(delivered.ID, token); + Assert.Null(afterFetched.KeyId); + Assert.Null(afterFetched.SignatureScheme); + Assert.Empty(afterFetched.SignedHeaders); + Assert.Null(afterFetched.ActivityContext); + Assert.Equal("https://x.example/u#k", afterDelivered.KeyId); + Assert.Contains("legacy-quote", afterFetched.Extensions); + Assert.Empty(afterDelivered.Extensions); + } } } diff --git a/PrivaPub.Tests/Support/Harness.cs b/PrivaPub.Tests/Support/Harness.cs index 13ad357..a7e5178 100644 --- a/PrivaPub.Tests/Support/Harness.cs +++ b/PrivaPub.Tests/Support/Harness.cs @@ -63,7 +63,7 @@ namespace PrivaPub.Tests.Support new QuoteRequestHandler(Quotes), new DislikeHandler(Db), new JoinHandler(Db, Local, Delivery), - new AnnounceHandler(Db, Local, RemotePosts, Fanout, Remote), + new AnnounceHandler(Db, Local, RemotePosts, Fanout, Remote, Records, Quotes), new CreateHandler(Db, Local, Remote, Delivery, new NoBlocks(), Fanout, RemotePosts, Groups, Records, Polls, new NoPreviews(), Quotes), new DeleteHandler(Db, Local, Remote, Delivery, Groups, Quotes), new UpdateHandler(Db, Local, Remote, Groups, Records, Quotes), diff --git a/PrivaPub/Api/Mastodon/Controllers/ProvenanceController.cs b/PrivaPub/Api/Mastodon/Controllers/ProvenanceController.cs index ba3f470..9b3b185 100644 --- a/PrivaPub/Api/Mastodon/Controllers/ProvenanceController.cs +++ b/PrivaPub/Api/Mastodon/Controllers/ProvenanceController.cs @@ -64,15 +64,16 @@ namespace PrivaPub.Api.Mastodon.Controllers Id = record.ActivityId, Type = record.ActivityType, Actor = record.ActivityActorURI }, Inbox = record?.Inbox, - Signature = record?.KeyId == default ? default : new ProvenanceSignature + FetchedBy = record?.Path == ObjectPath.Fetched ? "instance-actor" : default, + Signature = record?.KeyId == default || record.Path == ObjectPath.Fetched ? default : new ProvenanceSignature { Scheme = record.SignatureScheme, KeyId = record.KeyId, Algorithm = record.Algorithm, Headers = record.SignedHeaders }, ReceivedAt = record == default ? default : MastodonJson.Time(record.ReceivedAt), Published = record?.Published is { } published ? MastodonJson.Time(published) : default, Updated = record?.Updated is { } updated ? MastodonJson.Time(updated) : default, - Extensions = ObjectFeatures.Detect(raw), - ContextNamespaces = ObjectFeatures.Namespaces(raw, record?.ActivityContext), + Extensions = record?.Extensions ?? ObjectFeatures.Detect(raw), + ContextNamespaces = record?.ContextNamespaces ?? ObjectFeatures.Namespaces(raw, record?.ActivityContext), Raw = raw, RawBytes = record?.RawBytes ?? 0, RawHash = record?.RawHash, @@ -118,6 +119,7 @@ namespace PrivaPub.Api.Mastodon.Controllers public bool Refetched { get; set; } public ProvenanceActivity Activity { get; set; } public string Inbox { get; set; } + public string FetchedBy { get; set; } public ProvenanceSignature Signature { get; set; } public string ReceivedAt { get; set; } public string Published { get; set; } diff --git a/PrivaPub/Federation/Actors/ActorDocument.cs b/PrivaPub/Federation/Actors/ActorDocument.cs index 0b95d01..18825d7 100644 --- a/PrivaPub/Federation/Actors/ActorDocument.cs +++ b/PrivaPub/Federation/Actors/ActorDocument.cs @@ -37,6 +37,7 @@ namespace PrivaPub.Federation.Actors public List Emojis { get; init; } = new(); public Dictionary Fields { get; init; } = new(); public IReadOnlyList Keys { get; init; } = Array.Empty(); + public List Features { get; init; } = new(); public ActorKey Key(string keyId) => Keys.FirstOrDefault(k => k.Id == keyId); @@ -54,6 +55,7 @@ namespace PrivaPub.Federation.Actors { Id = id, Type = type, + Features = ActorFeatures.Detect(root), PreferredUsername = RemoteActorService.Text(root, "preferredUsername"), Name = RemoteActorService.Text(root, "name"), Summary = RemoteActorService.Text(root, "summary"), diff --git a/PrivaPub/Federation/Actors/ActorFeatures.cs b/PrivaPub/Federation/Actors/ActorFeatures.cs new file mode 100644 index 0000000..9b75137 --- /dev/null +++ b/PrivaPub/Federation/Actors/ActorFeatures.cs @@ -0,0 +1,72 @@ +using System.Security.Cryptography; +using System.Text.Json; + +namespace PrivaPub.Federation.Actors +{ + public static class ActorFeatures + { + public static List Detect(JsonElement root) + { + var found = new List(); + if (root.ValueKind != JsonValueKind.Object) + return found; + void When(bool condition, string feature) + { + if (condition) + found.Add(feature); + } + bool Has(string name) => root.TryGetProperty(name, out var value) && value.ValueKind is not (JsonValueKind.Null or JsonValueKind.Undefined); + bool True(string name) => root.TryGetProperty(name, out var value) && value.ValueKind == JsonValueKind.True; + IEnumerable Items(string name) => !root.TryGetProperty(name, out var value) + ? Enumerable.Empty() + : value.ValueKind == JsonValueKind.Array ? value.EnumerateArray() : new[] { value }; + bool Typed(JsonElement item, string type) => + item.ValueKind == JsonValueKind.Object && item.TryGetProperty("type", out var t) && t.ValueKind == JsonValueKind.String && t.GetString() == type; + + When(Has("featured"), "featured"); + When(Has("featuredTags"), "featured-tags"); + When(Items("alsoKnownAs").Any(), "also-known-as"); + When(Has("movedTo"), "moved-to"); + When(root.TryGetProperty("endpoints", out var endpoints) && endpoints.ValueKind == JsonValueKind.Object && endpoints.TryGetProperty("sharedInbox", out _), "shared-inbox"); + When(Has("assertionMethod"), "fep-521a-assertion-method"); + When(Has("proof"), "fep-8b32-proof"); + When(Items("attachment").Any(a => Typed(a, "PropertyValue")), "property-value"); + When(Items("attachment").Any(a => Typed(a, "VerifiableIdentityStatement") || Typed(a, "IdentityProof")), "fep-c390-identity-proof"); + When(Items("tag").Any(t => Typed(t, "Emoji")), "custom-emoji"); + When(root.EnumerateObject().Any(p => p.Name.StartsWith("_misskey", StringComparison.Ordinal)), "misskey"); + When(True("isCat"), "is-cat"); + When(Has("webfinger"), "fep-2c59-webfinger"); + When(Has("attributionDomains"), "attribution-domains"); + When(Has("searchableBy"), "fep-268d-searchable-by"); + When(True("indexable"), "indexable"); + When(root.TryGetProperty("discoverable", out var discoverable) && discoverable.ValueKind == JsonValueKind.False, "undiscoverable"); + When(True("manuallyApprovesFollowers"), "locked"); + When(True("memorial"), "memorial"); + When(Has("generator"), "generator"); + When(root.TryGetProperty("publicKey", out var keys) && keys.ValueKind == JsonValueKind.Array, "public-key-array"); + if (KeyBits(root) is { } bits) + found.Add($"key:rsa-{bits}"); + return found; + } + + static int? KeyBits(JsonElement root) + { + if (!root.TryGetProperty("publicKey", out var key)) + return default; + if (key.ValueKind == JsonValueKind.Array) + key = key.EnumerateArray().FirstOrDefault(k => k.ValueKind == JsonValueKind.Object); + if (key.ValueKind != JsonValueKind.Object || !key.TryGetProperty("publicKeyPem", out var pem) || pem.ValueKind != JsonValueKind.String) + return default; + try + { + using var rsa = RSA.Create(); + rsa.ImportFromPem(pem.GetString()); + return rsa.KeySize; + } + catch (Exception ex) when (ex is ArgumentException or CryptographicException) + { + return default; + } + } + } +} diff --git a/PrivaPub/Federation/Actors/RemoteActorService.cs b/PrivaPub/Federation/Actors/RemoteActorService.cs index 21e72f2..cb2c91f 100644 --- a/PrivaPub/Federation/Actors/RemoteActorService.cs +++ b/PrivaPub/Federation/Actors/RemoteActorService.cs @@ -192,6 +192,7 @@ namespace PrivaPub.Federation.Actors .Modify(a => a.Published, actor.Published) .Modify(a => a.Emojis, actor.Emojis) .Modify(a => a.Fields, actor.Fields) + .Modify(a => a.Features, actor.Features) .Modify(a => a.AvatarType, Enum.TryParse(actor.Type, out var type) ? type : AvatarType.Person) .Modify(a => a.PublicKeyId, key?.Id) .Modify(a => a.PublicKey, key?.Pem) diff --git a/PrivaPub/Federation/Inbox/Handlers/AnnounceHandler.cs b/PrivaPub/Federation/Inbox/Handlers/AnnounceHandler.cs index 5aa881e..d895f0e 100644 --- a/PrivaPub/Federation/Inbox/Handlers/AnnounceHandler.cs +++ b/PrivaPub/Federation/Inbox/Handlers/AnnounceHandler.cs @@ -1,6 +1,7 @@ using MongoDB.Driver; using MongoDB.Entities; +using PrivaPub.Domain.Statuses; using PrivaPub.Domain.Social; using PrivaPub.Domain.Timelines; using PrivaPub.Federation.Actors; @@ -28,8 +29,14 @@ namespace PrivaPub.Federation.Inbox.Handlers readonly IFanout _fanout; readonly IRemoteActorService _remoteActors; - public AnnounceHandler(DbEntities dbEntities, ILocalActorService localActors, IRemotePosts remotePosts, IFanout fanout, IRemoteActorService remoteActors) + readonly IObjectRecords _records; + readonly IQuoteService _quotes; + + public AnnounceHandler(DbEntities dbEntities, ILocalActorService localActors, IRemotePosts remotePosts, IFanout fanout, IRemoteActorService remoteActors, + IObjectRecords records, IQuoteService quotes) { + _records = records; + _quotes = quotes; _remoteActors = remoteActors; _dbEntities = dbEntities; _localActors = localActors; @@ -172,24 +179,7 @@ namespace PrivaPub.Federation.Inbox.Handlers Arrival.Drop(note == default ? "fetch-failed" : "misattributed"); return; } - Arrival.Accept("edit"); - stored.Revisions.Add(new PostRevision - { - Title = stored.Title, - SpoilerText = stored.SpoilerText, - ContentHtml = stored.ContentHtml, - HasContentWarning = stored.HasContentWarning, - EditedAt = stored.EditedAt ?? stored.CreationDate - }); - stored.Title = note.Title; - stored.SpoilerText = note.SpoilerText; - stored.HasContentWarning = note.Sensitive; - stored.Text = note.ContentHtml; - stored.ContentHtml = note.ContentHtml; - stored.Tags = note.Tags.ToList(); - stored.Media = note.Attachments.ToList(); - stored.EditedAt = note.Updated ?? DateTime.UtcNow; - await DB.Default.SaveAsync(stored, token); + Arrival.Accept(await RemoteEdits.Apply(stored, note, Id(inner), _localActors, _records, _quotes, token) ? "edit" : "refresh"); } break; case "Delete" when objectUri != default: @@ -207,8 +197,7 @@ namespace PrivaPub.Federation.Inbox.Handlers return; } Arrival.Accept("removed"); - await DB.Default.DeleteAsync(deleted.ID); - await DB.Default.DeleteAsync(e => e.PostId == deleted.ID); + await RemoteDeletes.Remove(deleted, objectUri, token); break; default: Arrival.Drop("unsupported"); diff --git a/PrivaPub/Federation/Inbox/Handlers/DeleteHandler.cs b/PrivaPub/Federation/Inbox/Handlers/DeleteHandler.cs index 929c427..01eb932 100644 --- a/PrivaPub/Federation/Inbox/Handlers/DeleteHandler.cs +++ b/PrivaPub/Federation/Inbox/Handlers/DeleteHandler.cs @@ -73,11 +73,7 @@ namespace PrivaPub.Federation.Inbox.Handlers return; } - await DB.Default.Update() - .Match(d => d.ObjectURI == objectUri) - .Modify(d => d.DeletedAt, DateTime.UtcNow) - .Option(o => o.IsUpsert = true) - .ExecuteAsync(token); + await RemoteDeletes.Tombstone(objectUri, token); var post = await _dbEntities.Posts.Match(p => p.ObjectURI == objectUri && p.ActorURI == actor.ActorURI).ExecuteFirstAsync(token); if (post == default) { @@ -87,14 +83,9 @@ namespace PrivaPub.Federation.Inbox.Handlers } Arrival.Accept("removed"); Arrival.About(post.ObjectType, post.Visibility, post.CreationDate); - await DB.Default.DeleteAsync(post.ID); - await DB.Default.DeleteAsync(r => r.PostId == post.ID || r.ObjectURI == objectUri); - await DB.Default.DeleteAsync(e => e.PostId == post.ID || e.ReblogOfPostId == post.ID); + await RemoteDeletes.Remove(post, objectUri, token); if (!string.IsNullOrEmpty(post.GroupId) && await _localActors.FindById(LocalActorKind.Group, post.GroupId, token) is { IsCircle: false } community) await _groups.Announce(community, activity.AsObject(), post.ObjectURI, isNewPost: false, token); - await DB.Default.DeleteAsync(p => p.ReblogOfPostId == post.ID); - if (!string.IsNullOrEmpty(post.AnsweringToPostId)) - await DB.Default.Update().MatchID(post.AnsweringToPostId).Modify(b => b.Inc(p => p.RepliesCount, -1)).ExecuteAsync(token); } } } diff --git a/PrivaPub/Federation/Inbox/Handlers/UpdateHandler.cs b/PrivaPub/Federation/Inbox/Handlers/UpdateHandler.cs index b4f1131..871faaf 100644 --- a/PrivaPub/Federation/Inbox/Handlers/UpdateHandler.cs +++ b/PrivaPub/Federation/Inbox/Handlers/UpdateHandler.cs @@ -18,7 +18,6 @@ namespace PrivaPub.Federation.Inbox.Handlers { public class UpdateHandler : IActivityHandler { - const int MaxRevisions = 20; readonly DbEntities _dbEntities; readonly ILocalActorService _localActors; @@ -70,69 +69,10 @@ namespace PrivaPub.Federation.Inbox.Handlers } Arrival.About(note.Type, post.Visibility, post.CreationDate); - post.Poll = note.Poll ?? post.Poll; - post.QuotePolicy = note.QuotePolicy; - post.Video = note.Video ?? post.Video; - post.Audio = note.Audio ?? post.Audio; - post.Event = note.Event ?? post.Event; - if (!IsEdit(note, post)) - { - Arrival.Accept("refresh"); - await DB.Default.SaveAsync(post, token); - await _records.Revise(note, Id(activity), token); - await Requote(post, note, token); - return; - } - - Arrival.Accept("edit"); - post.Revisions.Add(new PostRevision - { - Title = post.Title, - SpoilerText = post.SpoilerText, - ContentHtml = post.ContentHtml, - HasContentWarning = post.HasContentWarning, - EditedAt = post.EditedAt ?? post.CreationDate - }); - if (post.Revisions.Count > MaxRevisions) - post.Revisions.RemoveRange(0, post.Revisions.Count - MaxRevisions); - - var mentions = new List(); - foreach (var mention in note.Mentions) - { - var local = await _localActors.FindByUri(mention.ActorURI, token); - mentions.Add(new PostMention { ActorURI = mention.ActorURI, Handle = mention.Handle, IsLocal = local != default, AccountId = local?.Id }); - } - - post.Title = note.Title; - post.SpoilerText = note.SpoilerText; - post.Excerpt = note.Excerpt; - post.Source = note.Source; - post.Emojis = note.Emojis.ToList(); - post.CoverURL = note.CoverURL; - post.Link = note.Link; - post.HasContentWarning = note.Sensitive; - post.Text = note.ContentHtml; - post.ContentHtml = note.ContentHtml; - post.Language = note.Language; - post.Tags = note.Tags.ToList(); - post.Mentions = mentions; - post.Media = note.Attachments.ToList(); - post.EditedAt = note.Updated ?? DateTime.UtcNow; - post.UpdateDate = DateTime.UtcNow; - await DB.Default.SaveAsync(post, token); - await _records.Revise(note, Id(activity), token); - await Requote(post, note, token); - if (!string.IsNullOrEmpty(post.GroupId) && await _localActors.FindById(Models.Federation.LocalActorKind.Group, post.GroupId, token) is { IsCircle: false } community) + var edited = await RemoteEdits.Apply(post, note, Id(activity), _localActors, _records, _quotes, token); + Arrival.Accept(edited ? "edit" : "refresh"); + if (edited && !string.IsNullOrEmpty(post.GroupId) && await _localActors.FindById(Models.Federation.LocalActorKind.Group, post.GroupId, token) is { IsCircle: false } community) await _groups.Announce(community, activity.AsObject(), post.ObjectURI, isNewPost: false, token); } - - static bool IsEdit(NoteDocument note, PostEntity post) => - note.Updated is { } updated && updated > (post.EditedAt ?? post.CreationDate); - - async Task Requote(PostEntity post, NoteDocument note, CancellationToken token) - { - if (note.QuoteUri != post.QuoteURI || note.QuoteAuthorization != post.QuoteAuthorizationURI || post.QuoteState != QuoteState.Accepted) - await _quotes.Resolve(post, note, token); - } } } diff --git a/PrivaPub/Federation/Inbox/RemoteEdits.cs b/PrivaPub/Federation/Inbox/RemoteEdits.cs new file mode 100644 index 0000000..bd1ab33 --- /dev/null +++ b/PrivaPub/Federation/Inbox/RemoteEdits.cs @@ -0,0 +1,106 @@ +using MongoDB.Entities; + +using PrivaPub.Domain.Statuses; +using PrivaPub.Federation.Actors; +using PrivaPub.Federation.Objects; +using PrivaPub.Models.Federation; +using PrivaPub.Models.Post; +using PrivaPub.Models.Social; + +using PostEntity = PrivaPub.Models.Post.Post; + +namespace PrivaPub.Federation.Inbox +{ + public static class RemoteEdits + { + public const int MaxRevisions = 20; + + public static bool IsEdit(NoteDocument note, PostEntity post) => + note.Updated is { } updated && updated > (post.EditedAt ?? post.CreationDate); + + public static async Task Apply(PostEntity post, NoteDocument note, string activityId, ILocalActorService localActors, IObjectRecords records, + IQuoteService quotes, CancellationToken token) + { + post.Poll = note.Poll ?? post.Poll; + post.QuotePolicy = note.QuotePolicy; + post.Video = note.Video ?? post.Video; + post.Audio = note.Audio ?? post.Audio; + post.Event = note.Event ?? post.Event; + if (!IsEdit(note, post)) + { + await DB.Default.SaveAsync(post, token); + await records.Revise(note, activityId, token); + await Requote(post, note, quotes, token); + return false; + } + + post.Revisions.Add(new PostRevision + { + Title = post.Title, + SpoilerText = post.SpoilerText, + ContentHtml = post.ContentHtml, + HasContentWarning = post.HasContentWarning, + EditedAt = post.EditedAt ?? post.CreationDate + }); + if (post.Revisions.Count > MaxRevisions) + post.Revisions.RemoveRange(0, post.Revisions.Count - MaxRevisions); + + var mentions = new List(); + foreach (var mention in note.Mentions) + { + var local = await localActors.FindByUri(mention.ActorURI, token); + mentions.Add(new PostMention { ActorURI = mention.ActorURI, Handle = mention.Handle, IsLocal = local != default, AccountId = local?.Id }); + } + + post.Title = note.Title; + post.SpoilerText = note.SpoilerText; + post.Excerpt = note.Excerpt; + post.Source = note.Source; + post.Emojis = note.Emojis.ToList(); + post.CoverURL = note.CoverURL; + post.Link = note.Link; + post.HasContentWarning = note.Sensitive; + post.Text = note.ContentHtml; + post.ContentHtml = note.ContentHtml; + post.Language = note.Language; + post.Tags = note.Tags.ToList(); + post.Mentions = mentions; + post.Media = note.Attachments.ToList(); + post.EditedAt = note.Updated ?? DateTime.UtcNow; + post.UpdateDate = DateTime.UtcNow; + await DB.Default.SaveAsync(post, token); + await records.Revise(note, activityId, token); + await Requote(post, note, quotes, token); + return true; + } + + static async Task Requote(PostEntity post, NoteDocument note, IQuoteService quotes, CancellationToken token) + { + if (note.QuoteUri != post.QuoteURI || note.QuoteAuthorization != post.QuoteAuthorizationURI || post.QuoteState != QuoteState.Accepted) + await quotes.Resolve(post, note, token); + } + } + + public static class RemoteDeletes + { + public static async Task Tombstone(string objectUri, CancellationToken token) => + await DB.Default.Update() + .Match(d => d.ObjectURI == objectUri) + .Modify(d => d.DeletedAt, DateTime.UtcNow) + .Option(o => o.IsUpsert = true) + .ExecuteAsync(token); + + public static async Task Remove(PostEntity post, string objectUri, CancellationToken token) + { + await Tombstone(objectUri, token); + await DB.Default.DeleteAsync(post.ID); + await DB.Default.DeleteAsync(r => r.PostId == post.ID || r.ObjectURI == objectUri); + await DB.Default.DeleteAsync(e => e.PostId == post.ID || e.ReblogOfPostId == post.ID); + await DB.Default.DeleteAsync(p => p.ReblogOfPostId == post.ID); + if (!string.IsNullOrEmpty(post.AnsweringToPostId)) + await DB.Default.Update().MatchID(post.AnsweringToPostId).Modify(b => b.Inc(p => p.RepliesCount, -1)).ExecuteAsync(token); + if (post.QuoteState == QuoteState.Accepted && !string.IsNullOrEmpty(post.QuotedPostId)) + await DB.Default.Update().MatchID(post.QuotedPostId).Modify(b => b.Inc(p => p.QuotesCount, -1)).ExecuteAsync(token); + } + } +} diff --git a/PrivaPub/Federation/Objects/ObjectRecords.cs b/PrivaPub/Federation/Objects/ObjectRecords.cs index e517e18..3ebe25a 100644 --- a/PrivaPub/Federation/Objects/ObjectRecords.cs +++ b/PrivaPub/Federation/Objects/ObjectRecords.cs @@ -34,8 +34,10 @@ namespace PrivaPub.Federation.Objects public async Task Record(NoteDocument note, PostEntity post, ObjectPath path, bool refetched, CancellationToken token) { var arrival = Arrival.Current; + var delivered = path == ObjectPath.Delivered; var raw = Capture(note.Raw); var host = new Uri(note.Id).Host.ToLowerInvariant(); + var activityContext = !delivered || note.Raw?.ContainsKey("@context") == true || arrival?.Context?.Length > 16 * 1024 ? default : arrival?.Context; var record = new ObjectRecord { ObjectURI = note.Id, @@ -46,20 +48,22 @@ namespace PrivaPub.Federation.Objects RawHash = raw.Hash, RawBytes = raw.Bytes, RawTruncated = raw.Truncated, - ActivityContext = note.Raw?.ContainsKey("@context") == true || arrival?.Context?.Length > 16 * 1024 ? default : arrival?.Context, + ActivityContext = activityContext, Path = path, Refetched = refetched, ActivityId = arrival?.ActivityId, ActivityType = arrival?.ActivityType, ActivityActorURI = arrival?.ActorURI, - Inbox = path == ObjectPath.Delivered ? arrival?.Inbox : default, - SignatureScheme = arrival?.KeyId == default ? default : "draft-cavage", - KeyId = arrival?.KeyId, - Algorithm = arrival?.Algorithm, - SignedHeaders = arrival?.SignedHeaders?.ToList() ?? new(), - ReceivedAt = arrival?.ReceivedAt ?? DateTime.UtcNow, + Inbox = delivered ? arrival?.Inbox : default, + SignatureScheme = !delivered || arrival?.KeyId == default ? default : "draft-cavage", + KeyId = delivered ? arrival?.KeyId : default, + Algorithm = delivered ? arrival?.Algorithm : default, + SignedHeaders = delivered ? arrival?.SignedHeaders?.ToList() ?? new() : new(), + ReceivedAt = delivered ? arrival?.ReceivedAt ?? DateTime.UtcNow : DateTime.UtcNow, Published = note.Published, - Updated = note.Updated + Updated = note.Updated, + Extensions = ObjectFeatures.Detect(note.Raw), + ContextNamespaces = ObjectFeatures.Namespaces(note.Raw, activityContext) }; try { diff --git a/PrivaPub/Infrastructure/Data/Migrations/_008_fetched_records_lose_the_trigger_signature.cs b/PrivaPub/Infrastructure/Data/Migrations/_008_fetched_records_lose_the_trigger_signature.cs new file mode 100644 index 0000000..aaff8a3 --- /dev/null +++ b/PrivaPub/Infrastructure/Data/Migrations/_008_fetched_records_lose_the_trigger_signature.cs @@ -0,0 +1,19 @@ +using MongoDB.Entities; + +using PrivaPub.Models.Federation; + +namespace PrivaPub.Infrastructure.Data.Migrations +{ + public class _008_fetched_records_lose_the_trigger_signature : IMigration + { + public async Task UpgradeAsync() => + await DB.Default.Update() + .Match(r => r.Path == ObjectPath.Fetched) + .Modify(r => r.SignatureScheme, null) + .Modify(r => r.KeyId, null) + .Modify(r => r.Algorithm, null) + .Modify(r => r.SignedHeaders, new List()) + .Modify(r => r.ActivityContext, null) + .ExecuteAsync(); + } +} diff --git a/PrivaPub/Infrastructure/Data/Migrations/_009_object_records_keep_their_extensions.cs b/PrivaPub/Infrastructure/Data/Migrations/_009_object_records_keep_their_extensions.cs new file mode 100644 index 0000000..799417e --- /dev/null +++ b/PrivaPub/Infrastructure/Data/Migrations/_009_object_records_keep_their_extensions.cs @@ -0,0 +1,41 @@ +using MongoDB.Entities; + +using PrivaPub.Federation.Objects; +using PrivaPub.Models.Federation; + +using System.Text.Json; +using System.Text.Json.Nodes; + +namespace PrivaPub.Infrastructure.Data.Migrations +{ + public class _009_object_records_keep_their_extensions : IMigration + { + public async Task UpgradeAsync() + { + while (true) + { + var batch = await DB.Default.Find() + .Match(r => r.Extensions == null) + .Limit(500) + .ExecuteAsync(); + if (batch.Count == 0) + return; + foreach (var record in batch) + { + JsonObject raw = default; + try + { + raw = record.Raw == default ? default : JsonNode.Parse(record.Raw) as JsonObject; + } + catch (JsonException) + { + } + await DB.Default.Update().MatchID(record.ID) + .Modify(r => r.Extensions, ObjectFeatures.Detect(raw)) + .Modify(r => r.ContextNamespaces, ObjectFeatures.Namespaces(raw, record.ActivityContext)) + .ExecuteAsync(); + } + } + } + } +} diff --git a/PrivaPub/Models/Federation/ObjectRecord.cs b/PrivaPub/Models/Federation/ObjectRecord.cs index cf470af..8af03f0 100644 --- a/PrivaPub/Models/Federation/ObjectRecord.cs +++ b/PrivaPub/Models/Federation/ObjectRecord.cs @@ -30,6 +30,8 @@ namespace PrivaPub.Models.Federation public DateTime? Published { get; set; } public DateTime? Updated { get; set; } public List Revisions { get; set; } = new(); + public List Extensions { get; set; }//ObjectFeatures, stored for statistics; null on records older than v1.16 + public List ContextNamespaces { get; set; } } public class ObjectRevision diff --git a/PrivaPub/Models/User/Avatar.cs b/PrivaPub/Models/User/Avatar.cs index 73aab76..e32dfdf 100644 --- a/PrivaPub/Models/User/Avatar.cs +++ b/PrivaPub/Models/User/Avatar.cs @@ -66,6 +66,7 @@ namespace PrivaPub.Models.User public string PictureDescription { get; set; } public string HeaderDescription { get; set; } public DateTime? Published { get; set; }//the actor's own published + public List Features { get; set; }//protocol extensions its document uses, for statistics (ActorFeatures) public string ActorURI { get; set; }//id public string PublicKeyId { get; set; }