M6: provenance fixes, stored features, community edits that are real edits

- A fetched ObjectRecord no longer wears the signature, key, signed headers and @context
  of the activity that caused the fetch, and its ReceivedAt is the fetch's own time. Its
  activity fields now name the trigger. Provenance answers signature null and
  fetchedBy "instance-actor". Migration _008 clears the old fetched records.
- ObjectRecords store their ObjectFeatures extensions and context namespaces (migration
  _009 fills older records in batches), so statistics can count them. Provenance reads the
  stored lists.
- ForeignAvatar.Features records what an actor document uses (ActorFeatures): featured,
  shared inbox, FEP-521a, FEP-8b32, identity proofs, Misskey fields, indexable,
  undiscoverable, locked, key size, and more.
- RemoteEdits.Apply and RemoteDeletes.Remove are shared by the Update, Delete and Announce
  handlers. A community-wrapped Update is now an edit only when newer, refreshes polls
  and media otherwise, revises the ObjectRecord and re-resolves quotes. A
  community-wrapped Delete now tombstones the object, removes its ObjectRecord, reblogs
  and timeline rows, and lowers the reply count. Any deleted quoting post lowers the
  quoted post's quote count.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01ELjqpznMFMNrJoJUj6K5p2

CLAUDE.md

@@ -272,13 +272,15 @@ cd /var/www/privapub.thepra.dev && sudo -u www-data ASPNETCORE_ENVIRONMENT=Produ
   activity, inbox, key and signed headers; received time; up to 10 later revisions). Write it right after the post's
   save: `CreateHandler` (delivered) and `RemotePosts.StoreContext` (fetched) do, and `UpdateHandler` appends a
   revision. Delivery details reach them through `Arrival.Current`, which `InboxProcessor` sets for the handler's
-  duration. The raw form lives outside `Post` so timelines never load it. Read through
+  duration. A fetched record has no signature, key or `@context` of the activity that caused the fetch (its activity
+  fields name that trigger), and every record stores its `Extensions` and `ContextNamespaces` for statistics. The raw form lives outside `Post` so timelines never load it. Read through
   `/api/privapub/v1/statuses/:id/provenance` and `/api/privapub/v1/instances/:host`.
 - **Remote objects are parsed for every shape in `ObjectShapes`:** `url`/`icon`/`image` as a value, an object or an
   array; Markdown `content`; missing `mediaType`s inferred; thumbnails from any of their five places; and the typed
   `Link`, `Video`, `Audio` and `Event` details. A Mastodon API card is built from those, never by fetching the linked page
   (that fetch is the owner's decision 1, for P6).
-- **An `Update` is an edit only when its `updated` is newer than ours** (`UpdateHandler.IsEdit`). Otherwise it refreshes
+- **An `Update` is an edit only when its `updated` is newer than ours** (`RemoteEdits.IsEdit`; plain and community-wrapped
+  Updates both go through `RemoteEdits.Apply`, and deletes through `RemoteDeletes.Remove`). Otherwise it refreshes
   the poll, video, audio and event details and nothing else, and leaves no revision: Mastodon and Misskey refresh poll
   counts with bare Updates.
 - **A poll vote is a `Note` with a `name`, an `inReplyTo` that is a poll we hold, and no content.** `CreateHandler`

PrivaPub.Tests/Federation/ActorFeaturesTests.cs

@@ -0,0 +1,50 @@
+using PrivaPub.Federation.Actors;
+
+using System.Security.Cryptography;
+using System.Text.Json;
+using System.Text.Json.Nodes;
+
+namespace PrivaPub.Tests.Federation
+{
+	public class ActorFeaturesTests
+	{
+		[Fact]
+		public void A_rich_actor_reports_the_extensions_it_uses()
+		{
+			using var rsa = RSA.Create(4096);
+			var actor = new JsonObject
+			{
+				["id"] = "https://social.example/users/a",
+				["type"] = "Person",
+				["featured"] = "https://social.example/users/a/featured",
+				["alsoKnownAs"] = new JsonArray("https://old.example/users/a"),
+				["endpoints"] = new JsonObject { ["sharedInbox"] = "https://social.example/inbox" },
+				["assertionMethod"] = new JsonArray(),
+				["attachment"] = new JsonArray(new JsonObject { ["type"] = "PropertyValue", ["name"] = "x", ["value"] = "y" }),
+				["manuallyApprovesFollowers"] = true,
+				["discoverable"] = false,
+				["indexable"] = true,
+				["isCat"] = true,
+				["_misskey_summary"] = "hi",
+				["publicKey"] = new JsonObject { ["id"] = "https://social.example/users/a#main-key", ["publicKeyPem"] = rsa.ExportSubjectPublicKeyInfoPem() }
+			};
+
+			var features = ActorFeatures.Detect(JsonDocument.Parse(actor.ToJsonString()).RootElement);
+
+			Assert.Equal(new[]
+			{
+				"featured", "also-known-as", "shared-inbox", "fep-521a-assertion-method", "property-value", "misskey", "is-cat",
+				"indexable", "undiscoverable", "locked", "key:rsa-4096"
+			}, features);
+		}
+
+		[Fact]
+		public void A_bare_actor_reports_nothing_and_junk_is_survived()
+		{
+			Assert.Empty(ActorFeatures.Detect(JsonDocument.Parse("{\"id\":\"https://a.example/u\",\"type\":\"Person\"}").RootElement));
+			Assert.Equal(new[] { "public-key-array" },
+				ActorFeatures.Detect(JsonDocument.Parse("{\"publicKey\":[{\"publicKeyPem\":\"not a key\"}]}").RootElement));
+			Assert.Empty(ActorFeatures.Detect(JsonDocument.Parse("[1,2]").RootElement));
+		}
+	}
+}

PrivaPub.Tests/Federation/CommunityEditsTests.cs

@@ -0,0 +1,131 @@
+using MongoDB.Entities;
+
+using PrivaPub.ClientModels.Social;
+using PrivaPub.Federation.Objects;
+using PrivaPub.Models.Federation;
+using PrivaPub.Models.Post;
+using PrivaPub.Models.Social;
+using PrivaPub.Tests.Support;
+
+using System.Text.Json.Nodes;
+
+namespace PrivaPub.Tests.Federation
+{
+	[Trait("Category", "Integration")]
+	public sealed class CommunityEditsTests : IAsyncLifetime
+	{
+		Harness _harness;
+
+		public async ValueTask InitializeAsync()
+		{
+			Assert.SkipUnless(MongoFixture.Enabled, MongoFixture.Skip);
+			_harness = await Harness.Start();
+		}
+
+		public async ValueTask DisposeAsync()
+		{
+			if (_harness != default)
+				await _harness.DisposeAsync();
+		}
+
+		static string Origin(RemoteActor actor) => new Uri(actor.Id).GetLeftPart(UriPartial.Authority);
+
+		async Task<(RemoteActor Community, RemoteActor Poster, JsonObject Note, Post Post)> Announced()
+		{
+			var token = TestContext.Current.CancellationToken;
+			var (root, alice) = await _harness.Persona("alice");
+			var community = new RemoteActor(_harness.Peer, "cats", type: "Group");
+			var poster = new RemoteActor(_harness.Peer, "poster");
+			await _harness.Follows.Follow(root, new FollowForm { AvatarId = alice.Id, Target = community.Id }, token);
+			await DB.Default.Update<Following>().Match(f => f.AvatarId == alice.Id).Modify(f => f.State, FollowState.Accepted).ExecuteAsync(token);
+			var note = new JsonObject
+			{
+				["id"] = $"{Origin(poster)}/notes/{Guid.NewGuid():N}",
+				["type"] = "Page",
+				["name"] = "a title",
+				["attributedTo"] = poster.Id,
+				["content"] = "<p>first</p>",
+				["to"] = new JsonArray(community.Id, Addressing.Public),
+				["audience"] = community.Id,
+				["published"] = DateTime.UtcNow.AddMinutes(-5).ToString("O")
+			};
+			_harness.Peer.Serve(new Uri(note["id"]!.GetValue<string>()).AbsolutePath, note.ToJsonString());
+			await Announce(community, new JsonObject { ["id"] = $"{Origin(poster)}/create/{Guid.NewGuid():N}", ["type"] = "Create", ["actor"] = poster.Id, ["object"] = note.DeepClone() });
+			var post = await DB.Default.Find<Post>().Match(p => p.ObjectURI == note["id"]!.GetValue<string>()).ExecuteSingleAsync(token);
+			return (community, poster, note, post);
+		}
+
+		Task Announce(RemoteActor community, JsonObject inner) =>
+			_harness.Deliver(community, "/human-centipede", new JsonObject
+			{
+				["id"] = $"{Origin(community)}/announce/{Guid.NewGuid():N}",
+				["type"] = "Announce",
+				["actor"] = community.Id,
+				["to"] = new JsonArray(Addressing.Public),
+				["object"] = inner
+			});
+
+		[Fact]
+		public async Task A_community_edit_is_an_edit_only_when_newer_and_keeps_its_history()
+		{
+			var token = TestContext.Current.CancellationToken;
+			var (community, poster, note, post) = await Announced();
+
+			var refresh = (JsonObject)note.DeepClone();
+			_harness.Peer.Serve(new Uri(note["id"]!.GetValue<string>()).AbsolutePath, refresh.ToJsonString());
+			await Announce(community, new JsonObject { ["id"] = $"{Origin(poster)}/update/{Guid.NewGuid():N}", ["type"] = "Update", ["actor"] = poster.Id, ["object"] = note["id"]!.GetValue<string>() });
+			var afterRefresh = await DB.Default.Find<Post>().OneAsync(post.ID, token);
+
+			var edit = (JsonObject)note.DeepClone();
+			edit["content"] = "<p>second</p>";
+			edit["updated"] = DateTime.UtcNow.ToString("O");
+			_harness.Peer.Serve(new Uri(note["id"]!.GetValue<string>()).AbsolutePath, edit.ToJsonString());
+			await Announce(community, new JsonObject { ["id"] = $"{Origin(poster)}/update/{Guid.NewGuid():N}", ["type"] = "Update", ["actor"] = poster.Id, ["object"] = note["id"]!.GetValue<string>() });
+			var afterEdit = await DB.Default.Find<Post>().OneAsync(post.ID, token);
+			var record = await DB.Default.Find<ObjectRecord>().Match(r => r.ObjectURI == post.ObjectURI).ExecuteSingleAsync(token);
+
+			Assert.Empty(afterRefresh.Revisions);
+			Assert.Null(afterRefresh.EditedAt);
+			Assert.Equal("<p>second</p>", afterEdit.ContentHtml);
+			Assert.Equal("a title", afterEdit.Title);
+			Assert.Single(afterEdit.Revisions);
+			Assert.NotNull(afterEdit.EditedAt);
+			Assert.Equal(2, record.Revisions.Count);
+			Assert.Equal(new[] { "refresh", "edit" }, _harness.Ledger.Of("in").Where(e => e.Activity == "Announce" && e.Object == "Update").Select(e => e.Reason));
+		}
+
+		[Fact]
+		public async Task A_community_delete_leaves_a_tombstone_and_no_trace()
+		{
+			var token = TestContext.Current.CancellationToken;
+			var (community, poster, note, post) = await Announced();
+			_harness.Peer.Serve(new Uri(note["id"]!.GetValue<string>()).AbsolutePath, new JsonObject { ["id"] = note["id"]!.GetValue<string>(), ["type"] = "Tombstone" }.ToJsonString());
+
+			await Announce(community, new JsonObject { ["id"] = $"{Origin(poster)}/delete/{Guid.NewGuid():N}", ["type"] = "Delete", ["actor"] = poster.Id, ["object"] = note["id"]!.GetValue<string>() });
+
+			Assert.False(await DB.Default.Find<Post>().Match(p => p.ID == post.ID).ExecuteAnyAsync(token));
+			Assert.False(await DB.Default.Find<ObjectRecord>().Match(r => r.ObjectURI == post.ObjectURI).ExecuteAnyAsync(token));
+			Assert.False(await DB.Default.Find<TimelineEntry>().Match(e => e.PostId == post.ID).ExecuteAnyAsync(token));
+			Assert.True(await DB.Default.Find<DeletedObject>().Match(d => d.ObjectURI == post.ObjectURI).ExecuteAnyAsync(token));
+		}
+
+		[Fact]
+		public async Task A_fetched_object_does_not_wear_the_signature_of_the_activity_that_caused_the_fetch()
+		{
+			var token = TestContext.Current.CancellationToken;
+			var (_, _, _, post) = await Announced();
+
+			var record = await DB.Default.Find<ObjectRecord>().Match(r => r.ObjectURI == post.ObjectURI).ExecuteSingleAsync(token);
+
+			Assert.Equal(ObjectPath.Fetched, record.Path);
+			Assert.Null(record.KeyId);
+			Assert.Null(record.SignatureScheme);
+			Assert.Empty(record.SignedHeaders);
+			Assert.Null(record.ActivityContext);
+			Assert.Equal("Announce", record.ActivityType);
+			Assert.InRange(record.ReceivedAt, DateTime.UtcNow.AddMinutes(-1), DateTime.UtcNow.AddSeconds(1));
+			Assert.Contains("fep-1b12-audience", record.Extensions);
+			Assert.NotNull(record.ContextNamespaces);
+		}
+	}
+}

PrivaPub.Tests/Federation/RemoteActorServiceTests.cs

@@ -32,6 +32,16 @@ namespace PrivaPub.Tests.Federation
 				await _peer.DisposeAsync();
 		}
 
+		[Fact]
+		public async Task A_stored_actor_remembers_the_extensions_its_document_uses()
+		{
+			var bob = new RemoteActor(_peer, "bob");
+
+			var stored = await _service.GetActor(bob.Id, refresh: true, TestContext.Current.CancellationToken);
+
+			Assert.Contains("key:rsa-2048", stored.Features);
+		}
+
 		static string Actor(string id, string keyId = default, string owner = default, string pem = default, string type = "Person") =>
 			new JsonObject
 			{

PrivaPub.Tests/Infrastructure/MigrationTests.cs

@@ -1,6 +1,7 @@
 using MongoDB.Entities;
 
 using PrivaPub.Infrastructure.Data.Migrations;
+using PrivaPub.Models.Federation;
 using PrivaPub.Models.Post;
 using PrivaPub.Tests.Support;
 
@@ -46,5 +47,37 @@ namespace PrivaPub.Tests.Infrastructure
 			Assert.Equal("conversation1", post.ConversationId);
 			Assert.Equal("a1", post.AuthorAccountId);
 		}
+	
+		[Fact]
+		public async Task Fetched_records_lose_the_signature_they_never_had_and_every_record_gets_its_extensions()
+		{
+			Assert.SkipUnless(MongoFixture.Enabled, MongoFixture.Skip);
+			var token = TestContext.Current.CancellationToken;
+			var fetched = new ObjectRecord
+			{
+				ObjectURI = $"https://r.example/{Guid.NewGuid():N}", Path = ObjectPath.Fetched, KeyId = "https://x.example/u#k", SignatureScheme = "draft-cavage",
+				Algorithm = "rsa-sha256", SignedHeaders = new() { "host" }, ActivityContext = "\"https://www.w3.org/ns/activitystreams\"",
+				Raw = "{\"type\":\"Note\",\"quoteUrl\":\"https://q.example/1\"}"
+			};
+			var delivered = new ObjectRecord
+			{
+				ObjectURI = $"https://r.example/{Guid.NewGuid():N}", Path = ObjectPath.Delivered, KeyId = "https://x.example/u#k", SignatureScheme = "draft-cavage",
+				Raw = "not json"
+			};
+			await DB.Default.SaveAsync(new[] { fetched, delivered }, token);
+
+			await new _008_fetched_records_lose_the_trigger_signature().UpgradeAsync();
+			await new _009_object_records_keep_their_extensions().UpgradeAsync();
+
+			var afterFetched = await DB.Default.Find<ObjectRecord>().OneAsync(fetched.ID, token);
+			var afterDelivered = await DB.Default.Find<ObjectRecord>().OneAsync(delivered.ID, token);
+			Assert.Null(afterFetched.KeyId);
+			Assert.Null(afterFetched.SignatureScheme);
+			Assert.Empty(afterFetched.SignedHeaders);
+			Assert.Null(afterFetched.ActivityContext);
+			Assert.Equal("https://x.example/u#k", afterDelivered.KeyId);
+			Assert.Contains("legacy-quote", afterFetched.Extensions);
+			Assert.Empty(afterDelivered.Extensions);
+		}
 	}
 }

PrivaPub.Tests/Support/Harness.cs

@@ -63,7 +63,7 @@ namespace PrivaPub.Tests.Support
 				new QuoteRequestHandler(Quotes),
 				new DislikeHandler(Db),
 				new JoinHandler(Db, Local, Delivery),
-				new AnnounceHandler(Db, Local, RemotePosts, Fanout, Remote),
+				new AnnounceHandler(Db, Local, RemotePosts, Fanout, Remote, Records, Quotes),
 				new CreateHandler(Db, Local, Remote, Delivery, new NoBlocks(), Fanout, RemotePosts, Groups, Records, Polls, new NoPreviews(), Quotes),
 				new DeleteHandler(Db, Local, Remote, Delivery, Groups, Quotes),
 				new UpdateHandler(Db, Local, Remote, Groups, Records, Quotes),

PrivaPub/Api/Mastodon/Controllers/ProvenanceController.cs

@@ -64,15 +64,16 @@ namespace PrivaPub.Api.Mastodon.Controllers
 					Id = record.ActivityId, Type = record.ActivityType, Actor = record.ActivityActorURI
 				},
 				Inbox = record?.Inbox,
-				Signature = record?.KeyId == default ? default : new ProvenanceSignature
+				FetchedBy = record?.Path == ObjectPath.Fetched ? "instance-actor" : default,
+				Signature = record?.KeyId == default || record.Path == ObjectPath.Fetched ? default : new ProvenanceSignature
 				{
 					Scheme = record.SignatureScheme, KeyId = record.KeyId, Algorithm = record.Algorithm, Headers = record.SignedHeaders
 				},
 				ReceivedAt = record == default ? default : MastodonJson.Time(record.ReceivedAt),
 				Published = record?.Published is { } published ? MastodonJson.Time(published) : default,
 				Updated = record?.Updated is { } updated ? MastodonJson.Time(updated) : default,
-				Extensions = ObjectFeatures.Detect(raw),
-				ContextNamespaces = ObjectFeatures.Namespaces(raw, record?.ActivityContext),
+				Extensions = record?.Extensions ?? ObjectFeatures.Detect(raw),
+				ContextNamespaces = record?.ContextNamespaces ?? ObjectFeatures.Namespaces(raw, record?.ActivityContext),
 				Raw = raw,
 				RawBytes = record?.RawBytes ?? 0,
 				RawHash = record?.RawHash,
@@ -118,6 +119,7 @@ namespace PrivaPub.Api.Mastodon.Controllers
 			public bool Refetched { get; set; }
 			public ProvenanceActivity Activity { get; set; }
 			public string Inbox { get; set; }
+			public string FetchedBy { get; set; }
 			public ProvenanceSignature Signature { get; set; }
 			public string ReceivedAt { get; set; }
 			public string Published { get; set; }

PrivaPub/Federation/Actors/ActorDocument.cs

@@ -37,6 +37,7 @@ namespace PrivaPub.Federation.Actors
 		public List<CustomEmoji> Emojis { get; init; } = new();
 		public Dictionary<string, string> Fields { get; init; } = new();
 		public IReadOnlyList<ActorKey> Keys { get; init; } = Array.Empty<ActorKey>();
+		public List<string> Features { get; init; } = new();
 
 		public ActorKey Key(string keyId) => Keys.FirstOrDefault(k => k.Id == keyId);
 
@@ -54,6 +55,7 @@ namespace PrivaPub.Federation.Actors
 			{
 				Id = id,
 				Type = type,
+				Features = ActorFeatures.Detect(root),
 				PreferredUsername = RemoteActorService.Text(root, "preferredUsername"),
 				Name = RemoteActorService.Text(root, "name"),
 				Summary = RemoteActorService.Text(root, "summary"),

PrivaPub/Federation/Actors/ActorFeatures.cs

@@ -0,0 +1,72 @@
+using System.Security.Cryptography;
+using System.Text.Json;
+
+namespace PrivaPub.Federation.Actors
+{
+	public static class ActorFeatures
+	{
+		public static List<string> Detect(JsonElement root)
+		{
+			var found = new List<string>();
+			if (root.ValueKind != JsonValueKind.Object)
+				return found;
+			void When(bool condition, string feature)
+			{
+				if (condition)
+					found.Add(feature);
+			}
+			bool Has(string name) => root.TryGetProperty(name, out var value) && value.ValueKind is not (JsonValueKind.Null or JsonValueKind.Undefined);
+			bool True(string name) => root.TryGetProperty(name, out var value) && value.ValueKind == JsonValueKind.True;
+			IEnumerable<JsonElement> Items(string name) => !root.TryGetProperty(name, out var value)
+				? Enumerable.Empty<JsonElement>()
+				: value.ValueKind == JsonValueKind.Array ? value.EnumerateArray() : new[] { value };
+			bool Typed(JsonElement item, string type) =>
+				item.ValueKind == JsonValueKind.Object && item.TryGetProperty("type", out var t) && t.ValueKind == JsonValueKind.String && t.GetString() == type;
+
+			When(Has("featured"), "featured");
+			When(Has("featuredTags"), "featured-tags");
+			When(Items("alsoKnownAs").Any(), "also-known-as");
+			When(Has("movedTo"), "moved-to");
+			When(root.TryGetProperty("endpoints", out var endpoints) && endpoints.ValueKind == JsonValueKind.Object && endpoints.TryGetProperty("sharedInbox", out _), "shared-inbox");
+			When(Has("assertionMethod"), "fep-521a-assertion-method");
+			When(Has("proof"), "fep-8b32-proof");
+			When(Items("attachment").Any(a => Typed(a, "PropertyValue")), "property-value");
+			When(Items("attachment").Any(a => Typed(a, "VerifiableIdentityStatement") || Typed(a, "IdentityProof")), "fep-c390-identity-proof");
+			When(Items("tag").Any(t => Typed(t, "Emoji")), "custom-emoji");
+			When(root.EnumerateObject().Any(p => p.Name.StartsWith("_misskey", StringComparison.Ordinal)), "misskey");
+			When(True("isCat"), "is-cat");
+			When(Has("webfinger"), "fep-2c59-webfinger");
+			When(Has("attributionDomains"), "attribution-domains");
+			When(Has("searchableBy"), "fep-268d-searchable-by");
+			When(True("indexable"), "indexable");
+			When(root.TryGetProperty("discoverable", out var discoverable) && discoverable.ValueKind == JsonValueKind.False, "undiscoverable");
+			When(True("manuallyApprovesFollowers"), "locked");
+			When(True("memorial"), "memorial");
+			When(Has("generator"), "generator");
+			When(root.TryGetProperty("publicKey", out var keys) && keys.ValueKind == JsonValueKind.Array, "public-key-array");
+			if (KeyBits(root) is { } bits)
+				found.Add($"key:rsa-{bits}");
+			return found;
+		}
+
+		static int? KeyBits(JsonElement root)
+		{
+			if (!root.TryGetProperty("publicKey", out var key))
+				return default;
+			if (key.ValueKind == JsonValueKind.Array)
+				key = key.EnumerateArray().FirstOrDefault(k => k.ValueKind == JsonValueKind.Object);
+			if (key.ValueKind != JsonValueKind.Object || !key.TryGetProperty("publicKeyPem", out var pem) || pem.ValueKind != JsonValueKind.String)
+				return default;
+			try
+			{
+				using var rsa = RSA.Create();
+				rsa.ImportFromPem(pem.GetString());
+				return rsa.KeySize;
+			}
+			catch (Exception ex) when (ex is ArgumentException or CryptographicException)
+			{
+				return default;
+			}
+		}
+	}
+}

PrivaPub/Federation/Actors/RemoteActorService.cs

@@ -192,6 +192,7 @@ namespace PrivaPub.Federation.Actors
 				.Modify(a => a.Published, actor.Published)
 				.Modify(a => a.Emojis, actor.Emojis)
 				.Modify(a => a.Fields, actor.Fields)
+				.Modify(a => a.Features, actor.Features)
 				.Modify(a => a.AvatarType, Enum.TryParse<AvatarType>(actor.Type, out var type) ? type : AvatarType.Person)
 				.Modify(a => a.PublicKeyId, key?.Id)
 				.Modify(a => a.PublicKey, key?.Pem)

PrivaPub/Federation/Inbox/Handlers/AnnounceHandler.cs

@@ -1,6 +1,7 @@
 using MongoDB.Driver;
 using MongoDB.Entities;
 
+using PrivaPub.Domain.Statuses;
 using PrivaPub.Domain.Social;
 using PrivaPub.Domain.Timelines;
 using PrivaPub.Federation.Actors;
@@ -28,8 +29,14 @@ namespace PrivaPub.Federation.Inbox.Handlers
 		readonly IFanout _fanout;
 		readonly IRemoteActorService _remoteActors;
 
-		public AnnounceHandler(DbEntities dbEntities, ILocalActorService localActors, IRemotePosts remotePosts, IFanout fanout, IRemoteActorService remoteActors)
+		readonly IObjectRecords _records;
+		readonly IQuoteService _quotes;
+
+		public AnnounceHandler(DbEntities dbEntities, ILocalActorService localActors, IRemotePosts remotePosts, IFanout fanout, IRemoteActorService remoteActors,
+			IObjectRecords records, IQuoteService quotes)
 		{
+			_records = records;
+			_quotes = quotes;
 			_remoteActors = remoteActors;
 			_dbEntities = dbEntities;
 			_localActors = localActors;
@@ -172,24 +179,7 @@ namespace PrivaPub.Federation.Inbox.Handlers
 							Arrival.Drop(note == default ? "fetch-failed" : "misattributed");
 							return;
 						}
-						Arrival.Accept("edit");
-						stored.Revisions.Add(new PostRevision
-						{
-							Title = stored.Title,
-							SpoilerText = stored.SpoilerText,
-							ContentHtml = stored.ContentHtml,
-							HasContentWarning = stored.HasContentWarning,
-							EditedAt = stored.EditedAt ?? stored.CreationDate
-						});
-						stored.Title = note.Title;
-						stored.SpoilerText = note.SpoilerText;
-						stored.HasContentWarning = note.Sensitive;
-						stored.Text = note.ContentHtml;
-						stored.ContentHtml = note.ContentHtml;
-						stored.Tags = note.Tags.ToList();
-						stored.Media = note.Attachments.ToList();
-						stored.EditedAt = note.Updated ?? DateTime.UtcNow;
-						await DB.Default.SaveAsync(stored, token);
+						Arrival.Accept(await RemoteEdits.Apply(stored, note, Id(inner), _localActors, _records, _quotes, token) ? "edit" : "refresh");
 					}
 					break;
 				case "Delete" when objectUri != default:
@@ -207,8 +197,7 @@ namespace PrivaPub.Federation.Inbox.Handlers
 							return;
 						}
 					Arrival.Accept("removed");
-					await DB.Default.DeleteAsync<PostEntity>(deleted.ID);
-					await DB.Default.DeleteAsync<TimelineEntry>(e => e.PostId == deleted.ID);
+					await RemoteDeletes.Remove(deleted, objectUri, token);
 					break;
 				default:
 					Arrival.Drop("unsupported");

PrivaPub/Federation/Inbox/Handlers/DeleteHandler.cs

@@ -73,11 +73,7 @@ namespace PrivaPub.Federation.Inbox.Handlers
 				return;
 			}
 
-			await DB.Default.Update<DeletedObject>()
-				.Match(d => d.ObjectURI == objectUri)
-				.Modify(d => d.DeletedAt, DateTime.UtcNow)
-				.Option(o => o.IsUpsert = true)
-				.ExecuteAsync(token);
+			await RemoteDeletes.Tombstone(objectUri, token);
 			var post = await _dbEntities.Posts.Match(p => p.ObjectURI == objectUri && p.ActorURI == actor.ActorURI).ExecuteFirstAsync(token);
 			if (post == default)
 			{
@@ -87,14 +83,9 @@ namespace PrivaPub.Federation.Inbox.Handlers
 			}
 			Arrival.Accept("removed");
 			Arrival.About(post.ObjectType, post.Visibility, post.CreationDate);
-			await DB.Default.DeleteAsync<PostEntity>(post.ID);
-			await DB.Default.DeleteAsync<ObjectRecord>(r => r.PostId == post.ID || r.ObjectURI == objectUri);
-			await DB.Default.DeleteAsync<TimelineEntry>(e => e.PostId == post.ID || e.ReblogOfPostId == post.ID);
+			await RemoteDeletes.Remove(post, objectUri, token);
 			if (!string.IsNullOrEmpty(post.GroupId) && await _localActors.FindById(LocalActorKind.Group, post.GroupId, token) is { IsCircle: false } community)
 				await _groups.Announce(community, activity.AsObject(), post.ObjectURI, isNewPost: false, token);
-			await DB.Default.DeleteAsync<PostEntity>(p => p.ReblogOfPostId == post.ID);
-			if (!string.IsNullOrEmpty(post.AnsweringToPostId))
-				await DB.Default.Update<PostEntity>().MatchID(post.AnsweringToPostId).Modify(b => b.Inc(p => p.RepliesCount, -1)).ExecuteAsync(token);
 		}
 	}
 }

PrivaPub/Federation/Inbox/Handlers/UpdateHandler.cs

@@ -18,7 +18,6 @@ namespace PrivaPub.Federation.Inbox.Handlers
 {
 	public class UpdateHandler : IActivityHandler
 	{
-		const int MaxRevisions = 20;
 
 		readonly DbEntities _dbEntities;
 		readonly ILocalActorService _localActors;
@@ -70,69 +69,10 @@ namespace PrivaPub.Federation.Inbox.Handlers
 			}
 			Arrival.About(note.Type, post.Visibility, post.CreationDate);
 
-			post.Poll = note.Poll ?? post.Poll;
-			post.QuotePolicy = note.QuotePolicy;
-			post.Video = note.Video ?? post.Video;
-			post.Audio = note.Audio ?? post.Audio;
-			post.Event = note.Event ?? post.Event;
-			if (!IsEdit(note, post))
-			{
-				Arrival.Accept("refresh");
-				await DB.Default.SaveAsync(post, token);
-				await _records.Revise(note, Id(activity), token);
-				await Requote(post, note, token);
-				return;
-			}
-
-			Arrival.Accept("edit");
-			post.Revisions.Add(new PostRevision
-			{
-				Title = post.Title,
-				SpoilerText = post.SpoilerText,
-				ContentHtml = post.ContentHtml,
-				HasContentWarning = post.HasContentWarning,
-				EditedAt = post.EditedAt ?? post.CreationDate
-			});
-			if (post.Revisions.Count > MaxRevisions)
-				post.Revisions.RemoveRange(0, post.Revisions.Count - MaxRevisions);
-
-			var mentions = new List<PostMention>();
-			foreach (var mention in note.Mentions)
-			{
-				var local = await _localActors.FindByUri(mention.ActorURI, token);
-				mentions.Add(new PostMention { ActorURI = mention.ActorURI, Handle = mention.Handle, IsLocal = local != default, AccountId = local?.Id });
-			}
-
-			post.Title = note.Title;
-			post.SpoilerText = note.SpoilerText;
-			post.Excerpt = note.Excerpt;
-			post.Source = note.Source;
-			post.Emojis = note.Emojis.ToList();
-			post.CoverURL = note.CoverURL;
-			post.Link = note.Link;
-			post.HasContentWarning = note.Sensitive;
-			post.Text = note.ContentHtml;
-			post.ContentHtml = note.ContentHtml;
-			post.Language = note.Language;
-			post.Tags = note.Tags.ToList();
-			post.Mentions = mentions;
-			post.Media = note.Attachments.ToList();
-			post.EditedAt = note.Updated ?? DateTime.UtcNow;
-			post.UpdateDate = DateTime.UtcNow;
-			await DB.Default.SaveAsync(post, token);
-			await _records.Revise(note, Id(activity), token);
-			await Requote(post, note, token);
-			if (!string.IsNullOrEmpty(post.GroupId) && await _localActors.FindById(Models.Federation.LocalActorKind.Group, post.GroupId, token) is { IsCircle: false } community)
+			var edited = await RemoteEdits.Apply(post, note, Id(activity), _localActors, _records, _quotes, token);
+			Arrival.Accept(edited ? "edit" : "refresh");
+			if (edited && !string.IsNullOrEmpty(post.GroupId) && await _localActors.FindById(Models.Federation.LocalActorKind.Group, post.GroupId, token) is { IsCircle: false } community)
 				await _groups.Announce(community, activity.AsObject(), post.ObjectURI, isNewPost: false, token);
 		}
-
-		static bool IsEdit(NoteDocument note, PostEntity post) =>
-			note.Updated is { } updated && updated > (post.EditedAt ?? post.CreationDate);
-
-		async Task Requote(PostEntity post, NoteDocument note, CancellationToken token)
-		{
-			if (note.QuoteUri != post.QuoteURI || note.QuoteAuthorization != post.QuoteAuthorizationURI || post.QuoteState != QuoteState.Accepted)
-				await _quotes.Resolve(post, note, token);
-		}
 	}
 }

PrivaPub/Federation/Inbox/RemoteEdits.cs

@@ -0,0 +1,106 @@
+using MongoDB.Entities;
+
+using PrivaPub.Domain.Statuses;
+using PrivaPub.Federation.Actors;
+using PrivaPub.Federation.Objects;
+using PrivaPub.Models.Federation;
+using PrivaPub.Models.Post;
+using PrivaPub.Models.Social;
+
+using PostEntity = PrivaPub.Models.Post.Post;
+
+namespace PrivaPub.Federation.Inbox
+{
+	public static class RemoteEdits
+	{
+		public const int MaxRevisions = 20;
+
+		public static bool IsEdit(NoteDocument note, PostEntity post) =>
+			note.Updated is { } updated && updated > (post.EditedAt ?? post.CreationDate);
+
+		public static async Task<bool> Apply(PostEntity post, NoteDocument note, string activityId, ILocalActorService localActors, IObjectRecords records,
+			IQuoteService quotes, CancellationToken token)
+		{
+			post.Poll = note.Poll ?? post.Poll;
+			post.QuotePolicy = note.QuotePolicy;
+			post.Video = note.Video ?? post.Video;
+			post.Audio = note.Audio ?? post.Audio;
+			post.Event = note.Event ?? post.Event;
+			if (!IsEdit(note, post))
+			{
+				await DB.Default.SaveAsync(post, token);
+				await records.Revise(note, activityId, token);
+				await Requote(post, note, quotes, token);
+				return false;
+			}
+
+			post.Revisions.Add(new PostRevision
+			{
+				Title = post.Title,
+				SpoilerText = post.SpoilerText,
+				ContentHtml = post.ContentHtml,
+				HasContentWarning = post.HasContentWarning,
+				EditedAt = post.EditedAt ?? post.CreationDate
+			});
+			if (post.Revisions.Count > MaxRevisions)
+				post.Revisions.RemoveRange(0, post.Revisions.Count - MaxRevisions);
+
+			var mentions = new List<PostMention>();
+			foreach (var mention in note.Mentions)
+			{
+				var local = await localActors.FindByUri(mention.ActorURI, token);
+				mentions.Add(new PostMention { ActorURI = mention.ActorURI, Handle = mention.Handle, IsLocal = local != default, AccountId = local?.Id });
+			}
+
+			post.Title = note.Title;
+			post.SpoilerText = note.SpoilerText;
+			post.Excerpt = note.Excerpt;
+			post.Source = note.Source;
+			post.Emojis = note.Emojis.ToList();
+			post.CoverURL = note.CoverURL;
+			post.Link = note.Link;
+			post.HasContentWarning = note.Sensitive;
+			post.Text = note.ContentHtml;
+			post.ContentHtml = note.ContentHtml;
+			post.Language = note.Language;
+			post.Tags = note.Tags.ToList();
+			post.Mentions = mentions;
+			post.Media = note.Attachments.ToList();
+			post.EditedAt = note.Updated ?? DateTime.UtcNow;
+			post.UpdateDate = DateTime.UtcNow;
+			await DB.Default.SaveAsync(post, token);
+			await records.Revise(note, activityId, token);
+			await Requote(post, note, quotes, token);
+			return true;
+		}
+
+		static async Task Requote(PostEntity post, NoteDocument note, IQuoteService quotes, CancellationToken token)
+		{
+			if (note.QuoteUri != post.QuoteURI || note.QuoteAuthorization != post.QuoteAuthorizationURI || post.QuoteState != QuoteState.Accepted)
+				await quotes.Resolve(post, note, token);
+		}
+	}
+
+	public static class RemoteDeletes
+	{
+		public static async Task Tombstone(string objectUri, CancellationToken token) =>
+			await DB.Default.Update<DeletedObject>()
+				.Match(d => d.ObjectURI == objectUri)
+				.Modify(d => d.DeletedAt, DateTime.UtcNow)
+				.Option(o => o.IsUpsert = true)
+				.ExecuteAsync(token);
+
+		public static async Task Remove(PostEntity post, string objectUri, CancellationToken token)
+		{
+			await Tombstone(objectUri, token);
+			await DB.Default.DeleteAsync<PostEntity>(post.ID);
+			await DB.Default.DeleteAsync<ObjectRecord>(r => r.PostId == post.ID || r.ObjectURI == objectUri);
+			await DB.Default.DeleteAsync<TimelineEntry>(e => e.PostId == post.ID || e.ReblogOfPostId == post.ID);
+			await DB.Default.DeleteAsync<PostEntity>(p => p.ReblogOfPostId == post.ID);
+			if (!string.IsNullOrEmpty(post.AnsweringToPostId))
+				await DB.Default.Update<PostEntity>().MatchID(post.AnsweringToPostId).Modify(b => b.Inc(p => p.RepliesCount, -1)).ExecuteAsync(token);
+			if (post.QuoteState == QuoteState.Accepted && !string.IsNullOrEmpty(post.QuotedPostId))
+				await DB.Default.Update<PostEntity>().MatchID(post.QuotedPostId).Modify(b => b.Inc(p => p.QuotesCount, -1)).ExecuteAsync(token);
+		}
+	}
+}

PrivaPub/Federation/Objects/ObjectRecords.cs

@@ -34,8 +34,10 @@ namespace PrivaPub.Federation.Objects
 		public async Task Record(NoteDocument note, PostEntity post, ObjectPath path, bool refetched, CancellationToken token)
 		{
 			var arrival = Arrival.Current;
+			var delivered = path == ObjectPath.Delivered;
 			var raw = Capture(note.Raw);
 			var host = new Uri(note.Id).Host.ToLowerInvariant();
+			var activityContext = !delivered || note.Raw?.ContainsKey("@context") == true || arrival?.Context?.Length > 16 * 1024 ? default : arrival?.Context;
 			var record = new ObjectRecord
 			{
 				ObjectURI = note.Id,
@@ -46,20 +48,22 @@ namespace PrivaPub.Federation.Objects
 				RawHash = raw.Hash,
 				RawBytes = raw.Bytes,
 				RawTruncated = raw.Truncated,
-				ActivityContext = note.Raw?.ContainsKey("@context") == true || arrival?.Context?.Length > 16 * 1024 ? default : arrival?.Context,
+				ActivityContext = activityContext,
 				Path = path,
 				Refetched = refetched,
 				ActivityId = arrival?.ActivityId,
 				ActivityType = arrival?.ActivityType,
 				ActivityActorURI = arrival?.ActorURI,
-				Inbox = path == ObjectPath.Delivered ? arrival?.Inbox : default,
-				SignatureScheme = arrival?.KeyId == default ? default : "draft-cavage",
-				KeyId = arrival?.KeyId,
-				Algorithm = arrival?.Algorithm,
-				SignedHeaders = arrival?.SignedHeaders?.ToList() ?? new(),
-				ReceivedAt = arrival?.ReceivedAt ?? DateTime.UtcNow,
+				Inbox = delivered ? arrival?.Inbox : default,
+				SignatureScheme = !delivered || arrival?.KeyId == default ? default : "draft-cavage",
+				KeyId = delivered ? arrival?.KeyId : default,
+				Algorithm = delivered ? arrival?.Algorithm : default,
+				SignedHeaders = delivered ? arrival?.SignedHeaders?.ToList() ?? new() : new(),
+				ReceivedAt = delivered ? arrival?.ReceivedAt ?? DateTime.UtcNow : DateTime.UtcNow,
 				Published = note.Published,
-				Updated = note.Updated
+				Updated = note.Updated,
+				Extensions = ObjectFeatures.Detect(note.Raw),
+				ContextNamespaces = ObjectFeatures.Namespaces(note.Raw, activityContext)
 			};
 			try
 			{

PrivaPub/Infrastructure/Data/Migrations/_008_fetched_records_lose_the_trigger_signature.cs

@@ -0,0 +1,19 @@
+using MongoDB.Entities;
+
+using PrivaPub.Models.Federation;
+
+namespace PrivaPub.Infrastructure.Data.Migrations
+{
+	public class _008_fetched_records_lose_the_trigger_signature : IMigration
+	{
+		public async Task UpgradeAsync() =>
+			await DB.Default.Update<ObjectRecord>()
+				.Match(r => r.Path == ObjectPath.Fetched)
+				.Modify(r => r.SignatureScheme, null)
+				.Modify(r => r.KeyId, null)
+				.Modify(r => r.Algorithm, null)
+				.Modify(r => r.SignedHeaders, new List<string>())
+				.Modify(r => r.ActivityContext, null)
+				.ExecuteAsync();
+	}
+}

PrivaPub/Infrastructure/Data/Migrations/_009_object_records_keep_their_extensions.cs

@@ -0,0 +1,41 @@
+using MongoDB.Entities;
+
+using PrivaPub.Federation.Objects;
+using PrivaPub.Models.Federation;
+
+using System.Text.Json;
+using System.Text.Json.Nodes;
+
+namespace PrivaPub.Infrastructure.Data.Migrations
+{
+	public class _009_object_records_keep_their_extensions : IMigration
+	{
+		public async Task UpgradeAsync()
+		{
+			while (true)
+			{
+				var batch = await DB.Default.Find<ObjectRecord>()
+					.Match(r => r.Extensions == null)
+					.Limit(500)
+					.ExecuteAsync();
+				if (batch.Count == 0)
+					return;
+				foreach (var record in batch)
+				{
+					JsonObject raw = default;
+					try
+					{
+						raw = record.Raw == default ? default : JsonNode.Parse(record.Raw) as JsonObject;
+					}
+					catch (JsonException)
+					{
+					}
+					await DB.Default.Update<ObjectRecord>().MatchID(record.ID)
+						.Modify(r => r.Extensions, ObjectFeatures.Detect(raw))
+						.Modify(r => r.ContextNamespaces, ObjectFeatures.Namespaces(raw, record.ActivityContext))
+						.ExecuteAsync();
+				}
+			}
+		}
+	}
+}

PrivaPub/Models/Federation/ObjectRecord.cs

@@ -30,6 +30,8 @@ namespace PrivaPub.Models.Federation
 		public DateTime? Published { get; set; }
 		public DateTime? Updated { get; set; }
 		public List<ObjectRevision> Revisions { get; set; } = new();
+		public List<string> Extensions { get; set; }//ObjectFeatures, stored for statistics; null on records older than v1.16
+		public List<string> ContextNamespaces { get; set; }
 	}
 
 	public class ObjectRevision

PrivaPub/Models/User/Avatar.cs

@@ -66,6 +66,7 @@ namespace PrivaPub.Models.User
 		public string PictureDescription { get; set; }
 		public string HeaderDescription { get; set; }
 		public DateTime? Published { get; set; }//the actor's own published
+		public List<string> Features { get; set; }//protocol extensions its document uses, for statistics (ActorFeatures)
 
 		public string ActorURI { get; set; }//id
 		public string PublicKeyId { get; set; }