Every remote object keeps its raw form and how it reached us, for the client's details view

- ObjectRecord, one per stored remote object: the raw JSON (up to 256 KB, always hashed), delivered or fetched,
  refetched from origin or not, the activity that brought it (or caused the fetch), shared or personal inbox, the
  signature's key, algorithm and signed headers, received time, published and updated, the delivering activity's
  @context, and up to ten later revisions from Update.
- Delivery details travel from InboxReceiver through the inbox job to the handlers as Arrival.Current.
- A host is described from its NodeInfo when we first hear from it, at most weekly (DescribeInstance job), never when
  someone opens the details view.
- GET /api/privapub/v1/statuses/:id/provenance and /api/privapub/v1/instances/:host, with the extensions an object
  used detected from its raw form (044f quotes, interaction policies, contexts, proofs, Misskey fields, MFM, FEP-8967
  links, emoji, polls, language maps, url variants, Markdown content).

Checked live: a GoToSocial reply shows as delivered to the shared inbox, signed hs2019 with GoToSocial's fragment-less
key id, with its interaction policy detected, and gts.test is described as gotosocial 0.22.1.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012CzABvBkbcFqoHdmi8b9WB

PrivaPub/Federation/Objects/InstanceDescriber.cs

@@ -0,0 +1,84 @@
+using MongoDB.Entities;
+
+using PrivaPub.Infrastructure.Http;
+using PrivaPub.Infrastructure.Jobs;
+using PrivaPub.Models.Jobs;
+
+using System.Text.Json;
+
+namespace PrivaPub.Federation.Objects
+{
+	public class InstanceDescriber : IJobHandler
+	{
+		const int MaxNodeInfoChars = 64 * 1024;
+		static readonly string[] Schemas =
+		{
+			"http://nodeinfo.diaspora.software/ns/schema/2.1",
+			"http://nodeinfo.diaspora.software/ns/schema/2.0"
+		};
+
+		readonly IFederationHttp _http;
+
+		public InstanceDescriber(IFederationHttp http) => _http = http;
+
+		public JobKind Kind => JobKind.DescribeInstance;
+		public int Concurrency => 1;
+		public int MaxAttempts => 2;
+		public int PerHostLimit => 1;
+
+		public async Task<JobOutcome> Handle(Job job, CancellationToken token)
+		{
+			var host = job.Payload;
+			using var links = await _http.GetJson($"https://{host}/.well-known/nodeinfo", "application/json", sign: default, token);
+			var href = links == default ? default : Schemas.Select(schema => LinkFor(links.Root, schema)).FirstOrDefault(link => link != default);
+			if (href == default)
+				return await Described(host, default, "no NodeInfo", token);
+
+			using var nodeInfo = await _http.GetJson(href, "application/json", sign: default, token);
+			return nodeInfo == default ? await Described(host, default, "NodeInfo unreadable", token) : await Described(host, nodeInfo.Root, default, token);
+		}
+
+		static async Task<JobOutcome> Described(string host, JsonElement? nodeInfo, string error, CancellationToken token)
+		{
+			var update = DB.Default.Update<RemoteInstance>()
+				.Match(i => i.Host == host)
+				.Modify(i => i.DescribedAt, DateTime.UtcNow)
+				.Modify(i => i.DescriptionError, error)
+				.Option(o => o.IsUpsert = true);
+			if (nodeInfo is { } root)
+			{
+				var raw = root.GetRawText();
+				update = update
+					.Modify(i => i.Software, Text(root, "software", "name"))
+					.Modify(i => i.SoftwareVersion, Text(root, "software", "version"))
+					.Modify(i => i.NodeName, Text(root, "metadata", "nodeName"))
+					.Modify(i => i.Protocols, root.TryGetProperty("protocols", out var protocols) && protocols.ValueKind == JsonValueKind.Array
+						? protocols.EnumerateArray().Where(p => p.ValueKind == JsonValueKind.String).Select(p => p.GetString()).Take(16).ToList()
+						: new List<string>())
+					.Modify(i => i.OpenRegistrations, root.TryGetProperty("openRegistrations", out var open) && open.ValueKind is JsonValueKind.True or JsonValueKind.False
+						? open.GetBoolean()
+						: default(bool?))
+					.Modify(i => i.NodeInfo, raw.Length <= MaxNodeInfoChars ? raw : default);
+			}
+			await update.ExecuteAsync(token);
+			return JobOutcome.Done;
+		}
+
+		static string LinkFor(JsonElement root, string schema) =>
+			root.TryGetProperty("links", out var links) && links.ValueKind == JsonValueKind.Array
+				? links.EnumerateArray()
+					.Where(l => l.ValueKind == JsonValueKind.Object && l.TryGetProperty("rel", out var rel) && rel.ValueKind == JsonValueKind.String && rel.GetString() == schema)
+					.Select(l => l.TryGetProperty("href", out var href) && href.ValueKind == JsonValueKind.String ? href.GetString() : default)
+					.FirstOrDefault(h => h != default && h.StartsWith("https://", StringComparison.OrdinalIgnoreCase))
+				: default;
+
+		static string Text(JsonElement root, string parent, string name)
+		{
+			if (!root.TryGetProperty(parent, out var section) || section.ValueKind != JsonValueKind.Object
+				|| !section.TryGetProperty(name, out var value) || value.ValueKind != JsonValueKind.String)
+				return default;
+			var text = value.GetString();
+			return text.Length <= 200 ? text : text[..200];
+		}
+	}
+}

PrivaPub/Federation/Objects/NoteParser.cs

@@ -11,6 +11,7 @@ namespace PrivaPub.Federation.Objects
 	{
 		public string Id { get; init; }
 		public string Type { get; init; }
+		public JsonObject Raw { get; init; }
 		public string AttributedTo { get; init; }
 		public string ContentHtml { get; init; }
 		public string Title { get; init; }
@@ -55,6 +56,7 @@ namespace PrivaPub.Federation.Objects
 			{
 				Id = id,
 				Type = type,
+				Raw = note,
 				AttributedTo = Attribution(note["attributedTo"]),
 				ContentHtml = ContentSanitizer.Html(contentHtml),
 				Title = Plain(Value(note, "name"), 500),

PrivaPub/Federation/Objects/ObjectFeatures.cs

@@ -0,0 +1,68 @@
+using System.Text.Json.Nodes;
+
+using static PrivaPub.Federation.Objects.ActivityJson;
+
+namespace PrivaPub.Federation.Objects
+{
+	public static class ObjectFeatures
+	{
+		public static List<string> Detect(JsonObject raw)
+		{
+			var found = new List<string>();
+			if (raw == default)
+				return found;
+			void When(bool condition, string feature)
+			{
+				if (condition)
+					found.Add(feature);
+			}
+
+			When(raw.ContainsKey("quote"), "fep-044f-quote");
+			When(raw.ContainsKey("quoteAuthorization"), "fep-044f-authorization");
+			When(raw.ContainsKey("_misskey_quote") || raw.ContainsKey("quoteUrl") || raw.ContainsKey("quoteUri"), "legacy-quote");
+			When(raw.ContainsKey("interactionPolicy"), "interaction-policy");
+			When(raw.ContainsKey("context"), "context");
+			When(raw.ContainsKey("proof"), "fep-8b32-proof");
+			When(raw.Any(p => p.Key.StartsWith("_misskey_", StringComparison.Ordinal)), "misskey");
+			When(raw["source"] is JsonObject, "source:" + (Value(raw["source"], "mediaType") ?? "unknown"));
+			When(raw["source"] is JsonValue, "source:url");
+			When(raw.ContainsKey("htmlMfm"), "fep-c16b-mfm");
+			When(raw.ContainsKey("formerRepresentations"), "former-representations");
+			When(raw.ContainsKey("searchableBy"), "fep-268d-searchable-by");
+			When(raw.ContainsKey("audience"), "fep-1b12-audience");
+			When(Items(raw["attachment"]).Any(a => Value(a, "type") == "Link"), "fep-8967-link");
+			When(Items(raw["tag"]).Any(t => Value(t, "type") == "Emoji"), "custom-emoji");
+			When(Value(raw, "type") == "Question", "poll");
+			When(raw["contentMap"] is JsonObject, "content-map");
+			When(raw["url"] is JsonArray, "url-variants");
+			var contentType = Value(raw, "mediaType");
+			When(contentType is not (null or "text/html"), "content:" + contentType);
+			return found;
+		}
+
+		public static List<string> Namespaces(JsonObject raw, string activityContext = default) =>
+			Namespaces(raw?["@context"] ?? (activityContext == default ? default : JsonNode.Parse(activityContext)));
+
+		static List<string> Namespaces(JsonNode context) => context switch
+		{
+			JsonValue single when single.TryGetValue<string>(out var url) => new List<string> { url },
+			JsonArray array => array.SelectMany(entry => entry switch
+				{
+					JsonValue value when value.TryGetValue<string>(out var url) => new[] { url },
+					JsonObject terms => terms.Where(t => t.Value is JsonValue v && v.TryGetValue<string>(out var target) && target.EndsWith('#'))
+						.Select(t => t.Key + "=" + t.Value!.GetValue<string>()),
+					_ => Enumerable.Empty<string>()
+				})
+				.Take(32)
+				.ToList(),
+			_ => new List<string>()
+		};
+
+		static IEnumerable<JsonObject> Items(JsonNode node) => node switch
+		{
+			JsonArray array => array.OfType<JsonObject>(),
+			JsonObject single => new[] { single },
+			_ => Enumerable.Empty<JsonObject>()
+		};
+	}
+}

PrivaPub/Federation/Objects/ObjectRecords.cs

@@ -0,0 +1,112 @@
+using MongoDB.Driver;
+using MongoDB.Entities;
+
+using PrivaPub.Federation.Inbox;
+using PrivaPub.Infrastructure.Jobs;
+using PrivaPub.Models.Federation;
+using PrivaPub.Models.Jobs;
+
+using System.Globalization;
+using System.Security.Cryptography;
+using System.Text;
+using System.Text.Json.Nodes;
+
+using PostEntity = PrivaPub.Models.Post.Post;
+
+namespace PrivaPub.Federation.Objects
+{
+	public interface IObjectRecords
+	{
+		Task Record(NoteDocument note, PostEntity post, ObjectPath path, bool refetched, CancellationToken token);
+		Task Revise(NoteDocument note, string activityId, CancellationToken token);
+	}
+
+	public class ObjectRecords : IObjectRecords
+	{
+		public const int MaxRawBytes = 256 * 1024;
+		const int MaxRevisions = 10;
+		static readonly TimeSpan DescriptionAge = TimeSpan.FromDays(7);
+
+		readonly IJobQueue _queue;
+
+		public ObjectRecords(IJobQueue queue) => _queue = queue;
+
+		public async Task Record(NoteDocument note, PostEntity post, ObjectPath path, bool refetched, CancellationToken token)
+		{
+			var arrival = Arrival.Current;
+			var raw = Capture(note.Raw);
+			var host = new Uri(note.Id).Host.ToLowerInvariant();
+			var record = new ObjectRecord
+			{
+				ObjectURI = note.Id,
+				PostId = post.ID,
+				ObjectType = note.Type,
+				Host = host,
+				Raw = raw.Text,
+				RawHash = raw.Hash,
+				RawBytes = raw.Bytes,
+				RawTruncated = raw.Truncated,
+				ActivityContext = note.Raw?.ContainsKey("@context") == true || arrival?.Context?.Length > 16 * 1024 ? default : arrival?.Context,
+				Path = path,
+				Refetched = refetched,
+				ActivityId = arrival?.ActivityId,
+				ActivityType = arrival?.ActivityType,
+				ActivityActorURI = arrival?.ActorURI,
+				Inbox = path == ObjectPath.Delivered ? arrival?.Inbox : default,
+				SignatureScheme = arrival?.KeyId == default ? default : "draft-cavage",
+				KeyId = arrival?.KeyId,
+				Algorithm = arrival?.Algorithm,
+				SignedHeaders = arrival?.SignedHeaders?.ToList() ?? new(),
+				ReceivedAt = arrival?.ReceivedAt ?? DateTime.UtcNow,
+				Published = note.Published,
+				Updated = note.Updated
+			};
+			try
+			{
+				await DB.Default.SaveAsync(record, token);
+			}
+			catch (MongoWriteException ex) when (ex.WriteError?.Category == ServerErrorCategory.DuplicateKey)
+			{
+				return;
+			}
+			await Describe(host, token);
+		}
+
+		public async Task Revise(NoteDocument note, string activityId, CancellationToken token)
+		{
+			var raw = Capture(note.Raw);
+			var revision = new ObjectRevision
+			{
+				Raw = raw.Text,
+				RawHash = raw.Hash,
+				RawTruncated = raw.Truncated,
+				ActivityId = activityId,
+				Updated = note.Updated
+			};
+			await DB.Default.Update<ObjectRecord>()
+				.Match(r => r.ObjectURI == note.Id)
+				.Modify(b => b.PushEach(r => r.Revisions, new[] { revision }, -MaxRevisions))
+				.Modify(r => r.Updated, note.Updated)
+				.ExecuteAsync(token);
+		}
+
+		async Task Describe(string host, CancellationToken token)
+		{
+			var known = await DB.Default.Find<RemoteInstance>().Match(i => i.Host == host).ExecuteFirstAsync(token);
+			if (known?.DescribedAt > DateTime.UtcNow - DescriptionAge)
+				return;
+			var week = ISOWeek.GetYear(DateTime.UtcNow) + "-" + ISOWeek.GetWeekOfYear(DateTime.UtcNow);
+			await _queue.Enqueue(JobKind.DescribeInstance, host, host, $"describe|{host}|{week}", token);
+		}
+
+		public static (string Text, string Hash, int Bytes, bool Truncated) Capture(JsonNode raw)
+		{
+			if (raw == default)
+				return default;
+			var text = raw.ToJsonString();
+			var bytes = Encoding.UTF8.GetBytes(text);
+			var hash = Convert.ToHexStringLower(SHA256.HashData(bytes));
+			return bytes.Length > MaxRawBytes ? (default, hash, bytes.Length, true) : (text, hash, bytes.Length, false);
+		}
+	}
+}