From dcd024100a6ecc46297f295f148ceeb403de7918 Mon Sep 17 00:00:00 2001 From: thepra Date: Thu, 1 Oct 2026 17:49:22 +0200 Subject: [PATCH] Every remote object keeps its raw form and how it reached us, for the client's details view - ObjectRecord, one per stored remote object: the raw JSON (up to 256 KB, always hashed), delivered or fetched, refetched from origin or not, the activity that brought it (or caused the fetch), shared or personal inbox, the signature's key, algorithm and signed headers, received time, published and updated, the delivering activity's @context, and up to ten later revisions from Update. - Delivery details travel from InboxReceiver through the inbox job to the handlers as Arrival.Current. - A host is described from its NodeInfo when we first hear from it, at most weekly (DescribeInstance job), never when someone opens the details view. - GET /api/privapub/v1/statuses/:id/provenance and /api/privapub/v1/instances/:host, with the extensions an object used detected from its raw form (044f quotes, interaction policies, contexts, proofs, Misskey fields, MFM, FEP-8967 links, emoji, polls, language maps, url variants, Markdown content). Checked live: a GoToSocial reply shows as delivered to the shared inbox, signed hs2019 with GoToSocial's fragment-less key id, with its interaction policy detected, and gts.test is described as gotosocial 0.22.1. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012CzABvBkbcFqoHdmi8b9WB --- CLAUDE.md | 8 + .../Federation/InboxScenarioTests.cs | 4 +- PrivaPub.Tests/Federation/ProvenanceTests.cs | 85 ++++++++ PrivaPub.Tests/Support/Harness.cs | 9 +- .../Controllers/ProvenanceController.cs | 181 ++++++++++++++++++ PrivaPub/Federation/Inbox/Arrival.cs | 14 ++ .../Inbox/Handlers/CreateHandler.cs | 8 +- .../Inbox/Handlers/UpdateHandler.cs | 6 +- PrivaPub/Federation/Inbox/InboxProcessor.cs | 11 +- PrivaPub/Federation/Inbox/InboxReceiver.cs | 7 +- PrivaPub/Federation/Inbox/RemotePosts.cs | 6 +- .../Federation/Objects/InstanceDescriber.cs | 84 ++++++++ PrivaPub/Federation/Objects/NoteParser.cs | 2 + PrivaPub/Federation/Objects/ObjectFeatures.cs | 68 +++++++ PrivaPub/Federation/Objects/ObjectRecords.cs | 112 +++++++++++ PrivaPub/Infrastructure/Data/Indexes.cs | 2 + .../Middleware/SocialPubConfigurations.cs | 3 + PrivaPub/Models/Federation/ObjectRecord.cs | 50 +++++ PrivaPub/Models/Jobs/Job.cs | 3 +- PrivaPub/Models/Jobs/RemoteInstance.cs | 9 + docs/ROADMAP.md | 6 +- 21 files changed, 662 insertions(+), 16 deletions(-) create mode 100644 PrivaPub.Tests/Federation/ProvenanceTests.cs create mode 100644 PrivaPub/Api/Mastodon/Controllers/ProvenanceController.cs create mode 100644 PrivaPub/Federation/Inbox/Arrival.cs create mode 100644 PrivaPub/Federation/Objects/InstanceDescriber.cs create mode 100644 PrivaPub/Federation/Objects/ObjectFeatures.cs create mode 100644 PrivaPub/Federation/Objects/ObjectRecords.cs create mode 100644 PrivaPub/Models/Federation/ObjectRecord.cs diff --git a/CLAUDE.md b/CLAUDE.md index d3bae61..8f7a4f3 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -238,6 +238,14 @@ cd /var/www/privapub.thepra.dev && sudo -u www-data ASPNETCORE_ENVIRONMENT=Produ - **Collection name = class name, so never rename an entity class.** - `Entity.ID` is a 24-character lowercase hex string; `GenerateNewID()` returns `object`, so cast it. - New fields must be additive: a deploy rollback restores the binary, not the database. +- **Every stored remote object has an `ObjectRecord`** (raw JSON up to 256 KB plus its hash; delivered or fetched; the + activity, inbox, key and signed headers; received time; up to 10 later revisions). Write it right after the post's + save: `CreateHandler` (delivered) and `RemotePosts.StoreContext` (fetched) do, and `UpdateHandler` appends a + revision. Delivery details reach them through `Arrival.Current`, which `InboxProcessor` sets for the handler's + duration. The raw form lives outside `Post` so timelines never load it. Read through + `/api/privapub/v1/statuses/:id/provenance` and `/api/privapub/v1/instances/:host`. +- **A server is described on arrival, never on read.** The first record from a host enqueues `DescribeInstance` (its + NodeInfo, at most once a week, into `RemoteInstance`), so opening the details view tells nobody anything. - **Post ids are the timeline order, so they follow arrival, not `published`.** `PrivacyIds.Arrived` gives a remote post published within the last hour (or in the future) a fresh `ObjectId`, which sorts after every post already stored, and only backfill keeps a `published`-derived id. With `published` ids a reply arriving in the same second could sort diff --git a/PrivaPub.Tests/Federation/InboxScenarioTests.cs b/PrivaPub.Tests/Federation/InboxScenarioTests.cs index 9b6733c..b7debb1 100644 --- a/PrivaPub.Tests/Federation/InboxScenarioTests.cs +++ b/PrivaPub.Tests/Federation/InboxScenarioTests.cs @@ -54,9 +54,9 @@ namespace PrivaPub.Tests.Federation { new FollowHandler(db, _local, remote, delivery), new UndoHandler(db, _local), - new CreateHandler(db, _local, remote, delivery, _blocks, new Fanout(db), new RemotePosts(db, _local, remote, _blocks, queue), new GroupDistributor(delivery)), + new CreateHandler(db, _local, remote, delivery, _blocks, new Fanout(db), new RemotePosts(db, _local, remote, _blocks, queue, new ObjectRecords(queue)), new GroupDistributor(delivery), new ObjectRecords(queue)), new DeleteHandler(db, _local, remote, delivery, new GroupDistributor(delivery)), - new UpdateHandler(db, _local, remote, new GroupDistributor(delivery)) + new UpdateHandler(db, _local, remote, new GroupDistributor(delivery), new ObjectRecords(queue)) }, NullLogger.Instance); } diff --git a/PrivaPub.Tests/Federation/ProvenanceTests.cs b/PrivaPub.Tests/Federation/ProvenanceTests.cs new file mode 100644 index 0000000..d695a42 --- /dev/null +++ b/PrivaPub.Tests/Federation/ProvenanceTests.cs @@ -0,0 +1,85 @@ +using MongoDB.Entities; + +using PrivaPub.Federation.Objects; +using PrivaPub.Models.Federation; +using PrivaPub.Models.Jobs; +using PrivaPub.Tests.Support; + +using System.Text.Json.Nodes; + +namespace PrivaPub.Tests.Federation +{ + [Trait("Category", "Integration")] + public sealed class ProvenanceTests : IAsyncLifetime + { + Harness _harness; + + public async ValueTask InitializeAsync() + { + Assert.SkipUnless(MongoFixture.Enabled, MongoFixture.Skip); + _harness = await Harness.Start(); + } + + public async ValueTask DisposeAsync() + { + if (_harness != default) + await _harness.DisposeAsync(); + } + + [Fact] + public async Task A_delivered_post_keeps_its_raw_object_how_it_came_and_every_edit() + { + var token = TestContext.Current.CancellationToken; + var (_, alice) = await _harness.Persona("alice"); + var mallory = new RemoteActor(_harness.Peer, "mallory"); + var origin = new Uri(mallory.Id).GetLeftPart(UriPartial.Authority); + var note = new JsonObject + { + ["id"] = $"{origin}/notes/{Guid.NewGuid():N}", + ["type"] = "Note", + ["attributedTo"] = mallory.Id, + ["content"] = "

hi

", + ["_misskey_quote"] = $"{origin}/notes/elsewhere", + ["to"] = new JsonArray(Addressing.Public), + ["cc"] = new JsonArray(alice.Uri), + ["tag"] = new JsonArray(new JsonObject { ["type"] = "Mention", ["href"] = alice.Uri, ["name"] = "@alice" }) + }; + var create = new JsonObject { ["id"] = $"{origin}/activities/{Guid.NewGuid():N}", ["type"] = "Create", ["actor"] = mallory.Id, ["object"] = note.DeepClone() }; + + await _harness.Deliver(mallory, "/human-centipede", create); + var edited = note.DeepClone().AsObject(); + edited["content"] = "

hi again

"; + edited["updated"] = DateTime.UtcNow.ToString("O"); + await _harness.Deliver(mallory, "/human-centipede", new JsonObject + { + ["id"] = $"{origin}/activities/{Guid.NewGuid():N}", ["type"] = "Update", ["actor"] = mallory.Id, ["object"] = edited + }); + + var record = await DB.Default.Find().Match(r => r.ObjectURI == note["id"]!.GetValue()).ExecuteFirstAsync(token); + Assert.NotNull(record); + Assert.Equal(ObjectPath.Delivered, record.Path); + Assert.False(record.Refetched); + Assert.Equal("shared", record.Inbox); + Assert.Equal(mallory.KeyId, record.KeyId); + Assert.Equal(create["id"]!.GetValue(), record.ActivityId); + Assert.Contains("digest", record.SignedHeaders); + Assert.Equal("

hi

", JsonNode.Parse(record.Raw)!["content"]!.GetValue()); + Assert.Contains("legacy-quote", ObjectFeatures.Detect(JsonNode.Parse(record.Raw)!.AsObject())); + Assert.Equal("

hi again

", JsonNode.Parse(Assert.Single(record.Revisions).Raw)!["content"]!.GetValue()); + Assert.True(await DB.Default.Find().Match(j => j.Kind == JobKind.DescribeInstance && j.Payload == record.Host).ExecuteAnyAsync(token)); + } + + [Fact] + public void An_object_larger_than_the_cap_keeps_only_its_hash() + { + var big = new JsonObject { ["content"] = new string('x', ObjectRecords.MaxRawBytes + 1) }; + + var (text, hash, bytes, truncated) = ObjectRecords.Capture(big); + + Assert.Null(text); + Assert.True(truncated); + Assert.True(bytes > ObjectRecords.MaxRawBytes); + Assert.Equal(64, hash.Length); + } + } +} diff --git a/PrivaPub.Tests/Support/Harness.cs b/PrivaPub.Tests/Support/Harness.cs index 202f51a..d8a5196 100644 --- a/PrivaPub.Tests/Support/Harness.cs +++ b/PrivaPub.Tests/Support/Harness.cs @@ -16,6 +16,7 @@ using PrivaPub.Federation.Inbox.Handlers; using PrivaPub.Federation.Outbox; using PrivaPub.Infrastructure.Jobs; using PrivaPub.Models; +using PrivaPub.Federation.Objects; using PrivaPub.Models.Federation; using PrivaPub.Models.Jobs; using PrivaPub.Models.User; @@ -42,7 +43,8 @@ namespace PrivaPub.Tests.Support Delivery = new DeliveryService(Db, Queue); Fanout = new Fanout(Db); Groups = new GroupDistributor(Delivery); - RemotePosts = new RemotePosts(Db, Local, Remote, new NoBlocks(), Queue); + Records = new ObjectRecords(Queue); + RemotePosts = new RemotePosts(Db, Local, Remote, new NoBlocks(), Queue, Records); Receiver = new InboxReceiver(Local, Remote, Queue, new NoBlocks(), NullLogger.Instance); Processor = new InboxProcessor(Remote, new IActivityHandler[] { @@ -52,9 +54,9 @@ namespace PrivaPub.Tests.Support new UndoHandler(Db, Local), new LikeHandler(Db), new AnnounceHandler(Db, Local, RemotePosts, Fanout, Remote), - new CreateHandler(Db, Local, Remote, Delivery, new NoBlocks(), Fanout, RemotePosts, Groups), + new CreateHandler(Db, Local, Remote, Delivery, new NoBlocks(), Fanout, RemotePosts, Groups, Records), new DeleteHandler(Db, Local, Remote, Delivery, Groups), - new UpdateHandler(Db, Local, Remote, Groups), + new UpdateHandler(Db, Local, Remote, Groups, Records), new FlagHandler(Db, Local) }, NullLogger.Instance); Follows = new FollowService(Db, Local, Remote, Delivery, new KeyLocalizer(), NullLogger.Instance); @@ -86,6 +88,7 @@ namespace PrivaPub.Tests.Support public GroupDistributor Groups { get; } public Fanout Fanout { get; } public RemotePosts RemotePosts { get; } + public ObjectRecords Records { get; } public TimelineService Timelines { get; } public RelationshipService Relationships { get; } public ReportService Reports { get; } diff --git a/PrivaPub/Api/Mastodon/Controllers/ProvenanceController.cs b/PrivaPub/Api/Mastodon/Controllers/ProvenanceController.cs new file mode 100644 index 0000000..ba3f470 --- /dev/null +++ b/PrivaPub/Api/Mastodon/Controllers/ProvenanceController.cs @@ -0,0 +1,181 @@ +using Microsoft.AspNetCore.Authorization; +using Microsoft.AspNetCore.Mvc; + +using MongoDB.Entities; + +using PrivaPub.Api.Mastodon.Infrastructure; +using PrivaPub.Domain.Privacy; +using PrivaPub.Federation.Objects; +using PrivaPub.Models.Federation; +using PrivaPub.Models.Jobs; +using PrivaPub.Models.Post; +using PrivaPub.StaticServices; + +using System.Text.Json.Nodes; + +using PostEntity = PrivaPub.Models.Post.Post; + +namespace PrivaPub.Api.Mastodon.Controllers +{ + public class ProvenanceController : MastodonController + { + readonly DbEntities _dbEntities; + + public ProvenanceController(DbEntities dbEntities) => _dbEntities = dbEntities; + + [HttpGet("/api/privapub/v1/statuses/{id}/provenance"), Scope("read:statuses", requiresUser: false), AllowAnonymous] + public async Task Status(string id, CancellationToken token) + { + var post = await _dbEntities.Posts.Match(p => p.ID == id && !p.DeletedAt.HasValue).ExecuteFirstAsync(token); + if (post?.ReblogOfPostId != default) + post = await _dbEntities.Posts.Match(p => p.ID == post.ReblogOfPostId && !p.DeletedAt.HasValue).ExecuteFirstAsync(token); + if (post == default || post.Visibility == PostVisibility.LocalGeo || !await VisibilityPolicy.CanSee(post, MyId, token)) + return NotFoundError(); + if (!post.IsFederatedCopy) + return Json(new Provenance { ObjectUri = post.ObjectURI, Url = post.Url, Path = "local" }); + + var record = await DB.Default.Find().Match(r => r.PostId == post.ID).ExecuteFirstAsync(token) + ?? await DB.Default.Find().Match(r => r.ObjectURI == post.ObjectURI).ExecuteFirstAsync(token); + var host = record?.Host ?? (Uri.TryCreate(post.ObjectURI, UriKind.Absolute, out var uri) ? uri.Host.ToLowerInvariant() : default); + var instance = host == default ? default : await DB.Default.Find().Match(i => i.Host == host).ExecuteFirstAsync(token); + return Json(Describe(post, record, instance)); + } + + [HttpGet("/api/privapub/v1/instances/{host}"), Scope("read", requiresUser: false), AllowAnonymous] + public async Task Instance(string host, CancellationToken token) + { + host = host?.Trim().ToLowerInvariant(); + var instance = await DB.Default.Find().Match(i => i.Host == host).ExecuteFirstAsync(token); + return instance == default ? NotFoundError() : Json(Describe(instance)); + } + + static Provenance Describe(PostEntity post, ObjectRecord record, RemoteInstance instance) + { + var raw = record?.Raw == default ? default : JsonNode.Parse(record.Raw) as JsonObject; + return new Provenance + { + ObjectUri = post.ObjectURI, + ObjectType = record?.ObjectType ?? post.ObjectType, + Url = post.Url, + Path = record == default ? "unrecorded" : record.Path == ObjectPath.Delivered ? "delivered" : "fetched", + Refetched = record?.Refetched == true, + Activity = record?.ActivityId == default ? default : new ProvenanceActivity + { + Id = record.ActivityId, Type = record.ActivityType, Actor = record.ActivityActorURI + }, + Inbox = record?.Inbox, + Signature = record?.KeyId == default ? default : new ProvenanceSignature + { + Scheme = record.SignatureScheme, KeyId = record.KeyId, Algorithm = record.Algorithm, Headers = record.SignedHeaders + }, + ReceivedAt = record == default ? default : MastodonJson.Time(record.ReceivedAt), + Published = record?.Published is { } published ? MastodonJson.Time(published) : default, + Updated = record?.Updated is { } updated ? MastodonJson.Time(updated) : default, + Extensions = ObjectFeatures.Detect(raw), + ContextNamespaces = ObjectFeatures.Namespaces(raw, record?.ActivityContext), + Raw = raw, + RawBytes = record?.RawBytes ?? 0, + RawHash = record?.RawHash, + RawTruncated = record?.RawTruncated == true, + Revisions = record?.Revisions.Select(r => new ProvenanceRevision + { + ActivityId = r.ActivityId, + Updated = r.Updated is { } at ? MastodonJson.Time(at) : default, + ReceivedAt = MastodonJson.Time(r.ReceivedAt), + RawHash = r.RawHash, + Raw = r.Raw == default ? default : JsonNode.Parse(r.Raw) + }).ToList() ?? new(), + Instance = instance == default ? default : Describe(instance) + }; + } + + static InstanceDescription Describe(RemoteInstance instance) => new() + { + Host = instance.Host, + Software = instance.Software, + Version = instance.SoftwareVersion, + NodeName = instance.NodeName, + Protocols = instance.Protocols ?? new(), + OpenRegistrations = instance.OpenRegistrations, + DescribedAt = instance.DescribedAt is { } described ? MastodonJson.Time(described) : default, + DescriptionError = instance.DescriptionError, + NodeInfo = instance.NodeInfo == default ? default : JsonNode.Parse(instance.NodeInfo), + Delivery = new InstanceDelivery + { + ConsecutiveFailures = instance.ConsecutiveFailures, + UnavailableUntil = instance.UnavailableUntil is { } until ? MastodonJson.Time(until) : default, + LastSuccessAt = instance.LastSuccessAt is { } success ? MastodonJson.Time(success) : default, + LastFailureAt = instance.LastFailureAt is { } failure ? MastodonJson.Time(failure) : default + } + }; + + public class Provenance + { + public string ObjectUri { get; set; } + public string ObjectType { get; set; } + public string Url { get; set; } + public string Path { get; set; } + public bool Refetched { get; set; } + public ProvenanceActivity Activity { get; set; } + public string Inbox { get; set; } + public ProvenanceSignature Signature { get; set; } + public string ReceivedAt { get; set; } + public string Published { get; set; } + public string Updated { get; set; } + public List Extensions { get; set; } = new(); + public List ContextNamespaces { get; set; } = new(); + public JsonNode Raw { get; set; } + public int RawBytes { get; set; } + public string RawHash { get; set; } + public bool RawTruncated { get; set; } + public List Revisions { get; set; } = new(); + public InstanceDescription Instance { get; set; } + } + + public class ProvenanceActivity + { + public string Id { get; set; } + public string Type { get; set; } + public string Actor { get; set; } + } + + public class ProvenanceSignature + { + public string Scheme { get; set; } + public string KeyId { get; set; } + public string Algorithm { get; set; } + public List Headers { get; set; } = new(); + } + + public class ProvenanceRevision + { + public string ActivityId { get; set; } + public string Updated { get; set; } + public string ReceivedAt { get; set; } + public string RawHash { get; set; } + public JsonNode Raw { get; set; } + } + + public class InstanceDescription + { + public string Host { get; set; } + public string Software { get; set; } + public string Version { get; set; } + public string NodeName { get; set; } + public List Protocols { get; set; } = new(); + public bool? OpenRegistrations { get; set; } + public string DescribedAt { get; set; } + public string DescriptionError { get; set; } + public JsonNode NodeInfo { get; set; } + public InstanceDelivery Delivery { get; set; } + } + + public class InstanceDelivery + { + public int ConsecutiveFailures { get; set; } + public string UnavailableUntil { get; set; } + public string LastSuccessAt { get; set; } + public string LastFailureAt { get; set; } + } + } +} diff --git a/PrivaPub/Federation/Inbox/Arrival.cs b/PrivaPub/Federation/Inbox/Arrival.cs new file mode 100644 index 0000000..a18d815 --- /dev/null +++ b/PrivaPub/Federation/Inbox/Arrival.cs @@ -0,0 +1,14 @@ +namespace PrivaPub.Federation.Inbox +{ + public sealed record Arrival(string ActivityId, string ActivityType, string ActorURI, string Inbox, string KeyId, string Algorithm, + IReadOnlyList SignedHeaders, DateTime ReceivedAt, string Context = default) + { + static readonly AsyncLocal current = new(); + + public static Arrival Current + { + get => current.Value; + set => current.Value = value; + } + } +} diff --git a/PrivaPub/Federation/Inbox/Handlers/CreateHandler.cs b/PrivaPub/Federation/Inbox/Handlers/CreateHandler.cs index f2bb30b..3dd9209 100644 --- a/PrivaPub/Federation/Inbox/Handlers/CreateHandler.cs +++ b/PrivaPub/Federation/Inbox/Handlers/CreateHandler.cs @@ -34,10 +34,12 @@ namespace PrivaPub.Federation.Inbox.Handlers readonly IFanout _fanout; readonly IRemotePosts _remotePosts; readonly IGroupDistributor _groups; + readonly IObjectRecords _records; public CreateHandler(DbEntities dbEntities, ILocalActorService localActors, IRemoteActorService remoteActors, IDeliveryService delivery, - IDomainBlocks domainBlocks, IFanout fanout, IRemotePosts remotePosts, IGroupDistributor groups) + IDomainBlocks domainBlocks, IFanout fanout, IRemotePosts remotePosts, IGroupDistributor groups, IObjectRecords records) { + _records = records; _groups = groups; _fanout = fanout; _remotePosts = remotePosts; @@ -53,7 +55,8 @@ namespace PrivaPub.Federation.Inbox.Handlers public async Task Handle(JsonNode activity, ForeignAvatar author, CancellationToken token) { var node = activity["object"]; - if (node is not JsonObject || !Origin.Same(Id(node), author.ActorURI)) + var refetched = node is not JsonObject || !Origin.Same(Id(node), author.ActorURI); + if (refetched) { using var fetched = await _remoteActors.FetchObject(Id(node), token); node = fetched == default ? default : JsonNode.Parse(fetched.Root.GetRawText()); @@ -131,6 +134,7 @@ namespace PrivaPub.Federation.Inbox.Handlers { return; } + await _records.Record(note, post, ObjectPath.Delivered, refetched, token); if (parent != default) await DB.Default.Update().MatchID(parent.ID).Modify(b => b.Inc(p => p.RepliesCount, 1)).ExecuteAsync(token); diff --git a/PrivaPub/Federation/Inbox/Handlers/UpdateHandler.cs b/PrivaPub/Federation/Inbox/Handlers/UpdateHandler.cs index f4d8937..d82cfb6 100644 --- a/PrivaPub/Federation/Inbox/Handlers/UpdateHandler.cs +++ b/PrivaPub/Federation/Inbox/Handlers/UpdateHandler.cs @@ -23,9 +23,12 @@ namespace PrivaPub.Federation.Inbox.Handlers readonly ILocalActorService _localActors; readonly IRemoteActorService _remoteActors; readonly IGroupDistributor _groups; + readonly IObjectRecords _records; - public UpdateHandler(DbEntities dbEntities, ILocalActorService localActors, IRemoteActorService remoteActors, IGroupDistributor groups) + public UpdateHandler(DbEntities dbEntities, ILocalActorService localActors, IRemoteActorService remoteActors, IGroupDistributor groups, + IObjectRecords records) { + _records = records; _groups = groups; _dbEntities = dbEntities; _localActors = localActors; @@ -83,6 +86,7 @@ namespace PrivaPub.Federation.Inbox.Handlers post.EditedAt = note.Updated ?? DateTime.UtcNow; post.UpdateDate = DateTime.UtcNow; await DB.Default.SaveAsync(post, token); + await _records.Revise(note, Id(activity), token); if (!string.IsNullOrEmpty(post.GroupId) && await _localActors.FindById(Models.Federation.LocalActorKind.Group, post.GroupId, token) is { IsCircle: false } community) await _groups.Announce(community, activity.AsObject(), post.ObjectURI, isNewPost: false, token); } diff --git a/PrivaPub/Federation/Inbox/InboxProcessor.cs b/PrivaPub/Federation/Inbox/InboxProcessor.cs index 1738950..267c046 100644 --- a/PrivaPub/Federation/Inbox/InboxProcessor.cs +++ b/PrivaPub/Federation/Inbox/InboxProcessor.cs @@ -46,7 +46,16 @@ namespace PrivaPub.Federation.Inbox if (actor == default) return JobOutcome.Retry("the actor could not be loaded"); - await handler.Handle(activity, actor, token); + Arrival.Current = new Arrival(Id(activity), type, actor.ActorURI, payload.Inbox, payload.KeyId, payload.Algorithm, + payload.SignedHeaders ?? Array.Empty(), payload.ReceivedAt ?? job.CreatedAt, activity["@context"]?.ToJsonString()); + try + { + await handler.Handle(activity, actor, token); + } + finally + { + Arrival.Current = default; + } _logger.LogInformation("Processed {Type} {Id} from {Actor}", type, Id(activity), actor.ActorURI); return JobOutcome.Done; } diff --git a/PrivaPub/Federation/Inbox/InboxReceiver.cs b/PrivaPub/Federation/Inbox/InboxReceiver.cs index 3c2a49b..aad8992 100644 --- a/PrivaPub/Federation/Inbox/InboxReceiver.cs +++ b/PrivaPub/Federation/Inbox/InboxReceiver.cs @@ -15,7 +15,8 @@ namespace PrivaPub.Federation.Inbox { public sealed record InboxResult(int StatusCode, string Error = default); - public sealed record InboxPayload(string ActorURI, string Activity); + public sealed record InboxPayload(string ActorURI, string Activity, string Inbox = default, string KeyId = default, string Algorithm = default, + string[] SignedHeaders = default, DateTime? ReceivedAt = default); public interface IInboxReceiver { @@ -99,7 +100,9 @@ namespace PrivaPub.Federation.Inbox return shapeProblem; var activityId = Id(activity); - await _queue.Enqueue(JobKind.ProcessInbox, JsonSerializer.Serialize(new InboxPayload(actorUri, activity.ToJsonString())), + var payload = new InboxPayload(actorUri, activity.ToJsonString(), recipient == default ? "shared" : "personal", parameters.KeyId, + parameters.Algorithm, parameters.Headers, DateTime.UtcNow); + await _queue.Enqueue(JobKind.ProcessInbox, JsonSerializer.Serialize(payload), new Uri(actorUri).Host.ToLowerInvariant(), activityId == default ? default : "inbox|" + activityId, token); _logger.LogInformation("Inbox {Recipient}: {Type} from {Actor} queued", recipient?.Handle ?? "shared", type, actorUri); return new(StatusCodes.Status202Accepted); diff --git a/PrivaPub/Federation/Inbox/RemotePosts.cs b/PrivaPub/Federation/Inbox/RemotePosts.cs index ce07f7d..cf9cb84 100644 --- a/PrivaPub/Federation/Inbox/RemotePosts.cs +++ b/PrivaPub/Federation/Inbox/RemotePosts.cs @@ -6,6 +6,7 @@ using PrivaPub.Federation.Moderation; using PrivaPub.Federation.Objects; using PrivaPub.Infrastructure.Ids; using PrivaPub.Infrastructure.Jobs; +using PrivaPub.Models.Federation; using PrivaPub.Models.Jobs; using PrivaPub.Models.Post; using PrivaPub.Models.User; @@ -37,10 +38,12 @@ namespace PrivaPub.Federation.Inbox readonly IRemoteActorService _remoteActors; readonly IDomainBlocks _domainBlocks; readonly IJobQueue _queue; + readonly IObjectRecords _records; public RemotePosts(DbEntities dbEntities, ILocalActorService localActors, IRemoteActorService remoteActors, IDomainBlocks domainBlocks, - IJobQueue queue) + IJobQueue queue, IObjectRecords records) { + _records = records; _dbEntities = dbEntities; _localActors = localActors; _remoteActors = remoteActors; @@ -131,6 +134,7 @@ namespace PrivaPub.Federation.Inbox { return await _dbEntities.Posts.Match(p => p.ObjectURI == objectUri).ExecuteFirstAsync(token); } + await _records.Record(note, post, ObjectPath.Fetched, refetched: false, token); if (grandparent == default && !string.IsNullOrEmpty(note.InReplyTo) && depth < MaxDepth) await _queue.Enqueue(JobKind.FetchAncestors, JsonSerializer.Serialize(new AncestorsPayload(post.ID, depth + 1)), new Uri(note.InReplyTo).Host, diff --git a/PrivaPub/Federation/Objects/InstanceDescriber.cs b/PrivaPub/Federation/Objects/InstanceDescriber.cs new file mode 100644 index 0000000..9a70bf8 --- /dev/null +++ b/PrivaPub/Federation/Objects/InstanceDescriber.cs @@ -0,0 +1,84 @@ +using MongoDB.Entities; + +using PrivaPub.Infrastructure.Http; +using PrivaPub.Infrastructure.Jobs; +using PrivaPub.Models.Jobs; + +using System.Text.Json; + +namespace PrivaPub.Federation.Objects +{ + public class InstanceDescriber : IJobHandler + { + const int MaxNodeInfoChars = 64 * 1024; + static readonly string[] Schemas = + { + "http://nodeinfo.diaspora.software/ns/schema/2.1", + "http://nodeinfo.diaspora.software/ns/schema/2.0" + }; + + readonly IFederationHttp _http; + + public InstanceDescriber(IFederationHttp http) => _http = http; + + public JobKind Kind => JobKind.DescribeInstance; + public int Concurrency => 1; + public int MaxAttempts => 2; + public int PerHostLimit => 1; + + public async Task Handle(Job job, CancellationToken token) + { + var host = job.Payload; + using var links = await _http.GetJson($"https://{host}/.well-known/nodeinfo", "application/json", sign: default, token); + var href = links == default ? default : Schemas.Select(schema => LinkFor(links.Root, schema)).FirstOrDefault(link => link != default); + if (href == default) + return await Described(host, default, "no NodeInfo", token); + + using var nodeInfo = await _http.GetJson(href, "application/json", sign: default, token); + return nodeInfo == default ? await Described(host, default, "NodeInfo unreadable", token) : await Described(host, nodeInfo.Root, default, token); + } + + static async Task Described(string host, JsonElement? nodeInfo, string error, CancellationToken token) + { + var update = DB.Default.Update() + .Match(i => i.Host == host) + .Modify(i => i.DescribedAt, DateTime.UtcNow) + .Modify(i => i.DescriptionError, error) + .Option(o => o.IsUpsert = true); + if (nodeInfo is { } root) + { + var raw = root.GetRawText(); + update = update + .Modify(i => i.Software, Text(root, "software", "name")) + .Modify(i => i.SoftwareVersion, Text(root, "software", "version")) + .Modify(i => i.NodeName, Text(root, "metadata", "nodeName")) + .Modify(i => i.Protocols, root.TryGetProperty("protocols", out var protocols) && protocols.ValueKind == JsonValueKind.Array + ? protocols.EnumerateArray().Where(p => p.ValueKind == JsonValueKind.String).Select(p => p.GetString()).Take(16).ToList() + : new List()) + .Modify(i => i.OpenRegistrations, root.TryGetProperty("openRegistrations", out var open) && open.ValueKind is JsonValueKind.True or JsonValueKind.False + ? open.GetBoolean() + : default(bool?)) + .Modify(i => i.NodeInfo, raw.Length <= MaxNodeInfoChars ? raw : default); + } + await update.ExecuteAsync(token); + return JobOutcome.Done; + } + + static string LinkFor(JsonElement root, string schema) => + root.TryGetProperty("links", out var links) && links.ValueKind == JsonValueKind.Array + ? links.EnumerateArray() + .Where(l => l.ValueKind == JsonValueKind.Object && l.TryGetProperty("rel", out var rel) && rel.ValueKind == JsonValueKind.String && rel.GetString() == schema) + .Select(l => l.TryGetProperty("href", out var href) && href.ValueKind == JsonValueKind.String ? href.GetString() : default) + .FirstOrDefault(h => h != default && h.StartsWith("https://", StringComparison.OrdinalIgnoreCase)) + : default; + + static string Text(JsonElement root, string parent, string name) + { + if (!root.TryGetProperty(parent, out var section) || section.ValueKind != JsonValueKind.Object + || !section.TryGetProperty(name, out var value) || value.ValueKind != JsonValueKind.String) + return default; + var text = value.GetString(); + return text.Length <= 200 ? text : text[..200]; + } + } +} diff --git a/PrivaPub/Federation/Objects/NoteParser.cs b/PrivaPub/Federation/Objects/NoteParser.cs index af83672..a3c83bb 100644 --- a/PrivaPub/Federation/Objects/NoteParser.cs +++ b/PrivaPub/Federation/Objects/NoteParser.cs @@ -11,6 +11,7 @@ namespace PrivaPub.Federation.Objects { public string Id { get; init; } public string Type { get; init; } + public JsonObject Raw { get; init; } public string AttributedTo { get; init; } public string ContentHtml { get; init; } public string Title { get; init; } @@ -55,6 +56,7 @@ namespace PrivaPub.Federation.Objects { Id = id, Type = type, + Raw = note, AttributedTo = Attribution(note["attributedTo"]), ContentHtml = ContentSanitizer.Html(contentHtml), Title = Plain(Value(note, "name"), 500), diff --git a/PrivaPub/Federation/Objects/ObjectFeatures.cs b/PrivaPub/Federation/Objects/ObjectFeatures.cs new file mode 100644 index 0000000..15c5f29 --- /dev/null +++ b/PrivaPub/Federation/Objects/ObjectFeatures.cs @@ -0,0 +1,68 @@ +using System.Text.Json.Nodes; + +using static PrivaPub.Federation.Objects.ActivityJson; + +namespace PrivaPub.Federation.Objects +{ + public static class ObjectFeatures + { + public static List Detect(JsonObject raw) + { + var found = new List(); + if (raw == default) + return found; + void When(bool condition, string feature) + { + if (condition) + found.Add(feature); + } + + When(raw.ContainsKey("quote"), "fep-044f-quote"); + When(raw.ContainsKey("quoteAuthorization"), "fep-044f-authorization"); + When(raw.ContainsKey("_misskey_quote") || raw.ContainsKey("quoteUrl") || raw.ContainsKey("quoteUri"), "legacy-quote"); + When(raw.ContainsKey("interactionPolicy"), "interaction-policy"); + When(raw.ContainsKey("context"), "context"); + When(raw.ContainsKey("proof"), "fep-8b32-proof"); + When(raw.Any(p => p.Key.StartsWith("_misskey_", StringComparison.Ordinal)), "misskey"); + When(raw["source"] is JsonObject, "source:" + (Value(raw["source"], "mediaType") ?? "unknown")); + When(raw["source"] is JsonValue, "source:url"); + When(raw.ContainsKey("htmlMfm"), "fep-c16b-mfm"); + When(raw.ContainsKey("formerRepresentations"), "former-representations"); + When(raw.ContainsKey("searchableBy"), "fep-268d-searchable-by"); + When(raw.ContainsKey("audience"), "fep-1b12-audience"); + When(Items(raw["attachment"]).Any(a => Value(a, "type") == "Link"), "fep-8967-link"); + When(Items(raw["tag"]).Any(t => Value(t, "type") == "Emoji"), "custom-emoji"); + When(Value(raw, "type") == "Question", "poll"); + When(raw["contentMap"] is JsonObject, "content-map"); + When(raw["url"] is JsonArray, "url-variants"); + var contentType = Value(raw, "mediaType"); + When(contentType is not (null or "text/html"), "content:" + contentType); + return found; + } + + public static List Namespaces(JsonObject raw, string activityContext = default) => + Namespaces(raw?["@context"] ?? (activityContext == default ? default : JsonNode.Parse(activityContext))); + + static List Namespaces(JsonNode context) => context switch + { + JsonValue single when single.TryGetValue(out var url) => new List { url }, + JsonArray array => array.SelectMany(entry => entry switch + { + JsonValue value when value.TryGetValue(out var url) => new[] { url }, + JsonObject terms => terms.Where(t => t.Value is JsonValue v && v.TryGetValue(out var target) && target.EndsWith('#')) + .Select(t => t.Key + "=" + t.Value!.GetValue()), + _ => Enumerable.Empty() + }) + .Take(32) + .ToList(), + _ => new List() + }; + + static IEnumerable Items(JsonNode node) => node switch + { + JsonArray array => array.OfType(), + JsonObject single => new[] { single }, + _ => Enumerable.Empty() + }; + } +} diff --git a/PrivaPub/Federation/Objects/ObjectRecords.cs b/PrivaPub/Federation/Objects/ObjectRecords.cs new file mode 100644 index 0000000..e517e18 --- /dev/null +++ b/PrivaPub/Federation/Objects/ObjectRecords.cs @@ -0,0 +1,112 @@ +using MongoDB.Driver; +using MongoDB.Entities; + +using PrivaPub.Federation.Inbox; +using PrivaPub.Infrastructure.Jobs; +using PrivaPub.Models.Federation; +using PrivaPub.Models.Jobs; + +using System.Globalization; +using System.Security.Cryptography; +using System.Text; +using System.Text.Json.Nodes; + +using PostEntity = PrivaPub.Models.Post.Post; + +namespace PrivaPub.Federation.Objects +{ + public interface IObjectRecords + { + Task Record(NoteDocument note, PostEntity post, ObjectPath path, bool refetched, CancellationToken token); + Task Revise(NoteDocument note, string activityId, CancellationToken token); + } + + public class ObjectRecords : IObjectRecords + { + public const int MaxRawBytes = 256 * 1024; + const int MaxRevisions = 10; + static readonly TimeSpan DescriptionAge = TimeSpan.FromDays(7); + + readonly IJobQueue _queue; + + public ObjectRecords(IJobQueue queue) => _queue = queue; + + public async Task Record(NoteDocument note, PostEntity post, ObjectPath path, bool refetched, CancellationToken token) + { + var arrival = Arrival.Current; + var raw = Capture(note.Raw); + var host = new Uri(note.Id).Host.ToLowerInvariant(); + var record = new ObjectRecord + { + ObjectURI = note.Id, + PostId = post.ID, + ObjectType = note.Type, + Host = host, + Raw = raw.Text, + RawHash = raw.Hash, + RawBytes = raw.Bytes, + RawTruncated = raw.Truncated, + ActivityContext = note.Raw?.ContainsKey("@context") == true || arrival?.Context?.Length > 16 * 1024 ? default : arrival?.Context, + Path = path, + Refetched = refetched, + ActivityId = arrival?.ActivityId, + ActivityType = arrival?.ActivityType, + ActivityActorURI = arrival?.ActorURI, + Inbox = path == ObjectPath.Delivered ? arrival?.Inbox : default, + SignatureScheme = arrival?.KeyId == default ? default : "draft-cavage", + KeyId = arrival?.KeyId, + Algorithm = arrival?.Algorithm, + SignedHeaders = arrival?.SignedHeaders?.ToList() ?? new(), + ReceivedAt = arrival?.ReceivedAt ?? DateTime.UtcNow, + Published = note.Published, + Updated = note.Updated + }; + try + { + await DB.Default.SaveAsync(record, token); + } + catch (MongoWriteException ex) when (ex.WriteError?.Category == ServerErrorCategory.DuplicateKey) + { + return; + } + await Describe(host, token); + } + + public async Task Revise(NoteDocument note, string activityId, CancellationToken token) + { + var raw = Capture(note.Raw); + var revision = new ObjectRevision + { + Raw = raw.Text, + RawHash = raw.Hash, + RawTruncated = raw.Truncated, + ActivityId = activityId, + Updated = note.Updated + }; + await DB.Default.Update() + .Match(r => r.ObjectURI == note.Id) + .Modify(b => b.PushEach(r => r.Revisions, new[] { revision }, -MaxRevisions)) + .Modify(r => r.Updated, note.Updated) + .ExecuteAsync(token); + } + + async Task Describe(string host, CancellationToken token) + { + var known = await DB.Default.Find().Match(i => i.Host == host).ExecuteFirstAsync(token); + if (known?.DescribedAt > DateTime.UtcNow - DescriptionAge) + return; + var week = ISOWeek.GetYear(DateTime.UtcNow) + "-" + ISOWeek.GetWeekOfYear(DateTime.UtcNow); + await _queue.Enqueue(JobKind.DescribeInstance, host, host, $"describe|{host}|{week}", token); + } + + public static (string Text, string Hash, int Bytes, bool Truncated) Capture(JsonNode raw) + { + if (raw == default) + return default; + var text = raw.ToJsonString(); + var bytes = Encoding.UTF8.GetBytes(text); + var hash = Convert.ToHexStringLower(SHA256.HashData(bytes)); + return bytes.Length > MaxRawBytes ? (default, hash, bytes.Length, true) : (text, hash, bytes.Length, false); + } + } +} diff --git a/PrivaPub/Infrastructure/Data/Indexes.cs b/PrivaPub/Infrastructure/Data/Indexes.cs index ac29e5c..af2a7cd 100644 --- a/PrivaPub/Infrastructure/Data/Indexes.cs +++ b/PrivaPub/Infrastructure/Data/Indexes.cs @@ -102,6 +102,8 @@ namespace PrivaPub.Infrastructure.Data await Plain(token, r => r.IsResolved, r => r.ID); await Unique(b => b.Domain, Builders.Filter.Type(b => b.Domain, BsonType.String), token); await Unique(i => i.Host, Builders.Filter.Type(i => i.Host, BsonType.String), token); + await Unique(r => r.ObjectURI, Builders.Filter.Type(r => r.ObjectURI, BsonType.String), token); + await Plain(token, r => r.PostId); } static async Task Unique(System.Linq.Expressions.Expression> key, FilterDefinition partial, diff --git a/PrivaPub/Middleware/SocialPubConfigurations.cs b/PrivaPub/Middleware/SocialPubConfigurations.cs index 5843c77..02ee58a 100644 --- a/PrivaPub/Middleware/SocialPubConfigurations.cs +++ b/PrivaPub/Middleware/SocialPubConfigurations.cs @@ -17,6 +17,7 @@ using PrivaPub.Federation.Actors; using PrivaPub.Federation.Outbox; using PrivaPub.Federation.Signing; using PrivaPub.Federation.Inbox; +using PrivaPub.Federation.Objects; using PrivaPub.Federation.Moderation; using PrivaPub.Federation.Inbox.Handlers; using PrivaPub.Domain.Content; @@ -82,6 +83,8 @@ namespace PrivaPub.Middleware .AddSingleton() .AddSingleton() .AddSingleton() + .AddSingleton() + .AddSingleton() .AddSingleton() .AddSingleton() .AddSingleton() diff --git a/PrivaPub/Models/Federation/ObjectRecord.cs b/PrivaPub/Models/Federation/ObjectRecord.cs new file mode 100644 index 0000000..cf470af --- /dev/null +++ b/PrivaPub/Models/Federation/ObjectRecord.cs @@ -0,0 +1,50 @@ +using MongoDB.Entities; + +namespace PrivaPub.Models.Federation +{ + public class ObjectRecord : Entity + { + public string ObjectURI { get; set; } + public string PostId { get; set; } + public string ObjectType { get; set; } + public string Host { get; set; } + + public string Raw { get; set; }//the object exactly as it reached us, unless it was larger than MaxRawBytes + public string RawHash { get; set; }//sha-256 of the raw bytes, hex + public int RawBytes { get; set; } + public bool RawTruncated { get; set; } + public string ActivityContext { get; set; }//the delivering activity's @context, which an embedded object does not repeat + + public ObjectPath Path { get; set; } + public bool Refetched { get; set; }//delivered, but read again from its origin because it came embedded from another one + public string ActivityId { get; set; }//the activity that brought it, or that caused the fetch + public string ActivityType { get; set; } + public string ActivityActorURI { get; set; } + public string Inbox { get; set; }//"shared" or "personal" when delivered + public string SignatureScheme { get; set; } + public string KeyId { get; set; } + public string Algorithm { get; set; } + public List SignedHeaders { get; set; } = new(); + + public DateTime ReceivedAt { get; set; } = DateTime.UtcNow; + public DateTime? Published { get; set; } + public DateTime? Updated { get; set; } + public List Revisions { get; set; } = new(); + } + + public class ObjectRevision + { + public string Raw { get; set; } + public string RawHash { get; set; } + public bool RawTruncated { get; set; } + public string ActivityId { get; set; } + public DateTime? Updated { get; set; } + public DateTime ReceivedAt { get; set; } = DateTime.UtcNow; + } + + public enum ObjectPath + { + Delivered, + Fetched + } +} diff --git a/PrivaPub/Models/Jobs/Job.cs b/PrivaPub/Models/Jobs/Job.cs index 5457803..7ff0a6b 100644 --- a/PrivaPub/Models/Jobs/Job.cs +++ b/PrivaPub/Models/Jobs/Job.cs @@ -22,7 +22,8 @@ namespace PrivaPub.Models.Jobs { Deliver, ProcessInbox, - FetchAncestors + FetchAncestors, + DescribeInstance } public enum JobState diff --git a/PrivaPub/Models/Jobs/RemoteInstance.cs b/PrivaPub/Models/Jobs/RemoteInstance.cs index 3241987..8a4b5ee 100644 --- a/PrivaPub/Models/Jobs/RemoteInstance.cs +++ b/PrivaPub/Models/Jobs/RemoteInstance.cs @@ -10,5 +10,14 @@ namespace PrivaPub.Models.Jobs public DateTime? LastSuccessAt { get; set; } public DateTime? LastFailureAt { get; set; } public string LastError { get; set; } + + public string Software { get; set; }//NodeInfo software.name: for display, never for deciding behaviour + public string SoftwareVersion { get; set; } + public string NodeName { get; set; } + public List Protocols { get; set; } = new(); + public bool? OpenRegistrations { get; set; } + public string NodeInfo { get; set; }//the raw document, when small enough + public DateTime? DescribedAt { get; set; } + public string DescriptionError { get; set; } } } diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index cac04a9..666cbbb 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -426,9 +426,9 @@ it, raw where it doesn't. - A null `id`, a 200 `Tombstone`, a `Delete` before its `Create` (W9, W13). - **`Post.Kind`** plus typed payloads for article, video, audio, event, link, review and thread (INTEROP §4.1). The Mastodon API view of each kind: content, a card made from the object without fetching, attachments. -- **Raw capture for the details view:** the object as received, how it arrived, the signature scheme and key, received - versus `published`, the extensions detected, and the origin's software (§4.3). Exposed at - `/api/privapub/v1/statuses/:id/provenance` and `/api/privapub/v1/instances/:host`. +- **Raw capture for the details view** (done in v1.8.0): the object as received, how it arrived, the signature scheme and + key, received versus `published`, the extensions detected, and the origin's software (§4.3). Exposed at + `/api/privapub/v1/statuses/:id/provenance` and `/api/privapub/v1/instances/:host`. Checked live against GoToSocial. - **Routing by object type:** - `Accept`/`Reject`/`TentativeAccept` routed by what their object is (W6). - `ChatMessage` in as a direct message.