diff --git a/CLAUDE.md b/CLAUDE.md index d3bae61..8f7a4f3 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -238,6 +238,14 @@ cd /var/www/privapub.thepra.dev && sudo -u www-data ASPNETCORE_ENVIRONMENT=Produ - **Collection name = class name, so never rename an entity class.** - `Entity.ID` is a 24-character lowercase hex string; `GenerateNewID()` returns `object`, so cast it. - New fields must be additive: a deploy rollback restores the binary, not the database. +- **Every stored remote object has an `ObjectRecord`** (raw JSON up to 256 KB plus its hash; delivered or fetched; the + activity, inbox, key and signed headers; received time; up to 10 later revisions). Write it right after the post's + save: `CreateHandler` (delivered) and `RemotePosts.StoreContext` (fetched) do, and `UpdateHandler` appends a + revision. Delivery details reach them through `Arrival.Current`, which `InboxProcessor` sets for the handler's + duration. The raw form lives outside `Post` so timelines never load it. Read through + `/api/privapub/v1/statuses/:id/provenance` and `/api/privapub/v1/instances/:host`. +- **A server is described on arrival, never on read.** The first record from a host enqueues `DescribeInstance` (its + NodeInfo, at most once a week, into `RemoteInstance`), so opening the details view tells nobody anything. - **Post ids are the timeline order, so they follow arrival, not `published`.** `PrivacyIds.Arrived` gives a remote post published within the last hour (or in the future) a fresh `ObjectId`, which sorts after every post already stored, and only backfill keeps a `published`-derived id. With `published` ids a reply arriving in the same second could sort diff --git a/PrivaPub.Tests/Federation/InboxScenarioTests.cs b/PrivaPub.Tests/Federation/InboxScenarioTests.cs index 9b6733c..b7debb1 100644 --- a/PrivaPub.Tests/Federation/InboxScenarioTests.cs +++ b/PrivaPub.Tests/Federation/InboxScenarioTests.cs @@ -54,9 +54,9 @@ namespace PrivaPub.Tests.Federation { new FollowHandler(db, _local, remote, delivery), new UndoHandler(db, _local), - new CreateHandler(db, _local, remote, delivery, _blocks, new Fanout(db), new RemotePosts(db, _local, remote, _blocks, queue), new GroupDistributor(delivery)), + new CreateHandler(db, _local, remote, delivery, _blocks, new Fanout(db), new RemotePosts(db, _local, remote, _blocks, queue, new ObjectRecords(queue)), new GroupDistributor(delivery), new ObjectRecords(queue)), new DeleteHandler(db, _local, remote, delivery, new GroupDistributor(delivery)), - new UpdateHandler(db, _local, remote, new GroupDistributor(delivery)) + new UpdateHandler(db, _local, remote, new GroupDistributor(delivery), new ObjectRecords(queue)) }, NullLogger.Instance); } diff --git a/PrivaPub.Tests/Federation/ProvenanceTests.cs b/PrivaPub.Tests/Federation/ProvenanceTests.cs new file mode 100644 index 0000000..d695a42 --- /dev/null +++ b/PrivaPub.Tests/Federation/ProvenanceTests.cs @@ -0,0 +1,85 @@ +using MongoDB.Entities; + +using PrivaPub.Federation.Objects; +using PrivaPub.Models.Federation; +using PrivaPub.Models.Jobs; +using PrivaPub.Tests.Support; + +using System.Text.Json.Nodes; + +namespace PrivaPub.Tests.Federation +{ + [Trait("Category", "Integration")] + public sealed class ProvenanceTests : IAsyncLifetime + { + Harness _harness; + + public async ValueTask InitializeAsync() + { + Assert.SkipUnless(MongoFixture.Enabled, MongoFixture.Skip); + _harness = await Harness.Start(); + } + + public async ValueTask DisposeAsync() + { + if (_harness != default) + await _harness.DisposeAsync(); + } + + [Fact] + public async Task A_delivered_post_keeps_its_raw_object_how_it_came_and_every_edit() + { + var token = TestContext.Current.CancellationToken; + var (_, alice) = await _harness.Persona("alice"); + var mallory = new RemoteActor(_harness.Peer, "mallory"); + var origin = new Uri(mallory.Id).GetLeftPart(UriPartial.Authority); + var note = new JsonObject + { + ["id"] = $"{origin}/notes/{Guid.NewGuid():N}", + ["type"] = "Note", + ["attributedTo"] = mallory.Id, + ["content"] = "

hi

", + ["_misskey_quote"] = $"{origin}/notes/elsewhere", + ["to"] = new JsonArray(Addressing.Public), + ["cc"] = new JsonArray(alice.Uri), + ["tag"] = new JsonArray(new JsonObject { ["type"] = "Mention", ["href"] = alice.Uri, ["name"] = "@alice" }) + }; + var create = new JsonObject { ["id"] = $"{origin}/activities/{Guid.NewGuid():N}", ["type"] = "Create", ["actor"] = mallory.Id, ["object"] = note.DeepClone() }; + + await _harness.Deliver(mallory, "/human-centipede", create); + var edited = note.DeepClone().AsObject(); + edited["content"] = "

hi again

"; + edited["updated"] = DateTime.UtcNow.ToString("O"); + await _harness.Deliver(mallory, "/human-centipede", new JsonObject + { + ["id"] = $"{origin}/activities/{Guid.NewGuid():N}", ["type"] = "Update", ["actor"] = mallory.Id, ["object"] = edited + }); + + var record = await DB.Default.Find().Match(r => r.ObjectURI == note["id"]!.GetValue()).ExecuteFirstAsync(token); + Assert.NotNull(record); + Assert.Equal(ObjectPath.Delivered, record.Path); + Assert.False(record.Refetched); + Assert.Equal("shared", record.Inbox); + Assert.Equal(mallory.KeyId, record.KeyId); + Assert.Equal(create["id"]!.GetValue(), record.ActivityId); + Assert.Contains("digest", record.SignedHeaders); + Assert.Equal("

hi

", JsonNode.Parse(record.Raw)!["content"]!.GetValue()); + Assert.Contains("legacy-quote", ObjectFeatures.Detect(JsonNode.Parse(record.Raw)!.AsObject())); + Assert.Equal("

hi again

", JsonNode.Parse(Assert.Single(record.Revisions).Raw)!["content"]!.GetValue()); + Assert.True(await DB.Default.Find().Match(j => j.Kind == JobKind.DescribeInstance && j.Payload == record.Host).ExecuteAnyAsync(token)); + } + + [Fact] + public void An_object_larger_than_the_cap_keeps_only_its_hash() + { + var big = new JsonObject { ["content"] = new string('x', ObjectRecords.MaxRawBytes + 1) }; + + var (text, hash, bytes, truncated) = ObjectRecords.Capture(big); + + Assert.Null(text); + Assert.True(truncated); + Assert.True(bytes > ObjectRecords.MaxRawBytes); + Assert.Equal(64, hash.Length); + } + } +} diff --git a/PrivaPub.Tests/Support/Harness.cs b/PrivaPub.Tests/Support/Harness.cs index 202f51a..d8a5196 100644 --- a/PrivaPub.Tests/Support/Harness.cs +++ b/PrivaPub.Tests/Support/Harness.cs @@ -16,6 +16,7 @@ using PrivaPub.Federation.Inbox.Handlers; using PrivaPub.Federation.Outbox; using PrivaPub.Infrastructure.Jobs; using PrivaPub.Models; +using PrivaPub.Federation.Objects; using PrivaPub.Models.Federation; using PrivaPub.Models.Jobs; using PrivaPub.Models.User; @@ -42,7 +43,8 @@ namespace PrivaPub.Tests.Support Delivery = new DeliveryService(Db, Queue); Fanout = new Fanout(Db); Groups = new GroupDistributor(Delivery); - RemotePosts = new RemotePosts(Db, Local, Remote, new NoBlocks(), Queue); + Records = new ObjectRecords(Queue); + RemotePosts = new RemotePosts(Db, Local, Remote, new NoBlocks(), Queue, Records); Receiver = new InboxReceiver(Local, Remote, Queue, new NoBlocks(), NullLogger.Instance); Processor = new InboxProcessor(Remote, new IActivityHandler[] { @@ -52,9 +54,9 @@ namespace PrivaPub.Tests.Support new UndoHandler(Db, Local), new LikeHandler(Db), new AnnounceHandler(Db, Local, RemotePosts, Fanout, Remote), - new CreateHandler(Db, Local, Remote, Delivery, new NoBlocks(), Fanout, RemotePosts, Groups), + new CreateHandler(Db, Local, Remote, Delivery, new NoBlocks(), Fanout, RemotePosts, Groups, Records), new DeleteHandler(Db, Local, Remote, Delivery, Groups), - new UpdateHandler(Db, Local, Remote, Groups), + new UpdateHandler(Db, Local, Remote, Groups, Records), new FlagHandler(Db, Local) }, NullLogger.Instance); Follows = new FollowService(Db, Local, Remote, Delivery, new KeyLocalizer(), NullLogger.Instance); @@ -86,6 +88,7 @@ namespace PrivaPub.Tests.Support public GroupDistributor Groups { get; } public Fanout Fanout { get; } public RemotePosts RemotePosts { get; } + public ObjectRecords Records { get; } public TimelineService Timelines { get; } public RelationshipService Relationships { get; } public ReportService Reports { get; } diff --git a/PrivaPub/Api/Mastodon/Controllers/ProvenanceController.cs b/PrivaPub/Api/Mastodon/Controllers/ProvenanceController.cs new file mode 100644 index 0000000..ba3f470 --- /dev/null +++ b/PrivaPub/Api/Mastodon/Controllers/ProvenanceController.cs @@ -0,0 +1,181 @@ +using Microsoft.AspNetCore.Authorization; +using Microsoft.AspNetCore.Mvc; + +using MongoDB.Entities; + +using PrivaPub.Api.Mastodon.Infrastructure; +using PrivaPub.Domain.Privacy; +using PrivaPub.Federation.Objects; +using PrivaPub.Models.Federation; +using PrivaPub.Models.Jobs; +using PrivaPub.Models.Post; +using PrivaPub.StaticServices; + +using System.Text.Json.Nodes; + +using PostEntity = PrivaPub.Models.Post.Post; + +namespace PrivaPub.Api.Mastodon.Controllers +{ + public class ProvenanceController : MastodonController + { + readonly DbEntities _dbEntities; + + public ProvenanceController(DbEntities dbEntities) => _dbEntities = dbEntities; + + [HttpGet("/api/privapub/v1/statuses/{id}/provenance"), Scope("read:statuses", requiresUser: false), AllowAnonymous] + public async Task Status(string id, CancellationToken token) + { + var post = await _dbEntities.Posts.Match(p => p.ID == id && !p.DeletedAt.HasValue).ExecuteFirstAsync(token); + if (post?.ReblogOfPostId != default) + post = await _dbEntities.Posts.Match(p => p.ID == post.ReblogOfPostId && !p.DeletedAt.HasValue).ExecuteFirstAsync(token); + if (post == default || post.Visibility == PostVisibility.LocalGeo || !await VisibilityPolicy.CanSee(post, MyId, token)) + return NotFoundError(); + if (!post.IsFederatedCopy) + return Json(new Provenance { ObjectUri = post.ObjectURI, Url = post.Url, Path = "local" }); + + var record = await DB.Default.Find().Match(r => r.PostId == post.ID).ExecuteFirstAsync(token) + ?? await DB.Default.Find().Match(r => r.ObjectURI == post.ObjectURI).ExecuteFirstAsync(token); + var host = record?.Host ?? (Uri.TryCreate(post.ObjectURI, UriKind.Absolute, out var uri) ? uri.Host.ToLowerInvariant() : default); + var instance = host == default ? default : await DB.Default.Find().Match(i => i.Host == host).ExecuteFirstAsync(token); + return Json(Describe(post, record, instance)); + } + + [HttpGet("/api/privapub/v1/instances/{host}"), Scope("read", requiresUser: false), AllowAnonymous] + public async Task Instance(string host, CancellationToken token) + { + host = host?.Trim().ToLowerInvariant(); + var instance = await DB.Default.Find().Match(i => i.Host == host).ExecuteFirstAsync(token); + return instance == default ? NotFoundError() : Json(Describe(instance)); + } + + static Provenance Describe(PostEntity post, ObjectRecord record, RemoteInstance instance) + { + var raw = record?.Raw == default ? default : JsonNode.Parse(record.Raw) as JsonObject; + return new Provenance + { + ObjectUri = post.ObjectURI, + ObjectType = record?.ObjectType ?? post.ObjectType, + Url = post.Url, + Path = record == default ? "unrecorded" : record.Path == ObjectPath.Delivered ? "delivered" : "fetched", + Refetched = record?.Refetched == true, + Activity = record?.ActivityId == default ? default : new ProvenanceActivity + { + Id = record.ActivityId, Type = record.ActivityType, Actor = record.ActivityActorURI + }, + Inbox = record?.Inbox, + Signature = record?.KeyId == default ? default : new ProvenanceSignature + { + Scheme = record.SignatureScheme, KeyId = record.KeyId, Algorithm = record.Algorithm, Headers = record.SignedHeaders + }, + ReceivedAt = record == default ? default : MastodonJson.Time(record.ReceivedAt), + Published = record?.Published is { } published ? MastodonJson.Time(published) : default, + Updated = record?.Updated is { } updated ? MastodonJson.Time(updated) : default, + Extensions = ObjectFeatures.Detect(raw), + ContextNamespaces = ObjectFeatures.Namespaces(raw, record?.ActivityContext), + Raw = raw, + RawBytes = record?.RawBytes ?? 0, + RawHash = record?.RawHash, + RawTruncated = record?.RawTruncated == true, + Revisions = record?.Revisions.Select(r => new ProvenanceRevision + { + ActivityId = r.ActivityId, + Updated = r.Updated is { } at ? MastodonJson.Time(at) : default, + ReceivedAt = MastodonJson.Time(r.ReceivedAt), + RawHash = r.RawHash, + Raw = r.Raw == default ? default : JsonNode.Parse(r.Raw) + }).ToList() ?? new(), + Instance = instance == default ? default : Describe(instance) + }; + } + + static InstanceDescription Describe(RemoteInstance instance) => new() + { + Host = instance.Host, + Software = instance.Software, + Version = instance.SoftwareVersion, + NodeName = instance.NodeName, + Protocols = instance.Protocols ?? new(), + OpenRegistrations = instance.OpenRegistrations, + DescribedAt = instance.DescribedAt is { } described ? MastodonJson.Time(described) : default, + DescriptionError = instance.DescriptionError, + NodeInfo = instance.NodeInfo == default ? default : JsonNode.Parse(instance.NodeInfo), + Delivery = new InstanceDelivery + { + ConsecutiveFailures = instance.ConsecutiveFailures, + UnavailableUntil = instance.UnavailableUntil is { } until ? MastodonJson.Time(until) : default, + LastSuccessAt = instance.LastSuccessAt is { } success ? MastodonJson.Time(success) : default, + LastFailureAt = instance.LastFailureAt is { } failure ? MastodonJson.Time(failure) : default + } + }; + + public class Provenance + { + public string ObjectUri { get; set; } + public string ObjectType { get; set; } + public string Url { get; set; } + public string Path { get; set; } + public bool Refetched { get; set; } + public ProvenanceActivity Activity { get; set; } + public string Inbox { get; set; } + public ProvenanceSignature Signature { get; set; } + public string ReceivedAt { get; set; } + public string Published { get; set; } + public string Updated { get; set; } + public List Extensions { get; set; } = new(); + public List ContextNamespaces { get; set; } = new(); + public JsonNode Raw { get; set; } + public int RawBytes { get; set; } + public string RawHash { get; set; } + public bool RawTruncated { get; set; } + public List Revisions { get; set; } = new(); + public InstanceDescription Instance { get; set; } + } + + public class ProvenanceActivity + { + public string Id { get; set; } + public string Type { get; set; } + public string Actor { get; set; } + } + + public class ProvenanceSignature + { + public string Scheme { get; set; } + public string KeyId { get; set; } + public string Algorithm { get; set; } + public List Headers { get; set; } = new(); + } + + public class ProvenanceRevision + { + public string ActivityId { get; set; } + public string Updated { get; set; } + public string ReceivedAt { get; set; } + public string RawHash { get; set; } + public JsonNode Raw { get; set; } + } + + public class InstanceDescription + { + public string Host { get; set; } + public string Software { get; set; } + public string Version { get; set; } + public string NodeName { get; set; } + public List Protocols { get; set; } = new(); + public bool? OpenRegistrations { get; set; } + public string DescribedAt { get; set; } + public string DescriptionError { get; set; } + public JsonNode NodeInfo { get; set; } + public InstanceDelivery Delivery { get; set; } + } + + public class InstanceDelivery + { + public int ConsecutiveFailures { get; set; } + public string UnavailableUntil { get; set; } + public string LastSuccessAt { get; set; } + public string LastFailureAt { get; set; } + } + } +} diff --git a/PrivaPub/Federation/Inbox/Arrival.cs b/PrivaPub/Federation/Inbox/Arrival.cs new file mode 100644 index 0000000..a18d815 --- /dev/null +++ b/PrivaPub/Federation/Inbox/Arrival.cs @@ -0,0 +1,14 @@ +namespace PrivaPub.Federation.Inbox +{ + public sealed record Arrival(string ActivityId, string ActivityType, string ActorURI, string Inbox, string KeyId, string Algorithm, + IReadOnlyList SignedHeaders, DateTime ReceivedAt, string Context = default) + { + static readonly AsyncLocal current = new(); + + public static Arrival Current + { + get => current.Value; + set => current.Value = value; + } + } +} diff --git a/PrivaPub/Federation/Inbox/Handlers/CreateHandler.cs b/PrivaPub/Federation/Inbox/Handlers/CreateHandler.cs index f2bb30b..3dd9209 100644 --- a/PrivaPub/Federation/Inbox/Handlers/CreateHandler.cs +++ b/PrivaPub/Federation/Inbox/Handlers/CreateHandler.cs @@ -34,10 +34,12 @@ namespace PrivaPub.Federation.Inbox.Handlers readonly IFanout _fanout; readonly IRemotePosts _remotePosts; readonly IGroupDistributor _groups; + readonly IObjectRecords _records; public CreateHandler(DbEntities dbEntities, ILocalActorService localActors, IRemoteActorService remoteActors, IDeliveryService delivery, - IDomainBlocks domainBlocks, IFanout fanout, IRemotePosts remotePosts, IGroupDistributor groups) + IDomainBlocks domainBlocks, IFanout fanout, IRemotePosts remotePosts, IGroupDistributor groups, IObjectRecords records) { + _records = records; _groups = groups; _fanout = fanout; _remotePosts = remotePosts; @@ -53,7 +55,8 @@ namespace PrivaPub.Federation.Inbox.Handlers public async Task Handle(JsonNode activity, ForeignAvatar author, CancellationToken token) { var node = activity["object"]; - if (node is not JsonObject || !Origin.Same(Id(node), author.ActorURI)) + var refetched = node is not JsonObject || !Origin.Same(Id(node), author.ActorURI); + if (refetched) { using var fetched = await _remoteActors.FetchObject(Id(node), token); node = fetched == default ? default : JsonNode.Parse(fetched.Root.GetRawText()); @@ -131,6 +134,7 @@ namespace PrivaPub.Federation.Inbox.Handlers { return; } + await _records.Record(note, post, ObjectPath.Delivered, refetched, token); if (parent != default) await DB.Default.Update().MatchID(parent.ID).Modify(b => b.Inc(p => p.RepliesCount, 1)).ExecuteAsync(token); diff --git a/PrivaPub/Federation/Inbox/Handlers/UpdateHandler.cs b/PrivaPub/Federation/Inbox/Handlers/UpdateHandler.cs index f4d8937..d82cfb6 100644 --- a/PrivaPub/Federation/Inbox/Handlers/UpdateHandler.cs +++ b/PrivaPub/Federation/Inbox/Handlers/UpdateHandler.cs @@ -23,9 +23,12 @@ namespace PrivaPub.Federation.Inbox.Handlers readonly ILocalActorService _localActors; readonly IRemoteActorService _remoteActors; readonly IGroupDistributor _groups; + readonly IObjectRecords _records; - public UpdateHandler(DbEntities dbEntities, ILocalActorService localActors, IRemoteActorService remoteActors, IGroupDistributor groups) + public UpdateHandler(DbEntities dbEntities, ILocalActorService localActors, IRemoteActorService remoteActors, IGroupDistributor groups, + IObjectRecords records) { + _records = records; _groups = groups; _dbEntities = dbEntities; _localActors = localActors; @@ -83,6 +86,7 @@ namespace PrivaPub.Federation.Inbox.Handlers post.EditedAt = note.Updated ?? DateTime.UtcNow; post.UpdateDate = DateTime.UtcNow; await DB.Default.SaveAsync(post, token); + await _records.Revise(note, Id(activity), token); if (!string.IsNullOrEmpty(post.GroupId) && await _localActors.FindById(Models.Federation.LocalActorKind.Group, post.GroupId, token) is { IsCircle: false } community) await _groups.Announce(community, activity.AsObject(), post.ObjectURI, isNewPost: false, token); } diff --git a/PrivaPub/Federation/Inbox/InboxProcessor.cs b/PrivaPub/Federation/Inbox/InboxProcessor.cs index 1738950..267c046 100644 --- a/PrivaPub/Federation/Inbox/InboxProcessor.cs +++ b/PrivaPub/Federation/Inbox/InboxProcessor.cs @@ -46,7 +46,16 @@ namespace PrivaPub.Federation.Inbox if (actor == default) return JobOutcome.Retry("the actor could not be loaded"); - await handler.Handle(activity, actor, token); + Arrival.Current = new Arrival(Id(activity), type, actor.ActorURI, payload.Inbox, payload.KeyId, payload.Algorithm, + payload.SignedHeaders ?? Array.Empty(), payload.ReceivedAt ?? job.CreatedAt, activity["@context"]?.ToJsonString()); + try + { + await handler.Handle(activity, actor, token); + } + finally + { + Arrival.Current = default; + } _logger.LogInformation("Processed {Type} {Id} from {Actor}", type, Id(activity), actor.ActorURI); return JobOutcome.Done; } diff --git a/PrivaPub/Federation/Inbox/InboxReceiver.cs b/PrivaPub/Federation/Inbox/InboxReceiver.cs index 3c2a49b..aad8992 100644 --- a/PrivaPub/Federation/Inbox/InboxReceiver.cs +++ b/PrivaPub/Federation/Inbox/InboxReceiver.cs @@ -15,7 +15,8 @@ namespace PrivaPub.Federation.Inbox { public sealed record InboxResult(int StatusCode, string Error = default); - public sealed record InboxPayload(string ActorURI, string Activity); + public sealed record InboxPayload(string ActorURI, string Activity, string Inbox = default, string KeyId = default, string Algorithm = default, + string[] SignedHeaders = default, DateTime? ReceivedAt = default); public interface IInboxReceiver { @@ -99,7 +100,9 @@ namespace PrivaPub.Federation.Inbox return shapeProblem; var activityId = Id(activity); - await _queue.Enqueue(JobKind.ProcessInbox, JsonSerializer.Serialize(new InboxPayload(actorUri, activity.ToJsonString())), + var payload = new InboxPayload(actorUri, activity.ToJsonString(), recipient == default ? "shared" : "personal", parameters.KeyId, + parameters.Algorithm, parameters.Headers, DateTime.UtcNow); + await _queue.Enqueue(JobKind.ProcessInbox, JsonSerializer.Serialize(payload), new Uri(actorUri).Host.ToLowerInvariant(), activityId == default ? default : "inbox|" + activityId, token); _logger.LogInformation("Inbox {Recipient}: {Type} from {Actor} queued", recipient?.Handle ?? "shared", type, actorUri); return new(StatusCodes.Status202Accepted); diff --git a/PrivaPub/Federation/Inbox/RemotePosts.cs b/PrivaPub/Federation/Inbox/RemotePosts.cs index ce07f7d..cf9cb84 100644 --- a/PrivaPub/Federation/Inbox/RemotePosts.cs +++ b/PrivaPub/Federation/Inbox/RemotePosts.cs @@ -6,6 +6,7 @@ using PrivaPub.Federation.Moderation; using PrivaPub.Federation.Objects; using PrivaPub.Infrastructure.Ids; using PrivaPub.Infrastructure.Jobs; +using PrivaPub.Models.Federation; using PrivaPub.Models.Jobs; using PrivaPub.Models.Post; using PrivaPub.Models.User; @@ -37,10 +38,12 @@ namespace PrivaPub.Federation.Inbox readonly IRemoteActorService _remoteActors; readonly IDomainBlocks _domainBlocks; readonly IJobQueue _queue; + readonly IObjectRecords _records; public RemotePosts(DbEntities dbEntities, ILocalActorService localActors, IRemoteActorService remoteActors, IDomainBlocks domainBlocks, - IJobQueue queue) + IJobQueue queue, IObjectRecords records) { + _records = records; _dbEntities = dbEntities; _localActors = localActors; _remoteActors = remoteActors; @@ -131,6 +134,7 @@ namespace PrivaPub.Federation.Inbox { return await _dbEntities.Posts.Match(p => p.ObjectURI == objectUri).ExecuteFirstAsync(token); } + await _records.Record(note, post, ObjectPath.Fetched, refetched: false, token); if (grandparent == default && !string.IsNullOrEmpty(note.InReplyTo) && depth < MaxDepth) await _queue.Enqueue(JobKind.FetchAncestors, JsonSerializer.Serialize(new AncestorsPayload(post.ID, depth + 1)), new Uri(note.InReplyTo).Host, diff --git a/PrivaPub/Federation/Objects/InstanceDescriber.cs b/PrivaPub/Federation/Objects/InstanceDescriber.cs new file mode 100644 index 0000000..9a70bf8 --- /dev/null +++ b/PrivaPub/Federation/Objects/InstanceDescriber.cs @@ -0,0 +1,84 @@ +using MongoDB.Entities; + +using PrivaPub.Infrastructure.Http; +using PrivaPub.Infrastructure.Jobs; +using PrivaPub.Models.Jobs; + +using System.Text.Json; + +namespace PrivaPub.Federation.Objects +{ + public class InstanceDescriber : IJobHandler + { + const int MaxNodeInfoChars = 64 * 1024; + static readonly string[] Schemas = + { + "http://nodeinfo.diaspora.software/ns/schema/2.1", + "http://nodeinfo.diaspora.software/ns/schema/2.0" + }; + + readonly IFederationHttp _http; + + public InstanceDescriber(IFederationHttp http) => _http = http; + + public JobKind Kind => JobKind.DescribeInstance; + public int Concurrency => 1; + public int MaxAttempts => 2; + public int PerHostLimit => 1; + + public async Task Handle(Job job, CancellationToken token) + { + var host = job.Payload; + using var links = await _http.GetJson($"https://{host}/.well-known/nodeinfo", "application/json", sign: default, token); + var href = links == default ? default : Schemas.Select(schema => LinkFor(links.Root, schema)).FirstOrDefault(link => link != default); + if (href == default) + return await Described(host, default, "no NodeInfo", token); + + using var nodeInfo = await _http.GetJson(href, "application/json", sign: default, token); + return nodeInfo == default ? await Described(host, default, "NodeInfo unreadable", token) : await Described(host, nodeInfo.Root, default, token); + } + + static async Task Described(string host, JsonElement? nodeInfo, string error, CancellationToken token) + { + var update = DB.Default.Update() + .Match(i => i.Host == host) + .Modify(i => i.DescribedAt, DateTime.UtcNow) + .Modify(i => i.DescriptionError, error) + .Option(o => o.IsUpsert = true); + if (nodeInfo is { } root) + { + var raw = root.GetRawText(); + update = update + .Modify(i => i.Software, Text(root, "software", "name")) + .Modify(i => i.SoftwareVersion, Text(root, "software", "version")) + .Modify(i => i.NodeName, Text(root, "metadata", "nodeName")) + .Modify(i => i.Protocols, root.TryGetProperty("protocols", out var protocols) && protocols.ValueKind == JsonValueKind.Array + ? protocols.EnumerateArray().Where(p => p.ValueKind == JsonValueKind.String).Select(p => p.GetString()).Take(16).ToList() + : new List()) + .Modify(i => i.OpenRegistrations, root.TryGetProperty("openRegistrations", out var open) && open.ValueKind is JsonValueKind.True or JsonValueKind.False + ? open.GetBoolean() + : default(bool?)) + .Modify(i => i.NodeInfo, raw.Length <= MaxNodeInfoChars ? raw : default); + } + await update.ExecuteAsync(token); + return JobOutcome.Done; + } + + static string LinkFor(JsonElement root, string schema) => + root.TryGetProperty("links", out var links) && links.ValueKind == JsonValueKind.Array + ? links.EnumerateArray() + .Where(l => l.ValueKind == JsonValueKind.Object && l.TryGetProperty("rel", out var rel) && rel.ValueKind == JsonValueKind.String && rel.GetString() == schema) + .Select(l => l.TryGetProperty("href", out var href) && href.ValueKind == JsonValueKind.String ? href.GetString() : default) + .FirstOrDefault(h => h != default && h.StartsWith("https://", StringComparison.OrdinalIgnoreCase)) + : default; + + static string Text(JsonElement root, string parent, string name) + { + if (!root.TryGetProperty(parent, out var section) || section.ValueKind != JsonValueKind.Object + || !section.TryGetProperty(name, out var value) || value.ValueKind != JsonValueKind.String) + return default; + var text = value.GetString(); + return text.Length <= 200 ? text : text[..200]; + } + } +} diff --git a/PrivaPub/Federation/Objects/NoteParser.cs b/PrivaPub/Federation/Objects/NoteParser.cs index af83672..a3c83bb 100644 --- a/PrivaPub/Federation/Objects/NoteParser.cs +++ b/PrivaPub/Federation/Objects/NoteParser.cs @@ -11,6 +11,7 @@ namespace PrivaPub.Federation.Objects { public string Id { get; init; } public string Type { get; init; } + public JsonObject Raw { get; init; } public string AttributedTo { get; init; } public string ContentHtml { get; init; } public string Title { get; init; } @@ -55,6 +56,7 @@ namespace PrivaPub.Federation.Objects { Id = id, Type = type, + Raw = note, AttributedTo = Attribution(note["attributedTo"]), ContentHtml = ContentSanitizer.Html(contentHtml), Title = Plain(Value(note, "name"), 500), diff --git a/PrivaPub/Federation/Objects/ObjectFeatures.cs b/PrivaPub/Federation/Objects/ObjectFeatures.cs new file mode 100644 index 0000000..15c5f29 --- /dev/null +++ b/PrivaPub/Federation/Objects/ObjectFeatures.cs @@ -0,0 +1,68 @@ +using System.Text.Json.Nodes; + +using static PrivaPub.Federation.Objects.ActivityJson; + +namespace PrivaPub.Federation.Objects +{ + public static class ObjectFeatures + { + public static List Detect(JsonObject raw) + { + var found = new List(); + if (raw == default) + return found; + void When(bool condition, string feature) + { + if (condition) + found.Add(feature); + } + + When(raw.ContainsKey("quote"), "fep-044f-quote"); + When(raw.ContainsKey("quoteAuthorization"), "fep-044f-authorization"); + When(raw.ContainsKey("_misskey_quote") || raw.ContainsKey("quoteUrl") || raw.ContainsKey("quoteUri"), "legacy-quote"); + When(raw.ContainsKey("interactionPolicy"), "interaction-policy"); + When(raw.ContainsKey("context"), "context"); + When(raw.ContainsKey("proof"), "fep-8b32-proof"); + When(raw.Any(p => p.Key.StartsWith("_misskey_", StringComparison.Ordinal)), "misskey"); + When(raw["source"] is JsonObject, "source:" + (Value(raw["source"], "mediaType") ?? "unknown")); + When(raw["source"] is JsonValue, "source:url"); + When(raw.ContainsKey("htmlMfm"), "fep-c16b-mfm"); + When(raw.ContainsKey("formerRepresentations"), "former-representations"); + When(raw.ContainsKey("searchableBy"), "fep-268d-searchable-by"); + When(raw.ContainsKey("audience"), "fep-1b12-audience"); + When(Items(raw["attachment"]).Any(a => Value(a, "type") == "Link"), "fep-8967-link"); + When(Items(raw["tag"]).Any(t => Value(t, "type") == "Emoji"), "custom-emoji"); + When(Value(raw, "type") == "Question", "poll"); + When(raw["contentMap"] is JsonObject, "content-map"); + When(raw["url"] is JsonArray, "url-variants"); + var contentType = Value(raw, "mediaType"); + When(contentType is not (null or "text/html"), "content:" + contentType); + return found; + } + + public static List Namespaces(JsonObject raw, string activityContext = default) => + Namespaces(raw?["@context"] ?? (activityContext == default ? default : JsonNode.Parse(activityContext))); + + static List Namespaces(JsonNode context) => context switch + { + JsonValue single when single.TryGetValue(out var url) => new List { url }, + JsonArray array => array.SelectMany(entry => entry switch + { + JsonValue value when value.TryGetValue(out var url) => new[] { url }, + JsonObject terms => terms.Where(t => t.Value is JsonValue v && v.TryGetValue(out var target) && target.EndsWith('#')) + .Select(t => t.Key + "=" + t.Value!.GetValue()), + _ => Enumerable.Empty() + }) + .Take(32) + .ToList(), + _ => new List() + }; + + static IEnumerable Items(JsonNode node) => node switch + { + JsonArray array => array.OfType(), + JsonObject single => new[] { single }, + _ => Enumerable.Empty() + }; + } +} diff --git a/PrivaPub/Federation/Objects/ObjectRecords.cs b/PrivaPub/Federation/Objects/ObjectRecords.cs new file mode 100644 index 0000000..e517e18 --- /dev/null +++ b/PrivaPub/Federation/Objects/ObjectRecords.cs @@ -0,0 +1,112 @@ +using MongoDB.Driver; +using MongoDB.Entities; + +using PrivaPub.Federation.Inbox; +using PrivaPub.Infrastructure.Jobs; +using PrivaPub.Models.Federation; +using PrivaPub.Models.Jobs; + +using System.Globalization; +using System.Security.Cryptography; +using System.Text; +using System.Text.Json.Nodes; + +using PostEntity = PrivaPub.Models.Post.Post; + +namespace PrivaPub.Federation.Objects +{ + public interface IObjectRecords + { + Task Record(NoteDocument note, PostEntity post, ObjectPath path, bool refetched, CancellationToken token); + Task Revise(NoteDocument note, string activityId, CancellationToken token); + } + + public class ObjectRecords : IObjectRecords + { + public const int MaxRawBytes = 256 * 1024; + const int MaxRevisions = 10; + static readonly TimeSpan DescriptionAge = TimeSpan.FromDays(7); + + readonly IJobQueue _queue; + + public ObjectRecords(IJobQueue queue) => _queue = queue; + + public async Task Record(NoteDocument note, PostEntity post, ObjectPath path, bool refetched, CancellationToken token) + { + var arrival = Arrival.Current; + var raw = Capture(note.Raw); + var host = new Uri(note.Id).Host.ToLowerInvariant(); + var record = new ObjectRecord + { + ObjectURI = note.Id, + PostId = post.ID, + ObjectType = note.Type, + Host = host, + Raw = raw.Text, + RawHash = raw.Hash, + RawBytes = raw.Bytes, + RawTruncated = raw.Truncated, + ActivityContext = note.Raw?.ContainsKey("@context") == true || arrival?.Context?.Length > 16 * 1024 ? default : arrival?.Context, + Path = path, + Refetched = refetched, + ActivityId = arrival?.ActivityId, + ActivityType = arrival?.ActivityType, + ActivityActorURI = arrival?.ActorURI, + Inbox = path == ObjectPath.Delivered ? arrival?.Inbox : default, + SignatureScheme = arrival?.KeyId == default ? default : "draft-cavage", + KeyId = arrival?.KeyId, + Algorithm = arrival?.Algorithm, + SignedHeaders = arrival?.SignedHeaders?.ToList() ?? new(), + ReceivedAt = arrival?.ReceivedAt ?? DateTime.UtcNow, + Published = note.Published, + Updated = note.Updated + }; + try + { + await DB.Default.SaveAsync(record, token); + } + catch (MongoWriteException ex) when (ex.WriteError?.Category == ServerErrorCategory.DuplicateKey) + { + return; + } + await Describe(host, token); + } + + public async Task Revise(NoteDocument note, string activityId, CancellationToken token) + { + var raw = Capture(note.Raw); + var revision = new ObjectRevision + { + Raw = raw.Text, + RawHash = raw.Hash, + RawTruncated = raw.Truncated, + ActivityId = activityId, + Updated = note.Updated + }; + await DB.Default.Update() + .Match(r => r.ObjectURI == note.Id) + .Modify(b => b.PushEach(r => r.Revisions, new[] { revision }, -MaxRevisions)) + .Modify(r => r.Updated, note.Updated) + .ExecuteAsync(token); + } + + async Task Describe(string host, CancellationToken token) + { + var known = await DB.Default.Find().Match(i => i.Host == host).ExecuteFirstAsync(token); + if (known?.DescribedAt > DateTime.UtcNow - DescriptionAge) + return; + var week = ISOWeek.GetYear(DateTime.UtcNow) + "-" + ISOWeek.GetWeekOfYear(DateTime.UtcNow); + await _queue.Enqueue(JobKind.DescribeInstance, host, host, $"describe|{host}|{week}", token); + } + + public static (string Text, string Hash, int Bytes, bool Truncated) Capture(JsonNode raw) + { + if (raw == default) + return default; + var text = raw.ToJsonString(); + var bytes = Encoding.UTF8.GetBytes(text); + var hash = Convert.ToHexStringLower(SHA256.HashData(bytes)); + return bytes.Length > MaxRawBytes ? (default, hash, bytes.Length, true) : (text, hash, bytes.Length, false); + } + } +} diff --git a/PrivaPub/Infrastructure/Data/Indexes.cs b/PrivaPub/Infrastructure/Data/Indexes.cs index ac29e5c..af2a7cd 100644 --- a/PrivaPub/Infrastructure/Data/Indexes.cs +++ b/PrivaPub/Infrastructure/Data/Indexes.cs @@ -102,6 +102,8 @@ namespace PrivaPub.Infrastructure.Data await Plain(token, r => r.IsResolved, r => r.ID); await Unique(b => b.Domain, Builders.Filter.Type(b => b.Domain, BsonType.String), token); await Unique(i => i.Host, Builders.Filter.Type(i => i.Host, BsonType.String), token); + await Unique(r => r.ObjectURI, Builders.Filter.Type(r => r.ObjectURI, BsonType.String), token); + await Plain(token, r => r.PostId); } static async Task Unique(System.Linq.Expressions.Expression> key, FilterDefinition partial, diff --git a/PrivaPub/Middleware/SocialPubConfigurations.cs b/PrivaPub/Middleware/SocialPubConfigurations.cs index 5843c77..02ee58a 100644 --- a/PrivaPub/Middleware/SocialPubConfigurations.cs +++ b/PrivaPub/Middleware/SocialPubConfigurations.cs @@ -17,6 +17,7 @@ using PrivaPub.Federation.Actors; using PrivaPub.Federation.Outbox; using PrivaPub.Federation.Signing; using PrivaPub.Federation.Inbox; +using PrivaPub.Federation.Objects; using PrivaPub.Federation.Moderation; using PrivaPub.Federation.Inbox.Handlers; using PrivaPub.Domain.Content; @@ -82,6 +83,8 @@ namespace PrivaPub.Middleware .AddSingleton() .AddSingleton() .AddSingleton() + .AddSingleton() + .AddSingleton() .AddSingleton() .AddSingleton() .AddSingleton() diff --git a/PrivaPub/Models/Federation/ObjectRecord.cs b/PrivaPub/Models/Federation/ObjectRecord.cs new file mode 100644 index 0000000..cf470af --- /dev/null +++ b/PrivaPub/Models/Federation/ObjectRecord.cs @@ -0,0 +1,50 @@ +using MongoDB.Entities; + +namespace PrivaPub.Models.Federation +{ + public class ObjectRecord : Entity + { + public string ObjectURI { get; set; } + public string PostId { get; set; } + public string ObjectType { get; set; } + public string Host { get; set; } + + public string Raw { get; set; }//the object exactly as it reached us, unless it was larger than MaxRawBytes + public string RawHash { get; set; }//sha-256 of the raw bytes, hex + public int RawBytes { get; set; } + public bool RawTruncated { get; set; } + public string ActivityContext { get; set; }//the delivering activity's @context, which an embedded object does not repeat + + public ObjectPath Path { get; set; } + public bool Refetched { get; set; }//delivered, but read again from its origin because it came embedded from another one + public string ActivityId { get; set; }//the activity that brought it, or that caused the fetch + public string ActivityType { get; set; } + public string ActivityActorURI { get; set; } + public string Inbox { get; set; }//"shared" or "personal" when delivered + public string SignatureScheme { get; set; } + public string KeyId { get; set; } + public string Algorithm { get; set; } + public List SignedHeaders { get; set; } = new(); + + public DateTime ReceivedAt { get; set; } = DateTime.UtcNow; + public DateTime? Published { get; set; } + public DateTime? Updated { get; set; } + public List Revisions { get; set; } = new(); + } + + public class ObjectRevision + { + public string Raw { get; set; } + public string RawHash { get; set; } + public bool RawTruncated { get; set; } + public string ActivityId { get; set; } + public DateTime? Updated { get; set; } + public DateTime ReceivedAt { get; set; } = DateTime.UtcNow; + } + + public enum ObjectPath + { + Delivered, + Fetched + } +} diff --git a/PrivaPub/Models/Jobs/Job.cs b/PrivaPub/Models/Jobs/Job.cs index 5457803..7ff0a6b 100644 --- a/PrivaPub/Models/Jobs/Job.cs +++ b/PrivaPub/Models/Jobs/Job.cs @@ -22,7 +22,8 @@ namespace PrivaPub.Models.Jobs { Deliver, ProcessInbox, - FetchAncestors + FetchAncestors, + DescribeInstance } public enum JobState diff --git a/PrivaPub/Models/Jobs/RemoteInstance.cs b/PrivaPub/Models/Jobs/RemoteInstance.cs index 3241987..8a4b5ee 100644 --- a/PrivaPub/Models/Jobs/RemoteInstance.cs +++ b/PrivaPub/Models/Jobs/RemoteInstance.cs @@ -10,5 +10,14 @@ namespace PrivaPub.Models.Jobs public DateTime? LastSuccessAt { get; set; } public DateTime? LastFailureAt { get; set; } public string LastError { get; set; } + + public string Software { get; set; }//NodeInfo software.name: for display, never for deciding behaviour + public string SoftwareVersion { get; set; } + public string NodeName { get; set; } + public List Protocols { get; set; } = new(); + public bool? OpenRegistrations { get; set; } + public string NodeInfo { get; set; }//the raw document, when small enough + public DateTime? DescribedAt { get; set; } + public string DescriptionError { get; set; } } } diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index cac04a9..666cbbb 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -426,9 +426,9 @@ it, raw where it doesn't. - A null `id`, a 200 `Tombstone`, a `Delete` before its `Create` (W9, W13). - **`Post.Kind`** plus typed payloads for article, video, audio, event, link, review and thread (INTEROP §4.1). The Mastodon API view of each kind: content, a card made from the object without fetching, attachments. -- **Raw capture for the details view:** the object as received, how it arrived, the signature scheme and key, received - versus `published`, the extensions detected, and the origin's software (§4.3). Exposed at - `/api/privapub/v1/statuses/:id/provenance` and `/api/privapub/v1/instances/:host`. +- **Raw capture for the details view** (done in v1.8.0): the object as received, how it arrived, the signature scheme and + key, received versus `published`, the extensions detected, and the origin's software (§4.3). Exposed at + `/api/privapub/v1/statuses/:id/provenance` and `/api/privapub/v1/instances/:host`. Checked live against GoToSocial. - **Routing by object type:** - `Accept`/`Reject`/`TentativeAccept` routed by what their object is (W6). - `ChatMessage` in as a direct message.