diff --git a/CLAUDE.md b/CLAUDE.md index 0f90b19..56ca81f 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -230,7 +230,9 @@ group www-data and reaches the private mongod; `sudo -u www-data` works too. error that leaves without a body (a challenge, a 404 from routing, a 429) gets Mastodon's `{"error": ...}` from `UseMastodonErrorBodies`. `NeverFiveHundredTests` walks every route with junk ids, anonymously, as a persona and with a junk token. -6. Advertise `4.2.0 (compatible; PrivaPub)` until grouped notifications exist. +6. Advertise `4.2.0 (compatible; PrivaPub)` until streaming and Web Push exist too; `api_versions.mastodon = 7` stays, + because clients show quotes from it, so everything a 4.3+ client calls because of it must answer: grouped + notifications (`/api/v2/notifications`, its unread count, groups, accounts and dismiss) and the notification policy. 7. **What a Mastodon `Status` cannot say goes in `Status.privapub`** (`PrivaPubStatus`): object type, title, excerpt, cover, the author's source, link, video, audio and event details, and up/down votes. Every media URL in it goes through the proxy; only page links (`link.url`, an event's online link) point at the remote site, because following diff --git a/FEDERATION.md b/FEDERATION.md index 895e928..4cb600f 100644 --- a/FEDERATION.md +++ b/FEDERATION.md @@ -204,9 +204,12 @@ Posts with a location (shown to nearby users of this server) never leave the ser - **HTML.** Received HTML is sanitised to Mastodon's allowlist. - **Keys we cannot fetch for now.** When a sender's key cannot be fetched because its server timed out or answered 5xx, the inbox answers 503 with `Retry-After: 300` rather than 401. -- **Activity ids.** `Create` and `Announce` ids dereference. `Follow`, `Like`, `Block` and the `Accept`, `Reject` and - `Undo` that answer them do not, because serving them would reveal who follows, likes and blocks whom; they are always - sent with their object embedded. +- **Activity ids.** `Create` and `Announce` ids dereference, a community's announces included, until the post they + carry is deleted (then 410); a persona's boost id sends a browser to the boosted post. `Follow`, `Like`, `Block` and + the `Accept`, `Reject` and `Undo` that answer them do not, because serving them would reveal who follows, likes and + blocks whom. Neither do `Update`, `Delete`, `EmojiReact`, `QuoteRequest` and its answers, `Flag`, `Ignore` or poll + votes. All of them are always sent with their object embedded. +- **Hashtags.** A post's `Hashtag` links go to `/tags/{tag}`, a public page of this server's public posts with that tag. - **Delivery.** Failed deliveries are retried with Mastodon's backoff (16 attempts). A host that keeps failing is paused, starting at an hour and growing to a week. diff --git a/PrivaPub.Tests/Http/OneAnswerTests.cs b/PrivaPub.Tests/Http/OneAnswerTests.cs index 6ff8c6b..564c9d1 100644 --- a/PrivaPub.Tests/Http/OneAnswerTests.cs +++ b/PrivaPub.Tests/Http/OneAnswerTests.cs @@ -151,6 +151,105 @@ namespace PrivaPub.Tests.Http Assert.Equal(HttpStatusCode.OK, (await _client.Get(path)).Status); } + [Fact] + public async Task Likes_of_one_post_come_as_one_group_of_notifications() + { + var author = await _host.Mastodon("grouped"); + var first = await _host.Mastodon("fanone"); + var second = await _host.Mastodon("fantwo"); + var post = await author.Status("like me twice"); + var id = post.Text("id"); + await first.Client.Post($"/api/v1/statuses/{id}/favourite"); + await second.Client.Post($"/api/v1/statuses/{id}/favourite"); + + var list = (await author.Client.Get("/api/v2/notifications")).Ok().Body; + var group = Assert.Single(list["notification_groups"]!.AsArray(), g => g!.Text("type") == "favourite")!; + var key = group.Text("group_key"); + + Assert.Equal($"grouped-favourite-{id}", key); + Assert.Equal(2, group.Number("notifications_count")); + Assert.Equal(2, group["sample_account_ids"]!.AsArray().Count); + Assert.Equal(id, group.Text("status_id")); + Assert.Contains(list["statuses"]!.AsArray(), s => s!.Text("id") == id); + Assert.True((await author.Client.Get("/api/v2/notifications/unread_count")).Ok().Body.Number("count") >= 1); + Assert.Equal(key, Assert.Single((await author.Client.Get($"/api/v2/notifications/{key}")).Ok().Body["notification_groups"]!.AsArray())!.Text("group_key")); + Assert.Equal(2, (await author.Client.Get($"/api/v2/notifications/{key}/accounts")).Ok().Array.Count); + Assert.Equal(HttpStatusCode.OK, (await author.Client.Post($"/api/v2/notifications/{key}/dismiss")).Status); + Assert.Equal(HttpStatusCode.NotFound, (await author.Client.Get($"/api/v2/notifications/{key}")).Status); + } + + [Fact] + public async Task A_hashtag_link_opens_a_page_of_public_posts_with_that_tag() + { + var author = await _host.Mastodon("tagger"); + var tag = $"tag{Guid.NewGuid():N}"[..16]; + await author.Status($"about #{tag}"); + await author.Status($"quietly about #{tag}", ("visibility", "private")); + + var page = await _client.Fetch($"/tags/{tag}", Browser); + + Assert.Equal(HttpStatusCode.OK, page.Status); + Assert.Contains("about", page.Text); + Assert.DoesNotContain("quietly", page.Text); + Assert.Contains("default-src 'none'", page.Response.Headers.GetValues("Content-Security-Policy").Single()); + } + + [Fact] + public async Task A_communitys_announces_resolve_while_the_post_is_shown() + { + var token = TestContext.Current.CancellationToken; + var owner = await _host.Persona(await _host.SignUp(), "announcer"); + var community = await _host.FederatedGroup(owner, community: true); + var post = await _host.Publish(owner, new PrivaPub.Domain.Statuses.StatusDraft { Text = "into the community", PlainText = true, GroupId = community.Id }); + var kept = await DB.Default.Find().Match(a => a.GroupId == community.Id && a.ObjectURI == post.ObjectURI).ExecuteAsync(token); + Assert.Equal(2, kept.Count);//the activity, and the object for Mastodon + + foreach (var announcement in kept) + { + var fetched = await _client.Fetch(PathOf(announcement.ActivityURI)); + Assert.Equal(HttpStatusCode.OK, fetched.Status); + Assert.Equal("Announce", fetched.Json["type"]!.GetValue()); + Assert.Equal(announcement.ActivityURI, fetched.Json["id"]!.GetValue()); + } + var listed = await _client.Fetch($"/peasants/{community.UserName}/grunts/announce-{post.ID}"); + Assert.Equal(HttpStatusCode.OK, listed.Status); + + await _host.Remove(owner, post); + Assert.Equal(HttpStatusCode.Gone, (await _client.Fetch(PathOf(kept[0].ActivityURI))).Status); + } + + [Fact] + public async Task A_boost_points_at_the_boosted_posts_page() + { + var author = await _host.Mastodon("boosted"); + var booster = await _host.Mastodon("booster"); + var post = await author.Status("boost me"); + + var boost = (await booster.Client.Post($"/api/v1/statuses/{post.Text("id")}/reblog")).Ok().Body; + + Assert.Equal(post.Text("url"), boost.Text("url")); + var browser = await _client.Fetch(PathOf(boost.Text("uri")), Browser); + Assert.Equal(HttpStatusCode.Redirect, browser.Status); + } + + [Fact] + public async Task A_remote_accounts_counts_are_what_its_server_publishes() + { + var token = TestContext.Current.CancellationToken; + var reader = await _host.Mastodon("counter"); + var popular = new RemoteActor(_peer, "popular"); + _peer.Serve(new Uri(popular.Id).AbsolutePath + "/followers", + $$"""{"@context":"https://www.w3.org/ns/activitystreams","id":"{{popular.Id}}/followers","type":"OrderedCollection","totalItems":42}"""); + var remote = _host.Get(); + var account = await remote.GetActor(popular.Id, refresh: true, token); + + await _host.Run(j => j.Kind == PrivaPub.Models.Jobs.JobKind.CountAccount && j.Payload == popular.Id, token); + + var shown = (await reader.Client.Get($"/api/v1/accounts/{account.ID}")).Ok().Body; + Assert.Equal(42, shown.Number("followers_count")); + Assert.Equal(0, shown.Number("following_count")); + } + [Fact] public async Task Notification_policy_and_requests_answer_a_signed_in_reader() { diff --git a/PrivaPub/Api/Mastodon/Controllers/TimelinesController.cs b/PrivaPub/Api/Mastodon/Controllers/TimelinesController.cs index 6168666..0d846e6 100644 --- a/PrivaPub/Api/Mastodon/Controllers/TimelinesController.cs +++ b/PrivaPub/Api/Mastodon/Controllers/TimelinesController.cs @@ -177,8 +177,7 @@ namespace PrivaPub.Api.Mastodon.Controllers _dbEntities = dbEntities; } - [HttpGet("/api/v1/notifications"), Scope("read:notifications")] - public async Task List(CancellationToken token) + Find Filtered() { var types = Params.List("types").Select(Parse).Where(t => t.HasValue).Select(t => t.Value).ToList(); var excluded = Params.List("exclude_types").Select(Parse).Where(t => t.HasValue).Select(t => t.Value).ToList(); @@ -189,11 +188,132 @@ namespace PrivaPub.Api.Mastodon.Controllers query.Match(n => !excluded.Contains(n.Type)); if (Params.Get("account_id") is { } accountId) query.Match(n => n.FromAccountId == accountId); - var notifications = await Page.From(Params, Limit(15, 30)).Fetch(query, n => n.ID, token); + return query; + } + + [HttpGet("/api/v1/notifications"), Scope("read:notifications")] + public async Task List(CancellationToken token) + { + var notifications = await Page.From(Params, Limit(15, 30)).Fetch(Filtered(), n => n.ID, token); Link("/api/v1/notifications", notifications.LastOrDefault()?.ID, notifications.FirstOrDefault()?.ID); return Json(await Map(notifications, token)); } + // Grouped notifications (Mastodon 4.3): api_versions.mastodon says we speak that API, so clients call it. Likes and + // boosts of one post group together, as do follows within an hour; everything else stands alone. + static readonly HashSet DefaultGrouped = new() { "favourite", "reblog", "follow" }; + + HashSet Grouped() => Params.List("grouped_types") is { Count: > 0 } chosen ? chosen.ToHashSet() : DefaultGrouped; + + static string GroupKey(NotificationEntity n, ISet grouped) => + !grouped.Contains(Names[n.Type]) ? $"ungrouped-{n.ID}" + : n.Type == NotificationType.Follow ? $"grouped-follow-{n.CreatedAt:yyyyMMddHH}" + : string.IsNullOrEmpty(n.PostId) ? $"ungrouped-{n.ID}" + : $"grouped-{Names[n.Type]}-{n.PostId}"; + + // the notifications a group key stands for + System.Linq.Expressions.Expression> InGroup(string key) + { + var me = MyId; + if (key.StartsWith("ungrouped-", StringComparison.Ordinal)) + { + var id = key["ungrouped-".Length..]; + return n => n.AvatarId == me && n.ID == id; + } + if (key.StartsWith("grouped-follow-", StringComparison.Ordinal) + && DateTime.TryParseExact(key["grouped-follow-".Length..], "yyyyMMddHH", System.Globalization.CultureInfo.InvariantCulture, + System.Globalization.DateTimeStyles.AdjustToUniversal | System.Globalization.DateTimeStyles.AssumeUniversal, out var hour)) + { + var end = hour.AddHours(1); + return n => n.AvatarId == me && n.Type == NotificationType.Follow && n.CreatedAt >= hour && n.CreatedAt < end; + } + if (key.Split('-') is ["grouped", var name, var postId] && Parse(name) is { } type) + return n => n.AvatarId == me && n.Type == type && n.PostId == postId; + return default; + } + + async Task> Members(string key, CancellationToken token) => + InGroup(key) is { } filter + ? await _dbEntities.Notifications.Match(filter).Sort(n => n.ID, Order.Descending).Limit(80).ExecuteAsync(token) + : new List(); + + async Task Groups(List page, ISet grouped, CancellationToken token) + { + var mapped = (await Map(page, token)).ToDictionary(n => n.Id); + var shown = page.Where(n => mapped.ContainsKey(n.ID)).ToList(); + var groups = new List(); + foreach (var group in shown.GroupBy(n => GroupKey(n, grouped))) + { + var newest = mapped[group.First().ID]; + var total = group.Key.StartsWith("ungrouped-", StringComparison.Ordinal) ? 1 : await DB.Default.CountAsync(InGroup(group.Key), token); + groups.Add(new + { + group_key = group.Key, + notifications_count = Math.Max(total, group.Count()), + type = newest.Type, + most_recent_notification_id = newest.Id, + page_min_id = group.Last().ID, + page_max_id = group.First().ID, + latest_page_notification_at = newest.CreatedAt, + sample_account_ids = group.Select(n => mapped[n.ID].Account.Id).Distinct().Take(8).ToList(), + status_id = newest.Status?.Id, + emoji = newest.Emoji, + emoji_url = newest.EmojiUrl + }); + } + return new + { + accounts = mapped.Values.Select(n => n.Account).DistinctBy(a => a.Id).ToList(), + statuses = mapped.Values.Where(n => n.Status != default).Select(n => n.Status).DistinctBy(s => s.Id).ToList(), + notification_groups = groups + }; + } + + [HttpGet("/api/v2/notifications"), Scope("read:notifications")] + public async Task GroupedList(CancellationToken token) + { + var notifications = await Page.From(Params, Limit(40, 80)).Fetch(Filtered(), n => n.ID, token); + Link("/api/v2/notifications", notifications.LastOrDefault()?.ID, notifications.FirstOrDefault()?.ID); + return Json(await Groups(notifications, Grouped(), token)); + } + + [HttpGet("/api/v2/notifications/unread_count"), Scope("read:notifications")] + public async Task GroupedUnreadCount(CancellationToken token) + { + var unread = await Filtered().Match(n => !n.IsRead).Sort(n => n.ID, Order.Descending) + .Limit(Math.Clamp(Params.Int("limit") ?? 100, 1, 1000)).ExecuteAsync(token); + var shown = (await Map(unread, token)).Select(n => n.Id).ToHashSet(); + var grouped = Grouped(); + return Json(new { count = unread.Where(n => shown.Contains(n.ID)).Select(n => GroupKey(n, grouped)).Distinct().Count() }); + } + + [HttpGet("/api/v2/notifications/{groupKey}"), Scope("read:notifications")] + public async Task Group(string groupKey, CancellationToken token) + { + var members = await Members(groupKey, token); + if (members.Count == 0) + return NotFoundError(); + return Json(await Groups(members, new HashSet(DefaultGrouped.Append(Names[members[0].Type])), token)); + } + + [HttpGet("/api/v2/notifications/{groupKey}/accounts"), Scope("read:notifications")] + public async Task GroupAccounts(string groupKey, CancellationToken token) + { + var members = await Members(groupKey, token); + if (members.Count == 0) + return NotFoundError(); + return Json((await Map(members, token)).Select(n => n.Account).DistinctBy(a => a.Id).ToList()); + } + + [HttpPost("/api/v2/notifications/{groupKey}/dismiss"), Scope("write:notifications")] + public async Task DismissGroup(string groupKey, CancellationToken token) + { + if (InGroup(groupKey) is not { } filter) + return NotFoundError(); + await DB.Default.DeleteAsync(filter); + return Json(new { }); + } + [HttpGet("/api/v1/notifications/{id}"), Scope("read:notifications")] public async Task Get(string id, CancellationToken token) { diff --git a/PrivaPub/Api/Mastodon/Mappers/MastodonMapper.cs b/PrivaPub/Api/Mastodon/Mappers/MastodonMapper.cs index 0502abe..8cef9ef 100644 --- a/PrivaPub/Api/Mastodon/Mappers/MastodonMapper.cs +++ b/PrivaPub/Api/Mastodon/Mappers/MastodonMapper.cs @@ -120,7 +120,10 @@ namespace PrivaPub.Api.Mastodon.Mappers Avatar = Proxied(foreign.PictureURL) ?? MissingAvatar, AvatarStatic = Proxied(foreign.PictureURL) ?? MissingAvatar, Header = Proxied(foreign.ThumbnailURL) ?? MissingHeader, - HeaderStatic = Proxied(foreign.ThumbnailURL) ?? MissingHeader + HeaderStatic = Proxied(foreign.ThumbnailURL) ?? MissingHeader, + FollowersCount = foreign.FollowersCount ?? 0,//as its server publishes them (AccountCountsJob); 0 while unknown + FollowingCount = foreign.FollowingCount ?? 0, + StatusesCount = foreign.StatusesCount ?? 0 }; public async Task Account(string id, CancellationToken token) => @@ -279,6 +282,7 @@ namespace PrivaPub.Api.Mastodon.Mappers continue; Quote(inner, original); status.Reblog = inner; + status.Url = inner.Url;//a boost has no page of its own; /grunts would answer JSON to a browser status.Content = string.Empty; status.Reblogged = reblogged.Contains(original.ID); } diff --git a/PrivaPub/Federation/Actors/AccountCountsJob.cs b/PrivaPub/Federation/Actors/AccountCountsJob.cs new file mode 100644 index 0000000..4575ef3 --- /dev/null +++ b/PrivaPub/Federation/Actors/AccountCountsJob.cs @@ -0,0 +1,60 @@ +using MongoDB.Entities; + +using PrivaPub.Federation.Objects; +using PrivaPub.Infrastructure.Http; +using PrivaPub.Infrastructure.Jobs; +using PrivaPub.Models.Jobs; +using PrivaPub.Models.User; +using PrivaPub.StaticServices; + +namespace PrivaPub.Federation.Actors +{ + // A remote account's follower, following and post counts, as its own server publishes them (the totalItems of its + // followers, following and outbox collections). Read when the account is fetched, at most once a day, never when + // someone looks at it; a hidden or missing collection leaves its count unknown. Mastodon and GoToSocial do the same. + public class AccountCountsJob : IJobHandler + { + readonly IRemoteActorService _remoteActors; + readonly DbEntities _dbEntities; + + public AccountCountsJob(IRemoteActorService remoteActors, DbEntities dbEntities) + { + _remoteActors = remoteActors; + _dbEntities = dbEntities; + } + + public JobKind Kind => JobKind.CountAccount; + public int Concurrency => 2; + public int MaxAttempts => 2; + public int PerHostLimit => 1; + + public static string DedupeKey(string actorUri, DateTime day) => $"count|{actorUri}|{day:yyyyMMdd}"; + + public async Task Handle(Job job, CancellationToken token) + { + var actor = await _dbEntities.ForeignAvatars.Match(a => a.ActorURI == job.Payload && !a.DeletionAt.HasValue).ExecuteFirstAsync(token); + if (actor == default) + return JobOutcome.Done; + await DB.Default.Update().MatchID(actor.ID) + .Modify(a => a.FollowersCount, await Total(actor, actor.FollowersURL, token)) + .Modify(a => a.FollowingCount, await Total(actor, actor.FollowingURL, token)) + .Modify(a => a.StatusesCount, await Total(actor, actor.OutboxURL, token)) + .Modify(a => a.CountedAt, DateTime.UtcNow) + .ExecuteAsync(token); + return JobOutcome.Done; + } + + // only from the actor's own server, as everything we believe about it + async Task Total(ForeignAvatar actor, string collection, CancellationToken token) + { + if (string.IsNullOrEmpty(collection) || !Origin.Same(collection, actor.ActorURI)) + return default; + using var scope = HttpScope.For("collection"); + using var fetched = await _remoteActors.FetchObject(collection, token); + return fetched != default && fetched.Root.ValueKind == System.Text.Json.JsonValueKind.Object + && fetched.Root.TryGetProperty("totalItems", out var total) && total.TryGetInt32(out var count) && count >= 0 + ? count + : default; + } + } +} diff --git a/PrivaPub/Federation/Actors/RemoteActorService.cs b/PrivaPub/Federation/Actors/RemoteActorService.cs index cb2c91f..065f835 100644 --- a/PrivaPub/Federation/Actors/RemoteActorService.cs +++ b/PrivaPub/Federation/Actors/RemoteActorService.cs @@ -36,9 +36,12 @@ namespace PrivaPub.Federation.Actors readonly DbEntities _dbEntities; readonly IOptionsMonitor _options; + readonly Infrastructure.Jobs.IJobQueue _jobs; + public RemoteActorService(IFederationHttp http, ILocalActorService localActors, IMemoryCache cache, DbEntities dbEntities, - IOptionsMonitor options = default) + IOptionsMonitor options = default, Infrastructure.Jobs.IJobQueue jobs = default) { + _jobs = jobs; _http = http; _localActors = localActors; _cache = cache; @@ -89,7 +92,16 @@ namespace PrivaPub.Federation.Actors return cached; var key = cached == default ? default : actor.Key(cached.PublicKeyId); - return await Upsert(actor, key ?? actor.Keys.FirstOrDefault(), token); + return await Counted(await Upsert(actor, key ?? actor.Keys.FirstOrDefault(), token), token); + } + + // its counts are read once a day, after it was fetched (AccountCountsJob) + async Task Counted(ForeignAvatar stored, CancellationToken token) + { + if (_jobs != default && stored is { DeletionAt: null } && Uri.TryCreate(stored.ActorURI, UriKind.Absolute, out var uri)) + await _jobs.Enqueue(Models.Jobs.JobKind.CountAccount, stored.ActorURI, uri.Host.ToLowerInvariant(), + AccountCountsJob.DedupeKey(stored.ActorURI, DateTime.UtcNow), token); + return stored; } public async Task GetActorByKeyId(string keyId, bool refresh, CancellationToken token) @@ -121,7 +133,7 @@ namespace PrivaPub.Federation.Actors var key = actor?.Key(keyId); if (key == default) return default; - return await Upsert(actor, key, token); + return await Counted(await Upsert(actor, key, token), token); } public bool KeyTemporarilyUnavailable(string keyId) => Origin.Of(keyId) != default && _http.FailedTemporarily(StripFragment(keyId)); diff --git a/PrivaPub/Federation/Controllers/PeasantsController.cs b/PrivaPub/Federation/Controllers/PeasantsController.cs index ddee4e3..cf482d5 100644 --- a/PrivaPub/Federation/Controllers/PeasantsController.cs +++ b/PrivaPub/Federation/Controllers/PeasantsController.cs @@ -101,7 +101,7 @@ namespace PrivaPub.Federation.Controllers { if (local.Kind == LocalActorKind.Group) { - items.Add(ActivityPubRenderer.Announce(local, post.ObjectURI, $"announce-{post.ID}")); + items.Add(ActivityPubRenderer.Announce(local, post.ObjectURI, $"announce-{post.ID}", post.CreationDate)); continue; } if (post.ReblogOfPostId != default) @@ -258,8 +258,12 @@ namespace PrivaPub.Federation.Controllers { var booster = await _localActors.FindByUserName(actor, token); var uri = booster?.ActivityUri(activityId); + if (booster is { Kind: LocalActorKind.Group, IsCircle: false }) + return await GroupAnnounce(booster, activityId, uri, token); var reblog = uri == default ? default : await _dbEntities.Posts.Match(p => p.ObjectURI == uri && !p.DeletedAt.HasValue).ExecuteFirstAsync(token); var rendered = reblog == default ? default : await AnnounceFor(reblog, token); + if (rendered != default && WantsHtml() && await _dbEntities.Posts.MatchID(reblog.ReblogOfPostId).ExecuteFirstAsync(token) is { } boosted) + return Redirect(boosted.Url ?? boosted.ObjectURI);//a boost has no page of its own: the boosted post's return rendered == default ? NotFound() : Activity(rendered); } if (!activityId.StartsWith("create-", StringComparison.Ordinal)) @@ -386,6 +390,24 @@ namespace PrivaPub.Federation.Controllers return new ContentResult { Content = tombstone.ToJsonString(), ContentType = ActivityContentType, StatusCode = StatusCodes.Status410Gone }; } + // A community's announce: one GroupDistributor sent (kept as sent), or one its outbox lists (announce-{postId}), + // while the post it is about is still shown; 410 once that post is gone. + async Task GroupAnnounce(LocalActor group, string activityId, string uri, CancellationToken token) + { + if (await DB.Default.Find().Match(a => a.ActivityURI == uri && a.GroupId == group.Id).ExecuteFirstAsync(token) is { } kept) + { + var about = await _dbEntities.Posts.Match(p => p.ObjectURI == kept.ObjectURI).ExecuteFirstAsync(token); + if (!VisibilityPolicy.Shown(about) || about.Visibility is not (PostVisibility.Public or PostVisibility.Unlisted)) + return StatusCode(StatusCodes.Status410Gone); + var body = JsonNode.Parse(kept.Body)!.AsObject(); + body["@context"] = ActivityPubRenderer.Context(); + return Activity(body); + } + var postId = activityId["announce-".Length..]; + var post = await _dbEntities.Posts.Match(p => p.ID == postId && p.GroupId == group.Id).Match(VisibilityPolicy.IsPublic).ExecuteFirstAsync(token); + return post == default ? NotFound() : Activity(ActivityPubRenderer.Announce(group, post.ObjectURI, activityId, post.CreationDate)); + } + async Task AnnounceFor(PostEntity reblog, CancellationToken token) { var original = await _dbEntities.Posts.MatchID(reblog.ReblogOfPostId).ExecuteFirstAsync(token); diff --git a/PrivaPub/Federation/Outbox/GroupDistributor.cs b/PrivaPub/Federation/Outbox/GroupDistributor.cs index caf9f76..67c1a00 100644 --- a/PrivaPub/Federation/Outbox/GroupDistributor.cs +++ b/PrivaPub/Federation/Outbox/GroupDistributor.cs @@ -1,5 +1,8 @@ +using MongoDB.Entities; + using PrivaPub.Federation.Actors; using PrivaPub.Federation.Rendering; +using PrivaPub.Models.Federation; using System.Security.Cryptography; using System.Text; @@ -14,6 +17,26 @@ namespace PrivaPub.Federation.Outbox public class GroupDistributor : IGroupDistributor { + // so the announce's id resolves (PeasantsController.Grunt); a repeat of the same announce keeps the first + static async Task Keep(LocalActor group, JsonObject announce, string objectUri, CancellationToken token) + { + var body = (JsonObject)announce.DeepClone(); + body.Remove("@context"); + try + { + await DB.Default.SaveAsync(new GroupAnnouncement + { + GroupId = group.Id, + ActivityURI = body["id"]!.GetValue(), + ObjectURI = objectUri, + Body = body.ToJsonString() + }, token); + } + catch (MongoDB.Driver.MongoWriteException ex) when (ex.WriteError?.Category == MongoDB.Driver.ServerErrorCategory.DuplicateKey) + { + } + } + readonly IDeliveryService _delivery; public GroupDistributor(IDeliveryService delivery) @@ -43,8 +66,13 @@ namespace PrivaPub.Federation.Outbox // Every follower, the author's own server included: Lemmy 1.0 keeps its user's post pending until the community // announces it back, and clears that before it answers the echo 400 ("Object is not remote"). await _delivery.EnqueueToFollowers(group, announce, token); + await Keep(group, announce, objectUri, token); if (isNewPost && !string.IsNullOrEmpty(objectUri)) - await _delivery.EnqueueToFollowers(group, ActivityPubRenderer.Announce(group, objectUri, $"announce-object-{key}"), token); + { + var boost = ActivityPubRenderer.Announce(group, objectUri, $"announce-object-{key}"); + await _delivery.EnqueueToFollowers(group, boost, token); + await Keep(group, boost, objectUri, token); + } } } } diff --git a/PrivaPub/Federation/Rendering/ActivityPubRenderer.cs b/PrivaPub/Federation/Rendering/ActivityPubRenderer.cs index b3d25ed..a799efb 100644 --- a/PrivaPub/Federation/Rendering/ActivityPubRenderer.cs +++ b/PrivaPub/Federation/Rendering/ActivityPubRenderer.cs @@ -357,13 +357,13 @@ namespace PrivaPub.Federation.Rendering ["object"] = note }; - public static JsonObject Announce(LocalActor group, string objectUri, string activityId) => new() + public static JsonObject Announce(LocalActor group, string objectUri, string activityId, DateTime? published = default) => new() { ["@context"] = ActivityStreams, ["id"] = group.ActivityUri(activityId), ["type"] = "Announce", ["actor"] = group.Uri, - ["published"] = Timestamp(DateTime.UtcNow), + ["published"] = Timestamp(published ?? DateTime.UtcNow), ["to"] = new JsonArray(Public), ["cc"] = new JsonArray(group.Followers), ["object"] = objectUri diff --git a/PrivaPub/Infrastructure/Data/Indexes.cs b/PrivaPub/Infrastructure/Data/Indexes.cs index 3af9b9e..eb0851c 100644 --- a/PrivaPub/Infrastructure/Data/Indexes.cs +++ b/PrivaPub/Infrastructure/Data/Indexes.cs @@ -54,6 +54,7 @@ namespace PrivaPub.Infrastructure.Data await Plain(token, g => g.InvitationCode); await Plain(token, g => g.ParticipantsKey); + await Unique(a => a.ActivityURI, Builders.Filter.Type(a => a.ActivityURI, BsonType.String), token); await Plain(token, d => d.DeliveredAt, d => d.AbandonedAt, d => d.NextAttemptAt); diff --git a/PrivaPub/Middleware/SocialPubConfigurations.cs b/PrivaPub/Middleware/SocialPubConfigurations.cs index e5442b1..a9b4b7d 100644 --- a/PrivaPub/Middleware/SocialPubConfigurations.cs +++ b/PrivaPub/Middleware/SocialPubConfigurations.cs @@ -100,6 +100,7 @@ namespace PrivaPub.Middleware .AddSingleton() .AddSingleton() .AddSingleton() + .AddSingleton() .AddSingleton() .AddSingleton() .AddSingleton(services => services.GetRequiredService()) diff --git a/PrivaPub/Models/Federation/GroupAnnouncement.cs b/PrivaPub/Models/Federation/GroupAnnouncement.cs new file mode 100644 index 0000000..d199b15 --- /dev/null +++ b/PrivaPub/Models/Federation/GroupAnnouncement.cs @@ -0,0 +1,15 @@ +using MongoDB.Entities; + +namespace PrivaPub.Models.Federation +{ + // An Announce a community sent (GroupDistributor), kept so its id resolves as both docs promise: announces are public, + // unlike follows, likes and blocks. Served only while the post it is about is still shown. + public class GroupAnnouncement : Entity + { + public string GroupId { get; set; } + public string ActivityURI { get; set; } + public string ObjectURI { get; set; }//the post it is about + public string Body { get; set; }//the Announce as sent, without @context + public DateTime PublishedAt { get; set; } = DateTime.UtcNow; + } +} diff --git a/PrivaPub/Models/Jobs/Job.cs b/PrivaPub/Models/Jobs/Job.cs index 1b1ff26..115790b 100644 --- a/PrivaPub/Models/Jobs/Job.cs +++ b/PrivaPub/Models/Jobs/Job.cs @@ -30,7 +30,8 @@ namespace PrivaPub.Models.Jobs RollupDay, CrawlPlan, CrawlInstance, - SendRecovery + SendRecovery, + CountAccount } public enum JobState diff --git a/PrivaPub/Models/User/Avatar.cs b/PrivaPub/Models/User/Avatar.cs index 287f340..d27f0a7 100644 --- a/PrivaPub/Models/User/Avatar.cs +++ b/PrivaPub/Models/User/Avatar.cs @@ -88,6 +88,10 @@ namespace PrivaPub.Models.User public DateTime? BannedAt { get; set; } public DateTime? DeletionAt { get; set; } public DateTime? ForgottenAt { get; set; }//its likes, votes, boosts and replies were taken out of our counts (GoneActors) + public int? FollowersCount { get; set; }//its own collections' totalItems, read at most daily (AccountCountsJob) + public int? FollowingCount { get; set; } + public int? StatusesCount { get; set; } + public DateTime? CountedAt { get; set; } } public class AvatarSettings diff --git a/PrivaPub/Web/Pages/Pages.cs b/PrivaPub/Web/Pages/Pages.cs index 8f4ca3d..576b8fd 100644 --- a/PrivaPub/Web/Pages/Pages.cs +++ b/PrivaPub/Web/Pages/Pages.cs @@ -92,6 +92,49 @@ namespace PrivaPub.Web.Pages } } + // The hashtag links our posts carry (/tags/{tag}): this server's public posts with that tag, as a page like a profile's. + public class TagModel : PublicPageModel + { + const int PageSize = 20; + + readonly ILocalActorService _localActors; + readonly DbEntities _dbEntities; + + public TagModel(ILocalActorService localActors, DbEntities dbEntities) + { + _localActors = localActors; + _dbEntities = dbEntities; + } + + public string Tag { get; private set; } + public IReadOnlyList Posts { get; private set; } = Array.Empty(); + + public async Task OnGetAsync(string tag, CancellationToken token) + { + Tag = tag?.TrimStart('#').ToLowerInvariant(); + if (string.IsNullOrEmpty(Tag) || Tag.Length > 100) + return NotFound(); + Harden(); + var posts = await _dbEntities.Posts + .Match(p => p.Tags.Contains(Tag) && !p.IsFederatedCopy && p.ReblogOfPostId == null && p.Visibility == PostVisibility.Public) + .Match(VisibilityPolicy.IsPublic) + .Sort(p => p.ID, Order.Descending) + .Limit(PageSize) + .ExecuteAsync(token); + var authors = new Dictionary(); + var views = new List(); + foreach (var post in posts) + { + if (!authors.TryGetValue(post.GroupUserId, out var author)) + authors[post.GroupUserId] = author = await _localActors.FindById(LocalActorKind.Person, post.GroupUserId, token); + if (author is { IsFederated: true }) + views.Add(PostView.From(post, author)); + } + Posts = views; + return Page(); + } + } + public class StatusModel : PublicPageModel { readonly ILocalActorService _localActors; diff --git a/PrivaPub/Web/Pages/Tag.cshtml b/PrivaPub/Web/Pages/Tag.cshtml new file mode 100644 index 0000000..cc01ceb --- /dev/null +++ b/PrivaPub/Web/Pages/Tag.cshtml @@ -0,0 +1,17 @@ +@page "/tags/{tag}" +@model PrivaPub.Web.Pages.TagModel +@{ + ViewData["Title"] = $"#{Model.Tag}"; +} +
+

#@Model.Tag

+
Public posts on this server with this tag.
+
+@foreach (var post in Model.Posts) +{ + +} +@if (Model.Posts.Count == 0) +{ +

Nothing public with this tag yet.

+} diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index 0a2a4b2..8ee6bdc 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -504,7 +504,10 @@ it, raw where it doesn't. so Mastodon 4.7 retries instead of switching to RFC 9421. - **Deliberately not done: dereferenceable `Follow`, `Like`, `Block`, `Accept`, `Reject` and `Undo` ids.** Serving them would publish who follows, likes and blocks whom, which our collections deliberately hide. Every one of them is sent - with its object embedded, which is all Misskey needs. `Create` and `Announce` ids do dereference. + with its object embedded, which is all Misskey needs. `Create` and `Announce` ids do dereference, a community's + announces included (kept since 2026-10-04 while the post is shown, 410 after). Update, Delete, EmojiReact, + QuoteRequest and its answers, Flag, Ignore and poll votes do not either: like Mastodon's, they are sent whole and + never fetched. #### P6 What people see: emoji, polls, quotes, reactions, cards, players - **Custom emoji** on posts, names, fields and poll options, proxied (done in v1.10.0), together with fuller remote