From 17922f4dfdf2bfb7a86f70906087596112d71adb Mon Sep 17 00:00:00 2001 From: thepra Date: Mon, 5 Oct 2026 05:37:28 +0200 Subject: [PATCH] Conversations page, and know what is unread /api/v1/conversations answered one page, never unread, its read endpoint did nothing, and DELETE was missing; each conversation cost a query per member. Now each conversation keeps its newest post (DmGroup.LastPostId, set as posts arrive, learnt once by migration _013) and pages by it as Mastodon does, and each persona's ConversationState holds what it read and what it took off its list: - unread when someone else wrote last, after what the persona read; - read marks it so, and writing in a conversation reads it; - DELETE takes it off the list until a newer message brings it back. The list reads its states, newest posts, members and accounts in a few queries per page. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01LsXgEaXee4GCU1hwYgPJXw --- CLAUDE.md | 2 + PrivaPub.Tests/Http/MastodonTimelinesTests.cs | 37 ++++++++ .../Controllers/TimelinesController.cs | 90 +++++++++++++------ .../Domain/Statuses/ConversationStates.cs | 59 ++++++++++++ PrivaPub/Domain/Statuses/StatusService.cs | 2 +- .../Inbox/Handlers/CreateHandler.cs | 2 +- PrivaPub/Infrastructure/Data/Indexes.cs | 6 ++ ...13_conversations_know_their_newest_post.cs | 24 +++++ PrivaPub/Models/Group/DmGroup.cs | 1 + PrivaPub/Models/Social/ConversationState.cs | 15 ++++ docs/ROADMAP.md | 3 + 11 files changed, 214 insertions(+), 27 deletions(-) create mode 100644 PrivaPub/Domain/Statuses/ConversationStates.cs create mode 100644 PrivaPub/Infrastructure/Data/Migrations/_013_conversations_know_their_newest_post.cs create mode 100644 PrivaPub/Models/Social/ConversationState.cs diff --git a/CLAUDE.md b/CLAUDE.md index b4f4bfa..26ab04b 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -199,6 +199,8 @@ group www-data and reaches the private mongod; `sudo -u www-data` works too. Located posts (`LocalGeo`) are the only local-only posts. 9. **A DM joins a conversation only by `DmGroup.ParticipantsKey`**, the exact set of its participants; a remote `context` decides nothing. DMs are `Post`s with `Visibility = Direct` and a `ConversationId` (`DmPost` is legacy). + `DmGroup.LastPostId` pages `/api/v1/conversations`, and `ConversationState` keeps each persona's read and removed + marks (`Domain/Statuses/ConversationStates.cs`; writing in a conversation reads it). 10. **Nothing slow happens inside a request.** Deliveries and inbox processing are `Job`s (`Infrastructure/Jobs`): leased, retried on Mastodon's curve, at most two per host, paused per host by `RemoteInstance`. The inbox answers 202 once it has verified and queued; a handler must be idempotent (unique `ObjectURI`, job `DedupeKey`). diff --git a/PrivaPub.Tests/Http/MastodonTimelinesTests.cs b/PrivaPub.Tests/Http/MastodonTimelinesTests.cs index abeb98a..6c4ecca 100644 --- a/PrivaPub.Tests/Http/MastodonTimelinesTests.cs +++ b/PrivaPub.Tests/Http/MastodonTimelinesTests.cs @@ -146,6 +146,43 @@ namespace PrivaPub.Tests.Http Assert.DoesNotContain(answer, (await stranger.Client.Get($"/api/v1/accounts/{bob.Id}/statuses")).Ok().Ids); } + [Fact] + public async Task Conversations_page_by_their_newest_message_and_know_what_is_unread_read_and_removed() + { + var alice = await _host.Mastodon("alice"); + var bob = await _host.Mastodon("bob"); + var carol = await _host.Mastodon("carol"); + await bob.Status($"@{alice.UserName} first", ("visibility", "direct")); + var fromCarol = (await carol.Status($"@{alice.UserName} second", ("visibility", "direct"))).Text("id"); + // writing in a conversation reads it + await alice.Status($"@{bob.UserName} third", ("visibility", "direct")); + async Task List() => (await alice.Client.Get("/api/v1/conversations")).Ok().Array; + + var all = await List(); + Assert.Equal(2, all.Count); + var (withBob, withCarol) = (all[0].Text("id"), all[1].Text("id")); + Assert.Equal(new[] { bob.Id }, all[0]!["accounts"]!.AsArray().Select(a => a.Text("id"))); + Assert.False(all[0].Flag("unread")); + Assert.True(all[1].Flag("unread")); + Assert.Equal(fromCarol, all[1]!["last_status"].Text("id")); + + var page = (await alice.Client.Get("/api/v1/conversations?limit=1")).Ok(); + Assert.Equal(withBob, Assert.Single(page.Array).Text("id")); + Assert.Equal(withCarol, Assert.Single((await alice.Client.Get(page.Link("next"))).Ok().Array).Text("id")); + + Assert.False((await alice.Client.Post($"/api/v1/conversations/{withCarol}/read")).Ok().Body.Flag("unread")); + Assert.False((await List()).Single(c => c.Text("id") == withCarol).Flag("unread")); + + // off the list until a newer message, which brings it back unread; carol's own list never changed + (await alice.Client.Delete($"/api/v1/conversations/{withCarol}")).Ok(); + Assert.Equal(new[] { withBob }, (await List()).Select(c => c.Text("id"))); + Assert.Equal(withCarol, Assert.Single((await carol.Client.Get("/api/v1/conversations")).Ok().Array).Text("id")); + var again = (await carol.Status($"@{alice.UserName} are you there?", ("visibility", "direct"))).Text("id"); + var back = (await List())[0]; + Assert.Equal((withCarol, again, true), (back.Text("id"), back!["last_status"].Text("id"), back.Flag("unread"))); + Assert.Equal(HttpStatusCode.NotFound, (await bob.Client.Delete($"/api/v1/conversations/{withCarol}")).Status); + } + [Fact] public async Task Markers_are_saved_versioned_and_read_back_per_timeline() { diff --git a/PrivaPub/Api/Mastodon/Controllers/TimelinesController.cs b/PrivaPub/Api/Mastodon/Controllers/TimelinesController.cs index c99f298..0222b08 100644 --- a/PrivaPub/Api/Mastodon/Controllers/TimelinesController.cs +++ b/PrivaPub/Api/Mastodon/Controllers/TimelinesController.cs @@ -6,6 +6,8 @@ using PrivaPub.Api.Mastodon.Entities; using PrivaPub.Api.Mastodon.Infrastructure; using PrivaPub.Api.Mastodon.Mappers; using PrivaPub.Domain.Privacy; +using PrivaPub.Domain.Statuses; +using PrivaPub.Models.Group; using PrivaPub.Models.Post; using PrivaPub.Models.Social; using PrivaPub.StaticServices; @@ -83,41 +85,79 @@ namespace PrivaPub.Api.Mastodon.Controllers return Json(await _mapper.Statuses(visible, MyId, token)); } + // the persona's direct conversations, by their newest post (paged like Mastodon's by that post's id), each with its + // other members and whether its newest post, someone else's, is newer than what the persona read; the ones it took + // off its list stay off until a newer message [HttpGet("/api/v1/conversations"), Scope("read:statuses")] public async Task Conversations(CancellationToken token) { - var groups = await _dbEntities.DmGroups.Match(g => !g.DeletionAt.HasValue && g.Members.Any(m => !m.IsForeign && m.AvatarId == MyId)) - .Sort(g => g.UpdatedAt, Order.Descending).Limit(Limit()).ExecuteAsync(token); - var conversations = new List(); - foreach (var group in groups) - { - var last = await _dbEntities.Posts.Match(p => p.ConversationId == group.ID).Match(VisibilityPolicy.IsShown).Sort(p => p.ID, Order.Descending).ExecuteFirstAsync(token); - if (last == default) - continue; - var others = new List(); - foreach (var member in group.Members.Where(m => !(m.AvatarId == MyId && !m.IsForeign))) - others.Add(member.IsForeign - ? (await _dbEntities.ForeignAvatars.Match(f => f.ActorURI == member.AvatarId).ExecuteFirstAsync(token))?.ID - : member.AvatarId); - var accounts = await _mapper.Accounts(others, token); - conversations.Add(new Conversation - { - Id = group.ID, - Accounts = accounts.Values.ToList(), - LastStatus = await _mapper.Status(last, MyId, token) - }); - } - return Json(conversations); + var groups = await Page.From(Params, Limit(20, 40)).Fetch( + _dbEntities.DmGroups.Match(g => !g.DeletionAt.HasValue && g.LastPostId != null && g.Members.Any(m => !m.IsForeign && m.AvatarId == MyId)), + g => g.LastPostId, token); + Link("/api/v1/conversations", groups.LastOrDefault()?.LastPostId, groups.FirstOrDefault()?.LastPostId); + return Json(await ConversationsOf(groups, token)); } + async Task> ConversationsOf(List groups, CancellationToken token) + { + var ids = groups.Select(g => g.ID).ToList(); + var states = (await DB.Default.Find().Match(s => s.AvatarId == MyId && ids.Contains(s.ConversationId)).ExecuteAsync(token)) + .ToDictionary(s => s.ConversationId); + groups = groups.Where(g => !ConversationStates.IsHidden(states.GetValueOrDefault(g.ID), g.LastPostId)).ToList(); + + // the newest post each conversation still shows: usually its LastPostId, else (deleted) the one before + var newestIds = groups.Select(g => g.LastPostId).ToList(); + var newest = (await _dbEntities.Posts.Match(p => newestIds.Contains(p.ID)).Match(VisibilityPolicy.IsShown).ExecuteAsync(token)) + .ToDictionary(p => p.ConversationId); + foreach (var group in groups.Where(g => !newest.ContainsKey(g.ID))) + if (await _dbEntities.Posts.Match(p => p.ConversationId == group.ID).Match(VisibilityPolicy.IsShown).Sort(p => p.ID, Order.Descending) + .ExecuteFirstAsync(token) is { } shown) + newest[group.ID] = shown; + + var foreignUris = groups.SelectMany(g => g.Members).Where(m => m.IsForeign).Select(m => m.AvatarId).Distinct().ToList(); + var foreignIds = (await _dbEntities.ForeignAvatars.Match(f => foreignUris.Contains(f.ActorURI)).ExecuteAsync(token)) + .GroupBy(f => f.ActorURI).ToDictionary(g => g.Key, g => g.First().ID); + List Others(DmGroup group) => group.Members.Where(m => !(m.AvatarId == MyId && !m.IsForeign)) + .Select(m => m.IsForeign ? foreignIds.GetValueOrDefault(m.AvatarId) : m.AvatarId).Where(id => id != default).ToList(); + var accounts = await _mapper.Accounts(groups.SelectMany(Others).Distinct(), token); + var statuses = (await _mapper.Statuses(newest.Values.ToList(), MyId, token)).ToDictionary(s => s.Id); + + return groups.Where(g => newest.ContainsKey(g.ID) && statuses.ContainsKey(newest[g.ID].ID)) + .Select(g => new Conversation + { + Id = g.ID, + Unread = ConversationStates.IsUnread(states.GetValueOrDefault(g.ID), newest[g.ID].ID, newest[g.ID].AuthorAccountId, MyId), + Accounts = Others(g).Select(id => accounts.GetValueOrDefault(id)).Where(a => a != default).ToList(), + LastStatus = statuses[newest[g.ID].ID] + }) + .ToList(); + } + + async Task MyConversation(string id, CancellationToken token) => + await _dbEntities.DmGroups.Match(g => g.ID == id && !g.DeletionAt.HasValue && g.Members.Any(m => !m.IsForeign && m.AvatarId == MyId)) + .ExecuteFirstAsync(token); + [HttpPost("/api/v1/conversations/{id}/read"), Scope("write:conversations")] public async Task ReadConversation(string id, CancellationToken token) { - var group = await _dbEntities.DmGroups.Match(g => g.ID == id && g.Members.Any(m => !m.IsForeign && m.AvatarId == MyId)).ExecuteFirstAsync(token); + var group = await MyConversation(id, token); if (group == default) return NotFoundError(); - var last = await _dbEntities.Posts.Match(p => p.ConversationId == group.ID).Match(VisibilityPolicy.IsShown).Sort(p => p.ID, Order.Descending).ExecuteFirstAsync(token); - return Json(new Conversation { Id = group.ID, LastStatus = last == default ? default : await _mapper.Status(last, MyId, token) }); + await ConversationStates.Read(MyId, group.ID, group.LastPostId, token); + var conversation = (await ConversationsOf(new List { group }, token)).FirstOrDefault(); + return Json(conversation ?? new Conversation { Id = group.ID }); + } + + // off the persona's list until a newer message; the messages themselves stay + [HttpDelete("/api/v1/conversations/{id}"), Scope("write:conversations")] + public async Task RemoveConversation(string id, CancellationToken token) + { + var group = await MyConversation(id, token); + if (group == default) + return NotFoundError(); + await ConversationStates.Hide(MyId, group.ID, group.LastPostId, token); + await ConversationStates.Read(MyId, group.ID, group.LastPostId, token); + return Json(new { }); } [HttpGet("/api/v1/markers"), Scope("read:statuses")] diff --git a/PrivaPub/Domain/Statuses/ConversationStates.cs b/PrivaPub/Domain/Statuses/ConversationStates.cs new file mode 100644 index 0000000..ab94343 --- /dev/null +++ b/PrivaPub/Domain/Statuses/ConversationStates.cs @@ -0,0 +1,59 @@ +using MongoDB.Driver; +using MongoDB.Entities; + +using PrivaPub.Models.Group; +using PrivaPub.Models.Social; + +namespace PrivaPub.Domain.Statuses +{ + // A direct conversation's newest post (DmGroup.LastPostId, which pages the conversations list) and each persona's + // read and removed marks on it (ConversationState). A conversation is unread for a persona when its newest post is + // someone else's and newer than what the persona read; writing in it reads it. + public static class ConversationStates + { + public static async Task Posted(string conversationId, string postId, string authorAvatarId, CancellationToken token) + { + await DB.Default.Update().MatchID(conversationId) + .Modify(g => g.UpdatedAt, DateTime.UtcNow) + .Modify(g => g.LastPostId, postId) + .ExecuteAsync(token); + if (authorAvatarId != default) + await Read(authorAvatarId, conversationId, postId, token); + } + + public static Task Read(string avatarId, string conversationId, string postId, CancellationToken token) => + Mark(avatarId, conversationId, s => s.LastReadPostId, postId, token); + + public static Task Hide(string avatarId, string conversationId, string postId, CancellationToken token) => + Mark(avatarId, conversationId, s => s.HiddenUpTo, postId, token); + + public static bool IsUnread(ConversationState state, string lastPostId, string lastAuthorId, string avatarId) => + lastPostId != default && lastAuthorId != avatarId && string.CompareOrdinal(lastPostId, state?.LastReadPostId) > 0; + + public static bool IsHidden(ConversationState state, string lastPostId) => + state?.HiddenUpTo != default && string.CompareOrdinal(lastPostId, state.HiddenUpTo) <= 0; + + // the mark only moves forward; the first mark makes the state (two at once: the unique index keeps one, and the + // loser marks again) + static async Task Mark(string avatarId, string conversationId, System.Linq.Expressions.Expression> field, + string postId, CancellationToken token) + { + if (postId == default) + return; + Task Write() => DB.Default.Update() + .Match(s => s.AvatarId == avatarId && s.ConversationId == conversationId) + .Modify(b => b.Max(field, postId)) + .Modify(s => s.UpdatedAt, DateTime.UtcNow) + .Option(o => o.IsUpsert = true) + .ExecuteAsync(token); + try + { + await Write(); + } + catch (MongoWriteException ex) when (ex.WriteError?.Category == ServerErrorCategory.DuplicateKey) + { + await Write(); + } + } + } +} diff --git a/PrivaPub/Domain/Statuses/StatusService.cs b/PrivaPub/Domain/Statuses/StatusService.cs index ef6d93d..f46fab0 100644 --- a/PrivaPub/Domain/Statuses/StatusService.cs +++ b/PrivaPub/Domain/Statuses/StatusService.cs @@ -237,7 +237,7 @@ namespace PrivaPub.Domain.Statuses var note = ActivityPubRenderer.DirectNote(post, author, recipients.Select(r => (r.Uri, r.Handle)).ToList(), dmGroup.ConversationURI); create = ActivityPubRenderer.Create(author, note, $"create-{post.ID}"); post.To = Strings(note["to"]); - await DB.Default.Update().MatchID(dmGroup.ID).Modify(g => g.UpdatedAt, DateTime.UtcNow).ExecuteAsync(token); + await ConversationStates.Posted(dmGroup.ID, post.ID, author.Id, token); } else if (!isLocalOnly) { diff --git a/PrivaPub/Federation/Inbox/Handlers/CreateHandler.cs b/PrivaPub/Federation/Inbox/Handlers/CreateHandler.cs index c08b63b..84338a1 100644 --- a/PrivaPub/Federation/Inbox/Handlers/CreateHandler.cs +++ b/PrivaPub/Federation/Inbox/Handlers/CreateHandler.cs @@ -186,7 +186,7 @@ namespace PrivaPub.Federation.Inbox.Handlers await DB.Default.Update().MatchID(parent.ID).Modify(b => b.Inc(p => p.RepliesCount, 1)).ExecuteAsync(token); await _fanout.Distribute(post, token); if (conversation != default) - await DB.Default.Update().MatchID(conversation.ID).Modify(g => g.UpdatedAt, DateTime.UtcNow).ExecuteAsync(token); + await Domain.Statuses.ConversationStates.Posted(conversation.ID, post.ID, default, token); if (group is { IsCircle: false }) await _groups.Announce(group, activity.AsObject(), note.Id, isNewPost: true, token); } diff --git a/PrivaPub/Infrastructure/Data/Indexes.cs b/PrivaPub/Infrastructure/Data/Indexes.cs index 1d56662..27e5fe2 100644 --- a/PrivaPub/Infrastructure/Data/Indexes.cs +++ b/PrivaPub/Infrastructure/Data/Indexes.cs @@ -54,6 +54,12 @@ namespace PrivaPub.Infrastructure.Data await Plain(token, g => g.InvitationCode); await Plain(token, g => g.ParticipantsKey); + await Plain(token, g => g.LastPostId); + await DB.Default.Index() + .Key(s => s.AvatarId, KeyType.Ascending) + .Key(s => s.ConversationId, KeyType.Ascending) + .Option(o => o.Unique = true) + .CreateAsync(token); await Unique(a => a.ActivityURI, Builders.Filter.Type(a => a.ActivityURI, BsonType.String), token); await Plain(token, d => d.DeliveredAt, d => d.AbandonedAt, d => d.NextAttemptAt); diff --git a/PrivaPub/Infrastructure/Data/Migrations/_013_conversations_know_their_newest_post.cs b/PrivaPub/Infrastructure/Data/Migrations/_013_conversations_know_their_newest_post.cs new file mode 100644 index 0000000..00ff3ce --- /dev/null +++ b/PrivaPub/Infrastructure/Data/Migrations/_013_conversations_know_their_newest_post.cs @@ -0,0 +1,24 @@ +using MongoDB.Entities; + +using PrivaPub.Models.Group; + +using PostEntity = PrivaPub.Models.Post.Post; + +namespace PrivaPub.Infrastructure.Data.Migrations +{ + // the conversations list pages by each conversation's newest post (DmGroup.LastPostId), set from now on as posts + // arrive; the conversations made before learn theirs once + public class _013_conversations_know_their_newest_post : IMigration + { + public async Task UpgradeAsync() + { + foreach (var group in await DB.Default.Find().Match(g => g.LastPostId == null).ExecuteAsync()) + { + var newest = await DB.Default.Find().Match(p => p.ConversationId == group.ID && !p.DeletedAt.HasValue) + .Sort(p => p.ID, Order.Descending).ExecuteFirstAsync(); + if (newest != default) + await DB.Default.Update().MatchID(group.ID).Modify(g => g.LastPostId, newest.ID).ExecuteAsync(); + } + } + } +} diff --git a/PrivaPub/Models/Group/DmGroup.cs b/PrivaPub/Models/Group/DmGroup.cs index eda8594..0e89e68 100644 --- a/PrivaPub/Models/Group/DmGroup.cs +++ b/PrivaPub/Models/Group/DmGroup.cs @@ -11,6 +11,7 @@ namespace PrivaPub.Models.Group public List Members { get; set; } = new(); public string ConversationURI { get; set; }//context of the federated thread public string ParticipantsKey { get; set; } + public string LastPostId { get; set; }//its newest post, which pages the conversations list (ConversationStates) public DateTime CreationDate { get; set; } = DateTime.UtcNow; public DateTime UpdatedAt { get; set; } = DateTime.UtcNow; diff --git a/PrivaPub/Models/Social/ConversationState.cs b/PrivaPub/Models/Social/ConversationState.cs new file mode 100644 index 0000000..ce0e3cf --- /dev/null +++ b/PrivaPub/Models/Social/ConversationState.cs @@ -0,0 +1,15 @@ +using MongoDB.Entities; + +namespace PrivaPub.Models.Social +{ + // how far a persona has read a direct conversation, and up to which post it took the conversation off its list + // (Mastodon's DELETE /api/v1/conversations/:id): a newer message brings it back, unread. Post ids, compared as strings. + public class ConversationState : Entity + { + public string AvatarId { get; set; } + public string ConversationId { get; set; } + public string LastReadPostId { get; set; } + public string HiddenUpTo { get; set; } + public DateTime UpdatedAt { get; set; } = DateTime.UtcNow; + } +} diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index 6b1c75e..e759158 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -59,6 +59,9 @@ Written 2026-10-01 from the original 2023 code, the decePubClient UI, a federati - three bugs the first village found, fixed: likes, reactions, votes and downvotes from followers on followers-only posts and from circle members were dropped (G-0001); a Follow refused with Mastodon's first-contact 422 was lost for good; a persona addressed without being named could not see the post (silent mentions). + - direct conversations as Mastodon keeps them (2026-10-05): paged by their newest message, `unread` when someone + else wrote last, `read` that sticks, and `DELETE` taking one off the list until a newer message; the list reads its + members and newest posts in a few queries, not several per conversation. - [ ] P7 Threads, communities, moderation, the social graph - [ ] P8 Signatures, discovery, the long tail - [x] P9 Reading at volume (owner decision 2026-10-04, back from "Cut"): lists, server-side filters, scheduled posts,