From d4226a289c2c455aeceeaca1288fcc6b0aefb816 Mon Sep 17 00:00:00 2001 From: thepra Date: Mon, 5 Oct 2026 03:16:40 +0200 Subject: [PATCH] Posts found by their words /api/v2/search now searches statuses by words for a signed-in persona, in Mastodon's scope: what it wrote, boosted, favourited, bookmarked or was named in, and public posts of authors who let themselves be indexed (indexable, off by default). Newest first, only what the persona may see, through a text index over the posts' words in every language alike. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01LsXgEaXee4GCU1hwYgPJXw --- .../Http/MastodonWordSearchTests.cs | 50 +++++++++++++++++++ .../Mastodon/Controllers/SearchController.cs | 42 +++++++++++++++- PrivaPub/Infrastructure/Data/Indexes.cs | 5 ++ docs/ROADMAP.md | 3 ++ 4 files changed, 99 insertions(+), 1 deletion(-) create mode 100644 PrivaPub.Tests/Http/MastodonWordSearchTests.cs diff --git a/PrivaPub.Tests/Http/MastodonWordSearchTests.cs b/PrivaPub.Tests/Http/MastodonWordSearchTests.cs new file mode 100644 index 0000000..c37cedb --- /dev/null +++ b/PrivaPub.Tests/Http/MastodonWordSearchTests.cs @@ -0,0 +1,50 @@ +using PrivaPub.Tests.Support; +using PrivaPub.Tests.Support.Host; + +namespace PrivaPub.Tests.Http +{ + // Status search by words, as Mastodon scopes it: the persona's own posts and those it boosted, favourited, bookmarked + // or was named in, and public posts of authors who let themselves be indexed + [Trait("Category", "Integration")] + public sealed class MastodonWordSearchTests : IAsyncLifetime + { + PrivaPubHost _host; + + public async ValueTask InitializeAsync() + { + Assert.SkipUnless(MongoFixture.Enabled, MongoFixture.Skip); + _host = await PrivaPubHost.Shared(); + } + + public ValueTask DisposeAsync() => ValueTask.CompletedTask; + + [Fact] + public async Task Words_find_what_the_persona_wrote_or_touched_and_what_indexable_authors_made_public() + { + var reader = await _host.Mastodon("reader"); + var open = await _host.Mastodon("open"); + var shy = await _host.Mastodon("shy"); + (await open.Client.Patch("/api/v1/accounts/update_credentials", ("indexable", "true"))).Ok(); + var word = $"w{Guid.NewGuid():N}"[..12]; + + var own = (await reader.Status($"my {word}")).Text("id"); + var indexed = (await open.Status($"open {word}")).Text("id"); + var quietIndexed = (await open.Status($"followers {word}", ("visibility", "private"))).Text("id"); + var unindexed = (await shy.Status($"shy {word}")).Text("id"); + var liked = (await shy.Status($"liked {word}")).Text("id"); + var named = (await shy.Status($"@{reader.UserName} about {word}")).Text("id"); + var warned = (await shy.Status("calm", ("spoiler_text", $"about {word}"))).Text("id"); + (await reader.Client.Post($"/api/v1/statuses/{liked}/favourite")).Ok(); + (await reader.Client.Post($"/api/v1/statuses/{warned}/bookmark")).Ok(); + + var found = (await reader.Client.Get($"/api/v2/search?q={word}&type=statuses&limit=40")).Ok().Body["statuses"]!.AsArray() + .Select(s => s.Text("id")).ToList(); + Assert.Equal(new[] { warned, named, liked, indexed, own }, found); + Assert.DoesNotContain(unindexed, found); + Assert.DoesNotContain(quietIndexed, found); + Assert.Empty((await _host.Client().Get($"/api/v2/search?q={word}&type=statuses")).Ok().Body["statuses"]!.AsArray()); + Assert.Equal(new[] { named }, + (await reader.Client.Get($"/api/v2/search?q={word}&type=statuses&limit=1&offset=1")).Ok().Body["statuses"]!.AsArray().Select(s => s.Text("id"))); + } + } +} diff --git a/PrivaPub/Api/Mastodon/Controllers/SearchController.cs b/PrivaPub/Api/Mastodon/Controllers/SearchController.cs index 9e61c2b..a9477db 100644 --- a/PrivaPub/Api/Mastodon/Controllers/SearchController.cs +++ b/PrivaPub/Api/Mastodon/Controllers/SearchController.cs @@ -15,7 +15,7 @@ using PostEntity = PrivaPub.Models.Post.Post; namespace PrivaPub.Api.Mastodon.Controllers { - public class SearchController : MastodonController + public partial class SearchController : MastodonController { readonly MastodonMapper _mapper; readonly DbEntities _dbEntities; @@ -73,12 +73,52 @@ namespace PrivaPub.Api.Mastodon.Controllers { results.Accounts = await _search.Find(q, resolve, Limit(), Math.Max(0, offset), token); } + if (type is null or "statuses" && MyId != default && !q.StartsWith('#')) + results.Statuses = await Words(q, Math.Max(0, offset), Limit(), token); if (type is null or "hashtags" && q.TrimStart('#') is { Length: > 0 } tag) results.Hashtags.Add(new { name = tag.ToLowerInvariant(), url = $"{_localActors.BaseAddress}/tags/{Uri.EscapeDataString(tag.ToLowerInvariant())}", history = Array.Empty() }); return Json(results); } } + public partial class SearchController + { + const int WordCandidates = 400; + + // Posts by their words, as Mastodon searches them: those the persona wrote, boosted, favourited, bookmarked or was + // named in, and public posts of authors who let themselves be indexed; newest first, what the persona may see only + async Task> Words(string q, int offset, int limit, CancellationToken token) + { + var candidates = await _dbEntities.Posts.Match(f => f.Text(q)).Match(p => !p.DeletedAt.HasValue && p.ReblogOfPostId == null && !p.AuthorGone) + .Sort(p => p.ID, MongoDB.Entities.Order.Descending).Limit(WordCandidates).ExecuteAsync(token); + if (candidates.Count == 0) + return new List(); + var ids = candidates.Select(p => p.ID).ToList(); + var mine = new HashSet(); + mine.UnionWith((await _dbEntities.Favourites.Match(f => f.AccountId == MyId && ids.Contains(f.PostId)).ExecuteAsync(token)).Select(f => f.PostId)); + mine.UnionWith((await MongoDB.Entities.DB.Default.Find().Match(b => b.AvatarId == MyId && ids.Contains(b.PostId)).ExecuteAsync(token)) + .Select(b => b.PostId)); + mine.UnionWith((await _dbEntities.Posts.Match(p => p.AuthorAccountId == MyId && ids.Contains(p.ReblogOfPostId) && !p.DeletedAt.HasValue).ExecuteAsync(token)) + .Select(p => p.ReblogOfPostId)); + var authors = candidates.Select(p => p.AuthorAccountId ?? p.GroupUserId).Where(a => a != default).Distinct().ToList(); + var indexable = (await _dbEntities.Avatars.Match(a => authors.Contains(a.ID) && a.Settings.IsIndexable).ExecuteAsync(token)).Select(a => a.ID) + .Concat((await _dbEntities.ForeignAvatars.Match(f => authors.Contains(f.ID) && f.IsIndexable).ExecuteAsync(token)).Select(f => f.ID)) + .ToHashSet(); + var found = new List(); + foreach (var post in candidates) + { + var author = post.AuthorAccountId ?? post.GroupUserId; + var reachable = author == MyId && !post.IsFederatedCopy || mine.Contains(post.ID) || post.Mentions.Any(m => m.AccountId == MyId) + || post.Visibility == PostVisibility.Public && indexable.Contains(author); + if (reachable && await VisibilityPolicy.CanSee(post, MyId, token)) + found.Add(post); + if (found.Count >= offset + limit) + break; + } + return await _mapper.Statuses(found.Skip(offset).Take(limit).ToList(), MyId, token); + } + } + public class StubsController : MastodonController { [HttpGet("/api/v1/custom_emojis"), Microsoft.AspNetCore.Authorization.AllowAnonymous] diff --git a/PrivaPub/Infrastructure/Data/Indexes.cs b/PrivaPub/Infrastructure/Data/Indexes.cs index 170d0f1..1d56662 100644 --- a/PrivaPub/Infrastructure/Data/Indexes.cs +++ b/PrivaPub/Infrastructure/Data/Indexes.cs @@ -110,6 +110,11 @@ namespace PrivaPub.Infrastructure.Data await Plain(token, f => f.AvatarId); await Plain(token, t => t.Name); await Plain(token, p => p.Tags); + //the words status search reads, in every language alike (no stemming, no stop words); Language is not the + //index's language field + await DB.Default.Index().Key(p => p.Text, KeyType.Text).Key(p => p.ContentHtml, KeyType.Text).Key(p => p.SpoilerText, KeyType.Text) + .Key(p => p.Title, KeyType.Text).Option(o => { o.DefaultLanguage = "none"; o.LanguageOverride = "textSearchLanguage"; o.Name = "words"; }) + .CreateAsync(token); await Plain(token, m => m.AvatarId); await Plain(token, m => m.TargetActorURI); await Plain(token, r => r.IsResolved, r => r.ID); diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index b6e1e07..0d69613 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -79,6 +79,9 @@ Written 2026-10-01 from the original 2023 code, the decePubClient UI, a federati holds count, nothing behind a content warning, and a post trends only if its author is discoverable. The directory lists discoverable accounts by their latest public post, or by the day they say they joined (a persona's published day, never its creation, so personas made together are not told apart by their order). + - [x] Status search by words, scoped as Mastodon scopes it: what the persona wrote, boosted, favourited, bookmarked or + was named in, and public posts of authors who let themselves be indexed (off by default); newest first, through a + text index over the posts' words in every language alike. ## Intent