From 5b3e456e096f17271bcaa621bf4e621841cca809 Mon Sep 17 00:00:00 2001 From: thepra Date: Mon, 5 Oct 2026 02:48:49 +0200 Subject: [PATCH] Filters: words and posts a persona would rather not see Mastodon's v2 filters replace the empty stubs: a filter's title, contexts, action (warn, hide, blur) and expiry, its keywords (whole words or not, taken as JSON objects, listed or numbered form fields, with id and _destroy on update) and its statuses, each with their own endpoints; the v1 API is the same filters seen keyword by keyword. Every status a persona reads carries the filters it matches in `filtered` (a boost as what it boosts), matched as Mastodon matches: warning, title, text, poll options and media descriptions. Clients apply context and action. Filters never federate, and go with a deleted persona. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01LsXgEaXee4GCU1hwYgPJXw --- PrivaPub.Tests/Http/MastodonFiltersTests.cs | 169 ++++++++ PrivaPub.Tests/Http/MastodonInstanceTests.cs | 2 - .../Mastodon/Controllers/FiltersController.cs | 362 ++++++++++++++++++ .../Mastodon/Controllers/SearchController.cs | 6 - PrivaPub/Api/Mastodon/Entities/Entities.cs | 44 ++- .../Mastodon/Infrastructure/MastodonParams.cs | 12 +- .../Api/Mastodon/Mappers/MastodonMapper.cs | 5 +- PrivaPub/Domain/Social/Filters.cs | 115 ++++++ PrivaPub/Infrastructure/Data/Indexes.cs | 1 + PrivaPub/Models/Social/Filters.cs | 50 +++ PrivaPub/Services/RootRemoval.cs | 1 + docs/ROADMAP.md | 5 +- 12 files changed, 759 insertions(+), 13 deletions(-) create mode 100644 PrivaPub.Tests/Http/MastodonFiltersTests.cs create mode 100644 PrivaPub/Api/Mastodon/Controllers/FiltersController.cs create mode 100644 PrivaPub/Domain/Social/Filters.cs create mode 100644 PrivaPub/Models/Social/Filters.cs diff --git a/PrivaPub.Tests/Http/MastodonFiltersTests.cs b/PrivaPub.Tests/Http/MastodonFiltersTests.cs new file mode 100644 index 0000000..127df7a --- /dev/null +++ b/PrivaPub.Tests/Http/MastodonFiltersTests.cs @@ -0,0 +1,169 @@ +using MongoDB.Entities; + +using PrivaPub.Models.Social; +using PrivaPub.Tests.Support; +using PrivaPub.Tests.Support.Host; + +using System.Net; +using System.Text.Json.Nodes; + +namespace PrivaPub.Tests.Http +{ + // Mastodon's filters: a persona's v2 filters with keywords and statuses, the v1 API over their keywords, and the + // `filtered` each status carries for the filters it matches + [Trait("Category", "Integration")] + public sealed class MastodonFiltersTests : IAsyncLifetime + { + PrivaPubHost _host; + + static CancellationToken Token => TestContext.Current.CancellationToken; + + public async ValueTask InitializeAsync() + { + Assert.SkipUnless(MongoFixture.Enabled, MongoFixture.Skip); + _host = await PrivaPubHost.Shared(); + } + + public ValueTask DisposeAsync() => ValueTask.CompletedTask; + + static Task Create(Mastodon owner, string title, string action, params (string Keyword, bool WholeWord)[] keywords) => + owner.Client.Json(HttpMethod.Post, "/api/v2/filters", new JsonObject + { + ["title"] = title, + ["context"] = new JsonArray("home", "public", "thread"), + ["filter_action"] = action, + ["keywords_attributes"] = new JsonArray(keywords.Select(k => (JsonNode)new JsonObject { ["keyword"] = k.Keyword, ["whole_word"] = k.WholeWord }).ToArray()) + }); + + [Fact] + public async Task A_filter_is_made_changed_and_deleted_by_its_owner_only() + { + var alice = await _host.Mastodon("alice"); + var mallory = await _host.Mastodon("mallory"); + + var made = (await Create(alice, "Spoilers", "hide", ("spoiler", true), ("leak", false))).Ok().Object; + var id = made.Text("id"); + Assert.Equal("hide", made.Text("filter_action")); + Assert.Equal(new[] { "home", "public", "thread" }, made["context"]!.AsArray().Select(c => c!.GetValue())); + var keywords = made["keywords"]!.AsArray(); + Assert.Equal(new[] { "spoiler", "leak" }, keywords.Select(k => k.Text("keyword"))); + Assert.Equal(new[] { true, false }, keywords.Select(k => k!.Flag("whole_word"))); + Assert.Null(made["expires_at"]); + + var changed = (await alice.Client.Json(HttpMethod.Put, $"/api/v2/filters/{id}", new JsonObject + { + ["title"] = "Leaks", + ["filter_action"] = "warn", + ["expires_in"] = 3600, + ["keywords_attributes"] = new JsonArray( + new JsonObject { ["id"] = keywords[0].Text("id"), ["_destroy"] = true }, + new JsonObject { ["keyword"] = "ending" }) + })).Ok().Object; + Assert.Equal("Leaks", changed.Text("title")); + Assert.Equal("warn", changed.Text("filter_action")); + Assert.NotNull(changed.Text("expires_at")); + Assert.Equal(new[] { "leak", "ending" }, changed["keywords"]!.AsArray().Select(k => k.Text("keyword"))); + + var added = (await alice.Client.Post($"/api/v2/filters/{id}/keywords", ("keyword", "finale"), ("whole_word", "false"))).Ok().Object; + Assert.False(added.Flag("whole_word")); + var keywordId = added.Text("id"); + Assert.Equal("finale", (await alice.Client.Get($"/api/v2/filters/keywords/{keywordId}")).Ok().Object.Text("keyword")); + Assert.Equal("last scene", (await alice.Client.Put($"/api/v2/filters/keywords/{keywordId}", ("keyword", "last scene"))).Ok().Object.Text("keyword")); + (await alice.Client.Delete($"/api/v2/filters/keywords/{keywordId}")).Ok(); + Assert.Equal(2, (await alice.Client.Get($"/api/v2/filters/{id}/keywords")).Ok().Array.Count); + + Assert.Equal(HttpStatusCode.UnprocessableEntity, (await Create(alice, " ", "warn", ("x", true))).Status); + Assert.Equal(HttpStatusCode.UnprocessableEntity, (await Create(alice, "x", "explode", ("x", true))).Status); + Assert.Equal(HttpStatusCode.UnprocessableEntity, (await alice.Client.Post("/api/v2/filters", ("title", "x"), ("context[]", "everywhere"))).Status); + Assert.Empty((await mallory.Client.Get("/api/v2/filters")).Ok().Array); + Assert.Equal(HttpStatusCode.NotFound, (await mallory.Client.Get($"/api/v2/filters/{id}")).Status); + Assert.Equal(HttpStatusCode.NotFound, (await mallory.Client.Delete($"/api/v2/filters/keywords/{changed["keywords"]![0].Text("id")}")).Status); + Assert.Equal(HttpStatusCode.Unauthorized, (await _host.Client().Get("/api/v2/filters")).Status); + + (await alice.Client.Delete($"/api/v2/filters/{id}")).Ok(); + Assert.Empty((await alice.Client.Get("/api/v2/filters")).Ok().Array); + } + + [Fact] + public async Task Statuses_carry_the_filters_they_match_by_word_and_by_id() + { + var alice = await _host.Mastodon("alice"); + var bob = await _host.Mastodon("bob"); + (await bob.Client.Post($"/api/v1/accounts/{alice.Id}/follow")).Ok(); + var filterId = (await Create(alice, "Spoilers", "warn", ("spoiler", true), ("#tv", true))).Ok().Object.Text("id"); + var other = (await Create(bob, "Not mine", "hide", ("spoiler", false))).Ok().Object.Text("id"); + var whole = (await alice.Status("a big SPOILER ahead")).Text("id"); + var partial = (await alice.Status("spoilers galore")).Text("id"); + var tagged = (await alice.Status("watching #TV tonight")).Text("id"); + var plain = (await alice.Status("nothing to see")).Text("id"); + var warned = (await alice.Status("calm text", ("spoiler_text", "spoiler inside"))).Text("id"); + var byId = (await alice.Client.Post($"/api/v2/filters/{filterId}/statuses", ("status_id", plain))).Ok().Object; + Assert.Equal(plain, byId.Text("status_id")); + var boost = (await bob.Client.Post($"/api/v1/statuses/{whole}/reblog")).Ok().Object.Text("id"); + + var home = (await alice.Client.Get("/api/v1/timelines/home?limit=40")).Ok().Array.ToDictionary(s => s.Text("id"), s => s!["filtered"]!.AsArray()); + var match = Assert.Single(home[whole]); + Assert.Equal(filterId, match!["filter"].Text("id")); + Assert.Equal("warn", match["filter"].Text("filter_action")); + Assert.Equal(new[] { "spoiler" }, match["keyword_matches"]!.AsArray().Select(k => k!.GetValue())); + Assert.Empty(home[partial]); + Assert.Equal(new[] { "#tv" }, Assert.Single(home[tagged])!["keyword_matches"]!.AsArray().Select(k => k!.GetValue())); + Assert.Equal(new[] { byId.Text("id") }, Assert.Single(home[plain])!["status_matches"]!.AsArray().Select(k => k!.GetValue())); + Assert.Single(home[warned]); + Assert.Single((await alice.Client.Get($"/api/v1/statuses/{boost}")).Ok().Body["filtered"]!.AsArray()); + + var asBob = (await bob.Client.Get($"/api/v1/statuses/{partial}")).Ok().Body["filtered"]!.AsArray(); + Assert.Equal(other, Assert.Single(asBob)!["filter"].Text("id")); + Assert.Empty((await _host.Client().Get($"/api/v1/statuses/{whole}")).Ok().Body["filtered"]!.AsArray()); + + // an expired filter matches nothing + await DB.Default.Update().MatchID(filterId).Modify(f => f.ExpiresAt, DateTime.UtcNow.AddMinutes(-1)).ExecuteAsync(Token); + Assert.Empty((await alice.Client.Get($"/api/v1/statuses/{whole}")).Ok().Body["filtered"]!.AsArray()); + } + + [Fact] + public async Task Keywords_come_as_form_fields_listed_or_numbered() + { + var alice = await _host.Mastodon("alice"); + + var listed = (await alice.Client.Post("/api/v2/filters", ("title", "Listed"), ("context[]", "home"), + ("keywords_attributes[][keyword]", "one"), ("keywords_attributes[][whole_word]", "false"), + ("keywords_attributes[][keyword]", "two"), ("keywords_attributes[][whole_word]", "true"))).Ok().Object; + Assert.Equal(new[] { ("one", false), ("two", true) }, + listed["keywords"]!.AsArray().Select(k => (k.Text("keyword"), k!.Flag("whole_word")))); + + var numbered = (await alice.Client.Post("/api/v2/filters", ("title", "Numbered"), ("context[]", "public"), + ("keywords_attributes[1][keyword]", "second"), ("keywords_attributes[0][keyword]", "first"), + ("keywords_attributes[0][whole_word]", "false"))).Ok().Object; + Assert.Equal(new[] { ("first", false), ("second", true) }, + numbered["keywords"]!.AsArray().Select(k => (k.Text("keyword"), k!.Flag("whole_word")))); + } + + [Fact] + public async Task A_v1_filter_is_one_keyword_of_a_v2_filter() + { + var alice = await _host.Mastodon("alice"); + + var made = (await alice.Client.Post("/api/v1/filters", ("phrase", "election"), ("context[]", "home"), ("context[]", "notifications"), + ("irreversible", "true"), ("whole_word", "false"))).Ok().Object; + var id = made.Text("id"); + Assert.Equal("election", made.Text("phrase")); + Assert.True(made.Flag("irreversible")); + Assert.False(made.Flag("whole_word")); + var v2 = Assert.Single((await alice.Client.Get("/api/v2/filters")).Ok().Array)!; + Assert.Equal("election", v2.Text("title")); + Assert.Equal("hide", v2.Text("filter_action")); + Assert.Equal(id, Assert.Single(v2["keywords"]!.AsArray()).Text("id")); + + var changed = (await alice.Client.Put($"/api/v1/filters/{id}", ("phrase", "elections"), ("irreversible", "false"))).Ok().Object; + Assert.Equal("elections", changed.Text("phrase")); + Assert.False(changed.Flag("irreversible")); + Assert.Equal(new[] { id }, (await alice.Client.Get("/api/v1/filters")).Ok().Ids); + Assert.Equal("elections", (await alice.Client.Get($"/api/v1/filters/{id}")).Ok().Object.Text("phrase")); + + (await alice.Client.Delete($"/api/v1/filters/{id}")).Ok(); + Assert.Empty((await alice.Client.Get("/api/v2/filters")).Ok().Array); + Assert.Equal(HttpStatusCode.NotFound, (await alice.Client.Get($"/api/v1/filters/{id}")).Status); + } + } +} diff --git a/PrivaPub.Tests/Http/MastodonInstanceTests.cs b/PrivaPub.Tests/Http/MastodonInstanceTests.cs index 554ec70..6c9812a 100644 --- a/PrivaPub.Tests/Http/MastodonInstanceTests.cs +++ b/PrivaPub.Tests/Http/MastodonInstanceTests.cs @@ -79,8 +79,6 @@ namespace PrivaPub.Tests.Http } [Theory] - [InlineData("/api/v1/filters")] - [InlineData("/api/v2/filters")] [InlineData("/api/v1/suggestions")] [InlineData("/api/v2/suggestions")] [InlineData("/api/v1/followed_tags")] diff --git a/PrivaPub/Api/Mastodon/Controllers/FiltersController.cs b/PrivaPub/Api/Mastodon/Controllers/FiltersController.cs new file mode 100644 index 0000000..00c1612 --- /dev/null +++ b/PrivaPub/Api/Mastodon/Controllers/FiltersController.cs @@ -0,0 +1,362 @@ +using System.Text.RegularExpressions; + +using Microsoft.AspNetCore.Mvc; + +using MongoDB.Bson; +using MongoDB.Entities; + +using PrivaPub.Api.Mastodon.Entities; +using PrivaPub.Api.Mastodon.Infrastructure; +using PrivaPub.Domain.Social; +using PrivaPub.Models.Social; +using PrivaPub.StaticServices; + +namespace PrivaPub.Api.Mastodon.Controllers +{ + // Mastodon's filters (owner decision 2026-10-04, back from the cut list): v2 filters with their keywords and statuses, + // and the v1 API, whose filter is one keyword of a v2 filter. Statuses carry what they match in `filtered`; clients + // apply the context and the action. Nothing here leaves the server. + public class FiltersController : MastodonController + { + const int MaxFilters = 100; + const int MaxEntries = 100; + const int MaxKeywordLength = 100; + static readonly Regex Attribute = new(@"^keywords_attributes\[(\d*)\]\[(\w+)\]$", RegexOptions.Compiled); + + readonly DbEntities _dbEntities; + + public FiltersController(DbEntities dbEntities) + { + _dbEntities = dbEntities; + } + + static string NewId() => ObjectId.GenerateNewId().ToString(); + + Task Mine(string id, CancellationToken token) => + DB.Default.Find().Match(f => f.ID == id && f.AvatarId == MyId).ExecuteFirstAsync(token); + + Task WithKeyword(string keywordId, CancellationToken token) => + DB.Default.Find().Match(f => f.AvatarId == MyId && f.Keywords.Any(k => k.Id == keywordId)).ExecuteFirstAsync(token); + + Task WithStatus(string statusFilterId, CancellationToken token) => + DB.Default.Find().Match(f => f.AvatarId == MyId && f.Statuses.Any(s => s.Id == statusFilterId)).ExecuteFirstAsync(token); + + static JsonResult Invalid(string message) => Error(StatusCodes.Status422UnprocessableEntity, "Validation failed: " + message); + + // title, context[], filter_action, expires_in: what a create or an update sets; an error message when one is wrong + string Apply(PersonaFilter filter, bool creating) + { + var title = Params.Get("title")?.Trim(); + if (title != default || creating) + { + if (string.IsNullOrEmpty(title) || title.Length > 200) + return "Title can't be blank"; + filter.Title = title; + } + if (Params.Has("context") || creating) + { + var context = Params.List("context").Distinct().ToList(); + if (context.Count == 0 || context.Any(c => !FilterContexts.IsValid(c))) + return "Context can't be blank"; + filter.Context = context; + } + if (Params.Get("filter_action") is { } action) + { + if (!FilterActions.IsValid(action)) + return "Filter action is not included in the list"; + filter.Action = action; + } + if (Params.Has("expires_in")) + filter.ExpiresAt = Params.Int("expires_in") is > 0 and var seconds ? DateTime.UtcNow.AddSeconds(seconds) : default; + return default; + } + + // keywords_attributes, as Rails takes them: a JSON array of objects, or form fields [][keyword], [][whole_word], [][id], + // [][_destroy], or numbered [0][keyword] + List> KeywordAttributes() + { + if (Params.Json is { ValueKind: System.Text.Json.JsonValueKind.Object } body + && body.TryGetProperty("keywords_attributes", out var array) && array.ValueKind == System.Text.Json.JsonValueKind.Array) + return array.EnumerateArray().Where(o => o.ValueKind == System.Text.Json.JsonValueKind.Object) + .Select(o => o.EnumerateObject().ToDictionary(p => p.Name, p => p.Value.ValueKind switch + { + System.Text.Json.JsonValueKind.String => p.Value.GetString(), + System.Text.Json.JsonValueKind.True => "true", + System.Text.Json.JsonValueKind.False => "false", + _ => p.Value.ToString() + })) + .ToList(); + var numbered = new SortedDictionary>(); + var listed = new Dictionary>(); + foreach (var name in Params.Names) + { + var match = Attribute.Match(name); + if (!match.Success) + continue; + if (match.Groups[1].Value.Length == 0) + listed[match.Groups[2].Value] = Params.List(name); + else + { + var index = int.Parse(match.Groups[1].Value); + if (!numbered.TryGetValue(index, out var item)) + numbered[index] = item = new Dictionary(); + item[match.Groups[2].Value] = Params.Get(name); + } + } + var items = numbered.Values.ToList(); + var count = listed.Count == 0 ? 0 : listed.Values.Max(v => v.Count); + for (var i = 0; i < count; i++) + items.Add(listed.Where(kv => i < kv.Value.Count).ToDictionary(kv => kv.Key, kv => kv.Value[i])); + return items; + } + + static bool IsTrue(string value) => value?.ToLowerInvariant() is "true" or "1" or "on" or "yes"; + + // adds, changes or removes keywords; an error message when one is wrong + static string ApplyKeywords(PersonaFilter filter, List> items) + { + foreach (var item in items) + { + var id = item.GetValueOrDefault("id"); + var existing = id == default ? default : filter.Keywords.FirstOrDefault(k => k.Id == id); + if (existing != default && IsTrue(item.GetValueOrDefault("_destroy"))) + { + filter.Keywords.Remove(existing); + continue; + } + var keyword = item.GetValueOrDefault("keyword")?.Trim(); + if (existing == default && string.IsNullOrEmpty(keyword)) + return "Keyword can't be blank"; + if (keyword?.Length > MaxKeywordLength) + return "Keyword is too long"; + var target = existing ?? new FilterKeyword { Id = NewId() }; + if (!string.IsNullOrEmpty(keyword)) + target.Keyword = keyword; + if (item.ContainsKey("whole_word")) + target.WholeWord = IsTrue(item["whole_word"]); + else if (existing == default) + target.WholeWord = true; + if (existing == default) + filter.Keywords.Add(target); + } + return filter.Keywords.Count > MaxEntries ? "Too many keywords" : default; + } + + // -- v2 + + [HttpGet("/api/v2/filters"), Scope("read:filters")] + public async Task All(CancellationToken token) => + Json((await DB.Default.Find().Match(f => f.AvatarId == MyId).Sort(f => f.ID, Order.Ascending).ExecuteAsync(token)) + .Select(Filters.View).ToList()); + + [HttpGet("/api/v2/filters/{id}"), Scope("read:filters")] + public async Task One(string id, CancellationToken token) => + await Mine(id, token) is { } filter ? Json(Filters.View(filter)) : NotFoundError(); + + [HttpPost("/api/v2/filters"), Scope("write:filters")] + public async Task Create(CancellationToken token) + { + if (await DB.Default.CountAsync(f => f.AvatarId == MyId, token) >= MaxFilters) + return Invalid("Too many filters"); + var filter = new PersonaFilter { AvatarId = MyId }; + if ((Apply(filter, creating: true) ?? ApplyKeywords(filter, KeywordAttributes())) is { } problem) + return Invalid(problem); + await DB.Default.SaveAsync(filter, token); + return Json(Filters.View(filter)); + } + + [HttpPut("/api/v2/filters/{id}"), Scope("write:filters")] + public async Task Update(string id, CancellationToken token) + { + if (await Mine(id, token) is not { } filter) + return NotFoundError(); + if ((Apply(filter, creating: false) ?? ApplyKeywords(filter, KeywordAttributes())) is { } problem) + return Invalid(problem); + await DB.Default.SaveAsync(filter, token); + return Json(Filters.View(filter)); + } + + [HttpDelete("/api/v2/filters/{id}"), Scope("write:filters")] + public async Task Delete(string id, CancellationToken token) + { + if (await Mine(id, token) is not { } filter) + return NotFoundError(); + await DB.Default.DeleteAsync(filter.ID); + return Json(new { }); + } + + [HttpGet("/api/v2/filters/{id}/keywords"), Scope("read:filters")] + public async Task Keywords(string id, CancellationToken token) => + await Mine(id, token) is { } filter ? Json(filter.Keywords.Select(Filters.Keyword).ToList()) : NotFoundError(); + + [HttpPost("/api/v2/filters/{id}/keywords"), Scope("write:filters")] + public async Task AddKeyword(string id, CancellationToken token) + { + if (await Mine(id, token) is not { } filter) + return NotFoundError(); + var item = new Dictionary { ["keyword"] = Params.Get("keyword") }; + if (Params.Has("whole_word")) + item["whole_word"] = Params.Get("whole_word"); + if (ApplyKeywords(filter, new List> { item }) is { } problem) + return Invalid(problem); + await DB.Default.SaveAsync(filter, token); + return Json(Filters.Keyword(filter.Keywords[^1])); + } + + [HttpGet("/api/v2/filters/keywords/{id}"), Scope("read:filters")] + public async Task Keyword(string id, CancellationToken token) => + await WithKeyword(id, token) is { } filter ? Json(Filters.Keyword(filter.Keywords.First(k => k.Id == id))) : NotFoundError(); + + [HttpPut("/api/v2/filters/keywords/{id}"), Scope("write:filters")] + public async Task UpdateKeyword(string id, CancellationToken token) + { + if (await WithKeyword(id, token) is not { } filter) + return NotFoundError(); + var item = new Dictionary { ["id"] = id, ["keyword"] = Params.Get("keyword") }; + if (Params.Has("whole_word")) + item["whole_word"] = Params.Get("whole_word"); + if (ApplyKeywords(filter, new List> { item }) is { } problem) + return Invalid(problem); + await DB.Default.SaveAsync(filter, token); + return Json(Filters.Keyword(filter.Keywords.First(k => k.Id == id))); + } + + [HttpDelete("/api/v2/filters/keywords/{id}"), Scope("write:filters")] + public async Task DeleteKeyword(string id, CancellationToken token) + { + if (await WithKeyword(id, token) is not { } filter) + return NotFoundError(); + filter.Keywords.RemoveAll(k => k.Id == id); + await DB.Default.SaveAsync(filter, token); + return Json(new { }); + } + + [HttpGet("/api/v2/filters/{id}/statuses"), Scope("read:filters")] + public async Task Statuses(string id, CancellationToken token) => + await Mine(id, token) is { } filter ? Json(filter.Statuses.Select(Filters.Status).ToList()) : NotFoundError(); + + // a status the persona can see, once per filter + [HttpPost("/api/v2/filters/{id}/statuses"), Scope("write:filters")] + public async Task AddStatus(string id, CancellationToken token) + { + if (await Mine(id, token) is not { } filter) + return NotFoundError(); + var statusId = Params.Get("status_id"); + var post = string.IsNullOrEmpty(statusId) ? default : await _dbEntities.Posts.Match(p => p.ID == statusId).ExecuteFirstAsync(token); + if (post == default || !await Domain.Privacy.VisibilityPolicy.CanSee(post, MyId, token)) + return NotFoundError(); + if (filter.Statuses.FirstOrDefault(s => s.PostId == statusId) is { } present) + return Json(Filters.Status(present)); + if (filter.Statuses.Count >= MaxEntries) + return Invalid("Too many statuses"); + var added = new FilterStatus { Id = NewId(), PostId = statusId }; + filter.Statuses.Add(added); + await DB.Default.SaveAsync(filter, token); + return Json(Filters.Status(added)); + } + + [HttpGet("/api/v2/filters/statuses/{id}"), Scope("read:filters")] + public async Task Status(string id, CancellationToken token) => + await WithStatus(id, token) is { } filter ? Json(Filters.Status(filter.Statuses.First(s => s.Id == id))) : NotFoundError(); + + [HttpDelete("/api/v2/filters/statuses/{id}"), Scope("write:filters")] + public async Task DeleteStatus(string id, CancellationToken token) + { + if (await WithStatus(id, token) is not { } filter) + return NotFoundError(); + filter.Statuses.RemoveAll(s => s.Id == id); + await DB.Default.SaveAsync(filter, token); + return Json(new { }); + } + + // -- v1: each keyword is a filter of its own, with its parent's context, action and expiry + + static FilterV1Entity V1(PersonaFilter filter, FilterKeyword keyword) => new() + { + Id = keyword.Id, + Phrase = keyword.Keyword, + Context = filter.Context, + WholeWord = keyword.WholeWord, + ExpiresAt = filter.ExpiresAt.HasValue ? MastodonJson.Time(filter.ExpiresAt.Value) : default, + Irreversible = filter.Action == FilterActions.Hide + }; + + [HttpGet("/api/v1/filters"), Scope("read:filters")] + public async Task AllV1(CancellationToken token) => + Json((await DB.Default.Find().Match(f => f.AvatarId == MyId).Sort(f => f.ID, Order.Ascending).ExecuteAsync(token)) + .SelectMany(f => f.Keywords.Select(k => V1(f, k))).ToList()); + + [HttpGet("/api/v1/filters/{id}"), Scope("read:filters")] + public async Task OneV1(string id, CancellationToken token) => + await WithKeyword(id, token) is { } filter ? Json(V1(filter, filter.Keywords.First(k => k.Id == id))) : NotFoundError(); + + // phrase, context[], irreversible, whole_word, expires_in + string ApplyV1(PersonaFilter filter, FilterKeyword keyword, bool creating) + { + var phrase = Params.Get("phrase")?.Trim(); + if (phrase != default || creating) + { + if (string.IsNullOrEmpty(phrase)) + return "Phrase can't be blank"; + if (phrase.Length > MaxKeywordLength) + return "Phrase is too long"; + keyword.Keyword = phrase; + if (creating) + filter.Title = phrase; + } + if (Params.Has("context") || creating) + { + var context = Params.List("context").Distinct().ToList(); + if (context.Count == 0 || context.Any(c => !FilterContexts.IsValid(c))) + return "Context can't be blank"; + filter.Context = context; + } + if (Params.Bool("whole_word") is { } wholeWord) + keyword.WholeWord = wholeWord; + if (Params.Bool("irreversible") is { } irreversible) + filter.Action = irreversible ? FilterActions.Hide : FilterActions.Warn; + if (Params.Has("expires_in")) + filter.ExpiresAt = Params.Int("expires_in") is > 0 and var seconds ? DateTime.UtcNow.AddSeconds(seconds) : default; + return default; + } + + [HttpPost("/api/v1/filters"), Scope("write:filters")] + public async Task CreateV1(CancellationToken token) + { + if (await DB.Default.CountAsync(f => f.AvatarId == MyId, token) >= MaxFilters) + return Invalid("Too many filters"); + var keyword = new FilterKeyword { Id = NewId(), WholeWord = true }; + var filter = new PersonaFilter { AvatarId = MyId, Keywords = new List { keyword } }; + if (ApplyV1(filter, keyword, creating: true) is { } problem) + return Invalid(problem); + await DB.Default.SaveAsync(filter, token); + return Json(V1(filter, keyword)); + } + + [HttpPut("/api/v1/filters/{id}"), Scope("write:filters")] + public async Task UpdateV1(string id, CancellationToken token) + { + if (await WithKeyword(id, token) is not { } filter) + return NotFoundError(); + var keyword = filter.Keywords.First(k => k.Id == id); + if (ApplyV1(filter, keyword, creating: false) is { } problem) + return Invalid(problem); + await DB.Default.SaveAsync(filter, token); + return Json(V1(filter, keyword)); + } + + // the keyword goes, and its filter with it when nothing else is left in it + [HttpDelete("/api/v1/filters/{id}"), Scope("write:filters")] + public async Task DeleteV1(string id, CancellationToken token) + { + if (await WithKeyword(id, token) is not { } filter) + return NotFoundError(); + filter.Keywords.RemoveAll(k => k.Id == id); + if (filter.Keywords.Count == 0 && filter.Statuses.Count == 0) + await DB.Default.DeleteAsync(filter.ID); + else + await DB.Default.SaveAsync(filter, token); + return Json(new { }); + } + } +} diff --git a/PrivaPub/Api/Mastodon/Controllers/SearchController.cs b/PrivaPub/Api/Mastodon/Controllers/SearchController.cs index 211db73..24d7329 100644 --- a/PrivaPub/Api/Mastodon/Controllers/SearchController.cs +++ b/PrivaPub/Api/Mastodon/Controllers/SearchController.cs @@ -84,12 +84,6 @@ namespace PrivaPub.Api.Mastodon.Controllers [HttpGet("/api/v1/custom_emojis"), Microsoft.AspNetCore.Authorization.AllowAnonymous] public IActionResult CustomEmojis() => Json(Array.Empty()); - [HttpGet("/api/v1/filters"), Scope("read:filters")] - public IActionResult FiltersV1() => Json(Array.Empty()); - - [HttpGet("/api/v2/filters"), Scope("read:filters")] - public IActionResult FiltersV2() => Json(Array.Empty()); - [HttpGet("/api/v1/announcements"), Microsoft.AspNetCore.Authorization.AllowAnonymous] public IActionResult Announcements() => Json(Array.Empty()); diff --git a/PrivaPub/Api/Mastodon/Entities/Entities.cs b/PrivaPub/Api/Mastodon/Entities/Entities.cs index a24ec6e..e98d7b3 100644 --- a/PrivaPub/Api/Mastodon/Entities/Entities.cs +++ b/PrivaPub/Api/Mastodon/Entities/Entities.cs @@ -178,6 +178,48 @@ public string CurrentUser { get; set; } = "denied"; } + public class FilterEntity + { + public string Id { get; set; } + public string Title { get; set; } + public List Context { get; set; } = new(); + public string ExpiresAt { get; set; } + public string FilterAction { get; set; } + public List Keywords { get; set; } = new(); + public List Statuses { get; set; } = new(); + } + + public class FilterKeywordEntity + { + public string Id { get; set; } + public string Keyword { get; set; } + public bool WholeWord { get; set; } + } + + public class FilterStatusEntity + { + public string Id { get; set; } + public string StatusId { get; set; } + } + + public class FilterResultEntity + { + public FilterEntity Filter { get; set; } + public List KeywordMatches { get; set; } + public List StatusMatches { get; set; } + } + + // the v1 filter, which is one keyword of a v2 filter + public class FilterV1Entity + { + public string Id { get; set; } + public string Phrase { get; set; } + public List Context { get; set; } = new(); + public bool WholeWord { get; set; } + public string ExpiresAt { get; set; } + public bool Irreversible { get; set; } + } + public class EmojiReactionEntity { public string Name { get; set; } @@ -266,7 +308,7 @@ public Application Application { get; set; } public string Language { get; set; } public string Text { get; set; } - public List Filtered { get; set; } = new(); + public List Filtered { get; set; } = new(); public PrivaPubStatus Privapub { get; set; } public List EmojiReactions { get; set; } = new(); public PleromaStatus Pleroma { get; set; } diff --git a/PrivaPub/Api/Mastodon/Infrastructure/MastodonParams.cs b/PrivaPub/Api/Mastodon/Infrastructure/MastodonParams.cs index 67f6019..4b4e1c2 100644 --- a/PrivaPub/Api/Mastodon/Infrastructure/MastodonParams.cs +++ b/PrivaPub/Api/Mastodon/Infrastructure/MastodonParams.cs @@ -8,14 +8,19 @@ namespace PrivaPub.Api.Mastodon.Infrastructure { readonly Dictionary> _values; - MastodonParams(Dictionary> values) + MastodonParams(Dictionary> values, JsonElement? json = default) { _values = values; + Json = json; } + // a JSON body as it came, for what flattening loses: which fields belong to the same object of an array + public JsonElement? Json { get; } + public static async Task Read(HttpRequest request, CancellationToken token) { var values = new Dictionary>(StringComparer.Ordinal); + JsonElement? json = default; foreach (var pair in request.Query) foreach (var value in pair.Value) Add(values, pair.Key, value); @@ -33,12 +38,13 @@ namespace PrivaPub.Api.Mastodon.Infrastructure { using var document = await JsonDocument.ParseAsync(request.Body, cancellationToken: token); Flatten(values, default, document.RootElement); + json = document.RootElement.Clone(); } catch (JsonException) { } } - return new MastodonParams(values); + return new MastodonParams(values, json); } public static MastodonParams From(IEnumerable> pairs) @@ -51,6 +57,8 @@ namespace PrivaPub.Api.Mastodon.Infrastructure public bool Has(string name) => _values.ContainsKey(name); + public IEnumerable Names => _values.Keys; + public string Get(string name) => _values.TryGetValue(name, out var list) && list.Count > 0 ? list[^1] : default; public IReadOnlyList List(string name) => _values.TryGetValue(name, out var list) ? list : (IReadOnlyList)Array.Empty(); diff --git a/PrivaPub/Api/Mastodon/Mappers/MastodonMapper.cs b/PrivaPub/Api/Mastodon/Mappers/MastodonMapper.cs index 6d9a074..04bd6b0 100644 --- a/PrivaPub/Api/Mastodon/Mappers/MastodonMapper.cs +++ b/PrivaPub/Api/Mastodon/Mappers/MastodonMapper.cs @@ -197,6 +197,7 @@ namespace PrivaPub.Api.Mastodon.Mappers var hidden = viewerId == default || !hideFromViewer ? new HashSet() : await Domain.Relationships.Hidden.AuthorsHiddenFrom(viewerId, all.Select(p => p.ActorURI), forNotifications: false, token); + var filters = await Domain.Social.Filters.Of(viewerId, token); var reblogged = viewerId == default ? new HashSet() : (await _dbEntities.Posts.Match(p => p.AuthorAccountId == viewerId && ids.Contains(p.ReblogOfPostId) && !p.DeletedAt.HasValue).ExecuteAsync(token)) @@ -238,7 +239,8 @@ namespace PrivaPub.Api.Mastodon.Mappers EmojiReactions = reactions.GetValueOrDefault(post.ID) ?? new(), Pleroma = new PleromaStatus { EmojiReactions = reactions.GetValueOrDefault(post.ID) ?? new() }, Mentions = post.Mentions.Where(m => !m.Silent).Select(m => Mention(m, mentionAccounts)).Where(m => m != default).ToList(), - Tags = post.Tags.Select(t => new StatusTag { Name = t, Url = ActivityPubRenderer.TagUrl(_localActors.BaseAddress, t) }).ToList() + Tags = post.Tags.Select(t => new StatusTag { Name = t, Url = ActivityPubRenderer.TagUrl(_localActors.BaseAddress, t) }).ToList(), + Filtered = filters.Of(post) }; return status; } @@ -287,6 +289,7 @@ namespace PrivaPub.Api.Mastodon.Mappers continue; Quote(inner, original); status.Reblog = inner; + status.Filtered = inner.Filtered;//a boost is filtered as what it boosts status.Url = inner.Url;//a boost has no page of its own; /grunts would answer JSON to a browser status.Content = string.Empty; status.Reblogged = reblogged.Contains(original.ID); diff --git a/PrivaPub/Domain/Social/Filters.cs b/PrivaPub/Domain/Social/Filters.cs new file mode 100644 index 0000000..b15d1af --- /dev/null +++ b/PrivaPub/Domain/Social/Filters.cs @@ -0,0 +1,115 @@ +using System.Text.RegularExpressions; + +using MongoDB.Entities; + +using PrivaPub.Api.Mastodon.Entities; +using PrivaPub.Models.Social; + +using PostEntity = PrivaPub.Models.Post.Post; + +namespace PrivaPub.Domain.Social +{ + // A viewer's active filters, matched against posts as Mastodon matches them: a keyword anywhere in the post's words + // (its warning, title, text, poll options and media descriptions), case-insensitive, whole words when asked (a word + // boundary only where the keyword starts or ends with a word character); a filtered status by its id. + public sealed class Filters + { + static readonly AngleSharp.Html.Parser.HtmlParser Html = new(); + + readonly List<(PersonaFilter Filter, FilterEntity View, List<(string Keyword, Regex Pattern)> Keywords, HashSet PostIds)> _active; + + Filters(List filters) + { + var now = DateTime.UtcNow; + _active = filters.Where(f => f.IsActive(now)) + .Select(f => (f, View(f), f.Keywords.Select(k => (k.Keyword, Pattern(k))).ToList(), f.Statuses.Select(s => s.PostId).ToHashSet())) + .ToList(); + } + + public static readonly Filters None = new(new List()); + + public static async Task Of(string viewerId, CancellationToken token) => + viewerId == default ? None : new Filters(await DB.Default.Find().Match(f => f.AvatarId == viewerId).ExecuteAsync(token)); + + public bool IsEmpty => _active.Count == 0; + + public static Regex Pattern(FilterKeyword keyword) + { + var escaped = Regex.Escape(keyword.Keyword ?? string.Empty); + if (keyword.WholeWord) + { + var start = Regex.IsMatch(keyword.Keyword ?? string.Empty, @"^\w") ? @"\b" : string.Empty; + var end = Regex.IsMatch(keyword.Keyword ?? string.Empty, @"\w$") ? @"\b" : string.Empty; + escaped = start + escaped + end; + } + return new Regex(escaped, RegexOptions.IgnoreCase | RegexOptions.CultureInvariant, TimeSpan.FromMilliseconds(50)); + } + + public List Of(PostEntity post) + { + if (_active.Count == 0 || post == default) + return new List(); + string words = default; + var results = new List(); + foreach (var (filter, view, keywords, postIds) in _active) + { + var keywordMatches = new List(); + if (keywords.Count > 0) + { + words ??= Words(post); + foreach (var (keyword, pattern) in keywords) + if (Matches(pattern, words)) + keywordMatches.Add(keyword); + } + var statusMatches = postIds.Contains(post.ID) ? new List { filter.Statuses.First(s => s.PostId == post.ID).Id } : new List(); + if (keywordMatches.Count > 0 || statusMatches.Count > 0) + results.Add(new FilterResultEntity + { + Filter = view, + KeywordMatches = keywordMatches.Count > 0 ? keywordMatches : default, + StatusMatches = statusMatches.Count > 0 ? statusMatches : default + }); + } + return results; + } + + static bool Matches(Regex pattern, string words) + { + try + { + return pattern.IsMatch(words); + } + catch (RegexMatchTimeoutException) + { + return false; + } + } + + static string Words(PostEntity post) + { + var text = !string.IsNullOrEmpty(post.Text) && post.IsFederatedCopy == false + ? post.Text + : string.IsNullOrEmpty(post.ContentHtml) ? post.Text : Html.ParseDocument($"{post.ContentHtml}").Body!.TextContent; + return string.Join("\n", new[] { post.SpoilerText, post.Title, text } + .Concat(post.Poll?.Options.Select(o => o.Title) ?? Enumerable.Empty()) + .Concat(post.Media.Select(m => m.Description)) + .Where(s => !string.IsNullOrEmpty(s))); + } + + public static FilterEntity View(PersonaFilter filter) => new() + { + Id = filter.ID, + Title = filter.Title, + Context = filter.Context, + ExpiresAt = filter.ExpiresAt.HasValue ? Api.Mastodon.Infrastructure.MastodonJson.Time(filter.ExpiresAt.Value) : default, + FilterAction = filter.Action, + Keywords = filter.Keywords.Select(Keyword).ToList(), + Statuses = filter.Statuses.Select(Status).ToList() + }; + + public static FilterKeywordEntity Keyword(FilterKeyword keyword) => + new() { Id = keyword.Id, Keyword = keyword.Keyword, WholeWord = keyword.WholeWord }; + + public static FilterStatusEntity Status(FilterStatus status) => new() { Id = status.Id, StatusId = status.PostId }; + } +} diff --git a/PrivaPub/Infrastructure/Data/Indexes.cs b/PrivaPub/Infrastructure/Data/Indexes.cs index 2b8c9d3..89d9be5 100644 --- a/PrivaPub/Infrastructure/Data/Indexes.cs +++ b/PrivaPub/Infrastructure/Data/Indexes.cs @@ -106,6 +106,7 @@ namespace PrivaPub.Infrastructure.Data await Plain(token, b => b.TargetActorURI); await Plain(token, b => b.ActorURI); await Plain(token, l => l.AvatarId); + await Plain(token, f => f.AvatarId); await Plain(token, m => m.AvatarId); await Plain(token, m => m.TargetActorURI); await Plain(token, r => r.IsResolved, r => r.ID); diff --git a/PrivaPub/Models/Social/Filters.cs b/PrivaPub/Models/Social/Filters.cs new file mode 100644 index 0000000..ecf1dd9 --- /dev/null +++ b/PrivaPub/Models/Social/Filters.cs @@ -0,0 +1,50 @@ +using MongoDB.Entities; + +namespace PrivaPub.Models.Social +{ + //a persona's filter (Mastodon's v2 filters, owner decision 2026-10-04): words and posts it would rather not see, in + //some contexts, to be warned about, hidden or blurred. Statuses carry the filters they match (`filtered`) and the client + //applies context and action, as Mastodon's do. Filters are the persona's own and never federate. + public class PersonaFilter : Entity + { + public string AvatarId { get; set; } + public string Title { get; set; } + public List Context { get; set; } = new(); + public DateTime? ExpiresAt { get; set; } + public string Action { get; set; } = FilterActions.Warn; + public List Keywords { get; set; } = new(); + public List Statuses { get; set; } = new(); + public DateTime CreatedAt { get; set; } = DateTime.UtcNow; + + public bool IsActive(DateTime now) => ExpiresAt == default || ExpiresAt > now; + } + + public class FilterKeyword + { + public string Id { get; set; } + public string Keyword { get; set; } + public bool WholeWord { get; set; } + } + + public class FilterStatus + { + public string Id { get; set; } + public string PostId { get; set; } + } + + public static class FilterActions + { + public const string Warn = "warn"; + public const string Hide = "hide"; + public const string Blur = "blur"; + + public static bool IsValid(string value) => value is Warn or Hide or Blur; + } + + public static class FilterContexts + { + public static readonly string[] All = { "home", "notifications", "public", "thread", "account" }; + + public static bool IsValid(string value) => All.Contains(value); + } +} diff --git a/PrivaPub/Services/RootRemoval.cs b/PrivaPub/Services/RootRemoval.cs index d72c725..73737b5 100644 --- a/PrivaPub/Services/RootRemoval.cs +++ b/PrivaPub/Services/RootRemoval.cs @@ -64,6 +64,7 @@ namespace PrivaPub.Services await Empty(avatar.ID, now, token); await DB.Default.DeleteAsync(m => m.AvatarId == avatar.ID); await DB.Default.DeleteAsync(l => l.AvatarId == avatar.ID); + await DB.Default.DeleteAsync(f => f.AvatarId == avatar.ID); } await DB.Default.Update().MatchID(root.ID) diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index bd6fd69..fe7d4ec 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -66,6 +66,8 @@ Written 2026-10-01 from the original 2023 code, the decePubClient UI, a federati - [x] Lists: CRUD, members (followed accounts only, dropped when the follow ends), `timelines/list` with Mastodon's replies policies, exclusive lists kept out of home, and `accounts/search?following=true` to fill them. Lists are the persona's own and never federate. + - [x] Filters: v2 (keywords, whole words, statuses, contexts, warn/hide/blur, expiry) and v1 over their keywords; + every status carries the filters it matches in `filtered`, and clients apply context and action, as Mastodon's do. ## Intent @@ -450,7 +452,8 @@ The first refactor commit is a pure move with namespaces only. Logic changes fol - **Timelines and the rest:** home, public, tag; notifications; markers; conversations; `/api/v2/search` with `resolve`. - **Lists:** CRUD, `:id/accounts`, `accounts/:id/lists`, `timelines/list/:id` (P9). - - **Stubs:** `custom_emojis`, filters, announcements, trends, suggestions, `followed_tags`, preferences. + - **Filters:** v2 with keywords and statuses, v1, `filtered` on statuses (P9). + - **Stubs:** `custom_emojis`, announcements, trends, suggestions, `followed_tags`, preferences. - **Mapping:** - Avatar, ForeignAvatar and Group all map to Account (`group: true` for groups). `acct` uses a WebFinger-verified handle; `created_at` is truncated to the day; avatar and header always have a placeholder URL.