From bf7c88ce710765286836837751d6dce025fce4fb Mon Sep 17 00:00:00 2001 From: thepra Date: Thu, 1 Oct 2026 10:54:17 +0200 Subject: [PATCH] Remote HTML is sanitized before it is stored S5 of the roadmap. ContentSanitizer wraps HtmlSanitizer with Mastodon's allowlist: the inline and list tags Mastodon keeps, href/rel/class and the list attributes, microformat and mention/hashtag/ellipsis/invisible classes, Mastodon's link schemes, every link rel=nofollow noopener noreferrer, relative links unlinked, headings folded to a bold paragraph, and the contents of script, style, svg, iframe and friends dropped rather than kept as text. Post and DmPost gain ContentHtml (what is shown) and ContentFormat (Markdown for local, Html for remote). Inbound Create and Update, and a remote actor's biography, are sanitized on the way in; local posts store their Markdig rendering. Migration _002 does the same to what is already stored, and migrations now run at startup. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012CzABvBkbcFqoHdmi8b9WB --- PrivaPub.ClientModels/Post/ViewPost.cs | 1 + .../Federation/ContentSanitizerTests.cs | 51 ++++++++++++ .../Infrastructure/MigrationTests.cs | 33 ++++++++ .../Federation/Actors/RemoteActorService.cs | 2 +- PrivaPub/Federation/Inbox/InboxService.cs | 15 +++- .../Federation/Objects/ContentSanitizer.cs | 80 +++++++++++++++++++ .../_002_sanitize_stored_content.cs | 40 ++++++++++ PrivaPub/Models/Post/ContentFormat.cs | 8 ++ PrivaPub/Models/Post/DmPost.cs | 2 + PrivaPub/Models/Post/Post.cs | 2 + PrivaPub/PrivaPub.csproj | 1 + PrivaPub/Program.cs | 1 + PrivaPub/Services/PostsService.cs | 7 ++ 13 files changed, 239 insertions(+), 4 deletions(-) create mode 100644 PrivaPub.Tests/Federation/ContentSanitizerTests.cs create mode 100644 PrivaPub.Tests/Infrastructure/MigrationTests.cs create mode 100644 PrivaPub/Federation/Objects/ContentSanitizer.cs create mode 100644 PrivaPub/Infrastructure/Data/Migrations/_002_sanitize_stored_content.cs create mode 100644 PrivaPub/Models/Post/ContentFormat.cs diff --git a/PrivaPub.ClientModels/Post/ViewPost.cs b/PrivaPub.ClientModels/Post/ViewPost.cs index 66890b2..14da3d4 100644 --- a/PrivaPub.ClientModels/Post/ViewPost.cs +++ b/PrivaPub.ClientModels/Post/ViewPost.cs @@ -11,6 +11,7 @@ namespace PrivaPub.ClientModels.Post public string AnsweringToPostId { get; set; } public string Title { get; set; } public string Text { get; set; } + public string ContentHtml { get; set; } public bool HasContentWarning { get; set; } public bool IsFederatedCopy { get; set; } public DateTime CreationDate { get; set; } diff --git a/PrivaPub.Tests/Federation/ContentSanitizerTests.cs b/PrivaPub.Tests/Federation/ContentSanitizerTests.cs new file mode 100644 index 0000000..8dccc89 --- /dev/null +++ b/PrivaPub.Tests/Federation/ContentSanitizerTests.cs @@ -0,0 +1,51 @@ +using PrivaPub.Federation.Objects; + +namespace PrivaPub.Tests.Federation +{ + public class ContentSanitizerTests + { + [Theory] + [InlineData("

hi

", "

hi

")] + [InlineData("

hi

", "

hi

")] + [InlineData("", "")] + [InlineData("x", "x")] + [InlineData("alert(1)\">x", "x")] + [InlineData("x", "x")] + [InlineData("text", "text")] + [InlineData("

x

", "

x

")] + [InlineData("", "")] + [InlineData("

x

", "

x

")] + [InlineData("
ok", "ok")] + [InlineData("

ab

", "

ab

")] + [InlineData("

kept

", "

kept

")] + [InlineData("

Title

", "

Title

")] + [InlineData("x", "x")] + public void Removes_what_is_not_allowed(string input, string expected) => + Assert.Equal(expected, ContentSanitizer.Html(input)); + + [Fact] + public void Keeps_mastodon_mention_and_hashtag_markup() + { + var html = "

@alice " + + "#fedi " + + "https://example.org/a-long

"; + + var sanitized = ContentSanitizer.Html(html); + + Assert.Contains("class=\"h-card\"", sanitized); + Assert.Contains("class=\"u-url mention\"", sanitized); + Assert.Contains("class=\"mention hashtag\"", sanitized); + Assert.Contains("class=\"invisible\"", sanitized); + Assert.Contains("class=\"ellipsis\"", sanitized); + Assert.Contains("href=\"https://m.example/@alice\"", sanitized); + Assert.DoesNotContain("rel=\"tag\"", sanitized); + } + + [Theory] + [InlineData("")] + [InlineData(" ")] + [InlineData(null)] + public void Empty_input_is_empty(string input) => + Assert.Equal(string.Empty, ContentSanitizer.Html(input)); + } +} diff --git a/PrivaPub.Tests/Infrastructure/MigrationTests.cs b/PrivaPub.Tests/Infrastructure/MigrationTests.cs new file mode 100644 index 0000000..c4eb46f --- /dev/null +++ b/PrivaPub.Tests/Infrastructure/MigrationTests.cs @@ -0,0 +1,33 @@ +using MongoDB.Entities; + +using PrivaPub.Infrastructure.Data.Migrations; +using PrivaPub.Models.Post; +using PrivaPub.Tests.Support; + +namespace PrivaPub.Tests.Infrastructure +{ + [Trait("Category", "Integration")] + public class MigrationTests + { + [Fact] + public async Task Stored_remote_content_is_sanitized_and_local_content_rendered() + { + Assert.SkipUnless(MongoFixture.Enabled, MongoFixture.Skip); + var token = TestContext.Current.CancellationToken; + var remote = new Post { IsFederatedCopy = true, Text = "

hi

", ObjectURI = $"https://r.example/{Guid.NewGuid():N}" }; + var local = new Post { Text = "**bold** raw", ObjectURI = $"https://privapub.test/{Guid.NewGuid():N}" }; + await DB.Default.SaveAsync(new[] { remote, local }, token); + + await new _002_sanitize_stored_content().UpgradeAsync(); + + var migratedRemote = await DB.Default.Find().OneAsync(remote.ID, token); + var migratedLocal = await DB.Default.Find().OneAsync(local.ID, token); + Assert.Equal("

hi

", migratedRemote.Text); + Assert.Equal("

hi

", migratedRemote.ContentHtml); + Assert.Equal(ContentFormat.Html, migratedRemote.ContentFormat); + Assert.Equal("**bold** raw", migratedLocal.Text); + Assert.Equal("

bold <b>raw</b>

", migratedLocal.ContentHtml); + Assert.Equal(ContentFormat.Markdown, migratedLocal.ContentFormat); + } + } +} diff --git a/PrivaPub/Federation/Actors/RemoteActorService.cs b/PrivaPub/Federation/Actors/RemoteActorService.cs index c37408a..90b36bf 100644 --- a/PrivaPub/Federation/Actors/RemoteActorService.cs +++ b/PrivaPub/Federation/Actors/RemoteActorService.cs @@ -158,7 +158,7 @@ namespace PrivaPub.Federation.Actors .Match(a => a.ActorURI == actor.Id) .Modify(a => a.UserName, actor.PreferredUsername) .Modify(a => a.Name, actor.Name) - .Modify(a => a.Biography, actor.Summary) + .Modify(a => a.Biography, ContentSanitizer.Html(actor.Summary)) .Modify(a => a.Url, actor.Url) .Modify(a => a.Domain, new Uri(actor.Id).Authority) .Modify(a => a.InboxURL, actor.Inbox) diff --git a/PrivaPub/Federation/Inbox/InboxService.cs b/PrivaPub/Federation/Inbox/InboxService.cs index daaeee3..5991a54 100644 --- a/PrivaPub/Federation/Inbox/InboxService.cs +++ b/PrivaPub/Federation/Inbox/InboxService.cs @@ -2,6 +2,7 @@ using MongoDB.Entities; using PrivaPub.Models.Federation; using PrivaPub.Models.Group; +using PrivaPub.Models.Post; using PrivaPub.Models.User; using PrivaPub.StaticServices; @@ -211,13 +212,16 @@ namespace PrivaPub.Federation.Inbox if (await _dbEntities.Posts.Match(p => p.ObjectURI == objectUri).ExecuteAnyAsync(token)) return new(StatusCodes.Status202Accepted); + var html = ContentSanitizer.Html(Value(note, "content")); var post = new PostEntity { ObjectURI = objectUri, ActorURI = author.ActorURI, GroupId = group?.Id, Title = Value(note, "summary") ?? Value(note, "name"), - Text = Value(note, "content"), + Text = html, + ContentHtml = html, + ContentFormat = ContentFormat.Html, HasContentWarning = note["sensitive"] is JsonValue sensitive && sensitive.TryGetValue(out var s) && s, AnsweringToPostId = await LocalPostId(inReplyTo, token) ?? inReplyTo, IsFederatedCopy = true, @@ -241,13 +245,16 @@ namespace PrivaPub.Federation.Inbox var participants = addressed.Where(a => a != ActivityPubRenderer.Public).Append(author.ActorURI).ToList(); var dmGroup = await FindOrCreateDmGroup(participants, Value(note, "context") ?? Value(note, "conversation"), token); + var dmHtml = ContentSanitizer.Html(Value(note, "content")); var dm = new DmPostEntity { ObjectURI = objectUri, ActorURI = author.ActorURI, GroupId = dmGroup.ID, Title = Value(note, "summary"), - Text = Value(note, "content"), + Text = dmHtml, + ContentHtml = dmHtml, + ContentFormat = ContentFormat.Html, HasContentWarning = note["sensitive"] is JsonValue dmSensitive && dmSensitive.TryGetValue(out var ds) && ds, AnsweringToPostId = inReplyTo, IsFederatedCopy = true, @@ -296,15 +303,17 @@ namespace PrivaPub.Federation.Inbox return new(StatusCodes.Status202Accepted); var objectUri = Id(inner); - var text = Value(inner, "content"); + var text = ContentSanitizer.Html(Value(inner, "content")); await DB.Default.Update() .Match(p => p.ObjectURI == objectUri && p.ActorURI == actor.ActorURI) .Modify(p => p.Text, text) + .Modify(p => p.ContentHtml, text) .Modify(p => p.UpdateDate, DateTime.UtcNow) .ExecuteAsync(token); await DB.Default.Update() .Match(p => p.ObjectURI == objectUri && p.ActorURI == actor.ActorURI) .Modify(p => p.Text, text) + .Modify(p => p.ContentHtml, text) .Modify(p => p.UpdateDate, DateTime.UtcNow) .ExecuteAsync(token); return new(StatusCodes.Status202Accepted); diff --git a/PrivaPub/Federation/Objects/ContentSanitizer.cs b/PrivaPub/Federation/Objects/ContentSanitizer.cs new file mode 100644 index 0000000..df53ec0 --- /dev/null +++ b/PrivaPub/Federation/Objects/ContentSanitizer.cs @@ -0,0 +1,80 @@ +using AngleSharp.Css.Dom; +using AngleSharp.Dom; + +using Ganss.Xss; + +using System.Text.RegularExpressions; + +namespace PrivaPub.Federation.Objects +{ + public static partial class ContentSanitizer + { + static readonly HtmlSanitizer Sanitizer = Build(); + + public static string Html(string html) => + string.IsNullOrWhiteSpace(html) ? string.Empty : Sanitizer.Sanitize(html).Trim(); + + static HtmlSanitizer Build() + { + var sanitizer = new HtmlSanitizer(new HtmlSanitizerOptions + { + AllowedTags = new HashSet(StringComparer.OrdinalIgnoreCase) + { + "p", "br", "span", "a", "abbr", "del", "s", "pre", "blockquote", "code", "b", "strong", "u", "i", "em", + "sub", "sup", "ul", "ol", "li", "ruby", "rt", "rp", "h1", "h2", "h3", "h4", "h5", "h6" + }, + AllowedAttributes = new HashSet(StringComparer.OrdinalIgnoreCase) + { + "href", "rel", "class", "translate", "start", "reversed", "value", "title" + }, + AllowedCssProperties = new HashSet(), + AllowedAtRules = new HashSet(), + AllowedSchemes = new HashSet(StringComparer.OrdinalIgnoreCase) + { + "http", "https", "dat", "dweb", "ipfs", "ipns", "ssb", "gopher", "xmpp", "magnet", "gemini" + }, + UriAttributes = new HashSet(StringComparer.OrdinalIgnoreCase) { "href" } + }) + { + KeepChildNodes = true + }; + foreach (var allowed in new[] { "mention", "hashtag", "ellipsis", "invisible" }) + sanitizer.AllowedClasses.Add(allowed); + sanitizer.RemovingCssClass += (_, e) => e.Cancel = MicroformatClass().IsMatch(e.CssClass); + sanitizer.RemovingTag += (_, e) => + { + if (e.Tag.LocalName is "script" or "style" or "template" or "iframe" or "object" or "embed" or "noscript" or "svg" or "math") + e.Tag.InnerHtml = string.Empty; + }; + sanitizer.PostProcessNode += (_, e) => + { + if (e.Node is not IElement element) + return; + switch (element.LocalName) + { + case "a": + if (!SchemePrefix().IsMatch(element.GetAttribute("href") ?? string.Empty)) + element.RemoveAttribute("href"); + element.SetAttribute("rel", "nofollow noopener noreferrer"); + element.SetAttribute("target", "_blank"); + break; + case "h1" or "h2" or "h3" or "h4" or "h5" or "h6": + var paragraph = e.Document.CreateElement("p"); + var strong = e.Document.CreateElement("strong"); + while (element.FirstChild != default) + strong.AppendChild(element.FirstChild); + paragraph.AppendChild(strong); + e.ReplacementNodes.Add(paragraph); + break; + } + }; + return sanitizer; + } + + [GeneratedRegex("^[a-z][a-z0-9+.-]*:", RegexOptions.IgnoreCase)] + private static partial Regex SchemePrefix(); + + [GeneratedRegex("^(h|p|u|dt|e)-[a-z0-9-]+$")] + private static partial Regex MicroformatClass(); + } +} diff --git a/PrivaPub/Infrastructure/Data/Migrations/_002_sanitize_stored_content.cs b/PrivaPub/Infrastructure/Data/Migrations/_002_sanitize_stored_content.cs new file mode 100644 index 0000000..be8fb4a --- /dev/null +++ b/PrivaPub/Infrastructure/Data/Migrations/_002_sanitize_stored_content.cs @@ -0,0 +1,40 @@ +using MongoDB.Entities; + +using PrivaPub.Federation.Objects; +using PrivaPub.Federation.Rendering; +using PrivaPub.Models.Post; +using PrivaPub.Models.User; + +namespace PrivaPub.Infrastructure.Data.Migrations +{ + public class _002_sanitize_stored_content : IMigration + { + public async Task UpgradeAsync() + { + foreach (var post in await DB.Default.Find().ExecuteAsync()) + { + var html = post.IsFederatedCopy ? ContentSanitizer.Html(post.Text) : ActivityPubRenderer.Html(post.Text); + await DB.Default.Update().MatchID(post.ID) + .Modify(p => p.Text, post.IsFederatedCopy ? html : post.Text) + .Modify(p => p.ContentHtml, html) + .Modify(p => p.ContentFormat, post.IsFederatedCopy ? ContentFormat.Html : ContentFormat.Markdown) + .ExecuteAsync(); + } + + foreach (var post in await DB.Default.Find().ExecuteAsync()) + { + var html = post.IsFederatedCopy ? ContentSanitizer.Html(post.Text) : ActivityPubRenderer.Html(post.Text); + await DB.Default.Update().MatchID(post.ID) + .Modify(p => p.Text, post.IsFederatedCopy ? html : post.Text) + .Modify(p => p.ContentHtml, html) + .Modify(p => p.ContentFormat, post.IsFederatedCopy ? ContentFormat.Html : ContentFormat.Markdown) + .ExecuteAsync(); + } + + foreach (var avatar in await DB.Default.Find().ExecuteAsync()) + await DB.Default.Update().MatchID(avatar.ID) + .Modify(a => a.Biography, ContentSanitizer.Html(avatar.Biography)) + .ExecuteAsync(); + } + } +} diff --git a/PrivaPub/Models/Post/ContentFormat.cs b/PrivaPub/Models/Post/ContentFormat.cs new file mode 100644 index 0000000..8d63605 --- /dev/null +++ b/PrivaPub/Models/Post/ContentFormat.cs @@ -0,0 +1,8 @@ +namespace PrivaPub.Models.Post +{ + public enum ContentFormat + { + Markdown, + Html + } +} diff --git a/PrivaPub/Models/Post/DmPost.cs b/PrivaPub/Models/Post/DmPost.cs index 79c230c..6feb7bd 100644 --- a/PrivaPub/Models/Post/DmPost.cs +++ b/PrivaPub/Models/Post/DmPost.cs @@ -9,6 +9,8 @@ namespace PrivaPub.Models.Post public string GroupId { get; set; } public string Title { get; set; } public string Text { get; set; } + public string ContentHtml { get; set; } + public ContentFormat ContentFormat { get; set; } public List Media { get; set; } = new(); public List Location { get; set; } = new(); public bool HasContentWarning { get; set; } = false; diff --git a/PrivaPub/Models/Post/Post.cs b/PrivaPub/Models/Post/Post.cs index a408f45..c8b52e7 100644 --- a/PrivaPub/Models/Post/Post.cs +++ b/PrivaPub/Models/Post/Post.cs @@ -9,6 +9,8 @@ namespace PrivaPub.Models.Post public string GroupId { get; set; } public string Title { get; set; } public string Text { get; set; } + public string ContentHtml { get; set; } + public ContentFormat ContentFormat { get; set; } public List Media { get; set; } = new(); public List Location { get; set; } = new(); public float RangeKm { get; set; } = 5.0f; diff --git a/PrivaPub/PrivaPub.csproj b/PrivaPub/PrivaPub.csproj index 086ef96..b4451a7 100644 --- a/PrivaPub/PrivaPub.csproj +++ b/PrivaPub/PrivaPub.csproj @@ -7,6 +7,7 @@ + diff --git a/PrivaPub/Program.cs b/PrivaPub/Program.cs index a8f055e..459d9d0 100644 --- a/PrivaPub/Program.cs +++ b/PrivaPub/Program.cs @@ -67,6 +67,7 @@ try BsonSerializer.RegisterSerializer(new GuidSerializer(GuidRepresentation.Standard)); var mongoSettings = builder.Configuration.GetSection(nameof(MongoSettings)).Get(); await DB.InitAsync(mongoSettings.Database, MongoClientSettings.FromConnectionString(mongoSettings.ConnectionString)); + await DB.Default.MigrateAsync(); } catch (Exception ex) { diff --git a/PrivaPub/Services/PostsService.cs b/PrivaPub/Services/PostsService.cs index a4496f2..74d16a6 100644 --- a/PrivaPub/Services/PostsService.cs +++ b/PrivaPub/Services/PostsService.cs @@ -6,6 +6,7 @@ using PrivaPub.ClientModels; using PrivaPub.ClientModels.Post; using PrivaPub.Models.Federation; using PrivaPub.Models.Group; +using PrivaPub.Models.Post; using PrivaPub.Resources; using PrivaPub.StaticServices; @@ -79,6 +80,8 @@ namespace PrivaPub.Services GroupId = group?.Id, Title = form.Title, Text = form.Text, + ContentHtml = ActivityPubRenderer.Html(form.Text), + ContentFormat = ContentFormat.Markdown, HasContentWarning = form.HasContentWarning, AnsweringToPostId = form.AnsweringToPostId, ActorURI = author.Uri @@ -213,6 +216,8 @@ namespace PrivaPub.Services GroupUserId = author.Id, GroupId = dmGroup.ID, Text = form.Text, + ContentHtml = ActivityPubRenderer.Html(form.Text), + ContentFormat = ContentFormat.Markdown, HasContentWarning = form.HasContentWarning, ActorURI = author.Uri }; @@ -345,6 +350,7 @@ namespace PrivaPub.Services AnsweringToPostId = post.AnsweringToPostId, Title = post.Title, Text = post.Text, + ContentHtml = post.ContentHtml, HasContentWarning = post.HasContentWarning, IsFederatedCopy = post.IsFederatedCopy, CreationDate = post.CreationDate @@ -360,6 +366,7 @@ namespace PrivaPub.Services AnsweringToPostId = post.AnsweringToPostId, Title = post.Title, Text = post.Text, + ContentHtml = post.ContentHtml, HasContentWarning = post.HasContentWarning, IsFederatedCopy = post.IsFederatedCopy, CreationDate = post.CreationDate