diff --git a/PrivaPub.ClientModels/Post/ViewPost.cs b/PrivaPub.ClientModels/Post/ViewPost.cs index 66890b2..14da3d4 100644 --- a/PrivaPub.ClientModels/Post/ViewPost.cs +++ b/PrivaPub.ClientModels/Post/ViewPost.cs @@ -11,6 +11,7 @@ namespace PrivaPub.ClientModels.Post public string AnsweringToPostId { get; set; } public string Title { get; set; } public string Text { get; set; } + public string ContentHtml { get; set; } public bool HasContentWarning { get; set; } public bool IsFederatedCopy { get; set; } public DateTime CreationDate { get; set; } diff --git a/PrivaPub.Tests/Federation/ContentSanitizerTests.cs b/PrivaPub.Tests/Federation/ContentSanitizerTests.cs new file mode 100644 index 0000000..8dccc89 --- /dev/null +++ b/PrivaPub.Tests/Federation/ContentSanitizerTests.cs @@ -0,0 +1,51 @@ +using PrivaPub.Federation.Objects; + +namespace PrivaPub.Tests.Federation +{ + public class ContentSanitizerTests + { + [Theory] + [InlineData("

hi

", "

hi

")] + [InlineData("

hi

", "

hi

")] + [InlineData("", "")] + [InlineData("x", "x")] + [InlineData("alert(1)\">x", "x")] + [InlineData("x", "x")] + [InlineData("text", "text")] + [InlineData("

x

", "

x

")] + [InlineData("", "")] + [InlineData("

x

", "

x

")] + [InlineData("
ok", "ok")] + [InlineData("

ab

", "

ab

")] + [InlineData("

kept

", "

kept

")] + [InlineData("

Title

", "

Title

")] + [InlineData("x", "x")] + public void Removes_what_is_not_allowed(string input, string expected) => + Assert.Equal(expected, ContentSanitizer.Html(input)); + + [Fact] + public void Keeps_mastodon_mention_and_hashtag_markup() + { + var html = "

@alice " + + "#fedi " + + "https://example.org/a-long

"; + + var sanitized = ContentSanitizer.Html(html); + + Assert.Contains("class=\"h-card\"", sanitized); + Assert.Contains("class=\"u-url mention\"", sanitized); + Assert.Contains("class=\"mention hashtag\"", sanitized); + Assert.Contains("class=\"invisible\"", sanitized); + Assert.Contains("class=\"ellipsis\"", sanitized); + Assert.Contains("href=\"https://m.example/@alice\"", sanitized); + Assert.DoesNotContain("rel=\"tag\"", sanitized); + } + + [Theory] + [InlineData("")] + [InlineData(" ")] + [InlineData(null)] + public void Empty_input_is_empty(string input) => + Assert.Equal(string.Empty, ContentSanitizer.Html(input)); + } +} diff --git a/PrivaPub.Tests/Infrastructure/MigrationTests.cs b/PrivaPub.Tests/Infrastructure/MigrationTests.cs new file mode 100644 index 0000000..c4eb46f --- /dev/null +++ b/PrivaPub.Tests/Infrastructure/MigrationTests.cs @@ -0,0 +1,33 @@ +using MongoDB.Entities; + +using PrivaPub.Infrastructure.Data.Migrations; +using PrivaPub.Models.Post; +using PrivaPub.Tests.Support; + +namespace PrivaPub.Tests.Infrastructure +{ + [Trait("Category", "Integration")] + public class MigrationTests + { + [Fact] + public async Task Stored_remote_content_is_sanitized_and_local_content_rendered() + { + Assert.SkipUnless(MongoFixture.Enabled, MongoFixture.Skip); + var token = TestContext.Current.CancellationToken; + var remote = new Post { IsFederatedCopy = true, Text = "

hi

", ObjectURI = $"https://r.example/{Guid.NewGuid():N}" }; + var local = new Post { Text = "**bold** raw", ObjectURI = $"https://privapub.test/{Guid.NewGuid():N}" }; + await DB.Default.SaveAsync(new[] { remote, local }, token); + + await new _002_sanitize_stored_content().UpgradeAsync(); + + var migratedRemote = await DB.Default.Find().OneAsync(remote.ID, token); + var migratedLocal = await DB.Default.Find().OneAsync(local.ID, token); + Assert.Equal("

hi

", migratedRemote.Text); + Assert.Equal("

hi

", migratedRemote.ContentHtml); + Assert.Equal(ContentFormat.Html, migratedRemote.ContentFormat); + Assert.Equal("**bold** raw", migratedLocal.Text); + Assert.Equal("

bold <b>raw</b>

", migratedLocal.ContentHtml); + Assert.Equal(ContentFormat.Markdown, migratedLocal.ContentFormat); + } + } +} diff --git a/PrivaPub/Federation/Actors/RemoteActorService.cs b/PrivaPub/Federation/Actors/RemoteActorService.cs index c37408a..90b36bf 100644 --- a/PrivaPub/Federation/Actors/RemoteActorService.cs +++ b/PrivaPub/Federation/Actors/RemoteActorService.cs @@ -158,7 +158,7 @@ namespace PrivaPub.Federation.Actors .Match(a => a.ActorURI == actor.Id) .Modify(a => a.UserName, actor.PreferredUsername) .Modify(a => a.Name, actor.Name) - .Modify(a => a.Biography, actor.Summary) + .Modify(a => a.Biography, ContentSanitizer.Html(actor.Summary)) .Modify(a => a.Url, actor.Url) .Modify(a => a.Domain, new Uri(actor.Id).Authority) .Modify(a => a.InboxURL, actor.Inbox) diff --git a/PrivaPub/Federation/Inbox/InboxService.cs b/PrivaPub/Federation/Inbox/InboxService.cs index daaeee3..5991a54 100644 --- a/PrivaPub/Federation/Inbox/InboxService.cs +++ b/PrivaPub/Federation/Inbox/InboxService.cs @@ -2,6 +2,7 @@ using MongoDB.Entities; using PrivaPub.Models.Federation; using PrivaPub.Models.Group; +using PrivaPub.Models.Post; using PrivaPub.Models.User; using PrivaPub.StaticServices; @@ -211,13 +212,16 @@ namespace PrivaPub.Federation.Inbox if (await _dbEntities.Posts.Match(p => p.ObjectURI == objectUri).ExecuteAnyAsync(token)) return new(StatusCodes.Status202Accepted); + var html = ContentSanitizer.Html(Value(note, "content")); var post = new PostEntity { ObjectURI = objectUri, ActorURI = author.ActorURI, GroupId = group?.Id, Title = Value(note, "summary") ?? Value(note, "name"), - Text = Value(note, "content"), + Text = html, + ContentHtml = html, + ContentFormat = ContentFormat.Html, HasContentWarning = note["sensitive"] is JsonValue sensitive && sensitive.TryGetValue(out var s) && s, AnsweringToPostId = await LocalPostId(inReplyTo, token) ?? inReplyTo, IsFederatedCopy = true, @@ -241,13 +245,16 @@ namespace PrivaPub.Federation.Inbox var participants = addressed.Where(a => a != ActivityPubRenderer.Public).Append(author.ActorURI).ToList(); var dmGroup = await FindOrCreateDmGroup(participants, Value(note, "context") ?? Value(note, "conversation"), token); + var dmHtml = ContentSanitizer.Html(Value(note, "content")); var dm = new DmPostEntity { ObjectURI = objectUri, ActorURI = author.ActorURI, GroupId = dmGroup.ID, Title = Value(note, "summary"), - Text = Value(note, "content"), + Text = dmHtml, + ContentHtml = dmHtml, + ContentFormat = ContentFormat.Html, HasContentWarning = note["sensitive"] is JsonValue dmSensitive && dmSensitive.TryGetValue(out var ds) && ds, AnsweringToPostId = inReplyTo, IsFederatedCopy = true, @@ -296,15 +303,17 @@ namespace PrivaPub.Federation.Inbox return new(StatusCodes.Status202Accepted); var objectUri = Id(inner); - var text = Value(inner, "content"); + var text = ContentSanitizer.Html(Value(inner, "content")); await DB.Default.Update() .Match(p => p.ObjectURI == objectUri && p.ActorURI == actor.ActorURI) .Modify(p => p.Text, text) + .Modify(p => p.ContentHtml, text) .Modify(p => p.UpdateDate, DateTime.UtcNow) .ExecuteAsync(token); await DB.Default.Update() .Match(p => p.ObjectURI == objectUri && p.ActorURI == actor.ActorURI) .Modify(p => p.Text, text) + .Modify(p => p.ContentHtml, text) .Modify(p => p.UpdateDate, DateTime.UtcNow) .ExecuteAsync(token); return new(StatusCodes.Status202Accepted); diff --git a/PrivaPub/Federation/Objects/ContentSanitizer.cs b/PrivaPub/Federation/Objects/ContentSanitizer.cs new file mode 100644 index 0000000..df53ec0 --- /dev/null +++ b/PrivaPub/Federation/Objects/ContentSanitizer.cs @@ -0,0 +1,80 @@ +using AngleSharp.Css.Dom; +using AngleSharp.Dom; + +using Ganss.Xss; + +using System.Text.RegularExpressions; + +namespace PrivaPub.Federation.Objects +{ + public static partial class ContentSanitizer + { + static readonly HtmlSanitizer Sanitizer = Build(); + + public static string Html(string html) => + string.IsNullOrWhiteSpace(html) ? string.Empty : Sanitizer.Sanitize(html).Trim(); + + static HtmlSanitizer Build() + { + var sanitizer = new HtmlSanitizer(new HtmlSanitizerOptions + { + AllowedTags = new HashSet(StringComparer.OrdinalIgnoreCase) + { + "p", "br", "span", "a", "abbr", "del", "s", "pre", "blockquote", "code", "b", "strong", "u", "i", "em", + "sub", "sup", "ul", "ol", "li", "ruby", "rt", "rp", "h1", "h2", "h3", "h4", "h5", "h6" + }, + AllowedAttributes = new HashSet(StringComparer.OrdinalIgnoreCase) + { + "href", "rel", "class", "translate", "start", "reversed", "value", "title" + }, + AllowedCssProperties = new HashSet(), + AllowedAtRules = new HashSet(), + AllowedSchemes = new HashSet(StringComparer.OrdinalIgnoreCase) + { + "http", "https", "dat", "dweb", "ipfs", "ipns", "ssb", "gopher", "xmpp", "magnet", "gemini" + }, + UriAttributes = new HashSet(StringComparer.OrdinalIgnoreCase) { "href" } + }) + { + KeepChildNodes = true + }; + foreach (var allowed in new[] { "mention", "hashtag", "ellipsis", "invisible" }) + sanitizer.AllowedClasses.Add(allowed); + sanitizer.RemovingCssClass += (_, e) => e.Cancel = MicroformatClass().IsMatch(e.CssClass); + sanitizer.RemovingTag += (_, e) => + { + if (e.Tag.LocalName is "script" or "style" or "template" or "iframe" or "object" or "embed" or "noscript" or "svg" or "math") + e.Tag.InnerHtml = string.Empty; + }; + sanitizer.PostProcessNode += (_, e) => + { + if (e.Node is not IElement element) + return; + switch (element.LocalName) + { + case "a": + if (!SchemePrefix().IsMatch(element.GetAttribute("href") ?? string.Empty)) + element.RemoveAttribute("href"); + element.SetAttribute("rel", "nofollow noopener noreferrer"); + element.SetAttribute("target", "_blank"); + break; + case "h1" or "h2" or "h3" or "h4" or "h5" or "h6": + var paragraph = e.Document.CreateElement("p"); + var strong = e.Document.CreateElement("strong"); + while (element.FirstChild != default) + strong.AppendChild(element.FirstChild); + paragraph.AppendChild(strong); + e.ReplacementNodes.Add(paragraph); + break; + } + }; + return sanitizer; + } + + [GeneratedRegex("^[a-z][a-z0-9+.-]*:", RegexOptions.IgnoreCase)] + private static partial Regex SchemePrefix(); + + [GeneratedRegex("^(h|p|u|dt|e)-[a-z0-9-]+$")] + private static partial Regex MicroformatClass(); + } +} diff --git a/PrivaPub/Infrastructure/Data/Migrations/_002_sanitize_stored_content.cs b/PrivaPub/Infrastructure/Data/Migrations/_002_sanitize_stored_content.cs new file mode 100644 index 0000000..be8fb4a --- /dev/null +++ b/PrivaPub/Infrastructure/Data/Migrations/_002_sanitize_stored_content.cs @@ -0,0 +1,40 @@ +using MongoDB.Entities; + +using PrivaPub.Federation.Objects; +using PrivaPub.Federation.Rendering; +using PrivaPub.Models.Post; +using PrivaPub.Models.User; + +namespace PrivaPub.Infrastructure.Data.Migrations +{ + public class _002_sanitize_stored_content : IMigration + { + public async Task UpgradeAsync() + { + foreach (var post in await DB.Default.Find().ExecuteAsync()) + { + var html = post.IsFederatedCopy ? ContentSanitizer.Html(post.Text) : ActivityPubRenderer.Html(post.Text); + await DB.Default.Update().MatchID(post.ID) + .Modify(p => p.Text, post.IsFederatedCopy ? html : post.Text) + .Modify(p => p.ContentHtml, html) + .Modify(p => p.ContentFormat, post.IsFederatedCopy ? ContentFormat.Html : ContentFormat.Markdown) + .ExecuteAsync(); + } + + foreach (var post in await DB.Default.Find().ExecuteAsync()) + { + var html = post.IsFederatedCopy ? ContentSanitizer.Html(post.Text) : ActivityPubRenderer.Html(post.Text); + await DB.Default.Update().MatchID(post.ID) + .Modify(p => p.Text, post.IsFederatedCopy ? html : post.Text) + .Modify(p => p.ContentHtml, html) + .Modify(p => p.ContentFormat, post.IsFederatedCopy ? ContentFormat.Html : ContentFormat.Markdown) + .ExecuteAsync(); + } + + foreach (var avatar in await DB.Default.Find().ExecuteAsync()) + await DB.Default.Update().MatchID(avatar.ID) + .Modify(a => a.Biography, ContentSanitizer.Html(avatar.Biography)) + .ExecuteAsync(); + } + } +} diff --git a/PrivaPub/Models/Post/ContentFormat.cs b/PrivaPub/Models/Post/ContentFormat.cs new file mode 100644 index 0000000..8d63605 --- /dev/null +++ b/PrivaPub/Models/Post/ContentFormat.cs @@ -0,0 +1,8 @@ +namespace PrivaPub.Models.Post +{ + public enum ContentFormat + { + Markdown, + Html + } +} diff --git a/PrivaPub/Models/Post/DmPost.cs b/PrivaPub/Models/Post/DmPost.cs index 79c230c..6feb7bd 100644 --- a/PrivaPub/Models/Post/DmPost.cs +++ b/PrivaPub/Models/Post/DmPost.cs @@ -9,6 +9,8 @@ namespace PrivaPub.Models.Post public string GroupId { get; set; } public string Title { get; set; } public string Text { get; set; } + public string ContentHtml { get; set; } + public ContentFormat ContentFormat { get; set; } public List Media { get; set; } = new(); public List Location { get; set; } = new(); public bool HasContentWarning { get; set; } = false; diff --git a/PrivaPub/Models/Post/Post.cs b/PrivaPub/Models/Post/Post.cs index a408f45..c8b52e7 100644 --- a/PrivaPub/Models/Post/Post.cs +++ b/PrivaPub/Models/Post/Post.cs @@ -9,6 +9,8 @@ namespace PrivaPub.Models.Post public string GroupId { get; set; } public string Title { get; set; } public string Text { get; set; } + public string ContentHtml { get; set; } + public ContentFormat ContentFormat { get; set; } public List Media { get; set; } = new(); public List Location { get; set; } = new(); public float RangeKm { get; set; } = 5.0f; diff --git a/PrivaPub/PrivaPub.csproj b/PrivaPub/PrivaPub.csproj index 086ef96..b4451a7 100644 --- a/PrivaPub/PrivaPub.csproj +++ b/PrivaPub/PrivaPub.csproj @@ -7,6 +7,7 @@ + diff --git a/PrivaPub/Program.cs b/PrivaPub/Program.cs index a8f055e..459d9d0 100644 --- a/PrivaPub/Program.cs +++ b/PrivaPub/Program.cs @@ -67,6 +67,7 @@ try BsonSerializer.RegisterSerializer(new GuidSerializer(GuidRepresentation.Standard)); var mongoSettings = builder.Configuration.GetSection(nameof(MongoSettings)).Get(); await DB.InitAsync(mongoSettings.Database, MongoClientSettings.FromConnectionString(mongoSettings.ConnectionString)); + await DB.Default.MigrateAsync(); } catch (Exception ex) { diff --git a/PrivaPub/Services/PostsService.cs b/PrivaPub/Services/PostsService.cs index a4496f2..74d16a6 100644 --- a/PrivaPub/Services/PostsService.cs +++ b/PrivaPub/Services/PostsService.cs @@ -6,6 +6,7 @@ using PrivaPub.ClientModels; using PrivaPub.ClientModels.Post; using PrivaPub.Models.Federation; using PrivaPub.Models.Group; +using PrivaPub.Models.Post; using PrivaPub.Resources; using PrivaPub.StaticServices; @@ -79,6 +80,8 @@ namespace PrivaPub.Services GroupId = group?.Id, Title = form.Title, Text = form.Text, + ContentHtml = ActivityPubRenderer.Html(form.Text), + ContentFormat = ContentFormat.Markdown, HasContentWarning = form.HasContentWarning, AnsweringToPostId = form.AnsweringToPostId, ActorURI = author.Uri @@ -213,6 +216,8 @@ namespace PrivaPub.Services GroupUserId = author.Id, GroupId = dmGroup.ID, Text = form.Text, + ContentHtml = ActivityPubRenderer.Html(form.Text), + ContentFormat = ContentFormat.Markdown, HasContentWarning = form.HasContentWarning, ActorURI = author.Uri }; @@ -345,6 +350,7 @@ namespace PrivaPub.Services AnsweringToPostId = post.AnsweringToPostId, Title = post.Title, Text = post.Text, + ContentHtml = post.ContentHtml, HasContentWarning = post.HasContentWarning, IsFederatedCopy = post.IsFederatedCopy, CreationDate = post.CreationDate @@ -360,6 +366,7 @@ namespace PrivaPub.Services AnsweringToPostId = post.AnsweringToPostId, Title = post.Title, Text = post.Text, + ContentHtml = post.ContentHtml, HasContentWarning = post.HasContentWarning, IsFederatedCopy = post.IsFederatedCopy, CreationDate = post.CreationDate