diff --git a/.gitignore b/.gitignore index ca1c7a3..7bb4345 100644 --- a/.gitignore +++ b/.gitignore @@ -398,3 +398,6 @@ FodyWeavers.xsd # JetBrains Rider *.sln.iml + +PrivaPub/media-store/ +PrivaPub/media-store-proxy/ diff --git a/PrivaPub.Tests/Domain/MediaFlowTests.cs b/PrivaPub.Tests/Domain/MediaFlowTests.cs new file mode 100644 index 0000000..6a5aed8 --- /dev/null +++ b/PrivaPub.Tests/Domain/MediaFlowTests.cs @@ -0,0 +1,102 @@ +using Microsoft.AspNetCore.Http; +using Microsoft.Extensions.Caching.Memory; + +using MongoDB.Entities; + +using NetVips; + +using PrivaPub.Domain.Media; +using PrivaPub.Domain.Statuses; +using PrivaPub.Federation.Rendering; +using PrivaPub.Models.Media; +using PrivaPub.Tests.Support; + +namespace PrivaPub.Tests.Domain +{ + [Trait("Category", "Integration")] + public sealed class MediaFlowTests : IAsyncLifetime + { + Harness _harness; + + public async ValueTask InitializeAsync() + { + Assert.SkipUnless(MongoFixture.Enabled, MongoFixture.Skip); + _harness = await Harness.Start(); + } + + public async ValueTask DisposeAsync() + { + if (_harness != default) + await _harness.DisposeAsync(); + } + + static byte[] Png(int width, int height) + { + using var image = (Image.Black(width, height, bands: 3) + new double[] { 10, 120, 200 }).Cast(Enums.BandFormat.Uchar); + return image.WriteToBuffer(".png"); + } + + static IFormFile Upload(byte[] bytes, string contentType) => + new FormFile(new MemoryStream(bytes), 0, bytes.Length, "file", "picture") { Headers = new HeaderDictionary(), ContentType = contentType }; + + [Fact] + public async Task An_upload_is_attached_once_and_federated_with_its_alt_text() + { + var token = TestContext.Current.CancellationToken; + var (_, alice) = await _harness.Persona("alice"); + var (_, mallory) = await _harness.Persona("mallory"); + + var upload = await _harness.Media.Upload(alice, Upload(Png(300, 200), "image/png"), "a blue square", "0.25,-0.5", token); + Assert.True(upload.Ok); + Assert.True(File.Exists(Path.Combine(_harness.Media.Root, upload.Attachment.FilePath))); + + var stolen = await _harness.Statuses.Publish(mallory, new StatusDraft { Text = "mine", MediaIds = new[] { upload.Attachment.ID } }, token); + Assert.False(stolen.Ok); + + var posted = await _harness.Statuses.Publish(alice, new StatusDraft { Text = "look", MediaIds = new[] { upload.Attachment.ID } }, token); + Assert.True(posted.Ok); + Assert.Equal(posted.Post.ID, (await DB.Default.Find().OneAsync(upload.Attachment.ID, token)).PostId); + + var note = ActivityPubRenderer.Note(posted.Post, alice, default, default); + var attachment = note["attachment"]![0]!; + Assert.Equal("a blue square", attachment["name"]!.GetValue()); + Assert.Equal(300, attachment["width"]!.GetValue()); + Assert.Equal(-0.5f, attachment["focalPoint"]![1]!.GetValue()); + Assert.StartsWith("https://privapub.test/media/files/", attachment["url"]!.GetValue()); + + var reused = await _harness.Statuses.Publish(alice, new StatusDraft { Text = "again", MediaIds = new[] { upload.Attachment.ID } }, token); + Assert.False(reused.Ok); + } + + [Fact] + public async Task Unsupported_files_are_refused() + { + var (_, alice) = await _harness.Persona("alice"); + + var outcome = await _harness.Media.Upload(alice, Upload(System.Text.Encoding.UTF8.GetBytes(""), "image/svg+xml"), default, default, TestContext.Current.CancellationToken); + + Assert.False(outcome.Ok); + Assert.Equal(422, outcome.Status); + } + + [Fact] + public async Task The_proxy_serves_signed_remote_media_and_nothing_else() + { + var token = TestContext.Current.CancellationToken; + var path = $"/files/{Guid.NewGuid():N}.png"; + _harness.Peer.ServeFile(path, Png(10, 10), "image/png"); + var proxy = new MediaProxy(_harness.Local, Peer.Http(), _harness.Media, new StaticOptions(new MediaOptions())); + + var wrapped = proxy.Wrap(_harness.Peer.A + path); + var parts = new Uri(wrapped).AbsolutePath.Split('/'); + var (file, contentType) = await proxy.Fetch(parts[3], parts[4], token); + var (tampered, _) = await proxy.Fetch(parts[3].Replace(parts[3][0], parts[3][0] == 'A' ? 'B' : 'A'), parts[4], token); + + Assert.StartsWith("https://privapub.test/media/proxy/", wrapped); + Assert.Equal("image/png", contentType); + Assert.True(File.Exists(file)); + Assert.StartsWith(_harness.Media.ProxyRoot, file); + Assert.Null(tampered); + } + } +} diff --git a/PrivaPub.Tests/Domain/MediaProcessingTests.cs b/PrivaPub.Tests/Domain/MediaProcessingTests.cs new file mode 100644 index 0000000..65c80af --- /dev/null +++ b/PrivaPub.Tests/Domain/MediaProcessingTests.cs @@ -0,0 +1,72 @@ +using NetVips; + +using PrivaPub.Domain.Media; + +namespace PrivaPub.Tests.Domain +{ + public class MediaProcessingTests + { + static byte[] JpegWithMetadata(int width, int height) + { + using var image = (Image.Black(width, height, bands: 3) + new double[] { 200, 40, 90 }).Cast(Enums.BandFormat.Uchar); + using var tagged = image.Mutate(m => + { + m.Set(GValue.GStrType, "exif-ifd0-ImageDescription", "where I live"); + m.Set(GValue.GStrType, "exif-ifd0-Artist", "Alice Smith"); + m.Set(GValue.GStrType, "exif-ifd2-UserComment", "at home"); + m.Set(GValue.BlobType, "xmp-data", System.Text.Encoding.UTF8.GetBytes("")); + }); + return tagged.WriteToBuffer(".jpg"); + } + + [Fact] + public void Uploaded_images_lose_every_kind_of_metadata() + { + var input = JpegWithMetadata(800, 600); + using (var original = Image.NewFromBuffer(input)) + Assert.Contains("exif-data", original.GetFields()); + + var processed = MediaService.ProcessImage(input, 4096, 640, animated: false); + + using var output = Image.NewFromBuffer(processed.Bytes); + var fields = output.GetFields(); + Assert.DoesNotContain("exif-data", fields); + Assert.DoesNotContain("xmp-data", fields); + Assert.DoesNotContain("iptc-data", fields); + Assert.DoesNotContain(fields, f => f.StartsWith("exif-ifd")); + Assert.Equal((800, 600), (processed.Width, processed.Height)); + Assert.Equal("image/jpeg", processed.ContentType); + using var preview = Image.NewFromBuffer(processed.Preview); + Assert.Equal(640, Math.Max(preview.Width, preview.Height)); + Assert.DoesNotContain("exif-data", preview.GetFields()); + } + + [Fact] + public void Large_images_are_capped() + { + var processed = MediaService.ProcessImage(JpegWithMetadata(5000, 2500), 4096, 640, animated: false); + + Assert.Equal((4096, 2048), (processed.Width, processed.Height)); + } + + [Fact] + public void A_blurhash_is_well_formed() + { + var processed = MediaService.ProcessImage(JpegWithMetadata(64, 64), 4096, 640, animated: false); + + Assert.Equal(28, processed.Blurhash.Length); + Assert.Equal('L', processed.Blurhash[0]); + } + + [Fact] + public void A_white_image_encodes_white_as_its_average() + { + var pixels = Enumerable.Range(0, 16 * 16).SelectMany(_ => new byte[] { 255, 255, 255 }).ToArray(); + + var hash = Blurhash.Encode(pixels, 16, 16); + + Assert.Equal(28, hash.Length); + Assert.Equal("TSUA", hash[2..6]); + } + } +} diff --git a/PrivaPub.Tests/Support/Harness.cs b/PrivaPub.Tests/Support/Harness.cs index 29078ed..42824b0 100644 --- a/PrivaPub.Tests/Support/Harness.cs +++ b/PrivaPub.Tests/Support/Harness.cs @@ -5,6 +5,7 @@ using Microsoft.Extensions.Logging.Abstractions; using MongoDB.Entities; using PrivaPub.Domain.Content; +using PrivaPub.Domain.Media; using PrivaPub.Domain.Relationships; using PrivaPub.Domain.Social; using PrivaPub.Domain.Statuses; @@ -58,7 +59,9 @@ namespace PrivaPub.Tests.Support Follows = new FollowService(Db, Local, Remote, Delivery, new KeyLocalizer(), NullLogger.Instance); Content = new ContentRenderer(Local, Remote); Outbox = new OutboxPublisher(Db, Local, Delivery); - Statuses = new StatusService(Db, Local, Remote, Delivery, Content, Outbox, Fanout); + Media = new MediaService(new StaticOptions(new MediaOptions { Root = Path.Combine(Path.GetTempPath(), $"privapub-media-{Guid.NewGuid():N}") }), + Local, default, NullLogger.Instance); + Statuses = new StatusService(Db, Local, Remote, Delivery, Content, Outbox, Fanout, Media); Posts = new PostsService(Db, Local, Statuses, new KeyLocalizer(), NullLogger.Instance); Timelines = new TimelineService(Db, new KeyLocalizer()); Relationships = new RelationshipService(Db, Follows, Delivery); @@ -78,6 +81,7 @@ namespace PrivaPub.Tests.Support public OutboxPublisher Outbox { get; } public PostsService Posts { get; } public StatusService Statuses { get; } + public MediaService Media { get; } public Fanout Fanout { get; } public RemotePosts RemotePosts { get; } public TimelineService Timelines { get; } diff --git a/PrivaPub.Tests/Support/Peer.cs b/PrivaPub.Tests/Support/Peer.cs index 5172a77..6923324 100644 --- a/PrivaPub.Tests/Support/Peer.cs +++ b/PrivaPub.Tests/Support/Peer.cs @@ -19,6 +19,7 @@ namespace PrivaPub.Tests.Support readonly WebApplication _app; readonly ConcurrentDictionary _documents = new(); readonly ConcurrentDictionary _answers = new(); + readonly ConcurrentDictionary _files = new(); public int Port { get; } public string A => $"http://127.0.0.1:{Port}"; @@ -41,6 +42,12 @@ namespace PrivaPub.Tests.Support { peer.Requests.Enqueue(new(context.Request.Method, context.Request.Path, context.Request.Headers["Signature"].ToString())); var key = context.Request.Path.Value; + if (peer._files.TryGetValue(key, out var file)) + { + context.Response.ContentType = file.ContentType; + await context.Response.Body.WriteAsync(file.Bytes); + return; + } if (peer._answers.TryGetValue(key, out var answer)) { if (answer.Delay > TimeSpan.Zero) @@ -63,6 +70,8 @@ namespace PrivaPub.Tests.Support public void Serve(string path, string json) => _documents[path] = json; + public void ServeFile(string path, byte[] bytes, string contentType) => _files[path] = (bytes, contentType); + public void Answer(string path, int status, TimeSpan delay = default) => _answers[path] = (status, delay); public static FederationHttp Http(IMemoryCache cache = default, IDomainBlocks blocks = default) diff --git a/PrivaPub/Api/Mastodon/Controllers/AccountsController.cs b/PrivaPub/Api/Mastodon/Controllers/AccountsController.cs index d2a0f41..25d8cbd 100644 --- a/PrivaPub/Api/Mastodon/Controllers/AccountsController.cs +++ b/PrivaPub/Api/Mastodon/Controllers/AccountsController.cs @@ -5,6 +5,7 @@ using MongoDB.Entities; using PrivaPub.Api.Mastodon.Entities; using PrivaPub.Api.Mastodon.Infrastructure; using PrivaPub.Api.Mastodon.Mappers; +using PrivaPub.Domain.Media; using PrivaPub.Domain.Privacy; using PrivaPub.Domain.Relationships; using PrivaPub.Domain.Social; @@ -45,10 +46,19 @@ namespace PrivaPub.Api.Mastodon.Controllers [HttpGet("/api/v1/accounts/verify_credentials"), Scope("read:accounts")] public async Task VerifyCredentials(CancellationToken token) => Json(await _mapper.Local(Me, withSource: true, token)); - [HttpPatch("/api/v1/accounts/update_credentials"), Scope("write:accounts")] - public async Task UpdateCredentials(CancellationToken token) + [HttpPatch("/api/v1/accounts/update_credentials"), Scope("write:accounts"), RequestSizeLimit(20 * 1024 * 1024), + RequestFormLimits(MultipartBodyLengthLimit = 20 * 1024 * 1024)] + public async Task UpdateCredentials([FromServices] IMediaService media, CancellationToken token) { var avatar = await _dbEntities.Avatars.MatchID(Me.Id).ExecuteFirstAsync(token); + if (Request.HasFormContentType) + { + var form = await Request.ReadFormAsync(token); + if (form.Files["avatar"] is { } picture && await media.ProfileImage(picture, 400, 400, token) is { } pictureUrl) + avatar.PictureURL = pictureUrl; + if (form.Files["header"] is { } header && await media.ProfileImage(header, 1500, 500, token) is { } headerUrl) + avatar.ThumbnailURL = headerUrl; + } if (Params.Has("display_name")) avatar.Name = Params.Get("display_name")?.Trim(); if (Params.Has("note")) diff --git a/PrivaPub/Api/Mastodon/Controllers/MediaController.cs b/PrivaPub/Api/Mastodon/Controllers/MediaController.cs new file mode 100644 index 0000000..5244771 --- /dev/null +++ b/PrivaPub/Api/Mastodon/Controllers/MediaController.cs @@ -0,0 +1,91 @@ +using Microsoft.AspNetCore.Authorization; +using Microsoft.AspNetCore.Mvc; + +using MongoDB.Entities; + +using PrivaPub.Api.Mastodon.Infrastructure; +using PrivaPub.Domain.Media; +using PrivaPub.Models.Media; + +using System.Globalization; + +namespace PrivaPub.Api.Mastodon.Controllers +{ + public class MediaController : MastodonController + { + const long UploadLimit = 100L * 1024 * 1024; + + readonly IMediaService _media; + readonly IMediaProxy _proxy; + + public MediaController(IMediaService media, IMediaProxy proxy) + { + _media = media; + _proxy = proxy; + } + + [HttpPost("/api/v1/media"), HttpPost("/api/v2/media"), Scope("write:media"), RequestSizeLimit(UploadLimit), + RequestFormLimits(MultipartBodyLengthLimit = UploadLimit)] + public async Task Upload(CancellationToken token) + { + if (!Request.HasFormContentType) + return Error(StatusCodes.Status422UnprocessableEntity, "Validation failed: File can't be blank"); + var form = await Request.ReadFormAsync(token); + var outcome = await _media.Upload(Me, form.Files["file"], form["description"], form["focus"], token); + return outcome.Ok ? Json(View(outcome.Attachment)) : Error(outcome.Status, outcome.Error); + } + + [HttpGet("/api/v1/media/{id}"), Scope("write:media")] + public async Task Get(string id, CancellationToken token) + { + var attachment = await DB.Default.Find().Match(m => m.ID == id && m.OwnerAvatarId == MyId).ExecuteFirstAsync(token); + return attachment == default ? NotFoundError() : Json(View(attachment)); + } + + [HttpPut("/api/v1/media/{id}"), Scope("write:media")] + public async Task Update(string id, CancellationToken token) + { + var attachment = await DB.Default.Find().Match(m => m.ID == id && m.OwnerAvatarId == MyId).ExecuteFirstAsync(token); + if (attachment == default) + return NotFoundError(); + if (Params.Has("description")) + attachment.Description = Params.Get("description")?.Trim() is { Length: > 0 } description ? description[..Math.Min(description.Length, 1500)] : default; + if (Params.Get("focus")?.Split(',') is [var x, var y] + && float.TryParse(x, NumberStyles.Float, CultureInfo.InvariantCulture, out var fx) && float.TryParse(y, NumberStyles.Float, CultureInfo.InvariantCulture, out var fy)) + attachment.Focus = new[] { Math.Clamp(fx, -1, 1), Math.Clamp(fy, -1, 1) }; + await DB.Default.SaveAsync(attachment, token); + return Json(View(attachment)); + } + + [HttpGet("/media/proxy/{signature}/{encoded}"), AllowAnonymous, ApiExplorerSettings(IgnoreApi = true)] + public async Task Proxy(string signature, string encoded, CancellationToken token) + { + var (path, contentType) = await _proxy.Fetch(signature, encoded, token); + if (path == default) + return NotFound(); + Response.Headers["X-Content-Type-Options"] = "nosniff"; + Response.Headers["Content-Security-Policy"] = "default-src 'none'; sandbox"; + Response.Headers["Cache-Control"] = "public, max-age=604800"; + return PhysicalFile(path, contentType); + } + + object View(MediaAttachment attachment) => new + { + id = attachment.ID, + type = attachment.Kind, + url = _media.Url(attachment.FilePath), + preview_url = _media.Url(attachment.PreviewPath ?? attachment.FilePath), + remote_url = default(string), + text_url = default(string), + meta = new + { + original = attachment.Width.HasValue && attachment.Height > 0 + ? new { width = attachment.Width, height = attachment.Height, size = $"{attachment.Width}x{attachment.Height}", aspect = (double)attachment.Width / attachment.Height.Value } + : default, + focus = attachment.Focus is { Length: 2 } ? new { x = attachment.Focus[0], y = attachment.Focus[1] } : default + }, + description = attachment.Description, + blurhash = attachment.Blurhash + }; + } +} diff --git a/PrivaPub/Api/Mastodon/Controllers/StatusesController.cs b/PrivaPub/Api/Mastodon/Controllers/StatusesController.cs index 7d98fc1..ae54d4e 100644 --- a/PrivaPub/Api/Mastodon/Controllers/StatusesController.cs +++ b/PrivaPub/Api/Mastodon/Controllers/StatusesController.cs @@ -44,8 +44,6 @@ namespace PrivaPub.Api.Mastodon.Controllers if (existing != default) return Json(await _mapper.Status(existing, MyId, token)); } - if (Params.List("media_ids").Count > 0) - return Error(StatusCodes.Status422UnprocessableEntity, "Validation failed: Media attachments are not supported yet"); if (Params.Has("poll[options]")) return Error(StatusCodes.Status422UnprocessableEntity, "Validation failed: Polls are not supported yet"); @@ -57,7 +55,8 @@ namespace PrivaPub.Api.Mastodon.Controllers Sensitive = Params.Bool("sensitive") ?? Me.Settings.DefaultSensitive, Visibility = Visibility(Params.Get("visibility") ?? Me.Settings.DefaultVisibility), InReplyTo = Params.Get("in_reply_to_id"), - Language = Params.Get("language") ?? Me.Settings.DefaultLanguage + Language = Params.Get("language") ?? Me.Settings.DefaultLanguage, + MediaIds = Params.List("media_ids") }, token); if (!outcome.Ok) return Error(outcome.Status, outcome.Error); @@ -88,15 +87,14 @@ namespace PrivaPub.Api.Mastodon.Controllers [HttpPut("/api/v1/statuses/{id}"), Scope("write:statuses")] public async Task Edit(string id, CancellationToken token) { - if (Params.List("media_ids").Count > 0) - return Error(StatusCodes.Status422UnprocessableEntity, "Validation failed: Media attachments are not supported yet"); var outcome = await _statuses.Edit(Me, id, new StatusDraft { Text = Params.Get("status"), PlainText = true, SpoilerText = Params.Get("spoiler_text"), Sensitive = Params.Bool("sensitive") ?? false, - Language = Params.Get("language") + Language = Params.Get("language"), + MediaIds = Params.Has("media_ids") ? Params.List("media_ids") : default }, token); return outcome.Ok ? Json(await _mapper.Status(outcome.Post, MyId, token)) : Error(outcome.Status, outcome.Error); } diff --git a/PrivaPub/Api/Mastodon/Mappers/MastodonMapper.cs b/PrivaPub/Api/Mastodon/Mappers/MastodonMapper.cs index 59591fc..b9193a0 100644 --- a/PrivaPub/Api/Mastodon/Mappers/MastodonMapper.cs +++ b/PrivaPub/Api/Mastodon/Mappers/MastodonMapper.cs @@ -2,6 +2,7 @@ using MongoDB.Entities; using PrivaPub.Api.Mastodon.Entities; using PrivaPub.Api.Mastodon.Infrastructure; +using PrivaPub.Domain.Media; using PrivaPub.Federation.Actors; using PrivaPub.Federation.Rendering; using PrivaPub.Models.Federation; @@ -21,13 +22,17 @@ namespace PrivaPub.Api.Mastodon.Mappers { readonly DbEntities _dbEntities; readonly ILocalActorService _localActors; + readonly IMediaProxy _proxy; - public MastodonMapper(DbEntities dbEntities, ILocalActorService localActors) + public MastodonMapper(DbEntities dbEntities, ILocalActorService localActors, IMediaProxy proxy = default) { _dbEntities = dbEntities; _localActors = localActors; + _proxy = proxy; } + string Proxied(string url) => _proxy == default ? url : _proxy.Wrap(url); + string MissingAvatar => $"{_localActors.BaseAddress}/media/missing-avatar.png"; string MissingHeader => $"{_localActors.BaseAddress}/media/missing-header.png"; @@ -101,10 +106,10 @@ namespace PrivaPub.Api.Mastodon.Mappers Note = foreign.Biography ?? string.Empty, Url = foreign.Url ?? foreign.ActorURI, Uri = foreign.ActorURI, - Avatar = foreign.PictureURL ?? MissingAvatar, - AvatarStatic = foreign.PictureURL ?? MissingAvatar, - Header = foreign.ThumbnailURL ?? MissingHeader, - HeaderStatic = foreign.ThumbnailURL ?? MissingHeader + Avatar = Proxied(foreign.PictureURL) ?? MissingAvatar, + AvatarStatic = Proxied(foreign.PictureURL) ?? MissingAvatar, + Header = Proxied(foreign.ThumbnailURL) ?? MissingHeader, + HeaderStatic = Proxied(foreign.ThumbnailURL) ?? MissingHeader }; public async Task Account(string id, CancellationToken token) => @@ -244,9 +249,9 @@ namespace PrivaPub.Api.Mastodon.Mappers return new StatusMention { Id = account.ID, Username = account.UserName, Acct = $"{account.UserName}@{account.Domain}", Url = account.Url ?? account.ActorURI }; } - static MediaAttachment Media(PostMedia media) => new() + MediaAttachment Media(PostMedia media) => new() { - Id = media.Id.ToString("N"), + Id = media.AttachmentId ?? media.Id.ToString("N"), Type = media.ContentType switch { { } type when type.StartsWith("image/gif") => "gifv", @@ -255,14 +260,18 @@ namespace PrivaPub.Api.Mastodon.Mappers { } type when type.StartsWith("audio/") => "audio", _ => "unknown" }, - Url = media.URL ?? media.RemoteURL, - PreviewUrl = media.URL ?? media.RemoteURL, + Url = media.URL ?? Proxied(media.RemoteURL), + PreviewUrl = media.PreviewURL ?? media.URL ?? Proxied(media.RemoteURL), RemoteUrl = media.RemoteURL, Description = media.Description, Blurhash = media.Blurhash, - Meta = media.Width.HasValue && media.Height.HasValue - ? new { original = new { width = media.Width, height = media.Height, aspect = (double)media.Width / media.Height.Value } } - : default + Meta = new + { + original = media.Width.HasValue && media.Height > 0 + ? new { width = media.Width, height = media.Height, size = $"{media.Width}x{media.Height}", aspect = (double)media.Width / media.Height.Value } + : default, + focus = media.Focus is { Length: 2 } ? new { x = media.Focus[0], y = media.Focus[1] } : default + } }; } } diff --git a/PrivaPub/Domain/Media/Blurhash.cs b/PrivaPub/Domain/Media/Blurhash.cs new file mode 100644 index 0000000..1ca75fe --- /dev/null +++ b/PrivaPub/Domain/Media/Blurhash.cs @@ -0,0 +1,91 @@ +namespace PrivaPub.Domain.Media +{ + public static class Blurhash + { + const string Characters = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz#$%*+,-.:;=?@[]^_{|}~"; + + public static string Encode(byte[] rgb, int width, int height, int componentsX = 4, int componentsY = 3) + { + var factors = new double[componentsY, componentsX, 3]; + for (var j = 0; j < componentsY; j++) + for (var i = 0; i < componentsX; i++) + { + var normalisation = i == 0 && j == 0 ? 1 : 2; + double r = 0, g = 0, b = 0; + for (var y = 0; y < height; y++) + for (var x = 0; x < width; x++) + { + var basis = normalisation * Math.Cos(Math.PI * i * x / width) * Math.Cos(Math.PI * j * y / height); + var offset = (y * width + x) * 3; + r += basis * ToLinear(rgb[offset]); + g += basis * ToLinear(rgb[offset + 1]); + b += basis * ToLinear(rgb[offset + 2]); + } + var scale = 1.0 / (width * height); + factors[j, i, 0] = r * scale; + factors[j, i, 1] = g * scale; + factors[j, i, 2] = b * scale; + } + + var hash = new System.Text.StringBuilder(); + hash.Append(Base83((componentsX - 1) + (componentsY - 1) * 9, 1)); + + double maximum = 0; + for (var j = 0; j < componentsY; j++) + for (var i = 0; i < componentsX; i++) + if (i != 0 || j != 0) + for (var c = 0; c < 3; c++) + maximum = Math.Max(maximum, Math.Abs(factors[j, i, c])); + double maximumValue; + if (componentsX * componentsY > 1) + { + var quantised = (int)Math.Max(0, Math.Min(82, Math.Floor(maximum * 166 - 0.5))); + maximumValue = (quantised + 1) / 166.0; + hash.Append(Base83(quantised, 1)); + } + else + { + maximumValue = 1; + hash.Append(Base83(0, 1)); + } + + var dc = (ToSrgb(factors[0, 0, 0]) << 16) + (ToSrgb(factors[0, 0, 1]) << 8) + ToSrgb(factors[0, 0, 2]); + hash.Append(Base83(dc, 4)); + for (var j = 0; j < componentsY; j++) + for (var i = 0; i < componentsX; i++) + { + if (i == 0 && j == 0) + continue; + int Quantise(double value) => (int)Math.Max(0, Math.Min(18, Math.Floor(SignPow(value / maximumValue, 0.5) * 9 + 9.5))); + var ac = Quantise(factors[j, i, 0]) * 19 * 19 + Quantise(factors[j, i, 1]) * 19 + Quantise(factors[j, i, 2]); + hash.Append(Base83(ac, 2)); + } + return hash.ToString(); + } + + static double ToLinear(byte value) + { + var v = value / 255.0; + return v <= 0.04045 ? v / 12.92 : Math.Pow((v + 0.055) / 1.055, 2.4); + } + + static int ToSrgb(double value) + { + var v = Math.Max(0, Math.Min(1, value)); + return v <= 0.0031308 ? (int)(v * 12.92 * 255 + 0.5) : (int)((1.055 * Math.Pow(v, 1 / 2.4) - 0.055) * 255 + 0.5); + } + + static double SignPow(double value, double exponent) => Math.CopySign(Math.Pow(Math.Abs(value), exponent), value); + + static string Base83(int value, int length) + { + var result = new char[length]; + for (var i = 1; i <= length; i++) + { + var digit = value / (int)Math.Pow(83, length - i) % 83; + result[i - 1] = Characters[digit]; + } + return new string(result); + } + } +} diff --git a/PrivaPub/Domain/Media/MediaOptions.cs b/PrivaPub/Domain/Media/MediaOptions.cs new file mode 100644 index 0000000..3200d37 --- /dev/null +++ b/PrivaPub/Domain/Media/MediaOptions.cs @@ -0,0 +1,13 @@ +namespace PrivaPub.Domain.Media +{ + public class MediaOptions + { + public string Root { get; set; } + public long MaxImageBytes { get; set; } = 16 * 1024 * 1024; + public long MaxVideoBytes { get; set; } = 99 * 1024 * 1024; + public long MaxProxiedBytes { get; set; } = 40 * 1024 * 1024; + public long ProxyCacheBytes { get; set; } = 5L * 1024 * 1024 * 1024; + public int MaxImageSide { get; set; } = 4096; + public int PreviewSide { get; set; } = 640; + } +} diff --git a/PrivaPub/Domain/Media/MediaProxy.cs b/PrivaPub/Domain/Media/MediaProxy.cs new file mode 100644 index 0000000..c5c3a22 --- /dev/null +++ b/PrivaPub/Domain/Media/MediaProxy.cs @@ -0,0 +1,160 @@ +using Microsoft.Extensions.Options; + +using MongoDB.Entities; + +using PrivaPub.Federation.Actors; +using PrivaPub.Infrastructure.Http; +using PrivaPub.Models.Media; + +using System.Security.Cryptography; +using System.Text; + +namespace PrivaPub.Domain.Media +{ + public interface IMediaProxy + { + string Wrap(string remoteUrl); + Task<(string Path, string ContentType)> Fetch(string signature, string encodedUrl, CancellationToken token); + } + + public class MediaProxy : IMediaProxy + { + readonly ILocalActorService _localActors; + readonly IFederationHttp _http; + readonly IMediaService _media; + readonly IOptionsMonitor _options; + byte[] _key; + + public MediaProxy(ILocalActorService localActors, IFederationHttp http, IMediaService media, IOptionsMonitor options) + { + _localActors = localActors; + _http = http; + _media = media; + _options = options; + } + + byte[] Key => _key ??= LoadKey(); + + public string Wrap(string remoteUrl) + { + if (string.IsNullOrEmpty(remoteUrl) || remoteUrl.StartsWith(_localActors.BaseAddress + "/", StringComparison.OrdinalIgnoreCase)) + return remoteUrl; + var encoded = Base64Url(Encoding.UTF8.GetBytes(remoteUrl)); + return $"{_localActors.BaseAddress}/media/proxy/{Sign(remoteUrl)}/{encoded}"; + } + + public async Task<(string Path, string ContentType)> Fetch(string signature, string encodedUrl, CancellationToken token) + { + string url; + try + { + url = Encoding.UTF8.GetString(FromBase64Url(encodedUrl)); + } + catch (FormatException) + { + return default; + } + if (!CryptographicOperations.FixedTimeEquals(Encoding.ASCII.GetBytes(signature ?? string.Empty), Encoding.ASCII.GetBytes(Sign(url)))) + return default; + + var name = Convert.ToHexStringLower(SHA256.HashData(Encoding.UTF8.GetBytes(url))); + var directory = System.IO.Path.Combine(_media.ProxyRoot, name[..2]); + var path = System.IO.Path.Combine(directory, name); + var typePath = path + ".type"; + if (File.Exists(path) && File.Exists(typePath)) + { + File.SetLastWriteTimeUtc(path, DateTime.UtcNow); + return (path, await File.ReadAllTextAsync(typePath, token)); + } + + var (bytes, contentType) = await _http.GetMedia(url, _options.CurrentValue.MaxProxiedBytes, token); + if (bytes == default) + return default; + Directory.CreateDirectory(directory); + await File.WriteAllBytesAsync(path, bytes, token); + await File.WriteAllTextAsync(typePath, contentType, token); + return (path, contentType); + } + + string Sign(string url) => Base64Url(HMACSHA256.HashData(Key, Encoding.UTF8.GetBytes(url))[..16]); + + static byte[] LoadKey() + { + var secret = DB.Default.Find().ExecuteFirstAsync().GetAwaiter().GetResult(); + if (secret == default) + { + secret = new MediaSecret { Key = Convert.ToBase64String(RandomNumberGenerator.GetBytes(32)) }; + DB.Default.SaveAsync(secret).GetAwaiter().GetResult(); + secret = DB.Default.Find().ExecuteFirstAsync().GetAwaiter().GetResult(); + } + return Convert.FromBase64String(secret.Key); + } + + static string Base64Url(byte[] bytes) => Convert.ToBase64String(bytes).TrimEnd('=').Replace('+', '-').Replace('/', '_'); + + static byte[] FromBase64Url(string value) + { + var padded = value.Replace('-', '+').Replace('_', '/'); + return Convert.FromBase64String(padded + new string('=', (4 - padded.Length % 4) % 4)); + } + } + + public class MediaJanitor : BackgroundService + { + static readonly TimeSpan Interval = TimeSpan.FromHours(1); + static readonly TimeSpan UnattachedLifetime = TimeSpan.FromDays(1); + + readonly IMediaService _media; + readonly IOptionsMonitor _options; + readonly ILogger _logger; + + public MediaJanitor(IMediaService media, IOptionsMonitor options, ILogger logger) + { + _media = media; + _options = options; + _logger = logger; + } + + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + while (!stoppingToken.IsCancellationRequested) + { + try + { + await Task.Delay(Interval, stoppingToken); + var cutoff = DateTime.UtcNow - UnattachedLifetime; + foreach (var stale in await DB.Default.Find().Match(m => m.PostId == null && m.CreatedAt < cutoff).Limit(500).ExecuteAsync(stoppingToken)) + await _media.Delete(stale); + TrimProxyCache(); + } + catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) + { + return; + } + catch (Exception ex) + { + _logger.LogWarning(ex, "{Service} pass failed", nameof(MediaJanitor)); + } + } + } + + void TrimProxyCache() + { + var directory = new DirectoryInfo(_media.ProxyRoot); + if (!directory.Exists) + return; + var files = directory.EnumerateFiles("*", SearchOption.AllDirectories).Where(f => f.Extension != ".type").OrderBy(f => f.LastWriteTimeUtc).ToList(); + var total = files.Sum(f => f.Length); + foreach (var file in files) + { + if (total <= _options.CurrentValue.ProxyCacheBytes) + break; + total -= file.Length; + file.Delete(); + var type = new FileInfo(file.FullName + ".type"); + if (type.Exists) + type.Delete(); + } + } + } +} diff --git a/PrivaPub/Domain/Media/MediaService.cs b/PrivaPub/Domain/Media/MediaService.cs new file mode 100644 index 0000000..444fe2a --- /dev/null +++ b/PrivaPub/Domain/Media/MediaService.cs @@ -0,0 +1,263 @@ +using FFMpegCore; + +using Microsoft.Extensions.Options; + +using MongoDB.Entities; + +using NetVips; + +using PrivaPub.Federation.Actors; +using PrivaPub.Models.Media; + +using System.Globalization; +using System.Security.Cryptography; + +namespace PrivaPub.Domain.Media +{ + public sealed record MediaOutcome(MediaAttachment Attachment, int Status = StatusCodes.Status200OK, string Error = default) + { + public bool Ok => Error == default; + public static MediaOutcome Fail(int status, string error) => new(default, status, error); + } + + public sealed record ProcessedImage(byte[] Bytes, string Extension, string ContentType, int Width, int Height, byte[] Preview, string Blurhash); + + public interface IMediaService + { + string Root { get; } + string ProxyRoot { get; } + string Url(string relativePath); + Task Upload(LocalActor owner, IFormFile file, string description, string focus, CancellationToken token); + Task ProfileImage(IFormFile file, int width, int height, CancellationToken token); + Task Delete(MediaAttachment attachment); + } + + public class MediaService : IMediaService + { + static readonly string[] ImageTypes = { "image/jpeg", "image/png", "image/gif", "image/webp", "image/heic", "image/heif", "image/avif" }; + static readonly Dictionary AvTypes = new() + { + ["video/mp4"] = "mp4", ["video/quicktime"] = "mp4", ["video/webm"] = "webm", + ["audio/mpeg"] = "mp3", ["audio/mp3"] = "mp3", ["audio/ogg"] = "ogg", ["audio/wav"] = "wav", ["audio/x-wav"] = "wav", + ["audio/mp4"] = "m4a", ["audio/x-m4a"] = "m4a", ["audio/flac"] = "flac", ["audio/webm"] = "webm" + }; + + readonly IOptionsMonitor _options; + readonly ILocalActorService _localActors; + readonly IWebHostEnvironment _environment; + readonly ILogger _logger; + + public MediaService(IOptionsMonitor options, ILocalActorService localActors, IWebHostEnvironment environment, ILogger logger) + { + _options = options; + _localActors = localActors; + _environment = environment; + _logger = logger; + } + + public string Root => Path.GetFullPath(_options.CurrentValue.Root ?? Path.Combine(_environment.ContentRootPath, "media-store")); + + public string ProxyRoot => Root.TrimEnd(Path.DirectorySeparatorChar) + "-proxy"; + + public string Url(string relativePath) => relativePath == default ? default : $"{_localActors.BaseAddress}/media/files/{relativePath.Replace('\\', '/')}"; + + public async Task Upload(LocalActor owner, IFormFile file, string description, string focus, CancellationToken token) + { + if (file == default || file.Length == 0) + return MediaOutcome.Fail(StatusCodes.Status422UnprocessableEntity, "Validation failed: File can't be blank"); + var options = _options.CurrentValue; + var contentType = file.ContentType?.Split(';')[0].Trim().ToLowerInvariant(); + + var attachment = new MediaAttachment + { + OwnerAvatarId = owner.Id, + Description = Clean(description, 1500), + Focus = Focus(focus) + }; + if (ImageTypes.Contains(contentType)) + { + if (file.Length > options.MaxImageBytes) + return MediaOutcome.Fail(StatusCodes.Status422UnprocessableEntity, "Validation failed: File is too big"); + await using var stream = file.OpenReadStream(); + using var buffer = new MemoryStream(); + await stream.CopyToAsync(buffer, token); + ProcessedImage processed; + try + { + processed = ProcessImage(buffer.ToArray(), options.MaxImageSide, options.PreviewSide, contentType == "image/gif"); + } + catch (VipsException ex) + { + _logger.LogInformation("Refused an image upload: {Error}", ex.Message); + return MediaOutcome.Fail(StatusCodes.Status422UnprocessableEntity, "Validation failed: The file is not a readable image"); + } + attachment.Kind = contentType == "image/gif" ? "gifv" : "image"; + attachment.ContentType = processed.ContentType; + attachment.FilePath = await Save(processed.Bytes, processed.Extension, token); + attachment.PreviewPath = await Save(processed.Preview, "jpg", token); + attachment.Width = processed.Width; + attachment.Height = processed.Height; + attachment.Blurhash = processed.Blurhash; + attachment.Size = processed.Bytes.Length; + } + else if (contentType != default && AvTypes.TryGetValue(contentType, out var extension)) + { + if (file.Length > options.MaxVideoBytes) + return MediaOutcome.Fail(StatusCodes.Status422UnprocessableEntity, "Validation failed: File is too big"); + var outcome = await ProcessAv(file, extension, contentType, attachment, token); + if (!outcome.Ok) + return outcome; + } + else + return MediaOutcome.Fail(StatusCodes.Status422UnprocessableEntity, "Validation failed: File type is not supported"); + + await DB.Default.SaveAsync(attachment, token); + return new MediaOutcome(attachment); + } + + public async Task ProfileImage(IFormFile file, int width, int height, CancellationToken token) + { + if (file == default || file.Length == 0 || file.Length > _options.CurrentValue.MaxImageBytes) + return default; + await using var stream = file.OpenReadStream(); + using var buffer = new MemoryStream(); + await stream.CopyToAsync(buffer, token); + try + { + using var image = Image.ThumbnailBuffer(buffer.ToArray(), width, height: height, crop: Enums.Interesting.Centre, size: Enums.Size.Down); + using var flat = Flatten(image); + return Url(await Save(flat.WriteToBuffer(".jpg[Q=85,keep=none]"), "jpg", token)); + } + catch (VipsException) + { + return default; + } + } + + public Task Delete(MediaAttachment attachment) + { + foreach (var path in new[] { attachment.FilePath, attachment.PreviewPath }.Where(p => p != default)) + { + var full = Path.Combine(Root, path); + if (File.Exists(full)) + File.Delete(full); + } + return DB.Default.DeleteAsync(attachment.ID); + } + + public static ProcessedImage ProcessImage(byte[] input, int maxSide, int previewSide, bool animated) + { + using var loaded = animated ? Image.NewFromBuffer(input, kwargs: new VOption { { "n", -1 } }) : Image.NewFromBuffer(input); + using var rotated = animated ? loaded.Copy() : loaded.Autorot(); + var pageHeight = animated && rotated.Contains("page-height") ? (int)rotated.Get("page-height") : rotated.Height; + var longest = Math.Max(rotated.Width, pageHeight); + + byte[] bytes; + string extension, contentType; + int width, height; + if (animated) + { + bytes = rotated.WriteToBuffer(".gif"); + (extension, contentType, width, height) = ("gif", "image/gif", rotated.Width, pageHeight); + } + else + { + using var resized = longest > maxSide ? rotated.ThumbnailImage(maxSide, height: maxSide, size: Enums.Size.Down) : rotated.Copy(); + var keepsAlpha = resized.HasAlpha(); + bytes = keepsAlpha ? resized.WriteToBuffer(".png[keep=none]") : resized.WriteToBuffer(".jpg[Q=88,keep=none]"); + (extension, contentType, width, height) = keepsAlpha ? ("png", "image/png", resized.Width, resized.Height) : ("jpg", "image/jpeg", resized.Width, resized.Height); + } + + using var firstFrame = animated ? rotated.Crop(0, 0, rotated.Width, pageHeight) : rotated.Copy(); + using var preview = firstFrame.ThumbnailImage(previewSide, height: previewSide, size: Enums.Size.Down); + using var previewFlat = Flatten(preview); + var previewBytes = previewFlat.WriteToBuffer(".jpg[Q=80,keep=none]"); + + using var tiny = firstFrame.ThumbnailImage(32, height: 32); + using var tinyFlat = Flatten(tiny); + var pixels = tinyFlat.WriteToMemory(); + return new ProcessedImage(bytes, extension, contentType, width, height, previewBytes, Blurhash.Encode(pixels, tinyFlat.Width, tinyFlat.Height)); + } + + static Image Flatten(Image image) + { + var srgb = image.Interpretation == Enums.Interpretation.Srgb ? image.Copy() : image.Colourspace(Enums.Interpretation.Srgb); + var flat = srgb.HasAlpha() ? srgb.Flatten(background: new double[] { 255, 255, 255 }) : srgb.Copy(); + srgb.Dispose(); + var bands = flat.Bands > 3 ? flat.ExtractBand(0, n: 3) : flat.Bands == 1 ? flat.Bandjoin(flat, flat) : flat.Copy(); + flat.Dispose(); + using var cast = bands; + return bands.Format == Enums.BandFormat.Uchar ? bands.Copy() : bands.Cast(Enums.BandFormat.Uchar); + } + + async Task ProcessAv(IFormFile file, string extension, string contentType, MediaAttachment attachment, CancellationToken token) + { + var temp = Path.Combine(Path.GetTempPath(), $"privapub-{Guid.NewGuid():N}"); + var input = temp + ".in"; + var output = temp + "." + extension; + try + { + await using (var target = File.Create(input)) + await file.CopyToAsync(target, token); + var probe = await FFProbe.AnalyseAsync(input, cancellationToken: token); + var isVideo = contentType.StartsWith("video/") && probe.PrimaryVideoStream != default; + await FFMpegArguments.FromFileInput(input) + .OutputToFile(output, true, o => o.WithCustomArgument("-map 0 -map_metadata -1 -map_chapters -1 -c copy" + (extension is "mp4" or "m4a" ? " -movflags +faststart" : string.Empty))) + .CancellableThrough(token) + .ProcessAsynchronously(); + + attachment.Kind = isVideo ? "video" : "audio"; + attachment.ContentType = extension switch { "mp4" => "video/mp4", "webm" => isVideo ? "video/webm" : "audio/webm", "m4a" => "audio/mp4", "mp3" => "audio/mpeg", _ => contentType }; + attachment.FilePath = await Save(await File.ReadAllBytesAsync(output, token), extension, token); + attachment.Size = new FileInfo(output).Length; + if (isVideo) + { + attachment.Width = probe.PrimaryVideoStream.Width; + attachment.Height = probe.PrimaryVideoStream.Height; + var frame = temp + ".png"; + await FFMpeg.SnapshotAsync(input, frame, captureTime: TimeSpan.FromSeconds(Math.Min(1, probe.Duration.TotalSeconds / 2))); + var still = ProcessImage(await File.ReadAllBytesAsync(frame, token), 640, _options.CurrentValue.PreviewSide, false); + attachment.PreviewPath = await Save(still.Preview, "jpg", token); + attachment.Blurhash = still.Blurhash; + File.Delete(frame); + } + return new MediaOutcome(attachment); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + _logger.LogInformation(ex, "Refused an audio or video upload"); + return MediaOutcome.Fail(StatusCodes.Status422UnprocessableEntity, "Validation failed: The file could not be processed"); + } + finally + { + foreach (var path in new[] { input, output }) + if (File.Exists(path)) + File.Delete(path); + } + } + + async Task Save(byte[] bytes, string extension, CancellationToken token) + { + var now = DateTime.UtcNow; + var relative = Path.Combine(now.ToString("yyyy", CultureInfo.InvariantCulture), now.ToString("MM", CultureInfo.InvariantCulture), + $"{Convert.ToHexStringLower(RandomNumberGenerator.GetBytes(16))}.{extension}"); + var full = Path.Combine(Root, relative); + Directory.CreateDirectory(Path.GetDirectoryName(full)!); + await File.WriteAllBytesAsync(full, bytes, token); + return relative.Replace('\\', '/'); + } + + static string Clean(string value, int max) => + string.IsNullOrWhiteSpace(value) ? default : value.Trim().Length <= max ? value.Trim() : value.Trim()[..max]; + + static float[] Focus(string focus) + { + var parts = focus?.Split(','); + if (parts is not { Length: 2 } + || !float.TryParse(parts[0], NumberStyles.Float, CultureInfo.InvariantCulture, out var x) + || !float.TryParse(parts[1], NumberStyles.Float, CultureInfo.InvariantCulture, out var y)) + return default; + return new[] { Math.Clamp(x, -1, 1), Math.Clamp(y, -1, 1) }; + } + } +} diff --git a/PrivaPub/Domain/Statuses/StatusService.cs b/PrivaPub/Domain/Statuses/StatusService.cs index 104b5ea..9e7d138 100644 --- a/PrivaPub/Domain/Statuses/StatusService.cs +++ b/PrivaPub/Domain/Statuses/StatusService.cs @@ -2,6 +2,7 @@ using MongoDB.Driver; using MongoDB.Entities; using PrivaPub.Domain.Content; +using PrivaPub.Domain.Media; using PrivaPub.Domain.Privacy; using PrivaPub.Domain.Social; using PrivaPub.Domain.Timelines; @@ -10,6 +11,7 @@ using PrivaPub.Federation.Outbox; using PrivaPub.Federation.Rendering; using PrivaPub.Models.Federation; using PrivaPub.Models.Group; +using PrivaPub.Models.Media; using PrivaPub.Models.Post; using PrivaPub.Models.Social; using PrivaPub.StaticServices; @@ -33,6 +35,7 @@ namespace PrivaPub.Domain.Statuses public string Language { get; init; } public string ConversationId { get; init; } public IReadOnlyList Recipients { get; init; } = Array.Empty(); + public IReadOnlyList MediaIds { get; init; } } public sealed record StatusOutcome(PostEntity Post, int Status = StatusCodes.Status200OK, string Error = default) @@ -61,10 +64,12 @@ namespace PrivaPub.Domain.Statuses readonly IContentRenderer _content; readonly IOutboxPublisher _outbox; readonly IFanout _fanout; + readonly IMediaService _media; public StatusService(DbEntities dbEntities, ILocalActorService localActors, IRemoteActorService remoteActors, IDeliveryService delivery, - IContentRenderer content, IOutboxPublisher outbox, IFanout fanout) + IContentRenderer content, IOutboxPublisher outbox, IFanout fanout, IMediaService media) { + _media = media; _dbEntities = dbEntities; _localActors = localActors; _remoteActors = remoteActors; @@ -76,7 +81,10 @@ namespace PrivaPub.Domain.Statuses public async Task Publish(LocalActor author, StatusDraft draft, CancellationToken token) { - if (string.IsNullOrWhiteSpace(draft.Text)) + var media = await Media(author, draft.MediaIds, default, token); + if (media == default) + return StatusOutcome.Fail(StatusCodes.Status422UnprocessableEntity, "Validation failed: Media attachments are invalid"); + if (string.IsNullOrWhiteSpace(draft.Text) && media.Count == 0) return StatusOutcome.Fail(StatusCodes.Status422UnprocessableEntity, "Validation failed: Text can't be blank"); LocalActor group = default; @@ -94,7 +102,7 @@ namespace PrivaPub.Domain.Statuses if (parent != default && !await VisibilityPolicy.CanSee(parent, author.Id, token)) return StatusOutcome.Fail(StatusCodes.Status404NotFound, "Record not found"); - var rendered = draft.PlainText ? await _content.PlainText(draft.Text, token) : await _content.Markdown(draft.Text, token); + var rendered = draft.PlainText ? await _content.PlainText(draft.Text ?? string.Empty, token) : await _content.Markdown(draft.Text ?? string.Empty, token); var isLocalOnly = group is { IsFederated: false }; var visibility = isLocalOnly ? PostVisibility.Circle : draft.Visibility is PostVisibility.Circle or PostVisibility.LocalGeo ? PostVisibility.Public : draft.Visibility; var post = new PostEntity @@ -112,6 +120,7 @@ namespace PrivaPub.Domain.Statuses Language = Clean(draft.Language), Mentions = rendered.Mentions.Select(ToMention).ToList(), Tags = rendered.Tags.ToList(), + Media = media.Select(ToPostMedia).ToList(), AnsweringToPostId = parent?.ID, InReplyToURI = parent?.ObjectURI ?? (IsRemoteUri(draft.InReplyTo) ? draft.InReplyTo : default), InReplyToAccountId = parent?.AuthorAccountId ?? parent?.GroupUserId, @@ -150,6 +159,7 @@ namespace PrivaPub.Domain.Statuses post.ActivityURI = create?["id"]?.GetValue(); await DB.Default.SaveAsync(post, token); + await Attach(media, post.ID, token); if (parent != default) await DB.Default.Update().MatchID(parent.ID).Modify(b => b.Inc(p => p.RepliesCount, 1)).ExecuteAsync(token); await _fanout.Distribute(post, token); @@ -165,7 +175,10 @@ namespace PrivaPub.Domain.Statuses var post = await Own(author, postId, token); if (post == default) return StatusOutcome.Fail(StatusCodes.Status404NotFound, "Record not found"); - if (string.IsNullOrWhiteSpace(draft.Text)) + var media = draft.MediaIds == default ? default : await Media(author, draft.MediaIds, post.ID, token); + if (draft.MediaIds != default && media == default) + return StatusOutcome.Fail(StatusCodes.Status422UnprocessableEntity, "Validation failed: Media attachments are invalid"); + if (string.IsNullOrWhiteSpace(draft.Text) && (media ?? new List()).Count == 0 && post.Media.Count == 0) return StatusOutcome.Fail(StatusCodes.Status422UnprocessableEntity, "Validation failed: Text can't be blank"); post.Revisions.Add(new PostRevision @@ -180,7 +193,12 @@ namespace PrivaPub.Domain.Statuses post.Revisions.RemoveRange(0, post.Revisions.Count - MaxRevisions); var plain = draft.PlainText || post.ContentFormat == ContentFormat.Plain; - var rendered = plain ? await _content.PlainText(draft.Text, token) : await _content.Markdown(draft.Text, token); + var rendered = plain ? await _content.PlainText(draft.Text ?? string.Empty, token) : await _content.Markdown(draft.Text ?? string.Empty, token); + if (media != default) + { + post.Media = media.Select(ToPostMedia).ToList(); + await Attach(media, post.ID, token); + } post.Title = draft.Title == default ? post.Title : Clean(draft.Title); post.SpoilerText = Clean(draft.SpoilerText); post.HasContentWarning = draft.Sensitive || post.SpoilerText != default; @@ -461,6 +479,40 @@ namespace PrivaPub.Domain.Statuses return (await _dbEntities.ForeignAvatars.MatchID(post.AuthorAccountId).ExecuteFirstAsync(token))?.InboxURL; } + async Task> Media(LocalActor author, IReadOnlyList ids, string postId, CancellationToken token) + { + if (ids == default || ids.Count == 0) + return new List(); + if (ids.Count > 4) + return default; + var wanted = ids.Distinct().ToList(); + var found = await DB.Default.Find() + .Match(m => wanted.Contains(m.ID) && m.OwnerAvatarId == author.Id && (m.PostId == null || m.PostId == postId)) + .ExecuteAsync(token); + return found.Count == wanted.Count ? wanted.Select(id => found.First(m => m.ID == id)).ToList() : default; + } + + static async Task Attach(IEnumerable media, string postId, CancellationToken token) + { + var ids = media.Select(m => m.ID).ToList(); + if (ids.Count > 0) + await DB.Default.Update().Match(m => ids.Contains(m.ID)) + .Modify(m => m.PostId, postId).Modify(m => m.AttachedAt, DateTime.UtcNow).ExecuteAsync(token); + } + + PostMedia ToPostMedia(MediaAttachment attachment) => new() + { + AttachmentId = attachment.ID, + ContentType = attachment.ContentType, + URL = _media.Url(attachment.FilePath), + PreviewURL = _media.Url(attachment.PreviewPath ?? attachment.FilePath), + Description = attachment.Description, + Blurhash = attachment.Blurhash, + Width = attachment.Width, + Height = attachment.Height, + Focus = attachment.Focus + }; + static JsonObject Undo(LocalActor actor, JsonObject inner, string activityId) => new() { ["@context"] = ActivityPubRenderer.ActivityStreams, diff --git a/PrivaPub/Federation/Rendering/ActivityPubRenderer.cs b/PrivaPub/Federation/Rendering/ActivityPubRenderer.cs index 9137f0f..e1fa56e 100644 --- a/PrivaPub/Federation/Rendering/ActivityPubRenderer.cs +++ b/PrivaPub/Federation/Rendering/ActivityPubRenderer.cs @@ -170,6 +170,28 @@ namespace PrivaPub.Federation.Rendering }; if (!string.IsNullOrEmpty(post.Language)) note["contentMap"] = new JsonObject { [post.Language] = content }; + var attachments = post.Media.Where(m => !string.IsNullOrEmpty(m.URL)).Select(m => + { + var attachment = new JsonObject + { + ["type"] = "Document", + ["mediaType"] = m.ContentType, + ["url"] = m.URL, + ["name"] = m.Description + }; + if (!string.IsNullOrEmpty(m.Blurhash)) + attachment["blurhash"] = m.Blurhash; + if (m.Focus is { Length: 2 }) + attachment["focalPoint"] = new JsonArray(m.Focus[0], m.Focus[1]); + if (m.Width.HasValue && m.Height.HasValue) + { + attachment["width"] = m.Width; + attachment["height"] = m.Height; + } + return (JsonNode)attachment; + }).ToArray(); + if (attachments.Length > 0) + note["attachment"] = new JsonArray(attachments); if (!string.IsNullOrEmpty(post.Title)) note["name"] = post.Title; var summary = post.SpoilerText ?? (post.HasContentWarning ? post.Title ?? ContentWarning : default); diff --git a/PrivaPub/Infrastructure/Http/FederationHttp.cs b/PrivaPub/Infrastructure/Http/FederationHttp.cs index 15e84fa..7f3b90e 100644 --- a/PrivaPub/Infrastructure/Http/FederationHttp.cs +++ b/PrivaPub/Infrastructure/Http/FederationHttp.cs @@ -22,6 +22,7 @@ namespace PrivaPub.Infrastructure.Http bool IsAllowed(Uri target); Task GetJson(string url, string accept, Action sign, CancellationToken token); Task Send(HttpRequestMessage request, CancellationToken token); + Task<(byte[] Bytes, string ContentType)> GetMedia(string url, long maxBytes, CancellationToken token); } public class FederationHttp : IFederationHttp @@ -132,6 +133,45 @@ namespace PrivaPub.Infrastructure.Http } } + public async Task<(byte[] Bytes, string ContentType)> GetMedia(string url, long maxBytes, CancellationToken token) + { + if (!Uri.TryCreate(url, UriKind.Absolute, out var target) || !IsAllowed(target)) + return default; + using var timeout = CancellationTokenSource.CreateLinkedTokenSource(token); + timeout.CancelAfter(TimeSpan.FromSeconds(60)); + try + { + for (var hop = 0; hop <= MaxRedirects; hop++) + { + using var request = new HttpRequestMessage(HttpMethod.Get, target); + request.Headers.Accept.ParseAdd("image/*, video/*, audio/*"); + using var response = await _httpClientFactory.CreateClient(ClientName).SendAsync(request, HttpCompletionOption.ResponseHeadersRead, timeout.Token); + if (IsRedirect(response.StatusCode)) + { + var location = response.Headers.Location; + var next = location == default ? default : location.IsAbsoluteUri ? location : new Uri(target, location); + if (!IsAllowed(next)) + return default; + target = next; + continue; + } + var mediaType = response.Content.Headers.ContentType?.MediaType?.ToLowerInvariant(); + if (!response.IsSuccessStatusCode || mediaType == default + || !(mediaType.StartsWith("image/") || mediaType.StartsWith("video/") || mediaType.StartsWith("audio/")) + || mediaType.Contains("svg") || response.Content.Headers.ContentLength > maxBytes) + return default; + var bytes = await ReadBounded(response.Content, (int)Math.Min(maxBytes, int.MaxValue), timeout.Token); + return bytes == default ? default : (bytes, mediaType); + } + return default; + } + catch (Exception ex) when (ex is HttpRequestException or BlockedDestinationException or OperationCanceledException && !token.IsCancellationRequested) + { + _logger.LogInformation("Media {Url} refused: {Reason}", url, ex.Message); + return default; + } + } + public async Task Send(HttpRequestMessage request, CancellationToken token) { if (!IsAllowed(request.RequestUri)) diff --git a/PrivaPub/Models/Media/MediaAttachment.cs b/PrivaPub/Models/Media/MediaAttachment.cs new file mode 100644 index 0000000..4bad40a --- /dev/null +++ b/PrivaPub/Models/Media/MediaAttachment.cs @@ -0,0 +1,27 @@ +using MongoDB.Entities; + +namespace PrivaPub.Models.Media +{ + public class MediaAttachment : Entity + { + public string OwnerAvatarId { get; set; } + public string PostId { get; set; } + public string Kind { get; set; } + public string ContentType { get; set; } + public string FilePath { get; set; } + public string PreviewPath { get; set; } + public long Size { get; set; } + public string Description { get; set; } + public string Blurhash { get; set; } + public int? Width { get; set; } + public int? Height { get; set; } + public float[] Focus { get; set; } + public DateTime CreatedAt { get; set; } = DateTime.UtcNow; + public DateTime? AttachedAt { get; set; } + } + + public class MediaSecret : Entity + { + public string Key { get; set; } + } +} diff --git a/PrivaPub/Models/Post/PostMedia.cs b/PrivaPub/Models/Post/PostMedia.cs index da8b8b8..a6b74cc 100644 --- a/PrivaPub/Models/Post/PostMedia.cs +++ b/PrivaPub/Models/Post/PostMedia.cs @@ -9,6 +9,8 @@ namespace PrivaPub.Models.Post public string Path { get; set; } public string URL { get; set; } public string RemoteURL { get; set; } + public string PreviewURL { get; set; } + public string AttachmentId { get; set; } public string Description { get; set; } public string Blurhash { get; set; } public int? Width { get; set; } diff --git a/PrivaPub/PrivaPub.csproj b/PrivaPub/PrivaPub.csproj index bc3bfe8..b433327 100644 --- a/PrivaPub/PrivaPub.csproj +++ b/PrivaPub/PrivaPub.csproj @@ -7,11 +7,14 @@ + + + diff --git a/PrivaPub/Program.cs b/PrivaPub/Program.cs index 2d72918..57d79cb 100644 --- a/PrivaPub/Program.cs +++ b/PrivaPub/Program.cs @@ -58,6 +58,10 @@ try .PrivaPubOAuth(builder.Environment) .AddScoped() .AddScoped() + .Configure(builder.Configuration.GetSection("Media")) + .AddSingleton() + .AddSingleton() + .AddHostedService() .PrivaPubMiddlewareConfiguration(); } catch (Exception ex) @@ -124,6 +128,19 @@ try app.UseCors("DefaultCORS"); app.UseStaticFiles(); + var mediaRoot = app.Services.GetRequiredService().Root; + Directory.CreateDirectory(mediaRoot); + app.UseStaticFiles(new StaticFileOptions + { + FileProvider = new Microsoft.Extensions.FileProviders.PhysicalFileProvider(mediaRoot), + RequestPath = "/media/files", + OnPrepareResponse = context => + { + context.Context.Response.Headers["X-Content-Type-Options"] = "nosniff"; + context.Context.Response.Headers["Content-Security-Policy"] = "default-src 'none'; sandbox"; + context.Context.Response.Headers["Cache-Control"] = "public, max-age=31536000, immutable"; + } + }); app.UseRequestLocalization(await localizationService.Get()); diff --git a/PrivaPub/appsettings.Production.json b/PrivaPub/appsettings.Production.json index ace7c9f..d257f77 100644 --- a/PrivaPub/appsettings.Production.json +++ b/PrivaPub/appsettings.Production.json @@ -1,4 +1,7 @@ { + "Media": { + "Root": "/var/lib/privapub/media" + }, "MongoSettings": { "Database": "PrivaPub", "LogsDatabase": "logs", diff --git a/deploy/nginx/privapub.thepra.dev.conf b/deploy/nginx/privapub.thepra.dev.conf index eb1db73..064c4d3 100644 --- a/deploy/nginx/privapub.thepra.dev.conf +++ b/deploy/nginx/privapub.thepra.dev.conf @@ -35,6 +35,19 @@ server { default_type "text/plain"; } + location ~ ^/api/v[12]/media$|^/api/v1/accounts/update_credentials$ { + client_max_body_size 100m; + proxy_request_buffering off; + proxy_pass http://127.0.0.1:6970; + proxy_http_version 1.1; + proxy_set_header Connection ""; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_read_timeout 300s; + } + location / { proxy_pass http://127.0.0.1:6970; proxy_http_version 1.1;