SocialPub/PrivaPub.Tests/Http/MastodonMediaTests.cs

using PrivaPub.Domain.Media;
using PrivaPub.Tests.Support;
using PrivaPub.Tests.Support.Host;

using System.Diagnostics;
using System.Net;
using System.Net.Http.Headers;
using System.Text;
using System.Text.Json.Nodes;

namespace PrivaPub.Tests.Http
{
	[Trait("Category", "Integration")]
	public sealed class MastodonMediaTests : IAsyncLifetime
	{
		PrivaPubHost _host;
		Peer _peer;

		public async ValueTask InitializeAsync()
		{
			Assert.SkipUnless(MongoFixture.Enabled, MongoFixture.Skip);
			_host = await PrivaPubHost.Shared();
			_peer = await Peer.Start();
		}

		public async ValueTask DisposeAsync()
		{
			if (_peer != default)
				await _peer.DisposeAsync();
		}

		static CancellationToken Token => TestContext.Current.CancellationToken;

		static Task<ApiAnswer> Upload(Mastodon account, string path, byte[] bytes, string contentType, string fileName, params (string Key, string Value)[] fields)
		{
			var form = MastodonHelpers.Multipart(("file", bytes, contentType, fileName));
			foreach (var (key, value) in fields)
				form.Add(new StringContent(value), key);
			return account.Client.Exchange(new HttpRequestMessage(HttpMethod.Post, path) { Content = form });
		}

		static async Task<(int ExitCode, string Output)> Run(string program, params string[] arguments)
		{
			var start = new ProcessStartInfo(program) { RedirectStandardOutput = true, RedirectStandardError = true };
			foreach (var argument in arguments)
				start.ArgumentList.Add(argument);
			try
			{
				using var process = Process.Start(start)!;
				var output = process.StandardOutput.ReadToEndAsync(Token);
				var errors = process.StandardError.ReadToEndAsync(Token);
				await process.WaitForExitAsync(Token);
				return (process.ExitCode, await output + await errors);
			}
			catch (System.ComponentModel.Win32Exception)
			{
				return (-1, $"{program} is not installed");
			}
		}

		static async Task<byte[]> Made(string extension, params string[] arguments)
		{
			if ((await Run("ffmpeg", "-version")).ExitCode != 0 || (await Run("ffprobe", "-version")).ExitCode != 0)
				Assert.Skip("ffmpeg and ffprobe are needed to make and inspect audio and video");
			var path = Path.Combine(Path.GetTempPath(), $"privapub-av-{Guid.NewGuid():N}.{extension}");
			try
			{
				var (exitCode, output) = await Run("ffmpeg", arguments.Append(path).Prepend("-nostdin").Prepend("-y").ToArray());
				Assert.True(exitCode == 0, output);
				return await File.ReadAllBytesAsync(path, Token);
			}
			finally
			{
				File.Delete(path);
			}
		}

		async Task<string> Probe(string url)
		{
			var path = Path.Combine(Path.GetTempPath(), $"privapub-probe-{Guid.NewGuid():N}");
			try
			{
				await File.WriteAllBytesAsync(path, await _host.Client().GetByteArrayAsync(url, Token), Token);
				var (exitCode, output) = await Run("ffprobe", "-v", "quiet", "-print_format", "json", "-show_format", "-show_streams", path);
				Assert.Equal(0, exitCode);
				return output;
			}
			finally
			{
				File.Delete(path);
			}
		}

		[Theory]
		[InlineData("/api/v1/media")]
		[InlineData("/api/v2/media")]
		public async Task Images_upload_with_their_description_and_focus_and_without_their_metadata(string route)
		{
			var alice = await _host.Mastodon("alice");

			var uploaded = (await Upload(alice, route, MastodonHelpers.JpegWithMetadata(640, 480), "image/jpeg", "holiday.jpg",
				("description", "a blue square"), ("focus", "0.5,-0.25"))).Ok();

			Assert.Equal("image", uploaded.Body.Text("type"));
			Assert.Equal("a blue square", uploaded.Body.Text("description"));
			Assert.Equal(640, uploaded.Body["meta"]!["original"].Number("width"));
			Assert.Equal(480, uploaded.Body["meta"]!["original"].Number("height"));
			Assert.Equal(0.5, uploaded.Body["meta"]!["focus"]!["x"]!.GetValue<double>());
			Assert.Equal(-0.25, uploaded.Body["meta"]!["focus"]!["y"]!.GetValue<double>());
			Assert.False(string.IsNullOrEmpty(uploaded.Body.Text("blurhash")));
			Assert.Null(uploaded.Body.Text("remote_url"));
			foreach (var field in new[] { "url", "preview_url" })
			{
				Assert.StartsWith($"{PrivaPubHost.Base}/media/files/", uploaded.Body.Text(field));
				MastodonHelpers.AssertNoMetadata(await _host.Client().GetByteArrayAsync(uploaded.Body.Text(field), Token));
			}
		}

		[Fact]
		public async Task Media_is_read_and_described_only_by_its_owner()
		{
			var alice = await _host.Mastodon("alice");
			var mallory = await _host.Mastodon("mallory");
			var id = (await Upload(alice, "/api/v2/media", MastodonHelpers.JpegWithMetadata(32, 32), "image/jpeg", "a.jpg")).Ok().Body.Text("id");

			Assert.Equal(id, (await alice.Client.Get($"/api/v1/media/{id}")).Ok().Body.Text("id"));
			Assert.Equal(HttpStatusCode.NotFound, (await mallory.Client.Get($"/api/v1/media/{id}")).Status);
			Assert.Equal(HttpStatusCode.NotFound, (await mallory.Client.Put($"/api/v1/media/{id}", ("description", "mine now"))).Status);
			Assert.Equal(HttpStatusCode.Unauthorized, (await _host.Client().Get($"/api/v1/media/{id}")).Status);

			var described = (await alice.Client.Put($"/api/v1/media/{id}", ("description", "  a tiny square  "), ("focus", "2,-3"))).Ok();

			Assert.Equal("a tiny square", described.Body.Text("description"));
			Assert.Equal((1.0, -1.0), (described.Body["meta"]!["focus"]!["x"]!.GetValue<double>(), described.Body["meta"]!["focus"]!["y"]!.GetValue<double>()));
			Assert.Equal("a tiny square", (await alice.Client.Get($"/api/v1/media/{id}")).Ok().Body.Text("description"));
			Assert.Null((await alice.Client.Put($"/api/v1/media/{id}", ("description", ""))).Ok().Body.Text("description"));
		}

		[Fact]
		public async Task Unsupported_unreadable_and_missing_files_are_refused_with_422()
		{
			var alice = await _host.Mastodon("alice");

			var text = await Upload(alice, "/api/v2/media", Encoding.UTF8.GetBytes("hello"), "text/plain", "a.txt");
			Assert.Equal(HttpStatusCode.UnprocessableEntity, text.Status);
			Assert.Contains("not supported", text.Body.Text("error"));
			Assert.Equal(HttpStatusCode.UnprocessableEntity, (await Upload(alice, "/api/v2/media", Encoding.UTF8.GetBytes("not a jpeg"), "image/jpeg", "a.jpg")).Status);
			Assert.Equal(HttpStatusCode.UnprocessableEntity, (await Upload(alice, "/api/v2/media", Encoding.UTF8.GetBytes("not a video"), "video/mp4", "a.mp4")).Status);
			var empty = new MultipartFormDataContent { { new StringContent("no file"), "description" } };
			Assert.Equal(HttpStatusCode.UnprocessableEntity, (await alice.Client.Exchange(new HttpRequestMessage(HttpMethod.Post, "/api/v2/media") { Content = empty })).Status);
			Assert.Equal(HttpStatusCode.UnprocessableEntity, (await alice.Client.Post("/api/v1/media")).Status);
		}

		[Fact]
		public async Task Video_is_remuxed_without_its_metadata()
		{
			var alice = await _host.Mastodon("alice");
			var video = await Made("mp4", "-f", "lavfi", "-i", "testsrc=duration=1:size=320x240:rate=10", "-f", "lavfi", "-i", "sine=frequency=440:duration=1",
				"-metadata", "title=secret title", "-metadata", "comment=filmed at home", "-metadata:s:v:0", "handler_name=hidden handler",
				"-c:v", "mpeg4", "-c:a", "aac", "-shortest");
			Assert.Contains("secret title", Encoding.Latin1.GetString(video));

			var uploaded = (await Upload(alice, "/api/v2/media", video, "video/mp4", "clip.mp4")).Ok();

			Assert.Equal("video", uploaded.Body.Text("type"));
			Assert.Equal(320, uploaded.Body["meta"]!["original"].Number("width"));
			Assert.EndsWith(".mp4", uploaded.Body.Text("url"));
			Assert.EndsWith(".jpg", uploaded.Body.Text("preview_url"));
			var probe = await Probe(uploaded.Body.Text("url"));
			Assert.Contains("\"codec_type\": \"video\"", probe);
			Assert.DoesNotContain("secret title", probe);
			Assert.DoesNotContain("filmed at home", probe);
			Assert.DoesNotContain("hidden handler", probe);
		}

		[Fact]
		public async Task Audio_is_remuxed_without_its_metadata()
		{
			var alice = await _host.Mastodon("alice");
			var audio = await Made("m4a", "-f", "lavfi", "-i", "sine=frequency=330:duration=1", "-metadata", "title=secret song", "-metadata", "artist=Alice Smith",
				"-c:a", "aac");

			var uploaded = (await Upload(alice, "/api/v2/media", audio, "audio/mp4", "song.m4a")).Ok();

			Assert.Equal("audio", uploaded.Body.Text("type"));
			var probe = await Probe(uploaded.Body.Text("url"));
			Assert.Contains("\"codec_type\": \"audio\"", probe);
			Assert.DoesNotContain("secret song", probe);
			Assert.DoesNotContain("Alice Smith", probe);
		}

		static byte[] Bytes(int length)
		{
			var bytes = new byte[length];
			Random.Shared.NextBytes(bytes);
			return bytes;
		}

		string Served(byte[] bytes, string contentType = "video/mp4")
		{
			var path = $"/media/{Guid.NewGuid():N}.bin";
			_peer.ServeFile(path, bytes, contentType);
			return _peer.A + path;
		}

		static HttpRequestMessage Ranged(string url, long from, long to)
		{
			var request = new HttpRequestMessage(HttpMethod.Get, url);
			request.Headers.Range = new RangeHeaderValue(from, to);
			return request;
		}

		[Fact]
		public async Task Every_remote_media_address_the_api_returns_goes_through_the_proxy()
		{
			var alice = await _host.Mastodon("alice");
			var bob = new RemoteActor(_peer, "bob");
			var document = bob.Document();
			document["icon"] = new JsonObject { ["type"] = "Image", ["url"] = _peer.A + "/avatar.png" };
			document["image"] = new JsonObject { ["type"] = "Image", ["url"] = _peer.A + "/header.png" };
			_peer.Serve($"/users/{bob.Name}", document.ToJsonString());
			var bobId = (await _host.Known(bob)).ID;
			var post = await _host.PublicPostFrom(bob, alice, "<p>look :blob:</p>", note =>
			{
				note["attachment"] = new JsonArray(new JsonObject
				{
					["type"] = "Document", ["mediaType"] = "image/png", ["url"] = _peer.A + "/picture.png", ["name"] = "a picture"
				});
				note["tag"]!.AsArray().Add(new JsonObject
				{
					["type"] = "Emoji", ["name"] = ":blob:", ["icon"] = new JsonObject { ["type"] = "Image", ["url"] = _peer.A + "/blob.png" }
				});
			});
			var proxied = $"{PrivaPubHost.Base}/media/proxy/";

			var account = (await alice.Client.Get($"/api/v1/accounts/{bobId}")).Ok().Body;
			var status = (await alice.Client.Get($"/api/v1/statuses/{post.ID}")).Ok().Body;

			foreach (var field in new[] { "avatar", "avatar_static", "header", "header_static" })
				Assert.StartsWith(proxied, account.Text(field));
			Assert.StartsWith(proxied, status["account"].Text("avatar"));
			var attachment = Assert.Single(status["media_attachments"]!.AsArray());
			Assert.StartsWith(proxied, attachment.Text("url"));
			Assert.StartsWith(proxied, attachment.Text("preview_url"));
			Assert.Equal(_peer.A + "/picture.png", attachment.Text("remote_url"));
			Assert.Equal("a picture", attachment.Text("description"));
			Assert.StartsWith(proxied, Assert.Single(status["emojis"]!.AsArray()).Text("url"));
			var everything = account.ToJsonString() + status.ToJsonString();
			foreach (var file in new[] { "/avatar.png", "/header.png", "/blob.png" })
				Assert.DoesNotContain(_peer.A + file, everything);
		}

		[Fact]
		public async Task The_proxy_refuses_a_url_it_did_not_sign()
		{
			var proxy = _host.Get<IMediaProxy>();
			var remote = Served(Bytes(100));
			var wrapped = proxy.Wrap(remote);
			var parts = new Uri(wrapped).AbsolutePath.Split('/');
			var other = new Uri(proxy.Wrap(Served(Bytes(100)))).AbsolutePath.Split('/');
			using var client = _host.Client();

			Assert.StartsWith($"{PrivaPubHost.Base}/media/proxy/", wrapped);
			Assert.Equal(HttpStatusCode.NotFound, (await client.GetAsync($"/media/proxy/AAAAAAAAAAAAAAAAAAAAAA/{parts[^1]}", Token)).StatusCode);
			Assert.Equal(HttpStatusCode.NotFound, (await client.GetAsync($"/media/proxy/{parts[^2]}/{other[^1]}", Token)).StatusCode);
			Assert.Equal(HttpStatusCode.NotFound, (await client.GetAsync($"/media/proxy/{parts[^2]}/!!!", Token)).StatusCode);
			Assert.Empty(_peer.Requests);
			Assert.Equal($"{PrivaPubHost.Base}/media/files/a.jpg", proxy.Wrap($"{PrivaPubHost.Base}/media/files/a.jpg"));
		}

		[Fact]
		public async Task A_ranged_request_is_streamed_as_206_and_never_cached()
		{
			var proxy = _host.Get<IMediaProxy>();
			var bytes = Bytes(10_000);
			var remote = Served(bytes);
			using var client = _host.Client();

			using var response = await client.SendAsync(Ranged(proxy.Wrap(remote), 100, 199), Token);

			Assert.Equal(HttpStatusCode.PartialContent, response.StatusCode);
			Assert.Equal("bytes 100-199/10000", response.Content.Headers.ContentRange!.ToString());
			Assert.Equal(bytes[100..200], await response.Content.ReadAsByteArrayAsync(Token));
			Assert.Equal("nosniff", response.Headers.GetValues("X-Content-Type-Options").Single());
			Assert.Equal(default, proxy.Cached(remote));
			using var again = await client.SendAsync(Ranged(proxy.Wrap(remote), 0, 9), Token);
			Assert.Equal(bytes[..10], await again.Content.ReadAsByteArrayAsync(Token));
			Assert.Equal(2, _peer.Requests.Count);
			Assert.All(_peer.Requests, r => Assert.StartsWith("bytes=", r.Headers["Range"]));
		}

		[Fact]
		public async Task A_whole_download_is_cached_and_then_served_with_ranges()
		{
			var proxy = _host.Get<IMediaProxy>();
			var bytes = Bytes(5_000);
			var remote = Served(bytes, "image/png");
			using var client = _host.Client();

			using var whole = await client.GetAsync(proxy.Wrap(remote), Token);

			Assert.Equal(HttpStatusCode.OK, whole.StatusCode);
			Assert.Equal("image/png", whole.Content.Headers.ContentType!.MediaType);
			Assert.Equal(bytes, await whole.Content.ReadAsByteArrayAsync(Token));
			Assert.NotNull(proxy.Cached(remote).Path);
			using var part = await client.SendAsync(Ranged(proxy.Wrap(remote), 10, 19), Token);
			Assert.Equal(HttpStatusCode.PartialContent, part.StatusCode);
			Assert.Equal(bytes[10..20], await part.Content.ReadAsByteArrayAsync(Token));
			using var cached = await client.GetAsync(proxy.Wrap(remote), Token);
			Assert.Equal(bytes, await cached.Content.ReadAsByteArrayAsync(Token));
			Assert.Single(_peer.Requests);
		}

		[Fact]
		public async Task Anything_over_the_proxy_limit_is_streamed_and_never_cached()
		{
			var host = await SmallProxyHost.Shared();
			var proxy = host.Get<IMediaProxy>();
			var bytes = Bytes(SmallProxyHost.Limit * 4);
			var remote = Served(bytes);
			using var client = host.Client();

			using var response = await client.GetAsync(proxy.Wrap(remote), Token);

			Assert.Equal(HttpStatusCode.OK, response.StatusCode);
			Assert.Equal(bytes.Length, response.Content.Headers.ContentLength);
			Assert.Equal(bytes, await response.Content.ReadAsByteArrayAsync(Token));
			Assert.Equal(default, proxy.Cached(remote));
			using var again = await client.GetAsync(proxy.Wrap(remote), Token);
			Assert.Equal(bytes, await again.Content.ReadAsByteArrayAsync(Token));
			Assert.Equal(4, _peer.Requests.Count);
			var small = Served(Bytes(SmallProxyHost.Limit / 2));
			using (var fits = await client.GetAsync(proxy.Wrap(small), Token))
				Assert.Equal(HttpStatusCode.OK, fits.StatusCode);
			Assert.NotNull(proxy.Cached(small).Path);
		}
	}

	public sealed class SmallProxyHost : PrivaPubHost
	{
		public const int Limit = 16 * 1024;

		static readonly SemaphoreSlim Boot = new(1, 1);
		static SmallProxyHost _shared;

		public static new async Task<SmallProxyHost> Shared()
		{
			await PrivaPubHost.Shared();
			await Boot.WaitAsync();
			try
			{
				if (_shared == default)
				{
					var host = new SmallProxyHost();
					_ = host.Services;
					_shared = host;
				}
				return _shared;
			}
			finally
			{
				Boot.Release();
			}
		}

		protected override IEnumerable<KeyValuePair<string, string>> Settings() =>
			base.Settings().Append(new KeyValuePair<string, string>("Media:MaxProxiedBytes", Limit.ToString(System.Globalization.CultureInfo.InvariantCulture)));
	}
}