SocialPub/PrivaPub.Tests/Http/MediaLifecycleTests.cs

using MongoDB.Entities;

using PrivaPub.Domain.Media;
using PrivaPub.Models.Media;
using PrivaPub.Services;
using PrivaPub.Tests.Support;
using PrivaPub.Tests.Support.Host;

using System.Net;

namespace PrivaPub.Tests.Http
{
	// A file lives exactly as long as something holds it: a deleted post, an edit leaving media out, a replaced picture and
	// a removed root each trash theirs, which stops /media/files serving them at once; the janitor deletes them later.
	[Trait("Category", "Integration")]
	public sealed class MediaLifecycleTests : IAsyncLifetime
	{
		PrivaPubHost _host;

		public async ValueTask InitializeAsync()
		{
			Assert.SkipUnless(MongoFixture.Enabled, MongoFixture.Skip);
			_host = await PrivaPubHost.Shared();
		}

		public ValueTask DisposeAsync() => ValueTask.CompletedTask;

		static CancellationToken Token => TestContext.Current.CancellationToken;

		async Task<string> Upload(Mastodon account, string name)
		{
			var form = MastodonHelpers.Multipart(("file", MastodonHelpers.JpegWithMetadata(64, 48), "image/jpeg", name));
			return (await account.Client.Exchange(new HttpRequestMessage(HttpMethod.Post, "/api/v2/media") { Content = form })).Ok().Body.Text("id");
		}

		async Task<HttpStatusCode> Served(string url) =>
			(await _host.Client().GetAsync(new Uri(url).PathAndQuery, Token)).StatusCode;

		[Fact]
		public async Task A_deleted_posts_media_stop_being_served()
		{
			var alice = await _host.Mastodon("alice");
			var id = await Upload(alice, "a.jpg");
			var status = (await alice.Client.Post("/api/v1/statuses", ("status", "with a picture"), ("media_ids[]", id))).Ok();
			var url = status.Body["media_attachments"]![0]!["url"]!.GetValue<string>();
			var preview = status.Body["media_attachments"]![0]!["preview_url"]!.GetValue<string>();
			Assert.Equal(HttpStatusCode.OK, await Served(url));

			(await alice.Client.Delete($"/api/v1/statuses/{status.Body.Text("id")}")).Ok();

			Assert.Equal(HttpStatusCode.NotFound, await Served(url));
			Assert.Equal(HttpStatusCode.NotFound, await Served(preview));
			Assert.Equal("post deleted", (await DB.Default.Find<MediaAttachment>().OneAsync(id, Token)).TrashReason);
			// the file waits in the trash, beside what /media/files serves
			var relative = url[(url.IndexOf("/media/files/", StringComparison.Ordinal) + "/media/files/".Length)..];
			Assert.True(File.Exists(Path.Combine(_host.Get<IMediaService>().TrashRoot, relative)));
			Assert.Equal(HttpStatusCode.NotFound, await Served(url.Replace("/media/files/", "/media/files/.trash/")));
		}

		[Fact]
		public async Task An_edit_that_leaves_media_out_trashes_them()
		{
			var alice = await _host.Mastodon("alice");
			var kept = await Upload(alice, "kept.jpg");
			var dropped = await Upload(alice, "dropped.jpg");
			var status = (await alice.Client.Post("/api/v1/statuses", ("status", "two pictures"), ("media_ids[]", kept), ("media_ids[]", dropped))).Ok();
			var urls = status.Body["media_attachments"]!.AsArray().ToDictionary(m => m!["id"]!.GetValue<string>(), m => m!["url"]!.GetValue<string>());

			(await alice.Client.Put($"/api/v1/statuses/{status.Body.Text("id")}", ("status", "one picture"), ("media_ids[]", kept))).Ok();

			Assert.Equal(HttpStatusCode.OK, await Served(urls[kept]));
			Assert.Equal(HttpStatusCode.NotFound, await Served(urls[dropped]));
			Assert.Null((await DB.Default.Find<MediaAttachment>().OneAsync(kept, Token)).TrashedAt);
			Assert.Equal("edited out", (await DB.Default.Find<MediaAttachment>().OneAsync(dropped, Token)).TrashReason);
		}

		// a posted picture's description and focus change by editing its post (media_attributes), not by PUT, as on Mastodon
		[Fact]
		public async Task An_edit_describes_its_media_again_and_keeps_what_it_is_not_sent()
		{
			var alice = await _host.Mastodon("alice");
			var id = await Upload(alice, "a.jpg");
			var status = (await alice.Client.Post("/api/v1/statuses", ("status", "described later"), ("media_ids[]", id),
				("spoiler_text", "a warning"), ("sensitive", "true"), ("language", "it"))).Ok();
			var statusId = status.Body.Text("id");
			Assert.Equal(HttpStatusCode.NotFound, (await alice.Client.Put($"/api/v1/media/{id}", ("description", "too late"))).Status);

			var edited = (await alice.Client.Json(HttpMethod.Put, $"/api/v1/statuses/{statusId}", new System.Text.Json.Nodes.JsonObject
			{
				["status"] = "described now",
				["spoiler_text"] = "a warning",
				["sensitive"] = true,
				["media_ids"] = new System.Text.Json.Nodes.JsonArray(id),
				["media_attributes"] = new System.Text.Json.Nodes.JsonArray(new System.Text.Json.Nodes.JsonObject { ["id"] = id, ["description"] = "a red square", ["focus"] = "0.5,0.5" })
			})).Ok();

			Assert.Equal("a red square", edited.Body["media_attachments"]![0]!.Text("description"));
			Assert.Equal(0.5, edited.Body["media_attachments"]![0]!["meta"]!["focus"]!["x"]!.GetValue<double>());
			Assert.Equal("a warning", edited.Body.Text("spoiler_text"));
			Assert.True(edited.Body["sensitive"]!.GetValue<bool>());
			Assert.Equal("it", edited.Body.Text("language"));
			Assert.Equal("a red square", (await DB.Default.Find<MediaAttachment>().OneAsync(id, Token)).Description);

			// an edit that doesn't send media keeps them, described as they were; an empty media_ids takes them away
			var kept = (await alice.Client.Put($"/api/v1/statuses/{statusId}", ("status", "media kept"))).Ok();
			Assert.Equal("a red square", kept.Body["media_attachments"]![0]!.Text("description"));
			var bare = (await alice.Client.Json(HttpMethod.Put, $"/api/v1/statuses/{statusId}", new System.Text.Json.Nodes.JsonObject
			{
				["status"] = "no picture now",
				["media_ids"] = new System.Text.Json.Nodes.JsonArray()
			})).Ok();
			Assert.Empty(bare.Body["media_attachments"]!.AsArray());
			Assert.Equal("edited out", (await DB.Default.Find<MediaAttachment>().OneAsync(id, Token)).TrashReason);
		}

		[Fact]
		public async Task A_replaced_avatar_is_trashed_and_a_picture_that_is_not_one_is_refused()
		{
			var alice = await _host.Mastodon("alice");
			async Task<ApiAnswer> Picture(byte[] bytes, string type, string name) =>
				await alice.Client.Exchange(new HttpRequestMessage(HttpMethod.Patch, "/api/v1/accounts/update_credentials")
				{
					Content = MastodonHelpers.Multipart(("avatar", bytes, type, name))
				});
			var first = (await Picture(MastodonHelpers.JpegWithMetadata(300, 300), "image/jpeg", "one.jpg")).Ok().Body.Text("avatar");
			var second = (await Picture(MastodonHelpers.JpegWithMetadata(320, 300), "image/jpeg", "two.jpg")).Ok().Body.Text("avatar");

			Assert.NotEqual(first, second);
			Assert.Equal(HttpStatusCode.NotFound, await Served(first));
			Assert.Equal(HttpStatusCode.OK, await Served(second));
			var row = await DB.Default.Find<MediaAttachment>().Match(m => m.ProfileOfAvatarId == alice.Persona.Id && m.Kind == "avatar" && m.TrashedAt == null).ExecuteSingleAsync(Token);
			Assert.EndsWith(row.FilePath, second);

			// not an image: 422, and the avatar stays as it was
			var refused = await Picture("<svg xmlns='http://www.w3.org/2000/svg'/>"u8.ToArray(), "image/svg+xml", "x.svg");
			Assert.Equal(HttpStatusCode.UnprocessableEntity, refused.Status);
			Assert.Equal(second, (await alice.Client.Get("/api/v1/accounts/verify_credentials")).Ok().Body.Text("avatar"));
		}

		// Mastodon's DELETE /api/v1/profile/avatar and /header: the picture goes, the default shows again
		[Fact]
		public async Task A_picture_is_removed_and_the_default_shows_again()
		{
			var alice = await _host.Mastodon($"pic{Guid.NewGuid():N}"[..12]);
			var form = MastodonHelpers.Multipart(("avatar", MastodonHelpers.JpegWithMetadata(200, 200), "image/jpeg", "me.jpg"),
				("header", MastodonHelpers.JpegWithMetadata(900, 300), "image/jpeg", "top.jpg"));
			var set = (await alice.Client.Exchange(new HttpRequestMessage(HttpMethod.Patch, "/api/v1/accounts/update_credentials") { Content = form })).Ok();
			var avatar = set.Body.Text("avatar");
			var header = set.Body.Text("header");

			var removed = (await alice.Client.Delete("/api/v1/profile/avatar")).Ok();

			Assert.Contains("missing", removed.Body.Text("avatar"));
			Assert.Equal(header, removed.Body.Text("header"));
			Assert.Equal(HttpStatusCode.NotFound, await Served(avatar));
			Assert.Equal(HttpStatusCode.OK, await Served(header));

			Assert.Contains("missing", (await alice.Client.Delete("/api/v1/profile/header")).Ok().Body.Text("header"));
			Assert.Equal(HttpStatusCode.NotFound, await Served(header));
			Assert.False(await DB.Default.Find<MediaAttachment>().Match(m => m.ProfileOfAvatarId == alice.Persona.Id && m.TrashedAt == null).ExecuteAnyAsync(Token));
		}

		[Fact]
		public async Task A_removed_roots_media_and_pictures_are_trashed_and_its_scheduled_posts_dropped()
		{
			var gone = await _host.Mastodon($"gone{Guid.NewGuid():N}"[..12]);
			var posted = await Upload(gone, "posted.jpg");
			var url = (await gone.Client.Post("/api/v1/statuses", ("status", "soon gone"), ("media_ids[]", posted))).Ok()
				.Body["media_attachments"]![0]!["url"]!.GetValue<string>();
			var held = await Upload(gone, "held.jpg");
			(await gone.Client.Post("/api/v1/statuses", ("status", "later"), ("media_ids[]", held),
				("scheduled_at", DateTime.UtcNow.AddHours(2).ToString("O")))).Ok();
			var avatar = (await gone.Client.Exchange(new HttpRequestMessage(HttpMethod.Patch, "/api/v1/accounts/update_credentials")
			{
				Content = MastodonHelpers.Multipart(("avatar", MastodonHelpers.JpegWithMetadata(200, 200), "image/jpeg", "me.jpg"))
			})).Ok().Body.Text("avatar");

			Assert.True(await _host.Get<IRootRemoval>().Remove(gone.Persona.Root.Id, Token));

			Assert.Equal(HttpStatusCode.NotFound, await Served(url));
			Assert.Equal(HttpStatusCode.NotFound, await Served(avatar));
			Assert.False(await DB.Default.Find<MediaAttachment>().Match(m => m.OwnerAvatarId == gone.Persona.Id && m.TrashedAt == null).ExecuteAnyAsync(Token));
			Assert.False(await DB.Default.Find<Models.Social.ScheduledStatus>().Match(s => s.AvatarId == gone.Persona.Id).ExecuteAnyAsync(Token));
		}

		// what a login's media take: every persona's uploads and pictures, trashed ones no longer
		[Fact]
		public async Task A_logins_storage_counts_what_its_personas_keep()
		{
			var alice = await _host.Mastodon($"store{Guid.NewGuid():N}"[..12]);
			using var root = _host.As(alice.Persona.Root.Jwt);
			async Task<System.Text.Json.Nodes.JsonObject> Storage() => await (await root.GetAsync("/clientapi/user/storage", Token)).JsonBody();
			Assert.Equal(0, Storage().Result["mediaBytes"]!.GetValue<long>());

			var id = await Upload(alice, "a.jpg");
			var stored = await Storage();
			Assert.True(stored["mediaBytes"]!.GetValue<long>() > 0);
			Assert.Equal(1, stored["mediaFiles"]!.GetValue<long>());
			Assert.Null(stored["quotaBytes"]);

			await _host.Get<IMediaService>().Trash(m => m.ID == id, "test", Token);
			Assert.Equal(0, (await Storage())["mediaFiles"]!.GetValue<long>());
		}

		[Fact]
		public async Task The_audit_adopts_todays_pictures_and_trashes_what_nothing_holds()
		{
			var media = _host.Get<IMediaService>();
			var bob = await _host.Mastodon($"bob{Guid.NewGuid():N}"[..12]);
			// a picture shown from before pictures had rows, and a leftover nothing holds
			var picture = $"2020/01/{Guid.NewGuid():N}.jpg";
			var leftover = $"2020/01/{Guid.NewGuid():N}.jpg";
			foreach (var relative in new[] { picture, leftover })
			{
				Directory.CreateDirectory(Path.GetDirectoryName(Path.Combine(media.Root, relative))!);
				await File.WriteAllBytesAsync(Path.Combine(media.Root, relative), new byte[] { 1, 2, 3 }, Token);
			}
			await DB.Default.Update<Models.User.Avatar>().MatchID(bob.Persona.Id).Modify(a => a.PictureURL, media.Url(picture)).ExecuteAsync(Token);

			var looked = await MediaAudit.Run(media, fix: false, Token);
			Assert.True(looked.Adopted >= 1);
			Assert.True(looked.Unheld >= 1);
			Assert.True(File.Exists(Path.Combine(media.Root, leftover)));

			await MediaAudit.Run(media, fix: true, Token);

			Assert.True(await DB.Default.Find<MediaAttachment>().Match(m => m.ProfileOfAvatarId == bob.Persona.Id && m.FilePath == picture && m.TrashedAt == null).ExecuteAnyAsync(Token));
			Assert.Equal(HttpStatusCode.OK, await Served(media.Url(picture)));
			Assert.Equal(HttpStatusCode.NotFound, await Served(media.Url(leftover)));
			Assert.True(File.Exists(Path.Combine(media.TrashRoot, leftover)));
			Assert.True(await DB.Default.Find<MediaAttachment>().Match(m => m.FilePath == leftover && m.TrashedAt != null).ExecuteAnyAsync(Token));
		}
	}
}