SocialPub/PrivaPub/Domain/Media/MediaProxy.cs

using Microsoft.Extensions.Options;

using MongoDB.Entities;

using PrivaPub.Federation.Actors;
using PrivaPub.Infrastructure.Http;
using PrivaPub.Models.Media;

using System.Security.Cryptography;
using System.Text;

namespace PrivaPub.Domain.Media
{
	public interface IMediaProxy
	{
		string Wrap(string remoteUrl);
		Task<(string Path, string ContentType)> Fetch(string signature, string encodedUrl, CancellationToken token);
		string Verified(string signature, string encodedUrl);
		(string Path, string ContentType) Cached(string url);
		Task<HttpResponseMessage> Open(string url, System.Net.Http.Headers.RangeHeaderValue range, CancellationToken token);
	}

	public class MediaProxy : IMediaProxy
	{
		readonly ILocalActorService _localActors;
		readonly IFederationHttp _http;
		readonly IMediaService _media;
		readonly IOptionsMonitor<MediaOptions> _options;
		byte[] _key;

		public MediaProxy(ILocalActorService localActors, IFederationHttp http, IMediaService media, IOptionsMonitor<MediaOptions> options)
		{
			_localActors = localActors;
			_http = http;
			_media = media;
			_options = options;
		}

		byte[] Key => _key ??= LoadKey();

		public string Wrap(string remoteUrl)
		{
			if (string.IsNullOrEmpty(remoteUrl) || remoteUrl.StartsWith(_localActors.BaseAddress + "/", StringComparison.OrdinalIgnoreCase))
				return remoteUrl;
			var encoded = Base64Url(Encoding.UTF8.GetBytes(remoteUrl));
			return $"{_localActors.BaseAddress}/media/proxy/{Sign(remoteUrl)}/{encoded}";
		}

		public string Verified(string signature, string encodedUrl)
		{
			string url;
			try
			{
				url = Encoding.UTF8.GetString(FromBase64Url(encodedUrl));
			}
			catch (FormatException)
			{
				return default;
			}
			return CryptographicOperations.FixedTimeEquals(Encoding.ASCII.GetBytes(signature ?? string.Empty), Encoding.ASCII.GetBytes(Sign(url))) ? url : default;
		}

		public (string Path, string ContentType) Cached(string url)
		{
			var (path, typePath) = CachePaths(url);
			if (!File.Exists(path) || !File.Exists(typePath))
				return default;
			File.SetLastWriteTimeUtc(path, DateTime.UtcNow);
			return (path, File.ReadAllText(typePath));
		}

		public Task<HttpResponseMessage> Open(string url, System.Net.Http.Headers.RangeHeaderValue range, CancellationToken token) =>
			_http.OpenMedia(url, range, token);

		(string Path, string TypePath) CachePaths(string url)
		{
			var name = Convert.ToHexStringLower(SHA256.HashData(Encoding.UTF8.GetBytes(url)));
			var path = System.IO.Path.Combine(_media.ProxyRoot, name[..2], name);
			return (path, path + ".type");
		}

		public async Task<(string Path, string ContentType)> Fetch(string signature, string encodedUrl, CancellationToken token)
		{
			var url = Verified(signature, encodedUrl);
			if (url == default)
				return default;
			if (Cached(url) is { Path: not null } cached)
				return cached;
			var (path, typePath) = CachePaths(url);
			var directory = System.IO.Path.GetDirectoryName(path);

			var (bytes, contentType) = await _http.GetMedia(url, _options.CurrentValue.MaxProxiedBytes, token);
			if (bytes == default)
				return default;
			Directory.CreateDirectory(directory);
			await File.WriteAllBytesAsync(path, bytes, token);
			await File.WriteAllTextAsync(typePath, contentType, token);
			return (path, contentType);
		}

		string Sign(string url) => Base64Url(HMACSHA256.HashData(Key, Encoding.UTF8.GetBytes(url))[..16]);

		static byte[] LoadKey()
		{
			var secret = DB.Default.Find<MediaSecret>().ExecuteFirstAsync().GetAwaiter().GetResult();
			if (secret == default)
			{
				secret = new MediaSecret { Key = Convert.ToBase64String(RandomNumberGenerator.GetBytes(32)) };
				DB.Default.SaveAsync(secret).GetAwaiter().GetResult();
				secret = DB.Default.Find<MediaSecret>().ExecuteFirstAsync().GetAwaiter().GetResult();
			}
			return Convert.FromBase64String(secret.Key);
		}

		static string Base64Url(byte[] bytes) => Convert.ToBase64String(bytes).TrimEnd('=').Replace('+', '-').Replace('/', '_');

		static byte[] FromBase64Url(string value)
		{
			var padded = value.Replace('-', '+').Replace('_', '/');
			return Convert.FromBase64String(padded + new string('=', (4 - padded.Length % 4) % 4));
		}
	}

	public class MediaJanitor : BackgroundService
	{
		static readonly TimeSpan FirstPass = TimeSpan.FromMinutes(5);
		static readonly TimeSpan Interval = TimeSpan.FromHours(1);
		static readonly TimeSpan UnattachedLifetime = TimeSpan.FromDays(1);
		// a trashed file waits this long before it is deleted, already out of what /media/files serves
		public static readonly TimeSpan TrashGrace = TimeSpan.FromDays(1);

		readonly IMediaService _media;
		readonly IOptionsMonitor<MediaOptions> _options;
		readonly ILogger<MediaJanitor> _logger;

		public MediaJanitor(IMediaService media, IOptionsMonitor<MediaOptions> options, ILogger<MediaJanitor> logger)
		{
			_media = media;
			_options = options;
			_logger = logger;
		}

		protected override async Task ExecuteAsync(CancellationToken stoppingToken)
		{
			var wait = FirstPass;
			while (!stoppingToken.IsCancellationRequested)
			{
				try
				{
					await Task.Delay(wait, stoppingToken);
					wait = Interval;
					await Sweep(stoppingToken);
				}
				catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
				{
					return;
				}
				catch (Exception ex)
				{
					_logger.LogWarning(ex, "{Service} pass failed", nameof(MediaJanitor));
				}
			}
		}

		// one pass:
		// - uploads never posted for a day (and not waiting for a scheduled post, nor a profile picture) go to the trash;
		// - trashed files still served (a crash between the mark and the move) are moved out;
		// - trashed files past their grace are deleted, with their rows;
		// - the proxy cache is trimmed to its size, oldest first
		public async Task Sweep(CancellationToken token)
		{
			var cutoff = DateTime.UtcNow - UnattachedLifetime;
			var unattached = await _media.Trash(m => m.PostId == null && m.ScheduledStatusId == null && m.ProfileOfAvatarId == null && m.CreatedAt < cutoff,
				"never posted", token);

			var trashed = await DB.Default.Find<MediaAttachment>().Match(m => m.TrashedAt != null).Limit(2000).ExecuteAsync(token);
			var purgeBefore = DateTime.UtcNow - TrashGrace;
			var purged = 0;
			foreach (var row in trashed)
			{
				if (row.TrashedAt < purgeBefore)
				{
					await _media.Purge(row, token);
					purged++;
				}
				else
					_media.Hide(row);
			}
			if (unattached > 0 || purged > 0)
				_logger.LogInformation("{Service}: {Unattached} uploads never posted trashed, {Purged} trashed files deleted",
					nameof(MediaJanitor), unattached, purged);
			TrimProxyCache();
		}

		void TrimProxyCache()
		{
			var directory = new DirectoryInfo(_media.ProxyRoot);
			if (!directory.Exists)
				return;
			var files = directory.EnumerateFiles("*", SearchOption.AllDirectories).Where(f => f.Extension != ".type").OrderBy(f => f.LastWriteTimeUtc).ToList();
			var total = files.Sum(f => f.Length);
			foreach (var file in files)
			{
				if (total <= _options.CurrentValue.ProxyCacheBytes)
					break;
				total -= file.Length;
				file.Delete();
				var type = new FileInfo(file.FullName + ".type");
				if (type.Exists)
					type.Delete();
			}
		}
	}
}