SocialPub/PrivaPub/Domain/Media/MediaProxy.cs

using Microsoft.Extensions.Options;

using MongoDB.Entities;

using PrivaPub.Federation.Actors;
using PrivaPub.Infrastructure.Http;
using PrivaPub.Models.Media;

using System.Security.Cryptography;
using System.Text;

namespace PrivaPub.Domain.Media
{
	public interface IMediaProxy
	{
		string Wrap(string remoteUrl);
		Task<(string Path, string ContentType)> Fetch(string signature, string encodedUrl, CancellationToken token);
		string Verified(string signature, string encodedUrl);
		(string Path, string ContentType) Cached(string url);
		Task<HttpResponseMessage> Open(string url, System.Net.Http.Headers.RangeHeaderValue range, CancellationToken token);
	}

	public class MediaProxy : IMediaProxy
	{
		readonly ILocalActorService _localActors;
		readonly IFederationHttp _http;
		readonly IMediaService _media;
		readonly IOptionsMonitor<MediaOptions> _options;
		byte[] _key;

		public MediaProxy(ILocalActorService localActors, IFederationHttp http, IMediaService media, IOptionsMonitor<MediaOptions> options)
		{
			_localActors = localActors;
			_http = http;
			_media = media;
			_options = options;
		}

		byte[] Key => _key ??= LoadKey();

		public string Wrap(string remoteUrl)
		{
			if (string.IsNullOrEmpty(remoteUrl) || remoteUrl.StartsWith(_localActors.BaseAddress + "/", StringComparison.OrdinalIgnoreCase))
				return remoteUrl;
			var encoded = Base64Url(Encoding.UTF8.GetBytes(remoteUrl));
			return $"{_localActors.BaseAddress}/media/proxy/{Sign(remoteUrl)}/{encoded}";
		}

		public string Verified(string signature, string encodedUrl)
		{
			string url;
			try
			{
				url = Encoding.UTF8.GetString(FromBase64Url(encodedUrl));
			}
			catch (FormatException)
			{
				return default;
			}
			return CryptographicOperations.FixedTimeEquals(Encoding.ASCII.GetBytes(signature ?? string.Empty), Encoding.ASCII.GetBytes(Sign(url))) ? url : default;
		}

		public (string Path, string ContentType) Cached(string url)
		{
			var (path, typePath) = CachePaths(url);
			if (!File.Exists(path) || !File.Exists(typePath))
				return default;
			File.SetLastWriteTimeUtc(path, DateTime.UtcNow);
			return (path, File.ReadAllText(typePath));
		}

		public Task<HttpResponseMessage> Open(string url, System.Net.Http.Headers.RangeHeaderValue range, CancellationToken token) =>
			_http.OpenMedia(url, range, token);

		(string Path, string TypePath) CachePaths(string url)
		{
			var name = Convert.ToHexStringLower(SHA256.HashData(Encoding.UTF8.GetBytes(url)));
			var path = System.IO.Path.Combine(_media.ProxyRoot, name[..2], name);
			return (path, path + ".type");
		}

		public async Task<(string Path, string ContentType)> Fetch(string signature, string encodedUrl, CancellationToken token)
		{
			var url = Verified(signature, encodedUrl);
			if (url == default)
				return default;
			if (Cached(url) is { Path: not null } cached)
				return cached;
			var (path, typePath) = CachePaths(url);
			var directory = System.IO.Path.GetDirectoryName(path);

			var (bytes, contentType) = await _http.GetMedia(url, _options.CurrentValue.MaxProxiedBytes, token);
			if (bytes == default)
				return default;
			Directory.CreateDirectory(directory);
			await File.WriteAllBytesAsync(path, bytes, token);
			await File.WriteAllTextAsync(typePath, contentType, token);
			return (path, contentType);
		}

		string Sign(string url) => Base64Url(HMACSHA256.HashData(Key, Encoding.UTF8.GetBytes(url))[..16]);

		static byte[] LoadKey()
		{
			var secret = DB.Default.Find<MediaSecret>().ExecuteFirstAsync().GetAwaiter().GetResult();
			if (secret == default)
			{
				secret = new MediaSecret { Key = Convert.ToBase64String(RandomNumberGenerator.GetBytes(32)) };
				DB.Default.SaveAsync(secret).GetAwaiter().GetResult();
				secret = DB.Default.Find<MediaSecret>().ExecuteFirstAsync().GetAwaiter().GetResult();
			}
			return Convert.FromBase64String(secret.Key);
		}

		static string Base64Url(byte[] bytes) => Convert.ToBase64String(bytes).TrimEnd('=').Replace('+', '-').Replace('/', '_');

		static byte[] FromBase64Url(string value)
		{
			var padded = value.Replace('-', '+').Replace('_', '/');
			return Convert.FromBase64String(padded + new string('=', (4 - padded.Length % 4) % 4));
		}
	}

	public class MediaJanitor : BackgroundService
	{
		static readonly TimeSpan Interval = TimeSpan.FromHours(1);
		static readonly TimeSpan UnattachedLifetime = TimeSpan.FromDays(1);

		readonly IMediaService _media;
		readonly IOptionsMonitor<MediaOptions> _options;
		readonly ILogger<MediaJanitor> _logger;

		public MediaJanitor(IMediaService media, IOptionsMonitor<MediaOptions> options, ILogger<MediaJanitor> logger)
		{
			_media = media;
			_options = options;
			_logger = logger;
		}

		protected override async Task ExecuteAsync(CancellationToken stoppingToken)
		{
			while (!stoppingToken.IsCancellationRequested)
			{
				try
				{
					await Task.Delay(Interval, stoppingToken);
					await Sweep(stoppingToken);
				}
				catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
				{
					return;
				}
				catch (Exception ex)
				{
					_logger.LogWarning(ex, "{Service} pass failed", nameof(MediaJanitor));
				}
			}
		}

		//one pass: uploads left unattached for a day go, then the proxy cache is trimmed to its size, oldest first
		public async Task Sweep(CancellationToken token)
		{
			var cutoff = DateTime.UtcNow - UnattachedLifetime;
			foreach (var stale in await DB.Default.Find<MediaAttachment>().Match(m => m.PostId == null && m.CreatedAt < cutoff).Limit(500).ExecuteAsync(token))
				await _media.Delete(stale);
			TrimProxyCache();
		}

		void TrimProxyCache()
		{
			var directory = new DirectoryInfo(_media.ProxyRoot);
			if (!directory.Exists)
				return;
			var files = directory.EnumerateFiles("*", SearchOption.AllDirectories).Where(f => f.Extension != ".type").OrderBy(f => f.LastWriteTimeUtc).ToList();
			var total = files.Sum(f => f.Length);
			foreach (var file in files)
			{
				if (total <= _options.CurrentValue.ProxyCacheBytes)
					break;
				total -= file.Length;
				file.Delete();
				var type = new FileInfo(file.FullName + ".type");
				if (type.Exists)
					type.Delete();
			}
		}
	}
}