using Microsoft.Extensions.Options; using MongoDB.Entities; using PrivaPub.Federation.Actors; using PrivaPub.Infrastructure.Http; using PrivaPub.Models.Media; using System.Security.Cryptography; using System.Text; namespace PrivaPub.Domain.Media { public interface IMediaProxy { string Wrap(string remoteUrl); Task<(string Path, string ContentType)> Fetch(string signature, string encodedUrl, CancellationToken token); string Verified(string signature, string encodedUrl); (string Path, string ContentType) Cached(string url); Task Open(string url, System.Net.Http.Headers.RangeHeaderValue range, CancellationToken token); } public class MediaProxy : IMediaProxy { readonly ILocalActorService _localActors; readonly IFederationHttp _http; readonly IMediaService _media; readonly IOptionsMonitor _options; byte[] _key; public MediaProxy(ILocalActorService localActors, IFederationHttp http, IMediaService media, IOptionsMonitor options) { _localActors = localActors; _http = http; _media = media; _options = options; } byte[] Key => _key ??= LoadKey(); public string Wrap(string remoteUrl) { if (string.IsNullOrEmpty(remoteUrl) || remoteUrl.StartsWith(_localActors.BaseAddress + "/", StringComparison.OrdinalIgnoreCase)) return remoteUrl; var encoded = Base64Url(Encoding.UTF8.GetBytes(remoteUrl)); return $"{_localActors.BaseAddress}/media/proxy/{Sign(remoteUrl)}/{encoded}"; } public string Verified(string signature, string encodedUrl) { string url; try { url = Encoding.UTF8.GetString(FromBase64Url(encodedUrl)); } catch (FormatException) { return default; } return CryptographicOperations.FixedTimeEquals(Encoding.ASCII.GetBytes(signature ?? string.Empty), Encoding.ASCII.GetBytes(Sign(url))) ? url : default; } public (string Path, string ContentType) Cached(string url) { var (path, typePath) = CachePaths(url); if (!File.Exists(path) || !File.Exists(typePath)) return default; File.SetLastWriteTimeUtc(path, DateTime.UtcNow); return (path, File.ReadAllText(typePath)); } public Task Open(string url, System.Net.Http.Headers.RangeHeaderValue range, CancellationToken token) => _http.OpenMedia(url, range, token); (string Path, string TypePath) CachePaths(string url) { var name = Convert.ToHexStringLower(SHA256.HashData(Encoding.UTF8.GetBytes(url))); var path = System.IO.Path.Combine(_media.ProxyRoot, name[..2], name); return (path, path + ".type"); } public async Task<(string Path, string ContentType)> Fetch(string signature, string encodedUrl, CancellationToken token) { var url = Verified(signature, encodedUrl); if (url == default) return default; if (Cached(url) is { Path: not null } cached) return cached; var (path, typePath) = CachePaths(url); var directory = System.IO.Path.GetDirectoryName(path); var (bytes, contentType) = await _http.GetMedia(url, _options.CurrentValue.MaxProxiedBytes, token); if (bytes == default) return default; Directory.CreateDirectory(directory); await File.WriteAllBytesAsync(path, bytes, token); await File.WriteAllTextAsync(typePath, contentType, token); return (path, contentType); } string Sign(string url) => Base64Url(HMACSHA256.HashData(Key, Encoding.UTF8.GetBytes(url))[..16]); static byte[] LoadKey() { var secret = DB.Default.Find().ExecuteFirstAsync().GetAwaiter().GetResult(); if (secret == default) { secret = new MediaSecret { Key = Convert.ToBase64String(RandomNumberGenerator.GetBytes(32)) }; DB.Default.SaveAsync(secret).GetAwaiter().GetResult(); secret = DB.Default.Find().ExecuteFirstAsync().GetAwaiter().GetResult(); } return Convert.FromBase64String(secret.Key); } static string Base64Url(byte[] bytes) => Convert.ToBase64String(bytes).TrimEnd('=').Replace('+', '-').Replace('/', '_'); static byte[] FromBase64Url(string value) { var padded = value.Replace('-', '+').Replace('_', '/'); return Convert.FromBase64String(padded + new string('=', (4 - padded.Length % 4) % 4)); } } public class MediaJanitor : BackgroundService { static readonly TimeSpan FirstPass = TimeSpan.FromMinutes(5); static readonly TimeSpan Interval = TimeSpan.FromHours(1); static readonly TimeSpan UnattachedLifetime = TimeSpan.FromDays(1); // a trashed file waits this long before it is deleted, already out of what /media/files serves public static readonly TimeSpan TrashGrace = TimeSpan.FromDays(1); readonly IMediaService _media; readonly IOptionsMonitor _options; readonly ILogger _logger; public MediaJanitor(IMediaService media, IOptionsMonitor options, ILogger logger) { _media = media; _options = options; _logger = logger; } protected override async Task ExecuteAsync(CancellationToken stoppingToken) { var wait = FirstPass; while (!stoppingToken.IsCancellationRequested) { try { await Task.Delay(wait, stoppingToken); wait = Interval; await Sweep(stoppingToken); } catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) { return; } catch (Exception ex) { _logger.LogWarning(ex, "{Service} pass failed", nameof(MediaJanitor)); } } } // one pass: // - uploads never posted for a day (and not waiting for a scheduled post, nor a profile picture) go to the trash; // - trashed files still served (a crash between the mark and the move) are moved out; // - trashed files past their grace are deleted, with their rows; // - the proxy cache is trimmed to its size, oldest first public async Task Sweep(CancellationToken token) { var cutoff = DateTime.UtcNow - UnattachedLifetime; var unattached = await _media.Trash(m => m.PostId == null && m.ScheduledStatusId == null && m.ProfileOfAvatarId == null && m.CreatedAt < cutoff, "never posted", token); var trashed = await DB.Default.Find().Match(m => m.TrashedAt != null).Limit(2000).ExecuteAsync(token); var purgeBefore = DateTime.UtcNow - TrashGrace; var purged = 0; foreach (var row in trashed) { if (row.TrashedAt < purgeBefore) { await _media.Purge(row, token); purged++; } else _media.Hide(row); } if (unattached > 0 || purged > 0) _logger.LogInformation("{Service}: {Unattached} uploads never posted trashed, {Purged} trashed files deleted", nameof(MediaJanitor), unattached, purged); TrimProxyCache(); } void TrimProxyCache() { var directory = new DirectoryInfo(_media.ProxyRoot); if (!directory.Exists) return; var files = directory.EnumerateFiles("*", SearchOption.AllDirectories).Where(f => f.Extension != ".type").OrderBy(f => f.LastWriteTimeUtc).ToList(); var total = files.Sum(f => f.Length); foreach (var file in files) { if (total <= _options.CurrentValue.ProxyCacheBytes) break; total -= file.Length; file.Delete(); var type = new FileInfo(file.FullName + ".type"); if (type.Exists) type.Delete(); } } } }