The media proxy is bounded

Anyone could mint signed proxy URLs (a remote account changes its icon, an anonymous lookup returns the URL), and each
anonymous request held up to 40 MB in memory; the cache grew without bound between hourly trims; two clients asking
for the same new file downloaded it twice and wrote over each other in place, so a reader could get half a file with a
7-day cache header; a file over the limit was downloaded twice on every request; a failed fetch, a 404, was cached by
browsers for a week; cached media of a server suspended later were still served, and RejectMedia skipped avatars,
emoji, covers, video variants, link cards and remote edits; /clientapi/group/members returned remote pictures raw.

Now a download is shared by everyone asking at once, streamed into a .part file and renamed into place
(FederationHttp.DownloadMedia copies bounded, never into memory), at most eight at a time; a file too big to cache is
remembered for an hour and only streamed, a failure for five minutes; the cache's size is counted as it grows and
trimmed as soon as it passes the cap; a cached file is opened before it is answered; browsers may cache only a
success; nothing of a suspended server, or of one whose media are rejected, is proxied (everything remote a client
sees goes through the proxy, so that covers every kind), and blocking one purges its cache; the proxy has its own rate
limit per client address; group members' pictures are proxied; the proxy's key is loaded once, the oldest if two
were made. This changes what PrivaPub serves its clients, not what it sends to other servers.

Tests: clients asking at once share one download, a failure isn't cached by browsers, an over-limit file is fetched
three times for two requests instead of four, a blocked server's media are refused and its cache purged.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01LsXgEaXee4GCU1hwYgPJXw

PrivaPub/Infrastructure/Http/FederationHttp.cs

@@ -29,6 +29,9 @@ namespace PrivaPub.Infrastructure.Http
 		Task<HttpResponseMessage> OpenMedia(string url, System.Net.Http.Headers.RangeHeaderValue range, CancellationToken token);
 		Task<HttpResponseMessage> Send(HttpRequestMessage request, CancellationToken token);
 		Task<(byte[] Bytes, string ContentType)> GetMedia(string url, long maxBytes, CancellationToken token);
+		/// <summary>Copies a media file into <paramref name="destination"/>, up to maxBytes: its type, or why it was refused
+		/// ("too-large", "content-type", ...).</summary>
+		Task<(string ContentType, string Refusal)> DownloadMedia(string url, long maxBytes, Stream destination, CancellationToken token);
 		Task<(int Status, string Text)> GetText(string url, int maxBytes, CancellationToken token);
 		Task<IReadOnlyList<string>> GetStringArray(string url, int maxItems, CancellationToken token);
 	}
@@ -389,14 +392,29 @@ namespace PrivaPub.Infrastructure.Http
 			Uri.TryCreate(url, UriKind.Absolute, out var target) && _cache.TryGetValue(NegativeKey(target), out Refusal refusal) && refusal.Transient;
 
 		public async Task<(byte[] Bytes, string ContentType)> GetMedia(string url, long maxBytes, CancellationToken token)
+		{
+			var bytes = default(byte[]);
+			var (contentType, _) = await ReadMedia(url, maxBytes, async (content, limit, t) =>
+			{
+				bytes = await ReadBounded(content, (int)Math.Min(limit, int.MaxValue), t);
+				return bytes?.Length ?? -1;
+			}, token);
+			return contentType == default ? default : (bytes, contentType);
+		}
+
+		public Task<(string ContentType, string Refusal)> DownloadMedia(string url, long maxBytes, Stream destination, CancellationToken token) =>
+			ReadMedia(url, maxBytes, (content, limit, t) => CopyBounded(content, destination, limit, t), token);
+
+		// a media file read through consume (which answers how many bytes it took, or -1 past the limit): its type, or why not
+		async Task<(string ContentType, string Refusal)> ReadMedia(string url, long maxBytes, Func<HttpContent, long, CancellationToken, Task<long>> consume, CancellationToken token)
 		{
 			var exchange = new Exchange(url, "media");
 			try
 			{
-				var media = await GetMedia(url, maxBytes, exchange, token);
-				if (media.Bytes == default && exchange.Outcome == Interactions.Ok)
+				var contentType = await ReadMedia(url, maxBytes, consume, exchange, token);
+				if (contentType == default && exchange.Outcome == Interactions.Ok)
 					exchange.Refused("unusable");
-				return media;
+				return (contentType, contentType == default ? exchange.Reason ?? "unusable" : default);
 			}
 			finally
 			{
@@ -404,7 +422,7 @@ namespace PrivaPub.Infrastructure.Http
 			}
 		}
 
-		async Task<(byte[] Bytes, string ContentType)> GetMedia(string url, long maxBytes, Exchange exchange, CancellationToken token)
+		async Task<string> ReadMedia(string url, long maxBytes, Func<HttpContent, long, CancellationToken, Task<long>> consume, Exchange exchange, CancellationToken token)
 		{
 			if (!Uri.TryCreate(url, UriKind.Absolute, out var target) || !IsAllowed(target))
 			{
@@ -451,14 +469,14 @@ namespace PrivaPub.Infrastructure.Http
 						exchange.Refused("too-large");
 						return default;
 					}
-					var bytes = await ReadBounded(response.Content, (int)Math.Min(maxBytes, int.MaxValue), timeout.Token);
-					if (bytes == default)
+					var taken = await consume(response.Content, maxBytes, timeout.Token);
+					if (taken < 0)
 					{
 						exchange.Refused("too-large");
 						return default;
 					}
-					exchange.Bytes = bytes.Length;
-					return (bytes, mediaType);
+					exchange.Bytes = taken;
+					return mediaType;
 				}
 				exchange.Refused("too-many-redirects");
 				return default;
@@ -608,6 +626,23 @@ namespace PrivaPub.Infrastructure.Http
 			return await _httpClientFactory.CreateClient(ClientName).SendAsync(request, HttpCompletionOption.ResponseHeadersRead, token);
 		}
 
+		// copies at most limit bytes: how many, or -1 once there are more
+		static async Task<long> CopyBounded(HttpContent content, Stream destination, long limit, CancellationToken token)
+		{
+			await using var stream = await content.ReadAsStreamAsync(token);
+			var chunk = new byte[64 * 1024];
+			var copied = 0L;
+			int read;
+			while ((read = await stream.ReadAsync(chunk, token)) > 0)
+			{
+				copied += read;
+				if (copied > limit)
+					return -1;
+				await destination.WriteAsync(chunk.AsMemory(0, read), token);
+			}
+			return copied;
+		}
+
 		public static async Task<byte[]> ReadBounded(HttpContent content, int limit, CancellationToken token)
 		{
 			await using var stream = await content.ReadAsStreamAsync(token);

PrivaPub/Infrastructure/RateLimiting.cs

@@ -17,6 +17,8 @@ namespace PrivaPub.Infrastructure
 		public int InboxPerTenSeconds { get; set; } = 50;//and the rate it earns them back
 		public int UploadsBurst { get; set; } = 30;//uploads (media, profile pictures) a session may make at once
 		public int UploadsPerMinute { get; set; } = 10;//and the rate it earns them back
+		public int ProxyBurst { get; set; } = 1200;//remote media a client address may ask for at once (a timeline is many pictures)
+		public int ProxyPerMinute { get; set; } = 600;//and the rate it earns them back
 	}
 
 	public static class RateLimiting
@@ -24,6 +26,7 @@ namespace PrivaPub.Infrastructure
 		public const string Accounts = "accounts";
 		public const string Inbox = "inbox";
 		public const string Uploads = "uploads";
+		public const string Proxy = "proxy";
 
 		static RateLimitOptions Limits(HttpContext context) => context.RequestServices.GetRequiredService<IOptions<RateLimitOptions>>().Value;
 
@@ -62,6 +65,15 @@ namespace PrivaPub.Infrastructure
 						ReplenishmentPeriod = TimeSpan.FromSeconds(10),
 						QueueLimit = 0
 					}));
+				options.AddPolicy(Proxy, context => RateLimitPartition.GetTokenBucketLimiter(
+					context.Connection.RemoteIpAddress?.ToString() ?? "unknown",
+					_ => new TokenBucketRateLimiterOptions
+					{
+						TokenLimit = Limits(context).ProxyBurst,
+						TokensPerPeriod = Limits(context).ProxyPerMinute,
+						ReplenishmentPeriod = TimeSpan.FromMinutes(1),
+						QueueLimit = 0
+					}));
 				// per session: the limiter runs before authentication, so the credential sent stands for whoever sends it
 				options.AddPolicy(Uploads, context => RateLimitPartition.GetTokenBucketLimiter(
 					Credential(context.Request) ?? "anonymous:" + context.Connection.RemoteIpAddress,