The media proxy is bounded
Anyone could mint signed proxy URLs (a remote account changes its icon, an anonymous lookup returns the URL), and each anonymous request held up to 40 MB in memory; the cache grew without bound between hourly trims; two clients asking for the same new file downloaded it twice and wrote over each other in place, so a reader could get half a file with a 7-day cache header; a file over the limit was downloaded twice on every request; a failed fetch, a 404, was cached by browsers for a week; cached media of a server suspended later were still served, and RejectMedia skipped avatars, emoji, covers, video variants, link cards and remote edits; /clientapi/group/members returned remote pictures raw. Now a download is shared by everyone asking at once, streamed into a .part file and renamed into place (FederationHttp.DownloadMedia copies bounded, never into memory), at most eight at a time; a file too big to cache is remembered for an hour and only streamed, a failure for five minutes; the cache's size is counted as it grows and trimmed as soon as it passes the cap; a cached file is opened before it is answered; browsers may cache only a success; nothing of a suspended server, or of one whose media are rejected, is proxied (everything remote a client sees goes through the proxy, so that covers every kind), and blocking one purges its cache; the proxy has its own rate limit per client address; group members' pictures are proxied; the proxy's key is loaded once, the oldest if two were made. This changes what PrivaPub serves its clients, not what it sends to other servers. Tests: clients asking at once share one download, a failure isn't cached by browsers, an over-limit file is fetched three times for two requests instead of four, a blocked server's media are refused and its cache purged. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01LsXgEaXee4GCU1hwYgPJXw
This commit is contained in:
1 parent
8e59145826
commit
3ca29603ed
11 files changed
+399
-73
No files matched your search
@@ -415,6 +415,36 @@ namespace PrivaPub.Tests.Http
|
||||
Assert.Single(_peer.Requests);
|
||||
}
|
||||
|
||||
// clients asking at once for a file not cached yet share one download
|
||||
[Fact]
|
||||
public async Task Clients_asking_at_once_share_one_download()
|
||||
{
|
||||
var proxy = _host.Get<IMediaProxy>();
|
||||
var bytes = Bytes(200_000);
|
||||
var remote = Served(bytes, "image/png");
|
||||
var path = new Uri(remote).AbsolutePath;
|
||||
using var client = _host.Client();
|
||||
|
||||
var answers = await Task.WhenAll(Enumerable.Range(0, 5).Select(_ => client.GetAsync(proxy.Wrap(remote), Token)));
|
||||
|
||||
foreach (var answer in answers)
|
||||
Assert.Equal(bytes, await answer.Content.ReadAsByteArrayAsync(Token));
|
||||
Assert.Single(_peer.Requests, r => r.Path == path);
|
||||
}
|
||||
|
||||
// a remote file that can't be had is a 404 no browser keeps
|
||||
[Fact]
|
||||
public async Task A_remote_file_that_fails_is_not_cached_by_browsers()
|
||||
{
|
||||
var proxy = _host.Get<IMediaProxy>();
|
||||
using var client = _host.Client();
|
||||
|
||||
using var missing = await client.GetAsync(proxy.Wrap($"{_peer.A}/media/{Guid.NewGuid():N}.png"), Token);
|
||||
|
||||
Assert.Equal(HttpStatusCode.NotFound, missing.StatusCode);
|
||||
Assert.Null(missing.Headers.CacheControl);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Anything_over_the_proxy_limit_is_streamed_and_never_cached()
|
||||
{
|
||||
@@ -432,7 +462,8 @@ namespace PrivaPub.Tests.Http
|
||||
Assert.Equal(default, proxy.Cached(remote));
|
||||
using var again = await client.GetAsync(proxy.Wrap(remote), Token);
|
||||
Assert.Equal(bytes, await again.Content.ReadAsByteArrayAsync(Token));
|
||||
Assert.Equal(4, _peer.Requests.Count);
|
||||
// the first time, an attempt to cache it and the stream; then it is known to be too big, and only streamed
|
||||
Assert.Equal(3, _peer.Requests.Count);
|
||||
var small = Served(Bytes(SmallProxyHost.Limit / 2));
|
||||
using (var fits = await client.GetAsync(proxy.Wrap(small), Token))
|
||||
Assert.Equal(HttpStatusCode.OK, fits.StatusCode);
|
||||
|
||||
Reference in new issue
Block a user