From dc63fa57b8afd73d1d791e854f507bbf0478d484 Mon Sep 17 00:00:00 2001 From: thepra Date: Wed, 7 Oct 2026 11:03:25 +0200 Subject: [PATCH] A login's storage is counted, and can have a quota MediaAttachment.Size was stored and never summed: nobody, the administrator included, could tell what media took, and nothing bounded it. Counted.Media and MediaOfRoot sum what is kept (not trashed). A login sees what its personas' uploads and pictures take at /clientapi/user/storage (ViewStorage), with its quota when the server sets one; Media:QuotaBytesPerRoot (0, no quota, by default) refuses an upload or a picture over it with 422, whichever persona sends it; the statistics overview gains the media totals, the proxy cache and the trash (ViewMediaTotals). Tests: a login's storage counts what its personas keep and forgets what is trashed; two uploads from two personas of one login are refused together past the quota. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01LsXgEaXee4GCU1hwYgPJXw --- CLAUDE.md | 4 +++ PrivaPub.ClientModels/Admin/ViewStatistics.cs | 10 ++++++++ PrivaPub.ClientModels/User/ViewStorage.cs | 11 ++++++++ PrivaPub.Tests/Domain/MediaFlowTests.cs | 25 +++++++++++++++++++ PrivaPub.Tests/Http/MediaLifecycleTests.cs | 19 ++++++++++++++ .../ClientToServer/RootUserController.cs | 9 +++++++ .../ClientToServer/StatisticsController.cs | 12 +++++++-- PrivaPub/Domain/Media/MediaOptions.cs | 1 + PrivaPub/Domain/Media/MediaService.cs | 17 +++++++++++++ PrivaPub/Domain/Privacy/Counted.cs | 19 ++++++++++++++ .../Domain/Statistics/StatisticsQueries.cs | 10 +++++++- 11 files changed, 134 insertions(+), 3 deletions(-) create mode 100644 PrivaPub.ClientModels/User/ViewStorage.cs diff --git a/CLAUDE.md b/CLAUDE.md index 74eb8e0..aa70ede 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -330,6 +330,10 @@ group www-data and reaches the private mongod; `sudo -u www-data` works too. - its files move into `media-trash` at once, so `/media/files` stops serving them; - `MediaJanitor` deletes them a day later (`TrashGrace`). + What a login's media take (every persona's, trashed ones no longer) is `Counted.MediaOfRoot`, shown to the login at + `/clientapi/user/storage`; `Media:QuotaBytesPerRoot` (0, none, by default) refuses an upload over it with 422; the + administrator's statistics overview has the totals, the proxy cache and the trash. + Nothing is deleted because it looks unused. `PrivaPub admin media audit [--fix]` compares disk and database: with `--fix` (as www-data) it gives pictures shown from before their rows a row, and trashes media of deleted posts or personas, rows whose files are missing, and files nothing holds. diff --git a/PrivaPub.ClientModels/Admin/ViewStatistics.cs b/PrivaPub.ClientModels/Admin/ViewStatistics.cs index 5328fcb..c2cdffb 100644 --- a/PrivaPub.ClientModels/Admin/ViewStatistics.cs +++ b/PrivaPub.ClientModels/Admin/ViewStatistics.cs @@ -12,6 +12,16 @@ namespace PrivaPub.ClientModels.Admin public ViewHostCounts Hosts { get; set; } = new(); public int Accounts { get; set; } public ViewLedgerCounts Ledger { get; set; } = new(); + public ViewMediaTotals Media { get; set; } = new(); + } + + // what media take on the server: the uploads and pictures kept, the remote media cache, the trash waiting its grace + public class ViewMediaTotals + { + public long MediaBytes { get; set; } + public long MediaFiles { get; set; } + public long ProxyCacheBytes { get; set; } + public long TrashBytes { get; set; } } public class ViewLatency diff --git a/PrivaPub.ClientModels/User/ViewStorage.cs b/PrivaPub.ClientModels/User/ViewStorage.cs new file mode 100644 index 0000000..a6f7ec2 --- /dev/null +++ b/PrivaPub.ClientModels/User/ViewStorage.cs @@ -0,0 +1,11 @@ +namespace PrivaPub.ClientModels.User +{ + // /clientapi/user/storage: what the login's media take (every persona's uploads and pictures), and its quota when the + // server sets one + public class ViewStorage + { + public long MediaBytes { get; set; } + public long MediaFiles { get; set; } + public long? QuotaBytes { get; set; } + } +} diff --git a/PrivaPub.Tests/Domain/MediaFlowTests.cs b/PrivaPub.Tests/Domain/MediaFlowTests.cs index 8ddc669..8bc2adb 100644 --- a/PrivaPub.Tests/Domain/MediaFlowTests.cs +++ b/PrivaPub.Tests/Domain/MediaFlowTests.cs @@ -100,6 +100,31 @@ namespace PrivaPub.Tests.Domain Assert.Null(tampered); } + // a login over its quota uploads nothing more, whichever persona tries + [Fact] + public async Task A_login_over_its_quota_uploads_nothing_more() + { + var token = TestContext.Current.CancellationToken; + var (root, alice) = await _harness.Persona("alice"); + var (_, sibling) = await _harness.Persona("sibling", root); + var quota = new MediaService(new StaticOptions(new MediaOptions { Root = _harness.Media.Root, QuotaBytesPerRoot = 3000 }), + _harness.Local, default, Microsoft.Extensions.Logging.Abstractions.NullLogger.Instance); + // noise: its PNG is about as big as its pixels, so two of them are over the quota and one is not + static byte[] Noise() + { + var pixels = new byte[30 * 20 * 3]; + Random.Shared.NextBytes(pixels); + using var image = NetVips.Image.NewFromMemory(pixels, 30, 20, 3, NetVips.Enums.BandFormat.Uchar); + return image.WriteToBuffer(".png"); + } + + Assert.True((await quota.Upload(alice, Upload(Noise(), "image/png"), default, default, false, token)).Ok); + var full = await quota.Upload(sibling, Upload(Noise(), "image/png"), default, default, false, token); + + Assert.False(full.Ok); + Assert.Contains("storage is full", full.Error); + } + // nothing of a suspended server, or of one whose media are rejected, is proxied; blocking one purges what was cached [Fact] public async Task A_blocked_servers_media_are_not_proxied_and_their_cache_goes() diff --git a/PrivaPub.Tests/Http/MediaLifecycleTests.cs b/PrivaPub.Tests/Http/MediaLifecycleTests.cs index c2dbc9b..0cbd82b 100644 --- a/PrivaPub.Tests/Http/MediaLifecycleTests.cs +++ b/PrivaPub.Tests/Http/MediaLifecycleTests.cs @@ -176,6 +176,25 @@ namespace PrivaPub.Tests.Http Assert.False(await DB.Default.Find().Match(s => s.AvatarId == gone.Persona.Id).ExecuteAnyAsync(Token)); } + // what a login's media take: every persona's uploads and pictures, trashed ones no longer + [Fact] + public async Task A_logins_storage_counts_what_its_personas_keep() + { + var alice = await _host.Mastodon($"store{Guid.NewGuid():N}"[..12]); + using var root = _host.As(alice.Persona.Root.Jwt); + async Task Storage() => await (await root.GetAsync("/clientapi/user/storage", Token)).JsonBody(); + Assert.Equal(0, Storage().Result["mediaBytes"]!.GetValue()); + + var id = await Upload(alice, "a.jpg"); + var stored = await Storage(); + Assert.True(stored["mediaBytes"]!.GetValue() > 0); + Assert.Equal(1, stored["mediaFiles"]!.GetValue()); + Assert.Null(stored["quotaBytes"]); + + await _host.Get().Trash(m => m.ID == id, "test", Token); + Assert.Equal(0, (await Storage())["mediaFiles"]!.GetValue()); + } + [Fact] public async Task The_audit_adopts_todays_pictures_and_trashes_what_nothing_holds() { diff --git a/PrivaPub/Controllers/ClientToServer/RootUserController.cs b/PrivaPub/Controllers/ClientToServer/RootUserController.cs index e4f12b5..6e02640 100644 --- a/PrivaPub/Controllers/ClientToServer/RootUserController.cs +++ b/PrivaPub/Controllers/ClientToServer/RootUserController.cs @@ -328,6 +328,15 @@ namespace PrivaPub.Controllers.ClientToServer // } //} + // what the login's media take, every persona's, and its quota when the server sets one + [HttpGet, Route("/clientapi/user/storage"), Authorize(Policy = Policies.IsUser)] + public async Task GetStorage([FromServices] Microsoft.Extensions.Options.IOptionsMonitor media, CancellationToken token) + { + var (bytes, files) = await Domain.Privacy.Counted.MediaOfRoot(User.GetUserId(), token); + var quota = media.CurrentValue.QuotaBytesPerRoot; + return Ok(new PrivaPub.ClientModels.User.ViewStorage { MediaBytes = bytes, MediaFiles = files, QuotaBytes = quota > 0 ? quota : null }); + } + [HttpGet, Route("/clientapi/user/settings"), Authorize(Policy = Policies.IsUser)] public async Task GetUserSettings() { diff --git a/PrivaPub/Controllers/ClientToServer/StatisticsController.cs b/PrivaPub/Controllers/ClientToServer/StatisticsController.cs index 7f64098..2b0f58b 100644 --- a/PrivaPub/Controllers/ClientToServer/StatisticsController.cs +++ b/PrivaPub/Controllers/ClientToServer/StatisticsController.cs @@ -32,8 +32,16 @@ namespace PrivaPub.Controllers.ClientToServer Ok(await _queries.Crawler(_options.CurrentValue.Crawler.Enabled, _options.CurrentValue.Crawler.RevisitDays, token)); [HttpGet, Route("/clientapi/admin/statistics/overview")] - public async Task Overview([FromQuery] int days = 30, CancellationToken token = default) => - Ok(await _queries.Overview(days, token)); + public async Task Overview([FromServices] Domain.Media.IMediaService media, [FromQuery] int days = 30, CancellationToken token = default) + { + var overview = await _queries.Overview(days, token); + overview.Media.ProxyCacheBytes = Bytes(media.ProxyRoot); + overview.Media.TrashBytes = Bytes(media.TrashRoot); + return Ok(overview); + } + + static long Bytes(string root) => + Directory.Exists(root) ? new DirectoryInfo(root).EnumerateFiles("*", SearchOption.AllDirectories).Sum(f => f.Length) : 0; [HttpGet, Route("/clientapi/admin/statistics/hosts")] public async Task Hosts([FromQuery] int days = 30, [FromQuery] string sort = default, [FromQuery] string software = default, diff --git a/PrivaPub/Domain/Media/MediaOptions.cs b/PrivaPub/Domain/Media/MediaOptions.cs index 07a65f6..abd4a91 100644 --- a/PrivaPub/Domain/Media/MediaOptions.cs +++ b/PrivaPub/Domain/Media/MediaOptions.cs @@ -16,5 +16,6 @@ namespace PrivaPub.Domain.Media public long MaxVideoPixels { get; set; } = 2_304_000;//a larger video is made smaller (as Mastodon's video_matrix_limit) public double MaxFrameRate { get; set; } = 60; public int MaxSeconds { get; set; } = 3600;//audio or video may not last longer + public long QuotaBytesPerRoot { get; set; }//what a login's media may take (0: no quota) } } diff --git a/PrivaPub/Domain/Media/MediaService.cs b/PrivaPub/Domain/Media/MediaService.cs index 2262282..f3be238 100644 --- a/PrivaPub/Domain/Media/MediaService.cs +++ b/PrivaPub/Domain/Media/MediaService.cs @@ -120,6 +120,8 @@ namespace PrivaPub.Domain.Media return MediaOutcome.Fail(StatusCodes.Status422UnprocessableEntity, "Validation failed: File can't be blank"); var options = _options.CurrentValue; var contentType = file.ContentType?.Split(';')[0].Trim().ToLowerInvariant(); + if (await OverQuota(owner.Id, file.Length, token) is { } full) + return full; var attachment = new MediaAttachment { @@ -203,6 +205,8 @@ namespace PrivaPub.Domain.Media return MediaOutcome.Fail(StatusCodes.Status422UnprocessableEntity, "Validation failed: File type is not supported"); if (file.Length > _options.CurrentValue.MaxImageBytes) return MediaOutcome.Fail(StatusCodes.Status422UnprocessableEntity, "Validation failed: File is too big"); + if (await OverQuota(avatarId, file.Length, token) is { } full) + return full; var input = await Read(file, token); if (Refusal(input, _options.CurrentValue) is { } refused) return refused; @@ -241,6 +245,19 @@ namespace PrivaPub.Domain.Media return new MediaOutcome(attachment); } + // a login over its quota (Media:QuotaBytesPerRoot; none when 0) uploads nothing more + async Task OverQuota(string avatarId, long incoming, CancellationToken token) + { + var quota = _options.CurrentValue.QuotaBytesPerRoot; + if (quota <= 0) + return default; + var rootId = (await DB.Default.Find().Match(r => r.AvatarId == avatarId).ExecuteFirstAsync(token))?.RootId; + if (rootId == default) + return default; + var (bytes, _) = await Privacy.Counted.MediaOfRoot(rootId, token); + return bytes + incoming > quota ? MediaOutcome.Fail(StatusCodes.Status422UnprocessableEntity, "Validation failed: Your storage is full") : default; + } + public async Task Trash(System.Linq.Expressions.Expression> which, string reason, CancellationToken token) { var trashed = 0L; diff --git a/PrivaPub/Domain/Privacy/Counted.cs b/PrivaPub/Domain/Privacy/Counted.cs index ffb0b7c..6056798 100644 --- a/PrivaPub/Domain/Privacy/Counted.cs +++ b/PrivaPub/Domain/Privacy/Counted.cs @@ -1,3 +1,4 @@ +using MongoDB.Driver; using MongoDB.Entities; using PrivaPub.Federation.Actors; @@ -64,6 +65,24 @@ namespace PrivaPub.Domain.Privacy return distinct.Count == 0 ? 0 : await DB.Default.CountAsync(a => distinct.Contains(a.ID) && !a.DeletionAt.HasValue, token); } + // the media kept (not trashed) that which holds: their bytes and how many + public static async Task<(long Bytes, long Files)> Media(Expression> which, CancellationToken token) + { + var totals = await DB.Default.Fluent() + .Match(m => m.TrashedAt == null) + .Match(which) + .Group(m => true, g => new { Bytes = g.Sum(m => m.Size), Files = g.LongCount() }) + .FirstOrDefaultAsync(token); + return totals == default ? (0, 0) : (totals.Bytes, totals.Files); + } + + // a login's media: every persona's uploads and pictures + public static async Task<(long Bytes, long Files)> MediaOfRoot(string rootId, CancellationToken token) + { + var personas = (await DB.Default.Find().Match(r => r.RootId == rootId).ExecuteAsync(token)).Select(r => r.AvatarId).ToList(); + return await Media(m => personas.Contains(m.OwnerAvatarId), token); + } + public static async Task LocalPosts(CancellationToken token) { var barred = await BarredPersonas(token); diff --git a/PrivaPub/Domain/Statistics/StatisticsQueries.cs b/PrivaPub/Domain/Statistics/StatisticsQueries.cs index 121abed..b75c7f2 100644 --- a/PrivaPub/Domain/Statistics/StatisticsQueries.cs +++ b/PrivaPub/Domain/Statistics/StatisticsQueries.cs @@ -74,10 +74,18 @@ namespace PrivaPub.Domain.Statistics Written = server.Sum(d => d.LedgerWritten), Dropped = server.Sum(d => d.LedgerDropped), Failed = server.Sum(d => d.LedgerFailed) - } + }, + Media = await MediaTotals(token) }; } + // the media rows kept; the disk's caches are added by whoever knows where they are + static async Task MediaTotals(CancellationToken token) + { + var (bytes, files) = await Domain.Privacy.Counted.Media(m => true, token); + return new ViewMediaTotals { MediaBytes = bytes, MediaFiles = files }; + } + public async Task Hosts(int days, string sort, string software, int page, int limit, CancellationToken token) { var hostDays = await Days(days, default, token);