From d2f0c7a14e5711a22856462fe23fcb694c795036 Mon Sep 17 00:00:00 2001 From: thepra Date: Thu, 1 Oct 2026 11:01:08 +0200 Subject: [PATCH] Unique indexes, one username space, and entity maps warmed before use Infrastructure/Data/Indexes runs at every start, after the migrations: unique on Post/DmPost ObjectURI, ForeignAvatar ActorURI, the Follower triple, RootToAvatar, RootUser UserName and ReservedName; plain indexes on the lookups the services actually make (PublicKeyId, author and group post listings, ParticipantsKey, the delivery queue). Migration _001 runs first and removes the duplicates the races could already have left (keeping the newest actor row, the oldest post, the accepted follower), then fills ReservedName from every avatar and group. ReservedName is one username space for personas, groups and the instance: a name is reserved by an insert the unique index arbitrates, before the avatar or group is saved, so two simultaneous sign-ups cannot both get it. A short list of names (admin, support, abuse, postmaster, ...) is never available. EntityMaps.Warm touches every entity's collection one at a time before anything else runs. MongoDB.Entities maps the Entity base class on first touch, and two types mapped at once throw "An item with the same key has already been added" and stay broken for the life of the process; the parallel test run hit it, and the delivery worker racing a request could have too. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012CzABvBkbcFqoHdmi8b9WB --- PrivaPub.Tests/Infrastructure/IndexTests.cs | 62 ++++++++++++++++ PrivaPub.Tests/Support/MongoFixture.cs | 3 + .../Federation/Actors/LocalActorService.cs | 28 +++++++- PrivaPub/Infrastructure/Data/EntityMaps.cs | 18 +++++ PrivaPub/Infrastructure/Data/Indexes.cs | 72 +++++++++++++++++++ .../_001_deduplicate_before_unique_indexes.cs | 59 +++++++++++++++ PrivaPub/Models/User/ReservedName.cs | 14 ++++ PrivaPub/Program.cs | 3 + .../Private/IPrivateAvatarUsersService.cs | 4 ++ PrivaPub/Services/GroupUsersService.cs | 3 + 10 files changed, 263 insertions(+), 3 deletions(-) create mode 100644 PrivaPub.Tests/Infrastructure/IndexTests.cs create mode 100644 PrivaPub/Infrastructure/Data/EntityMaps.cs create mode 100644 PrivaPub/Infrastructure/Data/Indexes.cs create mode 100644 PrivaPub/Infrastructure/Data/Migrations/_001_deduplicate_before_unique_indexes.cs create mode 100644 PrivaPub/Models/User/ReservedName.cs diff --git a/PrivaPub.Tests/Infrastructure/IndexTests.cs b/PrivaPub.Tests/Infrastructure/IndexTests.cs new file mode 100644 index 0000000..072216e --- /dev/null +++ b/PrivaPub.Tests/Infrastructure/IndexTests.cs @@ -0,0 +1,62 @@ +using MongoDB.Driver; +using MongoDB.Entities; + +using PrivaPub.Federation.Actors; +using PrivaPub.Infrastructure.Data; +using PrivaPub.Infrastructure.Data.Migrations; +using PrivaPub.Models; +using PrivaPub.Models.Federation; +using PrivaPub.Models.User; +using PrivaPub.StaticServices; +using PrivaPub.Tests.Support; + +namespace PrivaPub.Tests.Infrastructure +{ + [Trait("Category", "Integration")] + public class IndexTests + { + [Fact] + public async Task Duplicates_are_removed_and_the_unique_indexes_then_hold() + { + Assert.SkipUnless(MongoFixture.Enabled, MongoFixture.Skip); + var token = TestContext.Current.CancellationToken; + var actorUri = $"https://r.example/users/{Guid.NewGuid():N}"; + var localId = Guid.NewGuid().ToString("N")[..24]; + await DB.Default.SaveAsync(new[] + { + new ForeignAvatar { ActorURI = actorUri, Name = "old", UpdatedAt = DateTime.UtcNow.AddDays(-2) }, + new ForeignAvatar { ActorURI = actorUri, Name = "new", UpdatedAt = DateTime.UtcNow } + }, token); + await DB.Default.SaveAsync(new[] + { + new Follower { LocalActorId = localId, ActorURI = actorUri, IsAccepted = false }, + new Follower { LocalActorId = localId, ActorURI = actorUri, IsAccepted = true } + }, token); + + await new _001_deduplicate_before_unique_indexes().UpgradeAsync(); + await Indexes.Create(token); + + Assert.Equal("new", (await DB.Default.Find().Match(a => a.ActorURI == actorUri).ExecuteSingleAsync(token)).Name); + Assert.True((await DB.Default.Find().Match(f => f.LocalActorId == localId).ExecuteSingleAsync(token)).IsAccepted); + var duplicate = await Assert.ThrowsAsync(async () => + await DB.Default.SaveAsync(new ForeignAvatar { ActorURI = actorUri }, token)); + Assert.Equal(ServerErrorCategory.DuplicateKey, duplicate.WriteError.Category); + } + + [Fact] + public async Task A_username_is_reserved_once_across_personas_and_groups() + { + Assert.SkipUnless(MongoFixture.Enabled, MongoFixture.Skip); + var token = TestContext.Current.CancellationToken; + await Indexes.Create(token); + var local = new LocalActorService(new DbEntities(), new StaticOptions(new AppConfiguration { BackendBaseAddress = "https://privapub.test" })); + var name = $"n{Guid.NewGuid():N}"[..20]; + + Assert.True(await local.TryReserveUserName(name, LocalActorKind.Person, "a", token)); + Assert.False(await local.TryReserveUserName(name.ToUpperInvariant(), LocalActorKind.Group, "b", token)); + Assert.True(await local.IsUserNameTaken(name, token)); + Assert.False(await local.TryReserveUserName("admin", LocalActorKind.Person, "c", token)); + Assert.False(await local.TryReserveUserName(LocalActorService.InstanceUserName, LocalActorKind.Person, "c", token)); + } + } +} diff --git a/PrivaPub.Tests/Support/MongoFixture.cs b/PrivaPub.Tests/Support/MongoFixture.cs index b4972ca..2eeb807 100644 --- a/PrivaPub.Tests/Support/MongoFixture.cs +++ b/PrivaPub.Tests/Support/MongoFixture.cs @@ -4,6 +4,8 @@ using MongoDB.Bson.Serialization.Serializers; using MongoDB.Driver; using MongoDB.Entities; +using PrivaPub.Infrastructure.Data; + [assembly: AssemblyFixture(typeof(PrivaPub.Tests.Support.MongoFixture))] namespace PrivaPub.Tests.Support @@ -29,6 +31,7 @@ namespace PrivaPub.Tests.Support } var connection = Environment.GetEnvironmentVariable("PRIVAPUB_TEST_MONGO") ?? "mongodb://127.0.0.1:27017"; await DB.InitAsync(Database, MongoClientSettings.FromConnectionString(connection)); + EntityMaps.Warm(); } public async ValueTask DisposeAsync() diff --git a/PrivaPub/Federation/Actors/LocalActorService.cs b/PrivaPub/Federation/Actors/LocalActorService.cs index a20a9bb..1247880 100644 --- a/PrivaPub/Federation/Actors/LocalActorService.cs +++ b/PrivaPub/Federation/Actors/LocalActorService.cs @@ -1,5 +1,6 @@ using Microsoft.Extensions.Options; +using MongoDB.Driver; using MongoDB.Entities; using PrivaPub.Models; @@ -52,6 +53,7 @@ namespace PrivaPub.Federation.Actors Task FindByUri(string actorUri, CancellationToken token); Task GetInstanceActor(CancellationToken token); Task IsUserNameTaken(string userName, CancellationToken token); + Task TryReserveUserName(string userName, LocalActorKind kind, string ownerId, CancellationToken token); LocalActor FromAvatar(Avatar avatar); LocalActor FromGroup(GroupEntity group); } @@ -60,6 +62,12 @@ namespace PrivaPub.Federation.Actors { public const string InstanceUserName = "privapub"; + static readonly HashSet ReservedByInstance = new(StringComparer.Ordinal) + { + InstanceUserName, "admin", "administrator", "root", "system", "support", "help", "moderator", "mod", + "abuse", "postmaster", "webmaster", "hostmaster", "security", "noreply", "no_reply", "null", "undefined" + }; + readonly DbEntities _dbEntities; readonly IOptionsMonitor _appConfiguration; InstanceActor _instanceActor; @@ -149,11 +157,25 @@ namespace PrivaPub.Federation.Actors public async Task IsUserNameTaken(string userName, CancellationToken token) { userName = userName?.ToLowerInvariant(); - if (string.IsNullOrEmpty(userName) || userName == InstanceUserName) + if (string.IsNullOrEmpty(userName) || ReservedByInstance.Contains(userName)) return true; - if (await _dbEntities.Avatars.Match(a => a.UserName == userName).ExecuteAnyAsync(token)) + return await DB.Default.Find().Match(r => r.Name == userName).ExecuteAnyAsync(token); + } + + public async Task TryReserveUserName(string userName, LocalActorKind kind, string ownerId, CancellationToken token) + { + userName = userName?.ToLowerInvariant(); + if (string.IsNullOrEmpty(userName) || ReservedByInstance.Contains(userName)) + return false; + try + { + await DB.Default.SaveAsync(new ReservedName { Name = userName, OwnerKind = kind, OwnerId = ownerId }, token); return true; - return await _dbEntities.Groups.Match(g => g.UserName == userName).ExecuteAnyAsync(token); + } + catch (MongoWriteException ex) when (ex.WriteError?.Category == ServerErrorCategory.DuplicateKey) + { + return false; + } } public LocalActor FromAvatar(Avatar avatar) => new() diff --git a/PrivaPub/Infrastructure/Data/EntityMaps.cs b/PrivaPub/Infrastructure/Data/EntityMaps.cs new file mode 100644 index 0000000..bbad55c --- /dev/null +++ b/PrivaPub/Infrastructure/Data/EntityMaps.cs @@ -0,0 +1,18 @@ +using MongoDB.Entities; + +namespace PrivaPub.Infrastructure.Data +{ + public static class EntityMaps + { + public static void Warm() + { + var collection = typeof(DB).GetMethods() + .Single(m => m.Name == nameof(DB.Collection) && m.IsGenericMethodDefinition && m.GetParameters().Length == 0); + var entities = typeof(EntityMaps).Assembly.GetTypes() + .Where(t => t is { IsClass: true, IsAbstract: false, IsGenericTypeDefinition: false } && typeof(IEntity).IsAssignableFrom(t)) + .OrderBy(t => t.FullName, StringComparer.Ordinal); + foreach (var entity in entities) + collection.MakeGenericMethod(entity).Invoke(DB.Default, null); + } + } +} diff --git a/PrivaPub/Infrastructure/Data/Indexes.cs b/PrivaPub/Infrastructure/Data/Indexes.cs new file mode 100644 index 0000000..428d207 --- /dev/null +++ b/PrivaPub/Infrastructure/Data/Indexes.cs @@ -0,0 +1,72 @@ +using MongoDB.Bson; +using MongoDB.Driver; +using MongoDB.Entities; + +using PrivaPub.Models.Federation; +using PrivaPub.Models.Group; +using PrivaPub.Models.Post; +using PrivaPub.Models.User; + +namespace PrivaPub.Infrastructure.Data +{ + public static class Indexes + { + public static async Task Create(CancellationToken token = default) + { + await Unique(p => p.ObjectURI, Builders.Filter.Type(p => p.ObjectURI, BsonType.String), token); + await Plain(token, p => p.GroupUserId, p => p.ID); + await Plain(token, p => p.GroupId, p => p.ID); + await Plain(token, p => p.ActorURI); + + await Unique(p => p.ObjectURI, Builders.Filter.Type(p => p.ObjectURI, BsonType.String), token); + await Plain(token, p => p.GroupId, p => p.ID); + + await Unique(a => a.ActorURI, Builders.Filter.Type(a => a.ActorURI, BsonType.String), token); + await Plain(token, a => a.PublicKeyId); + + await DB.Default.Index() + .Key(f => f.LocalActorId, KeyType.Ascending) + .Key(f => f.LocalActorKind, KeyType.Ascending) + .Key(f => f.ActorURI, KeyType.Ascending) + .Option(o => o.Unique = true) + .CreateAsync(token); + await Plain(token, f => f.ActorURI); + + await DB.Default.Index() + .Key(r => r.RootId, KeyType.Ascending) + .Key(r => r.AvatarId, KeyType.Ascending) + .Option(o => o.Unique = true) + .CreateAsync(token); + await Plain(token, r => r.AvatarId); + + await Unique(r => r.Name, Builders.Filter.Type(r => r.Name, BsonType.String), token); + await Unique(u => u.UserName, Builders.Filter.Type(u => u.UserName, BsonType.String), token); + await Plain(token, a => a.UserName); + await Plain(token, g => g.UserName); + await Plain(token, g => g.InvitationCode); + + await Plain(token, g => g.ParticipantsKey); + + await Plain(token, d => d.DeliveredAt, d => d.AbandonedAt, d => d.NextAttemptAt); + } + + static async Task Unique(System.Linq.Expressions.Expression> key, FilterDefinition partial, + CancellationToken token) where T : IEntity => + await DB.Default.Index() + .Key(key, KeyType.Ascending) + .Option(o => + { + o.Unique = true; + o.PartialFilterExpression = partial; + }) + .CreateAsync(token); + + static async Task Plain(CancellationToken token, params System.Linq.Expressions.Expression>[] keys) where T : IEntity + { + var index = DB.Default.Index(); + foreach (var key in keys) + index.Key(key, KeyType.Ascending); + await index.CreateAsync(token); + } + } +} diff --git a/PrivaPub/Infrastructure/Data/Migrations/_001_deduplicate_before_unique_indexes.cs b/PrivaPub/Infrastructure/Data/Migrations/_001_deduplicate_before_unique_indexes.cs new file mode 100644 index 0000000..df5e188 --- /dev/null +++ b/PrivaPub/Infrastructure/Data/Migrations/_001_deduplicate_before_unique_indexes.cs @@ -0,0 +1,59 @@ +using MongoDB.Entities; + +using PrivaPub.Federation.Actors; +using PrivaPub.Models.Federation; +using PrivaPub.Models.Group; +using PrivaPub.Models.Post; +using PrivaPub.Models.User; + +namespace PrivaPub.Infrastructure.Data.Migrations +{ + public class _001_deduplicate_before_unique_indexes : IMigration + { + public async Task UpgradeAsync() + { + var foreignAvatars = await DB.Default.Find().Match(a => a.ActorURI != null).ExecuteAsync(); + await DeleteAllBut(foreignAvatars.GroupBy(a => a.ActorURI).Select(g => g.OrderByDescending(a => a.UpdatedAt)), DeleteForeignAvatar); + + var posts = await DB.Default.Find().Match(p => p.ObjectURI != null).ExecuteAsync(); + await DeleteAllBut(posts.GroupBy(p => p.ObjectURI).Select(g => g.OrderBy(p => p.ID, StringComparer.Ordinal)), DeletePost); + + var dmPosts = await DB.Default.Find().Match(p => p.ObjectURI != null).ExecuteAsync(); + await DeleteAllBut(dmPosts.GroupBy(p => p.ObjectURI).Select(g => g.OrderBy(p => p.ID, StringComparer.Ordinal)), DeleteDmPost); + + var followers = await DB.Default.Find().ExecuteAsync(); + await DeleteAllBut(followers.GroupBy(f => (f.LocalActorId, f.LocalActorKind, f.ActorURI)) + .Select(g => g.OrderByDescending(f => f.IsAccepted).ThenBy(f => f.ID, StringComparer.Ordinal)), DeleteFollower); + + var links = await DB.Default.Find().ExecuteAsync(); + await DeleteAllBut(links.GroupBy(r => (r.RootId, r.AvatarId)).Select(g => g.OrderBy(r => r.ID, StringComparer.Ordinal)), DeleteLink); + + var taken = new HashSet(StringComparer.Ordinal) { LocalActorService.InstanceUserName }; + var reservations = new List + { + new() { Name = LocalActorService.InstanceUserName, OwnerKind = LocalActorKind.Application } + }; + foreach (var avatar in (await DB.Default.Find().ExecuteAsync()).OrderBy(a => a.ID, StringComparer.Ordinal)) + if (!string.IsNullOrEmpty(avatar.UserName) && taken.Add(avatar.UserName.ToLowerInvariant())) + reservations.Add(new() { Name = avatar.UserName.ToLowerInvariant(), OwnerKind = LocalActorKind.Person, OwnerId = avatar.ID }); + foreach (var group in (await DB.Default.Find().ExecuteAsync()).OrderBy(g => g.ID, StringComparer.Ordinal)) + if (!string.IsNullOrEmpty(group.UserName) && taken.Add(group.UserName.ToLowerInvariant())) + reservations.Add(new() { Name = group.UserName.ToLowerInvariant(), OwnerKind = LocalActorKind.Group, OwnerId = group.ID }); + await DB.Default.DeleteAsync(_ => true); + await DB.Default.SaveAsync(reservations); + } + + static async Task DeleteAllBut(IEnumerable> groups, Func delete) + { + foreach (var group in groups) + foreach (var duplicate in group.Skip(1)) + await delete(duplicate); + } + + static Task DeleteForeignAvatar(ForeignAvatar a) => DB.Default.DeleteAsync(a.ID); + static Task DeletePost(Post p) => DB.Default.DeleteAsync(p.ID); + static Task DeleteDmPost(DmPost p) => DB.Default.DeleteAsync(p.ID); + static Task DeleteFollower(Follower f) => DB.Default.DeleteAsync(f.ID); + static Task DeleteLink(RootToAvatar r) => DB.Default.DeleteAsync(r.ID); + } +} diff --git a/PrivaPub/Models/User/ReservedName.cs b/PrivaPub/Models/User/ReservedName.cs new file mode 100644 index 0000000..7c4b917 --- /dev/null +++ b/PrivaPub/Models/User/ReservedName.cs @@ -0,0 +1,14 @@ +using MongoDB.Entities; + +using PrivaPub.Models.Federation; + +namespace PrivaPub.Models.User +{ + public class ReservedName : Entity + { + public string Name { get; set; } + public LocalActorKind OwnerKind { get; set; } + public string OwnerId { get; set; } + public DateTime CreationDate { get; set; } = DateTime.UtcNow; + } +} diff --git a/PrivaPub/Program.cs b/PrivaPub/Program.cs index 27f44a2..83aef11 100644 --- a/PrivaPub/Program.cs +++ b/PrivaPub/Program.cs @@ -12,6 +12,7 @@ using Serilog; using PrivaPub.Data; using PrivaPub.Extensions; using PrivaPub.Infrastructure.Cli; +using PrivaPub.Infrastructure.Data; using PrivaPub.Infrastructure.Http; using PrivaPub.Middleware; using PrivaPub.Models; @@ -68,7 +69,9 @@ try BsonSerializer.RegisterSerializer(new GuidSerializer(GuidRepresentation.Standard)); var mongoSettings = builder.Configuration.GetSection(nameof(MongoSettings)).Get(); await DB.InitAsync(mongoSettings.Database, MongoClientSettings.FromConnectionString(mongoSettings.ConnectionString)); + EntityMaps.Warm(); await DB.Default.MigrateAsync(); + await Indexes.Create(); if (args is ["admin", ..]) { diff --git a/PrivaPub/Services/ClientToServer/Private/IPrivateAvatarUsersService.cs b/PrivaPub/Services/ClientToServer/Private/IPrivateAvatarUsersService.cs index a8ff825..9eb2b21 100644 --- a/PrivaPub/Services/ClientToServer/Private/IPrivateAvatarUsersService.cs +++ b/PrivaPub/Services/ClientToServer/Private/IPrivateAvatarUsersService.cs @@ -5,6 +5,7 @@ using MongoDB.Entities; using PrivaPub.ClientModels; using PrivaPub.ClientModels.User; using PrivaPub.ClientModels.User.Avatar; +using PrivaPub.Models.Federation; using PrivaPub.Models.User; using PrivaPub.Resources; using PrivaPub.StaticServices; @@ -64,6 +65,9 @@ namespace PrivaPub.Services.ClientToServer.Private PublicKey = publicKey, Domain = _localActors.BaseAddress }; + newAvatar.ID = (string)newAvatar.GenerateNewID(); + if (!await _localActors.TryReserveUserName(userName, LocalActorKind.Person, newAvatar.ID, default)) + return result.Invalidate(_localizer["The username '{0}' is already take.", userName]); if (form.Settings is { IsDefault: false }) newAvatar.Settings = ToSettings(form.Settings); diff --git a/PrivaPub/Services/GroupUsersService.cs b/PrivaPub/Services/GroupUsersService.cs index be2b23f..d926bbb 100644 --- a/PrivaPub/Services/GroupUsersService.cs +++ b/PrivaPub/Services/GroupUsersService.cs @@ -123,6 +123,9 @@ namespace PrivaPub.Services HashedInvitationPassword = string.IsNullOrEmpty(form.InvitationPassword) ? default : _passwordHasher.Hash(form.InvitationPassword), Members = new() { new GroupMember { AvatarId = form.AvatarId, Role = GroupRole.Owner } } }; + group.ID = (string)group.GenerateNewID(); + if (!await _localActors.TryReserveUserName(userName, LocalActorKind.Group, group.ID, token)) + return result.Invalidate(_localizer["The username '{0}' is already take.", userName]); var actor = _localActors.FromGroup(group); group.Url = actor.Uri; group.InboxURL = actor.Inbox;