Load: a flood of fake servers; unique-key lookups no longer scan
tools/pasture/flood/flood.cs answers as twenty fake servers (flood1..20.test) and sends signed Creates, Likes and Follows at a set rate; load.sh measures the answers, the queue's wait and processing times, its drain and a persona's home timeline meanwhile (docs/LOAD.md has the method and the runs). What the runs found: - Every unique index was partial on $type: "string", which MongoDB never uses for an equality lookup, so every post by ObjectURI, actor by ActorURI, deleted object, domain block, remote instance and the rest was a collection scan (280 ms a post lookup at 30 000 posts). They are partial on $gt: "" now, which an equality on a string implies; MongoDB.Entities rebuilds them in place at the next start. - Two inbox workers capped intake near 110 activities a second: Federation:InboxConcurrency and DeliveryConcurrency (default 8) set them. - The indexes the plan listed as missing: a post's boosts and replies, a persona's boosts, who follows an actor, timeline rows by author, a post's likes and pins. At 300 activities a second (200 let through, the rest 429 by the per-origin limit) the queue wait went from 29 s to 6 ms at p50; with the limits lifted PrivaPub processes about 900 a second, each in under 10 ms, and the home timeline stays under 20 ms. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01LsXgEaXee4GCU1hwYgPJXw
This commit is contained in:
thepraandClaude Opus 5.5 committed
1 parent
f1e743c331
commit
eeb873b816
12 files changed
+497
-17
No files matched your search
@@ -18,18 +18,23 @@ namespace PrivaPub.Infrastructure.Data
|
||||
|
||||
public static async Task Create(CancellationToken token = default)
|
||||
{
|
||||
await Unique<Post>(p => p.ObjectURI, Builders<Post>.Filter.Type(p => p.ObjectURI, BsonType.String), token);
|
||||
await Unique<Post>(p => p.ObjectURI, Builders<Post>.Filter.Gt(p => p.ObjectURI, ""), token);
|
||||
await Plain<Post>(token, p => p.GroupUserId, p => p.ID);
|
||||
await Plain<Post>(token, p => p.GroupId, p => p.ID);
|
||||
await Plain<Post>(token, p => p.ActorURI);
|
||||
await Plain<Post>(token, p => p.ConversationId, p => p.ID);
|
||||
await Plain<Post>(token, p => p.InReplyToURI);
|
||||
// who boosted what: a persona's boosts of the posts a page shows were a scan of every post under load
|
||||
// (tools/pasture/load.sh), and a post's boosts and replies are read for its counts and context
|
||||
await Plain<Post>(token, p => p.AuthorAccountId, p => p.ReblogOfPostId);
|
||||
await Plain<Post>(token, p => p.ReblogOfPostId);
|
||||
await Plain<Post>(token, p => p.AnsweringToPostId);
|
||||
await DB.Default.Index<Post>().Key(p => p.Geo, KeyType.Geo2DSphere).CreateAsync(token);
|
||||
|
||||
await Unique<DmPost>(p => p.ObjectURI, Builders<DmPost>.Filter.Type(p => p.ObjectURI, BsonType.String), token);
|
||||
await Unique<DmPost>(p => p.ObjectURI, Builders<DmPost>.Filter.Gt(p => p.ObjectURI, ""), token);
|
||||
await Plain<DmPost>(token, p => p.GroupId, p => p.ID);
|
||||
|
||||
await Unique<ForeignAvatar>(a => a.ActorURI, Builders<ForeignAvatar>.Filter.Type(a => a.ActorURI, BsonType.String), token);
|
||||
await Unique<ForeignAvatar>(a => a.ActorURI, Builders<ForeignAvatar>.Filter.Gt(a => a.ActorURI, ""), token);
|
||||
await Plain<ForeignAvatar>(token, a => a.PublicKeyId);
|
||||
|
||||
await DB.Default.Index<Follower>()
|
||||
@@ -47,8 +52,8 @@ namespace PrivaPub.Infrastructure.Data
|
||||
.CreateAsync(token);
|
||||
await Plain<RootToAvatar>(token, r => r.AvatarId);
|
||||
|
||||
await Unique<ReservedName>(r => r.Name, Builders<ReservedName>.Filter.Type(r => r.Name, BsonType.String), token);
|
||||
await Unique<RootUser>(u => u.UserName, Builders<RootUser>.Filter.Type(u => u.UserName, BsonType.String), token);
|
||||
await Unique<ReservedName>(r => r.Name, Builders<ReservedName>.Filter.Gt(r => r.Name, ""), token);
|
||||
await Unique<RootUser>(u => u.UserName, Builders<RootUser>.Filter.Gt(u => u.UserName, ""), token);
|
||||
await Plain<Avatar>(token, a => a.UserName);
|
||||
await Plain<Group>(token, g => g.UserName);
|
||||
await Plain<Group>(token, g => g.InvitationCode);
|
||||
@@ -60,13 +65,13 @@ namespace PrivaPub.Infrastructure.Data
|
||||
.Key(s => s.ConversationId, KeyType.Ascending)
|
||||
.Option(o => o.Unique = true)
|
||||
.CreateAsync(token);
|
||||
await Unique<GroupAnnouncement>(a => a.ActivityURI, Builders<GroupAnnouncement>.Filter.Type(a => a.ActivityURI, BsonType.String), token);
|
||||
await Unique<GroupAnnouncement>(a => a.ActivityURI, Builders<GroupAnnouncement>.Filter.Gt(a => a.ActivityURI, ""), token);
|
||||
|
||||
await Plain<Delivery>(token, d => d.DeliveredAt, d => d.AbandonedAt, d => d.NextAttemptAt);
|
||||
|
||||
await Plain<Job>(token, j => j.Kind, j => j.State, j => j.RunAt);
|
||||
await Plain<Job>(token, j => j.State, j => j.LeasedUntil);
|
||||
await Unique<Job>(j => j.DedupeKey, Builders<Job>.Filter.Type(j => j.DedupeKey, BsonType.String), token);
|
||||
await Unique<Job>(j => j.DedupeKey, Builders<Job>.Filter.Gt(j => j.DedupeKey, ""), token);
|
||||
await DB.Default.Index<Job>()
|
||||
.Key(j => j.FinishedAt, KeyType.Ascending)
|
||||
.Option(o => o.ExpireAfter = TimeSpan.FromDays(7))
|
||||
@@ -78,6 +83,7 @@ namespace PrivaPub.Infrastructure.Data
|
||||
.CreateAsync(token);
|
||||
await Plain<Following>(token, f => f.TargetActorURI, f => f.State);
|
||||
await Plain<Following>(token, f => f.TargetAccountId, f => f.State);
|
||||
await Plain<Following>(token, f => f.TargetActorURI, f => f.State);//who here follows the author of what arrives
|
||||
await Plain<Following>(token, f => f.FollowActivityURI);
|
||||
await DB.Default.Index<TimelineEntry>()
|
||||
.Key(e => e.AvatarId, KeyType.Ascending)
|
||||
@@ -85,7 +91,8 @@ namespace PrivaPub.Infrastructure.Data
|
||||
.Option(o => o.Unique = true)
|
||||
.CreateAsync(token);
|
||||
await Plain<TimelineEntry>(token, e => e.PostId);
|
||||
await Unique<Notification>(n => n.DedupeKey, Builders<Notification>.Filter.Type(n => n.DedupeKey, BsonType.String), token);
|
||||
await Plain<TimelineEntry>(token, e => e.AuthorAccountId);
|
||||
await Unique<Notification>(n => n.DedupeKey, Builders<Notification>.Filter.Gt(n => n.DedupeKey, ""), token);
|
||||
await Plain<Notification>(token, n => n.AvatarId, n => n.ID);
|
||||
await DB.Default.Index<Favourite>()
|
||||
.Key(f => f.AccountId, KeyType.Ascending)
|
||||
@@ -93,6 +100,7 @@ namespace PrivaPub.Infrastructure.Data
|
||||
.Option(o => o.Unique = true)
|
||||
.CreateAsync(token);
|
||||
await Plain<Favourite>(token, f => f.ActivityURI);
|
||||
await Plain<Favourite>(token, f => f.PostId);
|
||||
await DB.Default.Index<Participation>()
|
||||
.Key(p => p.AvatarId, KeyType.Ascending)
|
||||
.Key(p => p.PostId, KeyType.Ascending)
|
||||
@@ -117,6 +125,7 @@ namespace PrivaPub.Infrastructure.Data
|
||||
})
|
||||
await pair.Item1();
|
||||
await Plain<Block>(token, b => b.TargetActorURI);
|
||||
await Plain<Pin>(token, p => p.PostId);
|
||||
await Plain<BlockedBy>(token, b => b.ActorURI);
|
||||
await Plain<PersonaList>(token, l => l.AvatarId);
|
||||
await Plain<PersonaFilter>(token, f => f.AvatarId);
|
||||
@@ -130,11 +139,11 @@ namespace PrivaPub.Infrastructure.Data
|
||||
await Plain<PersonaListMember>(token, m => m.AvatarId);
|
||||
await Plain<Mute>(token, m => m.TargetActorURI);
|
||||
await Plain<Report>(token, r => r.IsResolved, r => r.ID);
|
||||
await Unique<DomainBlock>(b => b.Domain, Builders<DomainBlock>.Filter.Type(b => b.Domain, BsonType.String), token);
|
||||
await Unique<RemoteInstance>(i => i.Host, Builders<RemoteInstance>.Filter.Type(i => i.Host, BsonType.String), token);
|
||||
await Unique<ObjectRecord>(r => r.ObjectURI, Builders<ObjectRecord>.Filter.Type(r => r.ObjectURI, BsonType.String), token);
|
||||
await Unique<DomainBlock>(b => b.Domain, Builders<DomainBlock>.Filter.Gt(b => b.Domain, ""), token);
|
||||
await Unique<RemoteInstance>(i => i.Host, Builders<RemoteInstance>.Filter.Gt(i => i.Host, ""), token);
|
||||
await Unique<ObjectRecord>(r => r.ObjectURI, Builders<ObjectRecord>.Filter.Gt(r => r.ObjectURI, ""), token);
|
||||
await Plain<ObjectRecord>(token, r => r.PostId);
|
||||
await Unique<DeletedObject>(d => d.ObjectURI, Builders<DeletedObject>.Filter.Type(d => d.ObjectURI, BsonType.String), token);
|
||||
await Unique<DeletedObject>(d => d.ObjectURI, Builders<DeletedObject>.Filter.Gt(d => d.ObjectURI, ""), token);
|
||||
await DB.Default.Index<DeletedObject>().Key(d => d.DeletedAt, KeyType.Ascending).Option(o => o.ExpireAfter = TombstoneLifetime).CreateAsync(token);
|
||||
await DB.Default.Index<Downvote>().Key(d => d.PostId, KeyType.Ascending).Key(d => d.ActorURI, KeyType.Ascending).Option(o => o.Unique = true).CreateAsync(token);
|
||||
await Plain<Downvote>(token, d => d.ActivityURI);
|
||||
@@ -145,7 +154,7 @@ namespace PrivaPub.Infrastructure.Data
|
||||
.Option(o => o.Unique = true).CreateAsync(token);
|
||||
await Plain<Reaction>(token, r => r.ActivityURI);
|
||||
await DB.Default.Index<QuoteLicence>().Key(l => l.PostId, KeyType.Ascending).Key(l => l.QuotingObjectURI, KeyType.Ascending).CreateAsync(token);
|
||||
await Unique<Domain.Content.LinkPreview>(p => p.Url, Builders<Domain.Content.LinkPreview>.Filter.Type(p => p.Url, BsonType.String), token);
|
||||
await Unique<Domain.Content.LinkPreview>(p => p.Url, Builders<Domain.Content.LinkPreview>.Filter.Gt(p => p.Url, ""), token);
|
||||
|
||||
await DB.Default.Index<InteractionEvent>().Key(e => e.At, KeyType.Ascending).Option(o => o.ExpireAfter = InteractionEvent.Retention).CreateAsync(token);
|
||||
await Plain<InteractionEvent>(token, e => e.Host, e => e.At);
|
||||
@@ -164,6 +173,10 @@ namespace PrivaPub.Infrastructure.Data
|
||||
await Plain<RemoteInstance>(token, i => i.LastCrawledAt);
|
||||
}
|
||||
|
||||
// unique among the documents that have the key: the partial filter is `$gt: ""` (any non-empty string) rather than
|
||||
// `$type: "string"`, because MongoDB uses a partial index for a lookup only when the lookup implies its filter, and an
|
||||
// equality on a string implies the first, never the second: every lookup by these keys scanned the collection
|
||||
// (tools/pasture/load.sh, 280 ms a lookup at 30 000 posts)
|
||||
static async Task Unique<T>(System.Linq.Expressions.Expression<Func<T, object>> key, FilterDefinition<T> partial,
|
||||
CancellationToken token) where T : IEntity =>
|
||||
await DB.Default.Index<T>()
|
||||
|
||||
Reference in new issue
Block a user