using MongoDB.Bson; using MongoDB.Driver; using MongoDB.Entities; using PrivaPub.Models.Federation; using PrivaPub.Models.Group; using PrivaPub.Models.Jobs; using PrivaPub.Models.Post; using PrivaPub.Models.Social; using PrivaPub.Models.Statistics; using PrivaPub.Models.User; namespace PrivaPub.Infrastructure.Data { public static class Indexes { public static readonly TimeSpan TombstoneLifetime = TimeSpan.FromDays(90); public static async Task Create(CancellationToken token = default) { await Unique(p => p.ObjectURI, Builders.Filter.Gt(p => p.ObjectURI, ""), token); await Plain(token, p => p.GroupUserId, p => p.ID); await Plain(token, p => p.GroupId, p => p.ID); await Plain(token, p => p.ActorURI); await Plain(token, p => p.WallURI, p => p.ID);//a persona's wall (FEP-400e) await Plain(token, p => p.ConversationId, p => p.ID); await Plain(token, p => p.InReplyToURI); // who boosted what: a persona's boosts of the posts a page shows were a scan of every post under load // (tools/pasture/load.sh), and a post's boosts and replies are read for its counts and context await Plain(token, p => p.AuthorAccountId, p => p.ReblogOfPostId); await Plain(token, p => p.ReblogOfPostId); await Plain(token, p => p.AnsweringToPostId); await DB.Default.Index().Key(p => p.Geo, KeyType.Geo2DSphere).CreateAsync(token); await Unique(p => p.ObjectURI, Builders.Filter.Gt(p => p.ObjectURI, ""), token); await Plain(token, p => p.GroupId, p => p.ID); await Unique(a => a.ActorURI, Builders.Filter.Gt(a => a.ActorURI, ""), token); await Plain(token, a => a.PublicKeyId); await DB.Default.Index() .Key(f => f.LocalActorId, KeyType.Ascending) .Key(f => f.LocalActorKind, KeyType.Ascending) .Key(f => f.ActorURI, KeyType.Ascending) .Option(o => o.Unique = true) .CreateAsync(token); await Plain(token, f => f.ActorURI); await DB.Default.Index() .Key(r => r.RootId, KeyType.Ascending) .Key(r => r.AvatarId, KeyType.Ascending) .Option(o => o.Unique = true) .CreateAsync(token); await Plain(token, r => r.AvatarId); await Unique(r => r.Name, Builders.Filter.Gt(r => r.Name, ""), token); await Unique(u => u.UserName, Builders.Filter.Gt(u => u.UserName, ""), token); await Plain(token, a => a.UserName); await Plain(token, g => g.UserName); await Plain(token, g => g.InvitationCode); await Plain(token, g => g.ParticipantsKey); await Plain(token, g => g.LastPostId); await DB.Default.Index() .Key(s => s.AvatarId, KeyType.Ascending) .Key(s => s.ConversationId, KeyType.Ascending) .Option(o => o.Unique = true) .CreateAsync(token); await Unique(a => a.ActivityURI, Builders.Filter.Gt(a => a.ActivityURI, ""), token); await Plain(token, d => d.DeliveredAt, d => d.AbandonedAt, d => d.NextAttemptAt); await Plain(token, j => j.Kind, j => j.State, j => j.RunAt); await Plain(token, j => j.State, j => j.LeasedUntil); await Unique(j => j.DedupeKey, Builders.Filter.Gt(j => j.DedupeKey, ""), token); await DB.Default.Index() .Key(j => j.FinishedAt, KeyType.Ascending) .Option(o => o.ExpireAfter = TimeSpan.FromDays(7)) .CreateAsync(token); await DB.Default.Index() .Key(f => f.AvatarId, KeyType.Ascending) .Key(f => f.TargetActorURI, KeyType.Ascending) .Option(o => o.Unique = true) .CreateAsync(token); await Plain(token, f => f.TargetActorURI, f => f.State); await Plain(token, f => f.TargetAccountId, f => f.State); await Plain(token, f => f.TargetActorURI, f => f.State);//who here follows the author of what arrives await Plain(token, f => f.FollowActivityURI); await DB.Default.Index() .Key(e => e.AvatarId, KeyType.Ascending) .Key(e => e.PostId, KeyType.Ascending) .Option(o => o.Unique = true) .CreateAsync(token); await Plain(token, e => e.PostId); await Plain(token, e => e.AuthorAccountId); await Unique(n => n.DedupeKey, Builders.Filter.Gt(n => n.DedupeKey, ""), token); await Plain(token, n => n.AvatarId, n => n.ID); await DB.Default.Index() .Key(f => f.AccountId, KeyType.Ascending) .Key(f => f.PostId, KeyType.Ascending) .Option(o => o.Unique = true) .CreateAsync(token); await Plain(token, f => f.ActivityURI); await Plain(token, f => f.PostId); await DB.Default.Index() .Key(p => p.AvatarId, KeyType.Ascending) .Key(p => p.PostId, KeyType.Ascending) .Option(o => o.Unique = true) .CreateAsync(token); await Plain(token, p => p.ActivityURI); await DB.Default.Index() .Key(m => m.AvatarId, KeyType.Ascending) .Key(m => m.Timeline, KeyType.Ascending) .Option(o => o.Unique = true) .CreateAsync(token); foreach (var pair in new (Func, string)[] { (() => DB.Default.Index().Key(b => b.AvatarId, KeyType.Ascending).Key(b => b.TargetActorURI, KeyType.Ascending).Option(o => o.Unique = true).CreateAsync(token), "block"), (() => DB.Default.Index().Key(t => t.AvatarId, KeyType.Ascending).Key(t => t.Name, KeyType.Ascending).Option(o => o.Unique = true).CreateAsync(token), "followed tag"), (() => DB.Default.Index().Key(m => m.ListId, KeyType.Ascending).Key(m => m.AccountId, KeyType.Ascending).Option(o => o.Unique = true).CreateAsync(token), "list member"), (() => DB.Default.Index().Key(b => b.AvatarId, KeyType.Ascending).Key(b => b.ActorURI, KeyType.Ascending).Option(o => o.Unique = true).CreateAsync(token), "blocked by"), (() => DB.Default.Index().Key(m => m.AvatarId, KeyType.Ascending).Key(m => m.TargetActorURI, KeyType.Ascending).Option(o => o.Unique = true).CreateAsync(token), "mute"), (() => DB.Default.Index().Key(b => b.AvatarId, KeyType.Ascending).Key(b => b.Domain, KeyType.Ascending).Option(o => o.Unique = true).CreateAsync(token), "domain block"), (() => DB.Default.Index().Key(b => b.AvatarId, KeyType.Ascending).Key(b => b.PostId, KeyType.Ascending).Option(o => o.Unique = true).CreateAsync(token), "bookmark"), (() => DB.Default.Index().Key(p => p.AvatarId, KeyType.Ascending).Key(p => p.PostId, KeyType.Ascending).Option(o => o.Unique = true).CreateAsync(token), "pin"), (() => DB.Default.Index().Key(r => r.Configured, KeyType.Ascending).Option(o => o.Unique = true).CreateAsync(token), "relay"), (() => DB.Default.Index().Key(f => f.ActorURI, KeyType.Ascending).Key(f => f.ObjectURI, KeyType.Ascending).Option(o => o.Unique = true).CreateAsync(token), "remote-featured"), (() => DB.Default.Index().Key(p => p.AvatarId, KeyType.Ascending).Option(o => o.Unique = true).CreateAsync(token), "notification policy"), (() => DB.Default.Index().Key(r => r.AvatarId, KeyType.Ascending).Key(r => r.FromActorURI, KeyType.Ascending).Option(o => o.Unique = true).CreateAsync(token), "notification request"), (() => DB.Default.Index().Key(p => p.AvatarId, KeyType.Ascending).Key(p => p.FromActorURI, KeyType.Ascending).Option(o => o.Unique = true).CreateAsync(token), "notification permission") }) await pair.Item1(); await Plain(token, b => b.TargetActorURI); await Plain(token, p => p.PostId); await Plain(token, b => b.ActorURI); await Plain(token, l => l.AvatarId); await Plain(token, f => f.AvatarId); await Plain(token, t => t.Name); await Plain(token, p => p.Tags); //the words status search reads, in every language alike (no stemming, no stop words); Language is not the //index's language field await DB.Default.Index().Key(p => p.Text, KeyType.Text).Key(p => p.ContentHtml, KeyType.Text).Key(p => p.SpoilerText, KeyType.Text) .Key(p => p.Title, KeyType.Text).Option(o => { o.DefaultLanguage = "none"; o.LanguageOverride = "textSearchLanguage"; o.Name = "words"; }) .CreateAsync(token); await Plain(token, m => m.AvatarId); await Plain(token, m => m.TargetActorURI); await Plain(token, r => r.IsResolved, r => r.ID); await Unique(b => b.Domain, Builders.Filter.Gt(b => b.Domain, ""), token); await Unique(i => i.Host, Builders.Filter.Gt(i => i.Host, ""), token); await Unique(r => r.ObjectURI, Builders.Filter.Gt(r => r.ObjectURI, ""), token); await Plain(token, r => r.PostId); await Unique(d => d.ObjectURI, Builders.Filter.Gt(d => d.ObjectURI, ""), token); await DB.Default.Index().Key(d => d.DeletedAt, KeyType.Ascending).Option(o => o.ExpireAfter = TombstoneLifetime).CreateAsync(token); await DB.Default.Index().Key(d => d.PostId, KeyType.Ascending).Key(d => d.ActorURI, KeyType.Ascending).Option(o => o.Unique = true).CreateAsync(token); await Plain(token, d => d.ActivityURI); await DB.Default.Index().Key(v => v.PostId, KeyType.Ascending).Key(v => v.ActorURI, KeyType.Ascending).Key(v => v.Choice, KeyType.Ascending) .Option(o => o.Unique = true).CreateAsync(token); await DB.Default.Index().Key(v => v.PostId, KeyType.Ascending).Key(v => v.VoterAccountId, KeyType.Ascending).CreateAsync(token); await DB.Default.Index().Key(r => r.PostId, KeyType.Ascending).Key(r => r.ActorURI, KeyType.Ascending).Key(r => r.Emoji, KeyType.Ascending) .Option(o => o.Unique = true).CreateAsync(token); await Plain(token, r => r.ActivityURI); await DB.Default.Index().Key(l => l.PostId, KeyType.Ascending).Key(l => l.QuotingObjectURI, KeyType.Ascending).CreateAsync(token); await Unique(p => p.Url, Builders.Filter.Gt(p => p.Url, ""), token); await DB.Default.Index().Key(e => e.At, KeyType.Ascending).Option(o => o.ExpireAfter = InteractionEvent.Retention).CreateAsync(token); await Plain(token, e => e.Host, e => e.At); await DB.Default.Index().Key(s => s.Day, KeyType.Ascending).Option(o => o.Unique = true).CreateAsync(token); await DB.Default.Index().Key(s => s.ExpiresAt, KeyType.Ascending).Option(o => o.ExpireAfter = TimeSpan.Zero).CreateAsync(token); await DB.Default.Index().Key(d => d.Day, KeyType.Ascending).Key(d => d.Host, KeyType.Ascending).Option(o => o.Unique = true).CreateAsync(token); await Plain(token, d => d.Host, d => d.Day); await DB.Default.Index().Key(d => d.Day, KeyType.Ascending).Option(o => o.Unique = true).CreateAsync(token); await DB.Default.Index().Key(s => s.Host, KeyType.Ascending).Key(s => s.Week, KeyType.Ascending) .Option(o => o.Unique = true).CreateAsync(token); await Plain(token, s => s.Week); await Plain(token, s => s.CdnDomain, s => s.Week); await Plain(token, s => s.Cdn, s => s.Week); await Plain(token, s => s.Key); await Plain(token, i => i.Seen); await Plain(token, i => i.LastCrawledAt); } // unique among the documents that have the key: the partial filter is `$gt: ""` (any non-empty string) rather than // `$type: "string"`, because MongoDB uses a partial index for a lookup only when the lookup implies its filter, and an // equality on a string implies the first, never the second: every lookup by these keys scanned the collection // (tools/pasture/load.sh, 280 ms a lookup at 30 000 posts) static async Task Unique(System.Linq.Expressions.Expression> key, FilterDefinition partial, CancellationToken token) where T : IEntity => await DB.Default.Index() .Key(key, KeyType.Ascending) .Option(o => { o.Unique = true; o.PartialFilterExpression = partial; }) .CreateAsync(token); static async Task Plain(CancellationToken token, params System.Linq.Expressions.Expression>[] keys) where T : IEntity { var index = DB.Default.Index(); foreach (var key in keys) index.Key(key, KeyType.Ascending); await index.CreateAsync(token); } } }