SocialPub/PrivaPub/Infrastructure/Data/Indexes.cs

using MongoDB.Bson;
using MongoDB.Driver;
using MongoDB.Entities;

using PrivaPub.Models.Federation;
using PrivaPub.Models.Group;
using PrivaPub.Models.Jobs;
using PrivaPub.Models.Post;
using PrivaPub.Models.Social;
using PrivaPub.Models.Statistics;
using PrivaPub.Models.User;

namespace PrivaPub.Infrastructure.Data
{
	public static class Indexes
	{
		public static readonly TimeSpan TombstoneLifetime = TimeSpan.FromDays(90);

		public static async Task Create(CancellationToken token = default)
		{
			await Unique<Post>(p => p.ObjectURI, Builders<Post>.Filter.Gt(p => p.ObjectURI, ""), token);
			await Plain<Post>(token, p => p.GroupUserId, p => p.ID);
			await Plain<Post>(token, p => p.GroupId, p => p.ID);
			await Plain<Post>(token, p => p.ActorURI);
			await Plain<Post>(token, p => p.ConversationId, p => p.ID);
			await Plain<Post>(token, p => p.InReplyToURI);
			// who boosted what: a persona's boosts of the posts a page shows were a scan of every post under load
			// (tools/pasture/load.sh), and a post's boosts and replies are read for its counts and context
			await Plain<Post>(token, p => p.AuthorAccountId, p => p.ReblogOfPostId);
			await Plain<Post>(token, p => p.ReblogOfPostId);
			await Plain<Post>(token, p => p.AnsweringToPostId);
			await DB.Default.Index<Post>().Key(p => p.Geo, KeyType.Geo2DSphere).CreateAsync(token);

			await Unique<DmPost>(p => p.ObjectURI, Builders<DmPost>.Filter.Gt(p => p.ObjectURI, ""), token);
			await Plain<DmPost>(token, p => p.GroupId, p => p.ID);

			await Unique<ForeignAvatar>(a => a.ActorURI, Builders<ForeignAvatar>.Filter.Gt(a => a.ActorURI, ""), token);
			await Plain<ForeignAvatar>(token, a => a.PublicKeyId);

			await DB.Default.Index<Follower>()
				.Key(f => f.LocalActorId, KeyType.Ascending)
				.Key(f => f.LocalActorKind, KeyType.Ascending)
				.Key(f => f.ActorURI, KeyType.Ascending)
				.Option(o => o.Unique = true)
				.CreateAsync(token);
			await Plain<Follower>(token, f => f.ActorURI);

			await DB.Default.Index<RootToAvatar>()
				.Key(r => r.RootId, KeyType.Ascending)
				.Key(r => r.AvatarId, KeyType.Ascending)
				.Option(o => o.Unique = true)
				.CreateAsync(token);
			await Plain<RootToAvatar>(token, r => r.AvatarId);

			await Unique<ReservedName>(r => r.Name, Builders<ReservedName>.Filter.Gt(r => r.Name, ""), token);
			await Unique<RootUser>(u => u.UserName, Builders<RootUser>.Filter.Gt(u => u.UserName, ""), token);
			await Plain<Avatar>(token, a => a.UserName);
			await Plain<Group>(token, g => g.UserName);
			await Plain<Group>(token, g => g.InvitationCode);

			await Plain<DmGroup>(token, g => g.ParticipantsKey);
			await Plain<DmGroup>(token, g => g.LastPostId);
			await DB.Default.Index<ConversationState>()
				.Key(s => s.AvatarId, KeyType.Ascending)
				.Key(s => s.ConversationId, KeyType.Ascending)
				.Option(o => o.Unique = true)
				.CreateAsync(token);
			await Unique<GroupAnnouncement>(a => a.ActivityURI, Builders<GroupAnnouncement>.Filter.Gt(a => a.ActivityURI, ""), token);

			await Plain<Delivery>(token, d => d.DeliveredAt, d => d.AbandonedAt, d => d.NextAttemptAt);

			await Plain<Job>(token, j => j.Kind, j => j.State, j => j.RunAt);
			await Plain<Job>(token, j => j.State, j => j.LeasedUntil);
			await Unique<Job>(j => j.DedupeKey, Builders<Job>.Filter.Gt(j => j.DedupeKey, ""), token);
			await DB.Default.Index<Job>()
				.Key(j => j.FinishedAt, KeyType.Ascending)
				.Option(o => o.ExpireAfter = TimeSpan.FromDays(7))
				.CreateAsync(token);
			await DB.Default.Index<Following>()
				.Key(f => f.AvatarId, KeyType.Ascending)
				.Key(f => f.TargetActorURI, KeyType.Ascending)
				.Option(o => o.Unique = true)
				.CreateAsync(token);
			await Plain<Following>(token, f => f.TargetActorURI, f => f.State);
			await Plain<Following>(token, f => f.TargetAccountId, f => f.State);
			await Plain<Following>(token, f => f.TargetActorURI, f => f.State);//who here follows the author of what arrives
			await Plain<Following>(token, f => f.FollowActivityURI);
			await DB.Default.Index<TimelineEntry>()
				.Key(e => e.AvatarId, KeyType.Ascending)
				.Key(e => e.PostId, KeyType.Ascending)
				.Option(o => o.Unique = true)
				.CreateAsync(token);
			await Plain<TimelineEntry>(token, e => e.PostId);
			await Plain<TimelineEntry>(token, e => e.AuthorAccountId);
			await Unique<Notification>(n => n.DedupeKey, Builders<Notification>.Filter.Gt(n => n.DedupeKey, ""), token);
			await Plain<Notification>(token, n => n.AvatarId, n => n.ID);
			await DB.Default.Index<Favourite>()
				.Key(f => f.AccountId, KeyType.Ascending)
				.Key(f => f.PostId, KeyType.Ascending)
				.Option(o => o.Unique = true)
				.CreateAsync(token);
			await Plain<Favourite>(token, f => f.ActivityURI);
			await Plain<Favourite>(token, f => f.PostId);
			await DB.Default.Index<Participation>()
				.Key(p => p.AvatarId, KeyType.Ascending)
				.Key(p => p.PostId, KeyType.Ascending)
				.Option(o => o.Unique = true)
				.CreateAsync(token);
			await Plain<Participation>(token, p => p.ActivityURI);
			await DB.Default.Index<Marker>()
				.Key(m => m.AvatarId, KeyType.Ascending)
				.Key(m => m.Timeline, KeyType.Ascending)
				.Option(o => o.Unique = true)
				.CreateAsync(token);
			foreach (var pair in new (Func<Task>, string)[]
			{
				(() => DB.Default.Index<Block>().Key(b => b.AvatarId, KeyType.Ascending).Key(b => b.TargetActorURI, KeyType.Ascending).Option(o => o.Unique = true).CreateAsync(token), "block"),
				(() => DB.Default.Index<FollowedTag>().Key(t => t.AvatarId, KeyType.Ascending).Key(t => t.Name, KeyType.Ascending).Option(o => o.Unique = true).CreateAsync(token), "followed tag"),
				(() => DB.Default.Index<PersonaListMember>().Key(m => m.ListId, KeyType.Ascending).Key(m => m.AccountId, KeyType.Ascending).Option(o => o.Unique = true).CreateAsync(token), "list member"),
				(() => DB.Default.Index<BlockedBy>().Key(b => b.AvatarId, KeyType.Ascending).Key(b => b.ActorURI, KeyType.Ascending).Option(o => o.Unique = true).CreateAsync(token), "blocked by"),
				(() => DB.Default.Index<Mute>().Key(m => m.AvatarId, KeyType.Ascending).Key(m => m.TargetActorURI, KeyType.Ascending).Option(o => o.Unique = true).CreateAsync(token), "mute"),
				(() => DB.Default.Index<AccountDomainBlock>().Key(b => b.AvatarId, KeyType.Ascending).Key(b => b.Domain, KeyType.Ascending).Option(o => o.Unique = true).CreateAsync(token), "domain block"),
				(() => DB.Default.Index<Bookmark>().Key(b => b.AvatarId, KeyType.Ascending).Key(b => b.PostId, KeyType.Ascending).Option(o => o.Unique = true).CreateAsync(token), "bookmark"),
				(() => DB.Default.Index<Pin>().Key(p => p.AvatarId, KeyType.Ascending).Key(p => p.PostId, KeyType.Ascending).Option(o => o.Unique = true).CreateAsync(token), "pin"),
				(() => DB.Default.Index<Models.Federation.RelaySubscription>().Key(r => r.Configured, KeyType.Ascending).Option(o => o.Unique = true).CreateAsync(token), "relay"),
				(() => DB.Default.Index<RemoteFeatured>().Key(f => f.ActorURI, KeyType.Ascending).Key(f => f.ObjectURI, KeyType.Ascending).Option(o => o.Unique = true).CreateAsync(token), "remote-featured")
			})
				await pair.Item1();
			await Plain<Block>(token, b => b.TargetActorURI);
			await Plain<Pin>(token, p => p.PostId);
			await Plain<BlockedBy>(token, b => b.ActorURI);
			await Plain<PersonaList>(token, l => l.AvatarId);
			await Plain<PersonaFilter>(token, f => f.AvatarId);
			await Plain<FollowedTag>(token, t => t.Name);
			await Plain<Post>(token, p => p.Tags);
			//the words status search reads, in every language alike (no stemming, no stop words); Language is not the
			//index's language field
			await DB.Default.Index<Post>().Key(p => p.Text, KeyType.Text).Key(p => p.ContentHtml, KeyType.Text).Key(p => p.SpoilerText, KeyType.Text)
				.Key(p => p.Title, KeyType.Text).Option(o => { o.DefaultLanguage = "none"; o.LanguageOverride = "textSearchLanguage"; o.Name = "words"; })
				.CreateAsync(token);
			await Plain<PersonaListMember>(token, m => m.AvatarId);
			await Plain<Mute>(token, m => m.TargetActorURI);
			await Plain<Report>(token, r => r.IsResolved, r => r.ID);
			await Unique<DomainBlock>(b => b.Domain, Builders<DomainBlock>.Filter.Gt(b => b.Domain, ""), token);
			await Unique<RemoteInstance>(i => i.Host, Builders<RemoteInstance>.Filter.Gt(i => i.Host, ""), token);
			await Unique<ObjectRecord>(r => r.ObjectURI, Builders<ObjectRecord>.Filter.Gt(r => r.ObjectURI, ""), token);
			await Plain<ObjectRecord>(token, r => r.PostId);
			await Unique<DeletedObject>(d => d.ObjectURI, Builders<DeletedObject>.Filter.Gt(d => d.ObjectURI, ""), token);
			await DB.Default.Index<DeletedObject>().Key(d => d.DeletedAt, KeyType.Ascending).Option(o => o.ExpireAfter = TombstoneLifetime).CreateAsync(token);
			await DB.Default.Index<Downvote>().Key(d => d.PostId, KeyType.Ascending).Key(d => d.ActorURI, KeyType.Ascending).Option(o => o.Unique = true).CreateAsync(token);
			await Plain<Downvote>(token, d => d.ActivityURI);
			await DB.Default.Index<PollVote>().Key(v => v.PostId, KeyType.Ascending).Key(v => v.ActorURI, KeyType.Ascending).Key(v => v.Choice, KeyType.Ascending)
				.Option(o => o.Unique = true).CreateAsync(token);
			await DB.Default.Index<PollVote>().Key(v => v.PostId, KeyType.Ascending).Key(v => v.VoterAccountId, KeyType.Ascending).CreateAsync(token);
			await DB.Default.Index<Reaction>().Key(r => r.PostId, KeyType.Ascending).Key(r => r.ActorURI, KeyType.Ascending).Key(r => r.Emoji, KeyType.Ascending)
				.Option(o => o.Unique = true).CreateAsync(token);
			await Plain<Reaction>(token, r => r.ActivityURI);
			await DB.Default.Index<QuoteLicence>().Key(l => l.PostId, KeyType.Ascending).Key(l => l.QuotingObjectURI, KeyType.Ascending).CreateAsync(token);
			await Unique<Domain.Content.LinkPreview>(p => p.Url, Builders<Domain.Content.LinkPreview>.Filter.Gt(p => p.Url, ""), token);

			await DB.Default.Index<InteractionEvent>().Key(e => e.At, KeyType.Ascending).Option(o => o.ExpireAfter = InteractionEvent.Retention).CreateAsync(token);
			await Plain<InteractionEvent>(token, e => e.Host, e => e.At);
			await DB.Default.Index<InteractionSalt>().Key(s => s.Day, KeyType.Ascending).Option(o => o.Unique = true).CreateAsync(token);
			await DB.Default.Index<InteractionSalt>().Key(s => s.ExpiresAt, KeyType.Ascending).Option(o => o.ExpireAfter = TimeSpan.Zero).CreateAsync(token);
			await DB.Default.Index<InstanceDay>().Key(d => d.Day, KeyType.Ascending).Key(d => d.Host, KeyType.Ascending).Option(o => o.Unique = true).CreateAsync(token);
			await Plain<InstanceDay>(token, d => d.Host, d => d.Day);
			await DB.Default.Index<ServerDay>().Key(d => d.Day, KeyType.Ascending).Option(o => o.Unique = true).CreateAsync(token);
			await DB.Default.Index<RemoteInstanceSnapshot>().Key(s => s.Host, KeyType.Ascending).Key(s => s.Week, KeyType.Ascending)
				.Option(o => o.Unique = true).CreateAsync(token);
			await Plain<RemoteInstanceSnapshot>(token, s => s.Week);
			await Plain<RemoteInstanceSnapshot>(token, s => s.CdnDomain, s => s.Week);
			await Plain<RemoteInstanceSnapshot>(token, s => s.Cdn, s => s.Week);
			await Plain<CdnRangeSet>(token, s => s.Key);
			await Plain<RemoteInstance>(token, i => i.Seen);
			await Plain<RemoteInstance>(token, i => i.LastCrawledAt);
		}

		// unique among the documents that have the key: the partial filter is `$gt: ""` (any non-empty string) rather than
		// `$type: "string"`, because MongoDB uses a partial index for a lookup only when the lookup implies its filter, and an
		// equality on a string implies the first, never the second: every lookup by these keys scanned the collection
		// (tools/pasture/load.sh, 280 ms a lookup at 30 000 posts)
		static async Task Unique<T>(System.Linq.Expressions.Expression<Func<T, object>> key, FilterDefinition<T> partial,
			CancellationToken token) where T : IEntity =>
			await DB.Default.Index<T>()
				.Key(key, KeyType.Ascending)
				.Option(o =>
				{
					o.Unique = true;
					o.PartialFilterExpression = partial;
				})
				.CreateAsync(token);

		static async Task Plain<T>(CancellationToken token, params System.Linq.Expressions.Expression<Func<T, object>>[] keys) where T : IEntity
		{
			var index = DB.Default.Index<T>();
			foreach (var key in keys)
				index.Key(key, KeyType.Ascending);
			await index.CreateAsync(token);
		}
	}
}