thepraandClaude Opus 5.5
6fccb6fe35
Votes out. A persona's downvote is a Dislike (POST /api/v1/statuses/:id/downvote and /undownvote, the viewer's own as privapub.votes.downvoted). One vote a post: a changed vote is sent as the new vote alone, as Lemmy sends one, since an Undo of the old one beside it could arrive after it and take the new one away; Undo goes only when a vote is taken back. A vote on a post in a remote community goes to the community, which counts it, and to the author only when on another server: one copy a server, since PieFed drops a second copy of an activity it has just seen. Mbin keeps a favourite apart from a vote, so there a favourite stays after a switch to a downvote. Feeds followed (owner decision 2026-10-07: through the server). Following a feed (Lemmy's multi-community, PieFed's feed) keeps a FeedSubscription for the persona and nothing else; the new Service privapub_feeds (LocalActorKind.Reader, reserved by migration _017) follows every community of the feeds read here, reconciled when a persona follows or leaves one, when a feed's list is read again (kept as it was when it cannot be read) and every six hours. Its Following rows carry FollowerKind, so nobody's home gets what it brings in and its unanswered follows are sent again as its own. The persona reads GET /api/v1/timelines/feed/:id (the feed's threads, ours included) and lists its feeds at GET /api/v1/feeds. Checked in the pasture, every scenario: 873 pass. The 14 failures are Hubzilla's (identical on the previous commit: Hubzilla no longer answers a follow in this pasture since its restore) and two activities Smithereen never sent; followsync passes once the pasture's restore is older than the 14-day pause. Live: alice's downvotes count as downvotes on Lemmy 1.0 and PieFed, replacing her upvote; Lemmy 1.0 and PieFed take privapub_feeds' follows and their threads reach the feed timelines. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_012CzABvBkbcFqoHdmi8b9WB
228 lines
14 KiB
C#
228 lines
14 KiB
C#
using MongoDB.Bson;
|
|
using MongoDB.Driver;
|
|
using MongoDB.Entities;
|
|
|
|
using PrivaPub.Models.Federation;
|
|
using PrivaPub.Models.Group;
|
|
using PrivaPub.Models.Jobs;
|
|
using PrivaPub.Models.Post;
|
|
using PrivaPub.Models.Social;
|
|
using PrivaPub.Models.Statistics;
|
|
using PrivaPub.Models.User;
|
|
|
|
namespace PrivaPub.Infrastructure.Data
|
|
{
|
|
public static class Indexes
|
|
{
|
|
public static readonly TimeSpan TombstoneLifetime = TimeSpan.FromDays(90);
|
|
public static readonly TimeSpan ThreadCollectionLifetime = TimeSpan.FromDays(30);
|
|
|
|
public static async Task Create(CancellationToken token = default)
|
|
{
|
|
await Unique<Post>(p => p.ObjectURI, Builders<Post>.Filter.Gt(p => p.ObjectURI, ""), token);
|
|
await Plain<Post>(token, p => p.GroupUserId, p => p.ID);
|
|
await Plain<Post>(token, p => p.GroupId, p => p.ID);
|
|
await Plain<Post>(token, p => p.ActorURI);
|
|
await Plain<Post>(token, p => p.WallURI, p => p.ID);//a persona's wall (FEP-400e)
|
|
await Plain<Post>(token, p => p.ConversationId, p => p.ID);
|
|
await Plain<Post>(token, p => p.InReplyToURI);
|
|
await Plain<Post>(token, p => p.AudienceURI, p => p.ID);//a feed's timeline: the posts of its communities
|
|
// who boosted what: a persona's boosts of the posts a page shows were a scan of every post under load
|
|
// (tools/pasture/load.sh), and a post's boosts and replies are read for its counts and context
|
|
await Plain<Post>(token, p => p.AuthorAccountId, p => p.ReblogOfPostId);
|
|
await Plain<Post>(token, p => p.ReblogOfPostId);
|
|
await Plain<Post>(token, p => p.AnsweringToPostId);
|
|
await DB.Default.Index<Post>().Key(p => p.Geo, KeyType.Geo2DSphere).CreateAsync(token);
|
|
// a post imported from another actor's archive, once per persona: importing it twice changes nothing
|
|
await DB.Default.Index<Post>().Key(p => p.GroupUserId, KeyType.Ascending).Key(p => p.ImportedFromURI, KeyType.Ascending)
|
|
.Option(o =>
|
|
{
|
|
o.Unique = true;
|
|
o.PartialFilterExpression = Builders<Post>.Filter.Gt(p => p.ImportedFromURI, "");
|
|
})
|
|
.CreateAsync(token);
|
|
|
|
// media: a post's, a persona's, what a scheduled post holds, the janitor's never-posted uploads and its trash
|
|
await Plain<Models.Media.MediaAttachment>(token, m => m.PostId, m => m.ScheduledStatusId, m => m.CreatedAt);
|
|
await Plain<Models.Media.MediaAttachment>(token, m => m.OwnerAvatarId);
|
|
await Plain<Models.Media.MediaAttachment>(token, m => m.ScheduledStatusId);
|
|
await Plain<Models.Media.MediaAttachment>(token, m => m.ProfileOfAvatarId);
|
|
await Plain<Models.Media.MediaAttachment>(token, m => m.TrashedAt);
|
|
|
|
await Unique<DmPost>(p => p.ObjectURI, Builders<DmPost>.Filter.Gt(p => p.ObjectURI, ""), token);
|
|
await Plain<DmPost>(token, p => p.GroupId, p => p.ID);
|
|
|
|
await Unique<ForeignAvatar>(a => a.ActorURI, Builders<ForeignAvatar>.Filter.Gt(a => a.ActorURI, ""), token);
|
|
await Plain<ForeignAvatar>(token, a => a.PublicKeyId);
|
|
|
|
await DB.Default.Index<Follower>()
|
|
.Key(f => f.LocalActorId, KeyType.Ascending)
|
|
.Key(f => f.LocalActorKind, KeyType.Ascending)
|
|
.Key(f => f.ActorURI, KeyType.Ascending)
|
|
.Option(o => o.Unique = true)
|
|
.CreateAsync(token);
|
|
await Plain<Follower>(token, f => f.ActorURI);
|
|
|
|
await DB.Default.Index<RootToAvatar>()
|
|
.Key(r => r.RootId, KeyType.Ascending)
|
|
.Key(r => r.AvatarId, KeyType.Ascending)
|
|
.Option(o => o.Unique = true)
|
|
.CreateAsync(token);
|
|
await Plain<RootToAvatar>(token, r => r.AvatarId);
|
|
|
|
await Unique<ReservedName>(r => r.Name, Builders<ReservedName>.Filter.Gt(r => r.Name, ""), token);
|
|
await Unique<RootUser>(u => u.UserName, Builders<RootUser>.Filter.Gt(u => u.UserName, ""), token);
|
|
await Plain<Avatar>(token, a => a.UserName);
|
|
await Plain<Group>(token, g => g.UserName);
|
|
await Plain<Group>(token, g => g.InvitationCode);
|
|
|
|
await Plain<DmGroup>(token, g => g.ParticipantsKey);
|
|
await Plain<DmGroup>(token, g => g.LastPostId);
|
|
await DB.Default.Index<ConversationState>()
|
|
.Key(s => s.AvatarId, KeyType.Ascending)
|
|
.Key(s => s.ConversationId, KeyType.Ascending)
|
|
.Option(o => o.Unique = true)
|
|
.CreateAsync(token);
|
|
await Unique<GroupAnnouncement>(a => a.ActivityURI, Builders<GroupAnnouncement>.Filter.Gt(a => a.ActivityURI, ""), token);
|
|
|
|
await Plain<Delivery>(token, d => d.DeliveredAt, d => d.AbandonedAt, d => d.NextAttemptAt);
|
|
|
|
await Plain<Job>(token, j => j.Kind, j => j.State, j => j.RunAt);
|
|
await Plain<Job>(token, j => j.State, j => j.LeasedUntil);
|
|
await Unique<Job>(j => j.DedupeKey, Builders<Job>.Filter.Gt(j => j.DedupeKey, ""), token);
|
|
await DB.Default.Index<Job>()
|
|
.Key(j => j.FinishedAt, KeyType.Ascending)
|
|
.Option(o => o.ExpireAfter = TimeSpan.FromDays(7))
|
|
.CreateAsync(token);
|
|
await DB.Default.Index<Following>()
|
|
.Key(f => f.AvatarId, KeyType.Ascending)
|
|
.Key(f => f.TargetActorURI, KeyType.Ascending)
|
|
.Option(o => o.Unique = true)
|
|
.CreateAsync(token);
|
|
await Plain<Following>(token, f => f.TargetActorURI, f => f.State);
|
|
await Plain<Following>(token, f => f.TargetAccountId, f => f.State);
|
|
await Plain<Following>(token, f => f.TargetActorURI, f => f.State);//who here follows the author of what arrives
|
|
await Plain<Following>(token, f => f.FollowActivityURI);
|
|
await DB.Default.Index<TimelineEntry>()
|
|
.Key(e => e.AvatarId, KeyType.Ascending)
|
|
.Key(e => e.PostId, KeyType.Ascending)
|
|
.Option(o => o.Unique = true)
|
|
.CreateAsync(token);
|
|
await Plain<TimelineEntry>(token, e => e.PostId);
|
|
await Plain<TimelineEntry>(token, e => e.AuthorAccountId);
|
|
await Unique<Notification>(n => n.DedupeKey, Builders<Notification>.Filter.Gt(n => n.DedupeKey, ""), token);
|
|
await Plain<Notification>(token, n => n.AvatarId, n => n.ID);
|
|
await DB.Default.Index<Favourite>()
|
|
.Key(f => f.AccountId, KeyType.Ascending)
|
|
.Key(f => f.PostId, KeyType.Ascending)
|
|
.Option(o => o.Unique = true)
|
|
.CreateAsync(token);
|
|
await Plain<Favourite>(token, f => f.ActivityURI);
|
|
await Plain<Favourite>(token, f => f.PostId);
|
|
await DB.Default.Index<Participation>()
|
|
.Key(p => p.AvatarId, KeyType.Ascending)
|
|
.Key(p => p.PostId, KeyType.Ascending)
|
|
.Option(o => o.Unique = true)
|
|
.CreateAsync(token);
|
|
await Plain<Participation>(token, p => p.ActivityURI);
|
|
await DB.Default.Index<Marker>()
|
|
.Key(m => m.AvatarId, KeyType.Ascending)
|
|
.Key(m => m.Timeline, KeyType.Ascending)
|
|
.Option(o => o.Unique = true)
|
|
.CreateAsync(token);
|
|
foreach (var pair in new (Func<Task>, string)[]
|
|
{
|
|
(() => DB.Default.Index<Block>().Key(b => b.AvatarId, KeyType.Ascending).Key(b => b.TargetActorURI, KeyType.Ascending).Option(o => o.Unique = true).CreateAsync(token), "block"),
|
|
(() => DB.Default.Index<FollowedTag>().Key(t => t.AvatarId, KeyType.Ascending).Key(t => t.Name, KeyType.Ascending).Option(o => o.Unique = true).CreateAsync(token), "followed tag"),
|
|
(() => DB.Default.Index<PersonaListMember>().Key(m => m.ListId, KeyType.Ascending).Key(m => m.AccountId, KeyType.Ascending).Option(o => o.Unique = true).CreateAsync(token), "list member"),
|
|
(() => DB.Default.Index<BlockedBy>().Key(b => b.AvatarId, KeyType.Ascending).Key(b => b.ActorURI, KeyType.Ascending).Option(o => o.Unique = true).CreateAsync(token), "blocked by"),
|
|
(() => DB.Default.Index<Mute>().Key(m => m.AvatarId, KeyType.Ascending).Key(m => m.TargetActorURI, KeyType.Ascending).Option(o => o.Unique = true).CreateAsync(token), "mute"),
|
|
(() => DB.Default.Index<AccountDomainBlock>().Key(b => b.AvatarId, KeyType.Ascending).Key(b => b.Domain, KeyType.Ascending).Option(o => o.Unique = true).CreateAsync(token), "domain block"),
|
|
(() => DB.Default.Index<Bookmark>().Key(b => b.AvatarId, KeyType.Ascending).Key(b => b.PostId, KeyType.Ascending).Option(o => o.Unique = true).CreateAsync(token), "bookmark"),
|
|
(() => DB.Default.Index<Pin>().Key(p => p.AvatarId, KeyType.Ascending).Key(p => p.PostId, KeyType.Ascending).Option(o => o.Unique = true).CreateAsync(token), "pin"),
|
|
(() => DB.Default.Index<Models.Federation.RelaySubscription>().Key(r => r.Configured, KeyType.Ascending).Option(o => o.Unique = true).CreateAsync(token), "relay"),
|
|
(() => DB.Default.Index<RemoteFeatured>().Key(f => f.ActorURI, KeyType.Ascending).Key(f => f.ObjectURI, KeyType.Ascending).Option(o => o.Unique = true).CreateAsync(token), "remote-featured"),
|
|
(() => DB.Default.Index<NotificationPolicy>().Key(p => p.AvatarId, KeyType.Ascending).Option(o => o.Unique = true).CreateAsync(token), "notification policy"),
|
|
(() => DB.Default.Index<NotificationRequest>().Key(r => r.AvatarId, KeyType.Ascending).Key(r => r.FromActorURI, KeyType.Ascending).Option(o => o.Unique = true).CreateAsync(token), "notification request"),
|
|
(() => DB.Default.Index<NotificationPermission>().Key(p => p.AvatarId, KeyType.Ascending).Key(p => p.FromActorURI, KeyType.Ascending).Option(o => o.Unique = true).CreateAsync(token), "notification permission")
|
|
})
|
|
await pair.Item1();
|
|
await Plain<Block>(token, b => b.TargetActorURI);
|
|
await Plain<Pin>(token, p => p.PostId);
|
|
await Plain<BlockedBy>(token, b => b.ActorURI);
|
|
await Plain<PersonaList>(token, l => l.AvatarId);
|
|
await Plain<PersonaFilter>(token, f => f.AvatarId);
|
|
await Plain<FollowedTag>(token, t => t.Name);
|
|
await Plain<Post>(token, p => p.Tags);
|
|
//the words status search reads, in every language alike (no stemming, no stop words); Language is not the
|
|
//index's language field
|
|
await DB.Default.Index<Post>().Key(p => p.Text, KeyType.Text).Key(p => p.ContentHtml, KeyType.Text).Key(p => p.SpoilerText, KeyType.Text)
|
|
.Key(p => p.Title, KeyType.Text).Option(o => { o.DefaultLanguage = "none"; o.LanguageOverride = "textSearchLanguage"; o.Name = "words"; })
|
|
.CreateAsync(token);
|
|
await Plain<PersonaListMember>(token, m => m.AvatarId);
|
|
await Plain<Mute>(token, m => m.TargetActorURI);
|
|
await Plain<Report>(token, r => r.IsResolved, r => r.ID);
|
|
await Unique<DomainBlock>(b => b.Domain, Builders<DomainBlock>.Filter.Gt(b => b.Domain, ""), token);
|
|
await Unique<RemoteInstance>(i => i.Host, Builders<RemoteInstance>.Filter.Gt(i => i.Host, ""), token);
|
|
await Unique<ObjectRecord>(r => r.ObjectURI, Builders<ObjectRecord>.Filter.Gt(r => r.ObjectURI, ""), token);
|
|
await Plain<ObjectRecord>(token, r => r.PostId);
|
|
await Unique<DeletedObject>(d => d.ObjectURI, Builders<DeletedObject>.Filter.Gt(d => d.ObjectURI, ""), token);
|
|
await DB.Default.Index<DeletedObject>().Key(d => d.DeletedAt, KeyType.Ascending).Option(o => o.ExpireAfter = TombstoneLifetime).CreateAsync(token);
|
|
await DB.Default.Index<FeedSubscription>().Key(s => s.AvatarId, KeyType.Ascending).Key(s => s.FeedActorURI, KeyType.Ascending)
|
|
.Option(o => o.Unique = true).CreateAsync(token);
|
|
await Plain<FeedSubscription>(token, s => s.FeedActorURI);
|
|
await Unique<ThreadCollection>(c => c.URI, Builders<ThreadCollection>.Filter.Gt(c => c.URI, ""), token);
|
|
await DB.Default.Index<ThreadCollection>().Key(c => c.ReadAt, KeyType.Ascending).Option(o => o.ExpireAfter = ThreadCollectionLifetime).CreateAsync(token);
|
|
await DB.Default.Index<Downvote>().Key(d => d.PostId, KeyType.Ascending).Key(d => d.ActorURI, KeyType.Ascending).Option(o => o.Unique = true).CreateAsync(token);
|
|
await Plain<Downvote>(token, d => d.ActivityURI);
|
|
await DB.Default.Index<PollVote>().Key(v => v.PostId, KeyType.Ascending).Key(v => v.ActorURI, KeyType.Ascending).Key(v => v.Choice, KeyType.Ascending)
|
|
.Option(o => o.Unique = true).CreateAsync(token);
|
|
await DB.Default.Index<PollVote>().Key(v => v.PostId, KeyType.Ascending).Key(v => v.VoterAccountId, KeyType.Ascending).CreateAsync(token);
|
|
await DB.Default.Index<Reaction>().Key(r => r.PostId, KeyType.Ascending).Key(r => r.ActorURI, KeyType.Ascending).Key(r => r.Emoji, KeyType.Ascending)
|
|
.Option(o => o.Unique = true).CreateAsync(token);
|
|
await Plain<Reaction>(token, r => r.ActivityURI);
|
|
await DB.Default.Index<QuoteLicence>().Key(l => l.PostId, KeyType.Ascending).Key(l => l.QuotingObjectURI, KeyType.Ascending).CreateAsync(token);
|
|
await Unique<Domain.Content.LinkPreview>(p => p.Url, Builders<Domain.Content.LinkPreview>.Filter.Gt(p => p.Url, ""), token);
|
|
|
|
await DB.Default.Index<InteractionEvent>().Key(e => e.At, KeyType.Ascending).Option(o => o.ExpireAfter = InteractionEvent.Retention).CreateAsync(token);
|
|
await Plain<InteractionEvent>(token, e => e.Host, e => e.At);
|
|
await DB.Default.Index<InteractionSalt>().Key(s => s.Day, KeyType.Ascending).Option(o => o.Unique = true).CreateAsync(token);
|
|
await DB.Default.Index<InteractionSalt>().Key(s => s.ExpiresAt, KeyType.Ascending).Option(o => o.ExpireAfter = TimeSpan.Zero).CreateAsync(token);
|
|
await DB.Default.Index<InstanceDay>().Key(d => d.Day, KeyType.Ascending).Key(d => d.Host, KeyType.Ascending).Option(o => o.Unique = true).CreateAsync(token);
|
|
await Plain<InstanceDay>(token, d => d.Host, d => d.Day);
|
|
await DB.Default.Index<ServerDay>().Key(d => d.Day, KeyType.Ascending).Option(o => o.Unique = true).CreateAsync(token);
|
|
await DB.Default.Index<RemoteInstanceSnapshot>().Key(s => s.Host, KeyType.Ascending).Key(s => s.Week, KeyType.Ascending)
|
|
.Option(o => o.Unique = true).CreateAsync(token);
|
|
await Plain<RemoteInstanceSnapshot>(token, s => s.Week);
|
|
await Plain<RemoteInstanceSnapshot>(token, s => s.CdnDomain, s => s.Week);
|
|
await Plain<RemoteInstanceSnapshot>(token, s => s.Cdn, s => s.Week);
|
|
await Plain<CdnRangeSet>(token, s => s.Key);
|
|
await Plain<RemoteInstance>(token, i => i.Seen);
|
|
await Plain<RemoteInstance>(token, i => i.LastCrawledAt);
|
|
}
|
|
|
|
// unique among the documents that have the key: the partial filter is `$gt: ""` (any non-empty string) rather than
|
|
// `$type: "string"`, because MongoDB uses a partial index for a lookup only when the lookup implies its filter, and an
|
|
// equality on a string implies the first, never the second: every lookup by these keys scanned the collection
|
|
// (tools/pasture/load.sh, 280 ms a lookup at 30 000 posts)
|
|
static async Task Unique<T>(System.Linq.Expressions.Expression<Func<T, object>> key, FilterDefinition<T> partial,
|
|
CancellationToken token) where T : IEntity =>
|
|
await DB.Default.Index<T>()
|
|
.Key(key, KeyType.Ascending)
|
|
.Option(o =>
|
|
{
|
|
o.Unique = true;
|
|
o.PartialFilterExpression = partial;
|
|
})
|
|
.CreateAsync(token);
|
|
|
|
static async Task Plain<T>(CancellationToken token, params System.Linq.Expressions.Expression<Func<T, object>>[] keys) where T : IEntity
|
|
{
|
|
var index = DB.Default.Index<T>();
|
|
foreach (var key in keys)
|
|
index.Key(key, KeyType.Ascending);
|
|
await index.CreateAsync(token);
|
|
}
|
|
}
|
|
}
|