thepraandClaude Opus 5.5
3d7d406834
A root exports one of its personas (a job) as Mastodon's account archive, so other servers' importers read it: the actor with its public key only, its own posts and boosts with their media, likes, bookmarks and Mastodon's CSV files, plus PrivaPub's filters, followed hashtags, notification policy, pins, scheduled and located posts; nothing of its root, its siblings, its keys or anyone's token. A ticket link downloads it, for a week. An archive (PrivaPub's or Mastodon's) uploaded in pieces is imported into a persona in a job, the parts the root picks, with progress and a stop. SafeArchive refuses links, escaping paths, duplicates, bombs and oversized items, and reads the outbox one item at a time. Imported posts are delivered to no one, put in no home and notify nobody, yet show on the profile, outbox, hashtags and search; back home a post keeps its id, from another actor it gets one of its date and ImportedFromURI, so importing twice changes nothing. Relationships go through the existing services; located, scheduled and likes only when asked; followers never. tools/pasture/scenarios/persona-archive.sh imports mastouser's real Mastodon archive (156 posts, 24 pictures) into a persona Mastodon follows: Mastodon receives none of it, and a second import changes nothing. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01LsXgEaXee4GCU1hwYgPJXw
221 lines
14 KiB
C#
221 lines
14 KiB
C#
using MongoDB.Bson;
|
|
using MongoDB.Driver;
|
|
using MongoDB.Entities;
|
|
|
|
using PrivaPub.Models.Federation;
|
|
using PrivaPub.Models.Group;
|
|
using PrivaPub.Models.Jobs;
|
|
using PrivaPub.Models.Post;
|
|
using PrivaPub.Models.Social;
|
|
using PrivaPub.Models.Statistics;
|
|
using PrivaPub.Models.User;
|
|
|
|
namespace PrivaPub.Infrastructure.Data
|
|
{
|
|
public static class Indexes
|
|
{
|
|
public static readonly TimeSpan TombstoneLifetime = TimeSpan.FromDays(90);
|
|
|
|
public static async Task Create(CancellationToken token = default)
|
|
{
|
|
await Unique<Post>(p => p.ObjectURI, Builders<Post>.Filter.Gt(p => p.ObjectURI, ""), token);
|
|
await Plain<Post>(token, p => p.GroupUserId, p => p.ID);
|
|
await Plain<Post>(token, p => p.GroupId, p => p.ID);
|
|
await Plain<Post>(token, p => p.ActorURI);
|
|
await Plain<Post>(token, p => p.WallURI, p => p.ID);//a persona's wall (FEP-400e)
|
|
await Plain<Post>(token, p => p.ConversationId, p => p.ID);
|
|
await Plain<Post>(token, p => p.InReplyToURI);
|
|
// who boosted what: a persona's boosts of the posts a page shows were a scan of every post under load
|
|
// (tools/pasture/load.sh), and a post's boosts and replies are read for its counts and context
|
|
await Plain<Post>(token, p => p.AuthorAccountId, p => p.ReblogOfPostId);
|
|
await Plain<Post>(token, p => p.ReblogOfPostId);
|
|
await Plain<Post>(token, p => p.AnsweringToPostId);
|
|
await DB.Default.Index<Post>().Key(p => p.Geo, KeyType.Geo2DSphere).CreateAsync(token);
|
|
// a post imported from another actor's archive, once per persona: importing it twice changes nothing
|
|
await DB.Default.Index<Post>().Key(p => p.GroupUserId, KeyType.Ascending).Key(p => p.ImportedFromURI, KeyType.Ascending)
|
|
.Option(o =>
|
|
{
|
|
o.Unique = true;
|
|
o.PartialFilterExpression = Builders<Post>.Filter.Gt(p => p.ImportedFromURI, "");
|
|
})
|
|
.CreateAsync(token);
|
|
|
|
// media: a post's, a persona's, what a scheduled post holds, the janitor's never-posted uploads and its trash
|
|
await Plain<Models.Media.MediaAttachment>(token, m => m.PostId, m => m.ScheduledStatusId, m => m.CreatedAt);
|
|
await Plain<Models.Media.MediaAttachment>(token, m => m.OwnerAvatarId);
|
|
await Plain<Models.Media.MediaAttachment>(token, m => m.ScheduledStatusId);
|
|
await Plain<Models.Media.MediaAttachment>(token, m => m.ProfileOfAvatarId);
|
|
await Plain<Models.Media.MediaAttachment>(token, m => m.TrashedAt);
|
|
|
|
await Unique<DmPost>(p => p.ObjectURI, Builders<DmPost>.Filter.Gt(p => p.ObjectURI, ""), token);
|
|
await Plain<DmPost>(token, p => p.GroupId, p => p.ID);
|
|
|
|
await Unique<ForeignAvatar>(a => a.ActorURI, Builders<ForeignAvatar>.Filter.Gt(a => a.ActorURI, ""), token);
|
|
await Plain<ForeignAvatar>(token, a => a.PublicKeyId);
|
|
|
|
await DB.Default.Index<Follower>()
|
|
.Key(f => f.LocalActorId, KeyType.Ascending)
|
|
.Key(f => f.LocalActorKind, KeyType.Ascending)
|
|
.Key(f => f.ActorURI, KeyType.Ascending)
|
|
.Option(o => o.Unique = true)
|
|
.CreateAsync(token);
|
|
await Plain<Follower>(token, f => f.ActorURI);
|
|
|
|
await DB.Default.Index<RootToAvatar>()
|
|
.Key(r => r.RootId, KeyType.Ascending)
|
|
.Key(r => r.AvatarId, KeyType.Ascending)
|
|
.Option(o => o.Unique = true)
|
|
.CreateAsync(token);
|
|
await Plain<RootToAvatar>(token, r => r.AvatarId);
|
|
|
|
await Unique<ReservedName>(r => r.Name, Builders<ReservedName>.Filter.Gt(r => r.Name, ""), token);
|
|
await Unique<RootUser>(u => u.UserName, Builders<RootUser>.Filter.Gt(u => u.UserName, ""), token);
|
|
await Plain<Avatar>(token, a => a.UserName);
|
|
await Plain<Group>(token, g => g.UserName);
|
|
await Plain<Group>(token, g => g.InvitationCode);
|
|
|
|
await Plain<DmGroup>(token, g => g.ParticipantsKey);
|
|
await Plain<DmGroup>(token, g => g.LastPostId);
|
|
await DB.Default.Index<ConversationState>()
|
|
.Key(s => s.AvatarId, KeyType.Ascending)
|
|
.Key(s => s.ConversationId, KeyType.Ascending)
|
|
.Option(o => o.Unique = true)
|
|
.CreateAsync(token);
|
|
await Unique<GroupAnnouncement>(a => a.ActivityURI, Builders<GroupAnnouncement>.Filter.Gt(a => a.ActivityURI, ""), token);
|
|
|
|
await Plain<Delivery>(token, d => d.DeliveredAt, d => d.AbandonedAt, d => d.NextAttemptAt);
|
|
|
|
await Plain<Job>(token, j => j.Kind, j => j.State, j => j.RunAt);
|
|
await Plain<Job>(token, j => j.State, j => j.LeasedUntil);
|
|
await Unique<Job>(j => j.DedupeKey, Builders<Job>.Filter.Gt(j => j.DedupeKey, ""), token);
|
|
await DB.Default.Index<Job>()
|
|
.Key(j => j.FinishedAt, KeyType.Ascending)
|
|
.Option(o => o.ExpireAfter = TimeSpan.FromDays(7))
|
|
.CreateAsync(token);
|
|
await DB.Default.Index<Following>()
|
|
.Key(f => f.AvatarId, KeyType.Ascending)
|
|
.Key(f => f.TargetActorURI, KeyType.Ascending)
|
|
.Option(o => o.Unique = true)
|
|
.CreateAsync(token);
|
|
await Plain<Following>(token, f => f.TargetActorURI, f => f.State);
|
|
await Plain<Following>(token, f => f.TargetAccountId, f => f.State);
|
|
await Plain<Following>(token, f => f.TargetActorURI, f => f.State);//who here follows the author of what arrives
|
|
await Plain<Following>(token, f => f.FollowActivityURI);
|
|
await DB.Default.Index<TimelineEntry>()
|
|
.Key(e => e.AvatarId, KeyType.Ascending)
|
|
.Key(e => e.PostId, KeyType.Ascending)
|
|
.Option(o => o.Unique = true)
|
|
.CreateAsync(token);
|
|
await Plain<TimelineEntry>(token, e => e.PostId);
|
|
await Plain<TimelineEntry>(token, e => e.AuthorAccountId);
|
|
await Unique<Notification>(n => n.DedupeKey, Builders<Notification>.Filter.Gt(n => n.DedupeKey, ""), token);
|
|
await Plain<Notification>(token, n => n.AvatarId, n => n.ID);
|
|
await DB.Default.Index<Favourite>()
|
|
.Key(f => f.AccountId, KeyType.Ascending)
|
|
.Key(f => f.PostId, KeyType.Ascending)
|
|
.Option(o => o.Unique = true)
|
|
.CreateAsync(token);
|
|
await Plain<Favourite>(token, f => f.ActivityURI);
|
|
await Plain<Favourite>(token, f => f.PostId);
|
|
await DB.Default.Index<Participation>()
|
|
.Key(p => p.AvatarId, KeyType.Ascending)
|
|
.Key(p => p.PostId, KeyType.Ascending)
|
|
.Option(o => o.Unique = true)
|
|
.CreateAsync(token);
|
|
await Plain<Participation>(token, p => p.ActivityURI);
|
|
await DB.Default.Index<Marker>()
|
|
.Key(m => m.AvatarId, KeyType.Ascending)
|
|
.Key(m => m.Timeline, KeyType.Ascending)
|
|
.Option(o => o.Unique = true)
|
|
.CreateAsync(token);
|
|
foreach (var pair in new (Func<Task>, string)[]
|
|
{
|
|
(() => DB.Default.Index<Block>().Key(b => b.AvatarId, KeyType.Ascending).Key(b => b.TargetActorURI, KeyType.Ascending).Option(o => o.Unique = true).CreateAsync(token), "block"),
|
|
(() => DB.Default.Index<FollowedTag>().Key(t => t.AvatarId, KeyType.Ascending).Key(t => t.Name, KeyType.Ascending).Option(o => o.Unique = true).CreateAsync(token), "followed tag"),
|
|
(() => DB.Default.Index<PersonaListMember>().Key(m => m.ListId, KeyType.Ascending).Key(m => m.AccountId, KeyType.Ascending).Option(o => o.Unique = true).CreateAsync(token), "list member"),
|
|
(() => DB.Default.Index<BlockedBy>().Key(b => b.AvatarId, KeyType.Ascending).Key(b => b.ActorURI, KeyType.Ascending).Option(o => o.Unique = true).CreateAsync(token), "blocked by"),
|
|
(() => DB.Default.Index<Mute>().Key(m => m.AvatarId, KeyType.Ascending).Key(m => m.TargetActorURI, KeyType.Ascending).Option(o => o.Unique = true).CreateAsync(token), "mute"),
|
|
(() => DB.Default.Index<AccountDomainBlock>().Key(b => b.AvatarId, KeyType.Ascending).Key(b => b.Domain, KeyType.Ascending).Option(o => o.Unique = true).CreateAsync(token), "domain block"),
|
|
(() => DB.Default.Index<Bookmark>().Key(b => b.AvatarId, KeyType.Ascending).Key(b => b.PostId, KeyType.Ascending).Option(o => o.Unique = true).CreateAsync(token), "bookmark"),
|
|
(() => DB.Default.Index<Pin>().Key(p => p.AvatarId, KeyType.Ascending).Key(p => p.PostId, KeyType.Ascending).Option(o => o.Unique = true).CreateAsync(token), "pin"),
|
|
(() => DB.Default.Index<Models.Federation.RelaySubscription>().Key(r => r.Configured, KeyType.Ascending).Option(o => o.Unique = true).CreateAsync(token), "relay"),
|
|
(() => DB.Default.Index<RemoteFeatured>().Key(f => f.ActorURI, KeyType.Ascending).Key(f => f.ObjectURI, KeyType.Ascending).Option(o => o.Unique = true).CreateAsync(token), "remote-featured"),
|
|
(() => DB.Default.Index<NotificationPolicy>().Key(p => p.AvatarId, KeyType.Ascending).Option(o => o.Unique = true).CreateAsync(token), "notification policy"),
|
|
(() => DB.Default.Index<NotificationRequest>().Key(r => r.AvatarId, KeyType.Ascending).Key(r => r.FromActorURI, KeyType.Ascending).Option(o => o.Unique = true).CreateAsync(token), "notification request"),
|
|
(() => DB.Default.Index<NotificationPermission>().Key(p => p.AvatarId, KeyType.Ascending).Key(p => p.FromActorURI, KeyType.Ascending).Option(o => o.Unique = true).CreateAsync(token), "notification permission")
|
|
})
|
|
await pair.Item1();
|
|
await Plain<Block>(token, b => b.TargetActorURI);
|
|
await Plain<Pin>(token, p => p.PostId);
|
|
await Plain<BlockedBy>(token, b => b.ActorURI);
|
|
await Plain<PersonaList>(token, l => l.AvatarId);
|
|
await Plain<PersonaFilter>(token, f => f.AvatarId);
|
|
await Plain<FollowedTag>(token, t => t.Name);
|
|
await Plain<Post>(token, p => p.Tags);
|
|
//the words status search reads, in every language alike (no stemming, no stop words); Language is not the
|
|
//index's language field
|
|
await DB.Default.Index<Post>().Key(p => p.Text, KeyType.Text).Key(p => p.ContentHtml, KeyType.Text).Key(p => p.SpoilerText, KeyType.Text)
|
|
.Key(p => p.Title, KeyType.Text).Option(o => { o.DefaultLanguage = "none"; o.LanguageOverride = "textSearchLanguage"; o.Name = "words"; })
|
|
.CreateAsync(token);
|
|
await Plain<PersonaListMember>(token, m => m.AvatarId);
|
|
await Plain<Mute>(token, m => m.TargetActorURI);
|
|
await Plain<Report>(token, r => r.IsResolved, r => r.ID);
|
|
await Unique<DomainBlock>(b => b.Domain, Builders<DomainBlock>.Filter.Gt(b => b.Domain, ""), token);
|
|
await Unique<RemoteInstance>(i => i.Host, Builders<RemoteInstance>.Filter.Gt(i => i.Host, ""), token);
|
|
await Unique<ObjectRecord>(r => r.ObjectURI, Builders<ObjectRecord>.Filter.Gt(r => r.ObjectURI, ""), token);
|
|
await Plain<ObjectRecord>(token, r => r.PostId);
|
|
await Unique<DeletedObject>(d => d.ObjectURI, Builders<DeletedObject>.Filter.Gt(d => d.ObjectURI, ""), token);
|
|
await DB.Default.Index<DeletedObject>().Key(d => d.DeletedAt, KeyType.Ascending).Option(o => o.ExpireAfter = TombstoneLifetime).CreateAsync(token);
|
|
await DB.Default.Index<Downvote>().Key(d => d.PostId, KeyType.Ascending).Key(d => d.ActorURI, KeyType.Ascending).Option(o => o.Unique = true).CreateAsync(token);
|
|
await Plain<Downvote>(token, d => d.ActivityURI);
|
|
await DB.Default.Index<PollVote>().Key(v => v.PostId, KeyType.Ascending).Key(v => v.ActorURI, KeyType.Ascending).Key(v => v.Choice, KeyType.Ascending)
|
|
.Option(o => o.Unique = true).CreateAsync(token);
|
|
await DB.Default.Index<PollVote>().Key(v => v.PostId, KeyType.Ascending).Key(v => v.VoterAccountId, KeyType.Ascending).CreateAsync(token);
|
|
await DB.Default.Index<Reaction>().Key(r => r.PostId, KeyType.Ascending).Key(r => r.ActorURI, KeyType.Ascending).Key(r => r.Emoji, KeyType.Ascending)
|
|
.Option(o => o.Unique = true).CreateAsync(token);
|
|
await Plain<Reaction>(token, r => r.ActivityURI);
|
|
await DB.Default.Index<QuoteLicence>().Key(l => l.PostId, KeyType.Ascending).Key(l => l.QuotingObjectURI, KeyType.Ascending).CreateAsync(token);
|
|
await Unique<Domain.Content.LinkPreview>(p => p.Url, Builders<Domain.Content.LinkPreview>.Filter.Gt(p => p.Url, ""), token);
|
|
|
|
await DB.Default.Index<InteractionEvent>().Key(e => e.At, KeyType.Ascending).Option(o => o.ExpireAfter = InteractionEvent.Retention).CreateAsync(token);
|
|
await Plain<InteractionEvent>(token, e => e.Host, e => e.At);
|
|
await DB.Default.Index<InteractionSalt>().Key(s => s.Day, KeyType.Ascending).Option(o => o.Unique = true).CreateAsync(token);
|
|
await DB.Default.Index<InteractionSalt>().Key(s => s.ExpiresAt, KeyType.Ascending).Option(o => o.ExpireAfter = TimeSpan.Zero).CreateAsync(token);
|
|
await DB.Default.Index<InstanceDay>().Key(d => d.Day, KeyType.Ascending).Key(d => d.Host, KeyType.Ascending).Option(o => o.Unique = true).CreateAsync(token);
|
|
await Plain<InstanceDay>(token, d => d.Host, d => d.Day);
|
|
await DB.Default.Index<ServerDay>().Key(d => d.Day, KeyType.Ascending).Option(o => o.Unique = true).CreateAsync(token);
|
|
await DB.Default.Index<RemoteInstanceSnapshot>().Key(s => s.Host, KeyType.Ascending).Key(s => s.Week, KeyType.Ascending)
|
|
.Option(o => o.Unique = true).CreateAsync(token);
|
|
await Plain<RemoteInstanceSnapshot>(token, s => s.Week);
|
|
await Plain<RemoteInstanceSnapshot>(token, s => s.CdnDomain, s => s.Week);
|
|
await Plain<RemoteInstanceSnapshot>(token, s => s.Cdn, s => s.Week);
|
|
await Plain<CdnRangeSet>(token, s => s.Key);
|
|
await Plain<RemoteInstance>(token, i => i.Seen);
|
|
await Plain<RemoteInstance>(token, i => i.LastCrawledAt);
|
|
}
|
|
|
|
// unique among the documents that have the key: the partial filter is `$gt: ""` (any non-empty string) rather than
|
|
// `$type: "string"`, because MongoDB uses a partial index for a lookup only when the lookup implies its filter, and an
|
|
// equality on a string implies the first, never the second: every lookup by these keys scanned the collection
|
|
// (tools/pasture/load.sh, 280 ms a lookup at 30 000 posts)
|
|
static async Task Unique<T>(System.Linq.Expressions.Expression<Func<T, object>> key, FilterDefinition<T> partial,
|
|
CancellationToken token) where T : IEntity =>
|
|
await DB.Default.Index<T>()
|
|
.Key(key, KeyType.Ascending)
|
|
.Option(o =>
|
|
{
|
|
o.Unique = true;
|
|
o.PartialFilterExpression = partial;
|
|
})
|
|
.CreateAsync(token);
|
|
|
|
static async Task Plain<T>(CancellationToken token, params System.Linq.Expressions.Expression<Func<T, object>>[] keys) where T : IEntity
|
|
{
|
|
var index = DB.Default.Index<T>();
|
|
foreach (var key in keys)
|
|
index.Key(key, KeyType.Ascending);
|
|
await index.CreateAsync(token);
|
|
}
|
|
}
|
|
}
|