Streaming: what happens, told as it happens

Mastodon's streaming API: /api/v1/streaming as a WebSocket (streams
subscribed in the URL or by message) and /api/v1/streaming/{stream} as
server-sent events, with health and the URL advertised. The user stream
tells posts reaching the persona's home (not those an exclusive list keeps
apart, which its list stream tells), notifications, edits and deletions;
public, hashtag and list streams tell what belongs in them. An in-process
hub carries ids only; each connection maps a post or a notification for its
own persona as it sends it, so nothing it may not see, or whose author it
blocked or muted, goes out. A deletion reaches only the streams that showed
the post. The token comes as access_token, header or WebSocket protocol.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01LsXgEaXee4GCU1hwYgPJXw

PrivaPub/Domain/Social/Notifications.cs

@@ -16,7 +16,7 @@ namespace PrivaPub.Domain.Social
 				return;
 			try
 			{
-				await DB.Default.SaveAsync(new Notification
+				var notification = new Notification
 				{
 					AvatarId = avatarId,
 					Type = type,
@@ -26,7 +26,11 @@ namespace PrivaPub.Domain.Social
 					Emoji = emoji,
 					EmojiURL = emojiUrl,
 					DedupeKey = emoji == default ? $"{type}|{avatarId}|{fromActorUri}|{postId}" : $"{type}|{avatarId}|{fromActorUri}|{postId}|{emoji}"
-				}, token);
+				};
+				await DB.Default.SaveAsync(notification, token);
+				var told = new Timelines.Streams.Event("notification", notification.ID);
+				Timelines.Streams.ToPersona(avatarId, new Timelines.Streams.Key(Timelines.Streams.User), told);
+				Timelines.Streams.ToPersona(avatarId, new Timelines.Streams.Key(Timelines.Streams.UserNotification), told);
 			}
 			catch (MongoWriteException ex) when (ex.WriteError?.Category == ServerErrorCategory.DuplicateKey)
 			{

PrivaPub/Domain/Statuses/StatusService.cs

@@ -310,6 +310,7 @@ namespace PrivaPub.Domain.Statuses
 			post.EditedAt = DateTime.UtcNow;
 			post.UpdateDate = post.EditedAt;
 			await DB.Default.SaveAsync(post, token);
+			await Fanout.Edited(post, token);
 
 			if (!post.IsLocalOnly)
 			{
@@ -354,6 +355,7 @@ namespace PrivaPub.Domain.Statuses
 				.Modify(p => p.Media, new List<PostMedia>())
 				.Modify(p => p.Revisions, new List<PostRevision>())
 				.ExecuteAsync(token);
+			await Fanout.Deleting(post, token);
 			await DB.Default.DeleteAsync<TimelineEntry>(e => e.PostId == post.ID || e.ReblogOfPostId == post.ID);
 			// boosts of it end with it, as on Mastodon, so no count keeps them
 			await DB.Default.Update<PostEntity>().Match(p => p.ReblogOfPostId == post.ID && !p.DeletedAt.HasValue)
@@ -493,6 +495,7 @@ namespace PrivaPub.Domain.Statuses
 
 			if (existing == default)
 				return new StatusOutcome(original);
+			await Fanout.Deleting(existing, token);
 			await DB.Default.DeleteAsync<PostEntity>(existing.ID);
 			await DB.Default.DeleteAsync<TimelineEntry>(e => e.PostId == existing.ID);
 			await DB.Default.Update<PostEntity>().MatchID(original.ID).Modify(b => b.Inc(p => p.ReblogsCount, -1)).ExecuteAsync(token);

PrivaPub/Domain/Timelines/Fanout.cs

@@ -85,6 +85,90 @@ namespace PrivaPub.Domain.Timelines
 			if (string.IsNullOrEmpty(post.ReblogOfPostId))
 				foreach (var mention in post.Mentions.Where(m => m.IsLocal && !string.IsNullOrEmpty(m.AccountId)))
 					await Notifications.Add(mention.AccountId, NotificationType.Mention, post.AuthorAccountId ?? post.GroupUserId, post.ActorURI, post.ID, token);
+			if (Streams.Anyone)
+				await Stream(post, recipients, token);
+		}
+
+		// the post to the streams it belongs in: the homes it reached (but not where an exclusive list keeps its author
+		// apart), the lists of those homes that hold its author, and the public feeds and hashtags when it is public
+		static async Task Stream(Post post, IReadOnlyCollection<string> recipients, CancellationToken token)
+		{
+			var update = new Streams.Event("update", post.ID);
+			var author = post.AuthorAccountId ?? post.GroupUserId;
+			var listening = recipients.Where(Streams.Listening).ToList();
+			if (listening.Count > 0)
+			{
+				var memberships = await DB.Default.Find<PersonaListMember>().Match(m => listening.Contains(m.AvatarId) && m.AccountId == author).ExecuteAsync(token);
+				var listIds = memberships.Select(m => m.ListId).Distinct().ToList();
+				var exclusive = listIds.Count == 0
+					? new HashSet<string>()
+					: (await DB.Default.Find<PersonaList>().Match(l => listIds.Contains(l.ID) && l.Exclusive).ExecuteAsync(token)).Select(l => l.ID).ToHashSet();
+				foreach (var avatarId in listening)
+				{
+					var lists = memberships.Where(m => m.AvatarId == avatarId).Select(m => m.ListId).ToList();
+					if (!lists.Any(exclusive.Contains))
+						Streams.ToPersona(avatarId, new Streams.Key(Streams.User), update);
+					foreach (var listId in lists)
+						Streams.ToPersona(avatarId, new Streams.Key(Streams.List, listId), update);
+				}
+			}
+			if (post.Visibility != PostVisibility.Public || !string.IsNullOrEmpty(post.ReblogOfPostId) || !Privacy.VisibilityPolicy.Shown(post))
+				return;
+			var local = !post.IsFederatedCopy;
+			if (!post.IsLocalOnly)
+				Streams.ToAll(new Streams.Key(Streams.Public), update);
+			Streams.ToAll(new Streams.Key(local ? Streams.PublicLocal : Streams.PublicRemote), update);
+			foreach (var tag in post.Tags.Distinct())
+			{
+				if (!post.IsLocalOnly)
+					Streams.ToAll(new Streams.Key(Streams.Hashtag, tag), update);
+				if (local)
+					Streams.ToAll(new Streams.Key(Streams.HashtagLocal, tag), update);
+			}
+		}
+
+		// a post's deletion, told before its home entries go: to the homes holding it or a boost of it (the boost's id
+		// there), to its author's own connections, and to the public feeds and hashtags when it was public. Nobody else
+		// hears of it, not even its id.
+		public static async Task Deleting(Post post, CancellationToken token)
+		{
+			if (!Streams.Anyone || post == default)
+				return;
+			var entries = await DB.Default.Find<TimelineEntry>().Match(e => e.PostId == post.ID || e.ReblogOfPostId == post.ID).ExecuteAsync(token);
+			foreach (var entry in entries.Where(e => Streams.Listening(e.AvatarId)))
+				Streams.ToPersonaEverywhere(entry.AvatarId, new Streams.Event("delete", entry.PostId));
+			var author = post.AuthorAccountId ?? post.GroupUserId;
+			if (!post.IsFederatedCopy && !string.IsNullOrEmpty(author))
+				Streams.ToPersonaEverywhere(author, new Streams.Event("delete", post.ID));
+			if (post.Visibility != PostVisibility.Public || !string.IsNullOrEmpty(post.ReblogOfPostId))
+				return;
+			var gone = new Streams.Event("delete", post.ID);
+			var local = !post.IsFederatedCopy;
+			Streams.ToAll(new Streams.Key(Streams.Public), gone);
+			Streams.ToAll(new Streams.Key(local ? Streams.PublicLocal : Streams.PublicRemote), gone);
+			foreach (var tag in post.Tags.Distinct())
+			{
+				Streams.ToAll(new Streams.Key(Streams.Hashtag, tag), gone);
+				if (local)
+					Streams.ToAll(new Streams.Key(Streams.HashtagLocal, tag), gone);
+			}
+		}
+
+		// an edit to the homes holding the post and, when it is public, to the public feeds (status.update)
+		public static async Task Edited(Post post, CancellationToken token)
+		{
+			if (!Streams.Anyone)
+				return;
+			var update = new Streams.Event("status.update", post.ID);
+			var homes = (await DB.Default.Find<TimelineEntry>().Match(e => e.PostId == post.ID).ExecuteAsync(token)).Select(e => e.AvatarId).Distinct();
+			foreach (var avatarId in homes.Where(Streams.Listening))
+				Streams.ToPersona(avatarId, new Streams.Key(Streams.User), update);
+			if (post.Visibility == PostVisibility.Public && Privacy.VisibilityPolicy.Shown(post))
+			{
+				if (!post.IsLocalOnly)
+					Streams.ToAll(new Streams.Key(Streams.Public), update);
+				Streams.ToAll(new Streams.Key(post.IsFederatedCopy ? Streams.PublicRemote : Streams.PublicLocal), update);
+			}
 		}
 
 		async Task<bool> Shows(Following following, Post post, CancellationToken token)

PrivaPub/Domain/Timelines/Streams.cs

@@ -0,0 +1,107 @@
+using System.Collections.Concurrent;
+using System.Threading.Channels;
+
+namespace PrivaPub.Domain.Timelines
+{
+	// Mastodon's streaming, inside this one process: what happens (a post arriving in someone's home or a public feed, a
+	// notification, an edit, a deletion) is told to the connections listening for it, by id only. Each connection maps a
+	// post or a notification for its own persona when it sends it (StreamingController), so blocks, mutes and visibility
+	// are decided there, never here. Nothing is kept: a connection that is gone misses what happened meanwhile, and a
+	// client reads the timeline again when it comes back.
+	public static class Streams
+	{
+		public const string User = "user";
+		public const string UserNotification = "user:notification";
+		public const string Public = "public";
+		public const string PublicLocal = "public:local";
+		public const string PublicRemote = "public:remote";
+		public const string Hashtag = "hashtag";
+		public const string HashtagLocal = "hashtag:local";
+		public const string List = "list";
+		public const string Direct = "direct";
+
+		public static readonly string[] Names = { User, UserNotification, Public, PublicLocal, PublicRemote, Hashtag, HashtagLocal, List, Direct };
+
+		// what happened: Mastodon's event name, and the id of what it is about
+		public sealed record Event(string Name, string Id);
+
+		// one stream a connection listens to: its name and, for a hashtag or a list, which one
+		public sealed record Key(string Name, string Parameter = default)
+		{
+			public string[] Wire => Parameter == default ? new[] { Name } : new[] { Name, Parameter };
+		}
+
+		public sealed class Listener
+		{
+			readonly ConcurrentDictionary<Key, byte> _keys = new();
+
+			public Listener(string avatarId)
+			{
+				AvatarId = avatarId;
+			}
+
+			public string AvatarId { get; }
+
+			// what reached it and was not sent yet; the oldest goes first when a slow connection falls behind
+			public Channel<(Key Key, Event Event)> Queue { get; } =
+				Channel.CreateBounded<(Key, Event)>(new BoundedChannelOptions(1000) { FullMode = BoundedChannelFullMode.DropOldest, SingleReader = true });
+
+			public void Listen(Key key) => _keys[key] = 0;
+
+			public void Stop(Key key) => _keys.TryRemove(key, out _);
+
+			public bool Hears(Key key) => _keys.ContainsKey(key);
+
+			public IEnumerable<Key> Keys => _keys.Keys;
+
+			public void Tell(Key key, Event e) => Queue.Writer.TryWrite((key, e));
+		}
+
+		static readonly ConcurrentDictionary<Listener, byte> Listeners = new();
+
+		public static Listener Open(string avatarId)
+		{
+			var listener = new Listener(avatarId);
+			Listeners[listener] = 0;
+			return listener;
+		}
+
+		public static void Close(Listener listener)
+		{
+			Listeners.TryRemove(listener, out _);
+			listener.Queue.Writer.TryComplete();
+		}
+
+		public static int Count => Listeners.Count;
+
+		// whether the persona has a connection open, so the work of finding its lists can be skipped when it has none
+		public static bool Listening(string avatarId) => Listeners.Keys.Any(l => l.AvatarId == avatarId);
+
+		// to the persona's own streams: its home (user) and, for a notification, user:notification too
+		public static void ToPersona(string avatarId, Key key, Event e)
+		{
+			foreach (var listener in Listeners.Keys)
+				if (listener.AvatarId == avatarId && listener.Hears(key))
+					listener.Tell(key, e);
+		}
+
+		// to everyone listening to a shared stream (public, a hashtag)
+		public static void ToAll(Key key, Event e)
+		{
+			foreach (var listener in Listeners.Keys)
+				if (listener.Hears(key))
+					listener.Tell(key, e);
+		}
+
+		// to every stream the persona listens to (a deletion: wherever its client may show the post)
+		public static void ToPersonaEverywhere(string avatarId, Event e)
+		{
+			foreach (var listener in Listeners.Keys)
+				if (listener.AvatarId == avatarId)
+					foreach (var key in listener.Keys)
+						listener.Tell(key, e);
+		}
+
+		public static bool Anyone => !Listeners.IsEmpty;
+	}
+}