SocialPub/PrivaPub/Federation/Actors/AccountCountsJob.cs

using MongoDB.Entities;

using PrivaPub.Federation.Objects;
using PrivaPub.Infrastructure.Http;
using PrivaPub.Infrastructure.Jobs;
using PrivaPub.Models.Jobs;
using PrivaPub.Models.User;
using PrivaPub.StaticServices;

namespace PrivaPub.Federation.Actors
{
	// A remote account's follower, following and post counts, as its own server publishes them (the totalItems of its
	// followers, following and outbox collections), and the posts it features. Read when the account is fetched, at most
	// once a day, never when someone looks at it; a hidden or missing collection leaves its count unknown. Mastodon and
	// GoToSocial do the same. A feed's following is its communities: they are kept, and read in turn, so its list shows.
	public class AccountCountsJob : IJobHandler
	{
		readonly IRemoteActorService _remoteActors;
		readonly DbEntities _dbEntities;
		readonly IFeaturedPosts _featured;
		readonly ILocalActorService _localActors;

		public const int MaxFeedCommunities = 50;

		public AccountCountsJob(IRemoteActorService remoteActors, DbEntities dbEntities, IFeaturedPosts featured = default,
			ILocalActorService localActors = default)
		{
			_remoteActors = remoteActors;
			_dbEntities = dbEntities;
			_featured = featured;
			_localActors = localActors;
		}

		public JobKind Kind => JobKind.CountAccount;
		public int Concurrency => 2;
		public int MaxAttempts => 2;
		public int PerHostLimit => 1;

		public static string DedupeKey(string actorUri, DateTime day) => $"count|{actorUri}|{day:yyyyMMdd}";

		public async Task<JobOutcome> Handle(Job job, CancellationToken token)
		{
			var actor = await _dbEntities.ForeignAvatars.Match(a => a.ActorURI == job.Payload && !a.DeletionAt.HasValue).ExecuteFirstAsync(token);
			if (actor == default)
				return JobOutcome.Done;
			var feed = actor.AvatarType == AvatarType.Feed;
			var (following, communities) = await Read(actor, actor.FollowingURL, feed, token);
			var update = DB.Default.Update<ForeignAvatar>().MatchID(actor.ID)
				.Modify(a => a.FollowersCount, (await Read(actor, actor.FollowersURL, false, token)).Total)
				.Modify(a => a.FollowingCount, following)
				.Modify(a => a.StatusesCount, (await Read(actor, actor.OutboxURL, false, token)).Total)
				.Modify(a => a.CountedAt, DateTime.UtcNow);
			if (feed)
				update = update.Modify(a => a.FeedCommunities, communities);
			await update.ExecuteAsync(token);
			if (feed)
				foreach (var community in communities)
					if (_localActors == default || !community.StartsWith(_localActors.BaseAddress + "/", StringComparison.Ordinal))
						await _remoteActors.GetActor(community, refresh: false, token);
			if (_featured != default)
				await _featured.Sync(actor, token);
			return JobOutcome.Done;
		}

		// only from the actor's own server, as everything we believe about it: its total, and the ids it lists when asked
		// (inline, or on its first page)
		async Task<(int? Total, List<string> Items)> Read(ForeignAvatar actor, string collection, bool items, CancellationToken token)
		{
			var listed = new List<string>();
			if (string.IsNullOrEmpty(collection) || !Origin.Same(collection, actor.ActorURI))
				return (default, listed);
			using var scope = HttpScope.For("collection");
			using var fetched = await _remoteActors.FetchObject(collection, token);
			if (fetched == default || fetched.Root.ValueKind != System.Text.Json.JsonValueKind.Object)
				return (default, listed);
			int? total = fetched.Root.TryGetProperty("totalItems", out var counted) && counted.TryGetInt32(out var count) && count >= 0 ? count : default;
			if (!items)
				return (total, listed);
			var document = System.Text.Json.Nodes.JsonNode.Parse(fetched.Root.GetRawText());
			var page = document["orderedItems"] ?? document["items"] ?? document["first"]?["orderedItems"] ?? document["first"]?["items"];
			if (page == default && ActivityJson.Id(document["first"]) is { } first && Origin.Same(first, actor.ActorURI))
			{
				using var next = await _remoteActors.FetchObject(first, token);
				var nextPage = next == default ? default : System.Text.Json.Nodes.JsonNode.Parse(next.Root.GetRawText());
				page = nextPage?["orderedItems"] ?? nextPage?["items"];
			}
			if (page is System.Text.Json.Nodes.JsonArray array)
				listed = array.Select(ActivityJson.Id)
					.Where(id => Uri.TryCreate(id, UriKind.Absolute, out var uri) && uri.Scheme is "https" or "http")
					.Distinct(StringComparer.Ordinal).Take(MaxFeedCommunities).ToList();
			return (total, listed);
		}
	}
}