SocialPub/PrivaPub/Domain/Statistics/ServerPlaces.cs

using Microsoft.Extensions.Caching.Memory;

using MongoDB.Entities;

using PrivaPub.Federation.Objects;
using PrivaPub.Infrastructure.Geo;
using PrivaPub.Models.Jobs;

namespace PrivaPub.Domain.Statistics
{
	// Where a server was before it went behind a CDN, as its weekly snapshots last saw it.
	public sealed record PlaceBeforeCdn(string Week, string Country, string City, double? Latitude, double? Longitude);

	// A CDN as a group of servers: those behind it now, and week by week how many were behind it, who joined and who left.
	public sealed record CdnGroup(string Key, string Name, string Domain, int Servers, int Ranges, DateTime? RangesUpdatedAt, List<string> DetectedBy);

	public sealed record CdnWeek(string Week, int Servers, List<string> Joined, List<string> Left);

	public sealed record CdnHistory(CdnGroup Group, List<string> Hosts, List<CdnWeek> Weeks);

	public interface IServerPlaces
	{
		Task<Dictionary<string, PlaceBeforeCdn>> BeforeCdn(IEnumerable<string> hosts, CancellationToken token);
		Task<List<RemoteInstanceSnapshot>> History(string host, int weeks, CancellationToken token);
		Task<List<CdnGroup>> Cdns(CancellationToken token);
		Task<CdnHistory> Cdn(string key, int weeks, CancellationToken token);
	}

	// Servers through time (owner decision 2026-10-04): each server's weekly snapshots (place, CDN, size), and the servers
	// grouped by the CDN in front of them, named by the CDN's own domain. Read from the weekly snapshots InstanceDescriber
	// writes; the CDN pages are kept for half an hour, because a large CDN reads many snapshots.
	public class ServerPlaces : IServerPlaces
	{
		public const int MaxHosts = 500;
		const int MaxChanges = 50;
		static readonly TimeSpan CacheFor = TimeSpan.FromMinutes(30);

		readonly ICdnRanges _ranges;
		readonly IMemoryCache _cache;

		public ServerPlaces(ICdnRanges ranges, IMemoryCache cache)
		{
			_ranges = ranges;
			_cache = cache;
		}

		// the CDN key a server's place belongs to: the CDN's domain, or its name for a CDN that has none
		public static string KeyOf(InstanceGeo geo) =>
			geo?.Cdn == default ? default : geo.CdnDomain ?? CdnCatalog.ByKey(geo.Cdn)?.Key ?? geo.Cdn.ToLowerInvariant();

		static string KeyOf(RemoteInstanceSnapshot snapshot) =>
			snapshot.Cdn == default ? default : snapshot.CdnDomain ?? CdnCatalog.ByKey(snapshot.Cdn)?.Key ?? snapshot.Cdn.ToLowerInvariant();

		public async Task<Dictionary<string, PlaceBeforeCdn>> BeforeCdn(IEnumerable<string> hosts, CancellationToken token)
		{
			var wanted = hosts.Where(h => h != default).Distinct().ToList();
			if (wanted.Count == 0)
				return new();
			var located = await DB.Default.Find<RemoteInstanceSnapshot>()
				.Match(s => wanted.Contains(s.Host) && s.Cdn == null && (s.Country != null || s.Latitude != null))
				.Sort(s => s.Week, Order.Descending)
				.ExecuteAsync(token);
			return located.GroupBy(s => s.Host)
				.Select(g => g.First())
				.ToDictionary(s => s.Host, s => new PlaceBeforeCdn(s.Week, s.Country, s.City, s.Latitude, s.Longitude));
		}

		public async Task<List<RemoteInstanceSnapshot>> History(string host, int weeks, CancellationToken token) =>
			await DB.Default.Find<RemoteInstanceSnapshot>()
				.Match(s => s.Host == host)
				.Sort(s => s.Week, Order.Descending)
				.Limit(weeks)
				.ExecuteAsync(token);

		public async Task<List<CdnGroup>> Cdns(CancellationToken token)
		{
			var fronted = await Fronted(token);
			var status = _ranges.Status;
			var keys = CdnCatalog.All.Select(p => p.Key).Concat(fronted.Select(i => KeyOf(i.Geo))).Distinct(StringComparer.OrdinalIgnoreCase);
			return keys.Select(key => Group(key, fronted, status)).OrderByDescending(g => g.Servers).ThenBy(g => g.Name).ToList();
		}

		public async Task<CdnHistory> Cdn(string key, int weeks, CancellationToken token)
		{
			key = key?.Trim().ToLowerInvariant();
			if (string.IsNullOrEmpty(key) || key.Length > 100)
				return default;
			return await _cache.GetOrCreateAsync($"cdn-history|{key}|{weeks}", async entry =>
			{
				entry.AbsoluteExpirationRelativeToNow = CacheFor;
				var fronted = await Fronted(token);
				var provider = CdnCatalog.ByKey(key);
				if (provider == default && fronted.All(i => KeyOf(i.Geo) != key))
					return default;
				var group = Group(provider?.Key ?? key, fronted, _ranges.Status);
				var hosts = fronted.Where(i => KeyOf(i.Geo) == group.Key).Select(i => i.Host).OrderBy(h => h).Take(MaxHosts).ToList();
				return new CdnHistory(group, hosts, await Weeks(group, weeks, token));
			});
		}

		async Task<List<CdnWeek>> Weeks(CdnGroup group, int weeks, CancellationToken token)
		{
			var since = InstanceDescriber.Week(DateTime.UtcNow.AddDays(-7 * weeks));
			var everBehind = (await DB.Default.Find<RemoteInstanceSnapshot>()
					.Match(f => f.Gte(s => s.Week, since) & (f.Eq(s => s.CdnDomain, group.Key) | f.Eq(s => s.Cdn, group.Name)))
					.Project(s => new RemoteInstanceSnapshot { Host = s.Host })
					.ExecuteAsync(token))
				.Select(s => s.Host).Distinct().ToList();
			if (everBehind.Count == 0)
				return new();

			var snapshots = await DB.Default.Find<RemoteInstanceSnapshot>()
				.Match(f => f.In(s => s.Host, everBehind) & f.Gte(s => s.Week, since))
				.Project(s => new RemoteInstanceSnapshot { Host = s.Host, Week = s.Week, Cdn = s.Cdn, CdnDomain = s.CdnDomain })
				.ExecuteAsync(token);

			// a server is behind the CDN from the week a snapshot shows it there until one shows it elsewhere
			var behind = new Dictionary<string, bool>();
			var history = new List<CdnWeek>();
			foreach (var week in snapshots.GroupBy(s => s.Week).OrderBy(w => w.Key, StringComparer.Ordinal))
			{
				var joined = new List<string>();
				var left = new List<string>();
				foreach (var snapshot in week)
				{
					var now = KeyOf(snapshot) == group.Key;
					var before = behind.GetValueOrDefault(snapshot.Host);
					if (now && !before)
						joined.Add(snapshot.Host);
					else if (!now && before)
						left.Add(snapshot.Host);
					behind[snapshot.Host] = now;
				}
				history.Add(new CdnWeek(week.Key, behind.Count(b => b.Value), joined.OrderBy(h => h).Take(MaxChanges).ToList(),
					left.OrderBy(h => h).Take(MaxChanges).ToList()));
			}
			history.Reverse();
			return history;
		}

		static async Task<List<RemoteInstance>> Fronted(CancellationToken token) =>
			await DB.Default.Find<RemoteInstance>()
				.Match(i => i.Geo.Cdn != null)
				.Project(i => new RemoteInstance { Host = i.Host, Geo = i.Geo })
				.ExecuteAsync(token);

		static CdnGroup Group(string key, List<RemoteInstance> fronted, IReadOnlyDictionary<string, (int Ranges, DateTime FetchedAt)> status)
		{
			var provider = CdnCatalog.ByKey(key);
			var servers = fronted.Where(i => KeyOf(i.Geo) == key).ToList();
			var detectedBy = new List<string>();
			if (provider?.Lists.Length > 0)
				detectedBy.Add("ranges");
			if (provider != default)
				detectedBy.Add("headers");
			if (provider?.Asns.Length > 0)
				detectedBy.Add("asn");
			var ranges = status.TryGetValue(key, out var known) ? known : default;
			return new CdnGroup(key, provider?.Name ?? servers.Select(i => i.Geo.Cdn).FirstOrDefault() ?? key, provider?.Domain, servers.Count, ranges.Ranges,
				ranges.Ranges > 0 ? ranges.FetchedAt : null, detectedBy);
		}
	}
}