SocialPub/PrivaPub/Infrastructure/Geo/CdnUpdater.cs

using Microsoft.Extensions.Options;

using MongoDB.Entities;

using PrivaPub.Infrastructure.Statistics;
using PrivaPub.Models.Jobs;

using System.Text;

namespace PrivaPub.Infrastructure.Geo
{
	// Keeps the CDNs' published address ranges current by itself (owner decision 2026-10-04): at start it loads the stored
	// ones, then once a day it downloads each list again. A list that fails to download or to parse, or comes back empty,
	// leaves the stored one in place and is tried the next day. Like GeoUpdater, plain HTTPS to fixed hosts, not federation.
	public sealed class CdnUpdater : BackgroundService
	{
		public const string ClientName = "cdn";
		static readonly TimeSpan FirstWait = TimeSpan.FromMinutes(1);
		static readonly TimeSpan Interval = TimeSpan.FromHours(24);
		const int MaxBytes = 16 * 1024 * 1024;
		const int MaxRanges = 200_000;

		readonly IHttpClientFactory _http;
		readonly IOptionsMonitor<StatisticsOptions> _options;
		readonly ICdnRanges _ranges;
		readonly ILogger<CdnUpdater> _logger;

		public CdnUpdater(IHttpClientFactory http, IOptionsMonitor<StatisticsOptions> options, ICdnRanges ranges, ILogger<CdnUpdater> logger)
		{
			_http = http;
			_options = options;
			_ranges = ranges;
			_logger = logger;
		}

		protected override async Task ExecuteAsync(CancellationToken stoppingToken)
		{
			try
			{
				await _ranges.Load(stoppingToken);
				await Task.Delay(FirstWait, stoppingToken);
				while (!stoppingToken.IsCancellationRequested)
				{
					if (_options.CurrentValue.Cdn.AutoUpdate)
						await Update(CdnCatalog.All, stoppingToken);
					await Task.Delay(Interval + TimeSpan.FromMinutes(Random.Shared.Next(60)), stoppingToken);
				}
			}
			catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
			{
			}
			catch (Exception ex)
			{
				_logger.LogWarning(ex, "{Service} stopped", nameof(CdnUpdater));
			}
		}

		// The number of CDNs whose ranges were replaced.
		public async Task<int> Update(IEnumerable<CdnProvider> providers, CancellationToken token)
		{
			var replaced = 0;
			foreach (var provider in providers.Where(p => p.Lists.Length > 0))
			{
				try
				{
					var ranges = new List<string>();
					foreach (var list in provider.Lists)
						ranges.AddRange(CdnCatalog.Parse(list.Format, await Download(list, token)).Select(n => n.ToString()));
					ranges = ranges.Distinct().ToList();
					if (ranges.Count == 0 || ranges.Count > MaxRanges)
						throw new InvalidDataException($"{ranges.Count} ranges");

					var set = await DB.Default.Find<CdnRangeSet>().Match(s => s.Key == provider.Key).ExecuteFirstAsync(token) ?? new CdnRangeSet { Key = provider.Key };
					set.Name = provider.Name;
					set.Ranges = ranges;
					set.Sources = provider.Lists.Select(l => l.Url).ToList();
					set.FetchedAt = DateTime.UtcNow;
					await DB.Default.SaveAsync(set, token);
					_ranges.Replace(set);
					replaced++;
				}
				catch (Exception ex) when (ex is HttpRequestException or InvalidDataException or FormatException or System.Text.Json.JsonException
					or TaskCanceledException && !token.IsCancellationRequested)
				{
					_logger.LogWarning(ex, "Could not update the published ranges of {Cdn}", provider.Name);
				}
			}
			return replaced;
		}

		async Task<string> Download(CdnList list, CancellationToken token)
		{
			using var request = new HttpRequestMessage(list.Method == "POST" ? HttpMethod.Post : HttpMethod.Get, list.Url);
			request.Headers.Accept.ParseAdd("application/json, text/plain;q=0.9");
			if (list.Body != default)
				request.Content = new StringContent(list.Body, Encoding.UTF8, "application/x-www-form-urlencoded");
			using var response = await _http.CreateClient(ClientName).SendAsync(request, HttpCompletionOption.ResponseHeadersRead, token);
			response.EnsureSuccessStatusCode();
			if (response.Content.Headers.ContentLength > MaxBytes)
				throw new InvalidDataException($"more than {MaxBytes} bytes");
			await using var stream = await response.Content.ReadAsStreamAsync(token);
			using var buffer = new MemoryStream();
			var chunk = new byte[81920];
			int read;
			while ((read = await stream.ReadAsync(chunk, token)) > 0)
			{
				if (buffer.Length + read > MaxBytes)
					throw new InvalidDataException($"more than {MaxBytes} bytes");
				buffer.Write(chunk, 0, read);
			}
			return Encoding.UTF8.GetString(buffer.ToArray());
		}
	}
}