using Microsoft.Extensions.Options; using MongoDB.Entities; using PrivaPub.Infrastructure.Statistics; using PrivaPub.Models.Jobs; using System.Text; namespace PrivaPub.Infrastructure.Geo { // Keeps the CDNs' published address ranges current by itself (owner decision 2026-10-04): at start it loads the stored // ones, then once a day it downloads each list again. A list that fails to download or to parse, or comes back empty, // leaves the stored one in place and is tried the next day. Like GeoUpdater, plain HTTPS to fixed hosts, not federation. public sealed class CdnUpdater : BackgroundService { public const string ClientName = "cdn"; static readonly TimeSpan FirstWait = TimeSpan.FromMinutes(1); static readonly TimeSpan Interval = TimeSpan.FromHours(24); const int MaxBytes = 16 * 1024 * 1024; const int MaxRanges = 200_000; readonly IHttpClientFactory _http; readonly IOptionsMonitor _options; readonly ICdnRanges _ranges; readonly ILogger _logger; public CdnUpdater(IHttpClientFactory http, IOptionsMonitor options, ICdnRanges ranges, ILogger logger) { _http = http; _options = options; _ranges = ranges; _logger = logger; } protected override async Task ExecuteAsync(CancellationToken stoppingToken) { try { await _ranges.Load(stoppingToken); await Task.Delay(FirstWait, stoppingToken); while (!stoppingToken.IsCancellationRequested) { if (_options.CurrentValue.Cdn.AutoUpdate) await Update(CdnCatalog.All, stoppingToken); await Task.Delay(Interval + TimeSpan.FromMinutes(Random.Shared.Next(60)), stoppingToken); } } catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) { } catch (Exception ex) { _logger.LogWarning(ex, "{Service} stopped", nameof(CdnUpdater)); } } // The number of CDNs whose ranges were replaced. public async Task Update(IEnumerable providers, CancellationToken token) { var replaced = 0; foreach (var provider in providers.Where(p => p.Lists.Length > 0)) { try { var ranges = new List(); foreach (var list in provider.Lists) ranges.AddRange(CdnCatalog.Parse(list.Format, await Download(list, token)).Select(n => n.ToString())); ranges = ranges.Distinct().ToList(); if (ranges.Count == 0 || ranges.Count > MaxRanges) throw new InvalidDataException($"{ranges.Count} ranges"); var set = await DB.Default.Find().Match(s => s.Key == provider.Key).ExecuteFirstAsync(token) ?? new CdnRangeSet { Key = provider.Key }; set.Name = provider.Name; set.Ranges = ranges; set.Sources = provider.Lists.Select(l => l.Url).ToList(); set.FetchedAt = DateTime.UtcNow; await DB.Default.SaveAsync(set, token); _ranges.Replace(set); replaced++; } catch (Exception ex) when (ex is HttpRequestException or InvalidDataException or FormatException or System.Text.Json.JsonException or TaskCanceledException && !token.IsCancellationRequested) { _logger.LogWarning(ex, "Could not update the published ranges of {Cdn}", provider.Name); } } return replaced; } async Task Download(CdnList list, CancellationToken token) { using var request = new HttpRequestMessage(list.Method == "POST" ? HttpMethod.Post : HttpMethod.Get, list.Url); request.Headers.Accept.ParseAdd("application/json, text/plain;q=0.9"); if (list.Body != default) request.Content = new StringContent(list.Body, Encoding.UTF8, "application/x-www-form-urlencoded"); using var response = await _http.CreateClient(ClientName).SendAsync(request, HttpCompletionOption.ResponseHeadersRead, token); response.EnsureSuccessStatusCode(); if (response.Content.Headers.ContentLength > MaxBytes) throw new InvalidDataException($"more than {MaxBytes} bytes"); await using var stream = await response.Content.ReadAsStreamAsync(token); using var buffer = new MemoryStream(); var chunk = new byte[81920]; int read; while ((read = await stream.ReadAsync(chunk, token)) > 0) { if (buffer.Length + read > MaxBytes) throw new InvalidDataException($"more than {MaxBytes} bytes"); buffer.Write(chunk, 0, read); } return Encoding.UTF8.GetString(buffer.ToArray()); } } }