M5: outbound requests, previews, the media proxy and served traffic

- HttpScope (AsyncLocal) tags each outbound request with a purpose and a trigger:
  - purpose is set by the caller: actor, key, object, webfinger, context, nodeinfo;
  - trigger is set by the job kind, by "verify" during inbox verification, or defaults
    to "request".
- FederationHttp records every JSON, media and stream fetch: status, time, bytes, hops,
  and an outcome of ok, refused or failed, with a reason: disallowed, remembered,
  bad-redirect, too-many-redirects, content-type, too-large, bad-json, private-address,
  timeout, network, or the status. A fetch a reader caused (trigger "request") is only
  counted per server per day.
- Link previews record a 'preview' event: card, no-card or failed.
- The media proxy counts cache hits.
- TrafficMeter counts the client API per endpoint group, method and status class. It
  counts our served documents (actor, outbox, collection, object, activity, licence,
  webfinger, nodeinfo) by kind, status and whether signed, per day and never per server,
  and never names a circle's collections.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01ELjqpznMFMNrJoJUj6K5p2

PrivaPub/Infrastructure/Http/FederationHttp.cs

@@ -2,6 +2,8 @@ using Microsoft.Extensions.Caching.Memory;
 using Microsoft.Extensions.Options;
 
 using PrivaPub.Federation.Moderation;
+using PrivaPub.Infrastructure.Statistics;
+using PrivaPub.Models.Statistics;
 
 using System.Net;
 using System.Text.Json;
@@ -49,17 +51,71 @@ namespace PrivaPub.Infrastructure.Http
 		readonly IOptionsMonitor<FederationOptions> _options;
 		readonly IDomainBlocks _domainBlocks;
 		readonly ILogger<FederationHttp> _logger;
+		readonly IInteractionLedger _ledger;
 
 		public FederationHttp(IHttpClientFactory httpClientFactory, IMemoryCache cache, IOptionsMonitor<FederationOptions> options,
-			IDomainBlocks domainBlocks, ILogger<FederationHttp> logger)
+			IDomainBlocks domainBlocks, ILogger<FederationHttp> logger, IInteractionLedger ledger = default)
 		{
 			_httpClientFactory = httpClientFactory;
 			_cache = cache;
 			_options = options;
 			_domainBlocks = domainBlocks;
 			_logger = logger;
+			_ledger = ledger;
 		}
 
+		sealed class Exchange
+		{
+			public Exchange(string url, string purpose)
+			{
+				Host = Interactions.HostOf(url);
+				Purpose = purpose;
+			}
+
+			public readonly string Host;
+			public readonly string Purpose;
+			public readonly long Started = System.Diagnostics.Stopwatch.GetTimestamp();
+			public int? Status;
+			public long? Bytes;
+			public int Hops;
+			public string Outcome = Interactions.Ok;
+			public string Reason;
+
+			public void Refused(string reason) => (Outcome, Reason) = (Interactions.Refused, reason);
+
+			public void Failed(string reason) => (Outcome, Reason) = (Interactions.Failed, reason);
+
+			public void Answered(HttpResponseMessage response) => Status = (int)response.StatusCode;
+		}
+
+		void Record(Exchange exchange)
+		{
+			if (_ledger == default)
+				return;
+			var trigger = HttpScope.Trigger;
+			if (trigger == HttpScope.Request)
+			{
+				_ledger.Count(exchange.Host, $"http:{exchange.Purpose}:{exchange.Outcome}", exchange.Bytes ?? 0);
+				return;
+			}
+			_ledger.Record(new InteractionEvent
+			{
+				Channel = Interactions.Http,
+				Host = exchange.Host,
+				Purpose = exchange.Purpose,
+				Trigger = trigger,
+				Outcome = exchange.Outcome,
+				Reason = exchange.Reason,
+				Status = exchange.Status,
+				LatencyMs = (int)System.Diagnostics.Stopwatch.GetElapsedTime(exchange.Started).TotalMilliseconds,
+				Bytes = exchange.Bytes,
+				Redirects = exchange.Hops,
+				Crawl = HttpScope.Crawling
+			});
+		}
+
+		static string StatusReason(HttpResponseMessage response) => ((int)response.StatusCode).ToString(System.Globalization.CultureInfo.InvariantCulture);
+
 		public bool IsAllowed(Uri target)
 		{
 			if (target is not { IsAbsoluteUri: true } || !string.IsNullOrEmpty(target.UserInfo))
@@ -79,12 +135,31 @@ namespace PrivaPub.Infrastructure.Http
 		}
 
 		public async Task<FetchedJson> GetJson(string url, string accept, Action<HttpRequestMessage> sign, CancellationToken token)
+		{
+			var exchange = new Exchange(url, HttpScope.Purpose ?? "object");
+			try
+			{
+				return await GetJson(url, accept, sign, exchange, token);
+			}
+			finally
+			{
+				Record(exchange);
+			}
+		}
+
+		async Task<FetchedJson> GetJson(string url, string accept, Action<HttpRequestMessage> sign, Exchange exchange, CancellationToken token)
 		{
 			if (!Uri.TryCreate(url, UriKind.Absolute, out var target) || !IsAllowed(target))
+			{
+				exchange.Refused("disallowed");
 				return default;
+			}
 			var negativeKey = NegativeKey(target);
 			if (_cache.TryGetValue(negativeKey, out _))
+			{
+				exchange.Refused("remembered");
 				return default;
+			}
 
 			using var timeout = CancellationTokenSource.CreateLinkedTokenSource(token);
 			timeout.CancelAfter(RequestTimeout);
@@ -98,46 +173,49 @@ namespace PrivaPub.Infrastructure.Http
 
 					using var response = await _httpClientFactory.CreateClient(ClientName)
 						.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, timeout.Token);
+					exchange.Answered(response);
 
 					if (IsRedirect(response.StatusCode))
 					{
 						var location = response.Headers.Location;
 						var next = location == default ? default : location.IsAbsoluteUri ? location : new Uri(target, location);
 						if (!IsAllowed(next))
-							return Refuse(negativeKey, url, "a redirect to a disallowed location");
+							return Refuse(negativeKey, url, "a redirect to a disallowed location", exchange, "bad-redirect");
 						target = next;
+						exchange.Hops++;
 						continue;
 					}
 
 					if (!response.IsSuccessStatusCode)
-						return Refuse(negativeKey, url, $"status {(int)response.StatusCode}",
+						return Refuse(negativeKey, url, $"status {(int)response.StatusCode}", exchange, StatusReason(response),
 							transient: (int)response.StatusCode is >= 500 or 429 or 408);
 
 					var mediaType = response.Content.Headers.ContentType?.MediaType;
 					if (mediaType == default || !JsonMediaTypes.Contains(mediaType, StringComparer.OrdinalIgnoreCase))
-						return Refuse(negativeKey, url, $"content type '{mediaType}'");
+						return Refuse(negativeKey, url, $"content type '{mediaType}'", exchange, "content-type");
 					if (response.Content.Headers.ContentLength > MaxResponseBytes)
-						return Refuse(negativeKey, url, "a body over the size limit");
+						return Refuse(negativeKey, url, "a body over the size limit", exchange, "too-large");
 
 					var body = await ReadBounded(response.Content, MaxResponseBytes, timeout.Token);
 					if (body == default)
-						return Refuse(negativeKey, url, "a body over the size limit");
+						return Refuse(negativeKey, url, "a body over the size limit", exchange, "too-large");
+					exchange.Bytes = body.Length;
 
 					return new FetchedJson { FinalUri = target, Document = JsonDocument.Parse(body) };
 				}
-				return Refuse(negativeKey, url, "too many redirects");
+				return Refuse(negativeKey, url, "too many redirects", exchange, "too-many-redirects");
 			}
 			catch (OperationCanceledException) when (!token.IsCancellationRequested)
 			{
-				return Refuse(negativeKey, url, "a timeout", transient: true);
+				return Refuse(negativeKey, url, "a timeout", exchange, "timeout", transient: true);
 			}
 			catch (HttpRequestException ex)
 			{
-				return Refuse(negativeKey, url, ex.Message, transient: true);
+				return Refuse(negativeKey, url, ex.Message, exchange, "network", transient: true);
 			}
 			catch (Exception ex) when (ex is JsonException or BlockedDestinationException)
 			{
-				return Refuse(negativeKey, url, ex.Message);
+				return Refuse(negativeKey, url, ex.Message, exchange, ex is JsonException ? "bad-json" : "private-address");
 			}
 		}
 
@@ -145,9 +223,28 @@ namespace PrivaPub.Infrastructure.Http
 			Uri.TryCreate(url, UriKind.Absolute, out var target) && _cache.TryGetValue(NegativeKey(target), out bool transient) && transient;
 
 		public async Task<(byte[] Bytes, string ContentType)> GetMedia(string url, long maxBytes, CancellationToken token)
+		{
+			var exchange = new Exchange(url, "media");
+			try
+			{
+				var media = await GetMedia(url, maxBytes, exchange, token);
+				if (media.Bytes == default && exchange.Outcome == Interactions.Ok)
+					exchange.Refused("unusable");
+				return media;
+			}
+			finally
+			{
+				Record(exchange);
+			}
+		}
+
+		async Task<(byte[] Bytes, string ContentType)> GetMedia(string url, long maxBytes, Exchange exchange, CancellationToken token)
 		{
 			if (!Uri.TryCreate(url, UriKind.Absolute, out var target) || !IsAllowed(target))
+			{
+				exchange.Refused("disallowed");
 				return default;
+			}
 			using var timeout = CancellationTokenSource.CreateLinkedTokenSource(token);
 			timeout.CancelAfter(TimeSpan.FromSeconds(60));
 			try
@@ -157,27 +254,52 @@ namespace PrivaPub.Infrastructure.Http
 					using var request = new HttpRequestMessage(HttpMethod.Get, target);
 					request.Headers.Accept.ParseAdd("image/*, video/*, audio/*");
 					using var response = await _httpClientFactory.CreateClient(ClientName).SendAsync(request, HttpCompletionOption.ResponseHeadersRead, timeout.Token);
+					exchange.Answered(response);
 					if (IsRedirect(response.StatusCode))
 					{
 						var location = response.Headers.Location;
 						var next = location == default ? default : location.IsAbsoluteUri ? location : new Uri(target, location);
 						if (!IsAllowed(next))
+						{
+							exchange.Refused("bad-redirect");
 							return default;
+						}
 						target = next;
+						exchange.Hops++;
 						continue;
 					}
 					var mediaType = response.Content.Headers.ContentType?.MediaType?.ToLowerInvariant();
-					if (!response.IsSuccessStatusCode || mediaType == default
-						|| !(mediaType.StartsWith("image/") || mediaType.StartsWith("video/") || mediaType.StartsWith("audio/"))
-						|| mediaType.Contains("svg") || response.Content.Headers.ContentLength > maxBytes)
+					if (!response.IsSuccessStatusCode)
+					{
+						exchange.Failed(StatusReason(response));
 						return default;
+					}
+					if (mediaType == default || !(mediaType.StartsWith("image/") || mediaType.StartsWith("video/") || mediaType.StartsWith("audio/"))
+						|| mediaType.Contains("svg"))
+					{
+						exchange.Refused("content-type");
+						return default;
+					}
+					if (response.Content.Headers.ContentLength > maxBytes)
+					{
+						exchange.Refused("too-large");
+						return default;
+					}
 					var bytes = await ReadBounded(response.Content, (int)Math.Min(maxBytes, int.MaxValue), timeout.Token);
-					return bytes == default ? default : (bytes, mediaType);
+					if (bytes == default)
+					{
+						exchange.Refused("too-large");
+						return default;
+					}
+					exchange.Bytes = bytes.Length;
+					return (bytes, mediaType);
 				}
+				exchange.Refused("too-many-redirects");
 				return default;
 			}
 			catch (Exception ex) when (ex is HttpRequestException or BlockedDestinationException or OperationCanceledException && !token.IsCancellationRequested)
 			{
+				exchange.Failed(ex switch { BlockedDestinationException => "private-address", OperationCanceledException => "timeout", _ => "network" });
 				_logger.LogInformation("Media {Url} refused: {Reason}", url, ex.Message);
 				return default;
 			}
@@ -231,9 +353,29 @@ namespace PrivaPub.Infrastructure.Http
 		}
 
 		public async Task<HttpResponseMessage> OpenMedia(string url, System.Net.Http.Headers.RangeHeaderValue range, CancellationToken token)
+		{
+			var exchange = new Exchange(url, "stream");
+			try
+			{
+				var response = await OpenMedia(url, range, exchange, token);
+				if (response == default && exchange.Outcome == Interactions.Ok)
+					exchange.Refused("unusable");
+				exchange.Bytes = response?.Content.Headers.ContentLength;
+				return response;
+			}
+			finally
+			{
+				Record(exchange);
+			}
+		}
+
+		async Task<HttpResponseMessage> OpenMedia(string url, System.Net.Http.Headers.RangeHeaderValue range, Exchange exchange, CancellationToken token)
 		{
 			if (!Uri.TryCreate(url, UriKind.Absolute, out var target) || !IsAllowed(target))
+			{
+				exchange.Refused("disallowed");
 				return default;
+			}
 			try
 			{
 				for (var hop = 0; hop <= MaxRedirects; hop++)
@@ -242,20 +384,29 @@ namespace PrivaPub.Infrastructure.Http
 					request.Headers.Accept.ParseAdd("video/*, audio/*, image/*");
 					request.Headers.Range = range;
 					var response = await _httpClientFactory.CreateClient(ClientName).SendAsync(request, HttpCompletionOption.ResponseHeadersRead, token);
+					exchange.Answered(response);
 					if (IsRedirect(response.StatusCode))
 					{
 						var location = response.Headers.Location;
 						response.Dispose();
 						var next = location == default ? default : location.IsAbsoluteUri ? location : new Uri(target, location);
 						if (!IsAllowed(next))
+						{
+							exchange.Refused("bad-redirect");
 							return default;
+						}
 						target = next;
+						exchange.Hops++;
 						continue;
 					}
 					var mediaType = response.Content.Headers.ContentType?.MediaType?.ToLowerInvariant();
 					if (!response.IsSuccessStatusCode || mediaType == default || mediaType.Contains("svg")
 						|| !(mediaType.StartsWith("video/") || mediaType.StartsWith("audio/") || mediaType.StartsWith("image/") || mediaType == "application/octet-stream"))
 					{
+						if (response.IsSuccessStatusCode)
+							exchange.Refused("content-type");
+						else
+							exchange.Failed(StatusReason(response));
 						response.Dispose();
 						return default;
 					}
@@ -265,6 +416,7 @@ namespace PrivaPub.Infrastructure.Http
 			}
 			catch (Exception ex) when (ex is HttpRequestException or BlockedDestinationException or OperationCanceledException && !token.IsCancellationRequested)
 			{
+				exchange.Failed(ex switch { BlockedDestinationException => "private-address", OperationCanceledException => "timeout", _ => "network" });
 				_logger.LogInformation("Media stream {Url} refused: {Reason}", url, ex.Message);
 				return default;
 			}
@@ -311,8 +463,12 @@ namespace PrivaPub.Infrastructure.Http
 
 		static string NegativeKey(Uri target) => "federation-http:refused:" + target.AbsoluteUri;
 
-		FetchedJson Refuse(string negativeKey, string url, string reason, bool transient = false)
+		FetchedJson Refuse(string negativeKey, string url, string reason, Exchange exchange, string code, bool transient = false)
 		{
+			if (transient)
+				exchange.Failed(code);
+			else
+				exchange.Refused(code);
 			_cache.Set(negativeKey, transient, NegativeCacheLifetime);
 			_logger.LogInformation("GET {Url} refused: {Reason}", url, reason);
 			return default;