M4: every delivery attempt is recorded

DeliveryJobHandler records each attempt as an 'out' event: the activity, object type and
audience read from the body (ActivityShape, shared with the inbox side), the receiving
server, the status, the time the POST took, the wait since it was queued, the bytes, the
attempt number and the signer's kind (none for circles or private traffic). The outcome is
ok, deferred (429 or 503 with Retry-After, or an open breaker: host-unavailable), retry
(5xx, 408, timeout, network), or dead: another 4xx, private-address, not-deliverable,
signer-gone, or a retry at the last attempt. Until now none of this outlived the job's
seven-day TTL, and the last error was overwritten on success.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01ELjqpznMFMNrJoJUj6K5p2

PrivaPub/Federation/Outbox/DeliveryService.cs

@@ -11,8 +11,12 @@ using PrivaPub.Federation.Actors;
 using PrivaPub.Federation.Signing;
 using PrivaPub.Infrastructure.Http;
 using PrivaPub.Infrastructure.Jobs;
+using PrivaPub.Infrastructure.Statistics;
+using PrivaPub.Federation.Objects;
+using PrivaPub.Models.Statistics;
 using PrivaPub.Models.Jobs;
 
+using System.Diagnostics;
 using System.Text.Json;
 
 namespace PrivaPub.Federation.Outbox
@@ -81,13 +85,25 @@ namespace PrivaPub.Federation.Outbox
 		readonly IFederationHttp _http;
 		readonly IHostCircuitBreaker _breaker;
 		readonly ILogger<DeliveryJobHandler> _logger;
+		readonly IInteractionLedger _ledger;
 
-		public DeliveryJobHandler(ILocalActorService actors, IFederationHttp http, IHostCircuitBreaker breaker, ILogger<DeliveryJobHandler> logger)
+		public DeliveryJobHandler(ILocalActorService actors, IFederationHttp http, IHostCircuitBreaker breaker, ILogger<DeliveryJobHandler> logger,
+			IInteractionLedger ledger = default)
 		{
 			_actors = actors;
 			_http = http;
 			_breaker = breaker;
 			_logger = logger;
+			_ledger = ledger;
+		}
+
+		sealed class Attempt
+		{
+			public DeliveryPayload Payload;
+			public LocalActor Signer;
+			public int? Status;
+			public string Reason;
+			public int? LatencyMs;
 		}
 
 		public JobKind Kind => JobKind.Deliver;
@@ -96,28 +112,97 @@ namespace PrivaPub.Federation.Outbox
 		public int PerHostLimit => 2;
 
 		public async Task<JobOutcome> Handle(Job job, CancellationToken token)
+		{
+			var attempt = new Attempt();
+			JobOutcome outcome = default;
+			try
+			{
+				outcome = await Deliver(job, attempt, token);
+				return outcome;
+			}
+			finally
+			{
+				Record(job, attempt, outcome);
+			}
+		}
+
+		void Record(Job job, Attempt attempt, JobOutcome outcome)
+		{
+			if (_ledger == default)
+				return;
+			var shape = attempt.Payload == default ? default : ActivityShape.Of(attempt.Payload.Body);
+			_ledger.Record(new InteractionEvent
+			{
+				Channel = Interactions.Out,
+				Host = job.Host,
+				Activity = shape?.Type,
+				Object = shape?.ObjectType,
+				Outcome = outcome?.Result switch
+				{
+					JobResult.Done => Interactions.Ok,
+					JobResult.Defer => Interactions.Deferred,
+					JobResult.Retry when job.Attempts >= MaxAttempts => Interactions.Dead,
+					JobResult.Retry => Interactions.Retry,
+					JobResult.Dead => Interactions.Dead,
+					_ => Interactions.Failed
+				},
+				Reason = attempt.Reason,
+				Status = attempt.Status,
+				LatencyMs = attempt.LatencyMs,
+				WaitMs = (int)Math.Min(int.MaxValue, Math.Max(0, (DateTime.UtcNow - job.CreatedAt).TotalMilliseconds)),
+				Bytes = attempt.Payload?.Body == default ? default : Encoding.UTF8.GetByteCount(attempt.Payload.Body),
+				Attempt = job.Attempts,
+				Audience = shape?.Audience,
+				LocalKind = attempt.Signer switch
+				{
+					null => default,
+					{ IsCircle: true } => default,
+					{ Kind: LocalActorKind.Person } => "person",
+					{ Kind: LocalActorKind.Group } => "group",
+					_ => "application"
+				},
+				Signature = "cavage:rsa-sha256"
+			});
+		}
+
+		async Task<JobOutcome> Deliver(Job job, Attempt attempt, CancellationToken token)
 		{
 			var payload = JsonSerializer.Deserialize<DeliveryPayload>(job.Payload);
+			attempt.Payload = payload;
 			if (payload == default || !Uri.TryCreate(payload.Inbox, UriKind.Absolute, out var inbox) || !_http.IsAllowed(inbox))
+			{
+				attempt.Reason = "not-deliverable";
 				return JobOutcome.Dead("not a deliverable inbox");
+			}
 
 			var unavailableUntil = await _breaker.UnavailableUntil(job.Host, token);
 			if (unavailableUntil.HasValue)
+			{
+				attempt.Reason = "host-unavailable";
 				return JobOutcome.Defer(unavailableUntil.Value, "the host is unavailable");
+			}
 
 			var signer = await _actors.FindById(payload.SignerKind, payload.SignerId, token);
+			attempt.Signer = signer;
 			if (signer == default)
+			{
+				attempt.Reason = "signer-gone";
 				return JobOutcome.Dead("the signing actor no longer exists");
+			}
 
 			var body = Encoding.UTF8.GetBytes(payload.Body);
 			using var request = new HttpRequestMessage(HttpMethod.Post, inbox) { Content = new ByteArrayContent(body) };
 			request.Content.Headers.ContentType = MediaTypeHeaderValue.Parse(RemoteActorService.ActivityJson);
 			HttpSignatures.Sign(request, signer, body);
 
+			var started = Stopwatch.GetTimestamp();
 			try
 			{
 				using var response = await _http.Send(request, token);
+				attempt.LatencyMs = (int)Stopwatch.GetElapsedTime(started).TotalMilliseconds;
 				var status = (int)response.StatusCode;
+				attempt.Status = status;
+				attempt.Reason = response.IsSuccessStatusCode ? default : status.ToString(System.Globalization.CultureInfo.InvariantCulture);
 				if (response.IsSuccessStatusCode)
 				{
 					await _breaker.Succeeded(job.Host, token);
@@ -142,10 +227,13 @@ namespace PrivaPub.Federation.Outbox
 			}
 			catch (BlockedDestinationException ex)
 			{
+				attempt.Reason = "private-address";
 				return JobOutcome.Dead(ex.Message);
 			}
 			catch (Exception ex) when (ex is HttpRequestException or TaskCanceledException && !token.IsCancellationRequested)
 			{
+				attempt.LatencyMs = (int)Stopwatch.GetElapsedTime(started).TotalMilliseconds;
+				attempt.Reason = ex is TaskCanceledException ? "timeout" : "network";
 				await _breaker.Failed(job.Host, ex.GetType().Name, token);
 				return JobOutcome.Retry(ex.Message);
 			}