A long job keeps its lease, and runs once

A lease lasted two minutes and was never renewed, so the reaper gave any longer job to a second worker while the first
still ran it, and both finished it. The worker now renews the lease every third of its length while the handler runs;
a lease found taken (reaped and leased again) cancels the handler. Each lease carries its own owner stamp, since every
worker of a process shares one name, and Finish only counts for the lease it was given. Media processing and persona
archives will run longer than two minutes. JobQueueTests: a job three times its lease runs once with the reaper finding
nothing, and a stolen lease stops its handler and drops its outcome.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01LsXgEaXee4GCU1hwYgPJXw

PrivaPub/Infrastructure/Jobs/JobWorker.cs

@@ -69,18 +69,27 @@ namespace PrivaPub.Infrastructure.Jobs
 
 				var host = job.Host ?? string.Empty;
 				inFlight.AddOrUpdate(host, 1, (_, count) => count + 1);
+				// the lease is renewed while the handler runs; once it is lost (another worker leased the job again) the
+				// handler is cancelled and its outcome dropped
+				using var running = CancellationTokenSource.CreateLinkedTokenSource(stoppingToken);
+				var renewing = KeepLease(job, running);
 				try
 				{
 					JobOutcome outcome;
 					try
 					{
 						using var scope = HttpScope.Triggered(handler.Kind.ToString().ToLowerInvariant());
-						outcome = await handler.Handle(job, stoppingToken);
+						outcome = await handler.Handle(job, running.Token);
 					}
 					catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
 					{
 						return;
 					}
+					catch (OperationCanceledException) when (running.IsCancellationRequested)
+					{
+						_logger.LogWarning("{Kind} job {Id} lost its lease and was stopped", handler.Kind, job.ID);
+						continue;
+					}
 					catch (Exception ex)
 					{
 						_logger.LogError(ex, "{Kind} job {Id} threw", handler.Kind, job.ID);
@@ -90,7 +99,8 @@ namespace PrivaPub.Infrastructure.Jobs
 					if (outcome.Result != JobResult.Done && job.Attempts >= handler.MaxAttempts && outcome.Result == JobResult.Retry)
 						_logger.LogWarning("{Kind} job {Id} for {Host} is dead after {Attempts} attempts: {Error}",
 							handler.Kind, job.ID, job.Host, job.Attempts, outcome.Error);
-					await _queue.Finish(job, outcome, handler.MaxAttempts, CancellationToken.None);
+					if (!await _queue.Finish(job, outcome, handler.MaxAttempts, CancellationToken.None))
+						_logger.LogWarning("{Kind} job {Id} lost its lease before it finished; its outcome is dropped", handler.Kind, job.ID);
 				}
 				catch (Exception ex)
 				{
@@ -98,11 +108,43 @@ namespace PrivaPub.Infrastructure.Jobs
 				}
 				finally
 				{
+					await running.CancelAsync();
+					await renewing;
 					inFlight.AddOrUpdate(host, 0, (_, count) => count - 1);
 				}
 			}
 		}
 
+		// renews the lease every third of its length until the handler ends; a lease found lost cancels the handler
+		async Task KeepLease(Job job, CancellationTokenSource running)
+		{
+			var every = _queue.LeaseFor / 3;
+			while (!running.IsCancellationRequested)
+			{
+				try
+				{
+					await Task.Delay(every, running.Token);
+				}
+				catch (OperationCanceledException)
+				{
+					return;
+				}
+
+				try
+				{
+					if (await _queue.Renew(job, CancellationToken.None))
+						continue;
+					await running.CancelAsync();
+					return;
+				}
+				catch (Exception ex)
+				{
+					// a passing database error: the lease has two more thirds to go, so try again at the next turn
+					_logger.LogWarning(ex, "{Worker} could not renew the lease of job {Id}", nameof(JobWorker), job.ID);
+				}
+			}
+		}
+
 		async Task Reap(CancellationToken stoppingToken)
 		{
 			while (!stoppingToken.IsCancellationRequested)