A long job keeps its lease, and runs once

A lease lasted two minutes and was never renewed, so the reaper gave any longer job to a second worker while the first
still ran it, and both finished it. The worker now renews the lease every third of its length while the handler runs;
a lease found taken (reaped and leased again) cancels the handler. Each lease carries its own owner stamp, since every
worker of a process shares one name, and Finish only counts for the lease it was given. Media processing and persona
archives will run longer than two minutes. JobQueueTests: a job three times its lease runs once with the reaper finding
nothing, and a stolen lease stops its handler and drops its outcome.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01LsXgEaXee4GCU1hwYgPJXw

PrivaPub.Tests/Infrastructure/JobQueueTests.cs

@@ -1,3 +1,5 @@
+using Microsoft.Extensions.Logging.Abstractions;
+
 using MongoDB.Entities;
 
 using PrivaPub.Infrastructure.Jobs;
@@ -74,6 +76,8 @@ namespace PrivaPub.Tests.Infrastructure
 			Assert.Equal(JobState.Pending, after.State);
 			Assert.True(after.RunAt > DateTime.UtcNow.AddSeconds(10));
 
+			// leased again for its third attempt (a finish only counts for the lease it was given)
+			await DB.Default.Update<Job>().MatchID(leased.ID).Modify(j => j.State, JobState.Running).Modify(j => j.LeaseOwner, leased.LeaseOwner).ExecuteAsync(token);
 			leased.Attempts = 3;
 			await queue.Finish(leased, JobOutcome.Retry("boom"), maxAttempts: 3, token);
 			Assert.Equal(JobState.Dead, (await DB.Default.Find<Job>().OneAsync(leased.ID, token)).State);
@@ -90,5 +94,100 @@ namespace PrivaPub.Tests.Infrastructure
 
 			Assert.Equal(JobState.Pending, (await DB.Default.Find<Job>().OneAsync(job.ID, token)).State);
 		}
+
+		// a handler running three times its lease keeps it: the reaper finds nothing to take back and it runs once
+		[Fact]
+		public async Task A_long_job_keeps_its_lease_and_runs_once()
+		{
+			var token = TestContext.Current.CancellationToken;
+			var kind = (JobKind)(2000 + Random.Shared.Next(100000));
+			var queue = new JobQueue(TimeSpan.FromSeconds(1.5), j => j.Kind == kind);
+			var handler = new SlowHandler(kind, TimeSpan.FromSeconds(4.5));
+			await queue.Enqueue(kind, "{}", default, default, token);
+
+			using var worker = new JobWorker(queue, new IJobHandler[] { handler }, NullLogger<JobWorker>.Instance);
+			await worker.StartAsync(token);
+			try
+			{
+				var deadline = DateTime.UtcNow.AddSeconds(20);
+				while (!await DB.Default.Find<Job>().Match(j => j.Kind == kind && j.State == JobState.Done).ExecuteAnyAsync(token))
+				{
+					Assert.True(DateTime.UtcNow < deadline, "the job never finished");
+					Assert.Equal(0, await queue.Reap(token));
+					await Task.Delay(250, token);
+				}
+			}
+			finally
+			{
+				await worker.StopAsync(CancellationToken.None);
+			}
+
+			Assert.Equal(1, handler.Started);
+		}
+
+		// a lease taken by another worker (reaped, then leased again) stops the handler, and its outcome is not recorded
+		[Fact]
+		public async Task A_lost_lease_stops_the_handler_and_drops_its_outcome()
+		{
+			var token = TestContext.Current.CancellationToken;
+			var kind = (JobKind)(2000 + Random.Shared.Next(100000));
+			var queue = new JobQueue(TimeSpan.FromSeconds(1.5), j => j.Kind == kind);
+			var handler = new SlowHandler(kind, TimeSpan.FromSeconds(30));
+			await queue.Enqueue(kind, "{}", default, default, token);
+
+			using var worker = new JobWorker(queue, new IJobHandler[] { handler }, NullLogger<JobWorker>.Instance);
+			await worker.StartAsync(token);
+			try
+			{
+				var deadline = DateTime.UtcNow.AddSeconds(20);
+				while (handler.Started == 0)
+				{
+					Assert.True(DateTime.UtcNow < deadline, "the job never started");
+					await Task.Delay(100, token);
+				}
+				await DB.Default.Update<Job>().Match(j => j.Kind == kind).Modify(j => j.LeaseOwner, "another worker").ExecuteAsync(token);
+				while (!handler.Cancelled)
+				{
+					Assert.True(DateTime.UtcNow < deadline, "the handler was never stopped");
+					await Task.Delay(100, token);
+				}
+				await Task.Delay(300, token);
+			}
+			finally
+			{
+				await worker.StopAsync(CancellationToken.None);
+			}
+
+			var job = await DB.Default.Find<Job>().Match(j => j.Kind == kind).ExecuteFirstAsync(token);
+			Assert.Equal(JobState.Running, job.State);
+			Assert.Equal("another worker", job.LeaseOwner);
+		}
+
+		sealed class SlowHandler(JobKind kind, TimeSpan takes) : IJobHandler
+		{
+			int _started;
+
+			public int Started => _started;
+			public bool Cancelled { get; private set; }
+			public JobKind Kind => kind;
+			public int Concurrency => 1;
+			public int MaxAttempts => 3;
+			public int PerHostLimit => 1;
+
+			public async Task<JobOutcome> Handle(Job job, CancellationToken token)
+			{
+				Interlocked.Increment(ref _started);
+				try
+				{
+					await Task.Delay(takes, token);
+				}
+				catch (OperationCanceledException)
+				{
+					Cancelled = true;
+					throw;
+				}
+				return JobOutcome.Done;
+			}
+		}
 	}
 }