A long job keeps its lease, and runs once

A lease lasted two minutes and was never renewed, so the reaper gave any longer job to a second worker while the first
still ran it, and both finished it. The worker now renews the lease every third of its length while the handler runs;
a lease found taken (reaped and leased again) cancels the handler. Each lease carries its own owner stamp, since every
worker of a process shares one name, and Finish only counts for the lease it was given. Media processing and persona
archives will run longer than two minutes. JobQueueTests: a job three times its lease runs once with the reaper finding
nothing, and a stolen lease stops its handler and drops its outcome.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01LsXgEaXee4GCU1hwYgPJXw

PrivaPub/Infrastructure/Jobs/JobQueue.cs

@@ -1,3 +1,4 @@
+using MongoDB.Bson;
 using MongoDB.Driver;
 using MongoDB.Entities;
 
@@ -29,8 +30,13 @@ namespace PrivaPub.Infrastructure.Jobs
 		Task<bool> Enqueue(JobKind kind, string payload, string host, string dedupeKey, CancellationToken token);
 		Task<string> Payload(string dedupeKey, CancellationToken token);
 		Task<int> EnqueueMany(IEnumerable<Job> jobs, CancellationToken token);
+		/// <summary>How long a lease lasts unless it is renewed.</summary>
+		TimeSpan LeaseFor { get; }
 		Task<Job> Lease(JobKind kind, IReadOnlyCollection<string> busyHosts, CancellationToken token);
-		Task Finish(Job job, JobOutcome outcome, int maxAttempts, CancellationToken token);
+		/// <summary>Extends a running job's lease; false once it is no longer this lease's (reaped and leased again).</summary>
+		Task<bool> Renew(Job job, CancellationToken token);
+		/// <summary>Records the outcome; false when the lease was lost meanwhile, and then nothing changes.</summary>
+		Task<bool> Finish(Job job, JobOutcome outcome, int maxAttempts, CancellationToken token);
 		Task<long> Reap(CancellationToken token);
 		Task WaitForWork(JobKind kind, TimeSpan poll, CancellationToken token);
 	}
@@ -39,6 +45,8 @@ namespace PrivaPub.Infrastructure.Jobs
 	{
 		public static readonly TimeSpan LeaseTime = TimeSpan.FromMinutes(2);
 
+		// every lease is stamped with its own owner (the process and a fresh id): two workers of one process, the second
+		// leasing a job the reaper took back from the first, must not renew nor finish each other's lease
 		readonly string _owner = $"{Environment.MachineName}:{Environment.ProcessId}";
 		readonly ConcurrentDictionary<JobKind, SemaphoreSlim> _signals = new();
 		readonly FilterDefinition<Job> _scope = Builders<Job>.Filter.Empty;
@@ -49,6 +57,15 @@ namespace PrivaPub.Infrastructure.Jobs
 
 		public JobQueue(Expression<Func<Job, bool>> scope) => _scope = Builders<Job>.Filter.Where(scope);
 
+		public JobQueue(TimeSpan leaseFor, Expression<Func<Job, bool>> scope = default)
+		{
+			LeaseFor = leaseFor;
+			if (scope != default)
+				_scope = Builders<Job>.Filter.Where(scope);
+		}
+
+		public TimeSpan LeaseFor { get; } = LeaseTime;
+
 		public async Task<bool> Enqueue(JobKind kind, string payload, string host, string dedupeKey, CancellationToken token) =>
 			await EnqueueMany(new[] { new Job { Kind = kind, Payload = payload, Host = host, DedupeKey = dedupeKey } }, token) == 1;
 
@@ -81,17 +98,26 @@ namespace PrivaPub.Infrastructure.Jobs
 			return await DB.Default.UpdateAndGet<Job>()
 				.Match(f => f.Where(j => j.Kind == kind && j.State == JobState.Pending && j.RunAt <= now && !busy.Contains(j.Host)) & _scope)
 				.Modify(j => j.State, JobState.Running)
-				.Modify(j => j.LeasedUntil, now + LeaseTime)
-				.Modify(j => j.LeaseOwner, _owner)
+				.Modify(j => j.LeasedUntil, now + LeaseFor)
+				.Modify(j => j.LeaseOwner, $"{_owner}/{ObjectId.GenerateNewId()}")
 				.Modify(b => b.Inc(j => j.Attempts, 1))
 				.Option(o => o.Sort = Builders<Job>.Sort.Ascending(j => j.RunAt))
 				.ExecuteAsync(token);
 		}
 
-		public async Task Finish(Job job, JobOutcome outcome, int maxAttempts, CancellationToken token)
+		public async Task<bool> Renew(Job job, CancellationToken token)
+		{
+			var result = await DB.Default.Update<Job>()
+				.Match(j => j.ID == job.ID && j.State == JobState.Running && j.LeaseOwner == job.LeaseOwner)
+				.Modify(j => j.LeasedUntil, DateTime.UtcNow + LeaseFor)
+				.ExecuteAsync(token);
+			return result.MatchedCount == 1;
+		}
+
+		public async Task<bool> Finish(Job job, JobOutcome outcome, int maxAttempts, CancellationToken token)
 		{
 			var now = DateTime.UtcNow;
-			var update = DB.Default.Update<Job>().MatchID(job.ID)
+			var update = DB.Default.Update<Job>().Match(j => j.ID == job.ID && j.LeaseOwner == job.LeaseOwner)
 				.Modify(j => j.LeasedUntil, null)
 				.Modify(j => j.LeaseOwner, null)
 				.Modify(j => j.LastError, outcome.Error);
@@ -112,7 +138,7 @@ namespace PrivaPub.Infrastructure.Jobs
 					update.Modify(j => j.State, JobState.Dead).Modify(j => j.FinishedAt, now);
 					break;
 			}
-			await update.ExecuteAsync(token);
+			return (await update.ExecuteAsync(token)).MatchedCount == 1;
 		}
 
 		public async Task<long> Reap(CancellationToken token)