Statistics counters survive two flushes at once

Two ledger flushes that upsert the same new day (or touch the same new
instance) at once made one of them fail on the unique key, losing its
counts; MongoDB retries that only for single-document updates. The loser
now tries again and adds to the document the winner made. Seen as a rare
failure of LedgerOverHttpTests in full runs.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01LsXgEaXee4GCU1hwYgPJXw

PrivaPub/Infrastructure/Statistics/InteractionLedger.cs

@@ -203,13 +203,13 @@ namespace PrivaPub.Infrastructure.Statistics
 				if (host == Interactions.Unknown || _touchedAt.TryGetValue(host, out var at) && now - at < TouchInterval)
 					continue;
 				_touchedAt[host] = now;
-				await DB.Default.Update<RemoteInstance>()
+				await Upsert(() => DB.Default.Update<RemoteInstance>()
 					.Match(i => i.Host == host)
 					.Modify(i => i.Seen, "touched")
 					.Modify(i => i.LastSeenAt, now)
 					.Modify(b => b.SetOnInsert(i => i.FirstSeenAt, now))
 					.Option(o => o.IsUpsert = true)
-					.ExecuteAsync(token);
+					.ExecuteAsync(token));
 				_knownHosts.Add(host);
 				if (_queue != default && _blocks?.IsSuspended(host) != true)
 					await _queue.Enqueue(JobKind.DescribeInstance, host, host, Federation.Objects.InstanceDescriber.DedupeKey(host, now), token);
@@ -270,17 +270,31 @@ namespace PrivaPub.Infrastructure.Statistics
 			return _knownHosts;
 		}
 
+		// two flushes upserting a new day's document at once: one inserts it and the other's insert fails on the unique
+		// key; the second try finds the document and adds to it (MongoDB retries this only for single-document updates)
+		static async Task Upsert(Func<Task> write)
+		{
+			try
+			{
+				await write();
+			}
+			catch (MongoWriteException ex) when (ex.WriteError?.Category == ServerErrorCategory.DuplicateKey)
+			{
+				await write();
+			}
+		}
+
 		async Task FlushCounters(CancellationToken token)
 		{
 			foreach (var key in _reads.Keys.ToList())
 			{
 				if (!_reads.TryRemove(key, out var count) || count == 0)
 					continue;
-				await DB.Default.Update<InstanceDay>()
+				await Upsert(() => DB.Default.Update<InstanceDay>()
 					.Match(d => d.Day == key.Day && d.Host == key.Host)
 					.Modify(b => b.Inc($"{nameof(InstanceDay.Reads)}.{key.Key}", count))
 					.Option(o => o.IsUpsert = true)
-					.ExecuteAsync(token);
+					.ExecuteAsync(token));
 			}
 
 			var day = DateTime.UtcNow.Date;
@@ -291,22 +305,22 @@ namespace PrivaPub.Infrastructure.Statistics
 			var server = _server.Keys.ToList();
 			if (server.Count == 0 && written == 0 && failed == 0 && newlyDropped == 0)
 				return;
-			var update = DB.Default.Update<ServerDay>()
+			await Upsert(() => DB.Default.Update<ServerDay>()
 				.Match(d => d.Day == day)
 				.Modify(b => b.Inc(d => d.LedgerWritten, written))
 				.Modify(b => b.Inc(d => d.LedgerFailed, failed))
 				.Modify(b => b.Inc(d => d.LedgerDropped, newlyDropped))
-				.Option(o => o.IsUpsert = true);
-			await update.ExecuteAsync(token);
+				.Option(o => o.IsUpsert = true)
+				.ExecuteAsync(token));
 			foreach (var key in server)
 			{
 				if (!_server.TryRemove(key, out var count) || count == 0)
 					continue;
-				await DB.Default.Update<ServerDay>()
+				await Upsert(() => DB.Default.Update<ServerDay>()
 					.Match(d => d.Day == key.Day)
 					.Modify(b => b.Inc($"{key.Field}.{key.Key}", count))
 					.Option(o => o.IsUpsert = true)
-					.ExecuteAsync(token);
+					.ExecuteAsync(token));
 			}
 		}