P7: thread backfill reads conversation containers and asks with If-None-Match

The FetchReplies job reads a thread's FEP-171b contextHistory before its FEP-7888 context: the posts each Add (or
Forte's plain Create) brought in, read from their own servers. A thread collection read whole keeps its ETag when the
document itself changes with every post (it counts them, or holds them all with no further page); the next read sends
it as If-None-Match and a 304 ends the job. The ETag is kept as sent, since NodeBB's has no quotes and the typed header
drops it. A context naming a post we hold (Forte's first post) is not fetched.

Checked in the pasture: Mastodon 4.7.3 answers the second read 304; NodeBB 4.16's unquoted ETag is kept (23/23 in its
scenario); a Forte thread completes through its replies.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012CzABvBkbcFqoHdmi8b9WB

PrivaPub/Infrastructure/Data/Indexes.cs

@@ -15,6 +15,7 @@ namespace PrivaPub.Infrastructure.Data
 	public static class Indexes
 	{
 		public static readonly TimeSpan TombstoneLifetime = TimeSpan.FromDays(90);
+		public static readonly TimeSpan ThreadCollectionLifetime = TimeSpan.FromDays(30);
 
 		public static async Task Create(CancellationToken token = default)
 		{
@@ -166,6 +167,8 @@ namespace PrivaPub.Infrastructure.Data
 			await Plain<ObjectRecord>(token, r => r.PostId);
 			await Unique<DeletedObject>(d => d.ObjectURI, Builders<DeletedObject>.Filter.Gt(d => d.ObjectURI, ""), token);
 			await DB.Default.Index<DeletedObject>().Key(d => d.DeletedAt, KeyType.Ascending).Option(o => o.ExpireAfter = TombstoneLifetime).CreateAsync(token);
+			await Unique<ThreadCollection>(c => c.URI, Builders<ThreadCollection>.Filter.Gt(c => c.URI, ""), token);
+			await DB.Default.Index<ThreadCollection>().Key(c => c.ReadAt, KeyType.Ascending).Option(o => o.ExpireAfter = ThreadCollectionLifetime).CreateAsync(token);
 			await DB.Default.Index<Downvote>().Key(d => d.PostId, KeyType.Ascending).Key(d => d.ActorURI, KeyType.Ascending).Option(o => o.Unique = true).CreateAsync(token);
 			await Plain<Downvote>(token, d => d.ActivityURI);
 			await DB.Default.Index<PollVote>().Key(v => v.PostId, KeyType.Ascending).Key(v => v.ActorURI, KeyType.Ascending).Key(v => v.Choice, KeyType.Ascending)

PrivaPub/Infrastructure/Http/FederationHttp.cs

@@ -15,6 +15,9 @@ namespace PrivaPub.Infrastructure.Http
 		public Uri FinalUri { get; init; }
 		public JsonDocument Document { get; init; }
 		public JsonElement Root => Document.RootElement;
+		public string ETag { get; init; }
+		// the origin answered 304 to an If-None-Match: no document, the one read before still stands
+		public bool NotModified { get; init; }
 
 		public void Dispose() => Document?.Dispose();
 	}
@@ -24,6 +27,9 @@ namespace PrivaPub.Infrastructure.Http
 		bool IsAllowed(Uri target);
 		Task<FetchedJson> GetJson(string url, string accept, Action<HttpRequestMessage> sign, CancellationToken token);
 		Task<(int Status, FetchedJson Json)> GetJsonStatus(string url, string accept, Action<HttpRequestMessage> sign, CancellationToken token);
+		/// <summary>The document, or one that is <see cref="FetchedJson.NotModified"/> when its origin still has the version
+		/// <paramref name="etag"/> names.</summary>
+		Task<FetchedJson> GetJsonIfChanged(string url, string accept, string etag, Action<HttpRequestMessage> sign, CancellationToken token);
 		bool FailedTemporarily(string url);
 		Task<(Uri FinalUri, string Html)> GetPage(string url, CancellationToken token);
 		Task<HttpResponseMessage> OpenMedia(string url, System.Net.Http.Headers.RangeHeaderValue range, CancellationToken token);
@@ -291,7 +297,20 @@ namespace PrivaPub.Infrastructure.Http
 			var exchange = new Exchange(url, HttpScope.Purpose ?? "object");
 			try
 			{
-				return await GetJson(url, accept, sign, exchange, token);
+				return await GetJson(url, accept, sign, default, exchange, token);
+			}
+			finally
+			{
+				Record(exchange);
+			}
+		}
+
+		public async Task<FetchedJson> GetJsonIfChanged(string url, string accept, string etag, Action<HttpRequestMessage> sign, CancellationToken token)
+		{
+			var exchange = new Exchange(url, HttpScope.Purpose ?? "object");
+			try
+			{
+				return await GetJson(url, accept, sign, etag, exchange, token);
 			}
 			finally
 			{
@@ -308,7 +327,7 @@ namespace PrivaPub.Infrastructure.Http
 			var exchange = new Exchange(url, HttpScope.Purpose ?? "object");
 			try
 			{
-				var json = await GetJson(url, accept, sign, exchange, token);
+				var json = await GetJson(url, accept, sign, default, exchange, token);
 				return (exchange.Status ?? 0, json);
 			}
 			finally
@@ -317,7 +336,7 @@ namespace PrivaPub.Infrastructure.Http
 			}
 		}
 
-		async Task<FetchedJson> GetJson(string url, string accept, Action<HttpRequestMessage> sign, Exchange exchange, CancellationToken token)
+		async Task<FetchedJson> GetJson(string url, string accept, Action<HttpRequestMessage> sign, string etag, Exchange exchange, CancellationToken token)
 		{
 			if (!Uri.TryCreate(url, UriKind.Absolute, out var target) || !IsAllowed(target))
 			{
@@ -338,11 +357,15 @@ namespace PrivaPub.Infrastructure.Http
 				for (var hop = 0; hop <= MaxRedirects; hop++)
 				{
 					using var request = Request(target, accept);
+					if (!string.IsNullOrEmpty(etag))
+						request.Headers.TryAddWithoutValidation("If-None-Match", etag);
 					sign?.Invoke(request);
 
 					using var response = await _httpClientFactory.CreateClient(ClientName)
 						.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, timeout.Token);
 					exchange.Answered(response);
+					if (response.StatusCode == HttpStatusCode.NotModified && !string.IsNullOrEmpty(etag))
+						return new FetchedJson { FinalUri = target, ETag = etag, NotModified = true };
 
 					if (IsRedirect(response.StatusCode))
 					{
@@ -370,7 +393,7 @@ namespace PrivaPub.Infrastructure.Http
 						return Refuse(negativeKey, url, "a body over the size limit", exchange, "too-large");
 					exchange.Bytes = body.Length;
 
-					return new FetchedJson { FinalUri = target, Document = JsonDocument.Parse(body) };
+					return new FetchedJson { FinalUri = target, Document = JsonDocument.Parse(body), ETag = RawETag(response) };
 				}
 				return Refuse(negativeKey, url, "too many redirects", exchange, "too-many-redirects");
 			}
@@ -658,6 +681,10 @@ namespace PrivaPub.Infrastructure.Http
 			return buffer.ToArray();
 		}
 
+		// as sent, since it is only ever sent back: NodeBB's carries no quotes, which the typed header rejects
+		static string RawETag(HttpResponseMessage response) =>
+			response.Headers.TryGetValues("ETag", out var values) ? values.FirstOrDefault() : default;
+
 		static bool IsRedirect(HttpStatusCode status) =>
 			status is HttpStatusCode.MovedPermanently or HttpStatusCode.Found or HttpStatusCode.SeeOther
 				or HttpStatusCode.TemporaryRedirect or HttpStatusCode.PermanentRedirect;