An import reads the archive's followers once, finds threads in linear time, and imports a located post once

A post's visibility no longer parses actor.json again for every outbox item; a reply's conversation is its thread's
first post's, found from what the first pass read instead of searching every planned post (quadratic on a large
archive); and a located post always records where it came from, so importing a persona's own archive twice made it
twice.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01LsXgEaXee4GCU1hwYgPJXw

PrivaPub.Tests/Http/PersonaArchiveTests.cs

@@ -157,26 +157,30 @@ namespace PrivaPub.Tests.Http
 			})).Ok().Body.Text("id");
 			var first = await alice.Status("the first words #moving", ("media_ids[]", picture));
 			await alice.Status("and a reply to them", ("in_reply_to_id", first.Text("id")));
+			await _host.Located(alice, "here and nowhere else");
 			var zip = await Exported(alice.Persona);
 
 			var carol = await _host.Mastodon("carol");
 			await follower.Client.Post($"/api/v1/accounts/{carol.Persona.Id}/follow");
 			var actor = $"/peasants/{carol.Persona.UserName}";
 			var jobs = await DB.Default.CountAsync<Job>(j => j.Kind == JobKind.Deliver && j.Payload.Contains(actor), Token);
-			var (_, state) = await Imported(carol.Persona, zip, "posts", "pins", "tags");
+			var (_, state) = await Imported(carol.Persona, zip, "posts", "pins", "tags", "located");
 
 			Assert.Equal("done", state["importState"]!.GetValue<string>());
 			Assert.False(state["importSameActor"]!.GetValue<bool>());
 			Assert.Equal(2, Count(state, "posts"));
 			Assert.Equal(1, Count(state, "media"));
-			var copies = await DB.Default.Find<PostEntity>().Match(p => p.GroupUserId == carol.Persona.Id).ExecuteAsync(Token);
+			Assert.Equal(1, Count(state, "located posts"));
+			var copies = await DB.Default.Find<PostEntity>().Match(p => p.GroupUserId == carol.Persona.Id && p.Visibility != Models.Post.PostVisibility.LocalGeo).ExecuteAsync(Token);
 			Assert.Equal(2, copies.Count);
 			var copy = copies.Single(p => p.ImportedFromURI == first.Text("uri"));
 			Assert.StartsWith($"{PrivaPubHost.Base}/peasants/{carol.Persona.UserName}/scribbles/", copy.ObjectURI);
 			Assert.Equal(["moving"], copy.Tags);
 			Assert.Single(copy.Media);
 			Assert.Equal(carol.Persona.Id, (await DB.Default.Find<MediaAttachment>().MatchID(copy.Media[0].AttachmentId).ExecuteFirstAsync(Token)).OwnerAvatarId);
-			Assert.Equal(copy.ID, copies.Single(p => p.ID != copy.ID).AnsweringToPostId);
+			var reply = copies.Single(p => p.ID != copy.ID);
+			Assert.Equal(copy.ID, reply.AnsweringToPostId);
+			Assert.Equal(copy.ObjectURI + "/context", reply.ContextURI);
 
 			// nobody told: no delivery, no home, no notification; on the profile all the same
 			Assert.Equal(jobs, await DB.Default.CountAsync<Job>(j => j.Kind == JobKind.Deliver && j.Payload.Contains(actor), Token));
@@ -185,9 +189,9 @@ namespace PrivaPub.Tests.Http
 			Assert.Contains((await carol.Client.Get($"/api/v1/accounts/{carol.Persona.Id}/statuses")).Ok().Body!.AsArray(), s => s!.Text("id") == copy.ID);
 
 			// twice changes nothing
-			var (_, again) = await Imported(carol.Persona, zip, "posts");
+			var (_, again) = await Imported(carol.Persona, zip, "posts", "located");
-			Assert.Equal((0, 2), (Count(again, "posts"), Count(again, "posts already here")));
+			Assert.Equal((0, 2, 1), (Count(again, "posts"), Count(again, "posts already here"), Count(again, "located posts already here")));
-			Assert.Equal(2, await DB.Default.CountAsync<PostEntity>(p => p.GroupUserId == carol.Persona.Id, Token));
+			Assert.Equal(3, await DB.Default.CountAsync<PostEntity>(p => p.GroupUserId == carol.Persona.Id, Token));
 		}
 
 		// Mastodon's own archive: its posts, not its boosts nor its direct messages

PrivaPub/Domain/Portability/ImportArchiveJob.cs

@@ -65,15 +65,18 @@ namespace PrivaPub.Domain.Portability
 			DedupeKey = $"import|{avatarId}|{DateTime.UtcNow.Ticks}"
 		};
 
-		sealed class Run(PersonaArchive state, LocalActor me, SafeArchive archive, string archiveActor, bool sameActor, IServiceProvider services)
+		sealed class Run(PersonaArchive state, LocalActor me, SafeArchive archive, string archiveActor, string archiveFollowers, bool sameActor,
+			IServiceProvider services)
 		{
 			public PersonaArchive State { get; } = state;
+			public string ArchiveFollowers { get; } = archiveFollowers;
 			public LocalActor Me { get; } = me;
 			public SafeArchive Archive { get; } = archive;
 			public string ArchiveActor { get; } = archiveActor;
 			public bool SameActor { get; } = sameActor;
 			public IServiceProvider Services { get; } = services;
 			public Dictionary<string, (string Id, string Uri)> Posts { get; } = new(StringComparer.Ordinal);//archive object id → here
+			public Dictionary<string, string> Parents { get; } = new(StringComparer.Ordinal);//archive object id → what it answers
 			public DateTime LastSaved { get; set; }
 
 			public void Count(string what, int by = 1) => State.ImportCounts[what] = State.ImportCounts.GetValueOrDefault(what) + by;
@@ -109,7 +112,8 @@ namespace PrivaPub.Domain.Portability
 						return JobOutcome.Done;
 					}
 					using var scope = scopes.CreateScope();
-					var run = new Run(state, me, archive, actorId, actorId == me.Uri, scope.ServiceProvider);
+					var run = new Run(state, me, archive, actorId, Text(actor["followers"]) ?? actorId.TrimEnd('/') + "/followers", actorId == me.Uri,
+						scope.ServiceProvider);
 					state.ImportState = "running";
 					state.ImportFrom = actorId;
 					state.ImportSameActor = run.SameActor;
@@ -224,10 +228,7 @@ namespace PrivaPub.Domain.Portability
 				return PostVisibility.Public;
 			if (cc.Any(Public.Contains))
 				return PostVisibility.Unlisted;
-			var followers = run.ArchiveActor.TrimEnd('/') is var actor && run.Archive.Json("actor.json") is JsonObject document
+			if (to.Concat(cc).Contains(run.ArchiveFollowers) && Text(note["audience"]) == default)
-				? Text(document["followers"]) ?? actor + "/followers"
-				: default;
-			if (followers != default && to.Concat(cc).Contains(followers) && Text(note["audience"]) == default)
 				return PostVisibility.FollowersOnly;
 			return default;
 		}
@@ -260,6 +261,8 @@ namespace PrivaPub.Domain.Portability
 					continue;
 				var original = Text(note["id"]);
 				var published = Date(note["published"]) ?? DateTime.UtcNow;
+				if (Id(note["inReplyTo"]) is { } answers)
+					run.Parents[original] = answers;
 				if (run.SameActor)
 				{
 					var id = original.TrimEnd('/').Split('/')[^1];
@@ -363,10 +366,9 @@ namespace PrivaPub.Domain.Portability
 				}
 			if (Text(note["type"]) == "Question")
 				post.Poll = Poll(note);
-			if (visibility is PostVisibility.Public or PostVisibility.Unlisted)
+			// the conversation (FEP-7888) of its thread's first post, when the thread starts in the archive
-				post.ContextURI = post.AnsweringToPostId != default && run.Posts.ContainsValue((post.AnsweringToPostId, post.InReplyToURI))
+			if (visibility is PostVisibility.Public or PostVisibility.Unlisted && Root(run, original) is { } root && run.Posts.TryGetValue(root, out var first))
-					? post.InReplyToURI + "/context"
+				post.ContextURI = first.Uri + "/context";
-					: post.ObjectURI + "/context";
 			post.Media = await Media(run, note["attachment"] as JsonArray, id, token);
 			var rendered = ActivityPubRenderer.Note(post, me, default, post.InReplyToURI);
 			post.To = Strings(rendered["to"]);
@@ -376,6 +378,19 @@ namespace PrivaPub.Domain.Portability
 				await DB.Default.Update<PostEntity>().MatchID(post.AnsweringToPostId).Modify(b => b.Inc(p => p.RepliesCount, 1)).ExecuteAsync(token);
 		}
 
+		// the first post of an archive post's thread, when the thread starts in the archive
+		static string Root(Run run, string original)
+		{
+			var seen = new HashSet<string>(StringComparer.Ordinal);
+			while (run.Parents.TryGetValue(original, out var parent) && seen.Add(original))
+			{
+				if (!run.Posts.ContainsKey(parent))
+					return default;
+				original = parent;
+			}
+			return original;
+		}
+
 		static PostPoll Poll(JsonObject note)
 		{
 			var multiple = note["anyOf"] is JsonArray;
@@ -490,7 +505,7 @@ namespace PrivaPub.Domain.Portability
 					RangeKm = (float)Math.Clamp(located["rangeKm"]?.GetValue<double>() ?? 5, 1, 50),
 					ObjectURI = run.Me.PostUri(id),
 					Url = run.Me.PostHtmlUrl(id),
-					ImportedFromURI = original == run.Me.PostUri(original?.Split('/')[^1] ?? string.Empty) ? default : original,
+					ImportedFromURI = original,
 					CreationDate = Clamp(Date(located["published"]) ?? DateTime.UtcNow)
 				};
 				var files = (located["media"] as JsonArray ?? []).OfType<JsonObject>()