SocialPub/PrivaPub/Domain/Portability/ImportArchiveJob.cs

using Microsoft.AspNetCore.Http;

using MongoDB.Driver;
using MongoDB.Entities;

using PrivaPub.Api.Mastodon.Controllers;
using PrivaPub.Domain.Media;
using PrivaPub.Domain.Relationships;
using PrivaPub.Domain.Social;
using PrivaPub.Domain.Statuses;
using PrivaPub.Federation.Actors;
using PrivaPub.Federation.Inbox;
using PrivaPub.Federation.Moderation;
using PrivaPub.Federation.Objects;
using PrivaPub.Federation.Outbox;
using PrivaPub.Federation.Rendering;
using PrivaPub.Infrastructure.Backup;
using PrivaPub.Infrastructure.Ids;
using PrivaPub.Infrastructure.Jobs;
using PrivaPub.Models.Federation;
using PrivaPub.Models.Jobs;
using PrivaPub.Models.Media;
using PrivaPub.Models.Post;
using PrivaPub.Models.Social;
using PrivaPub.Models.User;

using System.Net;
using System.Text.Json.Nodes;
using System.Text.RegularExpressions;

using PostEntity = PrivaPub.Models.Post.Post;

namespace PrivaPub.Domain.Portability
{
	// An archive restored into one of a root's personas (owner decision 2026-10-07): PrivaPub's own, or Mastodon's account
	// archive. Only the parts asked for, one at a time, with progress, until asked to stop.
	// Posts: only the archive actor's own Creates (boosts, direct and circle posts are counted, not imported); delivered to
	// no one, put in no home, notifying nobody, they show on the profile, the outbox, hashtags and search. From the same
	// actor (a persona's own archive, back home) a post keeps its id and address, and one already here (even deleted) or
	// whose address is a tombstone is left alone; from another actor it gets an id of its date and its address here, with
	// the original in ImportedFromURI, so importing twice changes nothing. Replies and self-quotes inside the archive point
	// at their copies; polls are closed, with no votes; mentions stay links; the HTML is sanitized as any remote post's;
	// media go through MediaService, its checks and the root's quota.
	// The rest: follows asked again (not on a blocked server), blocks, mutes, blocked servers, lists (with only accounts
	// followed), bookmarks (read by signed fetch), filters, followed hashtags, the notification policy, pins (not
	// announced), the profile; and only when asked, located and scheduled posts (PrivaPub's archives) and likes, which tell
	// their authors. Followers are never imported: they choose.
	public partial class ImportArchiveJob(IServiceScopeFactory scopes, ILocalActorService localActors, IMediaService media, IDomainBlocks domainBlocks,
		IJobQueue jobs, ILogger<ImportArchiveJob> logger) : IJobHandler
	{
		static readonly HashSet<string> Public = ["https://www.w3.org/ns/activitystreams#Public", "as:Public", "Public"];
		static readonly HashSet<string> Postable = ["Note", "Article", "Page", "Question"];
		const int MaxMedia = 4;

		public JobKind Kind => JobKind.ImportArchive;
		public int Concurrency => 1;
		public int MaxAttempts => 1;
		public int PerHostLimit => 1;

		public static Job JobFor(string avatarId, string host) => new()
		{
			Kind = JobKind.ImportArchive,
			Payload = avatarId,
			Host = host,
			DedupeKey = $"import|{avatarId}|{DateTime.UtcNow.Ticks}"
		};

		sealed class Run(PersonaArchive state, LocalActor me, SafeArchive archive, string archiveActor, string archiveFollowers, bool sameActor,
			IServiceProvider services)
		{
			public PersonaArchive State { get; } = state;
			public string ArchiveFollowers { get; } = archiveFollowers;
			public LocalActor Me { get; } = me;
			public SafeArchive Archive { get; } = archive;
			public string ArchiveActor { get; } = archiveActor;
			public bool SameActor { get; } = sameActor;
			public IServiceProvider Services { get; } = services;
			public Dictionary<string, (string Id, string Uri)> Posts { get; } = new(StringComparer.Ordinal);//archive object id → here
			public Dictionary<string, string> Parents { get; } = new(StringComparer.Ordinal);//archive object id → what it answers
			public DateTime LastSaved { get; set; }

			public void Count(string what, int by = 1) => State.ImportCounts[what] = State.ImportCounts.GetValueOrDefault(what) + by;
		}

		public async Task<JobOutcome> Handle(Job job, CancellationToken token)
		{
			var state = await DB.Default.Find<PersonaArchive>().MatchID(job.Payload).ExecuteFirstAsync(token);
			if (state is not { ImportState: "queued" or "running" })
				return JobOutcome.Done;
			var me = await localActors.FindById(LocalActorKind.Person, job.Payload, token);
			var path = state.ImportFile;
			try
			{
				if (me == default || !await RootMayImport(me.Id, token))
				{
					await End(state, "failed", "the persona or its login can no longer import", token);
					return JobOutcome.Done;
				}
				var (archive, refused) = path == default ? (default, "no archive") : SafeArchive.Open(path);
				if (archive == default)
				{
					await End(state, "failed", refused, token);
					return JobOutcome.Done;
				}
				using (archive)
				{
					var actor = archive.Json("actor.json") as JsonObject;
					var actorId = Text(actor?["id"]);
					if (actorId == default)
					{
						await End(state, "failed", "the archive has no actor.json", token);
						return JobOutcome.Done;
					}
					using var scope = scopes.CreateScope();
					var run = new Run(state, me, archive, actorId, Text(actor["followers"]) ?? actorId.TrimEnd('/') + "/followers", actorId == me.Uri,
						scope.ServiceProvider);
					state.ImportState = "running";
					state.ImportFrom = actorId;
					state.ImportSameActor = run.SameActor;
					state.ImportCounts = [];
					await Save(run, token, force: true);

					var parts = state.ImportParts.ToHashSet(StringComparer.Ordinal);
					var privapub = Text((archive.Json("privapub/archive.json") as JsonObject)?["software"]) == "privapub";
					if (parts.Contains("profile") && !await Stopped(run, token))
						await Profile(run, actor, token);
					if (parts.Contains("posts") && !await Stopped(run, token))
						await Posts(run, token);
					else
						await KnownPosts(run, token);
					if (parts.Contains("located") && privapub && !await Stopped(run, token))
						await Located(run, token);
					if (parts.Contains("scheduled") && privapub && !await Stopped(run, token))
						await Scheduled(run, token);
					if (parts.Contains("tags") && !await Stopped(run, token))
						await Tags(run, token);
					if (parts.Contains("notificationpolicy") && !await Stopped(run, token))
						await Policy(run, token);
					if (parts.Contains("filters") && !await Stopped(run, token))
						await Filters(run, token);
					if (parts.Contains("pins") && !await Stopped(run, token))
						await Pins(run, token);
					if (parts.Contains("domainblocks") && !await Stopped(run, token))
						await DomainBlocks(run, token);
					if (parts.Contains("blocks") && !await Stopped(run, token))
						await Accounts(run, "blocked_accounts.csv", "blocks", Block, token);
					if (parts.Contains("mutes") && !await Stopped(run, token))
						await Accounts(run, "muted_accounts.csv", "mutes", Mute, token);
					if (parts.Contains("follows") && !await Stopped(run, token))
						await Accounts(run, "following_accounts.csv", "follows", Follow, token);
					if (parts.Contains("lists") && !await Stopped(run, token))
						await Lists(run, token);
					if (parts.Contains("bookmarks") && !await Stopped(run, token))
						await Bookmarks(run, token);
					if (parts.Contains("likes") && !await Stopped(run, token))
						await Likes(run, token);
					await End(state, await Stopped(run, token) ? "stopped" : "done", default, token);
				}
			}
			catch (Exception ex) when (ex is not OperationCanceledException)
			{
				logger.LogError(ex, "The import into {Persona} failed", me?.Handle ?? job.Payload);
				await End(state, "failed", ex.Message, CancellationToken.None);
			}
			finally
			{
				if (path != default && File.Exists(path))
					File.Delete(path);
			}
			return JobOutcome.Done;
		}

		static async Task<bool> RootMayImport(string avatarId, CancellationToken token)
		{
			var link = await DB.Default.Find<RootToAvatar>().Match(r => r.AvatarId == avatarId).ExecuteFirstAsync(token);
			var root = link == default ? default : await DB.Default.Find<RootUser>().MatchID(link.RootId).ExecuteFirstAsync(token);
			return root is { IsBanned: false, DeletedAt: null };
		}

		static async Task End(PersonaArchive state, string how, string error, CancellationToken token) =>
			await DB.Default.Update<PersonaArchive>().MatchID(state.ID)
				.Modify(a => a.ImportState, how).Modify(a => a.ImportError, error).Modify(a => a.ImportEndedAt, DateTime.UtcNow)
				.Modify(a => a.ImportCounts, state.ImportCounts).Modify(a => a.ImportDone, state.ImportDone).Modify(a => a.ImportTotal, state.ImportTotal)
				.Modify(a => a.ImportFile, null).Modify(a => a.StopAsked, false)
				.ExecuteAsync(token);

		// progress written at most every two seconds; the answer to whether the root asked to stop
		static async Task Save(Run run, CancellationToken token, bool force = false)
		{
			if (!force && DateTime.UtcNow - run.LastSaved < TimeSpan.FromSeconds(2))
				return;
			run.LastSaved = DateTime.UtcNow;
			await DB.Default.Update<PersonaArchive>().MatchID(run.State.ID)
				.Modify(a => a.ImportState, run.State.ImportState).Modify(a => a.ImportFrom, run.State.ImportFrom)
				.Modify(a => a.ImportSameActor, run.State.ImportSameActor).Modify(a => a.ImportCounts, run.State.ImportCounts)
				.Modify(a => a.ImportDone, run.State.ImportDone).Modify(a => a.ImportTotal, run.State.ImportTotal)
				.ExecuteAsync(token);
		}

		static async Task<bool> Stopped(Run run, CancellationToken token)
		{
			await Save(run, token);
			return await DB.Default.Find<PersonaArchive>().MatchID(run.State.ID).Match(a => a.StopAsked).ExecuteAnyAsync(token);
		}

		// ---- posts

		// what an outbox item is to this import: the object to import, or why not
		(JsonObject Note, string Skipped) Importable(Run run, JsonObject item)
		{
			var type = Text(item["type"]);
			if (type == "Announce")
				return (default, "boosts skipped");
			if (type != "Create" || item["object"] is not JsonObject note || !Postable.Contains(Text(note["type"]) ?? string.Empty))
				return (default, "not a post");
			if (Text(item["actor"]) != run.ArchiveActor || Id(note["attributedTo"]) is { } author && author != run.ArchiveActor)
				return (default, "not the archive's own");
			if (Text(note["id"]) == default)
				return (default, "not a post");
			return Visibility(run, note) == default ? (default, "direct or circle posts skipped") : (note, default);
		}

		PostVisibility? Visibility(Run run, JsonObject note)
		{
			var to = Strings(note["to"]);
			var cc = Strings(note["cc"]);
			if (to.Any(Public.Contains))
				return PostVisibility.Public;
			if (cc.Any(Public.Contains))
				return PostVisibility.Unlisted;
			if (to.Concat(cc).Contains(run.ArchiveFollowers) && Text(note["audience"]) == default)
				return PostVisibility.FollowersOnly;
			return default;
		}

		// what is already here of the archive's posts, for pins, filters and bookmarks when posts are not imported again
		async Task KnownPosts(Run run, CancellationToken token)
		{
			await foreach (var item in run.Archive.Outbox(token))
			{
				if (Importable(run, item).Note is not { } note)
					continue;
				var original = Text(note["id"]);
				var here = run.SameActor
					? await DB.Default.Find<PostEntity>().Match(p => p.ObjectURI == original && p.GroupUserId == run.Me.Id).ExecuteFirstAsync(token)
					: await DB.Default.Find<PostEntity>().Match(p => p.GroupUserId == run.Me.Id && p.ImportedFromURI == original).ExecuteFirstAsync(token);
				if (here is { DeletedAt: null })
					run.Posts[original] = (here.ID, here.ObjectURI);
			}
		}

		async Task Posts(Run run, CancellationToken token)
		{
			// first the ids each post will have here, so a reply or a quote anywhere in the archive finds its copy
			var planned = new Dictionary<string, (string Id, string Uri, bool New)>(StringComparer.Ordinal);
			var total = 0L;
			await foreach (var item in run.Archive.Outbox(token))
			{
				total++;
				if (Importable(run, item).Note is not { } note)
					continue;
				var original = Text(note["id"]);
				var published = Date(note["published"]) ?? DateTime.UtcNow;
				if (Id(note["inReplyTo"]) is { } answers)
					run.Parents[original] = answers;
				if (run.SameActor)
				{
					var id = original.TrimEnd('/').Split('/')[^1];
					if (!IsObjectId(id) || original != run.Me.PostUri(id))
						id = PrivacyIds.At(Clamp(published));
					var here = await DB.Default.Find<PostEntity>().Match(p => p.ID == id || p.ObjectURI == original).ExecuteFirstAsync(token);
					var tombstone = await DB.Default.Find<DeletedObject>().Match(d => d.ObjectURI == original).ExecuteAnyAsync(token);
					if (here != default || tombstone)
					{
						if (here is { DeletedAt: null, GroupUserId: var owner } && owner == run.Me.Id)
							run.Posts[original] = (here.ID, here.ObjectURI);
						run.Count(here is { DeletedAt: not null } || tombstone ? "posts deleted here" : "posts already here");
						continue;
					}
					planned[original] = (id, original, true);
				}
				else
				{
					var here = await DB.Default.Find<PostEntity>().Match(p => p.GroupUserId == run.Me.Id && p.ImportedFromURI == original).ExecuteFirstAsync(token);
					if (here != default)
					{
						if (here.DeletedAt == default)
							run.Posts[original] = (here.ID, here.ObjectURI);
						run.Count("posts already here");
						continue;
					}
					var id = PrivacyIds.At(Clamp(published));
					planned[original] = (id, run.Me.PostUri(id), true);
				}
			}
			foreach (var (original, (id, uri, _)) in planned)
				run.Posts[original] = (id, uri);
			run.State.ImportTotal = total;
			run.State.ImportDone = 0;

			await foreach (var item in run.Archive.Outbox(token))
			{
				run.State.ImportDone++;
				if (run.State.ImportDone % 50 == 0 && await Stopped(run, token))
					return;
				var (note, skipped) = Importable(run, item);
				if (note == default)
				{
					run.Count(skipped);
					continue;
				}
				var original = Text(note["id"]);
				if (!planned.TryGetValue(original, out var target))
					continue;
				await Post(run, note, original, target.Id, target.Uri, Visibility(run, note).Value, token);
				run.Count("posts");
			}
		}

		async Task Post(Run run, JsonObject note, string original, string id, string uri, PostVisibility visibility, CancellationToken token)
		{
			var me = run.Me;
			var summary = Clean(Text(note["summary"]));
			var content = Text(note["content"]) ?? (note["contentMap"] as JsonObject)?.Select(c => Text(c.Value)).FirstOrDefault();
			var source = note["source"] as JsonObject;
			var post = new PostEntity
			{
				ID = id,
				GroupUserId = me.Id,
				AuthorAccountId = me.Id,
				ActorURI = me.Uri,
				Visibility = visibility,
				Title = Clean(Text(note["name"])),
				SpoilerText = summary,
				HasContentWarning = summary != default || note["sensitive"]?.GetValueKind() == System.Text.Json.JsonValueKind.True,
				ContentHtml = content == default ? default : ContentSanitizer.Html(content),
				Text = Text(source?["mediaType"]) is "text/plain" or "text/markdown" ? Text(source["content"]) : default,
				ContentFormat = ContentFormat.Html,
				Language = (note["contentMap"] as JsonObject)?.Select(c => c.Key).FirstOrDefault(),
				Tags = (note["tag"] as JsonArray ?? [])
					.OfType<JsonObject>().Where(t => Text(t["type"]) == "Hashtag").Select(t => TagsController.Normalise(Text(t["name"])))
					.Where(t => t.Length > 0).Distinct().ToList(),
				ObjectURI = uri,
				Url = me.PostHtmlUrl(id),
				ActivityURI = me.ActivityUri("create-" + id),
				ImportedFromURI = run.SameActor ? default : original,
				CreationDate = Clamp(Date(note["published"]) ?? DateTime.UtcNow),
				LocalQuotePolicy = me.Settings.QuotePolicy ?? QuotePolicies.Public
			};
			if (Id(note["inReplyTo"]) is { } inReplyTo)
			{
				if (run.Posts.TryGetValue(inReplyTo, out var parent))
					(post.AnsweringToPostId, post.InReplyToURI, post.InReplyToAccountId) = (parent.Id, parent.Uri, me.Id);
				else
				{
					post.InReplyToURI = inReplyTo;
					if (await DB.Default.Find<PostEntity>().Match(p => p.ObjectURI == inReplyTo).ExecuteFirstAsync(token) is { } held)
						(post.AnsweringToPostId, post.InReplyToAccountId) = (held.ID, held.AuthorAccountId ?? held.GroupUserId);
				}
			}
			foreach (var key in new[] { "quote", "quoteUrl", "quoteUri", "_misskey_quote" })
				if (Text(note[key]) is { } quoted && run.Posts.TryGetValue(quoted, out var copy))
				{
					(post.QuotedPostId, post.QuoteURI, post.QuoteState, post.QuoteByConsent) = (copy.Id, copy.Uri, QuoteState.Accepted, true);
					break;
				}
			if (Text(note["type"]) == "Question")
				post.Poll = Poll(note);
			// the conversation (FEP-7888) of its thread's first post, when the thread starts in the archive
			if (visibility is PostVisibility.Public or PostVisibility.Unlisted && Root(run, original) is { } root && run.Posts.TryGetValue(root, out var first))
				post.ContextURI = first.Uri + "/context";
			post.Media = await Media(run, note["attachment"] as JsonArray, id, token);
			var rendered = ActivityPubRenderer.Note(post, me, default, post.InReplyToURI);
			post.To = Strings(rendered["to"]);
			post.Cc = Strings(rendered["cc"]);
			await DB.Default.SaveAsync(post, token);
			if (post.AnsweringToPostId != default)
				await DB.Default.Update<PostEntity>().MatchID(post.AnsweringToPostId).Modify(b => b.Inc(p => p.RepliesCount, 1)).ExecuteAsync(token);
		}

		// the first post of an archive post's thread, when the thread starts in the archive
		static string Root(Run run, string original)
		{
			var seen = new HashSet<string>(StringComparer.Ordinal);
			while (run.Parents.TryGetValue(original, out var parent) && seen.Add(original))
			{
				if (!run.Posts.ContainsKey(parent))
					return default;
				original = parent;
			}
			return original;
		}

		static PostPoll Poll(JsonObject note)
		{
			var multiple = note["anyOf"] is JsonArray;
			var options = (note["anyOf"] ?? note["oneOf"]) as JsonArray ?? [];
			var ends = Date(note["endTime"]) ?? Date(note["closed"]) ?? DateTime.UtcNow;
			var now = DateTime.UtcNow;
			return new PostPoll
			{
				Options = options.OfType<JsonObject>().Select(o => new PollOption { Title = Text(o["name"]) }).Where(o => o.Title != default).ToList(),
				Multiple = multiple,
				ExpiresAt = ends < now ? ends : now,
				ClosedAt = ends < now ? ends : now,
				VotersCount = 0
			};
		}

		// the attachments the archive holds, uploaded as the persona's own (checked, processed, counted against its quota)
		async Task<List<PostMedia>> Media(Run run, JsonArray attachments, string postId, CancellationToken token)
		{
			var made = new List<PostMedia>();
			foreach (var attachment in (attachments ?? []).OfType<JsonObject>().Take(MaxMedia))
			{
				var url = Text(attachment["url"]) ?? Text((attachment["url"] as JsonArray)?.OfType<JsonObject>().FirstOrDefault()?["href"]);
				var file = url?.TrimStart('/');
				if (file == default || file.Contains("://") || !run.Archive.Has(file))
				{
					run.Count("media not in the archive");
					continue;
				}
				var focus = attachment["focalPoint"] is JsonArray { Count: 2 } point ? $"{point[0]},{point[1]}" : default;
				var outcome = await Upload(run, file, Text(attachment["mediaType"]), Clean(Text(attachment["name"])), focus, token);
				if (outcome?.Ok != true)
				{
					run.Count("media refused");
					continue;
				}
				var row = outcome.Attachment;
				await DB.Default.Update<MediaAttachment>().MatchID(row.ID).Modify(m => m.PostId, postId).Modify(m => m.AttachedAt, DateTime.UtcNow).ExecuteAsync(token);
				made.Add(new PostMedia
				{
					AttachmentId = row.ID,
					ContentType = row.ContentType,
					Kind = row.Kind,
					DurationSeconds = row.DurationSeconds,
					URL = media.Url(row.FilePath),
					PreviewURL = media.Url(row.PreviewPath ?? row.FilePath),
					Description = row.Description,
					Blurhash = row.Blurhash,
					Width = row.Width,
					Height = row.Height,
					Focus = row.Focus
				});
				run.Count("media");
			}
			return made;
		}

		// a file of the archive given to MediaService as an upload would be
		async Task<MediaOutcome> Upload(Run run, string file, string contentType, string description, string focus, CancellationToken token)
		{
			var temporary = Path.Combine(Path.GetTempPath(), $"privapub-import-{Guid.NewGuid():N}{Path.GetExtension(file)}");
			try
			{
				await using (var source = run.Archive.Read(file))
				await using (var target = new FileStream(temporary, FileMode.CreateNew, FileAccess.Write))
					await source.CopyToAsync(target, token);
				await using var stream = new FileStream(temporary, FileMode.Open, FileAccess.Read);
				var form = new FormFile(stream, 0, stream.Length, "file", Path.GetFileName(file))
				{
					Headers = new HeaderDictionary(),
					ContentType = contentType ?? (MediaService.ContentTypes.TryGetContentType(file, out var guessed) ? guessed : "application/octet-stream")
				};
				return await media.Upload(run.Me, form, description, focus, later: false, token);
			}
			finally
			{
				File.Delete(temporary);
			}
		}

		// ---- PrivaPub's own parts

		async Task Located(Run run, CancellationToken token)
		{
			foreach (var located in (run.Archive.Json("privapub/located.json") as JsonArray ?? []).OfType<JsonObject>())
			{
				var original = Text(located["id"]);
				if (original != default && await DB.Default.Find<PostEntity>().Match(p => p.GroupUserId == run.Me.Id
					&& (p.ObjectURI == original || p.ImportedFromURI == original)).ExecuteAnyAsync(token))
				{
					run.Count("located posts already here");
					continue;
				}
				if (located["latitude"]?.GetValue<double>() is not { } latitude || located["longitude"]?.GetValue<double>() is not { } longitude)
					continue;
				var id = PrivacyIds.At(Clamp(Date(located["published"]) ?? DateTime.UtcNow));
				var post = new PostEntity
				{
					ID = id,
					GroupUserId = run.Me.Id,
					AuthorAccountId = run.Me.Id,
					ActorURI = run.Me.Uri,
					Visibility = PostVisibility.LocalGeo,
					IsLocalOnly = true,
					Title = Clean(Text(located["title"])),
					SpoilerText = Clean(Text(located["spoilerText"])),
					HasContentWarning = Clean(Text(located["spoilerText"])) != default,
					Text = Text(located["text"]),
					ContentHtml = ActivityPubRenderer.Html(Text(located["text"]) ?? string.Empty),
					ContentFormat = ContentFormat.Markdown,
					Geo = new MongoDB.Entities.Coordinates2D(Math.Round(longitude, 2), Math.Round(latitude, 2)),
					RangeKm = (float)Math.Clamp(located["rangeKm"]?.GetValue<double>() ?? 5, 1, 50),
					ObjectURI = run.Me.PostUri(id),
					Url = run.Me.PostHtmlUrl(id),
					ImportedFromURI = original,
					CreationDate = Clamp(Date(located["published"]) ?? DateTime.UtcNow)
				};
				var files = (located["media"] as JsonArray ?? []).OfType<JsonObject>()
					.Select(m => new JsonObject { ["url"] = Text(m["url"]), ["name"] = Text(m["description"]) }).ToArray();
				post.Media = await Media(run, new JsonArray(files), id, token);
				await DB.Default.SaveAsync(post, token);
				run.Count("located posts");
			}
		}

		async Task Scheduled(Run run, CancellationToken token)
		{
			var made = new List<ScheduledStatus>();
			foreach (var scheduled in (run.Archive.Json("privapub/scheduled.json") as JsonArray ?? []).OfType<JsonObject>())
			{
				var at = Date(scheduled["scheduledAt"]);
				if (at is not { } when || when < DateTime.UtcNow.AddMinutes(5) || await ScheduledStatuses.Refusal(run.Me.Id, when, default, token) != default)
				{
					run.Count("scheduled posts past or refused");
					continue;
				}
				var status = new ScheduledStatus
				{
					AvatarId = run.Me.Id,
					ScheduledAt = when,
					Params = new ScheduledParams
					{
						Text = Text(scheduled["text"]),
						SpoilerText = Text(scheduled["spoilerText"]),
						Sensitive = scheduled["sensitive"]?.GetValueKind() == System.Text.Json.JsonValueKind.True,
						Visibility = Text(scheduled["visibility"]) is { } visibility && visibility is "public" or "unlisted" or "private" or "direct" ? visibility : "public",
						Language = Text(scheduled["language"])
					}
				};
				status.ID = (string)status.GenerateNewID();
				foreach (var file in (scheduled["media"] as JsonArray ?? []).OfType<JsonObject>().Take(MaxMedia))
				{
					var path = Text(file["url"])?.TrimStart('/');
					if (path == default || !run.Archive.Has(path))
						continue;
					var outcome = await Upload(run, path, default, Clean(Text(file["description"])), default, token);
					if (outcome?.Ok != true)
						continue;
					await DB.Default.Update<MediaAttachment>().MatchID(outcome.Attachment.ID).Modify(m => m.ScheduledStatusId, status.ID).ExecuteAsync(token);
					status.Params.MediaIds.Add(outcome.Attachment.ID);
				}
				await DB.Default.SaveAsync(status, token);
				made.Add(status);
				run.Count("scheduled posts");
			}
			if (made.Count > 0)
				await jobs.EnqueueMany(made.Select(s => ScheduledStatuses.JobFor(s, new Uri(localActors.BaseAddress).Host)), token);
		}

		async Task Tags(Run run, CancellationToken token)
		{
			var mine = (await DB.Default.Find<FollowedTag, string>().Match(t => t.AvatarId == run.Me.Id).Project(t => t.Name).ExecuteAsync(token)).ToHashSet();
			foreach (var name in (run.Archive.Json("privapub/followed_tags.json") as JsonArray ?? []).Select(t => TagsController.Normalise(Text(t))))
			{
				if (name.Length == 0 || mine.Contains(name) || mine.Count >= 500)
					continue;
				await DB.Default.SaveAsync(new FollowedTag { AvatarId = run.Me.Id, Name = name }, token);
				mine.Add(name);
				run.Count("followed hashtags");
			}
		}

		async Task Policy(Run run, CancellationToken token)
		{
			if (run.Archive.Json("privapub/notification_policy.json") is not JsonObject json)
				return;
			var policy = await NotificationPolicies.Of(run.Me.Id, token);
			NotificationAction Action(string field, NotificationAction kept) => Enum.TryParse<NotificationAction>(Text(json[field]), out var action) ? action : kept;
			policy.ForNotFollowing = Action("forNotFollowing", policy.ForNotFollowing);
			policy.ForNotFollowers = Action("forNotFollowers", policy.ForNotFollowers);
			policy.ForNewAccounts = Action("forNewAccounts", policy.ForNewAccounts);
			policy.ForPrivateMentions = Action("forPrivateMentions", policy.ForPrivateMentions);
			policy.ForLimitedAccounts = Action("forLimitedAccounts", policy.ForLimitedAccounts);
			await NotificationPolicies.Save(policy, token);
			run.Count("notification policy");
		}

		async Task Filters(Run run, CancellationToken token)
		{
			var titles = (await DB.Default.Find<PersonaFilter, string>().Match(f => f.AvatarId == run.Me.Id).Project(f => f.Title).ExecuteAsync(token)).ToHashSet();
			foreach (var filter in (run.Archive.Json("privapub/filters.json") as JsonArray ?? []).OfType<JsonObject>())
			{
				var title = Clean(Text(filter["title"]));
				if (title == default || titles.Contains(title))
					continue;
				var statuses = new List<FilterStatus>();
				foreach (var uri in Strings(filter["statuses"]))
					if (await Held(run, uri, token) is { } post)
						statuses.Add(new FilterStatus { Id = MongoDB.Bson.ObjectId.GenerateNewId().ToString(), PostId = post.ID });
				await DB.Default.SaveAsync(new PersonaFilter
				{
					AvatarId = run.Me.Id,
					Title = title,
					Context = Strings(filter["context"]).Where(c => c is "home" or "notifications" or "public" or "thread" or "account").ToList(),
					ExpiresAt = Date(filter["expiresAt"]),
					Action = Text(filter["action"]) is { } action && action is "warn" or "hide" or "blur" ? action : "warn",
					Keywords = (filter["keywords"] as JsonArray ?? []).OfType<JsonObject>().Where(k => Clean(Text(k["keyword"])) != default)
						.Select(k => new FilterKeyword { Id = MongoDB.Bson.ObjectId.GenerateNewId().ToString(), Keyword = Text(k["keyword"]).Trim(),
							WholeWord = k["wholeWord"]?.GetValueKind() == System.Text.Json.JsonValueKind.True }).ToList(),
					Statuses = statuses
				}, token);
				titles.Add(title);
				run.Count("filters");
			}
		}

		// pinned again without announcing it (an Add would tell every follower's server)
		async Task Pins(Run run, CancellationToken token)
		{
			var pinned = (await DB.Default.Find<Pin, string>().Match(p => p.AvatarId == run.Me.Id).Project(p => p.PostId).ExecuteAsync(token)).ToHashSet();
			foreach (var uri in Strings(run.Archive.Json("privapub/pins.json")))
			{
				if (pinned.Count >= 5)
					break;
				if (!run.Posts.TryGetValue(uri, out var copy) || pinned.Contains(copy.Id))
					continue;
				await DB.Default.SaveAsync(new Pin { AvatarId = run.Me.Id, PostId = copy.Id }, token);
				pinned.Add(copy.Id);
				run.Count("pins");
			}
		}

		// ---- relationships

		async Task DomainBlocks(Run run, CancellationToken token)
		{
			var relationships = run.Services.GetRequiredService<IRelationshipService>();
			var mine = (await DB.Default.Find<AccountDomainBlock, string>().Match(d => d.AvatarId == run.Me.Id).Project(d => d.Domain).ExecuteAsync(token)).ToHashSet();
			foreach (var line in run.Archive.Lines("blocked_domains.csv"))
			{
				var domain = Cells(line).FirstOrDefault()?.Trim().ToLowerInvariant();
				if (string.IsNullOrEmpty(domain) || domain == "#domain" || Uri.CheckHostName(domain) != UriHostNameType.Dns || mine.Contains(domain))
					continue;
				await relationships.BlockDomain(run.Me, domain, token);
				mine.Add(domain);
				run.Count("blocked servers");
			}
		}

		// one CSV of accounts, each resolved (WebFinger or its address) and acted on; a header line is skipped
		async Task Accounts(Run run, string file, string what, Func<Run, (LocalActor Local, ForeignAvatar Remote), string[], CancellationToken, Task<bool>> act,
			CancellationToken token)
		{
			var follows = run.Services.GetRequiredService<IFollowService>();
			var lines = run.Archive.Lines(file);
			foreach (var line in lines)
			{
				if (await Stopped(run, token))
					return;
				var cells = Cells(line);
				var address = cells.FirstOrDefault()?.Trim();
				if (string.IsNullOrEmpty(address) || address.Equals("Account address", StringComparison.OrdinalIgnoreCase))
					continue;
				var target = await follows.Resolve(address, token);
				if (target.Local == default && target.Remote == default)
				{
					run.Count($"{what} not found");
					continue;
				}
				if (target.Local?.Id == run.Me.Id)
					continue;
				run.Count(await act(run, target, cells, token) ? what : $"{what} skipped");
			}
		}

		static async Task<bool> Block(Run run, (LocalActor Local, ForeignAvatar Remote) target, string[] cells, CancellationToken token)
		{
			var (uri, id) = target.Local != default ? (target.Local.Uri, target.Local.Id) : (target.Remote.ActorURI, target.Remote.ID);
			if (await DB.Default.Find<Block>().Match(b => b.AvatarId == run.Me.Id && b.TargetActorURI == uri).ExecuteAnyAsync(token))
				return false;
			await run.Services.GetRequiredService<IRelationshipService>().Block(run.Me, uri, id, token);
			return true;
		}

		static async Task<bool> Mute(Run run, (LocalActor Local, ForeignAvatar Remote) target, string[] cells, CancellationToken token)
		{
			var (uri, id) = target.Local != default ? (target.Local.Uri, target.Local.Id) : (target.Remote.ActorURI, target.Remote.ID);
			var hide = cells.Length < 2 || !cells[1].Trim().Equals("false", StringComparison.OrdinalIgnoreCase);
			await run.Services.GetRequiredService<IRelationshipService>().Mute(run.Me, uri, id, hide, default, token);
			return true;
		}

		async Task<bool> Follow(Run run, (LocalActor Local, ForeignAvatar Remote) target, string[] cells, CancellationToken token)
		{
			var uri = target.Local?.Uri ?? target.Remote.ActorURI;
			var host = new Uri(uri).Host;
			if (domainBlocks.IsSuspended(host)
				|| await DB.Default.Find<AccountDomainBlock>().Match(d => d.AvatarId == run.Me.Id && d.Domain == host).ExecuteAnyAsync(token)
				|| await DB.Default.Find<Following>().Match(f => f.AvatarId == run.Me.Id && f.TargetActorURI == uri).ExecuteAnyAsync(token))
				return false;
			var boosts = cells.Length < 2 || !cells[1].Trim().Equals("false", StringComparison.OrdinalIgnoreCase);
			return await run.Services.GetRequiredService<IFollowService>().FollowAs(run.Me, uri, boosts, token) != default;
		}

		// lists by title, with only the accounts the persona follows
		async Task Lists(Run run, CancellationToken token)
		{
			var follows = run.Services.GetRequiredService<IFollowService>();
			var lists = (await DB.Default.Find<PersonaList>().Match(l => l.AvatarId == run.Me.Id).ExecuteAsync(token)).ToDictionary(l => l.Title, StringComparer.Ordinal);
			foreach (var line in run.Archive.Lines("lists.csv"))
			{
				var cells = Cells(line);
				if (cells.Length < 2 || Clean(cells[0]) is not { } title)
					continue;
				if (!lists.TryGetValue(title, out var list))
				{
					if (lists.Count >= 50)
						continue;
					list = new PersonaList { AvatarId = run.Me.Id, Title = title, RepliesPolicy = "list" };
					await DB.Default.SaveAsync(list, token);
					lists[title] = list;
					run.Count("lists");
				}
				var target = await follows.Resolve(cells[1].Trim(), token);
				var uri = target.Local?.Uri ?? target.Remote?.ActorURI;
				var following = uri == default ? default
					: await DB.Default.Find<Following>().Match(f => f.AvatarId == run.Me.Id && f.TargetActorURI == uri && f.State == FollowState.Accepted).ExecuteFirstAsync(token);
				if (following == default)
				{
					run.Count("list members not followed");
					continue;
				}
				if (await DB.Default.Find<PersonaListMember>().Match(m => m.ListId == list.ID && m.AccountId == following.TargetAccountId).ExecuteAnyAsync(token))
					continue;
				await DB.Default.SaveAsync(new PersonaListMember { ListId = list.ID, AvatarId = run.Me.Id, AccountId = following.TargetAccountId }, token);
				run.Count("list members");
			}
		}

		async Task Bookmarks(Run run, CancellationToken token)
		{
			var uris = Strings(run.Archive.Json("bookmarks.json")?["orderedItems"]);
			if (uris.Count == 0)
				uris = run.Archive.Lines("bookmarks.csv").Select(l => Cells(l).FirstOrDefault()?.Trim()).Where(u => u?.StartsWith("http") == true).ToList();
			foreach (var uri in uris.Distinct())
			{
				if (await Stopped(run, token))
					return;
				var post = await Fetched(run, uri, token);
				if (post == default)
				{
					run.Count("bookmarks not found");
					continue;
				}
				if (await DB.Default.Find<Bookmark>().Match(b => b.AvatarId == run.Me.Id && b.PostId == post.ID).ExecuteAnyAsync(token))
					continue;
				await DB.Default.SaveAsync(new Bookmark { AvatarId = run.Me.Id, PostId = post.ID }, token);
				run.Count("bookmarks");
			}
		}

		// asked for on purpose: each like tells the post's author
		async Task Likes(Run run, CancellationToken token)
		{
			var statuses = run.Services.GetRequiredService<IStatusService>();
			foreach (var uri in Strings(run.Archive.Json("likes.json")?["orderedItems"]).Distinct())
			{
				if (await Stopped(run, token))
					return;
				var post = await Fetched(run, uri, token);
				if (post == default)
				{
					run.Count("likes not found");
					continue;
				}
				run.Count((await statuses.Favourite(run.Me, post.ID, true, token)).Ok ? "likes" : "likes not found");
			}
		}

		// a post here by its address, or read from its server (signed) and kept
		static async Task<PostEntity> Fetched(Run run, string uri, CancellationToken token)
		{
			if (await Held(run, uri, token) is { } held)
				return held;
			return await run.Services.GetRequiredService<IRemotePosts>().StoreContext(uri, 0, token);
		}

		static async Task<PostEntity> Held(Run run, string uri, CancellationToken token)
		{
			if (uri == default)
				return default;
			if (run.Posts.TryGetValue(uri, out var copy))
				return await DB.Default.Find<PostEntity>().MatchID(copy.Id).ExecuteFirstAsync(token);
			return await DB.Default.Find<PostEntity>().Match(p => (p.ObjectURI == uri || p.Url == uri) && !p.DeletedAt.HasValue).ExecuteFirstAsync(token);
		}

		// ---- the profile: name, words, fields, flags and pictures, saved as Settings saves them (and announced the same way)

		async Task Profile(Run run, JsonObject actor, CancellationToken token)
		{
			var avatar = await DB.Default.Find<Avatar>().MatchID(run.Me.Id).ExecuteFirstAsync(token);
			if (Clean(Text(actor["name"])) is { } name)
				avatar.Name = name;
			if (Text(actor["summary"]) is { } summary)
				avatar.Biography = Plain(summary);
			var fields = (actor["attachment"] as JsonArray ?? []).OfType<JsonObject>().Where(f => Text(f["type"]) == "PropertyValue" && Clean(Text(f["name"])) != default)
				.Take(4).ToDictionary(f => Text(f["name"]).Trim(), f => Plain(Text(f["value"]) ?? string.Empty));
			if (fields.Count > 0)
				avatar.Fields = fields;
			if (actor["manuallyApprovesFollowers"]?.GetValueKind() is { } locked and (System.Text.Json.JsonValueKind.True or System.Text.Json.JsonValueKind.False))
				avatar.Settings.IsLocked = locked == System.Text.Json.JsonValueKind.True;
			if (actor["discoverable"]?.GetValueKind() is { } discoverable and (System.Text.Json.JsonValueKind.True or System.Text.Json.JsonValueKind.False))
				avatar.Settings.IsDiscoverable = discoverable == System.Text.Json.JsonValueKind.True;
			if (actor["indexable"]?.GetValueKind() is { } indexable and (System.Text.Json.JsonValueKind.True or System.Text.Json.JsonValueKind.False))
				avatar.Settings.IsIndexable = indexable == System.Text.Json.JsonValueKind.True;
			foreach (var (field, kind, width, height) in new[] { ("icon", "avatar", 400, 400), ("image", "header", 1500, 500) })
			{
				var file = (Text((actor[field] as JsonObject)?["url"]) ?? Text(actor[field]))?.TrimStart('/');
				if (file == default || file.Contains("://") || !run.Archive.Has(file))
					continue;
				var outcome = await Picture(run, file, kind, width, height, token);
				if (outcome?.Ok != true)
				{
					run.Count("pictures refused");
					continue;
				}
				if (kind == "avatar")
					avatar.PictureURL = media.Url(outcome.Attachment.FilePath);
				else
					avatar.ThumbnailURL = media.Url(outcome.Attachment.FilePath);
				await media.Trash(m => m.ProfileOfAvatarId == avatar.ID && m.Kind == kind && m.ID != outcome.Attachment.ID, "replaced", token);
			}
			avatar.UpdatedAt = DateTime.UtcNow;
			await DB.Default.SaveAsync(avatar, token);
			await run.Services.GetRequiredService<IOutboxPublisher>().PublishProfile(localActors.FromAvatar(avatar), token);
			run.Count("profile");
		}

		async Task<MediaOutcome> Picture(Run run, string file, string kind, int width, int height, CancellationToken token)
		{
			var temporary = Path.Combine(Path.GetTempPath(), $"privapub-import-{Guid.NewGuid():N}{Path.GetExtension(file)}");
			try
			{
				await using (var source = run.Archive.Read(file))
				await using (var target = new FileStream(temporary, FileMode.CreateNew, FileAccess.Write))
					await source.CopyToAsync(target, token);
				await using var stream = new FileStream(temporary, FileMode.Open, FileAccess.Read);
				var form = new FormFile(stream, 0, stream.Length, kind, Path.GetFileName(file))
				{
					Headers = new HeaderDictionary(),
					ContentType = MediaService.ContentTypes.TryGetContentType(file, out var guessed) ? guessed : "application/octet-stream"
				};
				return await media.ProfileImage(run.Me.Id, kind, form, width, height, token);
			}
			finally
			{
				File.Delete(temporary);
			}
		}

		// ---- helpers

		static string Text(JsonNode node) => node is JsonValue value && value.TryGetValue<string>(out var text) ? text : default;

		static string Id(JsonNode node) => node switch
		{
			JsonValue => Text(node),
			JsonObject json => Text(json["id"]),
			JsonArray list => list.Select(Id).FirstOrDefault(i => i != default),
			_ => default
		};

		static List<string> Strings(JsonNode node) => node switch
		{
			JsonArray list => list.Select(Id).Where(i => i != default).ToList(),
			JsonValue => Text(node) is { } one ? [one] : [],
			JsonObject json when Text(json["type"]) is "OrderedCollection" or "Collection" => Strings(json["orderedItems"] ?? json["items"]),
			_ => []
		};

		static DateTime? Date(JsonNode node) =>
			DateTime.TryParse(Text(node), System.Globalization.CultureInfo.InvariantCulture,
				System.Globalization.DateTimeStyles.AdjustToUniversal | System.Globalization.DateTimeStyles.AssumeUniversal, out var date) ? date : default;

		static DateTime Clamp(DateTime when) => when > DateTime.UtcNow ? DateTime.UtcNow : when;

		static string Clean(string text) => string.IsNullOrWhiteSpace(text) ? default : text.Trim();

		static bool IsObjectId(string id) => id is { Length: 24 } && id.All(char.IsAsciiHexDigitLower);

		// a CSV line's cells (quotes as Mastodon writes them)
		static string[] Cells(string line)
		{
			var cells = new List<string>();
			var cell = new System.Text.StringBuilder();
			var quoted = false;
			for (var i = 0; i < line.Length; i++)
			{
				var c = line[i];
				if (quoted)
				{
					if (c == '"' && i + 1 < line.Length && line[i + 1] == '"')
					{
						cell.Append('"');
						i++;
					}
					else if (c == '"')
						quoted = false;
					else
						cell.Append(c);
				}
				else if (c == '"')
					quoted = true;
				else if (c == ',')
				{
					cells.Add(cell.ToString());
					cell.Clear();
				}
				else
					cell.Append(c);
			}
			cells.Add(cell.ToString());
			return [.. cells];
		}

		// HTML to the plain words a persona writes in its profile
		static string Plain(string html)
		{
			var text = Breaks().Replace(html, "\n");
			text = Tags().Replace(text, string.Empty);
			return WebUtility.HtmlDecode(text).Trim();
		}

		[GeneratedRegex(@"<br\s*/?>|</p>\s*<p[^>]*>", RegexOptions.IgnoreCase)]
		private static partial Regex Breaks();

		[GeneratedRegex(@"<[^>]+>")]
		private static partial Regex Tags();
	}
}