thepraandClaude Opus 5.5
163f7e4e2b
A post's visibility no longer parses actor.json again for every outbox item; a reply's conversation is its thread's first post's, found from what the first pass read instead of searching every planned post (quadratic on a large archive); and a located post always records where it came from, so importing a persona's own archive twice made it twice. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01LsXgEaXee4GCU1hwYgPJXw
347 lines
18 KiB
C#
347 lines
18 KiB
C#
using MongoDB.Entities;
|
|
|
|
using PrivaPub.Models.Jobs;
|
|
using PrivaPub.Models.Media;
|
|
using PrivaPub.Models.Social;
|
|
using PrivaPub.Models.User;
|
|
using PrivaPub.Tests.Support;
|
|
using PrivaPub.Tests.Support.Host;
|
|
|
|
using System.IO.Compression;
|
|
using System.Net;
|
|
using System.Net.Http.Json;
|
|
using System.Text;
|
|
using System.Text.Json.Nodes;
|
|
|
|
using PostEntity = PrivaPub.Models.Post.Post;
|
|
|
|
namespace PrivaPub.Tests.Http
|
|
{
|
|
// A persona's archive (owner decision 2026-10-07): exported in Mastodon's layout with nothing of its root, its siblings
|
|
// or its keys; imported back home without bringing anything back, or into another persona as posts nobody is told of,
|
|
// once; Mastodon's own archives read too; and hostile ones refused.
|
|
[Trait("Category", "Integration")]
|
|
public sealed class PersonaArchiveTests : IAsyncLifetime
|
|
{
|
|
PrivaPubHost _host;
|
|
|
|
static CancellationToken Token => TestContext.Current.CancellationToken;
|
|
|
|
public async ValueTask InitializeAsync()
|
|
{
|
|
Assert.SkipUnless(MongoFixture.Enabled, MongoFixture.Skip);
|
|
_host = await PrivaPubHost.Shared();
|
|
}
|
|
|
|
public ValueTask DisposeAsync() => ValueTask.CompletedTask;
|
|
|
|
static string Base(Persona persona) => $"/clientapi/persona/{persona.Id}/archive";
|
|
|
|
async Task<JsonObject> Status(Persona persona)
|
|
{
|
|
using var client = _host.As(persona.Root.Jwt);
|
|
return (await client.GetFromJsonAsync<JsonObject>(Base(persona), Token))!;
|
|
}
|
|
|
|
async Task<byte[]> Exported(Persona persona)
|
|
{
|
|
using var client = _host.As(persona.Root.Jwt);
|
|
Assert.Equal(HttpStatusCode.Accepted, (await client.PostAsync(Base(persona), null, Token)).StatusCode);
|
|
await _host.Run(j => j.Kind == JobKind.ExportArchive && j.Payload == persona.Id, Token);
|
|
Assert.Equal("ready", (await Status(persona))["exportState"]!.GetValue<string>());
|
|
var ticket = (await (await client.PostAsync(Base(persona) + "/ticket", null, Token)).Content.ReadFromJsonAsync<JsonObject>(Token))!;
|
|
using var anonymous = _host.Client();
|
|
var response = await anonymous.GetAsync(ticket["url"]!.GetValue<string>(), Token);
|
|
Assert.Equal(HttpStatusCode.OK, response.StatusCode);
|
|
return await response.Content.ReadAsByteArrayAsync(Token);
|
|
}
|
|
|
|
// uploaded in two pieces, then imported with the parts asked for, the job run: the import's state
|
|
async Task<(HttpStatusCode Status, JsonObject State)> Imported(Persona persona, byte[] archive, params string[] parts)
|
|
{
|
|
using var client = _host.As(persona.Root.Jwt);
|
|
var upload = (await (await client.PostAsync("/clientapi/persona/archive/uploads", null, Token)).Content.ReadFromJsonAsync<JsonObject>(Token))!["id"]!.GetValue<string>();
|
|
var half = archive.Length / 2;
|
|
Assert.Equal(HttpStatusCode.OK, (await client.PutAsync($"/clientapi/persona/archive/uploads/{upload}?offset=0", new ByteArrayContent(archive[..half]), Token)).StatusCode);
|
|
Assert.Equal(HttpStatusCode.OK, (await client.PutAsync($"/clientapi/persona/archive/uploads/{upload}?offset={half}", new ByteArrayContent(archive[half..]), Token)).StatusCode);
|
|
var asked = await client.PostAsJsonAsync(Base(persona) + "/import", new { uploadId = upload, parts }, Token);
|
|
if (asked.StatusCode != HttpStatusCode.Accepted)
|
|
return (asked.StatusCode, default);
|
|
await _host.Run(j => j.Kind == JobKind.ImportArchive && j.Payload == persona.Id, Token);
|
|
return (asked.StatusCode, await Status(persona));
|
|
}
|
|
|
|
static int Count(JsonObject state, string what) => state["importCounts"]?[what]?.GetValue<int>() ?? 0;
|
|
|
|
static Dictionary<string, string> Texts(byte[] zip)
|
|
{
|
|
using var archive = new ZipArchive(new MemoryStream(zip));
|
|
return archive.Entries.Where(e => e.Name.EndsWith(".json") || e.Name.EndsWith(".csv"))
|
|
.ToDictionary(e => e.FullName, e => new StreamReader(e.Open()).ReadToEnd());
|
|
}
|
|
|
|
[Fact]
|
|
public async Task An_archive_holds_the_persona_and_nothing_of_its_root_its_siblings_or_its_keys()
|
|
{
|
|
var alice = await _host.Mastodon("alice");
|
|
var sibling = await _host.Persona(alice.Persona.Root, "sibling");
|
|
var bob = await _host.Mastodon("bob");
|
|
var picture = (await alice.Client.Exchange(new HttpRequestMessage(HttpMethod.Post, "/api/v2/media")
|
|
{
|
|
Content = MastodonHelpers.Multipart(("file", MastodonHelpers.JpegWithMetadata(64, 48), "image/jpeg", "dot.jpg"))
|
|
})).Ok().Body.Text("id");
|
|
var first = await alice.Status("a public post with a picture #archives", ("media_ids[]", picture));
|
|
await alice.Status("for my followers only", ("visibility", "private"));
|
|
await alice.Status("a reply to myself", ("in_reply_to_id", first.Text("id")));
|
|
await alice.Client.Post($"/api/v1/statuses/{first.Text("id")}/bookmark");
|
|
await alice.Client.Post($"/api/v1/statuses/{first.Text("id")}/pin");
|
|
await alice.Client.Post($"/api/v1/accounts/{bob.Persona.Id}/block");
|
|
await alice.Client.Post("/api/v1/domain_blocks", ("domain", "spam.example"));
|
|
await alice.Client.Post("/api/v1/tags/archives/follow");
|
|
|
|
var zip = await Exported(alice.Persona);
|
|
var texts = Texts(zip);
|
|
|
|
foreach (var file in new[] { "actor.json", "outbox.json", "likes.json", "bookmarks.json", "following_accounts.csv", "blocked_accounts.csv",
|
|
"muted_accounts.csv", "blocked_domains.csv", "lists.csv", "bookmarks.csv", "privapub/archive.json", "privapub/pins.json",
|
|
"privapub/followed_tags.json", "privapub/filters.json", "privapub/notification_policy.json" })
|
|
Assert.True(texts.ContainsKey(file), $"{file} is missing");
|
|
using (var archive = new ZipArchive(new MemoryStream(zip)))
|
|
Assert.Contains(archive.Entries, e => e.FullName.StartsWith("media_attachments/files/", StringComparison.Ordinal));
|
|
var outbox = JsonNode.Parse(texts["outbox.json"])!["orderedItems"]!.AsArray();
|
|
Assert.Equal(3, outbox.Count);
|
|
Assert.Contains(outbox, i => i!["object"]!["attachment"]?[0]?["url"]?.GetValue<string>().StartsWith("/media_attachments/files/") == true);
|
|
Assert.Contains(bob.Persona.UserName, texts["blocked_accounts.csv"]);
|
|
Assert.Contains("spam.example", texts["blocked_domains.csv"]);
|
|
Assert.Contains(first.Text("uri"), texts["privapub/pins.json"]);
|
|
|
|
var avatar = await DB.Default.Find<Avatar>().MatchID(alice.Persona.Id).ExecuteFirstAsync(Token);
|
|
var root = await DB.Default.Find<RootUser>().MatchID(alice.Persona.Root.Id).ExecuteFirstAsync(Token);
|
|
var all = string.Concat(texts.Values);
|
|
Assert.DoesNotContain(avatar.PrivateKey.Split('\n')[1], all);
|
|
Assert.DoesNotContain(avatar.SigningKey, all);
|
|
Assert.DoesNotContain(root.UserName, all);
|
|
Assert.DoesNotContain(root.HashedPassword, all);
|
|
Assert.DoesNotContain(sibling.UserName, all);
|
|
Assert.DoesNotContain(alice.Token, all);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Back_home_an_archive_brings_nothing_back()
|
|
{
|
|
var alice = await _host.Mastodon("alice");
|
|
await alice.Status("kept");
|
|
var regretted = await alice.Status("regretted");
|
|
var zip = await Exported(alice.Persona);
|
|
await alice.Client.Delete($"/api/v1/statuses/{regretted.Text("id")}");
|
|
var before = await DB.Default.CountAsync<PostEntity>(p => p.GroupUserId == alice.Persona.Id, Token);
|
|
|
|
var (status, state) = await Imported(alice.Persona, zip, "posts", "pins");
|
|
|
|
Assert.Equal(HttpStatusCode.Accepted, status);
|
|
Assert.Equal("done", state["importState"]!.GetValue<string>());
|
|
Assert.True(state["importSameActor"]!.GetValue<bool>());
|
|
Assert.Equal((0, 1, 1), (Count(state, "posts"), Count(state, "posts already here"), Count(state, "posts deleted here")));
|
|
Assert.Equal(before, await DB.Default.CountAsync<PostEntity>(p => p.GroupUserId == alice.Persona.Id, Token));
|
|
Assert.NotNull((await DB.Default.Find<PostEntity>().MatchID(regretted.Text("id")).ExecuteFirstAsync(Token)).DeletedAt);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Into_another_persona_posts_arrive_once_and_nobody_is_told()
|
|
{
|
|
var alice = await _host.Mastodon("alice");
|
|
var follower = await _host.Mastodon("follower");
|
|
var picture = (await alice.Client.Exchange(new HttpRequestMessage(HttpMethod.Post, "/api/v2/media")
|
|
{
|
|
Content = MastodonHelpers.Multipart(("file", MastodonHelpers.JpegWithMetadata(64, 48), "image/jpeg", "dot.jpg"))
|
|
})).Ok().Body.Text("id");
|
|
var first = await alice.Status("the first words #moving", ("media_ids[]", picture));
|
|
await alice.Status("and a reply to them", ("in_reply_to_id", first.Text("id")));
|
|
await _host.Located(alice, "here and nowhere else");
|
|
var zip = await Exported(alice.Persona);
|
|
|
|
var carol = await _host.Mastodon("carol");
|
|
await follower.Client.Post($"/api/v1/accounts/{carol.Persona.Id}/follow");
|
|
var actor = $"/peasants/{carol.Persona.UserName}";
|
|
var jobs = await DB.Default.CountAsync<Job>(j => j.Kind == JobKind.Deliver && j.Payload.Contains(actor), Token);
|
|
var (_, state) = await Imported(carol.Persona, zip, "posts", "pins", "tags", "located");
|
|
|
|
Assert.Equal("done", state["importState"]!.GetValue<string>());
|
|
Assert.False(state["importSameActor"]!.GetValue<bool>());
|
|
Assert.Equal(2, Count(state, "posts"));
|
|
Assert.Equal(1, Count(state, "media"));
|
|
Assert.Equal(1, Count(state, "located posts"));
|
|
var copies = await DB.Default.Find<PostEntity>().Match(p => p.GroupUserId == carol.Persona.Id && p.Visibility != Models.Post.PostVisibility.LocalGeo).ExecuteAsync(Token);
|
|
Assert.Equal(2, copies.Count);
|
|
var copy = copies.Single(p => p.ImportedFromURI == first.Text("uri"));
|
|
Assert.StartsWith($"{PrivaPubHost.Base}/peasants/{carol.Persona.UserName}/scribbles/", copy.ObjectURI);
|
|
Assert.Equal(["moving"], copy.Tags);
|
|
Assert.Single(copy.Media);
|
|
Assert.Equal(carol.Persona.Id, (await DB.Default.Find<MediaAttachment>().MatchID(copy.Media[0].AttachmentId).ExecuteFirstAsync(Token)).OwnerAvatarId);
|
|
var reply = copies.Single(p => p.ID != copy.ID);
|
|
Assert.Equal(copy.ID, reply.AnsweringToPostId);
|
|
Assert.Equal(copy.ObjectURI + "/context", reply.ContextURI);
|
|
|
|
// nobody told: no delivery, no home, no notification; on the profile all the same
|
|
Assert.Equal(jobs, await DB.Default.CountAsync<Job>(j => j.Kind == JobKind.Deliver && j.Payload.Contains(actor), Token));
|
|
var ids = copies.Select(p => p.ID).ToList();
|
|
Assert.False(await DB.Default.Find<TimelineEntry>().Match(e => ids.Contains(e.PostId)).ExecuteAnyAsync(Token));
|
|
Assert.Contains((await carol.Client.Get($"/api/v1/accounts/{carol.Persona.Id}/statuses")).Ok().Body!.AsArray(), s => s!.Text("id") == copy.ID);
|
|
|
|
// twice changes nothing
|
|
var (_, again) = await Imported(carol.Persona, zip, "posts", "located");
|
|
Assert.Equal((0, 2, 1), (Count(again, "posts"), Count(again, "posts already here"), Count(again, "located posts already here")));
|
|
Assert.Equal(3, await DB.Default.CountAsync<PostEntity>(p => p.GroupUserId == carol.Persona.Id, Token));
|
|
}
|
|
|
|
// Mastodon's own archive: its posts, not its boosts nor its direct messages
|
|
[Fact]
|
|
public async Task A_mastodon_archive_is_read()
|
|
{
|
|
var dave = await _host.Mastodon("dave");
|
|
const string actor = "https://mastodon.example/users/dave";
|
|
const string Public = "https://www.w3.org/ns/activitystreams#Public";
|
|
var outbox = new JsonObject
|
|
{
|
|
["@context"] = "https://www.w3.org/ns/activitystreams",
|
|
["id"] = "outbox.json",
|
|
["type"] = "OrderedCollection",
|
|
["orderedItems"] = new JsonArray(
|
|
Create(actor, "1", "<p>hello from <b>Mastodon</b><script>alert(1)</script></p>", [Public], [actor + "/followers"], "2023-04-01T10:00:00Z"),
|
|
Create(actor, "2", "<p>only for one</p>", ["https://elsewhere.example/users/x"], [], "2023-04-02T10:00:00Z"),
|
|
new JsonObject { ["type"] = "Announce", ["actor"] = actor, ["object"] = "https://elsewhere.example/notes/9" })
|
|
};
|
|
var zip = Zip(("actor.json", new JsonObject { ["id"] = actor, ["type"] = "Person", ["followers"] = actor + "/followers" }.ToJsonString()),
|
|
("outbox.json", outbox.ToJsonString()));
|
|
|
|
var (_, state) = await Imported(dave.Persona, zip, "posts");
|
|
|
|
Assert.Equal("done", state["importState"]!.GetValue<string>());
|
|
Assert.Equal((1, 1, 1), (Count(state, "posts"), Count(state, "boosts skipped"), Count(state, "direct or circle posts skipped")));
|
|
var post = await DB.Default.Find<PostEntity>().Match(p => p.GroupUserId == dave.Persona.Id).ExecuteFirstAsync(Token);
|
|
Assert.Equal(actor + "/statuses/1", post.ImportedFromURI);
|
|
Assert.Equal(new DateTime(2023, 4, 1, 10, 0, 0, DateTimeKind.Utc), post.CreationDate);
|
|
Assert.DoesNotContain("<script", post.ContentHtml);
|
|
Assert.Contains("Mastodon", post.ContentHtml);
|
|
}
|
|
|
|
static JsonObject Create(string actor, string id, string content, string[] to, string[] cc, string published) => new()
|
|
{
|
|
["type"] = "Create",
|
|
["actor"] = actor,
|
|
["object"] = new JsonObject
|
|
{
|
|
["id"] = $"{actor}/statuses/{id}",
|
|
["type"] = "Note",
|
|
["attributedTo"] = actor,
|
|
["content"] = content,
|
|
["published"] = published,
|
|
["to"] = new JsonArray(to.Select(t => (JsonNode)t).ToArray()),
|
|
["cc"] = new JsonArray(cc.Select(c => (JsonNode)c).ToArray())
|
|
}
|
|
};
|
|
|
|
static byte[] Zip(params (string Name, string Text)[] files)
|
|
{
|
|
var stream = new MemoryStream();
|
|
using (var archive = new ZipArchive(stream, ZipArchiveMode.Create, leaveOpen: true))
|
|
foreach (var (name, text) in files)
|
|
{
|
|
using var entry = archive.CreateEntry(name).Open();
|
|
entry.Write(Encoding.UTF8.GetBytes(text));
|
|
}
|
|
return stream.ToArray();
|
|
}
|
|
|
|
[Theory]
|
|
[InlineData("escape")]
|
|
[InlineData("twice")]
|
|
[InlineData("bomb")]
|
|
[InlineData("link")]
|
|
[InlineData("junk")]
|
|
public async Task A_hostile_archive_is_refused(string how)
|
|
{
|
|
var erin = await _host.Mastodon("erin");
|
|
var actorJson = new JsonObject { ["id"] = "https://elsewhere.example/users/erin" }.ToJsonString();
|
|
var zip = how switch
|
|
{
|
|
"escape" => Zip(("actor.json", actorJson), ("../escape.txt", "x")),
|
|
"twice" => Zip(("actor.json", actorJson), ("outbox.json", "{}"), ("outbox.json", "{}")),
|
|
"bomb" => Zip(("actor.json", actorJson), ("outbox.json", new string(' ', 30 * 1024 * 1024))),
|
|
"link" => Link(actorJson),
|
|
_ => "not a zip at all"u8.ToArray()
|
|
};
|
|
|
|
var (status, _) = await Imported(erin.Persona, zip, "posts");
|
|
|
|
Assert.Equal(HttpStatusCode.UnprocessableEntity, status);
|
|
Assert.False(await DB.Default.Find<PostEntity>().Match(p => p.GroupUserId == erin.Persona.Id).ExecuteAnyAsync(Token));
|
|
}
|
|
|
|
// a symbolic link, as zip on Unix stores one
|
|
static byte[] Link(string actorJson)
|
|
{
|
|
var stream = new MemoryStream();
|
|
using (var archive = new ZipArchive(stream, ZipArchiveMode.Create, leaveOpen: true))
|
|
{
|
|
using (var actor = archive.CreateEntry("actor.json").Open())
|
|
actor.Write(Encoding.UTF8.GetBytes(actorJson));
|
|
var link = archive.CreateEntry("media_attachments/files/passwd");
|
|
link.ExternalAttributes = unchecked((int)(0xA1FFu << 16));
|
|
using var target = link.Open();
|
|
target.Write("/etc/passwd"u8);
|
|
}
|
|
return stream.ToArray();
|
|
}
|
|
|
|
[Fact]
|
|
public async Task An_outbox_item_too_large_stops_the_import()
|
|
{
|
|
var frank = await _host.Mastodon("frank");
|
|
const string actor = "https://elsewhere.example/users/frank";
|
|
// (random, so it is no compression bomb: that is refused before the import starts)
|
|
var words = Convert.ToBase64String(System.Security.Cryptography.RandomNumberGenerator.GetBytes(1536 * 1024));
|
|
var huge = Create(actor, "1", words, ["https://www.w3.org/ns/activitystreams#Public"], [], "2024-01-01T00:00:00Z");
|
|
var zip = Zip(("actor.json", new JsonObject { ["id"] = actor }.ToJsonString()),
|
|
("outbox.json", new JsonObject { ["orderedItems"] = new JsonArray(huge) }.ToJsonString()));
|
|
|
|
var (_, state) = await Imported(frank.Persona, zip, "posts");
|
|
|
|
Assert.Equal("failed", state["importState"]!.GetValue<string>());
|
|
Assert.Contains("larger than", state["importError"]!.GetValue<string>());
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Only_the_personas_own_root_and_never_a_banned_one()
|
|
{
|
|
var grace = await _host.Mastodon("grace");
|
|
var stranger = await _host.SignUp("stranger");
|
|
using (var other = _host.As(stranger.Jwt))
|
|
Assert.Equal(HttpStatusCode.NotFound, (await other.GetAsync(Base(grace.Persona), Token)).StatusCode);
|
|
Assert.Equal(HttpStatusCode.OK, (await _host.As(grace.Persona.Root.Jwt).GetAsync(Base(grace.Persona), Token)).StatusCode);
|
|
await DB.Default.Update<RootUser>().MatchID(grace.Persona.Root.Id).Modify(u => u.IsBanned, true).ExecuteAsync(Token);
|
|
Assert.NotEqual(HttpStatusCode.OK, (await _host.As(grace.Persona.Root.Jwt).GetAsync(Base(grace.Persona), Token)).StatusCode);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task An_import_asked_to_stop_stops()
|
|
{
|
|
var henry = await _host.Mastodon("henry");
|
|
const string actor = "https://elsewhere.example/users/henry";
|
|
var zip = Zip(("actor.json", new JsonObject { ["id"] = actor }.ToJsonString()),
|
|
("outbox.json", new JsonObject { ["orderedItems"] = new JsonArray(Create(actor, "1", "one", ["https://www.w3.org/ns/activitystreams#Public"], [], "2024-01-01T00:00:00Z")) }.ToJsonString()));
|
|
using var client = _host.As(henry.Persona.Root.Jwt);
|
|
var upload = (await (await client.PostAsync("/clientapi/persona/archive/uploads", null, Token)).Content.ReadFromJsonAsync<JsonObject>(Token))!["id"]!.GetValue<string>();
|
|
await client.PutAsync($"/clientapi/persona/archive/uploads/{upload}?offset=0", new ByteArrayContent(zip), Token);
|
|
Assert.Equal(HttpStatusCode.Accepted, (await client.PostAsJsonAsync(Base(henry.Persona) + "/import", new { uploadId = upload, parts = new[] { "posts" } }, Token)).StatusCode);
|
|
Assert.Equal(HttpStatusCode.Conflict, (await client.PostAsJsonAsync(Base(henry.Persona) + "/import", new { uploadId = upload, parts = new[] { "posts" } }, Token)).StatusCode);
|
|
Assert.Equal(HttpStatusCode.OK, (await client.PostAsync(Base(henry.Persona) + "/import/stop", null, Token)).StatusCode);
|
|
|
|
await _host.Run(j => j.Kind == JobKind.ImportArchive && j.Payload == henry.Persona.Id, Token);
|
|
|
|
Assert.Equal("stopped", (await Status(henry.Persona))["importState"]!.GetValue<string>());
|
|
Assert.False(await DB.Default.Find<PostEntity>().Match(p => p.GroupUserId == henry.Persona.Id).ExecuteAnyAsync(Token));
|
|
}
|
|
}
|
|
}
|