A persona's archive: exported in Mastodon's layout, imported without telling anyone
A root exports one of its personas (a job) as Mastodon's account archive, so other servers' importers read it: the actor with its public key only, its own posts and boosts with their media, likes, bookmarks and Mastodon's CSV files, plus PrivaPub's filters, followed hashtags, notification policy, pins, scheduled and located posts; nothing of its root, its siblings, its keys or anyone's token. A ticket link downloads it, for a week. An archive (PrivaPub's or Mastodon's) uploaded in pieces is imported into a persona in a job, the parts the root picks, with progress and a stop. SafeArchive refuses links, escaping paths, duplicates, bombs and oversized items, and reads the outbox one item at a time. Imported posts are delivered to no one, put in no home and notify nobody, yet show on the profile, outbox, hashtags and search; back home a post keeps its id, from another actor it gets one of its date and ImportedFromURI, so importing twice changes nothing. Relationships go through the existing services; located, scheduled and likes only when asked; followers never. tools/pasture/scenarios/persona-archive.sh imports mastouser's real Mastodon archive (156 posts, 24 pictures) into a persona Mastodon follows: Mastodon receives none of it, and a second import changes nothing. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01LsXgEaXee4GCU1hwYgPJXw
This commit is contained in:
thepraandClaude Opus 5.5 committed
1 parent
bdc8be4508
commit
3d7d406834
18 files changed
+2377
-7
No files matched your search
@@ -0,0 +1,270 @@
|
||||
using System.IO.Compression;
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Nodes;
|
||||
|
||||
namespace PrivaPub.Domain.Portability
|
||||
{
|
||||
// An archive someone uploads, opened only if nothing in it can harm the server: no link, no path leaving it (.. or
|
||||
// absolute), no name twice, at most MaxEntries files and MaxBytes in all (and what the disk has), and no file
|
||||
// compressed more than MaxRatio to one. Every file is read through a stream that stops at its stated size, and the
|
||||
// outbox one item at a time, none larger than MaxItemBytes, so a large archive never sits in memory.
|
||||
public sealed class SafeArchive : IDisposable
|
||||
{
|
||||
public const int MaxEntries = 200_000;
|
||||
public const long MaxBytes = 64L * 1024 * 1024 * 1024;
|
||||
public const int MaxRatio = 100;
|
||||
public const int MaxItemBytes = 1024 * 1024;
|
||||
public const int MaxJsonBytes = 64 * 1024 * 1024;
|
||||
|
||||
readonly ZipArchive _zip;
|
||||
readonly Dictionary<string, ZipArchiveEntry> _entries;
|
||||
readonly string _root;//the folder the archive's files sit in, when it wraps them in one
|
||||
|
||||
SafeArchive(ZipArchive zip, Dictionary<string, ZipArchiveEntry> entries, string root)
|
||||
{
|
||||
_zip = zip;
|
||||
_entries = entries;
|
||||
_root = root;
|
||||
}
|
||||
|
||||
public static (SafeArchive Archive, string Error) Open(string path)
|
||||
{
|
||||
ZipArchive zip;
|
||||
try
|
||||
{
|
||||
zip = ZipFile.OpenRead(path);
|
||||
}
|
||||
catch (InvalidDataException)
|
||||
{
|
||||
return (default, "not a zip archive");
|
||||
}
|
||||
var entries = new Dictionary<string, ZipArchiveEntry>(StringComparer.Ordinal);
|
||||
var total = 0L;
|
||||
var free = new DriveInfo(Path.GetFullPath(path)).AvailableFreeSpace;
|
||||
string Refuse(string why)
|
||||
{
|
||||
zip.Dispose();
|
||||
return why;
|
||||
}
|
||||
if (zip.Entries.Count > MaxEntries)
|
||||
return (default, Refuse("too many files"));
|
||||
foreach (var entry in zip.Entries)
|
||||
{
|
||||
var name = entry.FullName;
|
||||
if (name.Contains('\\') || name.StartsWith('/') || name.Contains(':') || name.Split('/').Any(part => part is ".." or "."))
|
||||
return (default, Refuse($"{name}: a path that leaves the archive"));
|
||||
if (((entry.ExternalAttributes >> 16) & 0xF000) == 0xA000)
|
||||
return (default, Refuse($"{name}: a link"));
|
||||
if (name.EndsWith('/'))
|
||||
continue;
|
||||
if (!entries.TryAdd(name, entry))
|
||||
return (default, Refuse($"{name}: twice"));
|
||||
total += entry.Length;
|
||||
if (total > MaxBytes || total > free)
|
||||
return (default, Refuse("larger than the server can take"));
|
||||
if (entry.Length > 1024 * 1024 && entry.Length > (long)MaxRatio * Math.Max(1, entry.CompressedLength))
|
||||
return (default, Refuse($"{name}: compressed more than {MaxRatio} to one"));
|
||||
}
|
||||
// some archivers wrap everything in one folder
|
||||
var root = string.Empty;
|
||||
if (!entries.ContainsKey("actor.json") && entries.Keys.FirstOrDefault(k => k.EndsWith("/actor.json", StringComparison.Ordinal)) is { } nested
|
||||
&& nested.Count(c => c == '/') == 1)
|
||||
root = nested[..(nested.IndexOf('/') + 1)];
|
||||
return (new SafeArchive(zip, entries, root), default);
|
||||
}
|
||||
|
||||
public bool Has(string name) => _entries.ContainsKey(_root + name);
|
||||
|
||||
public long Length(string name) => _entries.TryGetValue(_root + name, out var entry) ? entry.Length : -1;
|
||||
|
||||
/// <summary>A file's contents, never more than its stated size.</summary>
|
||||
public Stream Read(string name) =>
|
||||
_entries.TryGetValue(_root + name, out var entry) ? new Bounded(entry.Open(), entry.Length) : default;
|
||||
|
||||
/// <summary>A JSON file, or null when it is missing, too large or not JSON.</summary>
|
||||
public JsonNode Json(string name)
|
||||
{
|
||||
if (!_entries.TryGetValue(_root + name, out var entry) || entry.Length > MaxJsonBytes)
|
||||
return default;
|
||||
try
|
||||
{
|
||||
using var stream = Read(name);
|
||||
return JsonNode.Parse(stream);
|
||||
}
|
||||
catch (JsonException)
|
||||
{
|
||||
return default;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>The lines of a text file (a CSV), none when it is missing or too large.</summary>
|
||||
public List<string> Lines(string name)
|
||||
{
|
||||
if (!_entries.TryGetValue(_root + name, out var entry) || entry.Length > MaxJsonBytes)
|
||||
return [];
|
||||
using var reader = new StreamReader(Read(name));
|
||||
var lines = new List<string>();
|
||||
while (reader.ReadLine() is { } line)
|
||||
if (line.Length > 0)
|
||||
lines.Add(line);
|
||||
return lines;
|
||||
}
|
||||
|
||||
/// <summary>outbox.json's orderedItems, one object at a time; an item larger than MaxItemBytes stops it.</summary>
|
||||
public async IAsyncEnumerable<JsonObject> Outbox([System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken token)
|
||||
{
|
||||
await using var stream = Read("outbox.json");
|
||||
if (stream == default)
|
||||
yield break;
|
||||
var buffer = new byte[64 * 1024];
|
||||
var filled = 0;
|
||||
var final = false;
|
||||
var state = new JsonReaderState();
|
||||
var phase = Phase.Seeking;
|
||||
while (phase != Phase.Done)
|
||||
{
|
||||
if (!final)
|
||||
{
|
||||
if (filled == buffer.Length)
|
||||
{
|
||||
if (buffer.Length > MaxItemBytes + 64 * 1024)
|
||||
throw new InvalidDataException($"an outbox item larger than {MaxItemBytes / 1024} KiB");
|
||||
Array.Resize(ref buffer, buffer.Length * 2);
|
||||
}
|
||||
var read = await stream.ReadAsync(buffer.AsMemory(filled), token);
|
||||
filled += read;
|
||||
final = read == 0;
|
||||
}
|
||||
var (items, consumed, next, after) = Scan(buffer.AsSpan(0, filled), final, state, phase);
|
||||
foreach (var item in items)
|
||||
yield return item;
|
||||
(state, phase) = (next, after);
|
||||
Buffer.BlockCopy(buffer, consumed, buffer, 0, filled - consumed);
|
||||
filled -= consumed;
|
||||
if (final && consumed == 0 && items.Count == 0)
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
enum Phase { Seeking, Array, Done }
|
||||
|
||||
static (List<JsonObject> Items, int Consumed, JsonReaderState State, Phase Phase) Scan(ReadOnlySpan<byte> data, bool final, JsonReaderState state, Phase phase)
|
||||
{
|
||||
var items = new List<JsonObject>();
|
||||
var reader = new Utf8JsonReader(data, final, state);
|
||||
while (phase != Phase.Done)
|
||||
{
|
||||
var mark = reader;
|
||||
if (!reader.Read())
|
||||
{
|
||||
reader = mark;
|
||||
break;
|
||||
}
|
||||
if (phase == Phase.Seeking)
|
||||
{
|
||||
if (reader.TokenType == JsonTokenType.PropertyName && reader.CurrentDepth == 1 && reader.ValueTextEquals("orderedItems"))
|
||||
{
|
||||
var before = reader;
|
||||
if (!reader.Read())
|
||||
{
|
||||
reader = mark;
|
||||
break;
|
||||
}
|
||||
if (reader.TokenType == JsonTokenType.StartArray)
|
||||
phase = Phase.Array;
|
||||
else
|
||||
{
|
||||
reader = before;
|
||||
if (!reader.TrySkip())
|
||||
{
|
||||
reader = mark;
|
||||
break;
|
||||
}
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if (reader.TokenType is JsonTokenType.StartObject or JsonTokenType.StartArray && reader.CurrentDepth >= 1 && !reader.TrySkip())
|
||||
{
|
||||
reader = mark;
|
||||
break;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if (reader.TokenType == JsonTokenType.EndArray)
|
||||
{
|
||||
phase = Phase.Done;
|
||||
break;
|
||||
}
|
||||
if (reader.TokenType != JsonTokenType.StartObject)
|
||||
{
|
||||
if (reader.TokenType == JsonTokenType.StartArray && !reader.TrySkip())
|
||||
{
|
||||
reader = mark;
|
||||
break;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
var start = (int)reader.TokenStartIndex;
|
||||
if (!reader.TrySkip())
|
||||
{
|
||||
reader = mark;
|
||||
break;
|
||||
}
|
||||
var length = (int)reader.BytesConsumed - start;
|
||||
if (length > MaxItemBytes)
|
||||
throw new InvalidDataException($"an outbox item larger than {MaxItemBytes / 1024} KiB");
|
||||
if (JsonNode.Parse(data.Slice(start, length)) is JsonObject item)
|
||||
items.Add(item);
|
||||
}
|
||||
return (items, (int)reader.BytesConsumed, reader.CurrentState, phase);
|
||||
}
|
||||
|
||||
public void Dispose() => _zip.Dispose();
|
||||
|
||||
// a stream that ends at the size the archive states, whatever the compressed data says
|
||||
sealed class Bounded(Stream inner, long length) : Stream
|
||||
{
|
||||
long _left = length;
|
||||
|
||||
public override int Read(byte[] buffer, int offset, int count) => Read(buffer.AsSpan(offset, count));
|
||||
|
||||
public override int Read(Span<byte> buffer)
|
||||
{
|
||||
if (_left <= 0)
|
||||
return 0;
|
||||
var read = inner.Read(buffer[..(int)Math.Min(buffer.Length, _left)]);
|
||||
_left -= read;
|
||||
return read;
|
||||
}
|
||||
|
||||
public override async ValueTask<int> ReadAsync(Memory<byte> buffer, CancellationToken token = default)
|
||||
{
|
||||
if (_left <= 0)
|
||||
return 0;
|
||||
var read = await inner.ReadAsync(buffer[..(int)Math.Min(buffer.Length, _left)], token);
|
||||
_left -= read;
|
||||
return read;
|
||||
}
|
||||
|
||||
public override Task<int> ReadAsync(byte[] buffer, int offset, int count, CancellationToken token) =>
|
||||
ReadAsync(buffer.AsMemory(offset, count), token).AsTask();
|
||||
|
||||
public override bool CanRead => true;
|
||||
public override bool CanSeek => false;
|
||||
public override bool CanWrite => false;
|
||||
public override long Length => length;
|
||||
public override long Position { get => length - _left; set => throw new NotSupportedException(); }
|
||||
public override void Flush() { }
|
||||
public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException();
|
||||
public override void SetLength(long value) => throw new NotSupportedException();
|
||||
public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException();
|
||||
|
||||
protected override void Dispose(bool disposing)
|
||||
{
|
||||
if (disposing)
|
||||
inner.Dispose();
|
||||
base.Dispose(disposing);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user