From bdc8be4508d839e048f4d4d9c233d3443aa8e5f1 Mon Sep 17 00:00:00 2001 From: thepra Date: Wed, 7 Oct 2026 12:24:23 +0200 Subject: [PATCH] A restore on the live pasture: its own scenario, and two fixes it found tools/pasture/scenarios/restore.sh backs the pasture up from the administrator's endpoint, then alice_restore deletes a post and makes another, mastouser follows her, she blocks bob_restore and carol_restore is made; the restore, asked for from the endpoint, keeps every protective act (19 checks). town.sh renew signs a peer's town accounts in again, since a restore ends every session. It found that a backup listed media files already missing when it was made, so verifying it failed and the restore was refused: a manifest now lists only the files it holds (refusals are cut to five lines). And a local post made after the backup now comes back as a deleted row, as a deletion leaves it, so it answers 410 and its id is never given again; DeletedObject holds remote tombstones only. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01LsXgEaXee4GCU1hwYgPJXw --- PrivaPub.Tests/Infrastructure/BackupTests.cs | 5 +- PrivaPub.Tests/Infrastructure/RestoreTests.cs | 8 +- .../Infrastructure/Backup/ProtectiveMerge.cs | 19 ++-- .../Infrastructure/Backup/ServerBackup.cs | 25 +++--- .../Infrastructure/Backup/ServerRestore.cs | 2 +- tools/pasture/scenarios/restore.sh | 89 +++++++++++++++++++ tools/pasture/town.sh | 3 +- tools/pasture/town/town.py | 8 ++ 8 files changed, 133 insertions(+), 26 deletions(-) create mode 100644 tools/pasture/scenarios/restore.sh diff --git a/PrivaPub.Tests/Infrastructure/BackupTests.cs b/PrivaPub.Tests/Infrastructure/BackupTests.cs index f779d28..b859825 100644 --- a/PrivaPub.Tests/Infrastructure/BackupTests.cs +++ b/PrivaPub.Tests/Infrastructure/BackupTests.cs @@ -158,7 +158,7 @@ namespace PrivaPub.Tests.Infrastructure Assert.Equal("preview", await File.ReadAllTextAsync(Path.Combine(media, "2026", "10", "live-preview.jpg"), Token)); Assert.Equal("moving", await File.ReadAllTextAsync(Path.Combine(media, "2026", "10", "moving.jpg"), Token)); Assert.False(File.Exists(Path.Combine(media, "2026", "10", "trashed.jpg"))); - Assert.Equal(["2026/10/gone.jpg", "2026/10/live-preview.jpg", "2026/10/live.jpg", "2026/10/moving.jpg"], backup.Manifest.Media.List); + Assert.Equal(["2026/10/live-preview.jpg", "2026/10/live.jpg", "2026/10/moving.jpg"], backup.Manifest.Media.List); Assert.Equal((3, 1), (backup.Manifest.Media.Files, backup.Manifest.Media.Missing)); // a link, not a copy: the live file deleted, the backup's stays @@ -168,8 +168,9 @@ namespace PrivaPub.Tests.Infrastructure // db-only lists them and links none var (listed, _) = await ServerBackup.Create(Context(), "pre-deploy", dbOnly: true, Token); Assert.False(Directory.Exists(Path.Combine(Backups, listed.Id, "media"))); - Assert.Equal(backup.Manifest.Media.List, listed.Manifest.Media.List); + Assert.Equal(["2026/10/live-preview.jpg", "2026/10/moving.jpg"], listed.Manifest.Media.List); Assert.Empty(await ServerBackup.Verify(Backups, listed.Id, Token)); + Assert.Empty(await ServerBackup.Verify(Backups, backup.Id, Token)); } [Fact] diff --git a/PrivaPub.Tests/Infrastructure/RestoreTests.cs b/PrivaPub.Tests/Infrastructure/RestoreTests.cs index 45f9c36..81df874 100644 --- a/PrivaPub.Tests/Infrastructure/RestoreTests.cs +++ b/PrivaPub.Tests/Infrastructure/RestoreTests.cs @@ -132,7 +132,9 @@ namespace PrivaPub.Tests.Infrastructure { // lost: what was merely made or changed since Assert.Equal("first words", (await One("Post", Id(_editedPost)))["Text"].AsString); - Assert.Null(await One("Post", Id(_laterPost))); + var later = await One("Post", Id(_laterPost)); + Assert.False(later["DeletedAt"].IsBsonNull); + Assert.True(later["Text"].IsBsonNull); Assert.Empty(await All("PersonaList")); Assert.Equal(16, (await One("_migration_history_", new BsonDocument()))["Number"].ToInt32()); @@ -142,9 +144,7 @@ namespace PrivaPub.Tests.Infrastructure Assert.True(deleted["Text"].IsBsonNull); Assert.Null(await One("TimelineEntry", new BsonDocument("PostId", _deletedPost.ToString()))); Assert.False((await One("MediaAttachment", Id(_deletedMedia)))["TrashedAt"].IsBsonNull); - Assert.NotNull(await One("DeletedObject", new BsonDocument("ObjectURI", Uri(_laterPost)))); - var later = await One("MediaAttachment", Id(_laterMedia)); - Assert.Equal("restore: made after the backup", later["TrashReason"].AsString); + Assert.Equal("restore: made after the backup", (await One("MediaAttachment", Id(_laterMedia)))["TrashReason"].AsString); Assert.Equal(["https://elsewhere.example/users/came"], (await All("Follower")).Select(f => f["ActorURI"].AsString)); Assert.NotNull(await One("Block", new BsonDocument("TargetActorURI", "https://elsewhere.example/users/troll"))); Assert.NotNull(await One("DomainBlock", new BsonDocument("Domain", "evil.example"))); diff --git a/PrivaPub/Infrastructure/Backup/ProtectiveMerge.cs b/PrivaPub/Infrastructure/Backup/ProtectiveMerge.cs index eabb587..a7d0a67 100644 --- a/PrivaPub/Infrastructure/Backup/ProtectiveMerge.cs +++ b/PrivaPub/Infrastructure/Backup/ProtectiveMerge.cs @@ -148,11 +148,12 @@ namespace PrivaPub.Infrastructure.Backup Builders.Update.Set("ConsecutiveFailures", 0).Set("UnavailableUntil", BsonNull.Value), cancellationToken: token); } - // posts deleted since: deleted again, out of timelines and pins; local posts made since: 410 + // posts deleted since: deleted again, out of timelines and pins; local posts made since: there as deleted, as a + // deletion leaves them, so they answer 410 and their ids and addresses are never given again static async Task Posts(IMongoDatabase database, IEnumerable previous, RestoreReport report, CancellationToken token) { var posts = database.GetCollection("Post"); - var gone = database.GetCollection("DeletedObject"); + var now = DateTime.UtcNow; foreach (var batch in previous.Where(p => IsSet(p, "DeletedAt") || IsLocal(p)).Chunk(500)) { var ids = new BsonArray(batch.Select(p => p["_id"])); @@ -162,12 +163,18 @@ namespace PrivaPub.Infrastructure.Backup { if (!restored.TryGetValue(post["_id"], out var mine)) { - if (IsLocal(post) && post.GetValue("ObjectURI", BsonNull.Value) is BsonString uri) + if (!IsLocal(post)) + continue; + if (!IsSet(post, "DeletedAt")) { - await gone.UpdateOneAsync(new BsonDocument("ObjectURI", uri), Builders.Update - .SetOnInsert("ObjectURI", uri).SetOnInsert("DeletedAt", DateTime.UtcNow), new UpdateOptions { IsUpsert = true }, token); - report.PostsGone++; + post["DeletedAt"] = now; + foreach (var field in new[] { "Text", "ContentHtml", "Title", "SpoilerText" }) + post[field] = BsonNull.Value; + post["Media"] = new BsonArray(); + post["Revisions"] = new BsonArray(); } + await posts.InsertOneAsync(post, cancellationToken: token); + report.PostsGone++; continue; } if (!IsSet(post, "DeletedAt") || IsSet(mine, "DeletedAt")) diff --git a/PrivaPub/Infrastructure/Backup/ServerBackup.cs b/PrivaPub/Infrastructure/Backup/ServerBackup.cs index de1d8b4..aa42087 100644 --- a/PrivaPub/Infrastructure/Backup/ServerBackup.cs +++ b/PrivaPub/Infrastructure/Backup/ServerBackup.cs @@ -111,20 +111,21 @@ namespace PrivaPub.Infrastructure.Backup } manifest.Consistent = replica; - manifest.Media.List = [.. media]; - if (!dbOnly) - foreach (var relative in media) + // the files it holds (or, db-only, lists); a row whose file was already gone is only counted + foreach (var relative in media) + { + var source = new[] { context.MediaRoot, context.TrashRoot }.Select(root => Inside(root, relative)).FirstOrDefault(File.Exists); + if (source == default) { - var source = new[] { context.MediaRoot, context.TrashRoot }.Select(root => Inside(root, relative)).FirstOrDefault(File.Exists); - if (source == default) - { - manifest.Media.Missing++; - continue; - } - HardLink.LinkOrCopy(source, Inside(Path.Combine(partial, "media"), relative)); - manifest.Media.Files++; - manifest.Media.Bytes += new FileInfo(source).Length; + manifest.Media.Missing++; + continue; } + if (!dbOnly) + HardLink.LinkOrCopy(source, Inside(Path.Combine(partial, "media"), relative)); + manifest.Media.List.Add(relative); + manifest.Media.Files++; + manifest.Media.Bytes += new FileInfo(source).Length; + } manifest.Write(partial); Directory.Move(partial, Path.Combine(context.BackupsRoot, id)); diff --git a/PrivaPub/Infrastructure/Backup/ServerRestore.cs b/PrivaPub/Infrastructure/Backup/ServerRestore.cs index d0e4121..3e50b46 100644 --- a/PrivaPub/Infrastructure/Backup/ServerRestore.cs +++ b/PrivaPub/Infrastructure/Backup/ServerRestore.cs @@ -52,7 +52,7 @@ namespace PrivaPub.Infrastructure.Backup { var problems = await Check(context, id, token); if (problems.Count > 0) - return string.Join("; ", problems); + return string.Join("; ", problems.Take(5)) + (problems.Count > 5 ? $"; and {problems.Count - 5} more" : string.Empty); var waiting = RestoreMarker.Read(context.BackupsRoot); if (waiting != default && waiting.Attempts < RestoreMarker.MaxAttempts) return $"the restore of {waiting.Backup} is already {waiting.State}"; diff --git a/tools/pasture/scenarios/restore.sh b/tools/pasture/scenarios/restore.sh new file mode 100644 index 0000000..cbffd43 --- /dev/null +++ b/tools/pasture/scenarios/restore.sh @@ -0,0 +1,89 @@ +# Backup and restore (owner decisions 2026-10-07), on the live pasture and through the administrator's endpoints: the +# server is backed up, then life goes on: alice_restore deletes a post and makes another, mastouser follows her, she +# blocks bob_restore, and carol_restore is made. The backup is restored: PrivaPub stops, restores it as it starts again, +# and nothing protective is undone: the deleted post stays gone, the post made since answers as deleted, mastouser still +# follows her, the block holds and carol's name stays taken. Every session ends, so the scenario signs in again, and at +# the end the town's PrivaPub accounts too (town.sh renew privapub). Run it alone: it restores the whole server. Needs +# the mastodon peer. +M=https://mastodon.test:6443 +mcurl() { curl -sk --resolve mastodon.test:6443:127.0.0.1 "$@"; } +. "$here/peers/mastodon.sh" +m_rails() { podman exec pasture-mastodon bin/rails runner "$1" 2>/dev/null | tail -1; } +p_mongo() { podman exec pasture-mongo mongosh --quiet PrivaPub --eval "$1"; } + +echo "restore" +podman exec -w /app pasture-privapub /app/PrivaPub admin promote "$ROOT_USER" >/dev/null 2>&1 || true +JWT=$(privapub_root) +RH="Authorization: Bearer $JWT" +AT=$(privapub_token alice_restore) +BT=$(privapub_token bob_restore) +AH="Authorization: Bearer $AT" +[ -n "$AT" ] && [ -n "$BT" ] && ok "PrivaPub tokens for alice_restore and bob_restore" || { ko "PrivaPub tokens for alice_restore and bob_restore"; return 1; } +run=$(date +%s) +alice=$(curl -s -H "$AH" "$P/api/v1/accounts/verify_credentials") +alice_id=$(echo "$alice" | j "print(d['id'])") +alice_uri=$(echo "$alice" | j "print(d['url'])" | sed 's|/@|/peasants/|') +bob_id=$(curl -s -H "Authorization: Bearer $BT" "$P/api/v1/accounts/verify_credentials" | j "print(d['id'])") +regret=$(curl -s -X POST -H "$AH" "$P/api/v1/statuses" -d "status=a post regretted later $run&visibility=public") +regret_id=$(echo "$regret" | j "print(d['id'])"); regret_uri=$(echo "$regret" | j "print(d['uri'])") +# mastouser follows nobody here yet: the follow comes after the backup +m_follows() { m_rails "puts Follow.exists?(account: Account.find_local(\"mastouser\"), target_account: Account.find_by(uri: \"$alice_uri\")) ? \"True\" : \"False\""; } +alice_on_m=$(mcurl -H "Authorization: Bearer $(mastodon_token)" "$M/api/v2/search?q=@alice_restore@privapub.test&resolve=true&type=accounts" | j "print(d['accounts'][0]['id'])") +if [ "$(m_follows)" = "True" ]; then + mcurl -o /dev/null -X POST -H "Authorization: Bearer $(mastodon_token)" "$M/api/v1/accounts/$alice_on_m/unfollow" + until_true 30 '[ "$(m_follows)" = "False" ]' +fi +curl -s -o /dev/null -X POST -H "$AH" "$P/api/v1/accounts/$bob_id/unblock" + +backups() { curl -s -H "$RH" "$P/clientapi/admin/backups"; } +before=$(backups | j "print(' '.join(b['id'] for b in d['backups']))") +[ "$(curl -s -o /dev/null -w '%{http_code}' -X POST -H "$RH" "$P/clientapi/admin/backups")" = "202" ] && ok "a backup is asked for" || ko "the backup was refused" +backup="" +newest() { backups | j " +ids=[b['id'] for b in d['backups'] if b['id'] not in '$before'.split()] +print(ids[0] if ids and d.get('running') is None else '')"; } +until_true 300 '[ -n "$(newest)" ]' && backup=$(newest) +[ -n "$backup" ] && ok "backed up as $backup" || { ko "the backup was never made"; return 1; } + +echo " life goes on" +curl -s -o /dev/null -X DELETE -H "$AH" "$P/api/v1/statuses/$regret_id" +since=$(curl -s -X POST -H "$AH" "$P/api/v1/statuses" -d "status=made after the backup $run&visibility=public") +since_uri=$(echo "$since" | j "print(d['uri'])") +mcurl -o /dev/null -X POST -H "Authorization: Bearer $(mastodon_token)" "$M/api/v1/accounts/$alice_on_m/follow" +until_true 45 '[ "$(m_follows)" = "True" ]' && ok "mastouser follows alice after the backup" || ko "mastouser never followed alice" +until_true 30 '[ "$(curl -s -H "$AH" "$P/api/v1/accounts/$alice_id/followers" | j "print(any(a[\"acct\"]==\"mastouser@mastodon.test\" for a in d))")" = "True" ]' \ + && ok "PrivaPub has mastouser following alice" || ko "PrivaPub never had the follower" +curl -s -o /dev/null -X POST -H "$AH" "$P/api/v1/accounts/$bob_id/block" +carol="carol_restore_$run" +curl -s -o /dev/null -X POST -H "$RH" -H 'Content-Type: application/json' "$P/clientapi/avatar/private/insert" \ + -d "{\"userName\":\"$carol\",\"name\":\"carol\",\"biography\":\"made after the backup\"}" + +echo " restored" +[ "$(curl -s -o /dev/null -w '%{http_code}' -X POST -H "$RH" -H 'Content-Type: application/json' "$P/clientapi/admin/backups/$backup/restore" \ + -d "{\"password\":\"$ROOT_PASS\",\"host\":\"elsewhere.test\"}")" = "422" ] && ok "a restore with the wrong host is refused" || ko "a restore with the wrong host was taken" +[ "$(curl -s -o /dev/null -w '%{http_code}' -X POST -H "$RH" -H 'Content-Type: application/json' "$P/clientapi/admin/backups/$backup/restore" \ + -d "{\"password\":\"$ROOT_PASS\",\"host\":\"privapub.test\"}")" = "202" ] && ok "the restore is asked for" || { ko "the restore was refused"; return 1; } +# it stops within seconds, restores as it starts again, and every session ends: the old token is refused once it is back +until_true 300 '[ "$(curl -s -o /dev/null -w "%{http_code}" -H "$RH" "$P/clientapi/admin/backups")" = "401" ]' \ + && ok "PrivaPub came back, and the administrator's session ended" || { ko "PrivaPub never came back from the restore"; podman logs --tail 30 pasture-privapub; return 1; } +JWT=$(privapub_root); RH="Authorization: Bearer $JWT" +[ "$(backups | j "print(d['lastRestore']['backup'])")" = "$backup" ] && ok "the last restore is $backup" || ko "the last restore is not $backup" +[ "$(curl -s -o /dev/null -w '%{http_code}' -H "$AH" "$P/api/v1/accounts/verify_credentials")" = "401" ] && ok "alice's old token is refused" || ko "alice's old token still works" +AT=$(privapub_token alice_restore); AH="Authorization: Bearer $AT" + +[ "$(curl -s -o /dev/null -w '%{http_code}' -H "$AH" "$P/api/v1/statuses/$regret_id")" = "404" ] && ok "the post deleted after the backup stays deleted" || ko "the deleted post came back" +gone_unsigned "$regret_uri" && ok "its address answers as gone" || ko "its address answers $(pstatus "$regret_uri")" +gone_unsigned "$since_uri" && ok "the post made after the backup answers as gone" || ko "the post made after the backup answers $(pstatus "$since_uri")" +[ "$(curl -s -H "$AH" "$P/api/v1/accounts/$alice_id/followers" | j "print(any(a['acct']=='mastouser@mastodon.test' for a in d))")" = "True" ] \ + && ok "mastouser still follows alice" || ko "the follower gained after the backup was lost" +[ "$(m_follows)" = "True" ] && ok "Mastodon still has the follow" || ko "Mastodon lost the follow" +[ "$(curl -s -H "$AH" "$P/api/v1/accounts/relationships?id[]=$bob_id" | j "print(d[0]['blocking'])")" = "True" ] \ + && ok "alice still blocks bob_restore" || ko "the block made after the backup was undone" +[ "$(curl -s -o /dev/null -w '%{http_code}' -X POST -H "$RH" -H 'Content-Type: application/json' "$P/clientapi/avatar/private/insert" \ + -d "{\"userName\":\"$carol\",\"name\":\"carol\",\"biography\":\"again\"}")" != "200" ] && ok "carol's name stays taken" || ko "carol's name was free again" +status=$(podman exec -w /app pasture-privapub /app/PrivaPub admin restore --status 2>/dev/null | grep -o "personas [^,;]*" | head -1) +echo "$status" | grep -q "$carol" && ok "the report names carol among the personas made since" || ko "the report: $status" + +# the town signs in again; the record goes, so the followers' digests (FEP-8fcf, followsync) are not resting for two weeks +"$here/town.sh" renew privapub >/dev/null && ok "the town's PrivaPub accounts signed in again" || ko "the town could not sign in again" +p_mongo "db.RestoreRecord.deleteMany({})" >/dev/null diff --git a/tools/pasture/town.sh b/tools/pasture/town.sh index d00083b..26ef9f6 100755 --- a/tools/pasture/town.sh +++ b/tools/pasture/town.sh @@ -1,4 +1,5 @@ #!/usr/bin/env bash # The town: a fake community seeded across every pasture peer, then checked for coherence (see town/town.py). -# usage: tools/pasture/town.sh [args] commands: selftest, seed, check, report, backlog, gaps, drive, stored, seen +# usage: tools/pasture/town.sh [args] commands: selftest, seed, check, report, backlog, gaps, drive, stored, seen, +# renew (its accounts signed in again) exec python3 "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/town/town.py" "$@" diff --git a/tools/pasture/town/town.py b/tools/pasture/town/town.py index 74da14c..1426993 100644 --- a/tools/pasture/town/town.py +++ b/tools/pasture/town/town.py @@ -5,6 +5,7 @@ drive '' one action as one town account, printed as JSON stored ... what a peer holds of each object (from its database, never fetching) seen ... what one account can see of each object + renew ... its accounts signed in again (after a PrivaPub restore ends every session) plan the deterministic plan of a spec, printed as JSON lines seed replays a spec's plan against the pasture, writing out//ledger.jsonl check [run] sweeps every peer for what the ledger expects @@ -47,6 +48,13 @@ def main(argv): rows = driver(args[0]).stored(args[1:]) print(json.dumps({u: {k: v for k, v in r.__dict__.items() if k != "raw"} for u, r in rows.items()}, default=str)) return 0 + if cmd == "renew": + # a peer's town accounts signed in again (a PrivaPub restore ends every session): new tokens, nothing else + for platform in args: + stored = state.sessions(platform) + state.remember(driver(platform).provision([s.account for s in stored])) + print(f"{platform}: {len(stored)} accounts signed in again") + return 0 if cmd == "seen": s = state.session(args[0], args[1]) print(json.dumps(driver(args[0]).seen(s, args[2:])))