A restore on the live pasture: its own scenario, and two fixes it found

tools/pasture/scenarios/restore.sh backs the pasture up from the administrator's endpoint, then alice_restore deletes a
post and makes another, mastouser follows her, she blocks bob_restore and carol_restore is made; the restore, asked for
from the endpoint, keeps every protective act (19 checks). town.sh renew <peer> signs a peer's town accounts in again,
since a restore ends every session.

It found that a backup listed media files already missing when it was made, so verifying it failed and the restore was
refused: a manifest now lists only the files it holds (refusals are cut to five lines). And a local post made after the
backup now comes back as a deleted row, as a deletion leaves it, so it answers 410 and its id is never given again;
DeletedObject holds remote tombstones only.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01LsXgEaXee4GCU1hwYgPJXw

PrivaPub.Tests/Infrastructure/BackupTests.cs

@@ -158,7 +158,7 @@ namespace PrivaPub.Tests.Infrastructure
 			Assert.Equal("preview", await File.ReadAllTextAsync(Path.Combine(media, "2026", "10", "live-preview.jpg"), Token));
 			Assert.Equal("moving", await File.ReadAllTextAsync(Path.Combine(media, "2026", "10", "moving.jpg"), Token));
 			Assert.False(File.Exists(Path.Combine(media, "2026", "10", "trashed.jpg")));
-			Assert.Equal(["2026/10/gone.jpg", "2026/10/live-preview.jpg", "2026/10/live.jpg", "2026/10/moving.jpg"], backup.Manifest.Media.List);
+			Assert.Equal(["2026/10/live-preview.jpg", "2026/10/live.jpg", "2026/10/moving.jpg"], backup.Manifest.Media.List);
 			Assert.Equal((3, 1), (backup.Manifest.Media.Files, backup.Manifest.Media.Missing));
 
 			// a link, not a copy: the live file deleted, the backup's stays
@@ -168,8 +168,9 @@ namespace PrivaPub.Tests.Infrastructure
 			// db-only lists them and links none
 			var (listed, _) = await ServerBackup.Create(Context(), "pre-deploy", dbOnly: true, Token);
 			Assert.False(Directory.Exists(Path.Combine(Backups, listed.Id, "media")));
-			Assert.Equal(backup.Manifest.Media.List, listed.Manifest.Media.List);
+			Assert.Equal(["2026/10/live-preview.jpg", "2026/10/moving.jpg"], listed.Manifest.Media.List);
 			Assert.Empty(await ServerBackup.Verify(Backups, listed.Id, Token));
+			Assert.Empty(await ServerBackup.Verify(Backups, backup.Id, Token));
 		}
 
 		[Fact]

PrivaPub.Tests/Infrastructure/RestoreTests.cs

@@ -132,7 +132,9 @@ namespace PrivaPub.Tests.Infrastructure
 		{
 			// lost: what was merely made or changed since
 			Assert.Equal("first words", (await One("Post", Id(_editedPost)))["Text"].AsString);
-			Assert.Null(await One("Post", Id(_laterPost)));
+			var later = await One("Post", Id(_laterPost));
+			Assert.False(later["DeletedAt"].IsBsonNull);
+			Assert.True(later["Text"].IsBsonNull);
 			Assert.Empty(await All("PersonaList"));
 			Assert.Equal(16, (await One("_migration_history_", new BsonDocument()))["Number"].ToInt32());
 
@@ -142,9 +144,7 @@ namespace PrivaPub.Tests.Infrastructure
 			Assert.True(deleted["Text"].IsBsonNull);
 			Assert.Null(await One("TimelineEntry", new BsonDocument("PostId", _deletedPost.ToString())));
 			Assert.False((await One("MediaAttachment", Id(_deletedMedia)))["TrashedAt"].IsBsonNull);
-			Assert.NotNull(await One("DeletedObject", new BsonDocument("ObjectURI", Uri(_laterPost))));
-			var later = await One("MediaAttachment", Id(_laterMedia));
-			Assert.Equal("restore: made after the backup", later["TrashReason"].AsString);
+			Assert.Equal("restore: made after the backup", (await One("MediaAttachment", Id(_laterMedia)))["TrashReason"].AsString);
 			Assert.Equal(["https://elsewhere.example/users/came"], (await All("Follower")).Select(f => f["ActorURI"].AsString));
 			Assert.NotNull(await One("Block", new BsonDocument("TargetActorURI", "https://elsewhere.example/users/troll")));
 			Assert.NotNull(await One("DomainBlock", new BsonDocument("Domain", "evil.example")));

PrivaPub/Infrastructure/Backup/ProtectiveMerge.cs

@@ -148,11 +148,12 @@ namespace PrivaPub.Infrastructure.Backup
 				Builders<BsonDocument>.Update.Set("ConsecutiveFailures", 0).Set("UnavailableUntil", BsonNull.Value), cancellationToken: token);
 		}
 
-		// posts deleted since: deleted again, out of timelines and pins; local posts made since: 410
+		// posts deleted since: deleted again, out of timelines and pins; local posts made since: there as deleted, as a
+		// deletion leaves them, so they answer 410 and their ids and addresses are never given again
 		static async Task Posts(IMongoDatabase database, IEnumerable<BsonDocument> previous, RestoreReport report, CancellationToken token)
 		{
 			var posts = database.GetCollection<BsonDocument>("Post");
-			var gone = database.GetCollection<BsonDocument>("DeletedObject");
+			var now = DateTime.UtcNow;
 			foreach (var batch in previous.Where(p => IsSet(p, "DeletedAt") || IsLocal(p)).Chunk(500))
 			{
 				var ids = new BsonArray(batch.Select(p => p["_id"]));
@@ -162,12 +163,18 @@ namespace PrivaPub.Infrastructure.Backup
 				{
 					if (!restored.TryGetValue(post["_id"], out var mine))
 					{
-						if (IsLocal(post) && post.GetValue("ObjectURI", BsonNull.Value) is BsonString uri)
+						if (!IsLocal(post))
+							continue;
+						if (!IsSet(post, "DeletedAt"))
 						{
-							await gone.UpdateOneAsync(new BsonDocument("ObjectURI", uri), Builders<BsonDocument>.Update
-								.SetOnInsert("ObjectURI", uri).SetOnInsert("DeletedAt", DateTime.UtcNow), new UpdateOptions { IsUpsert = true }, token);
-							report.PostsGone++;
+							post["DeletedAt"] = now;
+							foreach (var field in new[] { "Text", "ContentHtml", "Title", "SpoilerText" })
+								post[field] = BsonNull.Value;
+							post["Media"] = new BsonArray();
+							post["Revisions"] = new BsonArray();
 						}
+						await posts.InsertOneAsync(post, cancellationToken: token);
+						report.PostsGone++;
 						continue;
 					}
 					if (!IsSet(post, "DeletedAt") || IsSet(mine, "DeletedAt"))

PrivaPub/Infrastructure/Backup/ServerBackup.cs

@@ -111,8 +111,7 @@ namespace PrivaPub.Infrastructure.Backup
 				}
 				manifest.Consistent = replica;
 
-				manifest.Media.List = [.. media];
-				if (!dbOnly)
+				// the files it holds (or, db-only, lists); a row whose file was already gone is only counted
 				foreach (var relative in media)
 				{
 					var source = new[] { context.MediaRoot, context.TrashRoot }.Select(root => Inside(root, relative)).FirstOrDefault(File.Exists);
@@ -121,7 +120,9 @@ namespace PrivaPub.Infrastructure.Backup
 						manifest.Media.Missing++;
 						continue;
 					}
+					if (!dbOnly)
 						HardLink.LinkOrCopy(source, Inside(Path.Combine(partial, "media"), relative));
+					manifest.Media.List.Add(relative);
 					manifest.Media.Files++;
 					manifest.Media.Bytes += new FileInfo(source).Length;
 				}

PrivaPub/Infrastructure/Backup/ServerRestore.cs

@@ -52,7 +52,7 @@ namespace PrivaPub.Infrastructure.Backup
 		{
 			var problems = await Check(context, id, token);
 			if (problems.Count > 0)
-				return string.Join("; ", problems);
+				return string.Join("; ", problems.Take(5)) + (problems.Count > 5 ? $"; and {problems.Count - 5} more" : string.Empty);
 			var waiting = RestoreMarker.Read(context.BackupsRoot);
 			if (waiting != default && waiting.Attempts < RestoreMarker.MaxAttempts)
 				return $"the restore of {waiting.Backup} is already {waiting.State}";

tools/pasture/scenarios/restore.sh

@@ -0,0 +1,89 @@
+# Backup and restore (owner decisions 2026-10-07), on the live pasture and through the administrator's endpoints: the
+# server is backed up, then life goes on: alice_restore deletes a post and makes another, mastouser follows her, she
+# blocks bob_restore, and carol_restore is made. The backup is restored: PrivaPub stops, restores it as it starts again,
+# and nothing protective is undone: the deleted post stays gone, the post made since answers as deleted, mastouser still
+# follows her, the block holds and carol's name stays taken. Every session ends, so the scenario signs in again, and at
+# the end the town's PrivaPub accounts too (town.sh renew privapub). Run it alone: it restores the whole server. Needs
+# the mastodon peer.
+M=https://mastodon.test:6443
+mcurl() { curl -sk --resolve mastodon.test:6443:127.0.0.1 "$@"; }
+. "$here/peers/mastodon.sh"
+m_rails() { podman exec pasture-mastodon bin/rails runner "$1" 2>/dev/null | tail -1; }
+p_mongo() { podman exec pasture-mongo mongosh --quiet PrivaPub --eval "$1"; }
+
+echo "restore"
+podman exec -w /app pasture-privapub /app/PrivaPub admin promote "$ROOT_USER" >/dev/null 2>&1 || true
+JWT=$(privapub_root)
+RH="Authorization: Bearer $JWT"
+AT=$(privapub_token alice_restore)
+BT=$(privapub_token bob_restore)
+AH="Authorization: Bearer $AT"
+[ -n "$AT" ] && [ -n "$BT" ] && ok "PrivaPub tokens for alice_restore and bob_restore" || { ko "PrivaPub tokens for alice_restore and bob_restore"; return 1; }
+run=$(date +%s)
+alice=$(curl -s -H "$AH" "$P/api/v1/accounts/verify_credentials")
+alice_id=$(echo "$alice" | j "print(d['id'])")
+alice_uri=$(echo "$alice" | j "print(d['url'])" | sed 's|/@|/peasants/|')
+bob_id=$(curl -s -H "Authorization: Bearer $BT" "$P/api/v1/accounts/verify_credentials" | j "print(d['id'])")
+regret=$(curl -s -X POST -H "$AH" "$P/api/v1/statuses" -d "status=a post regretted later $run&visibility=public")
+regret_id=$(echo "$regret" | j "print(d['id'])"); regret_uri=$(echo "$regret" | j "print(d['uri'])")
+# mastouser follows nobody here yet: the follow comes after the backup
+m_follows() { m_rails "puts Follow.exists?(account: Account.find_local(\"mastouser\"), target_account: Account.find_by(uri: \"$alice_uri\")) ? \"True\" : \"False\""; }
+alice_on_m=$(mcurl -H "Authorization: Bearer $(mastodon_token)" "$M/api/v2/search?q=@alice_restore@privapub.test&resolve=true&type=accounts" | j "print(d['accounts'][0]['id'])")
+if [ "$(m_follows)" = "True" ]; then
+	mcurl -o /dev/null -X POST -H "Authorization: Bearer $(mastodon_token)" "$M/api/v1/accounts/$alice_on_m/unfollow"
+	until_true 30 '[ "$(m_follows)" = "False" ]'
+fi
+curl -s -o /dev/null -X POST -H "$AH" "$P/api/v1/accounts/$bob_id/unblock"
+
+backups() { curl -s -H "$RH" "$P/clientapi/admin/backups"; }
+before=$(backups | j "print(' '.join(b['id'] for b in d['backups']))")
+[ "$(curl -s -o /dev/null -w '%{http_code}' -X POST -H "$RH" "$P/clientapi/admin/backups")" = "202" ] && ok "a backup is asked for" || ko "the backup was refused"
+backup=""
+newest() { backups | j "
+ids=[b['id'] for b in d['backups'] if b['id'] not in '$before'.split()]
+print(ids[0] if ids and d.get('running') is None else '')"; }
+until_true 300 '[ -n "$(newest)" ]' && backup=$(newest)
+[ -n "$backup" ] && ok "backed up as $backup" || { ko "the backup was never made"; return 1; }
+
+echo "  life goes on"
+curl -s -o /dev/null -X DELETE -H "$AH" "$P/api/v1/statuses/$regret_id"
+since=$(curl -s -X POST -H "$AH" "$P/api/v1/statuses" -d "status=made after the backup $run&visibility=public")
+since_uri=$(echo "$since" | j "print(d['uri'])")
+mcurl -o /dev/null -X POST -H "Authorization: Bearer $(mastodon_token)" "$M/api/v1/accounts/$alice_on_m/follow"
+until_true 45 '[ "$(m_follows)" = "True" ]' && ok "mastouser follows alice after the backup" || ko "mastouser never followed alice"
+until_true 30 '[ "$(curl -s -H "$AH" "$P/api/v1/accounts/$alice_id/followers" | j "print(any(a[\"acct\"]==\"mastouser@mastodon.test\" for a in d))")" = "True" ]' \
+	&& ok "PrivaPub has mastouser following alice" || ko "PrivaPub never had the follower"
+curl -s -o /dev/null -X POST -H "$AH" "$P/api/v1/accounts/$bob_id/block"
+carol="carol_restore_$run"
+curl -s -o /dev/null -X POST -H "$RH" -H 'Content-Type: application/json' "$P/clientapi/avatar/private/insert" \
+	-d "{\"userName\":\"$carol\",\"name\":\"carol\",\"biography\":\"made after the backup\"}"
+
+echo "  restored"
+[ "$(curl -s -o /dev/null -w '%{http_code}' -X POST -H "$RH" -H 'Content-Type: application/json' "$P/clientapi/admin/backups/$backup/restore" \
+	-d "{\"password\":\"$ROOT_PASS\",\"host\":\"elsewhere.test\"}")" = "422" ] && ok "a restore with the wrong host is refused" || ko "a restore with the wrong host was taken"
+[ "$(curl -s -o /dev/null -w '%{http_code}' -X POST -H "$RH" -H 'Content-Type: application/json' "$P/clientapi/admin/backups/$backup/restore" \
+	-d "{\"password\":\"$ROOT_PASS\",\"host\":\"privapub.test\"}")" = "202" ] && ok "the restore is asked for" || { ko "the restore was refused"; return 1; }
+# it stops within seconds, restores as it starts again, and every session ends: the old token is refused once it is back
+until_true 300 '[ "$(curl -s -o /dev/null -w "%{http_code}" -H "$RH" "$P/clientapi/admin/backups")" = "401" ]' \
+	&& ok "PrivaPub came back, and the administrator's session ended" || { ko "PrivaPub never came back from the restore"; podman logs --tail 30 pasture-privapub; return 1; }
+JWT=$(privapub_root); RH="Authorization: Bearer $JWT"
+[ "$(backups | j "print(d['lastRestore']['backup'])")" = "$backup" ] && ok "the last restore is $backup" || ko "the last restore is not $backup"
+[ "$(curl -s -o /dev/null -w '%{http_code}' -H "$AH" "$P/api/v1/accounts/verify_credentials")" = "401" ] && ok "alice's old token is refused" || ko "alice's old token still works"
+AT=$(privapub_token alice_restore); AH="Authorization: Bearer $AT"
+
+[ "$(curl -s -o /dev/null -w '%{http_code}' -H "$AH" "$P/api/v1/statuses/$regret_id")" = "404" ] && ok "the post deleted after the backup stays deleted" || ko "the deleted post came back"
+gone_unsigned "$regret_uri" && ok "its address answers as gone" || ko "its address answers $(pstatus "$regret_uri")"
+gone_unsigned "$since_uri" && ok "the post made after the backup answers as gone" || ko "the post made after the backup answers $(pstatus "$since_uri")"
+[ "$(curl -s -H "$AH" "$P/api/v1/accounts/$alice_id/followers" | j "print(any(a['acct']=='mastouser@mastodon.test' for a in d))")" = "True" ] \
+	&& ok "mastouser still follows alice" || ko "the follower gained after the backup was lost"
+[ "$(m_follows)" = "True" ] && ok "Mastodon still has the follow" || ko "Mastodon lost the follow"
+[ "$(curl -s -H "$AH" "$P/api/v1/accounts/relationships?id[]=$bob_id" | j "print(d[0]['blocking'])")" = "True" ] \
+	&& ok "alice still blocks bob_restore" || ko "the block made after the backup was undone"
+[ "$(curl -s -o /dev/null -w '%{http_code}' -X POST -H "$RH" -H 'Content-Type: application/json' "$P/clientapi/avatar/private/insert" \
+	-d "{\"userName\":\"$carol\",\"name\":\"carol\",\"biography\":\"again\"}")" != "200" ] && ok "carol's name stays taken" || ko "carol's name was free again"
+status=$(podman exec -w /app pasture-privapub /app/PrivaPub admin restore --status 2>/dev/null | grep -o "personas [^,;]*" | head -1)
+echo "$status" | grep -q "$carol" && ok "the report names carol among the personas made since" || ko "the report: $status"
+
+# the town signs in again; the record goes, so the followers' digests (FEP-8fcf, followsync) are not resting for two weeks
+"$here/town.sh" renew privapub >/dev/null && ok "the town's PrivaPub accounts signed in again" || ko "the town could not sign in again"
+p_mongo "db.RestoreRecord.deleteMany({})" >/dev/null

tools/pasture/town.sh

@@ -1,4 +1,5 @@
 #!/usr/bin/env bash
 # The town: a fake community seeded across every pasture peer, then checked for coherence (see town/town.py).
-# usage: tools/pasture/town.sh <command> [args]      commands: selftest, seed, check, report, backlog, gaps, drive, stored, seen
+# usage: tools/pasture/town.sh <command> [args]      commands: selftest, seed, check, report, backlog, gaps, drive, stored, seen,
+#                                                    renew <peer> (its accounts signed in again)
 exec python3 "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/town/town.py" "$@"

tools/pasture/town/town.py

@@ -5,6 +5,7 @@
   drive <peer> <user> <verb> '<json>'   one action as one town account, printed as JSON
   stored <peer> <uri>...        what a peer holds of each object (from its database, never fetching)
   seen <peer> <user> <uri>...   what one account can see of each object
+  renew <peer>...                its accounts signed in again (after a PrivaPub restore ends every session)
   plan <spec>                   the deterministic plan of a spec, printed as JSON lines
   seed <spec>                   replays a spec's plan against the pasture, writing out/<run>/ledger.jsonl
   check [run]                   sweeps every peer for what the ledger expects
@@ -47,6 +48,13 @@ def main(argv):
 		rows = driver(args[0]).stored(args[1:])
 		print(json.dumps({u: {k: v for k, v in r.__dict__.items() if k != "raw"} for u, r in rows.items()}, default=str))
 		return 0
+	if cmd == "renew":
+		# a peer's town accounts signed in again (a PrivaPub restore ends every session): new tokens, nothing else
+		for platform in args:
+			stored = state.sessions(platform)
+			state.remember(driver(platform).provision([s.account for s in stored]))
+			print(f"{platform}: {len(stored)} accounts signed in again")
+		return 0
 	if cmd == "seen":
 		s = state.session(args[0], args[1])
 		print(json.dumps(driver(args[0]).seen(s, args[2:])))