From 5f150b7a591a527580d9d4cbc4b8c7c2b5141cad Mon Sep 17 00:00:00 2001 From: thepra Date: Mon, 5 Oct 2026 22:03:35 +0200 Subject: [PATCH] Pasture: Smithereen 1.0.3, and failed server descriptions retried Smithereen joins the pasture (tools/pasture/peers/smithereen.sh): its image on the shared MySQL, with imgproxy and a file server behind Caddy, a JDK trust store with the pasture's CA, accounts from its signup form, and a password grant for a local application. scenarios/smithereen.sh drives its VKontakte-like API: friends as mutual follows, wall posts, comments, likes, reposts (quotes there), polls, edits, deletions, private messages, the unfollow and statistics, 30 checks. A post on someone else's wall never reaches PrivaPub, since Smithereen sends it only to servers whose actors publish a wall: G-0009, waiting for the owner. PrivaPub: a server description that failed (Smithereen serves no NodeInfo until it has a description) frees its week, so the server's next arrival asks again instead of a week later. The shared Postgres takes 400 connections: at 100 the village seed ran it dry (Sharkey's API answered 500, Misskey dropped deliveries). The seeder records a follow that stands from an earlier seed when the step itself fails, so the checker no longer expects that follower to see nothing. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01LsXgEaXee4GCU1hwYgPJXw --- CLAUDE.md | 11 +- FEDERATION.md | 1 + PrivaPub.Tests/Statistics/DescribeTests.cs | 24 ++++ .../Federation/Objects/InstanceDescriber.cs | 5 +- docs/INTEROP.md | 25 ++++ docs/ROADMAP.md | 4 + tools/pasture/Caddyfile | 13 ++ tools/pasture/peers/shared.sh | 6 +- tools/pasture/peers/smithereen.sh | 125 +++++++++++++++++ tools/pasture/scenarios/smithereen.sh | 129 ++++++++++++++++++ tools/pasture/town/gaps.json | 14 ++ tools/pasture/town/seed.py | 15 +- 12 files changed, 367 insertions(+), 5 deletions(-) create mode 100644 tools/pasture/peers/smithereen.sh create mode 100644 tools/pasture/scenarios/smithereen.sh diff --git a/CLAUDE.md b/CLAUDE.md index 7663c1b..07a60fc 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -391,7 +391,8 @@ group www-data and reaches the private mongod; `sudo -u www-data` works too. same rule writes `canQuote`, answers `QuoteRequest`s (`QuoteService.ReceiveRequest`) and fills `quote_approval`, so they cannot disagree. A persona quoting another persona gets a parrot-licence too, so other servers see an approved quote. - **A server is described on arrival, never on read.** The first record from a host enqueues `DescribeInstance` (its - NodeInfo, at most once a week, into `RemoteInstance`), so opening the details view tells nobody anything. + NodeInfo, at most once a week, into `RemoteInstance`), so opening the details view tells nobody anything. A + description that failed frees its week, so the server's next arrival (an hour on at the soonest) asks again. - **Post ids are the timeline order, so they follow arrival, not `published`.** `PrivacyIds.Arrived` gives a remote post published within the last hour (or in the future) a fresh `ObjectId`, which sorts after every post already stored, and only backfill keeps a `published`-derived id. With `published` ids a reply arriving in the same second could sort @@ -558,6 +559,14 @@ tools/pasture/run.sh down # removes e `connection`) and its API is v3 (`/api/v3`, `sort=New`, `resolve_object` answering views). It takes private messages only as `ChatMessage`, which PrivaPub sends it, a first message too (invariant 17). `scenarios/lemmy19.sh`, 30 checks. +- **Smithereen (1.0.3):** its image on the shared MySQL (database `smithereen`, its schema from the image's commit), + with imgproxy and a file server behind Caddy as `smithereen.test` (`/i` and `/s`), trusting the CA through a JDK + store with it added (`JAVA_TOOL_OPTIONS`). MySQL takes its stored functions only with + `log_bin_trust_function_creators`, and a first start that failed there leaves the database without triggers (drop it + and start again). Accounts come from its signup form (opened, with a short description, which its NodeInfo needs) and + are renamed in its database; its API is VKontakte's (`/api/method/?v=1.0`), with a password grant for a local + application the peer script inserts. `scenarios/smithereen.sh`, 30 checks and one known gap (G-0009, posts on + someone else's wall). - **Load (`load.sh`, needs the `flood` peer):** `flood` (`flood/flood.cs`, published once into `.flood`) answers as twenty fake servers and sends signed activities at a set rate; `load.sh --rate=N --seconds=N` measures the answers, the queue's wait and processing times, its drain, and a persona's home timeline meanwhile, and keeps each run in diff --git a/FEDERATION.md b/FEDERATION.md index 86263ef..cf4ce1e 100644 --- a/FEDERATION.md +++ b/FEDERATION.md @@ -32,6 +32,7 @@ and every run starting clean, with signed fetches required (as privapub.thepra.d - **PieFed 1.7.17** - **Mbin 1.10.1** - **NodeBB 4.16.1** +- **Smithereen 1.0.3** - in the town only (a seeded community checked server by server): **Hollo 0.9.19**, **Iceshrimp.NET 2026.1.2-beta**, **Pleroma 2.10.2** diff --git a/PrivaPub.Tests/Statistics/DescribeTests.cs b/PrivaPub.Tests/Statistics/DescribeTests.cs index 95ab23c..a883c34 100644 --- a/PrivaPub.Tests/Statistics/DescribeTests.cs +++ b/PrivaPub.Tests/Statistics/DescribeTests.cs @@ -206,6 +206,30 @@ namespace PrivaPub.Tests.Statistics Assert.False((await DB.Default.Find().Match(s => s.Host == gone).ExecuteSingleAsync(token)).Reachable); } + // a server whose NodeInfo failed is asked again on its next arrival, not a week later + [Fact] + public async Task A_failed_description_frees_its_weeks_key_and_a_good_one_keeps_it() + { + var token = TestContext.Current.CancellationToken; + var broken = $"broken{Guid.NewGuid():N}.example"; + var fine = $"fine{Guid.NewGuid():N}.example"; + ServeServer("/" + fine, "smithereen", v2: false); + var queue = new JobQueue(); + async Task Run(string host) + { + var key = InstanceDescriber.DedupeKey(host, DateTime.UtcNow); + Assert.True(await queue.Enqueue(JobKind.DescribeInstance, host, host, key, token)); + var job = await DB.Default.Find().Match(j => j.DedupeKey == key).ExecuteSingleAsync(token); + await new PeerDescriber(Peer.Http(), default, default, _peer.A + "/" + host).Handle(job, token); + } + + await Run(broken); + await Run(fine); + + Assert.True(await queue.Enqueue(JobKind.DescribeInstance, broken, broken, InstanceDescriber.DedupeKey(broken, DateTime.UtcNow), token)); + Assert.False(await queue.Enqueue(JobKind.DescribeInstance, fine, fine, InstanceDescriber.DedupeKey(fine, DateTime.UtcNow), token)); + } + [Fact] public async Task A_crawled_server_never_downgrades_a_touched_one_and_robots_are_obeyed() { diff --git a/PrivaPub/Federation/Objects/InstanceDescriber.cs b/PrivaPub/Federation/Objects/InstanceDescriber.cs index eb5e04c..8fda365 100644 --- a/PrivaPub/Federation/Objects/InstanceDescriber.cs +++ b/PrivaPub/Federation/Objects/InstanceDescriber.cs @@ -42,7 +42,10 @@ namespace PrivaPub.Federation.Objects public async Task Handle(Job job, CancellationToken token) { - await Describe(job.Payload, crawled: false, default, token); + // a description that failed (Smithereen serves no NodeInfo until its admin writes a description) frees its + // week's key, so the server's next arrival, an hour on at the soonest, asks again + if (!await Describe(job.Payload, crawled: false, default, token) && !string.IsNullOrEmpty(job.DedupeKey)) + await DB.Default.Update().MatchID(job.ID).Modify(j => j.DedupeKey, $"{job.DedupeKey}|failed|{job.ID}").ExecuteAsync(token); return JobOutcome.Done; } diff --git a/docs/INTEROP.md b/docs/INTEROP.md index 4ac1a77..f48660a 100644 --- a/docs/INTEROP.md +++ b/docs/INTEROP.md @@ -753,6 +753,31 @@ PeerTube's own instance account announces each new video too, which we drop: nob follow only PeerTube-like channels and accounts, never a persona. It checks the `Host` header against its own name, without a port, before it gives out its OAuth client. +### Smithereen 1.0.3 + +- **Walls:** a post is a `Note`; one written on someone else's wall carries the wall (`sm:wall`, FEP-400e) as its + `target`, and the wall's owner tells its followers with `Add{Note}`. Smithereen sends both only to servers whose + actors have a wall (`Server.Feature.WALL_POSTS`, learnt from an actor's `sm:wall`), so PrivaPub never sees a post on + someone else's wall (G-0009: a persona's wall waits for the owner). Comments are replies addressed to the post's + author, never to followers. +- **Friends are mutual follows.** A request is an `Offer{Follow}` sent only to actors with + `sm:supportsFriendRequests`; anyone else gets a plain `Follow`, so a persona is followed, and following back makes + them friends there. +- **A repost is a quote:** a `QuoteRequest` to the quoted author, then a `Create{Note}` quoting the post. Its API + answers 500 when `wall.repost` has no `message`, even an empty one. +- **Private messages** are `Note`s to their recipients, in threads. +- **Polls:** a vote is a `Note` with `name` and `inReplyTo`, as Mastodon's. +- **Running it:** its NodeInfo throws until the server has a short description (`Objects.requireNonNull` where + `requireNonNullElse` was meant); its HTTP client refuses private addresses; its schema updater makes stored + functions, which MySQL takes only with `log_bin_trust_function_creators`. A database whose first update failed is + left without its triggers, and the server then answers 404 to activities about anything it knows. +- **Pasture evidence (2026-10-05, Smithereen 1.0.3, `tools/pasture/scenarios/smithereen.sh`):** 30 checks pass and one + gap is expected (G-0009): smuser follows alice and she follows back, which makes them friends there; wall posts both + ways; comments both ways; likes both ways; smuser's repost is a quote alice's policy lets in, and alice's boost is a + repost there; smuser's poll and alice's vote; edits and deletions both ways; private messages both ways; the + unfollow; statistics. PrivaPub changed for it: a server description that failed is asked again on the server's next + arrival instead of a week later. + ### Loops (1.0.0-beta.14) and Pixelfed (0.14.4) - **Loops:** diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index 642d720..7580463 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -278,6 +278,10 @@ and circles (see Owner decisions). | An account a persona follows moves | **Move the follow, as Mastodon does.** After a verified `Move` the persona follows the new account (a Follow to its server) and unfollows the old, in the same lists; a mute or a block of the old account carries over. | | Should PrivaPub pass on replies to a persona's posts? | **Yes, "the fediverse is broken without".** A public or unlisted reply that someone on another server writes to a persona's post goes on to the persona's followers, as Mastodon forwards it: the activity as its author signed it, never to the replier's own server, never for a post that is local-only, in a group or not shown to followers. Its edits and deletion follow the same way. | +**Waiting for the owner:** a persona's wall (G-0009). Smithereen sends a post written on someone else's wall only to +servers whose actors publish a wall (`sm:wall`, FEP-400e). Publishing one would also let Smithereen's users write on a +persona's wall, and PrivaPub would then host their posts and announce them with `Add{Note}`. + ## Libraries (researched; no maintained .NET ActivityPub library exists, so Letterbook and Iceshrimp.NET both wrote their own) | Area | Choice | diff --git a/tools/pasture/Caddyfile b/tools/pasture/Caddyfile index 546e5ee..16eac0b 100644 --- a/tools/pasture/Caddyfile +++ b/tools/pasture/Caddyfile @@ -108,6 +108,19 @@ nodebb.test { reverse_proxy pasture-nodebb:4567 } +smithereen.test { + tls internal + handle /i/* { + reverse_proxy pasture-smithereen-imgproxy:8080 + } + handle_path /s/* { + reverse_proxy pasture-smithereen-files:80 + } + handle { + reverse_proxy pasture-smithereen:4567 + } +} + lemmy19.test { tls internal reverse_proxy pasture-lemmy19:8536 diff --git a/tools/pasture/peers/shared.sh b/tools/pasture/peers/shared.sh index e1b1857..14c28a6 100644 --- a/tools/pasture/peers/shared.sh +++ b/tools/pasture/peers/shared.sh @@ -1,9 +1,11 @@ # Services several peers share: one Postgres and one MySQL (each peer gets its own database), and one Redis (each its own -# db number). +# db number). A dozen peers keep their pools on the one Postgres, and Postgres's default 100 connections ran out while +# the town seeded (Sharkey's API answered 500, Misskey's queue dropped deliveries): it takes 400. shared_postgres_up() { podman container exists pasture-postgres && return 0 podman run -d --replace --name pasture-postgres --network $net --network-alias postgres \ - -e POSTGRES_USER=pasture -e POSTGRES_PASSWORD=pasture -e POSTGRES_DB=pasture docker.io/library/postgres:17-alpine >/dev/null + -e POSTGRES_USER=pasture -e POSTGRES_PASSWORD=pasture -e POSTGRES_DB=pasture docker.io/library/postgres:17-alpine \ + -c max_connections=400 >/dev/null for _ in $(seq 1 60); do podman exec pasture-postgres pg_isready -U pasture >/dev/null 2>&1 && return 0; sleep 1; done echo "postgres did not start" >&2; return 1 } diff --git a/tools/pasture/peers/smithereen.sh b/tools/pasture/peers/smithereen.sh new file mode 100644 index 0000000..892ed50 --- /dev/null +++ b/tools/pasture/peers/smithereen.sh @@ -0,0 +1,125 @@ +# Smithereen 1.0.3: a VKontakte-like network in Java. Walls (posts on someone else's wall carry a `target`, and the wall's +# owner announces them with Add{Note}), friends as mutual follows (requests as Offer{Follow}), groups and events, photo +# albums, threaded comments, polls, private messages. On the shared MySQL (database smithereen), with imgproxy for its +# pictures and a file server for its uploads, both behind Caddy as smithereen.test (/i and /s). Its API is VKontakte's +# (/api/method/?v=1.0) and takes a password grant for an Application its OAuth knows: the pasture makes a local +# one in its database. Its HTTP client refuses private addresses, which the pasture's subnet is not. +SMITHEREEN_IMAGE=${SMITHEREEN_IMAGE:-docker.io/grishkaa/smithereen@sha256:fcef096b78de21cc98936c2cebca5b5217d10976e59886be9e68d78f650581cf} +# the commit that image was built from (its version.properties), whose schema.sql makes the database +SMITHEREEN_COMMIT=d60c6e4c5abf2682d203f948770e6f63ce3bdeb5 +SMITHEREEN_IMGPROXY_IMAGE=${SMITHEREEN_IMGPROXY_IMAGE:-docker.io/darthsim/imgproxy:v3.31.4} +SMITHEREEN_PASSWORD=Smithereen-Pasture-Pass-1 +# imgproxy's URL signing pair, shared with Smithereen's config: not a secret in the pasture +SMITHEREEN_IMGPROXY_KEY=736d697468657265656e2d706173747572652d696d6770726f78792d6b65792d +SMITHEREEN_IMGPROXY_SALT=736d697468657265656e2d706173747572652d696d6770726f78792d73616c74 +. "$here/peers/shared.sh" + +smithereen_sql() { podman exec -i pasture-mysql mysql -uroot -ppasture -N smithereen "$@" 2>/dev/null; } + +smithereen_up() { + shared_mysql_up + # its schema updater makes stored functions, which MySQL refuses to a user without SUPER while binary logging is on + podman exec pasture-mysql mysql -uroot -ppasture -e "set persist log_bin_trust_function_creators = 1" 2>/dev/null + local st="$here/.state/smithereen" + mkdir -p "$st" + [ -s "$st/schema.sql" ] || curl -fsSL "https://raw.githubusercontent.com/grishka/Smithereen/$SMITHEREEN_COMMIT/schema.sql" -o "$st/schema.sql" \ + || { echo "smithereen: no schema.sql" >&2; return 1; } + if [ "$(echo "select count(*) from information_schema.tables where table_schema='smithereen' and table_name='users'" | smithereen_sql)" != "1" ]; then + mysql_db smithereen + smithereen_sql < "$st/schema.sql" + # anyone may sign up, without a captcha or a confirmation mail: the pasture makes its accounts through the form. Its + # NodeInfo fails (a NullPointerException) until the server has a short description + echo "insert into config (\`key\`, value) values ('SignupMode', 'OPEN'), ('SignupFormUseCaptcha', '0'), ('SignupConfirmEmail', '0'), + ('ServerDisplayName', 'Smithereen pasture'), ('ServerShortDescription', 'The pasture''s Smithereen') + on duplicate key update value=values(value)" | smithereen_sql + fi + cat > "$st/config.properties" <<-EOF + db.host=mysql + db.name=smithereen + db.user=pasture + db.password=pasture + server.ip=0.0.0.0 + domain=smithereen.test + upload.path=/uploads/uploads + upload.url_path=/s/uploads + media_cache.path=/uploads/media_cache + media_cache.url_path=/s/media_cache + media_cache.max_size=1G + media_cache.file_size_limit=50M + imgproxy.url_prefix=/i + imgproxy.local_uploads=/uploads + imgproxy.local_media_cache=/media_cache + imgproxy.key=$SMITHEREEN_IMGPROXY_KEY + imgproxy.salt=$SMITHEREEN_IMGPROXY_SALT + EOF + # Java trusts its own store only: the JDK's, with the pasture's CA added + if [ ! -s "$st/cacerts" ]; then + podman run --rm --entrypoint sh -v "$st:/st:Z" -v "$ca:/pasture/ca:z,ro" $SMITHEREEN_IMAGE -c \ + 'cp /opt/java/openjdk/lib/security/cacerts /st/cacerts && chmod u+w /st/cacerts && + /opt/java/openjdk/bin/keytool -importcert -noprompt -alias pasture -file /pasture/ca/root.crt -keystore /st/cacerts -storepass changeit' >/dev/null + fi + podman volume exists pasture-smithereen-uploads || podman volume create --label pasture=1 pasture-smithereen-uploads >/dev/null + podman run -d --replace --name pasture-smithereen-imgproxy --label pasture=1 --network $net -v pasture-smithereen-uploads:/uploads \ + -e IMGPROXY_PATH_PREFIX=/i -e IMGPROXY_ALLOWED_SOURCES=local:// -e IMGPROXY_LOCAL_FILESYSTEM_ROOT=/uploads \ + -e IMGPROXY_KEY=$SMITHEREEN_IMGPROXY_KEY -e IMGPROXY_SALT=$SMITHEREEN_IMGPROXY_SALT $SMITHEREEN_IMGPROXY_IMAGE >/dev/null + podman run -d --replace --name pasture-smithereen-files --label pasture=1 --network $net -v pasture-smithereen-uploads:/srv:ro \ + docker.io/library/caddy:2 caddy file-server --root /srv --listen :80 >/dev/null + podman run -d --replace --name pasture-smithereen --label pasture=1 --network $net -v pasture-smithereen-uploads:/uploads \ + -v "$st/config.properties:/usr/local/etc/config.properties:Z,ro" -v "$st/cacerts:/pasture/cacerts:Z,ro" \ + -e JAVA_TOOL_OPTIONS='-Djavax.net.ssl.trustStore=/pasture/cacerts -Djavax.net.ssl.trustStorePassword=changeit -Xmx512m' \ + $SMITHEREEN_IMAGE >/dev/null + podman exec pasture-smithereen mkdir -p /uploads/uploads /uploads/media_cache + smithereen_wait + smithereen_settle + echo "smithereen: https://smithereen.test:6443" +} + +smithereen_wait() { + for _ in $(seq 1 90); do + site smithereen.test -s -o /dev/null -w '%{http_code}' https://smithereen.test:6443/.well-known/nodeinfo 2>/dev/null | grep -q 200 && return 0 + sleep 2 + done + echo "smithereen did not start" >&2; return 1 +} + +# smithereen_account : an account made through the signup form, then renamed from its id to the +# name (Smithereen leaves the name to its settings page) +smithereen_account() { + local name=$1 first=$2 + [ "$(echo "select count(*) from users where username='$name' and domain=''" | smithereen_sql)" = "1" ] && return 0 + site smithereen.test -s -o /dev/null -X POST https://smithereen.test:6443/account/register \ + --data-urlencode "email=$name@smithereen.test" --data-urlencode "password=$SMITHEREEN_PASSWORD" \ + --data-urlencode "password2=$SMITHEREEN_PASSWORD" --data-urlencode "first_name=$first" --data-urlencode "last_name=Pasture" + echo "update users set username='$name' where id=(select user_id from accounts where email='$name@smithereen.test')" | smithereen_sql +} + +# smuser and smfriend, a local client application for the password grant, and smuser's token +smithereen_settle() { + local st="$here/.state/smithereen" + smithereen_account smuser Smitty + smithereen_account smfriend Freddie + if [ "$(echo "select count(*) from api_applications where name='pasture'" | smithereen_sql)" = "0" ]; then + openssl genpkey -algorithm RSA -pkeyopt rsa_keygen_bits:2048 -out "$st/app.pem" 2>/dev/null + local pub priv + pub=$(openssl pkey -in "$st/app.pem" -pubout -outform DER | python3 -c 'import sys; print(sys.stdin.buffer.read().hex())') + priv=$(openssl pkcs8 -topk8 -nocrypt -in "$st/app.pem" -outform DER | python3 -c 'import sys; print(sys.stdin.buffer.read().hex())') + echo "insert into api_applications (type, name, description, developer_id, public_key, private_key, extra) + values (0, 'pasture', 'the pasture', 1, x'$pub', x'$priv', '{\"redirectURIs\":[\"https://smithereen.test/\"]}')" | smithereen_sql + fi + # the new names are read at start + podman restart pasture-smithereen >/dev/null + smithereen_wait + local app + app=$(echo "select id from api_applications where name='pasture'" | smithereen_sql) + echo "https://smithereen.test/apps/$app" > "$st/client_id" + # (its login flood control may refuse a second grant at once: tried again) + for user in smuser smfriend; do + for _ in 1 2 3 4 5; do + site smithereen.test -s -X POST https://smithereen.test:6443/oauth/token --data-urlencode grant_type=password \ + --data-urlencode "client_id=https://smithereen.test/apps/$app" --data-urlencode "username=$user@smithereen.test" \ + --data-urlencode "password=$SMITHEREEN_PASSWORD" | python3 -c 'import json, sys; print(json.load(sys.stdin).get("access_token", ""))' > "$st/$user.token" + [ -s "$st/$user.token" ] && [ "$(wc -c < "$st/$user.token")" -gt 2 ] && break + sleep 3 + done + done +} diff --git a/tools/pasture/scenarios/smithereen.sh b/tools/pasture/scenarios/smithereen.sh new file mode 100644 index 0000000..fd170cb --- /dev/null +++ b/tools/pasture/scenarios/smithereen.sh @@ -0,0 +1,129 @@ +# Smithereen 1.0.3: walls, friends as mutual follows, threaded comments, likes, reposts, polls, private messages, edits +# and deletions, a post on someone else's wall, groups, the unfollow, statistics. Smithereen is driven through its +# VKontakte-like API (/api/method/, v=1.0) as smuser, with the token peers/smithereen.sh got by a password grant. +SM=https://smithereen.test:6443 +SMT=$(cat "$here/.state/smithereen/smuser.token" 2>/dev/null) +SFT=$(cat "$here/.state/smithereen/smfriend.token" 2>/dev/null) +# sm [name=value ...]: an API call as smuser (SMT names whose token), printed as Smithereen answers it +sm() { + local method=$1; shift + local args=() + for kv in "$@"; do args+=(--data-urlencode "$kv"); done + site smithereen.test -s -X POST "$SM/api/method/$method" -H "Authorization: Bearer ${SMTOKEN:-$SMT}" --data-urlencode v=1.0 "${args[@]}" +} +smf() { SMTOKEN=$SFT sm "$@"; } +sm_sql() { podman exec pasture-mysql mysql -uroot -ppasture -N smithereen -e "$1" 2>/dev/null; } +# sm_load : what Smithereen makes of it, as "type id" +sm_load() { sm utils.loadRemoteObject "q=$1" | j "r = d.get('response') or {}; print(r.get('type', ''), r.get('id', ''))"; } +p_home_has() { curl -s -H "$PH" "$P/api/v1/timelines/home?limit=40" | j "print(next(((s.get('reblog') or s)['id'] for s in d if '$1' in ((s.get('reblog') or s)['content'] or '')), ''))"; } +p_status() { curl -s -H "$PH" "$P/api/v1/statuses/$1" | j "print(d.get('$2'))"; } + +echo "smithereen" +[ "$(sm users.get fields=screen_name | j "print(d['response'][0]['screen_name'])")" = "smuser" ] && ok "Smithereen token for smuser" || { ko "Smithereen token"; return 1; } +PT=$(privapub_token alice_smithereen) +PH="Authorization: Bearer $PT" +[ -n "$PT" ] && ok "PrivaPub token for alice_smithereen" || { ko "PrivaPub token for alice_smithereen"; return 1; } +alice_uri="$(curl -s -H "$PH" "$P/api/v1/accounts/verify_credentials" | j "print(d['url'])" | sed 's|/@|/peasants/|')" +run=$(date +%s) +sm_self=$(sm users.get | j "print(d['response'][0]['id'])") + +echo " friends" +read -r kind alice_on_sm <<< "$(sm_load "$alice_uri")" +[ "$kind" = "user" ] && ok "Smithereen resolves alice as a user" || ko "Smithereen cannot resolve alice ($kind)" +sm friends.add "user_id=$alice_on_sm" >/dev/null +until_true 45 '[ "$(curl -s -H "$PH" "$P/api/v1/accounts/verify_credentials" | j "print(d[\"followers_count\"])")" -ge 1 ]' \ + && ok "smuser follows alice" || ko "Smithereen's follow never reached alice" +sm_on_p=$(curl -s -H "$PH" "$P/api/v2/search?q=smuser@smithereen.test&resolve=true&type=accounts" | j "print(d['accounts'][0]['id'])") +[ -n "$sm_on_p" ] && ok "PrivaPub resolves smuser" || ko "PrivaPub cannot resolve smuser" +curl -s -o /dev/null -X POST -H "$PH" "$P/api/v1/accounts/$sm_on_p/follow" +until_true 45 '[ "$(curl -s -H "$PH" "$P/api/v1/accounts/relationships?id[]=$sm_on_p" | j "print(d[0][\"following\"])")" = "True" ]' \ + && ok "alice follows smuser (Accept arrived)" || ko "smuser's Accept never arrived" +until_true 30 '[ "$(sm friends.areFriends "user_ids=$alice_on_sm" | j "print(d[\"response\"][0][\"friend_status\"])")" = "friends" ]' \ + && ok "the two follows make them friends on Smithereen" || ko "Smithereen does not count them as friends ($(sm friends.areFriends "user_ids=$alice_on_sm"))" + +echo " walls" +sm_post=$(sm wall.post "message=a Smithereen wall post $run" | j "print(d['response'])") +until_true 45 '[ -n "$(p_home_has "a Smithereen wall post $run")" ]' && ok "smuser's wall post reaches alice's home" || ko "smuser's wall post never reached alice" +sm_post_on_p=$(p_home_has "a Smithereen wall post $run") +p_post=$(curl -s -X POST -H "$PH" $P/api/v1/statuses -d "status=a PrivaPub post for Smithereen $run&visibility=public") +p_post_id=$(echo "$p_post" | j "print(d['id'])"); p_post_uri=$(echo "$p_post" | j "print(d['uri'])") +until_true 45 '[ "$(sm_sql "select count(*) from wall_posts where ap_id='"'$p_post_uri'"'")" = "1" ]' \ + && ok "alice's post reaches Smithereen" || ko "alice's post never reached Smithereen" +p_post_on_sm=$(sm_sql "select id from wall_posts where ap_id='$p_post_uri'") + +echo " comments" +sm wall.createComment "post_id=$p_post_on_sm" "message=a Smithereen comment $run" >/dev/null +until_true 45 '[ "$(curl -s -H "$PH" "$P/api/v1/statuses/$p_post_id/context" | j "print(any(\"a Smithereen comment $run\" in s[\"content\"] for s in d[\"descendants\"]))")" = "True" ]' \ + && ok "smuser's comment threads under alice's post" || ko "smuser's comment missing on PrivaPub" +curl -s -o /dev/null -X POST -H "$PH" $P/api/v1/statuses -d "status=a PrivaPub comment $run&in_reply_to_id=$sm_post_on_p&visibility=public" +until_true 45 '[ "$(sm wall.getComments "post_id=$sm_post" | j "print(any(\"a PrivaPub comment $run\" in (c.get(\"text\") or \"\") for c in d[\"response\"][\"items\"]))")" = "True" ]' \ + && ok "alice's reply becomes a comment on smuser's post" || ko "alice's reply never reached Smithereen ($(sm wall.getComments "post_id=$sm_post" | head -c 300))" + +echo " likes and reposts" +sm likes.add type=post "item_id=$p_post_on_sm" >/dev/null +until_true 45 '[ "$(p_status $p_post_id favourites_count)" = "1" ]' && ok "smuser's like counts on PrivaPub" || ko "smuser's like never counted" +curl -s -o /dev/null -X POST -H "$PH" "$P/api/v1/statuses/$sm_post_on_p/favourite" +until_true 45 '[ "$(sm_sql "select count(*) from likes where object_id=$sm_post and object_type=0")" = "1" ]' \ + && ok "alice's favourite is a like on Smithereen" || ko "alice's favourite never counted on Smithereen" +# a repost there is a quote: a QuoteRequest to alice, which her policy grants, then the quoting post (its API fails +# without a message, even an empty one) +sm wall.repost "post_id=$p_post_on_sm" message= >/dev/null +until_true 45 '[ "$(p_status $p_post_id quotes_count)" = "1" ]' && ok "smuser's repost quotes alice's post, which lets it" || ko "smuser's repost never counted as a quote" +curl -s -o /dev/null -X POST -H "$PH" "$P/api/v1/statuses/$sm_post_on_p/reblog" +until_true 45 '[ "$(sm_sql "select count(*) from wall_posts where repost_of=$sm_post")" = "1" ]' \ + && ok "alice's boost is a repost on Smithereen" || ko "alice's boost never reached Smithereen" + +echo " polls" +poll=$(sm polls.create "question=A Smithereen poll $run" 'answers=["hay","clover"]' | j "print(d['response'])") +sm wall.post "message=vote $run" "attachments=[{\"type\":\"poll\",\"poll_id\":$poll}]" >/dev/null +until_true 45 '[ -n "$(p_home_has "vote $run")" ]' && ok "smuser's poll reaches alice" || ko "smuser's poll never reached alice" +poll_on_p=$(p_home_has "vote $run") +p_poll=$(curl -s -H "$PH" "$P/api/v1/statuses/$poll_on_p" | j "print(d['poll']['id'] if d.get('poll') else '')") +[ -n "$p_poll" ] && ok "as a poll" || ko "the poll arrived without its question" +curl -s -o /dev/null -X POST -H "$PH" "$P/api/v1/polls/$p_poll/votes" -d 'choices[]=1' +until_true 45 '[ "$(sm_sql "select count(*) from poll_votes where poll_id=$poll")" = "1" ]' \ + && ok "alice's vote counts on Smithereen" || ko "alice's vote never counted on Smithereen" + +echo " edits and deletions" +sm wall.edit "post_id=$sm_post" "message=a Smithereen wall post $run, edited" >/dev/null +until_true 45 '[ "$(p_status $sm_post_on_p content | grep -c "edited")" = "1" ]' && ok "smuser's edit reaches PrivaPub" || ko "smuser's edit never reached PrivaPub" +curl -s -o /dev/null -X PUT -H "$PH" "$P/api/v1/statuses/$p_post_id" -d "status=a PrivaPub post for Smithereen $run, edited" +until_true 45 '[ "$(sm_sql "select text like '"'%edited%'"' from wall_posts where id=$p_post_on_sm")" = "1" ]' \ + && ok "alice's edit reaches Smithereen" || ko "alice's edit never reached Smithereen" +gone=$(sm wall.post "message=a Smithereen post to delete $run" | j "print(d['response'])") +until_true 45 '[ -n "$(p_home_has "a Smithereen post to delete $run")" ]' +gone_on_p=$(p_home_has "a Smithereen post to delete $run") +sm wall.delete "post_id=$gone" >/dev/null +until_true 45 '[ "$(curl -s -o /dev/null -w "%{http_code}" -H "$PH" "$P/api/v1/statuses/$gone_on_p")" = "404" ]' \ + && ok "smuser's deletion reaches PrivaPub" || ko "smuser's deleted post still shows on PrivaPub" +curl -s -o /dev/null -X DELETE -H "$PH" "$P/api/v1/statuses/$p_post_id" +until_true 45 '[ "$(sm_sql "select count(*) from wall_posts where ap_id='"'$p_post_uri'"'")" = "0" ]' \ + && ok "alice's deletion reaches Smithereen" || ko "alice's deleted post still on Smithereen" + +echo " someone else's wall" +# smfriend writes on smuser's wall: the post carries the wall as its target, and smuser tells its followers' servers +# with Add{Note}, which is no pin +read -r _ sm_on_friend <<< "$(SMTOKEN=$SFT sm_load "https://smithereen.test/users/$sm_self")" +smf wall.post "owner_id=$sm_self" "message=a word on smuser's wall $run" >/dev/null +# (Smithereen sends a post on someone else's wall, and its owner's Add, only to servers whose actors have a wall: G-0009) +until_true 20 '[ -n "$(p_home_has "a word on smuser'"'"'s wall $run")" ]' && ok "a post on smuser's wall reaches alice" \ + || xf "a post on smuser's wall never reaches alice: Smithereen sends it only to servers with walls (G-0009)" +[ "$(curl -s -H "$PH" "$P/api/v1/accounts/$sm_on_p/statuses?pinned=true" | j "print(len(d))")" = "0" ] \ + && ok "and is not taken for a pin" || ko "PrivaPub took the wall's Add for a pin" + +echo " private messages" +sm messages.send "to=$alice_on_sm" "body=a Smithereen message $run" >/dev/null +until_true 45 'curl -s -H "$PH" "$P/api/v1/conversations" | grep -q "a Smithereen message $run"' && ok "smuser's message arrives as a DM" || ko "smuser's message never reached alice" +dm=$(curl -s -H "$PH" "$P/api/v1/conversations" | j "print(next(c['last_status']['id'] for c in d if 'a Smithereen message $run' in c['last_status']['content']))") +curl -s -o /dev/null -X POST -H "$PH" $P/api/v1/statuses -d "status=@smuser@smithereen.test a PrivaPub answer $run&in_reply_to_id=$dm&visibility=direct" +until_true 45 '[ "$(sm_sql "select count(*) from mail_messages where text like '"'%a PrivaPub answer $run%'"'")" -ge 1 ]' \ + && ok "alice's answer arrives as a Smithereen message" || ko "alice's answer never reached Smithereen" + +echo " unfollow" +curl -s -o /dev/null -X POST -H "$PH" "$P/api/v1/accounts/$sm_on_p/unfollow" +# smuser still follows alice: no longer friends +until_true 45 '[ "$(sm friends.areFriends "user_ids=$alice_on_sm" | j "print(d[\"response\"][0][\"friend_status\"])")" = "following" ]' \ + && ok "alice's unfollow reaches Smithereen" || ko "Smithereen still counts alice ($(sm friends.areFriends "user_ids=$alice_on_sm"))" + +echo " statistics" +stats_check smithereen.test smithereen diff --git a/tools/pasture/town/gaps.json b/tools/pasture/town/gaps.json index 121cef9..bd827a5 100644 --- a/tools/pasture/town/gaps.json +++ b/tools/pasture/town/gaps.json @@ -122,5 +122,19 @@ "status": "closed", "note": "Closed by the owner's decision of 2026-10-05: a direct message to one account goes as a ChatMessage when the account writes to us that way, or when its server's NodeInfo names Lemmy before 1.0 or Mbin (the one place PrivaPub decides by software). Checked live: Lemmy 0.19's first message and Mbin's thread both ways.", "closed": "2026-10-05" + }, + { + "id": "G-0009", + "title": "A post someone writes on a Smithereen user's wall never reaches that user's followers here: Smithereen sends wall posts only to servers whose actors have a wall", + "match": { + "feature": "wall\\.others", + "observer": "smithereen" + }, + "kind": "server", + "phase": "P3", + "code": "smithereen 1.0.3 ActivityPubWorker.sendAddPostToWallActivity and sendActivityForPost: requireFeature(Server.Feature.WALL_POSTS), which ObjectLinkResolver.maybeUpdateServerFeaturesFromActor sets only for a server whose actors have sm:wall", + "opened": "2026-10-05", + "status": "open", + "note": "Waits for the owner: a persona would publish a wall (sm:wall, FEP-400e), which also offers Smithereen's users to write on it, and PrivaPub would host their posts and announce them with Add{Note}." } ] diff --git a/tools/pasture/town/seed.py b/tools/pasture/town/seed.py index 01dc81f..1eadfb6 100644 --- a/tools/pasture/town/seed.py +++ b/tools/pasture/town/seed.py @@ -203,7 +203,20 @@ class Seeder: # -- the graph def v_follow(self, s): platform = s["actor"].split("/")[0] - outcome = driver(platform).follow(self.session(s["actor"]), self.acct(s["args"]["target"])) + d, sess, acct = driver(platform), self.session(s["actor"]), self.acct(s["args"]["target"]) + try: + outcome = d.follow(sess, acct) + except Exception: + # the accounts outlive a run: a follow that failed here may stand from an earlier seed, and the checker must + # know it, or it expects the follower to see nothing of the target's followers-only posts + try: + rel = d.relationship(sess, acct) + except Exception: + rel = {} + if rel.get("following") or rel.get("requested"): + self.ledger.add(type="relation", n=s["n"], verb="follow", actor=s["actor"], target=s["args"]["target"], + state="accepted" if rel.get("following") else "requested", observed=True) + raise self.ledger.add(type="relation", n=s["n"], verb="follow", actor=s["actor"], target=s["args"]["target"], state=outcome) def _wait_pending(self, s):