From f6dbf71964add6056a3393a4c5d66ffd12aeb824 Mon Sep 17 00:00:00 2001 From: thepra Date: Sun, 4 Oct 2026 03:34:48 +0200 Subject: [PATCH] Everything on, phase 2 completed: SecureMode on in production, checked by the deploy Owner decision 2026-10-04: SecureMode on once the pasture passes with it. - Both clean pasture passes were run over all six peers: - normally: 246 passed, 0 failed; - with Federation__SecureMode=true: every federation check passed. The only failures were four checks expecting an unsigned GET to get 404 or 410 where SecureMode answers 401. Those checks now go through `unserved` and `gone_unsigned` (lib/interop.sh), which expect 401 when SecureMode is on. - Circle posts now reach their member on GoToSocial and Mastodon, and survive Mastodon's signed refetch, as does a followers-only post. The GoToSocial expected failure is gone. - appsettings.Production.json turns SecureMode on. - The deploy now checks that an unsigned GET of @thepra answers 401 and that a browser is redirected. It reads @thepra's discoverability through the Mastodon API, since the actor is no longer readable unsigned. - docs/INTEROP.md (Mastodon, GoToSocial), CLAUDE.md and ROADMAP updated. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01ELjqpznMFMNrJoJUj6K5p2 --- .gitea/workflows/deploy.yml | 7 ++++++- CLAUDE.md | 9 +++++++-- PrivaPub/appsettings.Production.json | 3 +++ docs/INTEROP.md | 15 +++++++++++---- docs/ROADMAP.md | 12 +++++++++--- tools/pasture/lib/interop.sh | 7 +++++++ tools/pasture/scenarios/gts.sh | 2 +- tools/pasture/scenarios/mastodon.sh | 6 +++--- 8 files changed, 47 insertions(+), 14 deletions(-) diff --git a/.gitea/workflows/deploy.yml b/.gitea/workflows/deploy.yml index 9fb6a5c..716c77d 100644 --- a/.gitea/workflows/deploy.yml +++ b/.gitea/workflows/deploy.yml @@ -95,6 +95,11 @@ jobs: [ "$served" = "$BUILD_COMMIT" ] || { echo "::error::served build is '$served', expected '$BUILD_COMMIT'"; exit 1; } code=$(curl -s -o /dev/null -w '%{http_code}' -H 'Accept: application/activity+json' "$PUBLIC_URL/peasants/privapub") [ "$code" = "200" ] || { echo "::error::the instance actor answered $code"; exit 1; } + # SecureMode (owner decision 2026-10-04): an unsigned reader gets 401 from a persona, a browser the public page + code=$(curl -s -o /dev/null -w '%{http_code}' -H 'Accept: application/activity+json' "$PUBLIC_URL/peasants/thepra") + [ "$code" = "401" ] || { echo "::error::an unsigned GET of a persona answered $code, not 401: SecureMode is off"; exit 1; } + code=$(curl -s -o /dev/null -w '%{http_code}' -H 'Accept: text/html' "$PUBLIC_URL/peasants/thepra") + [ "$code" = "302" ] || { echo "::error::a browser asking for a persona got $code, not a redirect"; exit 1; } code=$(curl -s -o /dev/null -w '%{http_code}' "$PUBLIC_URL/.well-known/nodeinfo") [ "$code" = "200" ] || { echo "::error::nodeinfo answered $code"; exit 1; } code=$(curl -s -o /dev/null -w '%{http_code}' "$PUBLIC_URL/swagger/index.html") @@ -123,7 +128,7 @@ jobs: read -r token cid cs <<< "$(tools/smoke/oauth.sh "$PUBLIC_URL" "$login" "$password" thepra read)" echo "::add-mask::$token" tools/smoke/mastodon-api.sh "$PUBLIC_URL" "$token" - discoverable=$(curl -fsS -H 'Accept: application/activity+json' "$PUBLIC_URL/peasants/thepra" | python3 -c "import json,sys; print(json.load(sys.stdin).get('discoverable'))") + discoverable=$(curl -fsS -H "Authorization: Bearer $token" "$PUBLIC_URL/api/v1/accounts/verify_credentials" | python3 -c "import json,sys; print(json.load(sys.stdin).get('discoverable'))") [ "$discoverable" = "False" ] || { echo "::error::@thepra is discoverable"; exit 1; } curl -fsS -o /dev/null -X POST "$PUBLIC_URL/oauth/revoke" --data-urlencode "token=$token" \ --data-urlencode "client_id=$cid" --data-urlencode "client_secret=$cs" diff --git a/CLAUDE.md b/CLAUDE.md index 3f9931e..5a43816 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -431,9 +431,11 @@ tools/pasture/run.sh down # removes e delete is checked as a 404 on `/api/v1/statuses/{id}`. - It creates its accounts locked, so the scenario approves alice's request through `/api/v1/follow_requests`, which also checks our pending (`requested`) state and the manual Accept. - - 37 checks: discovery and follows both ways; posts and CW; a reply and its notification; likes and boosts both + - 55 checks: discovery and follows both ways; posts and CW; a reply and its notification; likes and boosts both ways; DMs both ways and off public timelines; polls both ways; quote policy; link cards; edits and deletes both - ways; unfollow; block and unblock; statistics. + ways; communities and circles; locked personas; unfollow; block and unblock; statistics. + - It files a circle post as a direct message and shows it only to the accounts it mentions, so like a DM it is + checked in the member's `/api/v1/conversations`, never by URI. - **Mastodon (4.7.3):** web and sidekiq on the shared Postgres and Redis, `ALLOWED_PRIVATE_ADDRESSES` for the network. Its token comes from `rails runner` (no password grant). Without Elasticsearch its status search finds nothing, so deliveries are checked through `/api/v1/accounts/:id/statuses` of the sender as Mastodon knows them, or @@ -461,6 +463,9 @@ tools/pasture/run.sh down # removes e its own activity and every bare `Announce{object}` 400 by design, and the echo is still needed (see `docs/INTEROP.md`, Lemmy). A new Lemmy never sends what it queued for a server before its send worker for that server started, so the scenario waits for that worker (`lm_worker`) before its first follow. 20 checks; relayed votes and a moderator's removal are expected failures (P7). +- **SecureMode:** `PRIVAPUB_ENV="Federation__SecureMode=true" run.sh up …`, as production runs. A check of what an + unsigned reader sees uses `unserved` and `gone_unsigned` (`lib/interop.sh`), which expect 401 when SecureMode is on and + 404 or 410 when it is off. - **Crawler:** `PRIVAPUB_ENV="Statistics__Crawler__Enabled=true Statistics__Crawler__Seeds__0=mastodon.test" run.sh up mastodon`, then `interop.sh crawler`. `PRIVAPUB_ENV` passes any setting to the PrivaPub container. - `run.sh up` replaces every container, Mongo included, so each run starts clean. To keep the data, republish into diff --git a/PrivaPub/appsettings.Production.json b/PrivaPub/appsettings.Production.json index 81490ff..fb14057 100644 --- a/PrivaPub/appsettings.Production.json +++ b/PrivaPub/appsettings.Production.json @@ -5,6 +5,9 @@ "Registrations": { "Mode": "Invitations" }, + "Federation": { + "SecureMode": true + }, "Statistics": { "Crawler": { "Enabled": true, diff --git a/docs/INTEROP.md b/docs/INTEROP.md index cd0f44c..7318aa2 100644 --- a/docs/INTEROP.md +++ b/docs/INTEROP.md @@ -169,15 +169,19 @@ Priorities, used throughout: Findings: - **Mastodon 4.7 names its actors by number**: `https://mastodon.test/ap/users/`, inbox `…/ap/users//inbox`. Nothing here may assume `/users/`. -- **It drops circle posts** (expected failure in the scenario). +- **It dropped circle posts** until each member's copy named that member (owner decision 2026-10-04, v1.19.0). - A post addressed only to `[circle, circle/flock]` parses as `direct` there, and Mastodon keeps a `direct` post only if it names a local account or arrived in a known account's inbox (`Create#addresses_local_accounts?`). - But `ActivityPub::InboxesController#account_required?` looks only at `params[:account_username]`. A delivery to the numeric `/ap/users/:account_id/inbox` it now advertises therefore reaches the worker with no recipient and is rejected. - DMs are unaffected because they name the recipient. - - Fixing it on our side means naming the member (or that server's members) in each copy's `cc`, which changes what a - circle reveals: **owner decision pending**. Reporting it upstream is the other half. + - Each member's copy now names that member in `cc` and mentions them silently, so Mastodon keeps it, and its signed + refetch (`ActivityPub::FetchRemoteStatusService`, by its instance actor) gets the post back naming the members on + that server. The same holds for a followers-only post's refetch, which used to answer 404, and a 404 on refetch + makes Mastodon delete its copy. Reporting the numeric-inbox recipient loss upstream is still worth doing. +- **With SecureMode on** (all six peers, 2026-10-04) every federation check passes: Mastodon, GoToSocial, Misskey, + Sharkey, Akkoma and Lemmy all sign their fetches, and fetch our instance actor's key unsigned first. - Inbound `Block` from Mastodon is not enforced yet (P7). ### GoToSocial: 0.22.1 (2026-07-20) @@ -233,7 +237,10 @@ Findings: **Pasture evidence (2026-10-03, GoToSocial 0.22.1, `tools/pasture/scenarios/gts.sh`):** 37 checks pass, three runs in a row. That is the original 33 plus four on statistics: described as gotosocial, inbound and outbound traffic counted, -no account named. +no account named. Since 2026-10-04 (v1.19.0) circle posts reach a GoToSocial member too. GoToSocial files a post for +neither the public nor the author's followers as a direct message, like our DMs, and shows it only to the accounts it +mentions. Being in `cc` stored it but left it invisible, so each member's copy also mentions that member silently. +Such posts are then found in the member's conversations, never by a search on their URI. ### Misskey family: Misskey 2026.10.0, Sharkey 2025.4.7, Iceshrimp.NET 2026.1.2-beta, CherryPick 4.17 diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index c650b2c..0a2a4b2 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -31,9 +31,15 @@ Written 2026-10-01 from the original 2023 code, the decePubClient UI, a federati Both open items were closed without a root step (owner decisions, 2026-10-04): the server fetches its geolocation databases itself, and the deploy makes and signs in as @thepra. -- [ ] Everything on in production (owner decisions 2026-10-04): v1.18.0 self-updating geolocation, @thepra, the crawler - on, sign-up by invitation, one registrations switch; then signed audiences with circles for everyone and SecureMode on, - account privacy, and one answer everywhere (the mismatch sweep). +- [ ] Everything on in production (owner decisions 2026-10-04): + - v1.18.0, deployed and verified 2026-10-04: geolocation that updates itself (DB-IP Lite 2026-10 loaded), the deploy + signing in as @thepra (undiscoverable), the crawler on (1010 servers known within the hour), sign-up by invitation + with one registrations switch; + - v1.19.0, 2026-10-04: circle posts reach Mastodon and GoToSocial members (each copy names and mentions its member), + followers-only, direct and circle posts served to signed refetches from those they were for, SecureMode on (all six + pasture peers pass under it), and account privacy (sign-in and recovery say nothing, recovery codes hashed for an + hour, a recovered password ends every session, a deleted root's personas and groups are deleted everywhere); + - still to come: one answer everywhere (the mismatch sweep). - [ ] P7 Threads, communities, moderation, the social graph - [ ] P8 Signatures, discovery, the long tail diff --git a/tools/pasture/lib/interop.sh b/tools/pasture/lib/interop.sh index d416643..d21218e 100644 --- a/tools/pasture/lib/interop.sh +++ b/tools/pasture/lib/interop.sh @@ -15,6 +15,13 @@ site() { curl -k --resolve "$1:6443:127.0.0.1" "${@:2}"; } pfetch() { curl -sk --connect-to privapub.test:443:127.0.0.1:6443 "$@"; } # make_png : an 8x8 red PNG, for uploads +# the status an unsigned ActivityPub GET of a PrivaPub document gets +pstatus() { pfetch -o /dev/null -w '%{http_code}' -H 'Accept: application/activity+json' "$1"; } +# SecureMode (PRIVAPUB_ENV="Federation__SecureMode=true") answers every unsigned GET but the instance actor's 401, so a +# post's author answering 401 means it is on; then 401 is what "not served" and "gone" look like to an unsigned reader +secure_mode() { [ "$(pstatus "${1%%/scribbles/*}")" = "401" ]; } +unserved() { local code; code=$(pstatus "$1"); if secure_mode "$1"; then [ "$code" = 401 ]; else [ "$code" = 404 ]; fi; } +gone_unsigned() { local code; code=$(pstatus "$1"); if secure_mode "$1"; then [ "$code" = 401 ]; else [ "$code" = 410 ]; fi; } make_png() { python3 -c " import struct,zlib w=h=8 diff --git a/tools/pasture/scenarios/gts.sh b/tools/pasture/scenarios/gts.sh index 624cc78..3eeece8 100644 --- a/tools/pasture/scenarios/gts.sh +++ b/tools/pasture/scenarios/gts.sh @@ -170,7 +170,7 @@ circle_uri=$(podman exec pasture-mongo mongosh --quiet PrivaPub --eval 'print(db # is then in gtsuser's conversations, never in a search by URI. until_true 15 'gcurl -s -H "$GH" "$G/api/v1/conversations" | grep -q "only the circle on GoToSocial sees this"' \ && ok "a circle post, naming its member, reaches its GoToSocial member" || ko "circle post missing on GoToSocial" -[ "$(pfetch -o /dev/null -w '%{http_code}' -H 'Accept: application/activity+json' "$circle_uri")" = "404" ] && ok "the circle post is not served unsigned" || ko "circle post served unsigned" +unserved "$circle_uri" && ok "the circle post is not served unsigned" || ko "circle post served unsigned" echo "locked personas" LT=$(privapub_token locked_alice); LH="Authorization: Bearer $LT" diff --git a/tools/pasture/scenarios/mastodon.sh b/tools/pasture/scenarios/mastodon.sh index 479440b..c360f4a 100644 --- a/tools/pasture/scenarios/mastodon.sh +++ b/tools/pasture/scenarios/mastodon.sh @@ -39,7 +39,7 @@ cw_uri=$(curl -s -X POST -H "$AH" $P/api/v1/statuses -d 'status=behind a warning until_true 30 '[ "$(m_status_by_uri "$alice_on_m" "$cw_uri" | j "print(d and d[\"spoiler_text\"]==\"spoilers\" and d[\"sensitive\"])")" = "True" ]' && ok "a content warning survives to Mastodon" || ko "content warning lost on Mastodon" fo_uri=$(curl -s -X POST -H "$AH" $P/api/v1/statuses -d 'status=only for followers&visibility=private' | j "print(d['uri'])") until_true 30 '[ "$(m_status_by_uri "$alice_on_m" "$fo_uri" | j "print(d and d[\"visibility\"])")" = "private" ]' && ok "a followers-only post reaches Mastodon as private" || ko "followers-only post missing or widened on Mastodon" -[ "$(pfetch -o /dev/null -w '%{http_code}' -H 'Accept: application/activity+json' "$fo_uri")" = "404" ] && ok "the followers-only post is not served unsigned" || ko "followers-only post served unsigned" +unserved "$fo_uri" && ok "the followers-only post is not served unsigned" || ko "followers-only post served unsigned" # Mastodon deletes its copy when a refetch answers 404, so a signed refetch from a follower's server is answered fo_refetch=$(podman exec pasture-mastodon bin/rails runner "s = ActivityPub::FetchRemoteStatusService.new.call('$fo_uri'); puts(s.present? ? s.visibility : 'lost')" 2>/dev/null | tail -1) [ "$fo_refetch" = "private" ] && ok "Mastodon's signed refetch of the followers-only post keeps it private" || ko "Mastodon's refetch of the followers-only post: $fo_refetch" @@ -115,7 +115,7 @@ cw_on_m=$(m_status_by_uri "$alice_on_m" "$cw_uri" | j "print(d['id'])") cw_id=$(curl -s -H "$AH" "$P/api/v1/accounts/$a_poll_id/statuses" | j "print(next(s['id'] for s in d if s['uri']=='$cw_uri'))") curl -s -o /dev/null -X DELETE -H "$AH" "$P/api/v1/statuses/$cw_id" until_true 30 '[ "$(mcurl -o /dev/null -w "%{http_code}" -H "$MH" "$M/api/v1/statuses/$cw_on_m")" = "404" ]' && ok "alice_masto's delete reaches Mastodon" || ko "delete not applied on Mastodon" -[ "$(pfetch -o /dev/null -w '%{http_code}' -H 'Accept: application/activity+json' "$cw_uri")" = "410" ] && ok "the deleted post answers 410" || ko "deleted post does not answer 410" +gone_unsigned "$cw_uri" && ok "the deleted post answers 410" || ko "deleted post does not answer 410" mcurl -o /dev/null -X DELETE -H "$MH" "$M/api/v1/statuses/$m_poll" until_true 30 '! curl -s -H "$AH" "$P/api/v1/timelines/home" | grep -q "Mastodon asks"' && ok "mastouser's delete reaches PrivaPub" || ko "Mastodon's delete not applied on PrivaPub" @@ -159,7 +159,7 @@ OT=$(mastodon_token outsider) alice_on_m_o=$(mcurl -H "Authorization: Bearer $OT" "$M/api/v2/search?q=@alice_masto@privapub.test&resolve=true&type=accounts" | j "print(d['accounts'][0]['id'])") [ "$(mcurl -H "Authorization: Bearer $OT" "$M/api/v1/accounts/$alice_on_m_o/statuses?limit=40" | j "print(any(s['uri']=='$circle_uri' for s in d))")" = "False" ] \ && ok "another Mastodon user does not see the circle post" || ko "the circle post leaked to a non-member" -[ "$(pfetch -o /dev/null -w '%{http_code}' -H 'Accept: application/activity+json' "$circle_uri")" = "404" ] && ok "the circle post is not served unsigned" || ko "circle post served unsigned" +unserved "$circle_uri" && ok "the circle post is not served unsigned" || ko "circle post served unsigned" echo " reports" curl -s -o /dev/null -X POST -H "$AH" "$P/api/v1/reports" -d "account_id=$masto_on_p&comment=pasture-report-$circle_name&forward=true"