diff --git a/CLAUDE.md b/CLAUDE.md index 92b3b1f..de13642 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -491,6 +491,13 @@ tools/pasture/run.sh down # removes e its own activity and every bare `Announce{object}` 400 by design, and the echo is still needed (see `docs/INTEROP.md`, Lemmy). A new Lemmy never sends what it queued for a server before its send worker for that server started, so the scenario waits for that worker (`lm_worker`) before its first follow. 20 checks; relayed votes and a moderator's removal are expected failures (P7). +- **Hollo (0.9.19):** Fedify's microblog server on the shared Postgres, set up through its web form (which + checks `Origin` against `Host`, so the request names `hollo.test` without the port). It needs a 44-character + `SECRET_KEY`, a media directory and a `themeColor`; statuses and votes go as JSON. Town only, no scenario. +- **Iceshrimp.NET (2026.1.2-beta):** on the shared Postgres with AuthorizedFetch on and open registrations, trusting + Caddy's CA through `SSL_CERT_FILE`. Accounts come from its own `/api/iceshrimp/auth/register` (its login cuts the + connection short for a name it does not know), Mastodon API tokens from its OAuth form, which prints the out-of-band + code in the page. Its `jobs` table shows what it queued for whom. Town only, no scenario. - **SecureMode:** `PRIVAPUB_ENV="Federation__SecureMode=true" run.sh up …`, as production runs. A check of what an unsigned reader sees uses `unserved` and `gone_unsigned` (`lib/interop.sh`), which expect 401 when SecureMode is on and 404 or 410 when it is off. diff --git a/docs/INTEROP.md b/docs/INTEROP.md index 35c9342..b6588a5 100644 --- a/docs/INTEROP.md +++ b/docs/INTEROP.md @@ -338,6 +338,14 @@ Misskey scenario under Sharkey's name and then what only Sharkey does: - its edits arrive as edits, and ours reach it; - its quote carries the FEP-e232 `Link` tag and is understood. +**Town evidence (2026-10-05, Iceshrimp.NET 2026.1.2-beta, `town.sh seed iceshrimp-pair`):** 225 cells pass between +two personas and three Iceshrimp accounts (AuthorizedFetch on): delivery and confinement at every visibility both +ways, likes, boosts, emoji reactions and votes counted, threads, edits and deletes, follows of locked accounts on +both sides, and the privacy rows. Two cells are Iceshrimp's own gap (G-0004): a new note goes to its author's +followers only, so an account it mentions or answers gets it once it is edited, not when it is written (the Create's +`pre-deliver` job lists no recipients, through the Mastodon and the native API alike). Its followers and following +collections answer HTML to an ActivityPub `Accept`, so PrivaPub shows no counts for them. + ### Pleroma 2.10.2 and Akkoma 3.20.1 **Emits** diff --git a/tools/pasture/Caddyfile b/tools/pasture/Caddyfile index b4ad7fa..c95a520 100644 --- a/tools/pasture/Caddyfile +++ b/tools/pasture/Caddyfile @@ -47,3 +47,8 @@ hollo.test { tls internal reverse_proxy pasture-hollo:3000 } + +iceshrimp.test { + tls internal + reverse_proxy pasture-iceshrimp:3000 +} diff --git a/tools/pasture/peers/iceshrimp.sh b/tools/pasture/peers/iceshrimp.sh new file mode 100644 index 0000000..ab0d754 --- /dev/null +++ b/tools/pasture/peers/iceshrimp.sh @@ -0,0 +1,59 @@ +# Iceshrimp.NET (2026.1): the strict one. AuthorizedFetch on (every fetch from it is signed, and it answers unsigned +# reads 401), full JSON-LD expansion that drops undefined terms, several reactions per user. .NET, so it trusts Caddy's CA +# through SSL_CERT_FILE. On the shared Postgres (database iceshrimp); the town makes its users with its own CLI. +ICESHRIMP_IMAGE=${ICESHRIMP_IMAGE:-iceshrimp.dev/iceshrimp/iceshrimp.net:v2026.1.2-beta} +. "$here/peers/shared.sh" + +iceshrimp_up() { + shared_postgres_up + pg_db iceshrimp + mkdir -p "$here/.state/iceshrimp/media" + cat > "$here/.state/iceshrimp/configuration.ini" <<'INI' +[Instance] +ListenPort = 3000 +ListenHost = 0.0.0.0 +WebDomain = iceshrimp.test +AccountDomain = iceshrimp.test +CharacterLimit = 8192 + +[Security] +AuthorizedFetch = true +ValidateRequestSignatures = true +AllowLoopback = true +AllowLocalIPv4 = true +AllowLocalIPv6 = true +Registrations = Open +FederationMode = BlockList +PublicPreview = Public + +[Database] +Host = postgres +Port = 5432 +Database = iceshrimp +Username = pasture +Password = pasture +MaxConnections = 50 + +[Storage] +Provider = Local +MaxUploadSize = 100M +ProxyRemoteMedia = true + +[Storage:Local] +Path = /app/media + +[Logging:LogLevel] +Default = Information +Microsoft.AspNetCore = Warning +Microsoft.EntityFrameworkCore = Warning +INI + chmod -R a+rwX "$here/.state/iceshrimp" + podman run -d --replace --name pasture-iceshrimp --network $net -e SSL_CERT_FILE=/pasture/ca/bundle.pem \ + -v "$here/.state/iceshrimp/configuration.ini:/app/configuration.ini:z,ro" -v "$here/.state/iceshrimp/media:/app/media:z" \ + -v "$ca:/pasture/ca:z,ro" $ICESHRIMP_IMAGE >/dev/null + for _ in $(seq 1 90); do + site iceshrimp.test -s -o /dev/null -w '%{http_code}' https://iceshrimp.test:6443/api/v1/instance 2>/dev/null | grep -q 200 && break + sleep 2 + done + echo "iceshrimp: https://iceshrimp.test:6443" +} diff --git a/tools/pasture/town/check.py b/tools/pasture/town/check.py index c94d6fa..02c240f 100644 --- a/tools/pasture/town/check.py +++ b/tools/pasture/town/check.py @@ -48,11 +48,20 @@ class World: self.facts = [json.loads(line) for line in f if line.strip()] self.ok_steps = {f["n"] for f in self.facts if f.get("type") not in ("error", "unsupported") and "n" in f} self.failed = {f["n"]: f for f in self.facts if f.get("type") in ("error", "unsupported")} + # a fact's fetched_on names the objects its server had to fetch first (a reply's parent, a quote), and an + # interaction that arrived_by fetch made its actor's server fetch its object: both are kept per fetched object + fetched = defaultdict(set) + for f in self.facts: + for platform, refs in (f.get("fetched_on") or {}).items(): + for ref in refs: + fetched[ref].add(platform) + if f.get("type") == "interaction" and f.get("arrived_by") == "fetch": + fetched[f["ref"]].add(self.platform(f["actor"])) self.objects = {} for f in self.facts: if f.get("type") == "object": o = dict(self.planner.objects[f["ref"]]) - o.update(uri=f["uri"], origin=f["origin"], fetched_on=f.get("fetched_on", {})) + o.update(uri=f["uri"], origin=f["origin"], fetched_on=sorted(fetched[f["ref"]])) self.objects[f["ref"]] = o self.deleted = {f["ref"] for f in self.facts if f.get("type") == "mutation" and f["verb"] == "delete"} self.edited = {f["ref"] for f in self.facts if f.get("type") == "mutation" and f["verb"] == "edit"} @@ -197,8 +206,10 @@ class Sweep: continue if p in want and (receives is None or kind in receives): fetched = p in o.get("fetched_on", {}) - self.add(f"deliver.{kind}", o["origin"], p, held, ref, "held", "held" if held else "missing", through, - how="fetched by the seeder" if fetched else None) + # whether a follower of the author lives on p, or only the accounts it mentions or answers + followed = any(self.w.platform(k) == p for k in self.w.follows.get(o["author"], ())) + how = "fetched by the seeder" if fetched else None if followed or kind in ("circle", "community") else "addressed only" + self.add(f"deliver.{kind}", o["origin"], p, held, ref, "held", "held" if held else "missing", through, how=how) elif kind in ("followers", "direct", "circle", "located") and p not in want: self.add(f"confine.{kind}", o["origin"], p, not held, ref, "absent", "held" if held else "absent", through) diff --git a/tools/pasture/town/dialects/__init__.py b/tools/pasture/town/dialects/__init__.py index d14e454..e6cfab9 100644 --- a/tools/pasture/town/dialects/__init__.py +++ b/tools/pasture/town/dialects/__init__.py @@ -2,6 +2,7 @@ from dialects.akkoma import Akkoma from dialects.gts import Gts from dialects.hollo import Hollo +from dialects.iceshrimp import Iceshrimp from dialects.lemmy_api import LemmyApi from dialects.mastodon import Mastodon from dialects.misskey_api import Misskey, Sharkey @@ -16,6 +17,7 @@ DRIVERS = { "akkoma": (Akkoma, "akkoma.test"), "lemmy": (LemmyApi, "lemmy.test"), "hollo": (Hollo, "hollo.test"), + "iceshrimp": (Iceshrimp, "iceshrimp.test"), } _made = {} diff --git a/tools/pasture/town/dialects/iceshrimp.py b/tools/pasture/town/dialects/iceshrimp.py new file mode 100644 index 0000000..5d9a701 --- /dev/null +++ b/tools/pasture/town/dialects/iceshrimp.py @@ -0,0 +1,91 @@ +"""Iceshrimp.NET 2026.1: accounts through its own API (/api/iceshrimp/auth/register, registrations open in the pasture), +Mastodon API tokens through its OAuth page, which takes the user's name and password itself (an ASP.NET form with an +antiforgery token). Its database keeps Misskey's shape (`note`, `poll`), with likes counted apart from reactions.""" +import html +import re +import urllib.parse + +from core import podman +from dialects.base import Stored +from dialects.mastodon_api import MastodonApi + +OOB = "urn:ietf:wg:oauth:2.0:oob" +VIS = {"public": "public", "home": "unlisted", "followers": "followers", "specified": "direct"} + + +class Iceshrimp(MastodonApi): + platform = "iceshrimp" + caps = MastodonApi.caps | {"quote", "react"} + + def provision(self, accounts): + app = self.http.post(self.base + "/api/v1/apps", ok={200}, form={ + "client_name": "pasture-town", "redirect_uris": OOB, "scopes": "read write follow"}).json() + sessions = [] + existing = {r["username"] for r in podman.psql("iceshrimp", 'select username from "user" where host is null')} + for a in accounts: + # its login answers an unknown name by cutting the connection short, so only new names are registered + if a.username not in existing: + self.http.post(self.base + "/api/iceshrimp/auth/register", ok={200}, + json={"username": a.username, "password": a.password}) + sessions.append(self.session_from_token(a, self._token(app, a))) + return sessions + + def _token(self, app, a): + cookies = {} + + def send(method, path, form=None): + headers = {"Accept": "text/html,*/*"} + if cookies: + headers["Cookie"] = "; ".join(f"{k}={v}" for k, v in cookies.items()) + r = self.http.request(method, self.base + path, headers=headers, form=form) + for k, v in r.headers: + if k.lower() == "set-cookie": + name, _, value = v.split(";")[0].partition("=") + cookies[name.strip()] = value.strip() + return r + + query = urllib.parse.urlencode({"client_id": app["client_id"], "redirect_uri": OOB, "response_type": "code", + "scope": "read write follow"}) + page = send("GET", f"/oauth/authorize?{query}").text + antiforgery = re.search(r'name="__RequestVerificationToken" value="([^"]+)"', page).group(1) + r = send("POST", f"/oauth/authorize?{query}", { + "_handler": "oauth-authorize", "__RequestVerificationToken": html.unescape(antiforgery), + "Model.Username": a.username, "Model.Password": a.password}) + location = r.header("Location") or "" + code = urllib.parse.parse_qs(urllib.parse.urlsplit(location).query).get("code", [None])[0] + if code is None: + # the out-of-band page says "Your code is: " in its text + text = re.sub(r"<[^>]+>", " ", re.sub(r"<(style|script).*?", "", r.text, flags=re.S)) + found = re.search(r"Your code is:\s*([A-Za-z0-9_\-]{16,})", text) + code = found.group(1) if found else None + if code is None: + raise RuntimeError(f"Iceshrimp gave {a.username} no authorization code ({r.status}): {r.text[:300]}") + return self.http.post(self.base + "/oauth/token", ok={200}, form={ + "grant_type": "authorization_code", "code": code, "client_id": app["client_id"], + "client_secret": app["client_secret"], "redirect_uri": OOB, "scope": "read write follow"}).json()["access_token"] + + def react(self, s, uri, emoji): + sid = self.local_status_id(s, uri) or self.resolve(s, uri) + self.api(s, "POST", f"/api/v1/statuses/{sid}/react/{urllib.parse.quote(emoji)}", ok={200}) + + def _rows(self, uris): + if not uris: + return {} + local = {u: u.rsplit("/", 1)[1] for u in uris if u.startswith(f"{self.base}/notes/")} + rows = podman.psql("iceshrimp", f""" + select n.id, n.uri, n.visibility::text as visibility, n.text, n.cw, n."updatedAt" is not null as edited, + n."renoteCount" as boosts, n."repliesCount" as replies, n."likeCount" as likes, n.reactions, p.votes, + coalesce(r.uri, case when r.id is not null then '{self.base}/notes/' || r.id end) as parent_uri + from note n left join poll p on p."noteId" = n.id left join note r on r.id = n."replyId" + where (n."renoteId" is null or n.text is not null) + and (n.uri = any(string_to_array(:'p1', ' ')) or n.id = any(string_to_array(:'p2', ' ')))""", + " ".join(uris), " ".join(local.values()) or "-") + out = {} + for r in rows: + uri = r["uri"] or f"{self.base}/notes/{r['id']}" + if uri not in uris: + continue + out[uri] = Stored(True, False, r["id"], VIS.get(r["visibility"], r["visibility"]), r["text"], r["cw"], + bool(r["edited"]), r["parent_uri"], r["likes"], r["boosts"], r["replies"], r["votes"], + r["reactions"] or {}, r) + return out diff --git a/tools/pasture/town/dialects/privapub.py b/tools/pasture/town/dialects/privapub.py index ae23054..6cb6dd9 100644 --- a/tools/pasture/town/dialects/privapub.py +++ b/tools/pasture/town/dialects/privapub.py @@ -20,6 +20,7 @@ class PrivaPub(MastodonApi): # -- roots and personas def jwt(self, root, password): + """The root's JWT: signed up the first time, signed in again by any later command that reuses saved sessions.""" if root in self._jwts: return self._jwts[root] body = {"userName": root, "password": password} @@ -67,7 +68,7 @@ class PrivaPub(MastodonApi): # -- groups: communities (FEP-1b12) and circles def group(self, s, username, name, community, policy="followers", approve=False): - jwt = self._jwts[s.account.root] + jwt = self.jwt(s.account.root, s.account.password) r = self.http.post(self.base + "/clientapi/group/insert", headers={"Authorization": f"Bearer {jwt}"}, ok={200, 201}, json={"avatarId": s.local_id, "userName": username, "name": name, "description": name, "isCommunity": community, "postingPolicy": policy, "isDiscoverable": community, @@ -81,7 +82,7 @@ class PrivaPub(MastodonApi): return super().post(s, spec) def _client_post(self, s, spec): - jwt = self._jwts[s.account.root] + jwt = self.jwt(s.account.root, s.account.password) body = {"avatarId": s.local_id, "text": spec.text, "title": spec.title, "hasContentWarning": bool(spec.cw), "spoilerText": spec.cw} if spec.visibility == "located": diff --git a/tools/pasture/town/gaps.json b/tools/pasture/town/gaps.json index c0cdfa6..d716a7f 100644 --- a/tools/pasture/town/gaps.json +++ b/tools/pasture/town/gaps.json @@ -43,5 +43,20 @@ "code": "PrivaPub/Federation/Inbox/Handlers/AnnounceHandler.cs:203", "opened": "2026-10-01", "status": "open" + }, + { + "id": "G-0004", + "title": "Iceshrimp.NET sends a new note only to its author's followers: a remote account it mentions or answers gets nothing until the note is edited", + "match": { + "feature": "deliver\\..*", + "origin": "iceshrimp", + "how": "addressed only" + }, + "kind": "peer", + "phase": "upstream", + "code": "Iceshrimp.NET v2026.1.2-beta NoteService.PublishNoteAsync: recipients from note.Mentions (empty when the Create is queued) and note.Reply.ReplyUserId (the grandparent's author)", + "opened": "2026-10-05", + "status": "open", + "note": "seen in the jobs table: the Create's pre-deliver job has recipientIds [] for every visibility, through the Mastodon and the native API; the Update of the same note lists the mentioned account and reaches PrivaPub" } ] diff --git a/tools/pasture/town/gen.py b/tools/pasture/town/gen.py index 178db78..24646e3 100644 --- a/tools/pasture/town/gen.py +++ b/tools/pasture/town/gen.py @@ -19,10 +19,11 @@ from core.rng import Rng GENERATOR_VERSION = 1 HOSTS = {"privapub": "privapub.test", "gts": "gts.test", "mastodon": "mastodon.test", "misskey": "misskey.test", - "sharkey": "sharkey.test", "akkoma": "akkoma.test", "lemmy": "lemmy.test", "hollo": "hollo.test"} + "sharkey": "sharkey.test", "akkoma": "akkoma.test", "lemmy": "lemmy.test", "hollo": "hollo.test", + "iceshrimp": "iceshrimp.test"} SHORT = {"privapub": "pp", "gts": "gt", "mastodon": "ms", "misskey": "mk", "sharkey": "sk", "akkoma": "ak", "lemmy": "lm", - "hollo": "ho"} -MICRO = ("privapub", "gts", "mastodon", "misskey", "sharkey", "akkoma", "hollo") + "hollo": "ho", "iceshrimp": "is"} +MICRO = ("privapub", "gts", "mastodon", "misskey", "sharkey", "akkoma", "hollo", "iceshrimp") CAPS = { "privapub": {"post", "reply", "cw", "media", "poll", "like", "boost", "bookmark", "follow", "block", "mute", "report", "dm", "delete", "edit", "vote", "react", "quote", "profile", "circle", "community", "located"}, @@ -39,6 +40,8 @@ CAPS = { "lemmy": {"thread", "comment", "upvote", "downvote", "dm", "delete", "community", "edit", "block"}, "hollo": {"post", "reply", "cw", "media", "poll", "like", "boost", "bookmark", "follow", "block", "mute", "report", "dm", "delete", "edit", "vote", "quote", "react", "profile"}, + "iceshrimp": {"post", "reply", "cw", "media", "poll", "like", "boost", "bookmark", "follow", "block", "mute", "report", + "dm", "delete", "edit", "vote", "quote", "react", "profile"}, } FIRST = ["ada", "bo", "cleo", "dario", "elif", "femi", "gaia", "hiro", "ines", "jun", "kai", "lia", "milo", "nadia", @@ -192,7 +195,7 @@ class Planner: # circles: members are local siblings' neighbours and remote followers of the owner for ref, grp in sorted(self.groups.items()): if grp["kind"] == "circle": - pool = sorted(k for k in micro if k != grp["owner"] and k.split("/")[0] in ("privapub", "gts", "mastodon", "akkoma", "hollo")) + pool = sorted(k for k in micro if k != grp["owner"] and k.split("/")[0] in ("privapub", "gts", "mastodon", "akkoma", "hollo", "iceshrimp")) for member in rng.sample(pool, min(len(pool), self.spec.get("circleMembers", 4))): grp["members"].add(member) self.step("graph", member, "join", {"group": ref}) diff --git a/tools/pasture/town/seed.py b/tools/pasture/town/seed.py index 0183e39..2d8d0bd 100644 --- a/tools/pasture/town/seed.py +++ b/tools/pasture/town/seed.py @@ -211,6 +211,13 @@ class Seeder: raise TimeoutError(f"{follower}'s follow request never reached {s['actor']}") def v_accept(self, s): + # a follow that came back accepted needs no answer: the target was unlocked after all, or a run before this one + # on the same accounts already accepted it + if any(f.get("type") == "relation" and f["verb"] == "follow" and f["actor"] == s["args"]["target"] + and f["target"] == s["actor"] and f["state"] == "accepted" for f in self.ledger.facts): + self.ledger.add(type="relation", n=s["n"], verb="accept", actor=s["actor"], target=s["args"]["target"], + state="accepted", already=True) + return d, sess, follower = self._wait_pending(s) d.accept(sess, follower) self.ledger.add(type="relation", n=s["n"], verb="accept", actor=s["actor"], target=s["args"]["target"], state="accepted") diff --git a/tools/pasture/town/specs/iceshrimp-pair.json b/tools/pasture/town/specs/iceshrimp-pair.json new file mode 100644 index 0000000..f59a290 --- /dev/null +++ b/tools/pasture/town/specs/iceshrimp-pair.json @@ -0,0 +1,22 @@ +{ + "schema": "pasture-town/1", + "name": "iceshrimp-pair", + "seed": 20261006, + "peers": { + "privapub": {"roots": 1, "personas": [2, 2], "circles": 1, "communities": 0}, + "iceshrimp": {"accounts": 3} + }, + "profiles": {"locked": 0.2, "bot": 0.0, "fields": [0, 2], "avatar": 0.9, "header": 0.3, "bioWords": [5, 12], + "langs": {"en": 80, "it": 20}}, + "graph": {"follows": [2, 4], "mutual": 0.6, "pending": 0.0, "rejected": 0.0}, + "circleMembers": 2, + "content": { + "posts": 24, + "mix": {"text": 40, "cw": 10, "tags": 10, "mention": 10, "image": 10, "poll": 10, "quote": 10}, + "visibility": {"public": 50, "unlisted": 10, "followers": 20, "direct": 10, "circle": 10}, + "replyRounds": 2, "replies": 0.5, "deeperReplies": 0.4 + }, + "interactions": {"likes": [0, 3], "boosts": [0, 1], "react": 0.4, "vote": 0.8, "bookmark": 0.1}, + "mutations": {"edit": 0.15, "delete": 0.05, "blocks": 0, "mutes": 1, "reports": 0}, + "time": {"roundSettleSeconds": 15, "graphSettleSeconds": 20, "settleSeconds": 30, "deadlineSeconds": 120} +}