The town: a fake community across the pasture, checked for coherence

tools/pasture/town/ (run through tools/pasture/town.sh) seeds a fake
community across every running peer and checks that all of them, and
PrivaPub, agree on what happened:

- drivers per platform on four dialect bases (Mastodon API, Misskey API,
  Lemmy API, PrivaPub with /clientapi), each with a selftest against
  its own server; what a server holds is read from its database, never
  by making it fetch;
- a deterministic generator (specs/village.json: 23 accounts on seven
  servers, roots with several personas, circles and communities, a
  cross-server follow graph, posts of every kind and visibility, reply
  rounds, likes, boosts, reactions, votes, edits, deletes, blocks,
  mutes and a report) and a seeder that keeps a ledger of what happened;
- a sweep that expects delivery and confinement per server, what each
  account sees, counts, threads, edits, deletes, follows and privacy
  rows (sibling keys, published days, canary root credentials in every
  peer's database, located posts that never leave), with what the peers
  do on purpose modelled (Misskey drops orphan replies, Lemmy keeps only
  community content, edits go to the post's own audience);
- known gaps (gaps.json) turn failures into xfail and passes into xpass;
  a self-contained report.html, and docs/INTEROP-BACKLOG.md.

The pasture moves to a public-looking subnet (peers with no private
address switch can join), takes PASTURE_PORT when 6971 is in use, adds
peers to a running pasture (run.sh add, Caddy recreated with its CA
kept), removes its volumes on down, writes every scenario check to
out/scenarios.jsonl, serves decePub as decepub.test for its e2e tests,
lifts GoToSocial's and Lemmy's own rate limits, trusts Caddy in
Mastodon (TRUSTED_PROXY_IP) and gains Hollo (Fedify), whose one login
owning several accounts is the nearest peer to PrivaPub's personas.

The first village found the four PrivaPub bugs fixed in the commits
before this one; the second run, on the fixed server, passes 2319 checks
with 11 failures left, all between peers or from Lemmy's send worker,
which the seeder now warms up first. ROADMAP records the owner's
decisions of 2026-10-04 (the town, and P9 back from the cut list).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01LsXgEaXee4GCU1hwYgPJXw

tools/pasture/town/dialects/__init__.py

@@ -0,0 +1,28 @@
+"""The platform registry: name -> driver class and its site."""
+from dialects.akkoma import Akkoma
+from dialects.gts import Gts
+from dialects.hollo import Hollo
+from dialects.lemmy_api import LemmyApi
+from dialects.mastodon import Mastodon
+from dialects.misskey_api import Misskey, Sharkey
+from dialects.privapub import PrivaPub
+
+DRIVERS = {
+	"privapub": (PrivaPub, "privapub.test"),
+	"gts": (Gts, "gts.test"),
+	"mastodon": (Mastodon, "mastodon.test"),
+	"misskey": (Misskey, "misskey.test"),
+	"sharkey": (Sharkey, "sharkey.test"),
+	"akkoma": (Akkoma, "akkoma.test"),
+	"lemmy": (LemmyApi, "lemmy.test"),
+	"hollo": (Hollo, "hollo.test"),
+}
+
+_made = {}
+
+
+def driver(platform):
+	if platform not in _made:
+		cls, host = DRIVERS[platform]
+		_made[platform] = cls(host)
+	return _made[platform]

tools/pasture/town/dialects/akkoma.py

@@ -0,0 +1,94 @@
+"""Akkoma 3.20 (and Pleroma, which shares all of this): accounts made with pleroma_ctl, tokens through the password
+grant, objects read from its Postgres database (`akkoma`). Its Linkify never takes @name@host.test for a mention, so
+mentions are addressed with Pleroma's `to[]` as well. Status ids are the Create activity's FlakeId in base62."""
+import urllib.parse
+
+from core import podman
+from dialects.base import Stored
+from dialects.mastodon_api import MastodonApi
+
+PUBLIC = "https://www.w3.org/ns/activitystreams#Public"
+B62 = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
+
+
+def flake(uuid):
+	n = int(uuid.replace("-", ""), 16)
+	out = ""
+	while n:
+		n, r = divmod(n, 62)
+		out = B62[r] + out
+	return out or "0"
+
+
+class Akkoma(MastodonApi):
+	platform = "akkoma"
+	caps = MastodonApi.caps | {"react", "quote"}
+	container = "pasture-akkoma"
+	ctl = "/opt/akkoma/bin/pleroma_ctl"
+	db = "akkoma"
+
+	def provision(self, accounts):
+		existing = {r["nickname"] for r in podman.psql(self.db, "select nickname from users where local")}
+		for a in accounts:
+			if a.username in existing:
+				continue
+			# pleroma_ctl passes its arguments on unquoted: no value may hold a space
+			podman.exec_(self.container, self.ctl, "user", "new", a.username, f"{a.username}@{self.host}",
+						 "--password", a.password, "--name", a.username, "--assume-yes")
+		app = self.http.post(self.base + "/api/v1/apps", ok={200}, form={
+			"client_name": "pasture-town", "redirect_uris": "urn:ietf:wg:oauth:2.0:oob", "scopes": "read write follow"}).json()
+		sessions = []
+		for a in accounts:
+			token = self.http.post(self.base + "/oauth/token", ok={200}, form={
+				"grant_type": "password", "username": a.username, "password": a.password, "client_id": app["client_id"],
+				"client_secret": app["client_secret"], "scope": "read write follow"}).json()["access_token"]
+			sessions.append(self.session_from_token(a, token))
+		return sessions
+
+	def status_form(self, s, spec):
+		form = super().status_form(s, spec)
+		if spec.mentions:
+			# Pleroma's to[] takes a local user by bare nickname: name@own-host addresses nobody
+			form["to"] = [a.split("@")[0] if a.endswith("@" + self.host) else a for a in spec.mentions]
+		return form
+
+	def react(self, s, uri, emoji):
+		sid = self.local_status_id(s, uri) or self.resolve(s, uri)
+		self.api(s, "PUT", f"/api/v1/pleroma/statuses/{sid}/reactions/{urllib.parse.quote(emoji)}", ok={200})
+
+	def _rows(self, uris):
+		if not uris:
+			return {}
+		rows = podman.psql(self.db, """
+			select o.data->>'id' as uri, o.data as data, a.id::text as act
+			from objects o
+			left join activities a on a.data->>'type' = 'Create'
+				and coalesce(a.data->'object'->>'id', a.data->>'object') = o.data->>'id'
+			where o.data->>'id' = any(string_to_array(:'p1', ' '))""", " ".join(uris))
+		out = {}
+		for r in rows:
+			d = r["data"]
+			deleted = d.get("type") == "Tombstone"
+			to, cc = _list(d.get("to")), _list(d.get("cc"))
+			if PUBLIC in to:
+				vis = "public"
+			elif PUBLIC in cc:
+				vis = "unlisted"
+			elif any(x.endswith("/followers") for x in to + cc):
+				vis = "followers"
+			else:
+				vis = "direct"
+			options = d.get("oneOf") or d.get("anyOf")
+			votes = [((o.get("replies") or {}).get("totalItems") or 0) for o in options] if options else None
+			reactions = {e[0]: len(e[1]) for e in d.get("reactions") or [] if len(e) >= 2}
+			out[r["uri"]] = Stored(True, deleted, flake(r["act"]) if r["act"] else None, vis, d.get("content"),
+								   d.get("summary") or None, bool(d.get("formerRepresentations")), d.get("inReplyTo"),
+								   d.get("like_count") or 0, d.get("announcement_count") or 0, d.get("repliesCount") or 0,
+								   votes, reactions, d)
+		return out
+
+
+def _list(v):
+	if v is None:
+		return []
+	return v if isinstance(v, list) else [v]

tools/pasture/town/dialects/base.py

@@ -0,0 +1,212 @@
+"""What every peer driver offers. A driver acts as one account of its platform through that platform's own API, and reads
+back what the platform holds without ever making it fetch: `stored` asks its database, `seen` its API as one account.
+Anything a platform cannot do raises Unsupported, and the planner never asks for it."""
+from dataclasses import dataclass, field
+from typing import Optional
+
+from core.http import client
+
+
+class Unsupported(Exception):
+	def __init__(self, platform, what):
+		super().__init__(f"{platform} cannot {what}")
+		self.platform = platform
+		self.what = what
+
+
+@dataclass
+class Account:
+	platform: str
+	username: str
+	password: str
+	name: str = None
+	bio: str = None
+	fields: list = field(default_factory=list)  # [(name, value)]
+	locked: bool = False
+	bot: bool = False
+	lang: str = "en"
+	avatar_seed: str = None
+	header_seed: str = None
+	root: str = None  # PrivaPub only: the root login that owns this persona
+
+	def acct(self, host):
+		return f"{self.username}@{host}"
+
+
+@dataclass
+class Session:
+	account: Account
+	token: str
+	local_id: str  # the account's id in its own server's API
+	actor_uri: str  # its ActivityPub id
+
+
+@dataclass
+class PostSpec:
+	text: str
+	kind: str = "note"  # note | article | page | event | link | video | audio | image
+	visibility: str = "public"  # public | unlisted | followers | direct | circle | community | located
+	cw: str = None
+	mentions: list = field(default_factory=list)  # accts: "name@host"
+	tags: list = field(default_factory=list)
+	media: list = field(default_factory=list)  # [{"kind": "image|audio|video", "seed": str, "alt": str, "sensitive": bool}]
+	poll: dict = None  # {"options": [...], "multiple": bool, "expires_in": seconds}
+	quote_uri: str = None
+	reply_to_uri: str = None
+	title: str = None
+	link: str = None
+	event: dict = None  # {"start", "end", "place"}
+	language: str = None
+	group: str = None  # community or circle reference (driver-specific id)
+	location: dict = None  # PrivaPub located posts: {"lat", "lng", "range_km"}
+
+
+@dataclass
+class Made:
+	uri: str
+	local_id: str
+	url: Optional[str] = None
+
+
+@dataclass
+class Stored:
+	"""One object as a server holds it, read from its database."""
+	exists: bool
+	deleted: bool = False
+	local_id: str = None
+	visibility: str = None
+	text: str = None
+	cw: str = None
+	edited: bool = False
+	parent_uri: str = None
+	likes: int = None
+	boosts: int = None
+	replies: int = None
+	votes: list = None
+	reactions: dict = None
+	raw: dict = None
+
+
+class Driver:
+	"""Base for every platform. `host` is its site name (gts.test); `api` its base URL through Caddy."""
+	platform = "base"
+	caps = frozenset()
+
+	def __init__(self, host):
+		self.host = host
+		self.base = f"https://{host}"
+		self.http = client()
+
+	# -- accounts
+	def provision(self, accounts):
+		"""Creates the accounts (idempotently) and returns a Session for each, in order."""
+		raise Unsupported(self.platform, "provision accounts")
+
+	def update_profile(self, s, account):
+		raise Unsupported(self.platform, "update profiles")
+
+	def lookup(self, s, acct):
+		"""The local id of account `acct` (name@host) as `s` sees it, resolving it through WebFinger if needed."""
+		raise Unsupported(self.platform, "look up accounts")
+
+	# -- the graph
+	def follow(self, s, acct):
+		"""Follows acct; returns "accepted" or "requested"."""
+		raise Unsupported(self.platform, "follow")
+
+	def unfollow(self, s, acct):
+		raise Unsupported(self.platform, "unfollow")
+
+	def pending(self, s):
+		"""The accts waiting for `s` to accept their follow."""
+		raise Unsupported(self.platform, "list follow requests")
+
+	def accept(self, s, acct):
+		raise Unsupported(self.platform, "accept follow requests")
+
+	def reject(self, s, acct):
+		raise Unsupported(self.platform, "reject follow requests")
+
+	def relationship(self, s, acct):
+		"""{following, followed_by, requested, blocking, muting}"""
+		raise Unsupported(self.platform, "read relationships")
+
+	def block(self, s, acct):
+		raise Unsupported(self.platform, "block")
+
+	def unblock(self, s, acct):
+		raise Unsupported(self.platform, "unblock")
+
+	def mute(self, s, acct):
+		raise Unsupported(self.platform, "mute")
+
+	def report(self, s, acct, uris, comment):
+		raise Unsupported(self.platform, "report")
+
+	# -- content
+	def post(self, s, spec):
+		raise Unsupported(self.platform, "post")
+
+	def edit(self, s, uri, spec):
+		raise Unsupported(self.platform, "edit")
+
+	def delete(self, s, uri):
+		raise Unsupported(self.platform, "delete")
+
+	def like(self, s, uri):
+		raise Unsupported(self.platform, "like")
+
+	def unlike(self, s, uri):
+		raise Unsupported(self.platform, "unlike")
+
+	def boost(self, s, uri):
+		raise Unsupported(self.platform, "boost")
+
+	def unboost(self, s, uri):
+		raise Unsupported(self.platform, "unboost")
+
+	def react(self, s, uri, emoji):
+		raise Unsupported(self.platform, "react")
+
+	def vote(self, s, uri, choices):
+		raise Unsupported(self.platform, "vote")
+
+	def bookmark(self, s, uri):
+		raise Unsupported(self.platform, "bookmark")
+
+	# -- reading back, never fetching
+	def local_status_id(self, s, uri):
+		"""This server's API id for the object `uri`, or None if it does not hold it. Never fetches."""
+		raise Unsupported(self.platform, "find statuses by URI")
+
+	def resolve(self, s, uri):
+		"""This server's API id for `uri`, fetching it if it must (a deliberate fetch, recorded by the caller)."""
+		raise Unsupported(self.platform, "resolve statuses")
+
+	def stored(self, uris):
+		"""{uri: Stored} for every uri, from the database."""
+		raise Unsupported(self.platform, "read its database")
+
+	def seen(self, s, uris):
+		"""{uri: bool}: whether `s` can see each object through the API."""
+		out = {}
+		for uri in uris:
+			sid = self.local_status_id(s, uri)
+			out[uri] = sid is not None and self.visible(s, sid)
+		return out
+
+	def visible(self, s, local_id):
+		raise Unsupported(self.platform, "read statuses")
+
+	def notifications(self, s):
+		"""[{type, acct, uri}] newest first, normalised to Mastodon's types."""
+		raise Unsupported(self.platform, "read notifications")
+
+	def actor_uri(self, username):
+		"""The ActivityPub id of a local account, as this server's WebFinger names it."""
+		r = self.http.get(f"{self.base}/.well-known/webfinger", params={"resource": f"acct:{username}@{self.host}"},
+						  headers={"Accept": "application/jrd+json"})
+		for link in (r.json() or {}).get("links", []) if r.ok else []:
+			if link.get("rel") == "self" and "json" in (link.get("type") or ""):
+				return link["href"]
+		return None

tools/pasture/town/dialects/gts.py

@@ -0,0 +1,103 @@
+"""GoToSocial 0.22: accounts made with its admin CLI (then one restart, as the pasture does), tokens through the OAuth
+code flow with a signed-in cookie (it has no password grant), objects read from a copy of its sqlite database."""
+import json
+import re
+import urllib.parse
+
+from core import podman
+from dialects.base import Stored
+from dialects.mastodon_api import MastodonApi
+
+# gtsmodel.Visibility since 0.20: stored as small integers
+VIS = {2: "public", 3: "unlisted", 4: "followers", 5: "mutuals", 6: "direct"}
+OOB = "urn:ietf:wg:oauth:2.0:oob"
+
+
+class Gts(MastodonApi):
+	platform = "gts"
+
+	def provision(self, accounts):
+		created = False
+		existing = {r["username"] for r in self._sqlite("select username from accounts where domain is null")}
+		for a in accounts:
+			if a.username in existing:
+				continue
+			podman.exec_("pasture-gts", "/gotosocial/gotosocial", "admin", "account", "create", "--username", a.username,
+						 "--email", f"{a.username}@gts.test", "--password", a.password)
+			podman.exec_("pasture-gts", "/gotosocial/gotosocial", "admin", "account", "confirm", "--username", a.username)
+			created = True
+		if created:
+			# the running server keeps what it has cached about accounts; the pasture restarts it after the CLI too
+			podman.run("restart", "pasture-gts")
+			self._wait()
+		return [self.session_from_token(a, self._token(a)) for a in accounts]
+
+	def _wait(self):
+		import time
+		for _ in range(90):
+			try:
+				if self.http.get(self.base + "/api/v1/instance").status == 200:
+					return
+			except Exception:
+				pass
+			time.sleep(1)
+		raise TimeoutError("GoToSocial did not come back")
+
+	def _token(self, a):
+		app = self.http.post(self.base + "/api/v1/apps", ok={200}, form={
+			"client_name": "pasture-town", "redirect_uris": OOB, "scopes": "read write follow"}).json()
+		cookies = {}
+
+		def send(method, path, **kw):
+			# the sign-in pages are HTML: asked for JSON (the client's default) they answer 406 and set no session
+			headers = {"Accept": "text/html,*/*"}
+			if cookies:
+				headers["Cookie"] = "; ".join(f"{k}={v}" for k, v in cookies.items())
+			r = self.http.request(method, self.base + path, headers=headers, **kw)
+			for k, v in r.headers:
+				if k.lower() == "set-cookie":
+					name, _, value = v.split(";")[0].partition("=")
+					cookies[name.strip()] = value.strip()
+			return r
+
+		query = urllib.parse.urlencode({"client_id": app["client_id"], "redirect_uri": OOB, "response_type": "code",
+										"scope": "read write follow"})
+		send("GET", f"/oauth/authorize?{query}")
+		send("POST", "/auth/sign_in", form={"username": f"{a.username}@gts.test", "password": a.password})
+		r = send("POST", "/oauth/authorize")
+		location = r.header("Location") or ""
+		code = urllib.parse.parse_qs(urllib.parse.urlsplit(location).query).get("code", [None])[0]
+		if code is None:
+			found = re.search(r"code=([A-Za-z0-9_\-]+)", r.text)
+			code = found.group(1) if found else None
+		if code is None:
+			raise RuntimeError(f"GoToSocial gave {a.username} no authorization code ({r.status})")
+		token = self.http.post(self.base + "/oauth/token", ok={200}, form={
+			"grant_type": "authorization_code", "code": code, "client_id": app["client_id"],
+			"client_secret": app["client_secret"], "redirect_uri": OOB, "scope": "read write follow"}).json()
+		return token["access_token"]
+
+	def _sqlite(self, sql, *params):
+		with podman.SqliteCopy("pasture-gts", "/gotosocial/storage/sqlite.db") as db:
+			return db.rows(sql, *params)
+
+	def _rows(self, uris):
+		if not uris:
+			return {}
+		marks = ",".join("?" * len(uris))
+		rows = self._sqlite(f"""
+			select s.id as local_id, s.uri, s.visibility, s.flags, s.text, s.content, s.content_warning as cw,
+				s.edited_at is not null as edited, s.in_reply_to_uri as parent_uri,
+				(select count(*) from status_faves f where f.status_id = s.id) as likes,
+				(select count(*) from statuses b where b.boost_of_id = s.id) as boosts,
+				(select count(*) from statuses r where r.in_reply_to_id = s.id) as replies,
+				(select votes from polls p where p.status_id = s.id) as votes
+			from statuses s where s.boost_of_id is null and s.uri in ({marks})""", *uris)
+		out = {}
+		for r in rows:
+			votes = json.loads(r["votes"]) if r["votes"] else None
+			# 0.22 soft-deletes: flag 2 marks a deleted status (its statuses_deleted_idx), 8 a local one
+			out[r["uri"]] = Stored(True, bool(r["flags"] & 2), r["local_id"], VIS.get(r["visibility"], str(r["visibility"])),
+								   r["text"] or r["content"], r["cw"] or None, bool(r["edited"]), r["parent_uri"],
+								   r["likes"], r["boosts"], r["replies"], votes, None, r)
+		return out

tools/pasture/town/dialects/hollo.py

@@ -0,0 +1,124 @@
+"""Hollo 0.9 (Fedify): one login (made by peers/hollo.sh) owning every town account, each its own actor, the way a PrivaPub
+root owns its personas. Accounts are made through its /accounts form and tokens through its OAuth consent page, which
+asks the signed-in owner which account the token is for. Its forms check only that Origin is its own (Hono's csrf())."""
+import html
+import re
+import urllib.parse
+
+from core import podman
+from dialects.base import Stored
+from dialects.mastodon_api import MastodonApi
+
+OOB = "urn:ietf:wg:oauth:2.0:oob"
+OWNER = ("owner@hollo.test", "Hollo-Pasture-Pass-1")
+VIS = {"public": "public", "unlisted": "unlisted", "private": "followers", "direct": "direct"}
+
+
+class Hollo(MastodonApi):
+	platform = "hollo"
+	caps = MastodonApi.caps | {"quote", "react"}
+
+	def __init__(self, host):
+		super().__init__(host)
+		self._cookies = {}
+
+	def _form(self, method, path, form=None):
+		headers = {"Accept": "text/html,*/*", "Origin": self.base}
+		if self._cookies:
+			headers["Cookie"] = "; ".join(f"{k}={v}" for k, v in self._cookies.items())
+		r = self.http.request(method, self.base + path, headers=headers, form=form)
+		for k, v in r.headers:
+			if k.lower() == "set-cookie":
+				name, _, value = v.split(";")[0].partition("=")
+				self._cookies[name.strip()] = value.strip()
+		return r
+
+	def _login(self):
+		if "login" not in self._cookies:
+			self._form("POST", "/login", {"email": OWNER[0], "password": OWNER[1]})
+
+	def provision(self, accounts):
+		self._login()
+		existing = {r["handle"].split("@")[1] for r in podman.psql("hollo", "select handle from accounts where handle like '@%'")
+					if r["handle"].count("@") >= 2}
+		for a in accounts:
+			if a.username in existing:
+				continue
+			self._form("POST", "/accounts", {"username": a.username, "name": a.name or a.username, "bio": a.bio or "",
+											 "discoverable": "on", "language": a.lang or "en", "visibility": "public", "themeColor": "azure",
+											 **({"protected": "on"} if a.locked else {})})
+		app = self.http.post(self.base + "/api/v1/apps", ok={200}, form={
+			"client_name": "pasture-town", "redirect_uris": OOB, "scopes": "read write follow"}).json()
+		return [self.session_from_token(a, self._token(app, a.username)) for a in accounts]
+
+	def _token(self, app, username):
+		query = urllib.parse.urlencode({"client_id": app["client_id"], "redirect_uri": OOB, "response_type": "code",
+										"scope": "read write follow"})
+		page = self._form("GET", f"/oauth/authorize?{query}").text
+		form = dict((k, html.unescape(v)) for k, v in re.findall(r'<input type="hidden" name="([^"]+)" value="([^"]*)"', page))
+		account = None
+		for block in re.findall(r'<label[^>]*>(.*?)</label>', page, re.S):
+			if f"@{username}@" in block:
+				found = re.search(r'name="account_id"[^>]*value="([^"]+)"|value="([^"]+)"[^>]*name="account_id"', block)
+				if found:
+					account = found.group(1) or found.group(2)
+		if account is None:
+			raise RuntimeError(f"Hollo's consent page offers no account @{username}")
+		form.update({"account_id": account, "decision": "allow"})
+		r = self._form("POST", "/oauth/authorize", form)
+		location = r.header("Location") or ""
+		code = urllib.parse.parse_qs(urllib.parse.urlsplit(location).query).get("code", [None])[0]
+		if code is None:
+			found = re.search(r'<code[^>]*>([^<]+)</code>', r.text)
+			code = found.group(1).strip() if found else None
+		if code is None:
+			raise RuntimeError(f"Hollo gave @{username} no authorization code ({r.status})")
+		return self.http.post(self.base + "/oauth/token", ok={200}, form={
+			"grant_type": "authorization_code", "code": code, "client_id": app["client_id"],
+			"client_secret": app["client_secret"], "redirect_uri": OOB}).json()["access_token"]
+
+	def update_profile(self, s, account):
+		# Hollo has no update_credentials for fields and images beyond its own forms; name, bio and lock are enough here
+		form = {"display_name": account.name, "note": account.bio, "locked": account.locked}
+		self.api(s, "PATCH", "/api/v1/accounts/update_credentials", ok={200}, form=form)
+
+	def post(self, s, spec):
+		"""Hollo reads a status as JSON (its poll is a nested object a form cannot carry)."""
+		from dialects.base import Made, Unsupported
+		if spec.kind not in ("note", "image", "video", "audio"):
+			raise Unsupported(self.platform, f"post a {spec.kind}")
+		form = self.status_form(s, spec)
+		body = {k: v for k, v in form.items() if not k.startswith("poll[")}
+		if spec.poll:
+			body["poll"] = {"options": spec.poll["options"], "expires_in": spec.poll.get("expires_in", 86400),
+							"multiple": bool(spec.poll.get("multiple"))}
+		status = self.api_json(s, "POST", "/api/v1/statuses", json=body)
+		return Made(status["uri"], status["id"], status.get("url"))
+
+	def vote(self, s, uri, choices):
+		sid = self.local_status_id(s, uri) or self.resolve(s, uri)
+		poll = (self.api_json(s, "GET", f"/api/v1/statuses/{sid}").get("poll") or {})
+		if not poll:
+			raise LookupError(f"{self.host} shows no poll on {uri}")
+		self.api(s, "POST", f"/api/v1/polls/{poll['id']}/votes", ok={200}, json={"choices": choices})
+
+	def react(self, s, uri, emoji):
+		sid = self.local_status_id(s, uri) or self.resolve(s, uri)
+		self.api(s, "PUT", f"/api/v1/statuses/{sid}/emoji_reactions/{urllib.parse.quote(emoji)}", ok={200})
+
+	def _rows(self, uris):
+		if not uris:
+			return {}
+		rows = podman.psql("hollo", """
+			select p.id::text as local_id, p.iri as uri, p.visibility::text as visibility, p.content_html as text,
+				p.summary as cw, p.updated > p.published as edited, r.iri as parent_uri,
+				(select count(*) from likes l where l.post_id = p.id) as likes,
+				p.shares_count as boosts, p.replies_count as replies,
+				(select array_agg(o.votes_count order by o.index) from poll_options o where o.poll_id = p.poll_id) as votes,
+				(select json_object_agg(e.emoji, e.n) from (select emoji, count(*) n from reactions x where x.post_id = p.id
+					group by emoji) e) as reactions
+			from posts p left join posts r on r.id = p.reply_target_id
+			where p.sharing_id is null and p.iri = any(string_to_array(:'p1', ' '))""", " ".join(uris))
+		return {r["uri"]: Stored(True, False, r["local_id"], VIS.get(r["visibility"], r["visibility"]), r["text"], r["cw"],
+								 bool(r["edited"]), r["parent_uri"], r["likes"], r["boosts"], r["replies"], r["votes"], r["reactions"] or {}, r)
+				for r in rows}

tools/pasture/town/dialects/lemmy_api.py

@@ -0,0 +1,211 @@
+"""Lemmy 1.0 (API v4; PieFed's Lemmy-compatible API later): threads in communities, comments, votes, private
+messages. Its admin is made by `setup`; the town opens registration without email or captcha and registers everyone
+else. Objects are read from `post`, `comment` and `private_message` by `ap_id`."""
+import os
+
+from core import podman
+from core.http import HttpError
+from dialects.base import Driver, Made, Session, Stored, Unsupported
+
+
+class LemmyApi(Driver):
+	platform = "lemmy"
+	caps = frozenset({"thread", "comment", "upvote", "downvote", "dm", "delete", "community", "edit", "report",
+					  "block"})
+	db = "lemmy"
+	prefix = "/api/v4"
+
+	def __init__(self, host):
+		super().__init__(host)
+		self._ids = {}
+
+	def update_profile(self, s, account):
+		self.lm(s, "PUT", "account/settings/save", {"display_name": account.name, "bio": account.bio, "bot_account": account.bot})
+
+	def lm(self, s, method, path, body=None, params=None, ok=(200,)):
+		token = s.token if isinstance(s, Session) else s
+		headers = {"Authorization": f"Bearer {token}"} if token else {}
+		r = self.http.request(method, f"{self.base}{self.prefix}/{path}", headers=headers, json=body, params=params,
+							  template=f"{self.prefix}/{path}")
+		if ok and r.status not in ok:
+			raise HttpError(method, path, r.status, r.text)
+		return r.json()
+
+	def admin_token(self):
+		here = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+		with open(os.path.join(here, ".state", "lemmy.token")) as f:
+			return f.read().strip()
+
+	def provision(self, accounts):
+		admin = self.admin_token()
+		self.lm(admin, "PUT", "site", {"registration_mode": "open", "email_verification_required": False,
+									   "captcha_enabled": False})
+		sessions = []
+		for a in accounts:
+			r = self.http.post(f"{self.base}{self.prefix}/account/auth/login",
+							   json={"username_or_email": a.username, "password": a.password})
+			jwt = (r.json() or {}).get("jwt") if r.ok else None
+			if not jwt:
+				jwt = self.lm(None, "POST", "account/auth/register", {
+					"username": a.username, "password": a.password, "password_verify": a.password,
+					"show_nsfw": False})["jwt"]
+			me = self.lm(jwt, "GET", "account")
+			person = (me.get("local_user_view") or me.get("my_user", {}).get("local_user_view") or {}).get("person") or me.get("person")
+			sessions.append(Session(a, jwt, str(person["id"]), person["ap_id"]))
+		return sessions
+
+	def warm(self, s, accts, seconds=120):
+		"""Lemmy starts its send worker for a server at the newest activity and never sends what it queued for that server
+		before then (Accepts and Follows included), so the town makes it meet every server first and waits for the
+		workers. Lemmy only knows a server once it has resolved an account there."""
+		import time
+		hosts = set()
+		for acct in accts:
+			try:
+				self.lookup(s, acct)
+				hosts.add(acct.split("@")[1])
+			except Exception:
+				pass
+		deadline = time.time() + seconds
+		while time.time() < deadline:
+			ready = {r["domain"] for r in podman.psql(self.db, """
+				select i.domain from federation_queue_state q join instance i on i.id = q.instance_id""")}
+			if hosts <= ready:
+				return sorted(hosts)
+			time.sleep(5)
+		return sorted(ready & hosts)
+
+	# -- communities and people
+	def community(self, s, name, title):
+		return self.lm(s, "POST", "community", {"name": name, "title": title})["community_view"]["community"]
+
+	def resolve_community(self, s, ref):
+		"""ref: !name@host"""
+		return self.lm(s, "GET", "resolve_object", params={"q": ref})["community"]["id"]
+
+	def follow_community(self, s, community_id):
+		self.lm(s, "POST", "community/follow", {"community_id": community_id, "follow": True})
+
+	def lookup(self, s, acct):
+		key = (s.token, acct)
+		if key not in self._ids:
+			self._ids[key] = self.lm(s, "GET", "resolve_object", params={"q": f"@{acct}"})["person"]["id"]
+		return self._ids[key]
+
+	def follow(self, s, acct):
+		raise Unsupported(self.platform, "follow people")
+
+	def block(self, s, acct):
+		self.lm(s, "POST", "account/block/person", {"person_id": self.lookup(s, acct), "block": True})
+
+	def unblock(self, s, acct):
+		self.lm(s, "POST", "account/block/person", {"person_id": self.lookup(s, acct), "block": False})
+
+	# -- content
+	def post(self, s, spec):
+		if spec.visibility == "direct":
+			if len(spec.mentions) != 1:
+				raise Unsupported(self.platform, "message several people at once")
+			pm = self.lm(s, "POST", "private_message", {"content": spec.text, "recipient_id": self.lookup(s, spec.mentions[0])})
+			pm = pm.get("private_message_view", pm)["private_message"]
+			return Made(pm["ap_id"], str(pm["id"]))
+		if spec.reply_to_uri:
+			parent = self._find(s, spec.reply_to_uri)
+			body = {"content": spec.text, "post_id": parent["post_id"]}
+			if parent["kind"] == "comment":
+				body["parent_id"] = parent["id"]
+			c = self.lm(s, "POST", "comment", body)["comment_view"]["comment"]
+			return Made(c["ap_id"], f"c{c['id']}")
+		if spec.group is None:
+			raise Unsupported(self.platform, "post outside a community")
+		body = {"name": spec.title or spec.text[:80], "body": spec.text, "community_id": spec.group}
+		if spec.link:
+			body["url"] = spec.link
+		p = self.lm(s, "POST", "post", body)["post_view"]["post"]
+		return Made(p["ap_id"], f"p{p['id']}")
+
+	def _find(self, s, uri):
+		row = self._rows([uri]).get(uri)
+		if row is None:
+			found = self.lm(s, "GET", "resolve_object", params={"q": uri})
+			row = self._rows([uri]).get(uri)
+			if row is None:
+				raise LookupError(f"{self.host} cannot resolve {uri}: {found}")
+		return row.raw
+
+	def edit(self, s, uri, spec):
+		row = self._find(s, uri)
+		if row["kind"] == "comment":
+			self.lm(s, "PUT", "comment", {"comment_id": row["id"], "content": spec.text})
+		else:
+			self.lm(s, "PUT", "post", {"post_id": row["id"], "body": spec.text})
+
+	def delete(self, s, uri):
+		row = self._find(s, uri)
+		# 1.0 deletes with DELETE on the object's own route (0.19 had POST .../delete)
+		if row["kind"] == "comment":
+			self.lm(s, "DELETE", "comment", {"comment_id": row["id"], "deleted": True})
+		elif row["kind"] == "post":
+			self.lm(s, "DELETE", "post", {"post_id": row["id"], "deleted": True})
+		else:
+			self.lm(s, "DELETE", "private_message", {"private_message_id": row["id"], "deleted": True})
+
+	def _vote(self, s, uri, up):
+		row = self._find(s, uri)
+		if row["kind"] == "comment":
+			self.lm(s, "POST", "comment/like", {"comment_id": row["id"], "is_upvote": up})
+		else:
+			self.lm(s, "POST", "post/like", {"post_id": row["id"], "is_upvote": up})
+
+	def like(self, s, uri):
+		self._vote(s, uri, True)
+
+	def downvote(self, s, uri):
+		self._vote(s, uri, False)
+
+	def report(self, s, acct, uris, comment):
+		for uri in uris:
+			row = self._find(s, uri)
+			path = "comment/report" if row["kind"] == "comment" else "post/report"
+			key = "comment_id" if row["kind"] == "comment" else "post_id"
+			self.lm(s, "POST", path, {key: row["id"], "reason": comment})
+
+	# -- reading back
+	def local_status_id(self, s, uri):
+		row = self._rows([uri]).get(uri)
+		return row.local_id if row and not row.deleted else None
+
+	def stored(self, uris):
+		rows = self._rows(uris)
+		return {u: rows.get(u) or Stored(False) for u in uris}
+
+	def seen(self, s, uris):
+		return {u: self.local_status_id(s, u) is not None for u in uris}
+
+	def actor_uri(self, username):
+		return f"{self.base}/u/{username}"
+
+	def _rows(self, uris):
+		if not uris:
+			return {}
+		rows = podman.psql(self.db, """
+			select 'post' as kind, p.id, p.id as post_id, p.ap_id, p.deleted or p.removed as deleted, p.body as text,
+				p.name as title, p.upvotes, p.downvotes, p.comments as replies, p.updated_at is not null as edited,
+				null::text as parent_uri
+			from post p where p.ap_id = any(string_to_array(:'p1', ' '))
+			union all
+			select 'comment', c.id, c.post_id, c.ap_id, c.deleted or c.removed, c.content, null, c.upvotes, c.downvotes,
+				c.child_count, c.updated_at is not null,
+				coalesce((select pc.ap_id from comment pc where pc.id::text = split_part(c.path::text, '.', nlevel(c.path) - 1)
+					and nlevel(c.path) > 2), (select pp.ap_id from post pp where pp.id = c.post_id))
+			from comment c where c.ap_id = any(string_to_array(:'p1', ' '))
+			union all
+			select 'pm', m.id, null, m.ap_id, m.deleted or m.removed, m.content, null, 0, 0, 0, m.updated_at is not null, null
+			from private_message m where m.ap_id = any(string_to_array(:'p1', ' '))""", " ".join(uris))
+		out = {}
+		for r in rows:
+			prefix = {"post": "p", "comment": "c", "pm": "m"}[r["kind"]]
+			out[r["ap_id"]] = Stored(True, bool(r["deleted"]), f"{prefix}{r['id']}", "public" if r["kind"] != "pm" else "direct",
+									 r["text"], None, bool(r["edited"]), r["parent_uri"], r["upvotes"], None, r["replies"],
+									 None, {"down": r["downvotes"]}, r)
+		return out

tools/pasture/town/dialects/mastodon.py

@@ -0,0 +1,57 @@
+"""Mastodon 4.7: accounts and tokens made in one `rails runner` (it has no password grant, and tootctl boots Rails once
+per account), objects read from its Postgres database (`mastodon`)."""
+import json
+
+from core import podman
+from dialects.base import Stored
+from dialects.mastodon_api import MastodonApi
+
+VIS = {0: "public", 1: "unlisted", 2: "followers", 3: "direct", 4: "limited"}
+
+PROVISION = r'''
+app = Doorkeeper::Application.find_or_create_by!(name: "pasture-town") { |a| a.redirect_uri = "urn:ietf:wg:oauth:2.0:oob"; a.scopes = "read write follow" }
+out = {}
+JSON.parse(STDIN.read).each do |u|
+  account = Account.find_local(u["username"])
+  if account.nil?
+    account = Account.new(username: u["username"])
+    user = User.new(email: "#{u["username"]}@mastodon.test", password: u["password"], agreement: true, approved: true,
+                    confirmed_at: Time.now.utc, account: account)
+    user.save!
+    account.reload
+  end
+  account.user.approve! unless account.user.approved?
+  account.user.confirm unless account.user.confirmed?
+  token = Doorkeeper::AccessToken.find_or_create_for(application: app, resource_owner: account.user, scopes: Doorkeeper::OAuth::Scopes.from_string("read write follow"), expires_in: nil, use_refresh_token: false)
+  out[u["username"]] = token.token
+end
+puts "TOWN" + out.to_json
+'''
+
+
+class Mastodon(MastodonApi):
+	platform = "mastodon"
+	caps = MastodonApi.caps | {"quote"}
+
+	def provision(self, accounts):
+		payload = json.dumps([{"username": a.username, "password": a.password} for a in accounts])
+		out = podman.exec_("pasture-mastodon", "bin/rails", "runner", PROVISION, input=payload, timeout=600)
+		tokens = json.loads(next(line[4:] for line in out.splitlines() if line.startswith("TOWN")))
+		return [self.session_from_token(a, tokens[a.username]) for a in accounts]
+
+	def _rows(self, uris):
+		if not uris:
+			return {}
+		rows = podman.psql("mastodon", """
+			select s.id::text as local_id, s.uri, s.deleted_at is not null as deleted, s.visibility, s.text,
+				s.spoiler_text as cw, s.edited_at is not null as edited, p.uri as parent_uri,
+				coalesce(st.favourites_count, 0) as likes, coalesce(st.reblogs_count, 0) as boosts,
+				coalesce(st.replies_count, 0) as replies, pl.cached_tallies as votes
+			from statuses s
+			left join statuses p on p.id = s.in_reply_to_id
+			left join status_stats st on st.status_id = s.id
+			left join polls pl on pl.status_id = s.id
+			where s.reblog_of_id is null and s.uri = any(string_to_array(:'p1', ' '))""", " ".join(uris))
+		return {r["uri"]: Stored(True, r["deleted"], r["local_id"], VIS.get(r["visibility"]), r["text"], r["cw"] or None,
+								 r["edited"], r["parent_uri"], r["likes"], r["boosts"], r["replies"], r["votes"], None, r)
+				for r in rows}

tools/pasture/town/dialects/mastodon_api.py

@@ -0,0 +1,252 @@
+"""Everything that speaks the Mastodon client API: Mastodon, GoToSocial, Akkoma, and later Pleroma, Iceshrimp.NET, Hollo,
+snac, Friendica, Pixelfed, Mitra. Subclasses provide accounts and tokens (each platform makes them its own way) and
+`_rows(uris)`, their database's view of a set of objects."""
+import time
+
+from core import media
+from core.http import HttpError
+from dialects.base import Driver, Made, Session, Stored, Unsupported
+
+VISIBILITY = {"public": "public", "unlisted": "unlisted", "followers": "private", "direct": "direct"}
+
+
+class MastodonApi(Driver):
+	platform = "mastodon-api"
+	caps = frozenset({"post", "reply", "cw", "media", "poll", "like", "boost", "bookmark", "follow", "block", "mute",
+					  "report", "dm", "delete", "edit", "vote", "profile"})
+	media_endpoint = "/api/v2/media"
+	max_media = 4
+
+	def __init__(self, host):
+		super().__init__(host)
+		self._ids = {}  # (session token, acct) -> local account id
+
+	# -- plumbing
+	def api(self, s, method, path, ok=None, **kw):
+		headers = kw.pop("headers", {})
+		if s is not None:
+			headers["Authorization"] = f"Bearer {s.token}"
+		return self.http.request(method, self.base + path, headers=headers, ok=ok, template=path.split("?")[0], **kw)
+
+	def api_json(self, s, method, path, **kw):
+		return self.api(s, method, path, ok={200, 201, 202}, **kw).json()
+
+	def session_from_token(self, account, token):
+		me = self.api_json(None, "GET", "/api/v1/accounts/verify_credentials", headers={"Authorization": f"Bearer {token}"})
+		actor = self.actor_uri(account.username) or me.get("url")
+		return Session(account, token, me["id"], actor)
+
+	# -- profiles
+	def update_profile(self, s, account):
+		form = {"display_name": account.name, "note": account.bio, "locked": account.locked, "bot": account.bot}
+		for i, (k, v) in enumerate(account.fields[:4]):
+			form[f"fields_attributes[{i}][name]"] = k
+			form[f"fields_attributes[{i}][value]"] = v
+		files = {}
+		if account.avatar_seed:
+			files["avatar"] = media.upload("image", account.avatar_seed)
+		if account.header_seed:
+			files["header"] = media.upload("image", account.header_seed)
+		self.api(s, "PATCH", "/api/v1/accounts/update_credentials", ok={200}, form=form, files=files or None)
+
+	# -- accounts as this server knows them
+	def lookup(self, s, acct):
+		key = (s.token, acct)
+		if key in self._ids:
+			return self._ids[key]
+		found = None
+		r = self.api(s, "GET", "/api/v1/accounts/lookup", params={"acct": acct})
+		if r.ok and r.json():
+			found = r.json()["id"]
+		if found is None:
+			r = self.api(s, "GET", "/api/v2/search", params={"q": f"@{acct}", "resolve": "true", "type": "accounts"})
+			for a in (r.json() or {}).get("accounts", []) if r.ok else []:
+				if _same_acct(a.get("acct"), acct, self.host):
+					found = a["id"]
+					break
+		if found is None:
+			raise LookupError(f"{self.host} cannot find {acct}")
+		self._ids[key] = found
+		return found
+
+	def follow(self, s, acct):
+		rel = self.api_json(s, "POST", f"/api/v1/accounts/{self.lookup(s, acct)}/follow")
+		return "accepted" if rel.get("following") else "requested"
+
+	def unfollow(self, s, acct):
+		self.api(s, "POST", f"/api/v1/accounts/{self.lookup(s, acct)}/unfollow", ok={200})
+
+	def pending(self, s):
+		r = self.api(s, "GET", "/api/v1/follow_requests", params={"limit": 80}, ok={200})
+		return [_full_acct(a["acct"], self.host) for a in r.json() or []]
+
+	def accept(self, s, acct):
+		self.api(s, "POST", f"/api/v1/follow_requests/{self.lookup(s, acct)}/authorize", ok={200})
+
+	def reject(self, s, acct):
+		self.api(s, "POST", f"/api/v1/follow_requests/{self.lookup(s, acct)}/reject", ok={200})
+
+	def relationship(self, s, acct):
+		rels = self.api_json(s, "GET", "/api/v1/accounts/relationships", params={"id": [self.lookup(s, acct)]})
+		r = rels[0] if rels else {}
+		return {k: bool(r.get(k)) for k in ("following", "followed_by", "requested", "blocking", "muting", "blocked_by")}
+
+	def block(self, s, acct):
+		self.api(s, "POST", f"/api/v1/accounts/{self.lookup(s, acct)}/block", ok={200})
+
+	def unblock(self, s, acct):
+		self.api(s, "POST", f"/api/v1/accounts/{self.lookup(s, acct)}/unblock", ok={200})
+
+	def mute(self, s, acct):
+		self.api(s, "POST", f"/api/v1/accounts/{self.lookup(s, acct)}/mute", ok={200})
+
+	def report(self, s, acct, uris, comment):
+		ids = [i for i in (self.local_status_id(s, u) for u in uris) if i]
+		self.api(s, "POST", "/api/v1/reports", ok={200}, form={"account_id": self.lookup(s, acct), "status_ids": ids,
+																 "comment": comment, "forward": True})
+
+	# -- content
+	def upload(self, s, item):
+		filename, content, ctype = media.upload(item["kind"], item["seed"])
+		r = self.api(s, "POST", self.media_endpoint, files={"file": (filename, content, ctype)},
+					 form={"description": item.get("alt")})
+		if r.status == 404 and self.media_endpoint != "/api/v1/media":
+			r = self.api(s, "POST", "/api/v1/media", files={"file": (filename, content, ctype)},
+						 form={"description": item.get("alt")})
+		if not r.ok:
+			raise HttpError("POST", self.media_endpoint, r.status, r.text)
+		attachment = r.json()
+		for _ in range(60):
+			if attachment.get("url"):
+				break
+			time.sleep(1)
+			attachment = self.api_json(s, "GET", f"/api/v1/media/{attachment['id']}")
+		return attachment["id"]
+
+	def status_form(self, s, spec):
+		text = spec.text
+		for acct in spec.mentions:
+			if f"@{acct}" not in text:
+				text = f"@{acct} {text}"
+		for tag in spec.tags:
+			if f"#{tag}" not in text:
+				text = f"{text} #{tag}"
+		visibility = VISIBILITY.get(spec.visibility)
+		if visibility is None:
+			raise Unsupported(self.platform, f"post with visibility {spec.visibility}")
+		form = {"status": text, "visibility": visibility, "language": spec.language}
+		if spec.cw:
+			form["spoiler_text"] = spec.cw
+			form["sensitive"] = True
+		if spec.media:
+			form["media_ids"] = [self.upload(s, m) for m in spec.media[:self.max_media]]
+			if any(m.get("sensitive") for m in spec.media):
+				form["sensitive"] = True
+		if spec.poll:
+			form["poll[options]"] = spec.poll["options"]
+			form["poll[expires_in]"] = spec.poll.get("expires_in", 86400)
+			form["poll[multiple]"] = bool(spec.poll.get("multiple"))
+		if spec.reply_to_uri:
+			form["in_reply_to_id"] = self.local_status_id(s, spec.reply_to_uri) or self.resolve(s, spec.reply_to_uri)
+		if spec.quote_uri:
+			if "quote" not in self.caps:
+				raise Unsupported(self.platform, "quote")
+			form["quoted_status_id"] = self.local_status_id(s, spec.quote_uri) or self.resolve(s, spec.quote_uri)
+		return form
+
+	def post(self, s, spec):
+		if spec.kind not in ("note", "image", "video", "audio"):
+			raise Unsupported(self.platform, f"post a {spec.kind}")
+		status = self.api_json(s, "POST", "/api/v1/statuses", form=self.status_form(s, spec))
+		return Made(status["uri"], status["id"], status.get("url"))
+
+	def edit(self, s, uri, spec):
+		sid = self._own(s, uri)
+		text = spec.text
+		for acct in spec.mentions:
+			if f"@{acct}" not in text:
+				text = f"@{acct} {text}"
+		form = {"status": text}
+		if spec.cw:
+			form["spoiler_text"] = spec.cw
+		self.api(s, "PUT", f"/api/v1/statuses/{sid}", ok={200}, form=form)
+
+	def delete(self, s, uri):
+		self.api(s, "DELETE", f"/api/v1/statuses/{self._own(s, uri)}", ok={200})
+
+	def _act(self, s, uri, action):
+		sid = self.local_status_id(s, uri) or self.resolve(s, uri)
+		self.api(s, "POST", f"/api/v1/statuses/{sid}/{action}", ok={200})
+
+	def like(self, s, uri):
+		self._act(s, uri, "favourite")
+
+	def unlike(self, s, uri):
+		self._act(s, uri, "unfavourite")
+
+	def boost(self, s, uri):
+		self._act(s, uri, "reblog")
+
+	def unboost(self, s, uri):
+		self._act(s, uri, "unreblog")
+
+	def bookmark(self, s, uri):
+		self._act(s, uri, "bookmark")
+
+	def vote(self, s, uri, choices):
+		sid = self.local_status_id(s, uri) or self.resolve(s, uri)
+		status = self.api_json(s, "GET", f"/api/v1/statuses/{sid}")
+		poll = status.get("poll")
+		if not poll:
+			raise LookupError(f"{self.host} shows no poll on {uri}")
+		self.api(s, "POST", f"/api/v1/polls/{poll['id']}/votes", ok={200}, form={"choices": choices})
+
+	def _own(self, s, uri):
+		sid = self.local_status_id(s, uri)
+		if sid is None:
+			raise LookupError(f"{self.host} does not hold {uri}")
+		return sid
+
+	# -- reading back
+	def resolve(self, s, uri):
+		r = self.api(s, "GET", "/api/v2/search", params={"q": uri, "resolve": "true", "type": "statuses"})
+		for st in (r.json() or {}).get("statuses", []) if r.ok else []:
+			if st.get("uri") == uri or st.get("url") == uri:
+				return st["id"]
+		raise LookupError(f"{self.host} cannot resolve {uri}")
+
+	def local_status_id(self, s, uri):
+		row = self._rows([uri]).get(uri)
+		return row.local_id if row and row.exists and not row.deleted else None
+
+	def stored(self, uris):
+		rows = self._rows(uris)
+		return {u: rows.get(u) or Stored(False) for u in uris}
+
+	def visible(self, s, local_id):
+		return self.api(s, "GET", f"/api/v1/statuses/{local_id}").status == 200
+
+	def status(self, s, local_id):
+		r = self.api(s, "GET", f"/api/v1/statuses/{local_id}")
+		return r.json() if r.ok else None
+
+	def notifications(self, s):
+		r = self.api(s, "GET", "/api/v1/notifications", params={"limit": 80}, ok={200})
+		out = []
+		for n in r.json() or []:
+			out.append({"type": n["type"], "acct": _full_acct((n.get("account") or {}).get("acct", ""), self.host),
+						"uri": (n.get("status") or {}).get("uri")})
+		return out
+
+	def _rows(self, uris):
+		raise Unsupported(self.platform, "read its database")
+
+
+def _full_acct(acct, host):
+	return acct if "@" in acct else f"{acct}@{host}"
+
+
+def _same_acct(found, wanted, host):
+	if not found:
+		return False
+	return _full_acct(found, host).lower() == wanted.lower()

tools/pasture/town/dialects/misskey_api.py

@@ -0,0 +1,273 @@
+"""Misskey and its forks (Sharkey, later CherryPick, Iceshrimp's Misskey API): every call is POST /api/<endpoint> with
+the token as `i`. Accounts are made by the admin made at setup (`admin/accounts/create`); objects are read from the
+`note` table. A local note has no `uri` there: its id is the last part of https://<host>/notes/<id>."""
+from core import media, podman
+from core.http import HttpError
+from dialects.base import Driver, Made, Session, Stored, Unsupported
+import os
+
+VISIBILITY = {"public": "public", "unlisted": "home", "followers": "followers", "direct": "specified"}
+VIS = {"public": "public", "home": "unlisted", "followers": "followers", "specified": "direct"}
+LIKE = "❤"
+
+
+class MisskeyApi(Driver):
+	platform = "misskey"
+	caps = frozenset({"post", "reply", "cw", "media", "poll", "like", "react", "boost", "bookmark", "follow", "block",
+					  "mute", "report", "dm", "delete", "vote", "quote", "profile"})
+	db = "misskey"
+
+	def __init__(self, host, db=None):
+		super().__init__(host)
+		self.db = db or host.split(".")[0]
+		self._ids = {}
+
+	def mk(self, s_or_token, endpoint, body=None, ok=None):
+		body = dict(body or {})
+		token = s_or_token.token if isinstance(s_or_token, Session) else s_or_token
+		if token:
+			body["i"] = token
+		r = self.http.post(f"{self.base}/api/{endpoint}", json=body, template=f"/api/{endpoint}")
+		if ok is not False and r.status not in (200, 204):
+			raise HttpError("POST", endpoint, r.status, r.text)
+		return r.json() if r.body else None
+
+	def admin_token(self):
+		here = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+		with open(os.path.join(here, ".state", f"{self.db}.token")) as f:
+			return f.read().strip()
+
+	def provision(self, accounts):
+		admin = self.admin_token()
+		sessions = []
+		existing = {r["username"] for r in podman.psql(self.db, 'select username from "user" where host is null')}
+		for a in accounts:
+			if a.username in existing:
+				r = self.http.post(f"{self.base}/api/signin-flow", json={"username": a.username, "password": a.password})
+				token = (r.json() or {}).get("i")
+				if not token:
+					raise RuntimeError(f"{self.host}: cannot sign {a.username} in ({r.status} {r.text[:200]})")
+			else:
+				token = self.mk(admin, "admin/accounts/create", {"username": a.username, "password": a.password})["token"]
+			me = self.mk(token, "i")
+			sessions.append(Session(a, token, me["id"], f"{self.base}/users/{me['id']}"))
+		return sessions
+
+	def update_profile(self, s, account):
+		body = {"name": account.name, "description": account.bio, "isLocked": account.locked, "isBot": account.bot,
+				"fields": [{"name": k, "value": v} for k, v in account.fields[:4]]}
+		if account.avatar_seed:
+			body["avatarId"] = self._upload(s, {"kind": "image", "seed": account.avatar_seed, "alt": None})
+		if account.header_seed:
+			body["bannerId"] = self._upload(s, {"kind": "image", "seed": account.header_seed, "alt": None})
+		self.mk(s, "i/update", body)
+
+	def _upload(self, s, item):
+		filename, content, ctype = media.upload(item["kind"], item["seed"])
+		form = {"i": s.token, "isSensitive": bool(item.get("sensitive"))}
+		if item.get("alt"):
+			form["comment"] = item["alt"]
+		r = self.http.post(f"{self.base}/api/drive/files/create", files={"file": (filename, content, ctype)}, form=form, ok={200})
+		return r.json()["id"]
+
+	# -- accounts
+	def lookup(self, s, acct):
+		key = (s.token, acct)
+		if key not in self._ids:
+			name, _, host = acct.partition("@")
+			body = {"username": name}
+			if host and host != self.host:
+				body["host"] = host
+			self._ids[key] = self.mk(s, "users/show", body)["id"]
+		return self._ids[key]
+
+	def follow(self, s, acct):
+		self.mk(s, "following/create", {"userId": self.lookup(s, acct)}, ok=False)
+		rel = self.relationship(s, acct)
+		return "accepted" if rel["following"] else "requested"
+
+	def unfollow(self, s, acct):
+		self.mk(s, "following/delete", {"userId": self.lookup(s, acct)})
+
+	def pending(self, s):
+		out = []
+		for req in self.mk(s, "following/requests/list", {"limit": 100}) or []:
+			u = req["follower"]
+			out.append(f"{u['username']}@{u.get('host') or self.host}")
+		return out
+
+	def accept(self, s, acct):
+		self.mk(s, "following/requests/accept", {"userId": self.lookup(s, acct)})
+
+	def reject(self, s, acct):
+		self.mk(s, "following/requests/reject", {"userId": self.lookup(s, acct)})
+
+	def relationship(self, s, acct):
+		r = self.mk(s, "users/relation", {"userId": self.lookup(s, acct)})
+		r = r[0] if isinstance(r, list) else r
+		return {"following": r.get("isFollowing"), "followed_by": r.get("isFollowed"),
+				"requested": r.get("hasPendingFollowRequestFromYou"), "blocking": r.get("isBlocking"),
+				"muting": r.get("isMuted"), "blocked_by": r.get("isBlocked")}
+
+	def block(self, s, acct):
+		self.mk(s, "blocking/create", {"userId": self.lookup(s, acct)})
+
+	def unblock(self, s, acct):
+		self.mk(s, "blocking/delete", {"userId": self.lookup(s, acct)})
+
+	def mute(self, s, acct):
+		self.mk(s, "mute/create", {"userId": self.lookup(s, acct)})
+
+	def report(self, s, acct, uris, comment):
+		self.mk(s, "users/report-abuse", {"userId": self.lookup(s, acct), "comment": comment})
+
+	# -- content
+	def post(self, s, spec):
+		if spec.kind not in ("note", "image", "video", "audio"):
+			raise Unsupported(self.platform, f"post a {spec.kind}")
+		visibility = VISIBILITY.get(spec.visibility)
+		if visibility is None:
+			raise Unsupported(self.platform, f"post with visibility {spec.visibility}")
+		text = spec.text
+		for acct in spec.mentions:
+			if f"@{acct}" not in text:
+				text = f"@{acct} {text}"
+		for tag in spec.tags:
+			if f"#{tag}" not in text:
+				text = f"{text} #{tag}"
+		body = {"text": text, "visibility": visibility}
+		if spec.cw:
+			body["cw"] = spec.cw
+		if visibility == "specified":
+			body["visibleUserIds"] = [self.lookup(s, a) for a in spec.mentions]
+		if spec.media:
+			body["fileIds"] = [self._upload(s, m) for m in spec.media[:16]]
+		if spec.poll:
+			body["poll"] = {"choices": spec.poll["options"], "multiple": bool(spec.poll.get("multiple")),
+							"expiredAfter": spec.poll.get("expires_in", 86400) * 1000}
+		if spec.reply_to_uri:
+			body["replyId"] = self.local_status_id(s, spec.reply_to_uri) or self.resolve(s, spec.reply_to_uri)
+		if spec.quote_uri:
+			body["renoteId"] = self.local_status_id(s, spec.quote_uri) or self.resolve(s, spec.quote_uri)
+		note = self.mk(s, "notes/create", body)["createdNote"]
+		return Made(note.get("uri") or f"{self.base}/notes/{note['id']}", note["id"], note.get("url"))
+
+	def edit(self, s, uri, spec):
+		if "edit" not in self.caps:
+			raise Unsupported(self.platform, "edit")
+		text = spec.text
+		for acct in spec.mentions:
+			if f"@{acct}" not in text:
+				text = f"@{acct} {text}"
+		body = {"editId": self._own(uri), "text": text, "cw": spec.cw}
+		self.mk(s, "notes/edit", body)
+
+	def delete(self, s, uri):
+		self.mk(s, "notes/delete", {"noteId": self._own(uri)})
+
+	def _note(self, s, uri):
+		return self.local_status_id(s, uri) or self.resolve(s, uri)
+
+	def like(self, s, uri):
+		self.mk(s, "notes/reactions/create", {"noteId": self._note(s, uri), "reaction": LIKE})
+
+	def unlike(self, s, uri):
+		self.mk(s, "notes/reactions/delete", {"noteId": self._note(s, uri)})
+
+	def react(self, s, uri, emoji):
+		self.mk(s, "notes/reactions/create", {"noteId": self._note(s, uri), "reaction": emoji})
+
+	def boost(self, s, uri):
+		self.mk(s, "notes/create", {"renoteId": self._note(s, uri), "visibility": "public"})
+
+	def unboost(self, s, uri):
+		self.mk(s, "notes/unrenote", {"noteId": self._note(s, uri)})
+
+	def bookmark(self, s, uri):
+		self.mk(s, "notes/favorites/create", {"noteId": self._note(s, uri)})
+
+	def vote(self, s, uri, choices):
+		nid = self._note(s, uri)
+		for c in choices:
+			self.mk(s, "notes/polls/vote", {"noteId": nid, "choice": c})
+
+	def _own(self, uri):
+		nid = self.local_status_id(None, uri)
+		if nid is None:
+			raise LookupError(f"{self.host} does not hold {uri}")
+		return nid
+
+	# -- reading back
+	def resolve(self, s, uri):
+		r = self.mk(s, "ap/show", {"uri": uri}, ok=False)
+		if r and r.get("type") == "Note":
+			return r["object"]["id"]
+		raise LookupError(f"{self.host} cannot resolve {uri}")
+
+	def local_status_id(self, s, uri):
+		row = self._rows([uri]).get(uri)
+		return row.local_id if row else None
+
+	def stored(self, uris):
+		rows = self._rows(uris)
+		return {u: rows.get(u) or Stored(False) for u in uris}
+
+	def seen(self, s, uris):
+		out = {}
+		for uri in uris:
+			nid = self.local_status_id(s, uri)
+			if nid is None:
+				out[uri] = False
+				continue
+			r = self.http.post(f"{self.base}/api/notes/show", json={"i": s.token, "noteId": nid})
+			note = r.json() if r.ok else None
+			out[uri] = bool(note) and not note.get("isHidden")
+		return out
+
+	def notifications(self, s):
+		kinds = {"reaction": "favourite", "renote": "reblog", "reply": "mention", "mention": "mention",
+				 "follow": "follow", "receiveFollowRequest": "follow_request", "quote": "quote", "pollEnded": "poll"}
+		out = []
+		for n in self.mk(s, "i/notifications", {"limit": 100}) or []:
+			u = n.get("user") or {}
+			note = n.get("note") or {}
+			out.append({"type": kinds.get(n["type"], n["type"]), "acct": f"{u.get('username')}@{u.get('host') or self.host}",
+						"uri": note.get("uri") or (f"{self.base}/notes/{note['id']}" if note.get("id") else None)})
+		return out
+
+	def actor_uri(self, username):
+		row = podman.psql(self.db, 'select id from "user" where host is null and username = :\'p1\'', username)
+		return f"{self.base}/users/{row[0]['id']}" if row else None
+
+	def _rows(self, uris):
+		if not uris:
+			return {}
+		local = {u: u.rsplit("/", 1)[1] for u in uris if u.startswith(f"{self.base}/notes/")}
+		edited = "n.\"updatedAt\" is not null" if self.platform == "sharkey" else "false"
+		rows = podman.psql(self.db, f"""
+			select n.id, n.uri, n.visibility, n.text, n.cw, {edited} as edited, n."renoteCount" as boosts,
+				n."repliesCount" as replies, n.reactions, p.votes, coalesce(r.uri, case when r.id is not null
+					then 'https://{self.host}/notes/' || r.id end) as parent_uri
+			from note n left join poll p on p."noteId" = n.id left join note r on r.id = n."replyId"
+			where (n."renoteId" is null or n.text is not null)
+				and (n.uri = any(string_to_array(:'p1', ' ')) or n.id = any(string_to_array(:'p2', ' ')))""",
+						   " ".join(uris), " ".join(local.values()) or "-")
+		out = {}
+		for r in rows:
+			uri = r["uri"] or f"{self.base}/notes/{r['id']}"
+			if uri not in uris:
+				continue
+			reactions = r["reactions"] or {}
+			out[uri] = Stored(True, False, r["id"], VIS.get(r["visibility"], r["visibility"]), r["text"], r["cw"],
+							  bool(r["edited"]), r["parent_uri"], sum(reactions.values()), r["boosts"], r["replies"],
+							  r["votes"], reactions, r)
+		return out
+
+
+class Misskey(MisskeyApi):
+	platform = "misskey"
+
+
+class Sharkey(MisskeyApi):
+	platform = "sharkey"
+	caps = MisskeyApi.caps | {"edit"}

tools/pasture/town/dialects/privapub.py

@@ -0,0 +1,133 @@
+"""PrivaPub: roots signed up on /clientapi, personas made under them, and each persona's Mastodon token exchanged for the
+root's JWT exactly as decePubClient does (RFC 8693, client `decepub`). Everything a Mastodon client can say goes
+through the Mastodon API; circles, communities and located posts through /clientapi. Objects are read from Mongo."""
+from core import podman
+from dialects.base import Made, Session, Stored, Unsupported
+from dialects.mastodon_api import MastodonApi
+
+VIS = {0: "public", 1: "unlisted", 2: "followers", 3: "direct", 4: "circle", 5: "located",
+	   "Public": "public", "Unlisted": "unlisted", "FollowersOnly": "followers", "Direct": "direct", "Circle": "circle",
+	   "LocalGeo": "located"}
+
+
+class PrivaPub(MastodonApi):
+	platform = "privapub"
+	caps = MastodonApi.caps | {"react", "quote", "circle", "community", "located"}
+
+	def __init__(self, host="privapub.test"):
+		super().__init__(host)
+		self._jwts = {}
+
+	# -- roots and personas
+	def jwt(self, root, password):
+		if root in self._jwts:
+			return self._jwts[root]
+		body = {"userName": root, "password": password}
+		r = self.http.post(self.base + "/clientapi/user/signup", json=body)
+		token = (r.json() or {}).get("token") if r.ok else None
+		if not token:
+			token = self.http.post(self.base + "/clientapi/user/login", json=body, ok={200}).json()["token"]
+		self._jwts[root] = token
+		return token
+
+	def personas(self, root, password):
+		jwt = self.jwt(root, password)
+		r = self.http.get(self.base + "/clientapi/avatar/private/list", headers={"Authorization": f"Bearer {jwt}"}, ok={200})
+		return {p["userName"]: p for p in r.json() or []}
+
+	def provision(self, accounts):
+		"""Each account is a persona; `account.root` names its root, whose password is `account.password`."""
+		sessions = []
+		for root in dict.fromkeys(a.root for a in accounts):
+			mine = [a for a in accounts if a.root == root]
+			password = mine[0].password
+			jwt = self.jwt(root, password)
+			existing = self.personas(root, password)
+			for a in mine:
+				if a.username not in existing:
+					self.http.post(self.base + "/clientapi/avatar/private/insert", ok={200, 201},
+								   headers={"Authorization": f"Bearer {jwt}"},
+								   json={"userName": a.username, "name": a.name or a.username, "biography": a.bio or a.name or a.username,
+										 "fields": dict(a.fields)})
+			existing = self.personas(root, password)
+			for a in mine:
+				token = self.exchange(jwt, existing[a.username]["id"])
+				sessions.append(self.session_from_token(a, token))
+		by_name = {s.account.username: s for s in sessions}
+		return [by_name[a.username] for a in accounts]
+
+	def exchange(self, jwt, avatar_id):
+		return self.http.post(self.base + "/oauth/token", ok={200}, form={
+			"grant_type": "urn:ietf:params:oauth:grant-type:token-exchange", "client_id": "decepub",
+			"subject_token": jwt, "subject_token_type": "urn:ietf:params:oauth:token-type:jwt",
+			"avatar_id": avatar_id, "scope": "read write follow"}).json()["access_token"]
+
+	def actor_uri(self, username):
+		return f"https://{self.host}/peasants/{username}"
+
+	# -- groups: communities (FEP-1b12) and circles
+	def group(self, s, username, name, community, policy="followers", approve=False):
+		jwt = self._jwts[s.account.root]
+		r = self.http.post(self.base + "/clientapi/group/insert", headers={"Authorization": f"Bearer {jwt}"}, ok={200, 201},
+						   json={"avatarId": s.local_id, "userName": username, "name": name, "description": name,
+								 "isCommunity": community, "postingPolicy": policy, "isDiscoverable": community,
+								 "manuallyApprovesMembers": approve})
+		return r.json()
+
+	# -- content
+	def post(self, s, spec):
+		if spec.visibility in ("circle", "community", "located"):
+			return self._client_post(s, spec)
+		return super().post(s, spec)
+
+	def _client_post(self, s, spec):
+		jwt = self._jwts[s.account.root]
+		body = {"avatarId": s.local_id, "text": spec.text, "title": spec.title, "hasContentWarning": bool(spec.cw),
+				"spoilerText": spec.cw}
+		if spec.visibility == "located":
+			loc = spec.location
+			body.update({"latitude": loc["lat"], "longitude": loc["lng"], "rangeKm": loc.get("range_km", 5)})
+		else:
+			body["groupId"] = spec.group
+		if spec.reply_to_uri:
+			body["answeringToPostId"] = self.local_status_id(s, spec.reply_to_uri)
+		r = self.http.post(self.base + "/clientapi/post/insert", headers={"Authorization": f"Bearer {jwt}"},
+						   ok={200, 201}, json=body).json()
+		post_id = r.get("id") if isinstance(r, dict) else None
+		if not post_id:
+			raise RuntimeError(f"/clientapi/post/insert answered {r!r}")
+		row = podman.mongo(f"db.Post.findOne({{_id: ObjectId('{post_id}')}}, {{ObjectURI: 1}})")
+		return Made(row["ObjectURI"], post_id, None)
+
+	def react(self, s, uri, emoji):
+		import urllib.parse
+		sid = self.local_status_id(s, uri) or self.resolve(s, uri)
+		self.api(s, "PUT", f"/api/v1/pleroma/statuses/{sid}/reactions/{urllib.parse.quote(emoji)}", ok={200})
+
+	# -- reading back
+	def _rows(self, uris):
+		if not uris:
+			return {}
+		import json
+		docs = podman.mongo(f"db.Post.find({{ObjectURI: {{$in: {json.dumps(list(uris))}}}}}, "
+							"{ObjectURI: 1, Visibility: 1, Text: 1, ContentHtml: 1, SpoilerText: 1, EditedAt: 1, "
+							"InReplyToURI: 1, FavouritesCount: 1, ReblogsCount: 1, RepliesCount: 1, DownvotesCount: 1, "
+							"Poll: 1, DeletedAt: 1, ReblogOfPostId: 1, AuthorGone: 1, GroupId: 1, ConversationId: 1}).toArray()") or []
+		ids = [d["_id"]["$oid"] if isinstance(d["_id"], dict) else d["_id"] for d in docs]
+		reactions = {}
+		if ids:
+			for g in podman.mongo(f"db.Reaction.aggregate([{{$match: {{PostId: {{$in: {json.dumps(ids)}}}}}}}, "
+								  "{$group: {_id: {p: '$PostId', e: '$Emoji'}, n: {$sum: 1}}}]).toArray()") or []:
+				reactions.setdefault(g["_id"]["p"], {})[g["_id"]["e"]] = g["n"]
+		out = {}
+		for d in docs:
+			if d.get("ReblogOfPostId"):
+				continue
+			pid = d["_id"]["$oid"] if isinstance(d["_id"], dict) else d["_id"]
+			poll = d.get("Poll") or {}
+			votes = [o.get("Votes", o.get("VotesCount", 0)) for o in poll.get("Options", [])] if poll else None
+			out[d["ObjectURI"]] = Stored(True, bool(d.get("DeletedAt")), pid, VIS.get(d.get("Visibility"), str(d.get("Visibility"))),
+										 d.get("Text") or d.get("ContentHtml"), d.get("SpoilerText") or None,
+										 bool(d.get("EditedAt")), d.get("InReplyToURI"), d.get("FavouritesCount", 0),
+										 d.get("ReblogsCount", 0), d.get("RepliesCount", 0), votes, reactions.get(pid, {}), d)
+		return out