The town: a fake community across the pasture, checked for coherence

tools/pasture/town/ (run through tools/pasture/town.sh) seeds a fake
community across every running peer and checks that all of them, and
PrivaPub, agree on what happened:

- drivers per platform on four dialect bases (Mastodon API, Misskey API,
  Lemmy API, PrivaPub with /clientapi), each with a selftest against
  its own server; what a server holds is read from its database, never
  by making it fetch;
- a deterministic generator (specs/village.json: 23 accounts on seven
  servers, roots with several personas, circles and communities, a
  cross-server follow graph, posts of every kind and visibility, reply
  rounds, likes, boosts, reactions, votes, edits, deletes, blocks,
  mutes and a report) and a seeder that keeps a ledger of what happened;
- a sweep that expects delivery and confinement per server, what each
  account sees, counts, threads, edits, deletes, follows and privacy
  rows (sibling keys, published days, canary root credentials in every
  peer's database, located posts that never leave), with what the peers
  do on purpose modelled (Misskey drops orphan replies, Lemmy keeps only
  community content, edits go to the post's own audience);
- known gaps (gaps.json) turn failures into xfail and passes into xpass;
  a self-contained report.html, and docs/INTEROP-BACKLOG.md.

The pasture moves to a public-looking subnet (peers with no private
address switch can join), takes PASTURE_PORT when 6971 is in use, adds
peers to a running pasture (run.sh add, Caddy recreated with its CA
kept), removes its volumes on down, writes every scenario check to
out/scenarios.jsonl, serves decePub as decepub.test for its e2e tests,
lifts GoToSocial's and Lemmy's own rate limits, trusts Caddy in
Mastodon (TRUSTED_PROXY_IP) and gains Hollo (Fedify), whose one login
owning several accounts is the nearest peer to PrivaPub's personas.

The first village found the four PrivaPub bugs fixed in the commits
before this one; the second run, on the fixed server, passes 2319 checks
with 11 failures left, all between peers or from Lemmy's send worker,
which the seeder now warms up first. ROADMAP records the owner's
decisions of 2026-10-04 (the town, and P9 back from the cut list).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01LsXgEaXee4GCU1hwYgPJXw

tools/pasture/town/core/http.py

@@ -0,0 +1,219 @@
+"""HTTP to the pasture: every https://<name>.test site through Caddy on 127.0.0.1:6443, verified against Caddy's own CA
+(.ca/root.crt), with one kept-alive connection per site and thread. 429s wait for the server's reset; connection drops
+are retried. Every request is timed into `Client.timings` for the load mode."""
+import http.client
+import json as jsonlib
+import os
+import socket
+import ssl
+import threading
+import time
+import urllib.parse
+import uuid
+
+CADDY = ("127.0.0.1", 6443)
+
+
+class HttpError(Exception):
+	def __init__(self, method, url, status, body):
+		super().__init__(f"{method} {url} -> {status}: {body[:300]!r}")
+		self.status = status
+		self.body = body
+
+
+class Response:
+	def __init__(self, status, headers, body, url):
+		self.status = status
+		self.headers = headers
+		self.body = body
+		self.url = url
+
+	@property
+	def ok(self):
+		return 200 <= self.status < 300
+
+	@property
+	def text(self):
+		return self.body.decode("utf-8", "replace")
+
+	def json(self):
+		if not self.body:
+			return None
+		try:
+			return jsonlib.loads(self.body)
+		except ValueError:
+			return None
+
+	def header(self, name):
+		for k, v in self.headers:
+			if k.lower() == name.lower():
+				return v
+		return None
+
+
+class _CaddyConnection(http.client.HTTPSConnection):
+	"""Connects to Caddy's port but speaks TLS (SNI and certificate check) as the named site."""
+
+	def __init__(self, site, context, timeout):
+		super().__init__(site, 443, context=context, timeout=timeout)
+		self._site = site
+		self._ctx = context
+
+	def connect(self):
+		sock = socket.create_connection(CADDY, self.timeout)
+		self.sock = self._ctx.wrap_socket(sock, server_hostname=self._site)
+
+
+class Client:
+	def __init__(self, ca_file, timeout=60):
+		self.ctx = ssl.create_default_context(cafile=ca_file)
+		self.timeout = timeout
+		self.local = threading.local()
+		self.timings = []  # (method, site, path template, status, ms)
+		self.lock = threading.Lock()
+		self.record = False
+
+	def _connection(self, scheme, netloc):
+		conns = getattr(self.local, "conns", None)
+		if conns is None:
+			conns = self.local.conns = {}
+		key = (scheme, netloc)
+		conn = conns.get(key)
+		if conn is None:
+			if scheme == "https":
+				conn = _CaddyConnection(netloc.split(":")[0], self.ctx, self.timeout)
+			else:
+				host, _, port = netloc.partition(":")
+				conn = http.client.HTTPConnection(host, int(port or 80), timeout=self.timeout)
+			conns[key] = conn
+		return conn
+
+	def _drop(self, scheme, netloc):
+		conn = self.local.conns.pop((scheme, netloc), None)
+		if conn is not None:
+			conn.close()
+
+	def request(self, method, url, *, headers=None, json=None, form=None, data=None, files=None, params=None,
+				ok=None, attempts=4, wait_429=True, template=None):
+		"""Sends one request. `json` is a JSON body, `form` urlencoded pairs (a dict or a list of pairs, lists expand to
+		key[]=v as Rails expects), `files` a multipart body ({name: (filename, bytes, type)} plus `form` fields).
+		`ok` is a set of accepted statuses: anything else raises HttpError."""
+		parts = urllib.parse.urlsplit(url)
+		path = parts.path or "/"
+		query = parts.query
+		if params:
+			extra = urllib.parse.urlencode(_pairs(params), doseq=True)
+			query = f"{query}&{extra}" if query else extra
+		target = path + (f"?{query}" if query else "")
+		hdrs = {"Accept": "application/json", "User-Agent": "pasture-town/1"}
+		body = None
+		if json is not None:
+			body = jsonlib.dumps(json).encode()
+			hdrs["Content-Type"] = "application/json"
+		elif files is not None:
+			boundary = uuid.uuid4().hex
+			body = _multipart(boundary, _pairs(form or {}), files)
+			hdrs["Content-Type"] = f"multipart/form-data; boundary={boundary}"
+		elif form is not None:
+			body = urllib.parse.urlencode(_pairs(form), doseq=True).encode()
+			hdrs["Content-Type"] = "application/x-www-form-urlencoded"
+		elif data is not None:
+			body = data
+		if headers:
+			hdrs.update(headers)
+		last = None
+		for attempt in range(attempts):
+			conn = self._connection(parts.scheme, parts.netloc)
+			started = time.monotonic()
+			try:
+				conn.request(method, target, body=body, headers=hdrs)
+				raw = conn.getresponse()
+				payload = raw.read()
+				resp = Response(raw.status, raw.getheaders(), payload, url)
+			except (ConnectionError, http.client.HTTPException, socket.timeout, ssl.SSLError, OSError) as e:
+				self._drop(parts.scheme, parts.netloc)
+				last = e
+				time.sleep(0.5 * (attempt + 1))
+				continue
+			ms = (time.monotonic() - started) * 1000
+			if self.record:
+				with self.lock:
+					self.timings.append((method, parts.netloc, template or path, resp.status, ms))
+			if resp.status == 429 and wait_429 and attempt < attempts - 1:
+				time.sleep(_retry_after(resp))
+				continue
+			if resp.status in (502, 503, 504) and attempt < attempts - 1:
+				time.sleep(1 + attempt)
+				continue
+			if ok is not None and resp.status not in ok:
+				raise HttpError(method, url, resp.status, resp.text)
+			return resp
+		raise HttpError(method, url, 0, repr(last))
+
+	def get(self, url, **kw):
+		return self.request("GET", url, **kw)
+
+	def post(self, url, **kw):
+		return self.request("POST", url, **kw)
+
+
+def _pairs(values):
+	"""A dict or list of pairs, with list values as Rails' key[]=v (unless the key already ends in [])."""
+	items = values.items() if isinstance(values, dict) else values
+	out = []
+	for k, v in items:
+		if v is None:
+			continue
+		if isinstance(v, (list, tuple)):
+			key = k if k.endswith("[]") else f"{k}[]"
+			out.extend((key, _scalar(x)) for x in v)
+		else:
+			out.append((k, _scalar(v)))
+	return out
+
+
+def _scalar(v):
+	if v is True:
+		return "true"
+	if v is False:
+		return "false"
+	return str(v)
+
+
+def _multipart(boundary, fields, files):
+	chunks = []
+	for k, v in fields:
+		chunks.append(f'--{boundary}\r\nContent-Disposition: form-data; name="{k}"\r\n\r\n{v}\r\n'.encode())
+	for name, (filename, content, ctype) in files.items():
+		chunks.append(f'--{boundary}\r\nContent-Disposition: form-data; name="{name}"; filename="{filename}"\r\n'
+					  f"Content-Type: {ctype}\r\n\r\n".encode() + content + b"\r\n")
+	chunks.append(f"--{boundary}--\r\n".encode())
+	return b"".join(chunks)
+
+
+def _retry_after(resp):
+	value = resp.header("Retry-After")
+	if value and value.isdigit():
+		return min(int(value), 60)
+	reset = resp.header("X-RateLimit-Reset")
+	if reset:
+		try:
+			from email.utils import parsedate_to_datetime
+			import datetime
+			when = datetime.datetime.fromisoformat(reset.replace("Z", "+00:00")) if "T" in reset \
+				else parsedate_to_datetime(reset)
+			return max(1, min(60, (when - datetime.datetime.now(datetime.timezone.utc)).total_seconds()))
+		except (ValueError, TypeError):
+			pass
+	return 5
+
+
+_client = None
+
+
+def client():
+	global _client
+	if _client is None:
+		here = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+		_client = Client(os.path.join(here, ".ca", "root.crt"))
+	return _client

tools/pasture/town/core/media.py

@@ -0,0 +1,67 @@
+"""Media made from a seed, so two runs upload the same bytes: identicon PNGs, a sine-tone WAV, and the committed
+media/tiny.mp4 and media/tiny.ogg (made once with ffmpeg; the command is in media/SOURCES.md)."""
+import hashlib
+import math
+import os
+import struct
+import zlib
+
+HERE = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+
+
+def png(seed, size=96, cells=6):
+	"""A symmetric identicon: a colour and a mirrored grid from the SHA-256 of the seed."""
+	h = hashlib.sha256(str(seed).encode()).digest()
+	fg = (h[0], h[1], h[2])
+	bg = (240, 240, 236) if sum(fg) < 380 else (34, 34, 40)
+	half = (cells + 1) // 2
+	bits = int.from_bytes(h[3:11], "big")
+	grid = [[False] * cells for _ in range(cells)]
+	for y in range(cells):
+		for x in range(half):
+			on = bool(bits & 1)
+			bits >>= 1
+			grid[y][x] = grid[y][cells - 1 - x] = on
+	cell = size // cells
+	rows = []
+	for py in range(cell * cells):
+		row = bytearray(b"\x00")
+		for px in range(cell * cells):
+			row += bytes(fg if grid[py // cell][px // cell] else bg)
+		rows.append(bytes(row))
+	w = hgt = cell * cells
+	return _png_bytes(w, hgt, b"".join(rows))
+
+
+def _png_bytes(w, h, raw):
+	def chunk(t, c):
+		return struct.pack(">I", len(c)) + t + c + struct.pack(">I", zlib.crc32(t + c) & 0xffffffff)
+	return (b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", struct.pack(">IIBBBBB", w, h, 8, 2, 0, 0, 0))
+			+ chunk(b"IDAT", zlib.compress(raw, 9)) + chunk(b"IEND", b""))
+
+
+def wav(seed, seconds=1.0, rate=8000):
+	"""A mono 8-bit tone whose pitch comes from the seed."""
+	h = hashlib.sha256(str(seed).encode()).digest()
+	freq = 220 + h[0] * 2
+	n = int(seconds * rate)
+	samples = bytes(int(128 + 90 * math.sin(2 * math.pi * freq * i / rate)) for i in range(n))
+	header = b"RIFF" + struct.pack("<I", 36 + n) + b"WAVEfmt " + struct.pack("<IHHIIHH", 16, 1, 1, rate, rate, 1, 8) \
+		+ b"data" + struct.pack("<I", n)
+	return header + samples
+
+
+def file(name):
+	with open(os.path.join(HERE, "media", name), "rb") as f:
+		return f.read()
+
+
+def upload(kind, seed):
+	"""(filename, bytes, content type) for an upload of kind image, audio or video."""
+	if kind == "image":
+		return (f"town-{seed}.png", png(seed), "image/png")
+	if kind == "audio":
+		return (f"town-{seed}.ogg", file("tiny.ogg"), "audio/ogg")
+	if kind == "video":
+		return (f"town-{seed}.mp4", file("tiny.mp4"), "video/mp4")
+	raise ValueError(kind)

tools/pasture/town/core/podman.py

@@ -0,0 +1,99 @@
+"""The pasture's containers from the workstation: commands, Postgres rows as JSON, Mongo documents as JSON, a copy of
+GoToSocial's sqlite. Only reads go through here; nothing a peer does is faked in its database."""
+import json
+import os
+import shutil
+import sqlite3
+import subprocess
+import tempfile
+
+
+def run(*args, input=None, check=True, timeout=300):
+	proc = subprocess.run(["podman", *args], input=input, capture_output=True, timeout=timeout,
+						  text=isinstance(input, str) or input is None)
+	if check and proc.returncode != 0:
+		raise RuntimeError(f"podman {' '.join(args[:3])}… failed: {proc.stderr.strip()[:500]}")
+	return proc.stdout
+
+
+def exec_(container, *cmd, input=None, check=True, timeout=300, workdir=None):
+	args = ["exec"]
+	if input is not None:
+		args.append("-i")
+	if workdir:
+		args += ["-w", workdir]
+	return run(*args, container, *cmd, input=input, check=check, timeout=timeout)
+
+
+def exists(container):
+	return subprocess.run(["podman", "container", "exists", container]).returncode == 0
+
+
+def running():
+	out = run("ps", "--format", "{{.Names}}")
+	return [n for n in out.split() if n.startswith("pasture-")]
+
+
+def psql(db, sql, *params):
+	"""Rows of `sql` as a list of dicts. Parameters are bound by psql (:'p1', :'p2'...), never pasted into the SQL."""
+	args = ["exec", "-i", "pasture-postgres", "psql", "-U", "pasture", "-d", db, "-tAX", "-v", "ON_ERROR_STOP=1"]
+	for i, p in enumerate(params, 1):
+		args += ["-v", f"p{i}={p}"]
+	wrapped = f"select coalesce(json_agg(t), '[]'::json) from ({sql.rstrip().rstrip(';')}) t;\n"
+	out = run(*args, input=wrapped)
+	return json.loads(out.strip() or "[]")
+
+
+def psql_value(db, sql, *params):
+	rows = psql(db, sql, *params)
+	if not rows:
+		return None
+	return next(iter(rows[0].values()))
+
+
+def mongo(js, db="PrivaPub"):
+	"""The JSON value of a mongosh expression, e.g. `db.Post.find({...}).toArray()`."""
+	script = f"print(EJSON.stringify(({js}), {{relaxed: true}}))"
+	out = run("exec", "pasture-mongo", "mongosh", "--quiet", db, "--eval", script, timeout=120)
+	return json.loads(out.strip().splitlines()[-1]) if out.strip() else None
+
+
+class SqliteCopy:
+	"""A consistent copy of a container's sqlite database (with its WAL), opened read-only on the workstation."""
+
+	def __init__(self, container, path):
+		self.dir = tempfile.mkdtemp(prefix="town-sqlite-")
+		local = os.path.join(self.dir, os.path.basename(path))
+		run("cp", f"{container}:{path}", local)
+		for suffix in ("-wal", "-shm"):
+			subprocess.run(["podman", "cp", f"{container}:{path}{suffix}", local + suffix], capture_output=True)
+		self.db = sqlite3.connect(local)
+		self.db.row_factory = sqlite3.Row
+
+	def rows(self, sql, *params):
+		return [dict(r) for r in self.db.execute(sql, params)]
+
+	def close(self):
+		self.db.close()
+		shutil.rmtree(self.dir, ignore_errors=True)
+
+	def __enter__(self):
+		return self
+
+	def __exit__(self, *exc):
+		self.close()
+
+
+def logs(container, since=None):
+	args = ["logs"]
+	if since:
+		args += ["--since", since]
+	return run(*args, container, check=False)
+
+
+def stats():
+	out = run("stats", "--no-stream", "--format", "json", *running(), check=False)
+	try:
+		return json.loads(out)
+	except ValueError:
+		return []

tools/pasture/town/core/rng.py

@@ -0,0 +1,93 @@
+"""A seeded random generator that is the same on every Python: xoshiro256** seeded through splitmix64 from a SHA-256
+of the seed and a name. `sub(name)` gives an independent stream, so changing how many posts are drawn never reshuffles
+the follow graph."""
+import hashlib
+
+MASK = (1 << 64) - 1
+
+
+def _rotl(x, k):
+	return ((x << k) | (x >> (64 - k))) & MASK
+
+
+def _splitmix(state):
+	while True:
+		state = (state + 0x9E3779B97F4A7C15) & MASK
+		z = state
+		z = ((z ^ (z >> 30)) * 0xBF58476D1CE4E5B9) & MASK
+		z = ((z ^ (z >> 27)) * 0x94D049BB133111EB) & MASK
+		yield z ^ (z >> 31)
+
+
+class Rng:
+	def __init__(self, seed, name="root"):
+		self.seed = seed
+		self.name = name
+		digest = hashlib.sha256(f"{seed}/{name}".encode()).digest()
+		mix = _splitmix(int.from_bytes(digest[:8], "little"))
+		self.s = [next(mix) for _ in range(4)]
+
+	def sub(self, name):
+		return Rng(self.seed, f"{self.name}/{name}")
+
+	def next64(self):
+		s = self.s
+		result = (_rotl((s[1] * 5) & MASK, 7) * 9) & MASK
+		t = (s[1] << 17) & MASK
+		s[2] ^= s[0]
+		s[3] ^= s[1]
+		s[1] ^= s[2]
+		s[0] ^= s[3]
+		s[2] ^= t
+		s[3] = _rotl(s[3], 45)
+		return result
+
+	def random(self):
+		return (self.next64() >> 11) / float(1 << 53)
+
+	def below(self, n):
+		"""An integer in [0, n)."""
+		if n <= 0:
+			raise ValueError("below() needs n > 0")
+		limit = MASK - (MASK % n)
+		while True:
+			x = self.next64()
+			if x < limit:
+				return x % n
+
+	def between(self, lo, hi):
+		"""An integer in [lo, hi]."""
+		return lo + self.below(hi - lo + 1)
+
+	def chance(self, p):
+		return self.random() < p
+
+	def choice(self, seq):
+		return seq[self.below(len(seq))]
+
+	def weighted(self, weights):
+		"""A key of {key: weight}, keys taken in sorted order so the draw does not depend on dict order."""
+		keys = sorted(k for k, w in weights.items() if w > 0)
+		total = sum(weights[k] for k in keys)
+		x = self.random() * total
+		for k in keys:
+			x -= weights[k]
+			if x < 0:
+				return k
+		return keys[-1]
+
+	def shuffle(self, items):
+		items = list(items)
+		for i in range(len(items) - 1, 0, -1):
+			j = self.below(i + 1)
+			items[i], items[j] = items[j], items[i]
+		return items
+
+	def sample(self, seq, k):
+		return self.shuffle(seq)[:max(0, min(k, len(seq)))]
+
+	def span(self, pair):
+		"""A value drawn from [lo, hi] given as a two-item list, or the value itself."""
+		if isinstance(pair, (list, tuple)):
+			return self.between(pair[0], pair[1])
+		return pair