From b274ce12681f87c6dd961296762f0fcb0b6f11b9 Mon Sep 17 00:00:00 2001 From: thepra Date: Mon, 5 Oct 2026 03:54:29 +0200 Subject: [PATCH] Town: one report for several runs A village and the pair runs (Hollo, Iceshrimp.NET, Pleroma) are checked apart; town.sh report takes them together and adds their cells up, so one matrix shows every server. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01LsXgEaXee4GCU1hwYgPJXw --- CLAUDE.md | 1 + tools/pasture/town/report.py | 28 ++++++++++++++++++++++++---- tools/pasture/town/town.py | 2 +- 3 files changed, 26 insertions(+), 5 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index f13cf9e..d937c11 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -532,6 +532,7 @@ tools/pasture/town.sh selftest [peer...] # each driver against its own server; tools/pasture/town.sh seed village # specs/village.json: 23 accounts on seven servers, about 11 minutes tools/pasture/town.sh check village # out//results.json; --deadline=0 re-sweeps an old run at once tools/pasture/town.sh report village # out//report.html, publishable (fake data, no tokens) +tools/pasture/town.sh report village hollo-pair iceshrimp-pair pleroma-pair # several runs in one matrix tools/pasture/town.sh backlog --write # docs/INTEROP-BACKLOG.md ``` diff --git a/tools/pasture/town/report.py b/tools/pasture/town/report.py index 2ac123e..7268fbd 100644 --- a/tools/pasture/town/report.py +++ b/tools/pasture/town/report.py @@ -138,12 +138,32 @@ def render(result, scenarios, title): return "\n".join(out) +def merge(results): + """Several runs' results as one: each cell's counts added up (a pair run's cells and the village's meet in one + matrix), its examples kept, the runs named together.""" + if len(results) == 1: + return results[0] + cells, totals = {}, Counter() + for r in results: + totals.update(Counter(r["totals"])) + for key, v in r["cells"].items(): + cell = cells.setdefault(key, {"n": 0, "pass": 0, "fail": 0, "xfail": 0, "xpass": 0, "examples": []}) + for k in ("n", "pass", "fail", "xfail", "xpass"): + cell[k] += v.get(k, 0) + cell["examples"] = (cell["examples"] + v.get("examples", []))[:3] + return {"run": " + ".join(os.path.basename(r["run"]) for r in results), "when": max(r["when"] for r in results), + "cells": cells, "totals": dict(totals)} + + def main(args): import seed - name = next((a for a in args if not a.startswith("--")), "village") - run_dir = seed.run_dir(name) - with open(os.path.join(run_dir, "results.json")) as f: - result = json.load(f) + names = [a for a in args if not a.startswith("--")] or ["village"] + run_dir = seed.run_dir(names[0]) + results = [] + for name in names: + with open(os.path.join(seed.run_dir(name), "results.json")) as f: + results.append(json.load(f)) + result = merge(results) scenarios = [] path = os.path.join(seed.OUT, "scenarios.jsonl") if os.path.exists(path): diff --git a/tools/pasture/town/town.py b/tools/pasture/town/town.py index 016d4ff..74da14c 100644 --- a/tools/pasture/town/town.py +++ b/tools/pasture/town/town.py @@ -8,7 +8,7 @@ plan the deterministic plan of a spec, printed as JSON lines seed replays a spec's plan against the pasture, writing out//ledger.jsonl check [run] sweeps every peer for what the ledger expects - report [run] out//report.html + report [run...] out//report.html; several runs (a village and pairs) in one matrix backlog [run] [--write] docs/INTEROP-BACKLOG.md from the latest results """ import json