diff --git a/CLAUDE.md b/CLAUDE.md index f13cf9e..d937c11 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -532,6 +532,7 @@ tools/pasture/town.sh selftest [peer...] # each driver against its own server; tools/pasture/town.sh seed village # specs/village.json: 23 accounts on seven servers, about 11 minutes tools/pasture/town.sh check village # out//results.json; --deadline=0 re-sweeps an old run at once tools/pasture/town.sh report village # out//report.html, publishable (fake data, no tokens) +tools/pasture/town.sh report village hollo-pair iceshrimp-pair pleroma-pair # several runs in one matrix tools/pasture/town.sh backlog --write # docs/INTEROP-BACKLOG.md ``` diff --git a/tools/pasture/town/report.py b/tools/pasture/town/report.py index 2ac123e..7268fbd 100644 --- a/tools/pasture/town/report.py +++ b/tools/pasture/town/report.py @@ -138,12 +138,32 @@ def render(result, scenarios, title): return "\n".join(out) +def merge(results): + """Several runs' results as one: each cell's counts added up (a pair run's cells and the village's meet in one + matrix), its examples kept, the runs named together.""" + if len(results) == 1: + return results[0] + cells, totals = {}, Counter() + for r in results: + totals.update(Counter(r["totals"])) + for key, v in r["cells"].items(): + cell = cells.setdefault(key, {"n": 0, "pass": 0, "fail": 0, "xfail": 0, "xpass": 0, "examples": []}) + for k in ("n", "pass", "fail", "xfail", "xpass"): + cell[k] += v.get(k, 0) + cell["examples"] = (cell["examples"] + v.get("examples", []))[:3] + return {"run": " + ".join(os.path.basename(r["run"]) for r in results), "when": max(r["when"] for r in results), + "cells": cells, "totals": dict(totals)} + + def main(args): import seed - name = next((a for a in args if not a.startswith("--")), "village") - run_dir = seed.run_dir(name) - with open(os.path.join(run_dir, "results.json")) as f: - result = json.load(f) + names = [a for a in args if not a.startswith("--")] or ["village"] + run_dir = seed.run_dir(names[0]) + results = [] + for name in names: + with open(os.path.join(seed.run_dir(name), "results.json")) as f: + results.append(json.load(f)) + result = merge(results) scenarios = [] path = os.path.join(seed.OUT, "scenarios.jsonl") if os.path.exists(path): diff --git a/tools/pasture/town/town.py b/tools/pasture/town/town.py index 016d4ff..74da14c 100644 --- a/tools/pasture/town/town.py +++ b/tools/pasture/town/town.py @@ -8,7 +8,7 @@ plan the deterministic plan of a spec, printed as JSON lines seed replays a spec's plan against the pasture, writing out//ledger.jsonl check [run] sweeps every peer for what the ledger expects - report [run] out//report.html + report [run...] out//report.html; several runs (a village and pairs) in one matrix backlog [run] [--write] docs/INTEROP-BACKLOG.md from the latest results """ import json