Town: one report for several runs
A village and the pair runs (Hollo, Iceshrimp.NET, Pleroma) are checked apart; town.sh report takes them together and adds their cells up, so one matrix shows every server. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01LsXgEaXee4GCU1hwYgPJXw
This commit is contained in:
1 parent
40217f3261
commit
b274ce1268
3 files changed
+26
-5
No files matched your search
@@ -138,12 +138,32 @@ def render(result, scenarios, title):
|
||||
return "\n".join(out)
|
||||
|
||||
|
||||
def merge(results):
|
||||
"""Several runs' results as one: each cell's counts added up (a pair run's cells and the village's meet in one
|
||||
matrix), its examples kept, the runs named together."""
|
||||
if len(results) == 1:
|
||||
return results[0]
|
||||
cells, totals = {}, Counter()
|
||||
for r in results:
|
||||
totals.update(Counter(r["totals"]))
|
||||
for key, v in r["cells"].items():
|
||||
cell = cells.setdefault(key, {"n": 0, "pass": 0, "fail": 0, "xfail": 0, "xpass": 0, "examples": []})
|
||||
for k in ("n", "pass", "fail", "xfail", "xpass"):
|
||||
cell[k] += v.get(k, 0)
|
||||
cell["examples"] = (cell["examples"] + v.get("examples", []))[:3]
|
||||
return {"run": " + ".join(os.path.basename(r["run"]) for r in results), "when": max(r["when"] for r in results),
|
||||
"cells": cells, "totals": dict(totals)}
|
||||
|
||||
|
||||
def main(args):
|
||||
import seed
|
||||
name = next((a for a in args if not a.startswith("--")), "village")
|
||||
run_dir = seed.run_dir(name)
|
||||
with open(os.path.join(run_dir, "results.json")) as f:
|
||||
result = json.load(f)
|
||||
names = [a for a in args if not a.startswith("--")] or ["village"]
|
||||
run_dir = seed.run_dir(names[0])
|
||||
results = []
|
||||
for name in names:
|
||||
with open(os.path.join(seed.run_dir(name), "results.json")) as f:
|
||||
results.append(json.load(f))
|
||||
result = merge(results)
|
||||
scenarios = []
|
||||
path = os.path.join(seed.OUT, "scenarios.jsonl")
|
||||
if os.path.exists(path):
|
||||
|
||||
@@ -8,7 +8,7 @@
|
||||
plan <spec> the deterministic plan of a spec, printed as JSON lines
|
||||
seed <spec> replays a spec's plan against the pasture, writing out/<run>/ledger.jsonl
|
||||
check [run] sweeps every peer for what the ledger expects
|
||||
report [run] out/<run>/report.html
|
||||
report [run...] out/<run>/report.html; several runs (a village and pairs) in one matrix
|
||||
backlog [run] [--write] docs/INTEROP-BACKLOG.md from the latest results
|
||||
"""
|
||||
import json
|
||||
|
||||
Reference in new issue
Block a user