"]
out.append(f"{html.escape(title)}
"
f"Every object the town's accounts made, checked on every server it should and should not have reached, "
f"through each server's database and API. A cell's squares are the directions it was checked in: "
f"into PrivaPub, from PrivaPub, through PrivaPub, peer to peer, and on the author's own server.
"
f"{html.escape(os.path.basename(result['run']))} · checked {html.escape(result['when'])}
")
out.append('' + "".join(
f'{totals.get(s, 0)}{s}
' for s in STATUS_ORDER) + "")
out.append(' pass fail'
' known gap, still failing known gap, now passing'
' not checked
')
# the matrix
rows = ["| feature | " + "".join(f"{html.escape(o)} | " for o in observers) + "
|---|
"]
for prefix, label in GROUPS:
group = [f for f in features if f.split(".")[0] == prefix]
if not group:
continue
rows.append(f'| {label} |
|---|
')
for f in group:
rows.append(f"| {html.escape(f)} | " + "".join(
f'{cell_marks(cells, f, o)} | ' for o in observers) + "
|---|
")
rows.append("")
out.append('")
# failures
bad = sorted(((k, v) for k, v in cells.items() if v["fail"] or v["xpass"]), key=lambda kv: (-kv[1]["fail"], kv[0]))
items = []
for key, v in bad:
f, origin, obs, d = key.split("|")
status = "fail" if v["fail"] else "xpass"
examples = "\n".join(json.dumps(e, ensure_ascii=False, default=str) for e in v["examples"])
items.append(f'{v[status]}/{v["n"]} {html.escape(f)} · '
f'{html.escape(origin)} → {html.escape(obs)} · {html.escape(DIR_LABEL.get(d, d))}
'
f"{html.escape(examples)}Failing cells ({len(bad)})
' + ("".join(items) or "
None.
") + "
| {html.escape(p)} | " + "".join(f"{c.get(s, 0)} | " for s in ("pass", "fail", "xfail")) + "
|---|
"
for p, c in per.items())
failed = "".join(f"{html.escape(s['peer'])}: {html.escape(s['check'])} ({s['status']})"
for s in scenarios if s["status"] != "pass")
out.append('Scenarios
The pasture\'s scripted checks, one peer at a time.
'
'")
out.append("")
return "\n".join(out)
def merge(results):
"""Several runs' results as one: each cell's counts added up (a pair run's cells and the village's meet in one
matrix), its examples kept, the runs named together."""
if len(results) == 1:
return results[0]
cells, totals = {}, Counter()
for r in results:
totals.update(Counter(r["totals"]))
for key, v in r["cells"].items():
cell = cells.setdefault(key, {"n": 0, "pass": 0, "fail": 0, "xfail": 0, "xpass": 0, "examples": []})
for k in ("n", "pass", "fail", "xfail", "xpass"):
cell[k] += v.get(k, 0)
cell["examples"] = (cell["examples"] + v.get("examples", []))[:3]
return {"run": " + ".join(os.path.basename(r["run"]) for r in results), "when": max(r["when"] for r in results),
"cells": cells, "totals": dict(totals)}
def main(args):
import seed
names = [a for a in args if not a.startswith("--")] or ["village"]
run_dir = seed.run_dir(names[0])
results = []
for name in names:
with open(os.path.join(seed.run_dir(name), "results.json")) as f:
results.append(json.load(f))
result = merge(results)
scenarios = []
path = os.path.join(seed.OUT, "scenarios.jsonl")
if os.path.exists(path):
with open(path) as f:
scenarios = [json.loads(line) for line in f if line.strip()]
# decePub's e2e cells (tests/e2e) join the servers' in the same matrix, under the observer "decepub"
client = os.path.join(seed.OUT, "client.jsonl")
if os.path.exists(client):
import check
with open(client) as f:
lines = [json.loads(line) for line in f if line.strip()]
latest = {}
for c in lines:
latest[(c["cell"], c.get("ref"))] = c # the latest result of each test
checks = check.classify(list(latest.values()), check.load_gaps())
for cell, v in check.summarise(checks).items():
result["cells"][cell] = v
from collections import Counter
totals = Counter(result["totals"])
totals.update(Counter(c["status"] for c in checks))
result["totals"] = totals
page = render(result, scenarios, "PrivaPub Town Matrix")
target = os.path.join(run_dir, "report.html")
with open(target, "w") as f:
f.write(page)
print(target)
return 0
if __name__ == "__main__":
sys.exit(main(sys.argv[1:]))