thepraandClaude Opus 5.5
fb90558dec
decePub's e2e runs append to out/client.jsonl. The report kept the newest row per test and account, and the backlog every row, so a failure from a run that picked other accounts stayed in both after later runs passed. Each client cell now counts as its latest run left it (check.latest_client: the rows within an hour of its newest). The backlog comes from a fresh village on today's build: 2556 checks pass and 4 fail, three of them peer-to-peer counts, and one GoToSocial losing a status from its cache (an Update handled as a Create; a restart heals it), recorded in INTEROP.md rather than as a gap, which would hide our own edits failing there. The community vote and the Lemmy thread that failed before were data from older builds. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01LsXgEaXee4GCU1hwYgPJXw
195 lines
10 KiB
Python
195 lines
10 KiB
Python
"""out/<run>/report.html: one self-contained page (no script, fake pasture data only, no tokens) with the run's totals,
|
||
the feature × server matrix, every failing cell with its examples, and the bash scenarios' checks. Written so it can be
|
||
published as it is."""
|
||
import html
|
||
import json
|
||
import os
|
||
import sys
|
||
from collections import Counter, defaultdict
|
||
|
||
import gen
|
||
|
||
DIRS = ["in", "out", "via", "control", "local"]
|
||
DIR_LABEL = {"in": "into PrivaPub", "out": "from PrivaPub", "via": "through PrivaPub", "control": "peer to peer",
|
||
"local": "on its own server"}
|
||
GROUPS = [("client", "decePub"), ("deliver", "Delivery"), ("confine", "Confinement"), ("see", "Seen by the audience"),
|
||
("hide", "Hidden from outsiders"), ("count", "Counts"), ("thread", "Threads"), ("edit", "Edits"),
|
||
("delete", "Deletes"), ("graph", "Follows"), ("privacy", "Privacy")]
|
||
STATUS_ORDER = ["fail", "xpass", "xfail", "pass"]
|
||
|
||
CSS = """
|
||
/* layout: one column; a sticky summary, then the matrix in its own horizontal scroller, then failures and scenarios */
|
||
:root {
|
||
--bg: #f6f7f6; --panel: #ffffff; --fg: #1d2421; --muted: #5c6863; --line: #d9dfdc;
|
||
--pass: #2f7d57; --fail: #c2412d; --xfail: #a77a14; --xpass: #6a4fc0; --none: #e4e9e6;
|
||
--sans: "IBM Plex Sans", system-ui, sans-serif; --mono: "IBM Plex Mono", ui-monospace, monospace;
|
||
}
|
||
@media (prefers-color-scheme: dark) { :root:not([data-theme="light"]) {
|
||
--bg: #121715; --panel: #1a211e; --fg: #e3e9e6; --muted: #98a59f; --line: #2c3632;
|
||
--pass: #4fb184; --fail: #ec6c57; --xfail: #d6a940; --xpass: #9d86ef; --none: #26302c; color-scheme: dark } }
|
||
:root[data-theme="dark"] {
|
||
--bg: #121715; --panel: #1a211e; --fg: #e3e9e6; --muted: #98a59f; --line: #2c3632;
|
||
--pass: #4fb184; --fail: #ec6c57; --xfail: #d6a940; --xpass: #9d86ef; --none: #26302c; color-scheme: dark }
|
||
body { background: var(--bg); color: var(--fg); font: 15px/1.5 var(--sans); margin: 0; }
|
||
main { max-width: 1180px; margin: 0 auto; padding-inline: 16px; padding-block: 24px 48px; display: grid; gap: 28px; }
|
||
h1 { font-size: 1.6rem; margin: 0; text-wrap: balance; }
|
||
h2 { font-size: 1.15rem; margin: 0 0 10px; text-wrap: balance; }
|
||
p { margin: 0; max-width: 70ch; color: var(--muted); }
|
||
.meta { font: 13px var(--mono); color: var(--muted); overflow-wrap: anywhere; }
|
||
.totals { display: flex; flex-wrap: wrap; gap: 8px 18px; font-variant-numeric: tabular-nums; }
|
||
.total { display: flex; align-items: baseline; gap: 6px; }
|
||
.total b { font: 600 1.35rem var(--mono); }
|
||
.total span { text-transform: uppercase; letter-spacing: .06em; font-size: 12px; color: var(--muted); }
|
||
.legend { display: flex; flex-wrap: wrap; gap: 6px 16px; font-size: 13px; color: var(--muted); }
|
||
.legend i, .m { display: inline-block; width: 12px; height: 12px; border-radius: 2px; vertical-align: -1px; }
|
||
.s-pass { background: var(--pass); } .s-fail { background: var(--fail); } .s-xfail { background: var(--xfail); }
|
||
.s-xpass { background: var(--xpass); } .s-none { background: var(--none); }
|
||
.scroll { overflow-x: auto; background: var(--panel); border: 1px solid var(--line); border-radius: 6px; }
|
||
table { border-collapse: collapse; width: 100%; font-variant-numeric: tabular-nums; }
|
||
th, td { padding: 6px 10px; border-bottom: 1px solid var(--line); text-align: left; vertical-align: top; }
|
||
thead th { font-size: 12px; text-transform: uppercase; letter-spacing: .06em; color: var(--muted); white-space: nowrap; }
|
||
tbody th { font: 13px var(--mono); white-space: nowrap; }
|
||
tr.group th { font: 600 12px var(--sans); text-transform: uppercase; letter-spacing: .08em; color: var(--muted);
|
||
background: var(--bg); }
|
||
.cell { display: flex; gap: 3px; flex-wrap: nowrap; }
|
||
.cell .m { width: 14px; height: 14px; }
|
||
.fails { display: grid; gap: 10px; }
|
||
details { background: var(--panel); border: 1px solid var(--line); border-radius: 6px; padding: 8px 12px; }
|
||
details[open] { padding-bottom: 12px; }
|
||
summary { cursor: pointer; font: 13px var(--mono); overflow-wrap: anywhere; }
|
||
summary:focus-visible { outline: 2px solid var(--xpass); outline-offset: 2px; }
|
||
pre { font: 12px/1.45 var(--mono); white-space: pre-wrap; overflow-wrap: anywhere; margin: 8px 0 0; color: var(--muted); }
|
||
.tag { font: 12px var(--mono); padding: 1px 6px; border-radius: 3px; color: var(--bg); }
|
||
"""
|
||
|
||
|
||
def cell_marks(cells, feature, observer):
|
||
parts = []
|
||
for d in DIRS:
|
||
agg = Counter()
|
||
for key, v in cells.items():
|
||
f, origin, obs, direction = key.split("|")
|
||
if f == feature and obs == observer and direction == d:
|
||
for s in STATUS_ORDER:
|
||
agg[s] += v[s]
|
||
if not agg:
|
||
continue
|
||
worst = next(s for s in STATUS_ORDER if agg[s])
|
||
title = f"{DIR_LABEL[d]}: " + ", ".join(f"{agg[s]} {s}" for s in STATUS_ORDER if agg[s])
|
||
parts.append(f'<i class="m s-{worst}" title="{html.escape(title)}"></i>')
|
||
return "".join(parts) or '<i class="m s-none" title="not checked"></i>'
|
||
|
||
|
||
def render(result, scenarios, title):
|
||
cells = result["cells"]
|
||
totals = Counter(result["totals"])
|
||
observers = [p for p in gen.HOSTS if any(k.split("|")[2] == p for k in cells)] + \
|
||
[o for o in ("*", "decepub") if any(k.split("|")[2] == o for k in cells)]
|
||
features = sorted({k.split("|")[0] for k in cells})
|
||
out = [f"<title>{html.escape(title)}</title>",
|
||
'<link rel="stylesheet" href="https://fonts.googleapis.com/css2?family=IBM+Plex+Mono:wght@400;600&family=IBM+Plex+Sans:wght@400;600&display=swap">',
|
||
f"<style>{CSS}</style>", "<main>"]
|
||
out.append(f"<header style=\"display:grid;gap:10px\"><h1>{html.escape(title)}</h1>"
|
||
f"<p>Every object the town's accounts made, checked on every server it should and should not have reached, "
|
||
f"through each server's database and API. A cell's squares are the directions it was checked in: "
|
||
f"into PrivaPub, from PrivaPub, through PrivaPub, peer to peer, and on the author's own server.</p>"
|
||
f"<div class=\"meta\">{html.escape(os.path.basename(result['run']))} · checked {html.escape(result['when'])}</div></header>")
|
||
out.append('<section class="totals" aria-label="Totals">' + "".join(
|
||
f'<div class="total"><i class="m s-{s}"></i><b>{totals.get(s, 0)}</b><span>{s}</span></div>' for s in STATUS_ORDER) + "</section>")
|
||
out.append('<div class="legend"><span><i class="s-pass"></i> pass</span><span><i class="s-fail"></i> fail</span>'
|
||
'<span><i class="s-xfail"></i> known gap, still failing</span><span><i class="s-xpass"></i> known gap, now passing</span>'
|
||
'<span><i class="s-none"></i> not checked</span></div>')
|
||
# the matrix
|
||
rows = ["<thead><tr><th>feature</th>" + "".join(f"<th>{html.escape(o)}</th>" for o in observers) + "</tr></thead><tbody>"]
|
||
for prefix, label in GROUPS:
|
||
group = [f for f in features if f.split(".")[0] == prefix]
|
||
if not group:
|
||
continue
|
||
rows.append(f'<tr class="group"><th colspan="{len(observers) + 1}">{label}</th></tr>')
|
||
for f in group:
|
||
rows.append(f"<tr><th>{html.escape(f)}</th>" + "".join(
|
||
f'<td><span class="cell">{cell_marks(cells, f, o)}</span></td>' for o in observers) + "</tr>")
|
||
rows.append("</tbody>")
|
||
out.append('<section><h2>Matrix</h2><div class="scroll"><table>' + "".join(rows) + "</table></div></section>")
|
||
# failures
|
||
bad = sorted(((k, v) for k, v in cells.items() if v["fail"] or v["xpass"]), key=lambda kv: (-kv[1]["fail"], kv[0]))
|
||
items = []
|
||
for key, v in bad:
|
||
f, origin, obs, d = key.split("|")
|
||
status = "fail" if v["fail"] else "xpass"
|
||
examples = "\n".join(json.dumps(e, ensure_ascii=False, default=str) for e in v["examples"])
|
||
items.append(f'<details><summary><span class="tag s-{status}">{v[status]}/{v["n"]}</span> {html.escape(f)} · '
|
||
f'{html.escape(origin)} → {html.escape(obs)} · {html.escape(DIR_LABEL.get(d, d))}</summary>'
|
||
f"<pre>{html.escape(examples)}</pre></details>")
|
||
out.append(f'<section><h2>Failing cells ({len(bad)})</h2><div class="fails">' + ("".join(items) or "<p>None.</p>") + "</div></section>")
|
||
# bash scenarios
|
||
if scenarios:
|
||
per = defaultdict(Counter)
|
||
for s in scenarios:
|
||
per[s["peer"]][s["status"]] += 1
|
||
srows = "".join(f"<tr><th>{html.escape(p)}</th>" + "".join(f"<td>{c.get(s, 0)}</td>" for s in ("pass", "fail", "xfail")) + "</tr>"
|
||
for p, c in per.items())
|
||
failed = "".join(f"<li>{html.escape(s['peer'])}: {html.escape(s['check'])} ({s['status']})</li>"
|
||
for s in scenarios if s["status"] != "pass")
|
||
out.append('<section><h2>Scenarios</h2><p>The pasture\'s scripted checks, one peer at a time.</p>'
|
||
'<div class="scroll"><table><thead><tr><th>peer</th><th>pass</th><th>fail</th><th>known gap</th></tr></thead>'
|
||
f"<tbody>{srows}</tbody></table></div><ul>{failed}</ul></section>")
|
||
out.append("</main>")
|
||
return "\n".join(out)
|
||
|
||
|
||
def merge(results):
|
||
"""Several runs' results as one: each cell's counts added up (a pair run's cells and the village's meet in one
|
||
matrix), its examples kept, the runs named together."""
|
||
if len(results) == 1:
|
||
return results[0]
|
||
cells, totals = {}, Counter()
|
||
for r in results:
|
||
totals.update(Counter(r["totals"]))
|
||
for key, v in r["cells"].items():
|
||
cell = cells.setdefault(key, {"n": 0, "pass": 0, "fail": 0, "xfail": 0, "xpass": 0, "examples": []})
|
||
for k in ("n", "pass", "fail", "xfail", "xpass"):
|
||
cell[k] += v.get(k, 0)
|
||
cell["examples"] = (cell["examples"] + v.get("examples", []))[:3]
|
||
return {"run": " + ".join(os.path.basename(r["run"]) for r in results), "when": max(r["when"] for r in results),
|
||
"cells": cells, "totals": dict(totals)}
|
||
|
||
|
||
def main(args):
|
||
import seed
|
||
names = [a for a in args if not a.startswith("--")] or ["village"]
|
||
run_dir = seed.run_dir(names[0])
|
||
results = []
|
||
for name in names:
|
||
with open(os.path.join(seed.run_dir(name), "results.json")) as f:
|
||
results.append(json.load(f))
|
||
result = merge(results)
|
||
scenarios = []
|
||
path = os.path.join(seed.OUT, "scenarios.jsonl")
|
||
if os.path.exists(path):
|
||
with open(path) as f:
|
||
scenarios = [json.loads(line) for line in f if line.strip()]
|
||
# decePub's e2e cells (tests/e2e) join the servers' in the same matrix, under the observer "decepub"
|
||
client = os.path.join(seed.OUT, "client.jsonl")
|
||
if os.path.exists(client):
|
||
import check
|
||
with open(client) as f:
|
||
lines = [json.loads(line) for line in f if line.strip()]
|
||
checks = check.classify(check.latest_client(lines), check.load_gaps())
|
||
for cell, v in check.summarise(checks).items():
|
||
result["cells"][cell] = v
|
||
from collections import Counter
|
||
totals = Counter(result["totals"])
|
||
totals.update(Counter(c["status"] for c in checks))
|
||
result["totals"] = totals
|
||
page = render(result, scenarios, "PrivaPub Town Matrix")
|
||
target = os.path.join(run_dir, "report.html")
|
||
with open(target, "w") as f:
|
||
f.write(page)
|
||
print(target)
|
||
return 0
|
||
|
||
|
||
if __name__ == "__main__":
|
||
sys.exit(main(sys.argv[1:]))
|