thepraandClaude Opus 5.5
fb90558dec
decePub's e2e runs append to out/client.jsonl. The report kept the newest row per test and account, and the backlog every row, so a failure from a run that picked other accounts stayed in both after later runs passed. Each client cell now counts as its latest run left it (check.latest_client: the rows within an hour of its newest). The backlog comes from a fresh village on today's build: 2556 checks pass and 4 fail, three of them peer-to-peer counts, and one GoToSocial losing a status from its cache (an Update handled as a Create; a restart heals it), recorded in INTEROP.md rather than as a gap, which would hide our own edits failing there. The community vote and the Lemmy thread that failed before were data from older builds. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01LsXgEaXee4GCU1hwYgPJXw
87 lines
3.3 KiB
Python
87 lines
3.3 KiB
Python
"""docs/INTEROP-BACKLOG.md from the latest town results, the bash scenarios and decePub's e2e cells: new failures (to fix
|
|
or to record in gaps.json), known gaps by phase, known gaps now passing (close them), and the scenarios' failures.
|
|
|
|
town.sh backlog [run] [--write]"""
|
|
import json
|
|
import os
|
|
import sys
|
|
import time
|
|
from collections import defaultdict
|
|
|
|
import check
|
|
import seed
|
|
|
|
DOCS = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))), "docs")
|
|
|
|
|
|
def load_jsonl(path):
|
|
if not os.path.exists(path):
|
|
return []
|
|
with open(path) as f:
|
|
return [json.loads(line) for line in f if line.strip()]
|
|
|
|
|
|
def main(args):
|
|
name = next((a for a in args if not a.startswith("--")), "village")
|
|
run_dir = seed.run_dir(name)
|
|
with open(os.path.join(run_dir, "results.json")) as f:
|
|
result = json.load(f)
|
|
gaps = {g["id"]: g for g in check.load_gaps()}
|
|
client = check.classify(check.latest_client([dict(c) for c in load_jsonl(os.path.join(seed.OUT, "client.jsonl"))]), list(gaps.values()))
|
|
scenarios = load_jsonl(os.path.join(seed.OUT, "scenarios.jsonl"))
|
|
checks = result["checks"] + client
|
|
|
|
new = defaultdict(list)
|
|
known = defaultdict(list)
|
|
closing = defaultdict(list)
|
|
for c in checks:
|
|
if c["status"] == "fail":
|
|
new[c["cell"]].append(c)
|
|
elif c["status"] == "xfail":
|
|
known[c["gap"]].append(c)
|
|
elif c["status"] == "xpass":
|
|
closing[c["gap"]].append(c)
|
|
|
|
lines = ["# Interop backlog", "",
|
|
f"Generated by `tools/pasture/town.sh backlog --write` from `{os.path.basename(run_dir)}` on "
|
|
f"{time.strftime('%Y-%m-%d', time.gmtime())}. Do not edit by hand: fix the cause, or record a known gap in "
|
|
"`tools/pasture/town/gaps.json`, and generate it again.", "",
|
|
f"Checks: {sum(1 for c in checks if c['status'] == 'pass')} pass, {sum(1 for c in checks if c['status'] == 'fail')} "
|
|
f"fail, {sum(1 for c in checks if c['status'] == 'xfail')} known gaps, "
|
|
f"{sum(1 for c in checks if c['status'] == 'xpass')} known gaps now passing.", ""]
|
|
lines += ["## New failures", ""]
|
|
if not new:
|
|
lines.append("None.")
|
|
for cell, cs in sorted(new.items(), key=lambda kv: -len(kv[1])):
|
|
ex = cs[0]
|
|
got = " ".join(str(ex.get("got")).replace("`", "'").split())[:140]
|
|
lines.append(f"- **`{cell}`**: {len(cs)} failing. Example: {ex.get('ref') or ''} expected `{ex.get('expect')}`, "
|
|
f"got `{got}`" + (f" ({ex['how']})" if ex.get("how") else "") + ".")
|
|
lines += ["", "## Known gaps still failing", ""]
|
|
if not known:
|
|
lines.append("None.")
|
|
for gid, cs in sorted(known.items()):
|
|
g = gaps.get(gid, {})
|
|
lines.append(f"- **{gid}** ({g.get('kind')}, {g.get('phase')}): {g.get('title')}. {len(cs)} checks. "
|
|
+ (f"Code: `{g['code']}`." if g.get("code") else ""))
|
|
lines += ["", "## Known gaps now passing (close them)", ""]
|
|
if not closing:
|
|
lines.append("None.")
|
|
for gid, cs in sorted(closing.items()):
|
|
lines.append(f"- **{gid}**: {gaps.get(gid, {}).get('title')}. {len(cs)} checks pass.")
|
|
lines += ["", "## Scenario failures", ""]
|
|
bad = [s for s in scenarios if s["status"] == "fail"]
|
|
lines += [f"- {s['peer']}: {s['check']}" for s in bad] or ["None."]
|
|
text = "\n".join(lines) + "\n"
|
|
if "--write" in args:
|
|
with open(os.path.join(DOCS, "INTEROP-BACKLOG.md"), "w") as f:
|
|
f.write(text)
|
|
print(os.path.join(DOCS, "INTEROP-BACKLOG.md"))
|
|
else:
|
|
print(text)
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main(sys.argv[1:]))
|