"""Generates Markdown report from raw benchmark rows.""" import statistics from collections import defaultdict def summarise(values: list[float]) -> dict: if not values: return {"avg": 0, "median": 0, "stdev": 0, "min": 0, "max": 0} return { "avg": round(statistics.mean(values), 2), "median": round(statistics.median(values), 2), "stdev": round(statistics.stdev(values), 2) if len(values) > 1 else 0.0, "min": round(min(values), 2), "max": round(max(values), 2), } def build_markdown(rows: list[dict], ts: str) -> str: # Group by (scenario, framework) data: dict[str, dict[str, list[float]]] = defaultdict(lambda: defaultdict(list)) scenario_labels: dict[str, str] = {} frameworks: list[str] = [] for row in rows: sk = row["scenario_key"] sl = row["scenario"] fw = row["framework"] scenario_labels[sk] = sl if fw not in frameworks: frameworks.append(fw) data[sk][fw].append(row["ms"]) lines = [] lines.append(f"# Benchmark Results\n") lines.append(f"Generated: {ts}\n") lines.append(f"Frameworks: {' vs '.join(frameworks)}\n") COMMON = ["add_alert", "delete_alert", "search_filter"] EDGE = ["large_list_add", "rapid_fire", "empty_search"] for section, keys in [("Common scenarios", COMMON), ("Edge case scenarios", EDGE)]: lines.append(f"\n## {section}\n") # Header header = "| Scenario |" sep = "| --- |" for fw in frameworks: header += f" {fw} avg (ms) | {fw} median (ms) | {fw} stdev |" sep += " ---: | ---: | ---: |" lines.append(header) lines.append(sep) for key in keys: if key not in data: continue sl = scenario_labels.get(key, key) row_line = f"| {sl} |" for fw in frameworks: s = summarise(data[key].get(fw, [])) row_line += f" {s['avg']} | {s['median']} | {s['stdev']} |" lines.append(row_line) # WebSocket payload table lines.append("\n## WebSocket payload (avg bytes per action)\n") header = "| Scenario |" sep = "| --- |" for fw in frameworks: header += f" {fw} sent | {fw} received |" sep += " ---: | ---: |" lines.append(header) lines.append(sep) ws_data: dict[str, dict[str, list]] = defaultdict(lambda: defaultdict(list)) for row in rows: ws_data[row["scenario_key"]][row["framework"] + "_sent"].append(row["ws_sent_b"]) ws_data[row["scenario_key"]][row["framework"] + "_recv"].append(row["ws_recv_b"]) for key in COMMON + EDGE: if key not in ws_data: continue sl = scenario_labels.get(key, key) row_line = f"| {sl} |" for fw in frameworks: sent_avg = round(statistics.mean(ws_data[key].get(f"{fw}_sent", [0])), 0) recv_avg = round(statistics.mean(ws_data[key].get(f"{fw}_recv", [0])), 0) row_line += f" {int(sent_avg)} B | {int(recv_avg)} B |" lines.append(row_line) lines.append("\n## Notes\n") lines.append("- All times are wall-clock milliseconds (action → DOM update).") lines.append("- Warmup iterations are discarded.") lines.append("- WebSocket bytes measured per interaction via Playwright.") lines.append("- Large list scenario pre-loads 500 alerts.") lines.append("- Rapid fire: 5 consecutive clicks with 50ms interval.\n") return "\n".join(lines) if __name__ == "__main__": import sys, json rows = json.load(open(sys.argv[1])) print(build_markdown(rows, "manual"))