agentic-kvc/scripts/render_b3_report.py

"""Render a B3 comparison markdown from b3_policy_comparison.json.

Drops rows for policies not yet present in the sweep. Designed to be
re-run whenever a new policy finishes; the markdown lives next to
the sweep directory so future re-runs just overwrite it.
"""

from __future__ import annotations

import argparse
import json
from pathlib import Path

POLICY_ORDER = ["lmetric", "load_only", "sticky", "unified", "capped"]
POLICY_DESCR = {
    "lmetric":   "cache-aware P_tokens × BS (main baseline)",
    "load_only": "control: min(num_requests), no cache, no affinity",
    "sticky":    "control: hard session affinity (never break)",
    "unified":   "hybrid affinity + LMetric fallback",
    "capped":    "lmetric on per-session turn-capped trace",
}


def _fmt(v, kind="f2"):
    if v is None:
        return "—"
    if kind == "f2":
        return f"{v:.2f}"
    if kind == "f3":
        return f"{v:.3f}"
    if kind == "pct":
        return f"{v * 100:.1f}%"
    if kind == "ms":
        return f"{v * 1000:.1f}"
    return str(v)


def main() -> None:
    p = argparse.ArgumentParser()
    p.add_argument("--sweep-dir", type=Path, required=True)
    p.add_argument("--output", type=Path, default=None,
                   help="Markdown output; default <sweep-dir>/b3_report.md")
    args = p.parse_args()

    comp_path = args.sweep_dir / "b3_policy_comparison.json"
    if not comp_path.exists():
        raise SystemExit(f"missing {comp_path}; run b3_analyze.sh first")
    data = json.loads(comp_path.read_text())
    by_pol = {r["policy"]: r for r in data["rows"]}

    lines: list[str] = []
    lines.append(f"# B3 Routing Sweep Report")
    lines.append("")
    lines.append(f"Sweep dir: `{args.sweep_dir.name}`")
    lines.append(f"Trace: w600_r0.0015_st30.jsonl (~1.2k reqs, 8 × TP1)")
    lines.append(f"Policies present: {', '.join(p for p in POLICY_ORDER if p in by_pol)}")
    lines.append(f"Policies pending: {', '.join(p for p in POLICY_ORDER if p not in by_pol) or '—'}")
    lines.append("")

    lines.append("## Headline latencies + APC")
    lines.append("")
    lines.append("| policy | ok/total | TTFT p50/p90/p99 (s) | TPOT p50/p90/p99 (ms) | E2E p50/p90/p99 (s) | APC |")
    lines.append("|---|---:|---|---|---|---:|")
    for pol in POLICY_ORDER:
        r = by_pol.get(pol)
        if not r:
            lines.append(f"| **{pol}** | _pending_ | _pending_ | _pending_ | _pending_ | _pending_ |")
            continue
        lines.append(
            f"| **{pol}** | {r['n_ok']}/{r['n_total']} | "
            f"{_fmt(r['ttft_p50_s'])}/{_fmt(r['ttft_p90_s'])}/{_fmt(r['ttft_p99_s'])} | "
            f"{_fmt(r['tpot_p50_s'], 'ms')}/{_fmt(r['tpot_p90_s'], 'ms')}/{_fmt(r['tpot_p99_s'], 'ms')} | "
            f"{_fmt(r['e2e_p50_s'])}/{_fmt(r['e2e_p90_s'])}/{_fmt(r['e2e_p99_s'])} | "
            f"{_fmt(r['apc_ratio'], 'pct')} |"
        )
    lines.append("")

    lines.append("## Mechanism indices")
    lines.append("")
    lines.append("| policy | interference_index | hotspot_index (TTFT p90) | intra-session reuse | cross-session reuse | n_slow |")
    lines.append("|---|---:|---:|---:|---:|---:|")
    for pol in POLICY_ORDER:
        r = by_pol.get(pol)
        if not r:
            lines.append(f"| **{pol}** | _pending_ | _pending_ | _pending_ | _pending_ | _pending_ |")
            continue
        lines.append(
            f"| **{pol}** | {_fmt(r['interference_index'])} | "
            f"{_fmt(r['hotspot_index_ttft_p90'])} | "
            f"{_fmt(r['reuse_intra_frac'], 'pct')} | "
            f"{_fmt(r['reuse_cross_frac'], 'pct')} | "
            f"{r['n_slow']} |"
        )
    lines.append("")
    lines.append("- **interference_index** = TPOT_p90(decode overlapping same-worker prefill) / TPOT_p90(clean)")
    lines.append("- **hotspot_index** = max(worker TTFT_p90) / median(worker TTFT_p90)")
    lines.append("")

    lines.append("## Slow-request cause breakdown")
    lines.append("")
    lines.append("| policy | n_slow | same-worker overlap | hot worker queue | cache miss large append | high KV | unknown |")
    lines.append("|---|---:|---:|---:|---:|---:|---:|")
    for pol in POLICY_ORDER:
        r = by_pol.get(pol)
        if not r:
            lines.append(f"| **{pol}** | _pending_ | _pending_ | _pending_ | _pending_ | _pending_ | _pending_ |")
            continue
        fc = r.get("failure_counts") or {}
        def cnt(k): return fc.get(k, 0)
        lines.append(
            f"| **{pol}** | {r['n_slow']} | "
            f"{cnt('same_worker_prefill_overlap')} | {cnt('hot_worker_queue')} | "
            f"{cnt('cache_miss_large_append')} | {cnt('high_kv_occupancy')} | "
            f"{cnt('unknown')} |"
        )
    lines.append("")

    lines.append("## Policy notes")
    lines.append("")
    for pol in POLICY_ORDER:
        lines.append(f"- **{pol}** — {POLICY_DESCR[pol]}")
    lines.append("")

    lines.append("## Per-policy per-worker TTFT p90 (s)")
    lines.append("")
    for pol in POLICY_ORDER:
        r = by_pol.get(pol)
        if not r:
            lines.append(f"### {pol} _(pending)_")
            lines.append("")
            continue
        path = args.sweep_dir / pol / "joined" / "hotspot_index.json"
        if not path.exists():
            continue
        hot = json.loads(path.read_text())
        per = hot.get("per_worker_ttft_p90_s") or {}
        lines.append(f"### {pol}")
        lines.append("")
        lines.append("| worker | TTFT p90 (s) |")
        lines.append("|---|---:|")
        for w, v in sorted(per.items()):
            lines.append(f"| {w} | {v:.2f} |")
        lines.append("")

    out = args.output or args.sweep_dir / "b3_report.md"
    out.write_text("\n".join(lines))
    print(f"wrote {out}")


if __name__ == "__main__":
    main()