#!/usr/bin/env python3 """Re-derive summary.json from existing metrics.jsonl using the fixed metrics.py. Bug fixed: requests aborted by SGLang (e.g. input > max-input-len returns a fast 400 with latency_s ~ 0.08s) were previously counted in latency_stats as if successful, deflating mean/p50/p90. The fixed metrics.py excludes all failed requests (errors or aborts) from latency/ttft/tpot stats and exposes abort_count / failure_count. Usage: python3 scripts/analysis/recompute_summary.py path/to/metrics.jsonl ... python3 scripts/analysis/recompute_summary.py --diff path/to/metrics.jsonl path/to/old_summary.json """ from __future__ import annotations import argparse import json import sys from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "src")) from agentic_pd_hybrid.metrics import RequestMetrics, write_summary_json def load_rows(metrics_path: Path) -> list[RequestMetrics]: rows = [] field_names = {f for f in RequestMetrics.__dataclass_fields__} with metrics_path.open() as handle: for line in handle: line = line.strip() if not line: continue raw = json.loads(line) kwargs = {k: raw.get(k) for k in field_names} rows.append(RequestMetrics(**kwargs)) return rows def main() -> None: parser = argparse.ArgumentParser() parser.add_argument("metrics_paths", nargs="+", type=Path) parser.add_argument( "--out", type=Path, default=None, help="output summary path (default: alongside metrics with .recomputed_summary.json)", ) parser.add_argument( "--diff", action="store_true", help="print before/after diff against the old .summary.json", ) args = parser.parse_args() for metrics_path in args.metrics_paths: rows = load_rows(metrics_path) out_path = args.out or metrics_path.with_suffix(".recomputed_summary.json") write_summary_json( out_path, rows, trace_path=metrics_path, router_url=None, ) new = json.load(out_path.open()) print(f"\n=== {metrics_path} ===") print(f" written: {out_path}") print(f" total rows: {new['request_count']}") print(f" error_count: {new['error_count']}") print(f" abort_count: {new.get('abort_count', '?')}") print(f" failure_count: {new.get('failure_count', '?')}") ls = new.get("latency_stats_s", {}) or {} ts = new.get("ttft_stats_s", {}) or {} print(f" lat: n={ls.get('count')} mean={ls.get('mean'):.4f} p50={ls.get('p50'):.4f} p90={ls.get('p90'):.4f} p99={ls.get('p99'):.4f}") print(f" ttft: n={ts.get('count')} mean={ts.get('mean'):.4f} p50={ts.get('p50'):.4f} p90={ts.get('p90'):.4f} p99={ts.get('p99'):.4f}") if args.diff: # find old summary (sibling file) candidates = [ metrics_path.parent / f"{metrics_path.stem}.summary.json", metrics_path.with_suffix(".summary.json"), ] old_path = next((p for p in candidates if p.exists()), None) if old_path: old = json.load(old_path.open()) print(f" vs old {old_path}:") old_ls = old.get("latency_stats_s", {}) or {} old_ts = old.get("ttft_stats_s", {}) or {} for k in ("count", "mean", "p50", "p90", "p99"): o = old_ls.get(k) n = ls.get(k) if o is not None and n is not None: delta = n - o print(f" lat.{k}: {o:.4f} -> {n:.4f} ({delta:+.4f})") for k in ("count", "mean", "p50", "p90", "p99"): o = old_ts.get(k) n = ts.get(k) if o is not None and n is not None: delta = n - o print(f" ttft.{k}: {o:.4f} -> {n:.4f} ({delta:+.4f})") if __name__ == "__main__": main()