diff --git a/scripts/analysis/paired_compare.py b/scripts/analysis/paired_compare.py new file mode 100755 index 0000000..5ef6ada --- /dev/null +++ b/scripts/analysis/paired_compare.py @@ -0,0 +1,225 @@ +#!/usr/bin/env python3 +"""Paired latency comparison with bootstrap CI. + +Implements docs/EVALUATION_PROTOCOL_ZH.md ยง2.2 (M2 fix): when comparing +mechanism A vs B on the same trace, the only honest comparison is paired +on same-trial-mask. This script joins two metrics.jsonl by request_id, +keeps the rows where BOTH sides succeeded, and reports paired deltas +with 95% bootstrap CIs. + +Out vs the existing `compare_no_error.py`: + - works on raw metrics.jsonl, not pre-aggregated summary.json + - bootstrap CIs (not just point estimates) + - reports paired-mask size + per-side failure counts so the reader + sees how many rows were dropped from the comparison + +Usage: + scripts/analysis/paired_compare.py \ + --baseline outputs/run-dp/request-metrics.jsonl \ + --candidate outputs/run-kvc/request-metrics.jsonl + scripts/analysis/paired_compare.py ... --bootstrap 5000 --seed 42 + scripts/analysis/paired_compare.py ... --json > paired.json + +stdlib only โ€” no scipy/numpy. Runs without GPU and without SGLang. +""" + +from __future__ import annotations + +import argparse +import json +import math +import random +import sys +from pathlib import Path + + +def _load(path: Path) -> dict[str, dict]: + out: dict[str, dict] = {} + with path.open() as handle: + for line in handle: + line = line.strip() + if not line: + continue + row = json.loads(line) + rid = row.get("request_id") + if rid is None: + continue + out[rid] = row + return out + + +def _ok(row: dict) -> bool: + return row.get("error") is None and row.get("latency_s") is not None + + +def _quantile(values: list[float], q: float) -> float: + if not values: + return float("nan") + s = sorted(values) + if len(s) == 1: + return s[0] + pos = (len(s) - 1) * q + lo = math.floor(pos) + hi = math.ceil(pos) + if lo == hi: + return s[lo] + return s[lo] + (s[hi] - s[lo]) * (pos - lo) + + +def _stats(deltas: list[float]) -> dict[str, float]: + if not deltas: + return {"mean": float("nan"), "p50": float("nan"), "p90": float("nan"), "p99": float("nan")} + return { + "mean": sum(deltas) / len(deltas), + "p50": _quantile(deltas, 0.50), + "p90": _quantile(deltas, 0.90), + "p99": _quantile(deltas, 0.99), + } + + +def _bootstrap_ci( + deltas: list[float], statistic, n_boot: int, rng: random.Random +) -> tuple[float, float]: + """Return (lo, hi) 95% CI for `statistic(deltas)`.""" + if len(deltas) < 2: + return (float("nan"), float("nan")) + n = len(deltas) + samples = [] + for _ in range(n_boot): + # resample with replacement + resample = [deltas[rng.randrange(n)] for _ in range(n)] + samples.append(statistic(resample)) + samples.sort() + lo = samples[int(0.025 * (n_boot - 1))] + hi = samples[int(0.975 * (n_boot - 1))] + return (lo, hi) + + +def compare( + baseline: dict[str, dict], + candidate: dict[str, dict], + *, + metric: str, + n_boot: int, + seed: int, +) -> dict: + common_ids = set(baseline.keys()) & set(candidate.keys()) + paired_ids = [ + rid for rid in common_ids if _ok(baseline[rid]) and _ok(candidate[rid]) + ] + paired_ids.sort() + + base_only_fail = sum(1 for rid in common_ids if not _ok(baseline[rid])) + cand_only_fail = sum(1 for rid in common_ids if not _ok(candidate[rid])) + + deltas = [] + wins = losses = ties = 0 + for rid in paired_ids: + b = baseline[rid].get(metric) + c = candidate[rid].get(metric) + if b is None or c is None: + continue + d = float(c) - float(b) + deltas.append(d) + if d < 0: + wins += 1 + elif d > 0: + losses += 1 + else: + ties += 1 + + rng = random.Random(seed) + stats = _stats(deltas) + ci_mean = _bootstrap_ci(deltas, lambda x: sum(x) / len(x), n_boot, rng) + ci_p50 = _bootstrap_ci(deltas, lambda x: _quantile(x, 0.50), n_boot, rng) + ci_p90 = _bootstrap_ci(deltas, lambda x: _quantile(x, 0.90), n_boot, rng) + + return { + "metric": metric, + "baseline_size": len(baseline), + "candidate_size": len(candidate), + "intersection_size": len(common_ids), + "paired_size": len(paired_ids), + "baseline_fail_in_common": base_only_fail, + "candidate_fail_in_common": cand_only_fail, + "delta_stats": stats, + "delta_mean_ci95": ci_mean, + "delta_p50_ci95": ci_p50, + "delta_p90_ci95": ci_p90, + "wins_candidate": wins, + "losses_candidate": losses, + "ties": ties, + } + + +def _fmt(x: float, w: int = 6) -> str: + if x is None or (isinstance(x, float) and math.isnan(x)): + return " nan " + return f"{x:+{w}.3f}" + + +def render(result: dict) -> str: + s = result["delta_stats"] + mlo, mhi = result["delta_mean_ci95"] + p5lo, p5hi = result["delta_p50_ci95"] + p9lo, p9hi = result["delta_p90_ci95"] + n = result["paired_size"] + lines = [ + f"# paired comparison ({result['metric']})", + "", + f"baseline rows: {result['baseline_size']}", + f"candidate rows: {result['candidate_size']}", + f"intersection (rid): {result['intersection_size']}", + f"paired (both ok): {result['paired_size']}", + f" baseline fails in common: {result['baseline_fail_in_common']}", + f" candidate fails in common: {result['candidate_fail_in_common']}", + "", + "## delta (candidate - baseline) โ€” negative = candidate is faster", + "", + "| stat | value | 95% CI |", + "|---|---:|---:|", + f"| mean | {_fmt(s['mean'])} | [{_fmt(mlo)}, {_fmt(mhi)}] |", + f"| p50 | {_fmt(s['p50'])} | [{_fmt(p5lo)}, {_fmt(p5hi)}] |", + f"| p90 | {_fmt(s['p90'])} | [{_fmt(p9lo)}, {_fmt(p9hi)}] |", + f"| p99 | {_fmt(s['p99'])} | โ€” |", + "", + f"win/loss/tie: {result['wins_candidate']} / {result['losses_candidate']} / {result['ties']} (of {n})", + ] + return "\n".join(lines) + + +def main() -> None: + p = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + p.add_argument("--baseline", required=True, type=Path) + p.add_argument("--candidate", required=True, type=Path) + p.add_argument( + "--metric", + default="latency_s", + choices=["latency_s", "ttft_s", "tpot_s"], + help="which per-request field to compare (default: latency_s)", + ) + p.add_argument("--bootstrap", type=int, default=2000) + p.add_argument("--seed", type=int, default=20260512) + p.add_argument("--json", action="store_true") + args = p.parse_args() + + baseline = _load(args.baseline) + candidate = _load(args.candidate) + if not baseline or not candidate: + print("empty input on one side", file=sys.stderr) + sys.exit(1) + + result = compare( + baseline, candidate, + metric=args.metric, n_boot=args.bootstrap, seed=args.seed, + ) + + if args.json: + json.dump(result, sys.stdout, indent=2, default=lambda x: None if isinstance(x, float) and math.isnan(x) else x) + sys.stdout.write("\n") + else: + print(render(result)) + + +if __name__ == "__main__": + main()