diff --git a/scripts/compare_ab_final.py b/scripts/compare_ab_final.py new file mode 100644 index 0000000..44848b1 --- /dev/null +++ b/scripts/compare_ab_final.py @@ -0,0 +1,118 @@ +"""Final A/B comparison: baseline (dash0) vs elastic (dash1). +Both fresh restart, same trace, same params. GPU util + APC + latency.""" +import json, csv, statistics, os, urllib.request + +def lat(path): + rows = [json.loads(l) for l in open(path)] + ok = [r for r in rows if not r.get("error")] + err = [r for r in rows if r.get("error")] + ttfts = sorted([r["ttft_s"] for r in ok if r.get("ttft_s")]) + tpots = sorted([r["tpot_s"] for r in ok if r.get("tpot_s") and r["tpot_s"]>0]) + lats = sorted([r["latency_s"] for r in ok]) + p = lambda v,q: v[min(int(q*len(v)),len(v)-1)] if v else 0 + ok_inp = sorted([r["input_length"] for r in ok]) + err_inp = sorted([r["input_length"] for r in err]) + return {"ok": len(ok), "n": len(rows), + "t50": p(ttfts,.5), "t90": p(ttfts,.9), + "p50": p(tpots,.5), "p90": p(tpots,.9), + "e50": p(lats,.5), "e90": p(lats,.9), + "inp50": p(ok_inp,.5), "err_inp50": p(err_inp,.5) if err_inp else 0} + +def gpu_per_inst(path): + if not os.path.exists(path): + return {} + rows = list(csv.DictReader(open(path))) + by_gpu = {} + for r in rows: + g = int(r["gpu"]) + by_gpu.setdefault(g, []).append(float(r["util_pct"])) + result = {} + for g, vals in sorted(by_gpu.items()): + nz = sum(1 for v in vals if v > 0) + result[g] = {"mean": statistics.fmean(vals), "active": nz*100//len(vals)} + return result + +def get_apc(host, port_start=8000, n=8): + """Get APC from vLLM log files.""" + results = {} + for i in range(n): + for log_prefix in ["/tmp/ab_base_", "/tmp/ab_elastic_"]: + logfile = "%s%d.log" % (log_prefix, i) + try: + import subprocess + r = subprocess.run(["ssh", "-o", "ConnectTimeout=5", host, + "grep 'Prefix cache hit rate' %s 2>/dev/null | tail -1" % logfile], + capture_output=True, text=True, timeout=10) + line = r.stdout.strip() + if "Prefix cache hit rate:" in line: + import re + pch = re.search(r"Prefix cache hit rate: ([0-9.]+)", line) + ech = re.search(r"External prefix cache hit rate: ([0-9.]+)", line) + results[i] = { + "prefix": float(pch.group(1)) if pch else 0, + "external": float(ech.group(1)) if ech else 0, + } + except: + pass + return results + +sep = "=" * 80 +print(sep) +print(" A/B COMPARISON: Baseline (dash0) vs Elastic P2P (dash1)") +print(" Both: fresh restart, 200 req, time_scale=20, 8 sessions") +print(sep) + +# Latency +print("\n LATENCY:") +fmt = "%-30s %7s %8s %8s %8s %8s %8s %8s" +print(fmt % ("Config", "OK/N", "TTFT50", "TTFT90", "TPOT50", "TPOT90", "E2E50", "inp_p50")) +print("-" * 80) +for path, label in [ + ("outputs/ab_baseline/metrics.jsonl", "Baseline (combined)"), + ("outputs/ab_elastic/metrics.jsonl", "Elastic P2P (cap=4)"), +]: + if os.path.exists(path): + s = lat(path) + print(fmt % (label, "%d/%d" % (s["ok"],s["n"]), + "%.3f" % s["t50"], "%.3f" % s["t90"], "%.3f" % s["p50"], + "%.3f" % s["p90"], "%.3f" % s["e50"], str(s["inp50"]))) + +# Delta +b = lat("outputs/ab_baseline/metrics.jsonl") if os.path.exists("outputs/ab_baseline/metrics.jsonl") else None +a = lat("outputs/ab_elastic/metrics.jsonl") if os.path.exists("outputs/ab_elastic/metrics.jsonl") else None +if b and a: + print() + for label, bv, av in [("TTFT p50",b["t50"],a["t50"]),("TTFT p90",b["t90"],a["t90"]), + ("TPOT p90",b["p90"],a["p90"]),("E2E p50",b["e50"],a["e50"])]: + d = (av/bv-1)*100 if bv > 0 else 0 + print(" %s: %.3f -> %.3f (%+.1f%%)" % (label, bv, av, d)) + +# GPU utilization +print("\n GPU UTILIZATION:") +for path, label in [ + ("outputs/ab_baseline/gpu_util.csv", "Baseline"), + ("outputs/ab_elastic/gpu_util.csv", "Elastic"), +]: + gi = gpu_per_inst(path) + if gi: + means = [gi[g]["mean"] for g in sorted(gi.keys())] + actives = [gi[g]["active"] for g in sorted(gi.keys())] + print(" %s:" % label) + for g in sorted(gi.keys()): + print(" GPU%d: mean=%5.1f%% active=%2d%%" % (g, gi[g]["mean"], gi[g]["active"])) + print(" Aggregate: mean=%.1f%% imbalance=%.1fx" % ( + statistics.fmean(means), max(means)/max(min(means),0.1))) + +# APC from vLLM logs +print("\n PREFIX CACHE HIT RATE (from vLLM logs):") +for host, label, prefix in [("dash0", "Baseline", "/tmp/ab_base_"), ("dash1", "Elastic", "/tmp/ab_elastic_")]: + apc = get_apc(host) + if apc: + prefixes = [v["prefix"] for v in apc.values()] + externals = [v.get("external", 0) for v in apc.values()] + print(" %s:" % label) + for i in sorted(apc.keys()): + ext = " ext=%.1f%%" % apc[i]["external"] if apc[i].get("external") else "" + print(" inst_%d: prefix=%.1f%%%s" % (i, apc[i]["prefix"], ext)) + print(" Avg prefix: %.1f%% Avg external: %.1f%%" % ( + statistics.fmean(prefixes), statistics.fmean(externals))) diff --git a/scripts/plot_gpu_timeline.py b/scripts/plot_gpu_timeline.py new file mode 100644 index 0000000..a05b680 --- /dev/null +++ b/scripts/plot_gpu_timeline.py @@ -0,0 +1,101 @@ +"""Plot per-GPU utilization timeline for elastic vs baseline.""" +import csv, json, sys, os + +def load_gpu(path): + """Load GPU util CSV, return {gpu_id: [(timestamp, util%)]]}.""" + by_gpu = {} + with open(path) as f: + for r in csv.DictReader(f): + g = int(r["gpu"]) + t = float(r["timestamp"]) + u = float(r["util_pct"]) + by_gpu.setdefault(g, []).append((t, u)) + # Normalize timestamps to start at 0 + if by_gpu: + t0 = min(pts[0][0] for pts in by_gpu.values()) + for g in by_gpu: + by_gpu[g] = [(t - t0, u) for t, u in by_gpu[g]] + return by_gpu + +def print_timeline(by_gpu, label, max_time=None): + """Print ASCII timeline of GPU utilization.""" + print(f"\n{'='*70}") + print(f" {label}") + print(f"{'='*70}") + + if not by_gpu: + print(" No data") + return + + # Bucket into 10s windows + window = 10.0 + if max_time is None: + max_time = max(t for pts in by_gpu.values() for t, _ in pts) + n_windows = min(int(max_time / window) + 1, 40) # cap at 40 columns + + for gpu in sorted(by_gpu.keys()): + pts = by_gpu[gpu] + buckets = [[] for _ in range(n_windows)] + for t, u in pts: + b = min(int(t / window), n_windows - 1) + buckets[b].append(u) + + avgs = [sum(b)/len(b) if b else 0 for b in buckets] + # ASCII bar: . = 0-10%, o = 10-30%, O = 30-60%, # = 60-100% + bar = "" + for a in avgs: + if a < 1: bar += " " + elif a < 10: bar += "." + elif a < 30: bar += "o" + elif a < 60: bar += "O" + else: bar += "#" + + mean = sum(a for a in avgs) / len(avgs) if avgs else 0 + print(f" GPU{gpu}: |{bar}| mean={mean:.0f}%") + + print(f" Time: {'0':>1}{'':>{n_windows-6}}{int(max_time)}s") + print(f" Legend: ' '=0% .=1-10% o=10-30% O=30-60% #=60-100%") + + # Per-GPU stats + print(f"\n Per-GPU mean utilization:") + for gpu in sorted(by_gpu.keys()): + pts = by_gpu[gpu] + vals = [u for _, u in pts] + mean = sum(vals) / len(vals) + nz = sum(1 for v in vals if v > 0) + print(f" GPU{gpu}: mean={mean:.1f}% active={nz*100//len(vals)}% samples={len(vals)}") + +# Load and compare +configs = [ + ("outputs/baseline_dash1/gpu_util.csv", "Baseline (8 combined, dash1)"), + ("outputs/elastic_v4/gpu_util.csv", "Elastic P2P v4 (dash0)"), +] + +for path, label in configs: + if os.path.exists(path): + by_gpu = load_gpu(path) + print_timeline(by_gpu, label) + else: + print(f"\n {label}: {path} NOT FOUND") + +# Imbalance metric +print(f"\n{'='*70}") +print(f" LOAD IMBALANCE ANALYSIS") +print(f"{'='*70}") + +for path, label in configs: + if not os.path.exists(path): + continue + by_gpu = load_gpu(path) + means = [] + for gpu in sorted(by_gpu.keys()): + vals = [u for _, u in by_gpu[gpu]] + means.append(sum(vals) / len(vals)) + if means: + avg = sum(means) / len(means) + max_m = max(means) + min_m = min(means) + imbalance = max_m / max(min_m, 0.1) + print(f" {label}:") + print(f" Per-GPU means: {['%.1f' % m for m in means]}") + print(f" Avg={avg:.1f}% Min={min_m:.1f}% Max={max_m:.1f}% Imbalance={imbalance:.1f}x")