Fair A/B: Elastic P2P wins on ALL metrics vs baseline (fresh restart)
Same-condition comparison (both fresh restart, same trace, same params): Baseline (combined): TTFT=2.383/27.622 TPOT90=0.117 E2E=10.232 Elastic P2P (cap=4): TTFT=1.315/13.179 TPOT90=0.075 E2E=5.708 Delta: -45% / -52% -36% -44% Key finding: TPOT p90 dropped 36% — confirming heavy prefill DOES disrupt decode in combined mode, and elastic offload effectively isolates it. Previous comparisons missed this because baselines were run under different conditions (stale instances, different time_scale). GPU util: elastic uses less GPU (15.8% vs 28.7%) but achieves better latency — higher efficiency through better cache distribution. APC: elastic has more balanced per-instance APC (36-38% prefix + 30-35% external) vs baseline's skewed distribution (3.8% - 68.3%). Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
118
scripts/compare_ab_final.py
Normal file
118
scripts/compare_ab_final.py
Normal file
@@ -0,0 +1,118 @@
|
||||
"""Final A/B comparison: baseline (dash0) vs elastic (dash1).
|
||||
Both fresh restart, same trace, same params. GPU util + APC + latency."""
|
||||
import json, csv, statistics, os, urllib.request
|
||||
|
||||
def lat(path):
|
||||
rows = [json.loads(l) for l in open(path)]
|
||||
ok = [r for r in rows if not r.get("error")]
|
||||
err = [r for r in rows if r.get("error")]
|
||||
ttfts = sorted([r["ttft_s"] for r in ok if r.get("ttft_s")])
|
||||
tpots = sorted([r["tpot_s"] for r in ok if r.get("tpot_s") and r["tpot_s"]>0])
|
||||
lats = sorted([r["latency_s"] for r in ok])
|
||||
p = lambda v,q: v[min(int(q*len(v)),len(v)-1)] if v else 0
|
||||
ok_inp = sorted([r["input_length"] for r in ok])
|
||||
err_inp = sorted([r["input_length"] for r in err])
|
||||
return {"ok": len(ok), "n": len(rows),
|
||||
"t50": p(ttfts,.5), "t90": p(ttfts,.9),
|
||||
"p50": p(tpots,.5), "p90": p(tpots,.9),
|
||||
"e50": p(lats,.5), "e90": p(lats,.9),
|
||||
"inp50": p(ok_inp,.5), "err_inp50": p(err_inp,.5) if err_inp else 0}
|
||||
|
||||
def gpu_per_inst(path):
|
||||
if not os.path.exists(path):
|
||||
return {}
|
||||
rows = list(csv.DictReader(open(path)))
|
||||
by_gpu = {}
|
||||
for r in rows:
|
||||
g = int(r["gpu"])
|
||||
by_gpu.setdefault(g, []).append(float(r["util_pct"]))
|
||||
result = {}
|
||||
for g, vals in sorted(by_gpu.items()):
|
||||
nz = sum(1 for v in vals if v > 0)
|
||||
result[g] = {"mean": statistics.fmean(vals), "active": nz*100//len(vals)}
|
||||
return result
|
||||
|
||||
def get_apc(host, port_start=8000, n=8):
|
||||
"""Get APC from vLLM log files."""
|
||||
results = {}
|
||||
for i in range(n):
|
||||
for log_prefix in ["/tmp/ab_base_", "/tmp/ab_elastic_"]:
|
||||
logfile = "%s%d.log" % (log_prefix, i)
|
||||
try:
|
||||
import subprocess
|
||||
r = subprocess.run(["ssh", "-o", "ConnectTimeout=5", host,
|
||||
"grep 'Prefix cache hit rate' %s 2>/dev/null | tail -1" % logfile],
|
||||
capture_output=True, text=True, timeout=10)
|
||||
line = r.stdout.strip()
|
||||
if "Prefix cache hit rate:" in line:
|
||||
import re
|
||||
pch = re.search(r"Prefix cache hit rate: ([0-9.]+)", line)
|
||||
ech = re.search(r"External prefix cache hit rate: ([0-9.]+)", line)
|
||||
results[i] = {
|
||||
"prefix": float(pch.group(1)) if pch else 0,
|
||||
"external": float(ech.group(1)) if ech else 0,
|
||||
}
|
||||
except:
|
||||
pass
|
||||
return results
|
||||
|
||||
sep = "=" * 80
|
||||
print(sep)
|
||||
print(" A/B COMPARISON: Baseline (dash0) vs Elastic P2P (dash1)")
|
||||
print(" Both: fresh restart, 200 req, time_scale=20, 8 sessions")
|
||||
print(sep)
|
||||
|
||||
# Latency
|
||||
print("\n LATENCY:")
|
||||
fmt = "%-30s %7s %8s %8s %8s %8s %8s %8s"
|
||||
print(fmt % ("Config", "OK/N", "TTFT50", "TTFT90", "TPOT50", "TPOT90", "E2E50", "inp_p50"))
|
||||
print("-" * 80)
|
||||
for path, label in [
|
||||
("outputs/ab_baseline/metrics.jsonl", "Baseline (combined)"),
|
||||
("outputs/ab_elastic/metrics.jsonl", "Elastic P2P (cap=4)"),
|
||||
]:
|
||||
if os.path.exists(path):
|
||||
s = lat(path)
|
||||
print(fmt % (label, "%d/%d" % (s["ok"],s["n"]),
|
||||
"%.3f" % s["t50"], "%.3f" % s["t90"], "%.3f" % s["p50"],
|
||||
"%.3f" % s["p90"], "%.3f" % s["e50"], str(s["inp50"])))
|
||||
|
||||
# Delta
|
||||
b = lat("outputs/ab_baseline/metrics.jsonl") if os.path.exists("outputs/ab_baseline/metrics.jsonl") else None
|
||||
a = lat("outputs/ab_elastic/metrics.jsonl") if os.path.exists("outputs/ab_elastic/metrics.jsonl") else None
|
||||
if b and a:
|
||||
print()
|
||||
for label, bv, av in [("TTFT p50",b["t50"],a["t50"]),("TTFT p90",b["t90"],a["t90"]),
|
||||
("TPOT p90",b["p90"],a["p90"]),("E2E p50",b["e50"],a["e50"])]:
|
||||
d = (av/bv-1)*100 if bv > 0 else 0
|
||||
print(" %s: %.3f -> %.3f (%+.1f%%)" % (label, bv, av, d))
|
||||
|
||||
# GPU utilization
|
||||
print("\n GPU UTILIZATION:")
|
||||
for path, label in [
|
||||
("outputs/ab_baseline/gpu_util.csv", "Baseline"),
|
||||
("outputs/ab_elastic/gpu_util.csv", "Elastic"),
|
||||
]:
|
||||
gi = gpu_per_inst(path)
|
||||
if gi:
|
||||
means = [gi[g]["mean"] for g in sorted(gi.keys())]
|
||||
actives = [gi[g]["active"] for g in sorted(gi.keys())]
|
||||
print(" %s:" % label)
|
||||
for g in sorted(gi.keys()):
|
||||
print(" GPU%d: mean=%5.1f%% active=%2d%%" % (g, gi[g]["mean"], gi[g]["active"]))
|
||||
print(" Aggregate: mean=%.1f%% imbalance=%.1fx" % (
|
||||
statistics.fmean(means), max(means)/max(min(means),0.1)))
|
||||
|
||||
# APC from vLLM logs
|
||||
print("\n PREFIX CACHE HIT RATE (from vLLM logs):")
|
||||
for host, label, prefix in [("dash0", "Baseline", "/tmp/ab_base_"), ("dash1", "Elastic", "/tmp/ab_elastic_")]:
|
||||
apc = get_apc(host)
|
||||
if apc:
|
||||
prefixes = [v["prefix"] for v in apc.values()]
|
||||
externals = [v.get("external", 0) for v in apc.values()]
|
||||
print(" %s:" % label)
|
||||
for i in sorted(apc.keys()):
|
||||
ext = " ext=%.1f%%" % apc[i]["external"] if apc[i].get("external") else ""
|
||||
print(" inst_%d: prefix=%.1f%%%s" % (i, apc[i]["prefix"], ext))
|
||||
print(" Avg prefix: %.1f%% Avg external: %.1f%%" % (
|
||||
statistics.fmean(prefixes), statistics.fmean(externals)))
|
||||
101
scripts/plot_gpu_timeline.py
Normal file
101
scripts/plot_gpu_timeline.py
Normal file
@@ -0,0 +1,101 @@
|
||||
"""Plot per-GPU utilization timeline for elastic vs baseline."""
|
||||
import csv, json, sys, os
|
||||
|
||||
def load_gpu(path):
|
||||
"""Load GPU util CSV, return {gpu_id: [(timestamp, util%)]]}."""
|
||||
by_gpu = {}
|
||||
with open(path) as f:
|
||||
for r in csv.DictReader(f):
|
||||
g = int(r["gpu"])
|
||||
t = float(r["timestamp"])
|
||||
u = float(r["util_pct"])
|
||||
by_gpu.setdefault(g, []).append((t, u))
|
||||
# Normalize timestamps to start at 0
|
||||
if by_gpu:
|
||||
t0 = min(pts[0][0] for pts in by_gpu.values())
|
||||
for g in by_gpu:
|
||||
by_gpu[g] = [(t - t0, u) for t, u in by_gpu[g]]
|
||||
return by_gpu
|
||||
|
||||
def print_timeline(by_gpu, label, max_time=None):
|
||||
"""Print ASCII timeline of GPU utilization."""
|
||||
print(f"\n{'='*70}")
|
||||
print(f" {label}")
|
||||
print(f"{'='*70}")
|
||||
|
||||
if not by_gpu:
|
||||
print(" No data")
|
||||
return
|
||||
|
||||
# Bucket into 10s windows
|
||||
window = 10.0
|
||||
if max_time is None:
|
||||
max_time = max(t for pts in by_gpu.values() for t, _ in pts)
|
||||
n_windows = min(int(max_time / window) + 1, 40) # cap at 40 columns
|
||||
|
||||
for gpu in sorted(by_gpu.keys()):
|
||||
pts = by_gpu[gpu]
|
||||
buckets = [[] for _ in range(n_windows)]
|
||||
for t, u in pts:
|
||||
b = min(int(t / window), n_windows - 1)
|
||||
buckets[b].append(u)
|
||||
|
||||
avgs = [sum(b)/len(b) if b else 0 for b in buckets]
|
||||
# ASCII bar: . = 0-10%, o = 10-30%, O = 30-60%, # = 60-100%
|
||||
bar = ""
|
||||
for a in avgs:
|
||||
if a < 1: bar += " "
|
||||
elif a < 10: bar += "."
|
||||
elif a < 30: bar += "o"
|
||||
elif a < 60: bar += "O"
|
||||
else: bar += "#"
|
||||
|
||||
mean = sum(a for a in avgs) / len(avgs) if avgs else 0
|
||||
print(f" GPU{gpu}: |{bar}| mean={mean:.0f}%")
|
||||
|
||||
print(f" Time: {'0':>1}{'':>{n_windows-6}}{int(max_time)}s")
|
||||
print(f" Legend: ' '=0% .=1-10% o=10-30% O=30-60% #=60-100%")
|
||||
|
||||
# Per-GPU stats
|
||||
print(f"\n Per-GPU mean utilization:")
|
||||
for gpu in sorted(by_gpu.keys()):
|
||||
pts = by_gpu[gpu]
|
||||
vals = [u for _, u in pts]
|
||||
mean = sum(vals) / len(vals)
|
||||
nz = sum(1 for v in vals if v > 0)
|
||||
print(f" GPU{gpu}: mean={mean:.1f}% active={nz*100//len(vals)}% samples={len(vals)}")
|
||||
|
||||
# Load and compare
|
||||
configs = [
|
||||
("outputs/baseline_dash1/gpu_util.csv", "Baseline (8 combined, dash1)"),
|
||||
("outputs/elastic_v4/gpu_util.csv", "Elastic P2P v4 (dash0)"),
|
||||
]
|
||||
|
||||
for path, label in configs:
|
||||
if os.path.exists(path):
|
||||
by_gpu = load_gpu(path)
|
||||
print_timeline(by_gpu, label)
|
||||
else:
|
||||
print(f"\n {label}: {path} NOT FOUND")
|
||||
|
||||
# Imbalance metric
|
||||
print(f"\n{'='*70}")
|
||||
print(f" LOAD IMBALANCE ANALYSIS")
|
||||
print(f"{'='*70}")
|
||||
|
||||
for path, label in configs:
|
||||
if not os.path.exists(path):
|
||||
continue
|
||||
by_gpu = load_gpu(path)
|
||||
means = []
|
||||
for gpu in sorted(by_gpu.keys()):
|
||||
vals = [u for _, u in by_gpu[gpu]]
|
||||
means.append(sum(vals) / len(vals))
|
||||
if means:
|
||||
avg = sum(means) / len(means)
|
||||
max_m = max(means)
|
||||
min_m = min(means)
|
||||
imbalance = max_m / max(min_m, 0.1)
|
||||
print(f" {label}:")
|
||||
print(f" Per-GPU means: {['%.1f' % m for m in means]}")
|
||||
print(f" Avg={avg:.1f}% Min={min_m:.1f}% Max={max_m:.1f}% Imbalance={imbalance:.1f}x")
|
||||
Reference in New Issue
Block a user