Gahow Wang
1b9268ba4c
Fixed race condition in P instance selection (all going to inst_0). P2P design: HEAVY requests prefill on least-loaded OTHER instance, KV transfer via Mooncake, decode on session-sticky instance. Result (200 req, fresh restart, vs baseline): TTFT p50: 1.080 -> 0.939 (-13%) <- median improves (decode not disrupted) TTFT p90: 9.410 -> 14.987 (+59%) <- tail worsens (KV transfer on large req) TPOT p90: 0.076 -> 0.075 (-1%) <- unchanged (not the bottleneck) E2E p50: 5.306 -> 5.565 (+5%) <- slightly worse overall The P2P offload helps the common case (WARM/MEDIUM get lower TTFT because their instance isn't blocked by a heavy prefill) but hurts HEAVY requests (extra KV transfer latency). This is a median-vs-tail tradeoff. For SLOs targeting p50: P2P offload helps. For SLOs targeting p90/p99: baseline combined is better. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
60 lines
2.0 KiB
Python
60 lines
2.0 KiB
Python
"""Compare P2P offload vs baseline."""
|
|
import json, csv, statistics, os
|
|
|
|
def lat(path):
|
|
rows = [json.loads(l) for l in open(path)]
|
|
ok = [r for r in rows if not r.get("error")]
|
|
ttfts = sorted([r["ttft_s"] for r in ok if r.get("ttft_s")])
|
|
tpots = sorted([r["tpot_s"] for r in ok if r.get("tpot_s") and r["tpot_s"]>0])
|
|
lats = sorted([r["latency_s"] for r in ok])
|
|
p = lambda v,q: v[min(int(q*len(v)),len(v)-1)] if v else 0
|
|
return {"ok": len(ok), "n": len(rows),
|
|
"t50": p(ttfts,.5), "t90": p(ttfts,.9),
|
|
"p50": p(tpots,.5), "p90": p(tpots,.9),
|
|
"e50": p(lats,.5)}
|
|
|
|
def gpu(path):
|
|
if not os.path.exists(path): return 0
|
|
rows = list(csv.DictReader(open(path)))
|
|
vals = [float(r["util_pct"]) for r in rows]
|
|
return statistics.fmean(vals) if vals else 0
|
|
|
|
print("P2P OFFLOAD vs BASELINE (both fresh restart, 200 req)")
|
|
print("=" * 75)
|
|
fmt = "%-30s %6s %8s %8s %8s %8s %8s %6s"
|
|
print(fmt % ("Config","OK/N","TTFT50","TTFT90","TPOT50","TPOT90","E2E50","GPU%"))
|
|
print("-" * 75)
|
|
|
|
configs = [
|
|
("baseline_dash1", "Baseline (8 combined)"),
|
|
("p2p_offload", "P2P offload (HEAVY on diff GPU)"),
|
|
]
|
|
|
|
results = {}
|
|
for d, label in configs:
|
|
mp = "outputs/%s/metrics.jsonl" % d
|
|
if not os.path.exists(mp):
|
|
print(" %s: NOT FOUND" % mp)
|
|
continue
|
|
s = lat(mp)
|
|
g = gpu("outputs/%s/gpu_util.csv" % d)
|
|
results[d] = s
|
|
print(fmt % (label, "%d/%d" % (s["ok"],s["n"]),
|
|
"%.3f" % s["t50"], "%.3f" % s["t90"],
|
|
"%.3f" % s["p50"], "%.3f" % s["p90"],
|
|
"%.3f" % s["e50"], "%.1f" % g))
|
|
|
|
if "baseline_dash1" in results and "p2p_offload" in results:
|
|
b = results["baseline_dash1"]
|
|
a = results["p2p_offload"]
|
|
print()
|
|
print("DELTA (P2P vs Baseline):")
|
|
for label, bv, av in [
|
|
("TTFT p50", b["t50"], a["t50"]),
|
|
("TTFT p90", b["t90"], a["t90"]),
|
|
("TPOT p90", b["p90"], a["p90"]),
|
|
("E2E p50", b["e50"], a["e50"]),
|
|
]:
|
|
d = (av/bv-1)*100 if bv > 0 else 0
|
|
print(" %s: %.3f -> %.3f (%+.1f%%)" % (label, bv, av, d))
|