gahow/agentic-kvc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

MB2 inter-node: dash1↔dash2 transfer cost is identical to intra-node

Browse Source 
Sweep on dash1 GPU 0 → dash2 GPU 0 over 200 Gbps RoCE.
remote_bootstrap_addr=http://172.27.123.142:8998. Same 9-size × 5-rep
config as the 2026-05-27 intra-node run.

Per-size pure_transfer (p50) lines up within 1–3% of the intra-node
numbers across all sizes:

  size      intra p50   inter p50
   512 tok    5.3 ms      5.2 ms
  2048 tok   20.6         20.0
  8192 tok   83.7         80.9
 32k  tok  320.9        309.6
 64k  tok 1895          1734       (bimodal in both)
128k  tok 2835          2818       (bimodal in both)

=> Mooncake's batch_transfer_sync_write **does not use NVLink** for
intra-node peers; both paths go through the 200 Gbps RDMA NIC, with
the 200 Gbps NIC (not the GPU interconnect) being the bottleneck. The
~9.7 GB/s steady-state ceiling and the 6+ GiB variance regime are
identical across topologies.

Operational implication for §3.2: PD-disaggregation does not get
cheaper by co-locating P and D on the same node — every routed request
pays the same ~10 GB/s ceiling for KV transfer, no matter where it
lands. Halving the transfer cost cannot be bought back by topology.

Caveat: B's receive_kv events did not log on dash2 — `MB2_LOG_DIR`
env var did not propagate through vLLM's EngineCore subprocess on
the consumer host (cat /proc/$ENGINE_PID/environ is empty on dash2
for that var, but the producer host on dash1 worked). For this run
pure_transfer numbers are from A's send_blocks alone; full rx_total
breakdown is not available, but pure_transfer is the dominant term.

Adds:
- analyze_mb2_send_only.py — analyzer that works from A's send_blocks
  alone when B's receive_kv events are absent
- plot_mb2_compare.py — overlay intra vs inter on the same axes
- plot_mb2.py — tolerate the `rows`-less send-only schema
- figs/mb2_transfer_{time,bw}_inter.png — inter-node single-curve
- figs/mb2_transfer_{time,bw}_compare.png — intra vs inter overlay
- analysis/mb2/A_inter_kvboth.jsonl, inter_kvboth_client.json,
  inter_kvboth_breakdown.json
- analysis/mb2/README.md — Summary block updated to reference both
  paths, dated 2026-05-27 run-log entry appended with the full table
  and the topology-independence framing

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
Gahow Wang
2026-05-27 20:56:08 +08:00
parent 50f72d8875
commit 90127c3389
11 changed files with 1096 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

80
microbench/fresh_setup/analyze_mb2_send_only.py Normal file
View File

@@ -0,0 +1,80 @@
#!/usr/bin/env python3
"""Per-size pure_transfer aggregate from A's send_blocks events alone.
Used when B's receive_kv events are missing (e.g. EngineCore env-var
propagation failed on the consumer host). Pure transfer time still
recoverable from the producer side.
"""
from __future__ import annotations
import argparse
import json
import statistics
from pathlib import Path
def main() -> None:
p = argparse.ArgumentParser()
p.add_argument("--a-log", type=Path, required=True)
p.add_argument("--out", type=Path, required=True)
p.add_argument("--min-tokens", type=int, default=32,
help="Skip events smaller than this (drop the spurious "
"init-time tiny sends)")
args = p.parse_args()
events = []
with args.a_log.open() as f:
for line in f:
try:
events.append(json.loads(line))
except json.JSONDecodeError:
continue
by_size: dict[int, list[float]] = {}
for e in events:
if e.get("event") != "send_blocks":
continue
sz_tokens = e["total_bytes"] // 98304
if sz_tokens < args.min_tokens:
continue
by_size.setdefault(sz_tokens, []).append(e["duration_s"])
summary = []
for sz in sorted(by_size):
durs = by_size[sz]
sz_bytes = sz * 98304
sz_mib = sz_bytes / (1024 * 1024)
bw = [sz_bytes / d / 1e9 for d in durs]
summary.append({
"input_tokens": sz,
"kv_mib": round(sz_mib, 1),
"n": len(durs),
"pure_transfer_ms_mean": round(statistics.mean(durs) * 1000, 2),
"pure_transfer_ms_p50": round(statistics.median(durs) * 1000, 2),
"pure_transfer_ms_min": round(min(durs) * 1000, 2),
"pure_transfer_ms_max": round(max(durs) * 1000, 2),
"throughput_gbps_mean": round(statistics.mean(bw), 2),
"throughput_gbps_p50": round(statistics.median(bw), 2),
"throughput_gbps_max": round(max(bw), 2),
})
print(f"loaded {len(events)} events; kept {sum(s['n'] for s in summary)} send_blocks")
print()
print(f"{'in_tok':>8} {'KV_MiB':>8} {'n':>4} "
f"{'pure_p50':>10} {'pure_min':>10} {'pure_max':>10} "
f"{'GB/s_p50':>10} {'GB/s_max':>10}")
for s in summary:
print(f"{s['input_tokens']:>8} {s['kv_mib']:>8.1f} {s['n']:>4} "
f"{s['pure_transfer_ms_p50']:>10.1f} "
f"{s['pure_transfer_ms_min']:>10.1f} "
f"{s['pure_transfer_ms_max']:>10.1f} "
f"{s['throughput_gbps_p50']:>10.2f} "
f"{s['throughput_gbps_max']:>10.2f}")
args.out.parent.mkdir(parents=True, exist_ok=True)
args.out.write_text(json.dumps({"summary": summary}, indent=2))
print(f"\nwrote {args.out}")
if __name__ == "__main__":
main()

7
microbench/fresh_setup/plot_mb2.py
View File

@@ -22,9 +22,10 @@ def main() -> None:
args = p.parse_args()
d = json.loads(args.breakdown.read_text())
# Drop the spurious 16-token events (zero-byte sends produced by the
# connector during request init; not a real KV transfer).
rows = [r for r in d["rows"] if r["input_tokens_est"] >= 64]
# `rows` is optional (send-only analyzer skips per-request joining).
# Drop the spurious 16-token events from any rows present.
if "rows" in d:
_ = [r for r in d["rows"] if r["input_tokens_est"] >= 64]
summary = [s for s in d["summary"] if s["input_tokens"] >= 64]
kv_mib = [s["kv_mib"] for s in summary]

99
microbench/fresh_setup/plot_mb2_compare.py Normal file
View File

@@ -0,0 +1,99 @@
#!/usr/bin/env python3
"""Overlay intra-node and inter-node MB2 curves on the same axes."""
from __future__ import annotations
import argparse
import json
from pathlib import Path
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt
import numpy as np
def load(path: Path) -> list[dict]:
d = json.loads(path.read_text())
return [s for s in d["summary"] if s["input_tokens"] >= 64]
def main() -> None:
p = argparse.ArgumentParser()
p.add_argument("--intra", type=Path, required=True)
p.add_argument("--inter", type=Path, required=True)
p.add_argument("--out-time", type=Path, default=Path("figs/mb2_transfer_time_compare.png"))
p.add_argument("--out-bw", type=Path, default=Path("figs/mb2_transfer_bw_compare.png"))
args = p.parse_args()
intra = load(args.intra)
inter = load(args.inter)
def axis_arrays(rows):
kv = [r["kv_mib"] for r in rows]
p50 = [r["pure_transfer_ms_p50"] for r in rows]
mn = [r["pure_transfer_ms_min"] for r in rows]
mx = [r["pure_transfer_ms_max"] for r in rows]
bw_p50 = [r["throughput_gbps_p50"] for r in rows]
bw_max = [r["throughput_gbps_max"] for r in rows]
return kv, p50, mn, mx, bw_p50, bw_max
ai_kv, ai_p50, ai_mn, ai_mx, ai_bw_p50, ai_bw_max = axis_arrays(intra)
bi_kv, bi_p50, bi_mn, bi_mx, bi_bw_p50, bi_bw_max = axis_arrays(inter)
# ---- transfer time ----
fig, ax = plt.subplots(figsize=(8.5, 5))
ax.errorbar(ai_kv, ai_p50,
yerr=[np.array(ai_p50) - np.array(ai_mn),
np.array(ai_mx) - np.array(ai_p50)],
fmt="o-", color="#1f77b4", lw=2, markersize=7, capsize=4,
label="intra-node (dash1 GPU 0↔1)")
ax.errorbar(bi_kv, bi_p50,
yerr=[np.array(bi_p50) - np.array(bi_mn),
np.array(bi_mx) - np.array(bi_p50)],
fmt="s--", color="#d62728", lw=2, markersize=7, capsize=4,
label="inter-node (dash1 GPU0 → dash2 GPU0)")
# ideal 9.7 GB/s reference
ref_x = np.array(ai_kv)
ref_y_ms = (ref_x * 1024 * 1024) / (9.7 * 1e9) * 1000
ax.plot(ref_x, ref_y_ms, "--", color="#888", alpha=0.5,
label="9.7 GB/s reference")
ax.axvline(11500, color="#7a1d1d", lw=0.8, ls=":", alpha=0.5)
ax.text(11500, 0.7, "p99 agentic req\n11.5 GiB",
fontsize=8, color="#7a1d1d", ha="center")
ax.set_xscale("log"); ax.set_yscale("log")
ax.set_xlabel("KV transfer size (MiB)")
ax.set_ylabel("Pure transfer time (ms, log)")
ax.set_title("MB2 intra vs inter — Mooncake transfer cost is topology-independent\n"
"(both paths go through 200 Gbps RDMA NIC; intra-node does not use NVLink)")
ax.grid(True, which="both", alpha=0.3)
ax.legend(loc="upper left", fontsize=9)
args.out_time.parent.mkdir(parents=True, exist_ok=True)
fig.tight_layout(); fig.savefig(args.out_time, dpi=150); plt.close(fig)
print(f"wrote {args.out_time}")
# ---- bandwidth ----
fig, ax = plt.subplots(figsize=(8.5, 5))
ax.plot(ai_kv, ai_bw_p50, "o-", color="#1f77b4", lw=2, markersize=7,
label="intra p50")
ax.plot(ai_kv, ai_bw_max, "x--", color="#1f77b4", lw=1.2, markersize=8,
alpha=0.7, label="intra max")
ax.plot(bi_kv, bi_bw_p50, "s-", color="#d62728", lw=2, markersize=7,
label="inter p50")
ax.plot(bi_kv, bi_bw_max, "+--", color="#d62728", lw=1.2, markersize=8,
alpha=0.7, label="inter max")
ax.axhline(9.7, color="#888", ls="--", alpha=0.5,
label="steady-state ≈ 9.7 GB/s")
ax.set_xscale("log")
ax.set_xlabel("KV transfer size (MiB)")
ax.set_ylabel("Effective bandwidth (GB/s)")
ax.set_ylim(0, 12)
ax.set_title("MB2 intra vs inter — bandwidth")
ax.grid(True, which="both", alpha=0.3)
ax.legend(loc="lower left", fontsize=9)
args.out_bw.parent.mkdir(parents=True, exist_ok=True)
fig.tight_layout(); fig.savefig(args.out_bw, dpi=150); plt.close(fig)
print(f"wrote {args.out_bw}")
if __name__ == "__main__":
main()