Add NIXL substrate isolation control + attribution decomposition
Adds unified_nixl_both to elastic_migration_v2: same picker as unified_kv_both (never triggers PD-sep), but launches vLLM with NixlConnector instead of MooncakeConnector. Compared against plain unified and unified_kv_both (Mooncake) we can now attribute the substrate overhead between "v1 connector framework irreducible cost" (proxied by the leaner NIXL) and "Mooncake implementation extra" (Mooncake - NIXL). Result (vs plain unified, both substrates never PD-sep): metric plain NIXL Mooncake TTFT p90 7.35s +37.9% +45.3% (NIXL: +7pp better) TPOT p90 17.1ms +15.5% +24.5% (NIXL: +9pp better) E2E p90 18.03s +17.4% +27.0% (NIXL: +10pp better) hotspot 3.667 +0.2% +19.0% (NIXL: keeps it flat) APC 79.4% -0.3pp -1.1pp interference - 5.58 8.57 (NIXL: ~35% lower) The cleanest signal is hotspot: NIXL preserves plain-unified's distribution (3.674 vs 3.667), while Mooncake's per-scheduler-step O(|cache|) `set(self._block_pool.cache.keys())` diff against _known_hash_keys (mooncake_connector.py:432-456) inflates routing imbalance by 19%. The hash sync runs unconditionally even when no direct_read consumer is present. Attribution: NIXL-plain ~= v1 framework irreducible cost (kv_buffer GPU memory, per-step SchedulerOutput.kv_connector_metadata round-trip, altered kv_cache_manager block-lifecycle). Mooncake-NIXL ~= Mooncake-specific overhead (the hash-sync loop and stricter delay_free semantics). Practical implication: NIXL is meaningfully better than Mooncake on this stack, but even NIXL imposes 16-38% across metrics — too expensive for selective-PD-sep on agentic workloads where the trigger rate is < 0.5%. Launch fixes required for NIXL multi-instance: - VLLM_NIXL_SIDE_CHANNEL_PORT must be unique per instance (default 5600; we use 5600+i). Without this, 7 of 8 instances silently hang in `zmq.error.ZMQError: Address already in use` and the launcher trap kills all of them at health-check timeout. - Health-check timeout raised from 180s to 360s; NIXL initialization (UCX agent + memory registration) is ~100-150s per instance under 8-way concurrent load, vs Mooncake's ~30-60s. New figure: fig_connector_substrate_attribution.png stacks plain / framework / Mooncake-extra / v2-branch overhead per metric. Existing figures (fig_kv_both_overhead, fig_three_way_hotspot) updated to include NIXL as a fourth bar. README updated with 4-way table, Result 1 reframed as "the cost is mostly framework, not Mooncake — but Mooncake adds the hotspot penalty", and the substrate-vs-PD-sep tradeoff math. Refs: nixl_connector.py:700 handshake listener bind, factory.py register_connector for the NixlConnector entry. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
@@ -34,37 +34,39 @@ def _load(name: str):
|
||||
|
||||
|
||||
POLICY_COLORS = {
|
||||
"unified": "#2ca02c",
|
||||
"unified_kv_both": "#9467bd",
|
||||
"unified_v2": "#d62728",
|
||||
"unified_v2_strict": "#ff7f0e",
|
||||
"unified": "#2ca02c",
|
||||
"unified_kv_both": "#9467bd",
|
||||
"unified_nixl_both": "#1f77b4",
|
||||
"unified_v2": "#d62728",
|
||||
"unified_v2_strict": "#ff7f0e",
|
||||
}
|
||||
|
||||
|
||||
def fig_kv_both_overhead():
|
||||
comp = _load("b3_policy_comparison.json")
|
||||
by = {r["policy"]: r for r in comp["rows"]}
|
||||
pols = ["unified", "unified_kv_both", "unified_v2"]
|
||||
pols = ["unified", "unified_kv_both", "unified_nixl_both", "unified_v2"]
|
||||
metrics = [
|
||||
("TTFT p90 (s)", lambda r: r["ttft_p90_s"]),
|
||||
("TPOT p90 (ms)", lambda r: r["tpot_p90_s"] * 1000),
|
||||
("E2E p90 (s)", lambda r: r["e2e_p90_s"]),
|
||||
("hotspot index", lambda r: r["hotspot_index_ttft_p90"]),
|
||||
]
|
||||
fig, axes = plt.subplots(1, 4, figsize=(14, 4))
|
||||
fig, axes = plt.subplots(1, 4, figsize=(15, 4.2))
|
||||
for ax, (label, fn) in zip(axes, metrics):
|
||||
vals = [fn(by[p]) for p in pols]
|
||||
bars = ax.bar(pols, vals,
|
||||
labels_short = [p.replace("unified_", "") for p in pols]
|
||||
labels_short[0] = "plain"
|
||||
bars = ax.bar(labels_short, vals,
|
||||
color=[POLICY_COLORS[p] for p in pols],
|
||||
edgecolor="black", linewidth=0.5)
|
||||
ax.set_title(label)
|
||||
ax.tick_params(axis="x", rotation=20, labelsize=9)
|
||||
ax.tick_params(axis="x", rotation=15, labelsize=9)
|
||||
for b, v in zip(bars, vals):
|
||||
ax.text(b.get_x() + b.get_width() / 2, v,
|
||||
f"{v:.2f}" if v < 100 else f"{v:.0f}",
|
||||
ha="center", va="bottom", fontsize=9)
|
||||
ax.grid(alpha=0.3, axis="y")
|
||||
# delta annotation
|
||||
baseline = vals[0]
|
||||
for i, v in enumerate(vals):
|
||||
if i == 0:
|
||||
@@ -74,8 +76,8 @@ def fig_kv_both_overhead():
|
||||
fontsize=10, fontweight="bold",
|
||||
color="darkred" if pct > 0 else "darkgreen")
|
||||
fig.suptitle(
|
||||
"kv_both adds ~45% to TTFT p90 even without PD-sep firing.\n"
|
||||
"v2's PD-sep barely recovers the gap (and overshoots TTFT p99)."
|
||||
"Mooncake substrate adds 19-45% across metrics; NIXL is 5-19pp better but\n"
|
||||
"still 16-38% above plain. v2's 5 PD-sep events don't recover the substrate tax."
|
||||
)
|
||||
fig.tight_layout()
|
||||
fig.savefig(OUT / "fig_kv_both_overhead.png", dpi=120)
|
||||
@@ -203,27 +205,29 @@ def fig_v2_predicted_vs_actual():
|
||||
|
||||
|
||||
def fig_three_way_hotspot():
|
||||
pols = ["unified", "unified_kv_both", "unified_v2"]
|
||||
pols = ["unified", "unified_kv_both", "unified_nixl_both", "unified_v2"]
|
||||
per_worker = {p: _load(f"per_worker_{p}.json") for p in pols}
|
||||
workers = sorted(per_worker["unified"]["per_worker_ttft_p90_s"].keys())
|
||||
|
||||
x = range(len(workers))
|
||||
width = 0.27
|
||||
fig, ax = plt.subplots(figsize=(11, 5))
|
||||
n = len(pols)
|
||||
width = 0.85 / n
|
||||
fig, ax = plt.subplots(figsize=(12, 5))
|
||||
for i, p in enumerate(pols):
|
||||
d = per_worker[p]["per_worker_ttft_p90_s"]
|
||||
vals = [d[w] for w in workers]
|
||||
offset = (i - 1) * width
|
||||
offset = (i - (n - 1) / 2) * width
|
||||
label = p.replace("unified_", "") if p != "unified" else "plain"
|
||||
ax.bar([j + offset for j in x], vals, width,
|
||||
label=f"{p} (hotspot={per_worker[p]['hotspot_index_ttft_p90']:.2f})",
|
||||
label=f"{label} (hotspot={per_worker[p]['hotspot_index_ttft_p90']:.2f})",
|
||||
color=POLICY_COLORS[p], edgecolor="black", linewidth=0.4)
|
||||
short = [w.replace("http://127.0.0.1:", ":") for w in workers]
|
||||
ax.set_xticks(list(x))
|
||||
ax.set_xticklabels(short, rotation=0, fontsize=9)
|
||||
ax.set_ylabel("worker TTFT p90 (s)")
|
||||
ax.set_title(
|
||||
"Per-worker TTFT p90 distribution. kv_both alone makes the hot worker hotter\n"
|
||||
"(unified→kv_both: 37.7s→43.5s peak); v2's 5 PD-sep triggers nudge it back."
|
||||
"Per-worker TTFT p90 distribution across substrates. Mooncake (kv_both)\n"
|
||||
"amplifies the hot worker (hotspot 4.36); NIXL keeps it close to plain (3.67)."
|
||||
)
|
||||
ax.legend(loc="upper left", fontsize=9)
|
||||
ax.grid(alpha=0.3, axis="y")
|
||||
@@ -232,12 +236,64 @@ def fig_three_way_hotspot():
|
||||
plt.close(fig)
|
||||
|
||||
|
||||
def fig_connector_substrate_attribution():
|
||||
"""Decomposes overhead into v1-framework cost (shared by all connectors,
|
||||
proxied by NIXL since it's the leanest) and Mooncake-specific cost."""
|
||||
comp = _load("b3_policy_comparison.json")
|
||||
by = {r["policy"]: r for r in comp["rows"]}
|
||||
metrics = [
|
||||
("TTFT p90 (s)", "ttft_p90_s", False),
|
||||
("TPOT p90 (ms)", "tpot_p90_s", True),
|
||||
("E2E p90 (s)", "e2e_p90_s", False),
|
||||
("hotspot index", "hotspot_index_ttft_p90", False),
|
||||
]
|
||||
fig, axes = plt.subplots(1, 4, figsize=(15, 4))
|
||||
for ax, (label, key, scale_ms) in zip(axes, metrics):
|
||||
plain = by["unified"][key] * (1000 if scale_ms else 1)
|
||||
nixl = by["unified_nixl_both"][key] * (1000 if scale_ms else 1)
|
||||
moon = by["unified_kv_both"][key] * (1000 if scale_ms else 1)
|
||||
v2 = by["unified_v2"][key] * (1000 if scale_ms else 1)
|
||||
|
||||
framework_cost = nixl - plain # what NIXL adds = v1 framework cost
|
||||
mooncake_extra = moon - nixl # extra on top from Mooncake
|
||||
v2_branch_extra = v2 - moon # extra from PD-sep branch (Mooncake + 5 events)
|
||||
|
||||
bottom = 0
|
||||
ax.bar(["overhead"], [plain], color="#cccccc",
|
||||
edgecolor="black", linewidth=0.4,
|
||||
label=f"plain unified ({plain:.2f})")
|
||||
bottom += plain
|
||||
ax.bar(["overhead"], [framework_cost], bottom=[bottom],
|
||||
color="#1f77b4", edgecolor="black", linewidth=0.4,
|
||||
label=f"v1 framework (+{framework_cost:.2f})")
|
||||
bottom += framework_cost
|
||||
ax.bar(["overhead"], [mooncake_extra], bottom=[bottom],
|
||||
color="#9467bd", edgecolor="black", linewidth=0.4,
|
||||
label=f"Mooncake extra (+{mooncake_extra:.2f})")
|
||||
bottom += mooncake_extra
|
||||
ax.bar(["overhead"], [v2_branch_extra], bottom=[bottom],
|
||||
color="#d62728", edgecolor="black", linewidth=0.4,
|
||||
label=f"v2 PD-sep branch ({v2_branch_extra:+.2f})")
|
||||
ax.set_title(label)
|
||||
ax.legend(fontsize=8, loc="upper right")
|
||||
ax.grid(alpha=0.3, axis="y")
|
||||
ax.tick_params(axis="x", labelbottom=False)
|
||||
fig.suptitle(
|
||||
"Attribution: plain unified vs NIXL substrate vs Mooncake substrate vs v2.\n"
|
||||
"Blue: cost shared by any v1 connector. Purple: cost specific to Mooncake."
|
||||
)
|
||||
fig.tight_layout()
|
||||
fig.savefig(OUT / "fig_connector_substrate_attribution.png", dpi=120)
|
||||
plt.close(fig)
|
||||
|
||||
|
||||
def main():
|
||||
fig_kv_both_overhead()
|
||||
fig_v2_trigger_funnel()
|
||||
fig_v2_predicted_vs_actual()
|
||||
fig_three_way_hotspot()
|
||||
print(f"wrote 4 figures to {OUT}")
|
||||
fig_connector_substrate_attribution()
|
||||
print(f"wrote 5 figures to {OUT}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
Reference in New Issue
Block a user