MB5 PD-disagg pipeline: working end-to-end

Three independent bugs were blocking PD-disagg smoke; each fix is
isolated so the next PD experiment doesn't re-hit them.

1. mb5_launch.sh
   - stop_all() also kills mb5_pd_proxy.py (our vendored copy),
     not just the upstream filename, and asserts ports 8000-8007 +
     PROXY_PORT are free before launching — stale proxies were
     silently passing the readiness check.
   - Proxy readiness uses a generic "any HTTP response" probe;
     mooncake_connector_proxy only exposes /v1/completions so
     /v1/models 404 is expected.

2. mb5_pd_proxy.py (vendored from third_party so deploy.sh ships it)
   - Force min_tokens=1 on the prefill leg. Clients that set
     min_tokens == max_tokens (our replayer does) collide with
     vLLM's min_tokens<=max_tokens check after the proxy caps
     max_tokens=1.

3. instrument_kv_snapshot.py
   - Adds a second patch target: initialize
     MooncakeConnectorWorker.bootstrap_server = None in __init__.
     vLLM 0.18.1 only sets it under the is_kv_producer branch, so
     kv_consumer hits AttributeError as soon as the first remote
     prefill request lands.
   - apply/revert refactored to iterate over (path, patches) pairs.

plot_kv_pool_timeline.py also handles snapshot files that never
captured a running request (would otherwise IndexError on an empty
stackplot input).

Smoke: 4P+4D × 20 reqs → 20/20 success, mean 3.9s, p99 17s, 8 PIDs
all writing snapshots (601 total), well above the 8C baseline.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>

microbench/fresh_setup/instrument_kv_snapshot.py

@@ -32,6 +32,10 @@ from pathlib import Path
 
 DEFAULT_VENV = Path("/home/admin/cpfs/wjh/agentic-kv-fresh/.venv")
 TARGET_REL = "lib/python3.12/site-packages/vllm/v1/core/sched/scheduler.py"
+MOONCAKE_REL = (
+    "lib/python3.12/site-packages/vllm/distributed/kv_transfer/"
+    "kv_connector/v1/mooncake/mooncake_connector.py"
+)
 
 START_MARK = "# MB5_INSTRUMENT_START"
 END_MARK = "# MB5_INSTRUMENT_END"
@@ -165,45 +169,66 @@ SCHED_RET_REPLACE = f"""        {START_MARK}
     def _agentic_emit_step_log("""
 
 
-PATCHES = [
+SCHED_PATCHES = [
     ("header",            HEADER_ANCHOR,     HEADER_ANCHOR + HEADER_INSERT),
     ("schedule() return", SCHED_RET_TARGET,  SCHED_RET_REPLACE),
 ]
 
+# ---------- Patch 3: vLLM 0.18.1 kv_consumer AttributeError fix --------------
+# In MooncakeConnectorWorker.__init__, `self.bootstrap_server` is only assigned
+# inside the `is_kv_producer` branch (around line 615). For kv_consumer roles
+# the attribute is never set, but later code paths (e.g. line ~1060) check
+# `if self.bootstrap_server is not None:` and AttributeError. We initialize it
+# unconditionally just before the role-conditional branch.
+MOONCAKE_ANCHOR = "        self.reqs_need_send: dict[TransferId, SendBlockMeta] = {}\n"
+MOONCAKE_INSERT = (
+    f"        {START_MARK}\n"
+    f"        self.bootstrap_server = None  # vLLM 0.18.1 kv_consumer fix\n"
+    f"        {END_MARK}\n"
+)
 
-def find_target(venv_or_path: Path) -> Path:
-    candidates = [venv_or_path, DEFAULT_VENV / TARGET_REL]
+MOONCAKE_PATCHES = [
+    ("kv_consumer bootstrap_server init", MOONCAKE_ANCHOR,
+     MOONCAKE_ANCHOR + MOONCAKE_INSERT),
+]
+
+PATCH_FILES = [
+    (TARGET_REL, SCHED_PATCHES),
+    (MOONCAKE_REL, MOONCAKE_PATCHES),
+]
+
+
+def find_target(venv_or_path: Path, rel_path: str) -> Path:
+    candidates = [venv_or_path / rel_path, DEFAULT_VENV / rel_path]
     for c in candidates:
         if c.is_file():
             return c
-        if c.is_dir():
-            sub = c / TARGET_REL
-            if sub.is_file():
-                return sub
-    raise FileNotFoundError(f"cannot find vllm V1 scheduler at {venv_or_path}")
+    raise FileNotFoundError(
+        f"cannot find {rel_path} under {venv_or_path}"
+    )
 
 
 def is_patched(text: str) -> bool:
     return START_MARK in text
 
 
-def apply(target: Path) -> None:
+def apply_one(target: Path, patches: list) -> None:
     text = target.read_text()
     if is_patched(text):
         print(f"[mb5-instr] already patched: {target}")
         return
     new = text
-    for name, src, dst in PATCHES:
+    for name, src, dst in patches:
         if src not in new:
             raise RuntimeError(
                 f"patch {name!r}: anchor not found in {target}."
             )
         new = new.replace(src, dst, 1)
     target.write_text(new)
-    print(f"[mb5-instr] applied {len(PATCHES)} patches -> {target}")
+    print(f"[mb5-instr] applied {len(patches)} patches -> {target}")
 
 
-def revert(target: Path) -> None:
+def revert_one(target: Path) -> None:
     text = target.read_text()
     if not is_patched(text):
         print(f"[mb5-instr] not patched (nothing to revert): {target}")
@@ -225,16 +250,18 @@ def main() -> None:
     p.add_argument("--check", action="store_true")
     p.add_argument("--venv", type=Path, default=DEFAULT_VENV)
     args = p.parse_args()
-    target = find_target(args.venv)
-    if args.apply:
-        apply(target)
-    elif args.revert:
-        revert(target)
-    elif args.check:
-        text = target.read_text()
-        print(f"[mb5-instr] {'PATCHED' if is_patched(text) else 'CLEAN'}: {target}")
-    else:
-        p.error("specify --apply / --revert / --check")
+    for rel_path, patches in PATCH_FILES:
+        target = find_target(args.venv, rel_path)
+        if args.apply:
+            apply_one(target, patches)
+        elif args.revert:
+            revert_one(target)
+        elif args.check:
+            text = target.read_text()
+            state = 'PATCHED' if is_patched(text) else 'CLEAN'
+            print(f"[mb5-instr] {state}: {target}")
+        else:
+            p.error("specify --apply / --revert / --check")
 
 
 if __name__ == "__main__":