Layerwise under load: overlap benefit survives (bg=16)

mb7 with background decode load (8/instance). Critical-path transfer overhead
stays ~constant ~90ms for layerwise vs 158/239/749ms baseline (up to 7.9x at
32k), prefill not slowed, KV correct. Confirms the overlap holds on busy
instances. DESIGN.md updated with idle-vs-load table + the two blockers
(chunk-safety, concurrent-transfer safety) that the full 1200-req trace needs.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>

microbench/connector_tax/layerwise/mb7_layerwise.py

@@ -72,6 +72,56 @@ def cached_of(resp) -> int:
     return det.get("cached_tokens", 0) or usage.get("cached_tokens", 0) or 0
 
 
+async def _stream_completion(client, host, port, prompt, max_tokens):
+    payload = {"model": MODEL, "prompt": prompt, "max_tokens": max_tokens,
+               "min_tokens": 1, "temperature": 0.0, "stream": True}
+    async with client.stream("POST", f"http://{host}:{port}/v1/completions",
+                             json=payload, timeout=600.0) as r:
+        r.raise_for_status()
+        async for _ in r.aiter_bytes():
+            pass
+
+
+class BackgroundLoad:
+    """Hold N concurrent long-decode streams across endpoints to keep busy."""
+    def __init__(self, client, endpoints, n, prompt_tokens=2000, out_tokens=6000):
+        self.client, self.endpoints, self.n = client, endpoints, n
+        self.pt, self.ot = prompt_tokens, out_tokens
+        self._stop = asyncio.Event()
+        self._tasks = []
+
+    async def _w(self, idx):
+        host, port = self.endpoints[idx % len(self.endpoints)]
+        seed = 800000 + idx
+        while not self._stop.is_set():
+            try:
+                await _stream_completion(self.client, host, port,
+                                         synth_prompt(seed, self.pt), self.ot)
+            except Exception:
+                await asyncio.sleep(0.5)
+            seed += 1
+
+    def start(self):
+        self._tasks = [asyncio.create_task(self._w(i)) for i in range(self.n)]
+
+    async def stop(self):
+        self._stop.set()
+        for t in self._tasks:
+            t.cancel()
+        await asyncio.gather(*self._tasks, return_exceptions=True)
+
+
+async def num_running(client, host, port):
+    try:
+        r = await client.get(f"http://{host}:{port}/metrics", timeout=5.0)
+        for line in r.text.splitlines():
+            if line.startswith("vllm:num_requests_running"):
+                return int(float(line.split()[-1]))
+    except Exception:
+        pass
+    return -1
+
+
 async def prefill_only(client, host, port, prompt):
     """Reference: plain prefill cost on A, no transfer."""
     dt, _ = await completion(client, host, port, prompt, max_tokens=1)
@@ -128,7 +178,20 @@ async def main_async(a):
     async with httpx.AsyncClient(limits=limits, trust_env=False) as client:
         src_eid = await get_engine_id(client, a.src_host, a.src_bp)
         src_bp_addr = f"http://{a.src_host}:{a.src_bp}"
-        print(f"[mb7] mode={a.mode} src_eid={src_eid[:16]}...")
+        print(f"[mb7] mode={a.mode} bg_load={a.bg_load} src_eid={src_eid[:16]}...")
+
+        loader = None
+        if a.bg_load > 0:
+            loader = BackgroundLoad(client, [A, B], a.bg_load)
+            loader.start()
+            print(f"[mb7] ramping background load ({a.bg_load}) ...")
+            for _ in range(40):
+                await asyncio.sleep(1.0)
+                na = await num_running(client, *A)
+                nb = await num_running(client, *B)
+                if na >= 1 and nb >= 1:
+                    print(f"[mb7] busy: A_run={na} B_run={nb}")
+                    break
 
         results = []
         for sz in sizes:
@@ -155,8 +218,11 @@ async def main_async(a):
                       f"total={row['t_total_s']*1000:7.0f}ms {extra} "
                       f"cached={row['cached']}/{sz} correct={row['correct']}")
 
+        if loader:
+            await loader.stop()
+
     # summary
-    print(f"\n=== {a.mode} summary ===")
+    print(f"\n=== {a.mode} (bg={a.bg_load}) summary ===")
     print(f"{'size':>7} {'n':>2} {'pf_only_ms':>11} {'total_ms':>9} "
           f"{'overhead_ms':>12} {'correct':>8}")
     summary = []
@@ -191,6 +257,8 @@ def main():
     p.add_argument("--dst-bp", type=int, default=8999)
     p.add_argument("--sizes", default="8192,32768,65536")
     p.add_argument("--repeats", type=int, default=3)
+    p.add_argument("--bg-load", type=int, default=0,
+                   help="N concurrent background decode streams across A+B")
     p.add_argument("--out", default="mb7_result.json")
     args = p.parse_args()
     asyncio.run(main_async(args))