Paper section: system analysis + workload figures + KV-wall model

Adds the system-level argument resolving the roofline/PD-sep paradox.
Even at 95% cache reuse prefill stays compute-bound (the C6 roofline
fact), yet PD separation regresses TTFT 72%. The new system_analysis.md
walks through six layers showing why the roofline claim is necessary
but not sufficient, with the falsifiable condition being decode-side
KV memory budget: concurrent_decode * KV_per_req / (N_D * HBM_pool).

For chatbot this ratio is << 1 at any layout; for agentic at p90+
context it goes >> 1 under 4P+4D and 6P+2D, predicting the empirical
97% decode KV occupancy. fig_kv_memory_wall.pdf visualizes the model
with audit-able constants; fig_c1a/b ground the per-request KV-size
inputs in the actual sampled trace (input p50=33.5k, p90=101k,
intra-session reuse 79.2%).

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>

analysis/pd_sep_paper_section/scripts/plot_kv_memory_wall.py

@@ -0,0 +1,127 @@
+"""Decode-side KV cache memory budget as a function of per-request KV
+footprint and prefill/decode split.
+
+Idea: PD separation is equivalent to multiplying the per-D-instance KV
+demand by (N_total / N_D). For workloads with large per-request KV
+footprint (agentic), this concentration breaches the memory wall.
+
+The plot fixes the system-wide concurrent decode count to a steady-state
+estimate from the trace (QPS x avg_decode_seconds) and shows per-D-instance
+KV pool occupancy as a function of per-request KV footprint, one line per
+PD layout.
+
+All constants are documented at the top so they can be audited.
+"""
+import argparse
+from pathlib import Path
+
+import matplotlib
+matplotlib.use("Agg")
+import matplotlib.pyplot as plt
+import numpy as np
+
+# ---- Cluster constants (8x H20, vLLM 0.18.1) ----
+N_TOTAL_GPUS = 8
+HBM_PER_GPU_GB = 96
+MODEL_GB = 50            # Qwen3-30B-A3B MoE weights bf16
+ACTIVATION_OVERHEAD_GB = 18
+KV_POOL_PER_GPU_GB = HBM_PER_GPU_GB - MODEL_GB - ACTIVATION_OVERHEAD_GB  # ~28
+
+# ---- Workload steady-state ----
+# Peak QPS on the sampled trace = 1.6; mean E2E ~5s under Combined; both
+# numbers from REPORT.md. So at any instant ~8 decodes are alive.
+CONCURRENT_DECODE = 8
+
+# ---- KV footprint constants for Qwen3-30B-A3B ----
+# 2 (K+V) * 4 kv-heads * 128 head_dim * 2 bytes * 48 layers = 96 KB / token.
+KV_BYTES_PER_TOKEN = 2 * 4 * 128 * 2 * 48  # = 98304 bytes
+
+
+def kv_mb(seqlen_tokens):
+    return seqlen_tokens * KV_BYTES_PER_TOKEN / 1e6
+
+
+# Reference operating points (per-request KV size, MB)
+POINTS = [
+    ("chatbot avg (~2k input)",   kv_mb(2_000)),
+    ("agentic avg (33.6k input)", kv_mb(33_600)),
+    ("agentic p90 (101k input)",  kv_mb(101_000)),
+    ("agentic p99 (132k input)",  kv_mb(132_000)),
+]
+
+# PD layouts: (label, N_D, color, linestyle)
+LAYOUTS = [
+    ("Combined 8C  (N_D=8)",    8, "#2ca02c", "-"),
+    ("PD-sep 4P+4D (N_D=4)",    4, "#ff7f0e", "--"),
+    ("PD-sep 6P+2D (N_D=2)",    2, "#d62728", "-."),
+]
+
+
+def occupancy(kv_per_req_mb, n_d):
+    """Per-D-instance KV pool occupancy (fraction)."""
+    pool_mb = KV_POOL_PER_GPU_GB * 1024
+    demand_mb = CONCURRENT_DECODE * kv_per_req_mb / n_d
+    return demand_mb / pool_mb
+
+
+def plot(out_path):
+    fig, ax = plt.subplots(figsize=(8.5, 4.6))
+
+    kv_range_mb = np.logspace(0.0, 4.5, 400)  # 1 MB .. ~30 GB
+    for label, n_d, color, ls in LAYOUTS:
+        y = occupancy(kv_range_mb, n_d) * 100
+        ax.plot(kv_range_mb, y, color=color, lw=1.8, ls=ls, label=label)
+
+    # memory-wall threshold (vLLM starts queuing aggressively above ~90%)
+    ax.axhline(90, color="#888", ls=":", lw=1)
+    ax.text(1.2, 93, "memory wall (~90%, vLLM stops admitting new reqs)",
+            fontsize=8.5, color="#666")
+
+    # operating-point markers, labelled along the top edge
+    for name, kv in POINTS:
+        ax.axvline(kv, color="#777", lw=0.7, ls=(0, (1, 2)))
+        ax.text(kv, 198, name, fontsize=8, color="#333",
+                rotation=90, ha="right", va="top")
+
+    ax.set_xscale("log")
+    ax.set_xlim(50, 3e4)
+    ax.set_ylim(0, 200)
+    ax.set_xlabel("Per-request KV footprint (MB)")
+    ax.set_ylabel("Per-D-instance KV pool occupancy (%)")
+    ax.set_title(
+        "Decode-side KV concentration explains the PD-sep memory wall  "
+        f"(8x H20, KV pool ≈ {KV_POOL_PER_GPU_GB} GB/GPU, {CONCURRENT_DECODE} concurrent decodes)",
+        fontsize=10,
+    )
+    ax.grid(True, alpha=0.25, which="both")
+    ax.legend(loc="upper left", fontsize=9, framealpha=0.95)
+    fig.tight_layout()
+    fig.savefig(out_path, bbox_inches="tight")
+    plt.close(fig)
+    print(f"wrote {out_path}")
+    print(f"  KV/token: {KV_BYTES_PER_TOKEN/1024:.1f} KB  "
+          f"pool/GPU: {KV_POOL_PER_GPU_GB} GB  "
+          f"concurrent decodes: {CONCURRENT_DECODE}")
+    print()
+    print(f"  per-D occupancy at each operating point:")
+    print(f"  {'workload':28s}  {'KV/req':>10s}  "
+          f"{'Combined':>10s}  {'4P+4D':>10s}  {'6P+2D':>10s}")
+    for name, kv in POINTS:
+        c8 = occupancy(kv, 8) * 100
+        c4 = occupancy(kv, 4) * 100
+        c2 = occupancy(kv, 2) * 100
+        print(f"  {name:28s}  {kv:>7.0f} MB  "
+              f"{c8:>9.1f}%  {c4:>9.1f}%  {c2:>9.1f}%")
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--outdir", default="analysis/pd_sep_paper_section/figures")
+    args = ap.parse_args()
+    out = Path(args.outdir)
+    out.mkdir(parents=True, exist_ok=True)
+    plot(out / "fig_kv_memory_wall.pdf")
+
+
+if __name__ == "__main__":
+    main()

analysis/pd_sep_paper_section/scripts/plot_workload.py

@@ -160,29 +160,39 @@ def plot_reuse(rows, out_path):
     d = reuse_decomposition(rows)
     total = sum(d.values())
     parts = [
-        ("intra-session reuse",          d["intra_session_reuse_tokens"], "#2ca02c"),
-        ("cross-session reuse",          d["cross_session_reuse_tokens"], "#1f77b4"),
+        ("intra-session reuse",           d["intra_session_reuse_tokens"], "#2ca02c"),
+        ("cross-session reuse",           d["cross_session_reuse_tokens"], "#1f77b4"),
         ("first emission (reused later)", d["first_emission_will_reuse_tokens"], "#ff7f0e"),
         ("unique (never reused)",        d["unique_no_reuse_tokens"], "#d62728"),
     ]
 
-    fig, ax = plt.subplots(figsize=(8.5, 1.9))
+    fig, ax = plt.subplots(figsize=(9.0, 2.2))
     left = 0
+    handles = []
     for label, val, color in parts:
         frac = val / total
-        ax.barh(0, frac, left=left, color=color, edgecolor="white", height=0.6, label=label)
-        if frac > 0.025:
-            ax.text(left + frac / 2, 0,
-                    f"{label}\n{frac*100:.1f}%",
-                    ha="center", va="center", fontsize=8.5, color="white")
+        b = ax.barh(0, frac, left=left, color=color, edgecolor="white",
+                    height=0.55, label=f"{label}  ({frac*100:.1f}%)")
+        handles.append(b)
+        if frac > 0.04:
+            ax.text(left + frac / 2, 0, f"{frac*100:.1f}%",
+                    ha="center", va="center", fontsize=10,
+                    color="white", fontweight="bold")
         left += frac
 
     ax.set_xlim(0, 1)
+    ax.set_ylim(-0.6, 0.6)
     ax.set_yticks([])
-    ax.set_xlabel("share of total cacheable tokens (block-aligned, 512 tok blocks)")
-    ax.set_title("Where do prefix cache hits come from?  "
-                 f"(N={len(rows)} requests, sampled trace)")
-    ax.legend(loc="upper center", bbox_to_anchor=(0.5, -0.45), ncol=4, fontsize=8, frameon=False)
+    ax.set_xticks([0, 0.25, 0.5, 0.75, 1.0])
+    ax.set_xticklabels(["0%", "25%", "50%", "75%", "100%"])
+    ax.set_title(
+        f"Where do prefix cache hits come from?  (N={len(rows)} requests; "
+        "block-aligned 512-tok blocks)",
+        pad=8,
+    )
+    ax.legend(loc="upper center", bbox_to_anchor=(0.5, -0.20),
+              ncol=2, fontsize=9, frameon=False, handlelength=1.5,
+              columnspacing=2.5)
     for spine in ("top", "right", "left"):
         ax.spines[spine].set_visible(False)
     fig.tight_layout()