diff --git a/docs/V2_DEEP_ANALYSIS_ZH.md b/docs/V2_DEEP_ANALYSIS_ZH.md index 2381d8e..4b40768 100644 --- a/docs/V2_DEEP_ANALYSIS_ZH.md +++ b/docs/V2_DEEP_ANALYSIS_ZH.md @@ -312,26 +312,49 @@ delta (KVC vs DP) -0.8% -1.4% ← KVC 优势略放大 Critic 的 framing： > KVC 之所以赢，是因为它把 cache 集中到 3 个 D（每个 ~43M token），DP fragment 到 4 个 worker（每个 ~30M token）。两边 policy 都是 `kv-aware`，差异来自架构而非策略。 -**反驳**：KVC 整套机制的**核心设计就是主动选择 affinity 集中而非 fragment**。"差异来自架构"等价于"差异来自 KVC 是 KVC"——这正是要论证的设计点。 -- DP 的 hash 路由理论上能命中 prefix cache，但**单个 session 的 cache 散到 4 个 worker** = 命中率打 1/4 折扣 -- KVC 的 session affinity = 整段 KV 永远在同一个 D = 跨 turn 100% 命中 -- 同 `kv-aware` policy 在两种拓扑上的天花板根本不同——这是 KVC 的设计胜利，不是 measurement confound +**反驳**：KVC 整套机制的**核心设计就是主动选择 affinity 集中而非 fragment**。"差异来自架构"等价于"差异来自 KVC 是 KVC"——这正是要论证的设计点。更重要的：**KVC 的总 KV pool 实际上比 DP 少 27%**（KVC 3×92K=276K vs DP 4×87K=351K tokens），但 cache 命中率仍然更高（98.1% vs 96.8%）。 -**论文应当把这条作为 contribution 写出来，不是作为 caveat。** +![Cache efficiency paradox: KVC 用更少的总池子缓存更多](figures/cache_efficiency.png) + +**左图 — 命中率随 turn 的演化**揭示了 cache 效率不是"总池子大小"决定的，是"留什么"的策略决定的： +- KVC 的 session affinity → cache 在被钉定的 D 上**随 turn 累积**，hit rate 单调上升 +- DP 的 hash 路由 + radix LRU → 跨 session 共享 87K pool，hit rate 在 turn 8-25 区间（KVC 97.0% vs DP 95.8%，差 **1.24pp**）出现"中段 drift" +- 后期两边都稳定在 ~98-99%（session 长时间没换，cache 反复命中），但 DP 的 IQR band 更宽 → 不同请求 / 不同 session 之间命中波动更大 + +**右图 — uncached tokens 的 ECDF** 量化了 per-request 影响： +- KVC 50% 请求 uncached ≤ **187 tokens**，DP 50% 请求 uncached ≤ **781 tokens**（4× 差距） +- 在 uncached = 500 tokens 阈值上：**KVC 74% 请求落在该阈值以下，DP 只有 31%** +- KVC 的曲线 "撞墙" 在 ~200 token 处快速爬到 0.5；DP 的曲线在 100-10K 区间均匀展开 + +→ 论文里这是 **contribution**，不是 caveat：KVC 的 mechanism 让 27% 更少的总池子产生了更高的 retention 效率。 ### 4.5 [辩驳 critic] "Prefill GPU 90%+ 闲置" 是设计意图，不是浪费 Critic 的 framing： > KVC 1P3D 中 prefill GPU 只在 8.3% 请求时被激活；实际工作 GPU 只有 ~3.08 个，对比 4DP CA 的 4 个 fused GPU 不公平。 -**反驳**：在线 coding agent workload 下，**P 应该闲着**——P 一旦忙意味着 cache miss 太多。 -- P 的角色是 **reseed safety net + 初次 seed**，不是常态负载 -- "GPU 利用率高 = 好"在 throughput 视角对，**在 latency 视角错**——闲 GPU = burst 响应能力 = 用户体验更好 -- 生产部署可以给 P 用低规格 GPU（如 A100 vs D 用 H100），cost 上摊得开 +**反驳**：按"请求计数"看 P 确实稀疏，但按"实际工作量"看 P 的负载和每个 D 相当——P 是**低频高 cost 的 safety net**，不是 idle 容量。 -历史尝试：KVC 4D0P（取消 P 角色，所有 GPU 都做 P+D）已经实验过——整体性能下降，因为 prefill 与 decode 争 GPU 资源时 decode latency 抖动放大。 +![Per-GPU utilization: 请求计数视图 vs 工作量视图](figures/gpu_utilization.png) -**论文应当把这条作为 architectural rationale 写出来：KVC 用 P 闲置换 TTFT 稳定性。** +**左图 — 请求计数视图**：KVC P GPU 仅处理 328 个请求（7.4%），而 KVC D 各处理 ~1450 个（33%），DP 各处理 ~1100 个（25%）。**乍看像 critic 说的"P 闲着"**。 + +**右图 — 工作量视图（compute tokens）**： +- KVC P GPU：**1.07M tokens 的 prefill 工作**（仅 prefill，无 decode） +- KVC D GPU 每个：~0.80M tokens（小量 append-prefill + 全部 decode） +- DP 每个 worker：~1.30M tokens（全套 prefill + decode） + +→ **KVC P GPU 的 per-GPU 工作量与每个 KVC D GPU 相当**——只是分布在少数（328）个高强度请求上（每个 reseed 5K-90K tokens）。它不是空转，是 **low-frequency, high-cost safety net**。 + +**总工作量对比**： +- KVC 4 个 GPU 合计 ~3.47M tokens 工作 +- DP 4 个 GPU 合计 ~5.17M tokens 工作（**KVC 减少 33% compute**——这是 session affinity 带来的 cache 复用收益） + +这两点综合：KVC 用 **同样 4 个 GPU、更少总 KV pool、更少总 compute**，做到了 latency / TTFT mean/p50/p90 全胜。 + +**论文应当把这条作为 architectural rationale 写出来：KVC 用 P 的低频专用化换 D 端的 TTFT 稳定性。** + +历史尝试佐证：KVC 4D0P（取消 P 角色，所有 GPU 都做 P+D）已经实验过——整体性能下降，因为 prefill 与 decode 争 GPU 资源时 decode latency 抖动放大。 ### 4.6 v2 N=1 + 新代码路径未验证确定性 — **MINOR，方法学待办** diff --git a/docs/figures/cache_efficiency.png b/docs/figures/cache_efficiency.png new file mode 100644 index 0000000..1179786 Binary files /dev/null and b/docs/figures/cache_efficiency.png differ diff --git a/docs/figures/gpu_utilization.png b/docs/figures/gpu_utilization.png new file mode 100644 index 0000000..bde2d88 Binary files /dev/null and b/docs/figures/gpu_utilization.png differ diff --git a/scripts/analysis/plot_cache_efficiency.py b/scripts/analysis/plot_cache_efficiency.py new file mode 100644 index 0000000..56e074b --- /dev/null +++ b/scripts/analysis/plot_cache_efficiency.py @@ -0,0 +1,209 @@ +#!/usr/bin/env python3 +"""Cache efficiency comparison: KVC 1P3D v2 vs 4-way DP CA. + +Generates docs/figures/cache_efficiency.png — two-panel: + left: cache hit rate vs turn number (mechanism: affinity vs LRU) + right: ECDF of per-request uncached tokens (per-request impact) + +Resolves the apparent paradox: KVC has 27% less total KV pool capacity +(3 × 92K = 276K vs DP 4 × 87K = 351K) yet achieves higher cache hit rate +(98.1% vs 96.8%) and lower mean uncached tokens per request (560 vs 952). + +The left panel shows the mechanism: KVC's session affinity makes cache hit +rate grow with turn count (more cache accumulates on the pinned D), while +DP's hash + radix-LRU causes cache hit rate to decay through the middle +turns (other sessions' KV competes via LRU eviction). + +The right panel quantifies the impact: KVC's uncached tokens are +concentrated near 0 (mean 560), DP's are spread (mean 952). + +Aborted / errored requests are excluded. +""" + +from __future__ import annotations + +import json +from collections import defaultdict +from pathlib import Path + +import matplotlib.pyplot as plt +import numpy as np + +ROOT = Path(__file__).resolve().parents[2] +KVC = ROOT / "outputs/qwen3-30b-tp1-ts1-migration-v2/kvc_1p3d_migration_v2_run1_metrics.jsonl" +DP = ROOT / "outputs/qwen3-30b-tp1-ts1-validation/dp4_metrics.jsonl" +OUT = ROOT / "docs/figures/cache_efficiency.png" + + +def load(p: Path) -> list[dict]: + return [json.loads(line) for line in p.open()] + + +def is_failed(r: dict) -> bool: + if r.get("error"): + return True + fr = r.get("finish_reason") + if fr and ("abort" in str(fr).lower() or "badrequest" in str(fr).lower()): + return True + return False + + +def main() -> None: + kvc = [r for r in load(KVC) if not is_failed(r)] + dp = [r for r in load(DP) if not is_failed(r)] + + KVC_COLOR = "#1F77B4" + DP_COLOR = "#D62728" + + fig, axes = plt.subplots(1, 2, figsize=(15, 6.5)) + + # ------------------------------------------------------------------ + # Left panel: cache hit rate per turn + # Bin requests by turn_id, plot mean hit rate per bin with shaded band + # ------------------------------------------------------------------ + def bin_by_turn(rows: list[dict]) -> tuple[list[int], list[float], list[float], list[float]]: + per_turn: defaultdict[int, list[float]] = defaultdict(list) + for r in rows: + if r["input_length"] == 0: + continue + hit = r.get("cached_tokens", 0) / r["input_length"] + per_turn[r["turn_id"]].append(hit) + turns = sorted(per_turn.keys()) + means, p25s, p75s = [], [], [] + for t in turns: + arr = np.array(per_turn[t]) + means.append(float(np.mean(arr))) + p25s.append(float(np.quantile(arr, 0.25))) + p75s.append(float(np.quantile(arr, 0.75))) + return turns, means, p25s, p75s + + kvc_t, kvc_m, kvc_lo, kvc_hi = bin_by_turn(kvc) + dp_t, dp_m, dp_lo, dp_hi = bin_by_turn(dp) + + # Cap x-axis: tails get noisy below ~5 samples per bin + max_turn = 100 + + ax = axes[0] + ax.plot(kvc_t, kvc_m, color=KVC_COLOR, lw=2.5, + label=f"KVC 1P3D v2 (overall hit 98.1%)") + ax.fill_between(kvc_t, kvc_lo, kvc_hi, color=KVC_COLOR, alpha=0.18, + label="KVC IQR (p25-p75)") + ax.plot(dp_t, dp_m, color=DP_COLOR, lw=2.5, + label=f"4-way DP CA (overall hit 96.8%)") + ax.fill_between(dp_t, dp_lo, dp_hi, color=DP_COLOR, alpha=0.18, + label="DP IQR (p25-p75)") + + # Annotate the mid-turn drift gap + drift_turns = list(range(8, 25)) + drift_kvc = np.mean([m for t, m in zip(kvc_t, kvc_m) if t in drift_turns]) + drift_dp = np.mean([m for t, m in zip(dp_t, dp_m) if t in drift_turns]) + ax.axvspan(8, 25, color="#999", alpha=0.08, label="_nolegend_") + ax.text(16, 0.65, + f"Mid-turn region\n(turns 8-25):\nKVC {drift_kvc*100:.1f}% | DP {drift_dp*100:.1f}%\nGap {(drift_kvc-drift_dp)*100:+.1f} pp", + ha="center", va="center", fontsize=9.5, + bbox=dict(facecolor="white", edgecolor="gray", alpha=0.92, pad=4)) + + ax.set_xlim(1, max_turn) + ax.set_ylim(0.4, 1.02) + ax.set_xlabel("Turn number within session", fontsize=11) + ax.set_ylabel("Per-request cache hit rate (cached / input_length)", fontsize=11) + ax.set_title("Cache hit rate vs turn number\n(mechanism: session affinity vs hash-LRU)", + fontsize=12, pad=10) + ax.legend(loc="lower right", fontsize=9.5, framealpha=0.95) + ax.grid(True, linestyle=":", alpha=0.4) + ax.set_axisbelow(True) + + # ------------------------------------------------------------------ + # Right panel: ECDF of per-request uncached tokens (log x) + # ------------------------------------------------------------------ + def ecdf(rows: list[dict]) -> tuple[np.ndarray, np.ndarray]: + vals = np.array([ + max(1, r["input_length"] - r.get("cached_tokens", 0)) + for r in rows + ]) + vals = np.sort(vals) + return vals, np.arange(1, len(vals) + 1) / len(vals) + + kvc_x, kvc_y = ecdf(kvc) + dp_x, dp_y = ecdf(dp) + + ax = axes[1] + ax.plot(kvc_x, kvc_y, color=KVC_COLOR, lw=2.5, + label=f"KVC 1P3D v2 (mean {int(np.mean(kvc_x))} tokens)") + ax.plot(dp_x, dp_y, color=DP_COLOR, lw=2.5, + label=f"4-way DP CA (mean {int(np.mean(dp_x))} tokens)") + + # Median markers + kvc_p50 = np.quantile(kvc_x, 0.50) + dp_p50 = np.quantile(dp_x, 0.50) + ax.axhline(0.5, color="gray", linestyle=":", alpha=0.5) + ax.text(1.2, 0.52, "median (50% of requests below this)", + fontsize=8.5, color="gray", style="italic") + ax.axvline(kvc_p50, color=KVC_COLOR, ls="--", alpha=0.5, lw=1.0) + ax.axvline(dp_p50, color=DP_COLOR, ls="--", alpha=0.5, lw=1.0) + ax.text(kvc_p50, 0.06, f"KVC\nmedian\n{int(kvc_p50)}", + color=KVC_COLOR, fontsize=9, ha="center", va="bottom", + bbox=dict(facecolor="white", edgecolor="none", alpha=0.75, pad=1)) + ax.text(dp_p50, 0.06, f"DP\nmedian\n{int(dp_p50)}", + color=DP_COLOR, fontsize=9, ha="center", va="bottom", + bbox=dict(facecolor="white", edgecolor="none", alpha=0.75, pad=1)) + + # Annotate the separation: at uncached = 500 tokens, what fraction below? + sep_x = 500 + kvc_at_sep = (kvc_x <= sep_x).mean() + dp_at_sep = (dp_x <= sep_x).mean() + ax.axvline(sep_x, color="#666", linestyle=":", alpha=0.6, lw=1.0) + ax.annotate( + f"At uncached = {sep_x} tokens:\n" + f"KVC {kvc_at_sep*100:.0f}% of requests below\n" + f"DP {dp_at_sep*100:.0f}% of requests below", + xy=(sep_x, dp_at_sep), + xytext=(2500, 0.35), + fontsize=9.5, + bbox=dict(facecolor="white", edgecolor="gray", alpha=0.92, pad=4), + arrowprops=dict(arrowstyle="->", color="#666", lw=0.8), + ) + + ax.set_xscale("log") + ax.set_xlim(1, 1e5) + ax.set_xticks([1, 10, 100, 1000, 10000, 100000]) + ax.set_xticklabels(["1", "10", "100", "1K", "10K", "100K"]) + ax.set_ylim(0, 1.02) + ax.set_xlabel("Uncached tokens per request (log scale)", fontsize=11) + ax.set_ylabel("Cumulative fraction of requests", fontsize=11) + ax.set_title("ECDF of uncached tokens per request\n(impact: KVC concentrates near zero)", + fontsize=12, pad=10) + ax.legend(loc="lower right", fontsize=10, framealpha=0.95) + ax.grid(True, which="both", linestyle=":", alpha=0.4) + ax.set_axisbelow(True) + + fig.suptitle( + "Cache efficiency paradox: KVC has 27% LESS total KV pool (276K vs 351K tokens) yet caches MORE per request.\n" + "Left: session-affinity lets KVC's cache accumulate with turns; DP's hash-LRU loses cache to cross-session competition.\n" + "Right: net effect — KVC's uncached compute is concentrated near zero, DP's is spread over 100-10K tokens.", + fontsize=11.5, y=1.05, + ) + plt.tight_layout() + plt.savefig(OUT, dpi=150, bbox_inches="tight") + print(f"wrote {OUT}") + plt.close(fig) + + # ------------------------------------------------------------------ + # Print summary for doc reference + # ------------------------------------------------------------------ + print("\n=== Cache efficiency stats ===") + print(f"KVC v2: total_input={sum(r['input_length'] for r in kvc)/1e6:.1f}M tokens") + print(f" total_cached={sum(r.get('cached_tokens',0) for r in kvc)/1e6:.1f}M tokens") + print(f" hit rate {sum(r.get('cached_tokens',0) for r in kvc)/sum(r['input_length'] for r in kvc)*100:.2f}%") + print(f" mean uncached {np.mean(kvc_x):.0f} p50 {kvc_p50:.0f} p90 {np.quantile(kvc_x, 0.9):.0f}") + + print(f"\nDP 4w: total_input={sum(r['input_length'] for r in dp)/1e6:.1f}M tokens") + print(f" total_cached={sum(r.get('cached_tokens',0) for r in dp)/1e6:.1f}M tokens") + print(f" hit rate {sum(r.get('cached_tokens',0) for r in dp)/sum(r['input_length'] for r in dp)*100:.2f}%") + print(f" mean uncached {np.mean(dp_x):.0f} p50 {dp_p50:.0f} p90 {np.quantile(dp_x, 0.9):.0f}") + + print(f"\nMid-turn region (8-25): KVC {drift_kvc*100:.2f}% DP {drift_dp*100:.2f}% (gap {(drift_kvc-drift_dp)*100:+.2f}pp)") + + +if __name__ == "__main__": + main() diff --git a/scripts/analysis/plot_gpu_utilization.py b/scripts/analysis/plot_gpu_utilization.py new file mode 100644 index 0000000..7dc81d9 --- /dev/null +++ b/scripts/analysis/plot_gpu_utilization.py @@ -0,0 +1,234 @@ +#!/usr/bin/env python3 +"""Per-GPU utilization breakdown: KVC 1P3D v2 vs 4-way DP CA. + +Generates docs/figures/gpu_utilization.png — two-panel: + left: per-GPU request count + right: per-GPU compute work (uncached prefill tokens + decode tokens, stacked) + +The point of the figure is to push back on the naïve reading +"KVC's prefill GPU is idle 90% of the time, so KVC is using fewer GPUs." + +By request count, the prefill GPU is indeed touched by only ~8% of requests. +By compute work, the prefill GPU bears comparable per-GPU load to each +decode GPU — it is a low-frequency, high-cost safety net for cache misses, +not idle capacity. + +Work attribution: + KVC direct-to-D path: prefill happens locally on the assigned D worker + (append-prefill of `uncached_tokens` tokens). + KVC seed/reseed/fallback path: prefill happens on prefill-0 + (full uncached_tokens), decode on assigned D. + DP: all work on assigned direct-N worker. + +Aborted / errored requests are excluded. +""" + +from __future__ import annotations + +import json +from collections import defaultdict +from pathlib import Path + +import matplotlib.pyplot as plt +import numpy as np + +ROOT = Path(__file__).resolve().parents[2] +KVC = ROOT / "outputs/qwen3-30b-tp1-ts1-migration-v2/kvc_1p3d_migration_v2_run1_metrics.jsonl" +DP = ROOT / "outputs/qwen3-30b-tp1-ts1-validation/dp4_metrics.jsonl" +OUT = ROOT / "docs/figures/gpu_utilization.png" + + +def load(p: Path) -> list[dict]: + return [json.loads(line) for line in p.open()] + + +def is_failed(r: dict) -> bool: + if r.get("error"): + return True + fr = r.get("finish_reason") + if fr and ("abort" in str(fr).lower() or "badrequest" in str(fr).lower()): + return True + return False + + +def uncached(r: dict) -> int: + return max(0, r["input_length"] - r.get("cached_tokens", 0)) + + +def out_tokens(r: dict) -> int: + return r.get("actual_output_tokens") or r.get("output_length") or 0 + + +def main() -> None: + kvc = [r for r in load(KVC) if not is_failed(r)] + dp = [r for r in load(DP) if not is_failed(r)] + + # ------------------------------------------------------------------ + # KVC per-GPU attribution + # ------------------------------------------------------------------ + kvc_req_count = defaultdict(int) + kvc_prefill_tokens = defaultdict(int) # uncached prefill compute + kvc_decode_tokens = defaultdict(int) + + for r in kvc: + d = r["assigned_decode_node"] # decode-0/1/2 + p = r["assigned_prefill_node"] # prefill-0 + mode = r.get("execution_mode", "") + if mode == "kvcache-direct-to-d-session": + # P is bypassed entirely; D does the append-prefill + decode + kvc_req_count[d] += 1 + kvc_prefill_tokens[d] += uncached(r) + kvc_decode_tokens[d] += out_tokens(r) + else: + # P does the full prefill; D handles decode + kvc_req_count[p] += 1 + kvc_req_count[d] += 1 # decode side still counts + kvc_prefill_tokens[p] += uncached(r) + kvc_decode_tokens[d] += out_tokens(r) + + # ------------------------------------------------------------------ + # DP per-GPU attribution (fused P+D on every worker) + # ------------------------------------------------------------------ + dp_req_count = defaultdict(int) + dp_prefill_tokens = defaultdict(int) + dp_decode_tokens = defaultdict(int) + + for r in dp: + w = r["assigned_decode_node"] # direct-0..3 + dp_req_count[w] += 1 + dp_prefill_tokens[w] += uncached(r) + dp_decode_tokens[w] += out_tokens(r) + + # ------------------------------------------------------------------ + # Build ordered GPU list, KVC then DP + # ------------------------------------------------------------------ + kvc_gpus = ["prefill-0", "decode-0", "decode-1", "decode-2"] + dp_gpus = ["direct-0", "direct-1", "direct-2", "direct-3"] + all_gpus = kvc_gpus + dp_gpus + + def get(d, k): + return d.get(k, 0) + + counts = [get(kvc_req_count, g) for g in kvc_gpus] + \ + [get(dp_req_count, g) for g in dp_gpus] + prefill_tk = [get(kvc_prefill_tokens, g) for g in kvc_gpus] + \ + [get(dp_prefill_tokens, g) for g in dp_gpus] + decode_tk = [get(kvc_decode_tokens, g) for g in kvc_gpus] + \ + [get(dp_decode_tokens, g) for g in dp_gpus] + + # Display labels: P/D role + worker id + labels = [ + "KVC P\nprefill-0", + "KVC D\ndecode-0", + "KVC D\ndecode-1", + "KVC D\ndecode-2", + "DP P+D\ndirect-0", + "DP P+D\ndirect-1", + "DP P+D\ndirect-2", + "DP P+D\ndirect-3", + ] + kvc_mask = [True, True, True, True, False, False, False, False] + + KVC_P_COLOR = "#E89D44" # orange — P GPU stands out + KVC_D_COLOR = "#1F77B4" # blue + DP_COLOR = "#D62728" # red + + bar_colors = [KVC_P_COLOR, KVC_D_COLOR, KVC_D_COLOR, KVC_D_COLOR, + DP_COLOR, DP_COLOR, DP_COLOR, DP_COLOR] + + fig, axes = plt.subplots(1, 2, figsize=(15, 6.5)) + x = np.arange(len(all_gpus)) + + # -- Left: per-GPU request count ---------------------------------- + ax = axes[0] + bars = ax.bar(x, counts, color=bar_colors, edgecolor="black", linewidth=0.6) + for xi, c in zip(x, counts): + ax.text(xi, c + max(counts) * 0.015, f"{c:,}", + ha="center", va="bottom", fontsize=9.5) + ax.set_xticks(x) + ax.set_xticklabels(labels, fontsize=9.5) + ax.set_ylabel("Number of requests touching this GPU", fontsize=11) + ax.set_title("Per-GPU request count\n(naïve view: P seems idle)", fontsize=12, pad=10) + ax.grid(axis="y", linestyle=":", alpha=0.4) + ax.set_axisbelow(True) + + # Annotate: KVC P GPU is "low frequency" + p_idx = 0 + p_pct = counts[p_idx] / sum(counts[:4]) * 100 # vs KVC total + ax.annotate( + f"P GPU only sees\n" + f"{counts[p_idx]:,} requests\n" + f"({counts[p_idx]/len(kvc)*100:.1f}% of total)", + xy=(p_idx, counts[p_idx]), + xytext=(p_idx + 0.6, max(counts) * 0.55), + fontsize=9, color=KVC_P_COLOR, fontweight="bold", + arrowprops=dict(arrowstyle="->", color=KVC_P_COLOR, lw=1.0), + ) + + # -- Right: per-GPU compute work (stacked prefill + decode) ------- + ax = axes[1] + prefill_M = [t / 1e6 for t in prefill_tk] + decode_M = [t / 1e6 for t in decode_tk] + total_M = [p + d for p, d in zip(prefill_M, decode_M)] + + bars_p = ax.bar(x, prefill_M, color=[c for c in bar_colors], + edgecolor="black", linewidth=0.6, label="Uncached prefill tokens", + alpha=0.95) + bars_d = ax.bar(x, decode_M, bottom=prefill_M, color=[c for c in bar_colors], + edgecolor="black", linewidth=0.6, hatch="///", + label="Decode tokens", alpha=0.55) + + for xi, t in zip(x, total_M): + ax.text(xi, t + max(total_M) * 0.015, f"{t:.2f}M", + ha="center", va="bottom", fontsize=9.5) + + ax.set_xticks(x) + ax.set_xticklabels(labels, fontsize=9.5) + ax.set_ylabel("Compute tokens (millions)", fontsize=11) + ax.set_title("Per-GPU compute work\n(work view: P is comparable to each D)", + fontsize=12, pad=10) + ax.grid(axis="y", linestyle=":", alpha=0.4) + ax.set_axisbelow(True) + ax.legend(loc="upper left", fontsize=10, framealpha=0.95) + + # Annotate: KVC P GPU does similar work to each D + ax.annotate( + f"P GPU does {total_M[p_idx]:.2f}M tokens of\n" + f"prefill — comparable per-GPU\n" + f"load to each KVC D worker", + xy=(p_idx, total_M[p_idx]), + xytext=(p_idx + 0.6, max(total_M) * 0.62), + fontsize=9, color=KVC_P_COLOR, fontweight="bold", + arrowprops=dict(arrowstyle="->", color=KVC_P_COLOR, lw=1.0), + ) + + # Separator + group labels + for ax in axes: + ax.axvline(3.5, color="gray", linestyle="--", linewidth=1.0, alpha=0.5) + ymin, ymax = ax.get_ylim() + ax.text(1.5, ymax * 1.05, "KVC 1P3D", ha="center", fontsize=11, + fontweight="bold", color="#444") + ax.text(5.5, ymax * 1.05, "DP 4-way CA", ha="center", fontsize=11, + fontweight="bold", color="#444") + + fig.suptitle( + "Per-GPU utilization: \"is KVC's prefill GPU wasted?\"\n" + "Left view says yes (only 8% of requests); right view says no (comparable work to each D).", + fontsize=13, y=1.02, + ) + plt.tight_layout() + plt.savefig(OUT, dpi=150, bbox_inches="tight") + print(f"wrote {OUT}") + plt.close(fig) + + # ------------------------------------------------------------------ + # Print numbers for doc reference + # ------------------------------------------------------------------ + print("\n=== Per-GPU numbers ===") + print(f"{'GPU':<22} {'requests':>10} {'prefill(M)':>12} {'decode(M)':>12} {'total(M)':>10}") + for lbl, n, pM, dM in zip(labels, counts, prefill_M, decode_M): + print(f" {lbl.replace(chr(10), ' '):<20} {n:>10} {pM:>12.3f} {dM:>12.3f} {pM+dM:>10.3f}") + + +if __name__ == "__main__": + main()