gahow/agentic-kvc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Working-set figure: express footprint in node count, not GB

Browse Source 
Both axes now in "# nodes" (footprint / per-node KV pool) so the
cluster-size implication is direct: 1-node budget line + 14-node oracle
ceiling, instead of raw GB.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
Gahow Wang
2026-05-28 16:16:00 +08:00
parent dae98c6472
commit 3b8be5bb61
2 changed files with 27 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files

BIN
figs/working_set/glm5_fp8_tp8_b300.png
View File

Binary file not shown.
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 134 KiB

After

Width:  |  Height:  |  Size: 150 KiB

53
scripts/working_set_analysis.py
View File

@@ -154,48 +154,49 @@ def plot(ws, hw, block_bytes, label, out_path):
import matplotlib.pyplot as plt
bgb = block_bytes / GB
taus = [r["tau"] for r in ws["taus"]]
peak_gb = np.array([r["peak_blocks"] * bgb for r in ws["taus"]])
pool = hw["kv_pool_gb"] # KV pool per node (= per replica)
gpr = hw["gpus_per_replica"]
node_lbl = f"1 node = {gpr}x {hw['gpu']} = {pool:.0f} GB KV"
# everything in node units: nodes = footprint_GB / pool
peak_nodes = np.array([r["peak_blocks"] * bgb / pool for r in ws["taus"]])
apc = np.array([r["apc"] * 100 for r in ws["taus"]])
oracle_gb = ws["oracle_peak_blocks"] * bgb
oracle_nodes = ws["oracle_peak_blocks"] * bgb / pool
ceil = ws["apc_ceiling"] * 100
pool = hw["kv_pool_gb"] # per replica
fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(15, 6))
# --- panel 1: APC vs required KV footprint ---
ax1.plot(peak_gb, apc, "o-", color="#1f77b4", lw=2, ms=7, label="TTL-LRU W(T)")
for r, x, y in zip(ws["taus"], peak_gb, apc):
# --- panel 1: APC vs nodes of HBM needed ---
ax1.plot(peak_nodes, apc, "o-", color="#1f77b4", lw=2, ms=7, label="TTL-LRU W(T)")
for r, x, y in zip(ws["taus"], peak_nodes, apc):
ax1.annotate(f"{r['tau']:g}s", (x, y), fontsize=8,
textcoords="offset points", xytext=(4, 5))
ax1.scatter([oracle_gb], [ceil], marker="*", s=320, color="#d62728", zorder=5,
label=f"oracle / ceiling ({ceil:.1f}%)")
ax1.scatter([oracle_nodes], [ceil], marker="*", s=320, color="#d62728", zorder=5,
label=f"oracle / ceiling ({ceil:.1f}% @ {oracle_nodes:.0f} nodes)")
ax1.axhline(ceil, ls=":", color="#d62728", alpha=.5)
for k in (1, 2, 4, 8):
x = pool * k
ax1.axvline(x, ls="--", color="#2ca02c", alpha=.55)
ax1.text(x, 2, f"{k} replica\n{k*hw['gpus_per_replica']} GPU",
for k in (1, 2, 4, 8, 16, 32):
ax1.axvline(k, ls="--", color="#2ca02c", alpha=.45)
ax1.text(k, 2, f"{k} node / {k*gpr} GPU",
rotation=90, va="bottom", ha="right", fontsize=8, color="#2ca02c")
ax1.axvspan(0.1, 1, color="#2ca02c", alpha=.06) # "fits in 1 node" region
ax1.set_xscale("log")
ax1.set_xlabel("KV footprint that must be resident (GB, log)")
ax1.set_xlabel(f"# nodes of GPU HBM that must hold the KV ({node_lbl})")
ax1.set_ylabel("Achievable prefix-cache hit rate (APC %)")
ax1.set_title("APC vs KV-pool budget")
ax1.set_title("APC vs cluster size (nodes)")
ax1.grid(alpha=.3, which="both"); ax1.legend(loc="lower right"); ax1.set_ylim(0, 100)
# --- panel 2: footprint over time for a few T ---
span = ws["span"]; grid = np.linspace(0, span, 400)
# recompute series for a representative subset from stored peaks is not enough;
# show peak/p50 bars instead (compact, robust)
# --- panel 2: nodes needed by retention window ---
sel = [r for r in ws["taus"] if r["tau"] in (2, 30, 300, 600)]
xs = np.arange(len(sel)); w = 0.38
ax2.bar(xs - w/2, [r["peak_blocks"]*bgb for r in sel], w, label="peak", color="#1f77b4")
ax2.bar(xs + w/2, [r["p50_blocks"]*bgb for r in sel], w, label="median", color="#aec7e8")
ax2.axhline(pool, ls="--", color="#2ca02c", lw=2, label=f"1 replica KV pool ({pool:.0f} GB)")
ax2.axhline(oracle_gb, ls=":", color="#d62728", lw=2, label=f"oracle full-ceiling ({oracle_gb:.0f} GB)")
ax2.bar(xs - w/2, [r["peak_blocks"]*bgb/pool for r in sel], w, label="peak", color="#1f77b4")
ax2.bar(xs + w/2, [r["p50_blocks"]*bgb/pool for r in sel], w, label="median", color="#aec7e8")
ax2.axhline(1, ls="--", color="#2ca02c", lw=2, label="your budget: 1 node")
ax2.axhline(oracle_nodes, ls=":", color="#d62728", lw=2,
label=f"oracle full-ceiling ({oracle_nodes:.0f} nodes)")
ax2.set_xticks(xs); ax2.set_xticklabels([f"T={r['tau']:g}s\nAPC={r['apc']*100:.0f}%" for r in sel])
ax2.set_ylabel("KV footprint (GB)")
ax2.set_ylabel("# nodes of HBM needed")
ax2.set_yscale("log")
ax2.set_title("Footprint by retention window vs pool")
ax2.set_title("Cluster size by retention window")
ax2.grid(alpha=.3, axis="y", which="both"); ax2.legend(loc="upper left", fontsize=9)
fig.suptitle(label, fontsize=13, fontweight="bold")
@@ -230,7 +231,7 @@ def main():
gpus_per_replica = a.tp * a.pp
total_hbm = gpus_per_replica * GPU_HBM_GB[a.gpu]
kv_pool_gb = total_hbm - a.weight_gb - a.activation_gb
hw = {"gpus_per_replica": gpus_per_replica, "kv_pool_gb": kv_pool_gb}
hw = {"gpus_per_replica": gpus_per_replica, "kv_pool_gb": kv_pool_gb, "gpu": a.gpu}
taus = [1, 2, 5, 10, 30, 60, 300, 600, 1800]
n, ids, ts = load_trace(a.trace, a.min_ts, a.max_ts)