diff --git a/figs/working_set/glm5_fp8_tp8_b300.png b/figs/working_set/glm5_fp8_tp8_b300.png index 3e15512..dc2ca76 100644 Binary files a/figs/working_set/glm5_fp8_tp8_b300.png and b/figs/working_set/glm5_fp8_tp8_b300.png differ diff --git a/scripts/working_set_analysis.py b/scripts/working_set_analysis.py index a738ee3..5cc43ad 100644 --- a/scripts/working_set_analysis.py +++ b/scripts/working_set_analysis.py @@ -154,48 +154,49 @@ def plot(ws, hw, block_bytes, label, out_path): import matplotlib.pyplot as plt bgb = block_bytes / GB - taus = [r["tau"] for r in ws["taus"]] - peak_gb = np.array([r["peak_blocks"] * bgb for r in ws["taus"]]) + pool = hw["kv_pool_gb"] # KV pool per node (= per replica) + gpr = hw["gpus_per_replica"] + node_lbl = f"1 node = {gpr}x {hw['gpu']} = {pool:.0f} GB KV" + + # everything in node units: nodes = footprint_GB / pool + peak_nodes = np.array([r["peak_blocks"] * bgb / pool for r in ws["taus"]]) apc = np.array([r["apc"] * 100 for r in ws["taus"]]) - oracle_gb = ws["oracle_peak_blocks"] * bgb + oracle_nodes = ws["oracle_peak_blocks"] * bgb / pool ceil = ws["apc_ceiling"] * 100 - pool = hw["kv_pool_gb"] # per replica fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(15, 6)) - # --- panel 1: APC vs required KV footprint --- - ax1.plot(peak_gb, apc, "o-", color="#1f77b4", lw=2, ms=7, label="TTL-LRU W(T)") - for r, x, y in zip(ws["taus"], peak_gb, apc): + # --- panel 1: APC vs nodes of HBM needed --- + ax1.plot(peak_nodes, apc, "o-", color="#1f77b4", lw=2, ms=7, label="TTL-LRU W(T)") + for r, x, y in zip(ws["taus"], peak_nodes, apc): ax1.annotate(f"{r['tau']:g}s", (x, y), fontsize=8, textcoords="offset points", xytext=(4, 5)) - ax1.scatter([oracle_gb], [ceil], marker="*", s=320, color="#d62728", zorder=5, - label=f"oracle / ceiling ({ceil:.1f}%)") + ax1.scatter([oracle_nodes], [ceil], marker="*", s=320, color="#d62728", zorder=5, + label=f"oracle / ceiling ({ceil:.1f}% @ {oracle_nodes:.0f} nodes)") ax1.axhline(ceil, ls=":", color="#d62728", alpha=.5) - for k in (1, 2, 4, 8): - x = pool * k - ax1.axvline(x, ls="--", color="#2ca02c", alpha=.55) - ax1.text(x, 2, f"{k} replica\n{k*hw['gpus_per_replica']} GPU", + for k in (1, 2, 4, 8, 16, 32): + ax1.axvline(k, ls="--", color="#2ca02c", alpha=.45) + ax1.text(k, 2, f"{k} node / {k*gpr} GPU", rotation=90, va="bottom", ha="right", fontsize=8, color="#2ca02c") + ax1.axvspan(0.1, 1, color="#2ca02c", alpha=.06) # "fits in 1 node" region ax1.set_xscale("log") - ax1.set_xlabel("KV footprint that must be resident (GB, log)") + ax1.set_xlabel(f"# nodes of GPU HBM that must hold the KV ({node_lbl})") ax1.set_ylabel("Achievable prefix-cache hit rate (APC %)") - ax1.set_title("APC vs KV-pool budget") + ax1.set_title("APC vs cluster size (nodes)") ax1.grid(alpha=.3, which="both"); ax1.legend(loc="lower right"); ax1.set_ylim(0, 100) - # --- panel 2: footprint over time for a few T --- - span = ws["span"]; grid = np.linspace(0, span, 400) - # recompute series for a representative subset from stored peaks is not enough; - # show peak/p50 bars instead (compact, robust) + # --- panel 2: nodes needed by retention window --- sel = [r for r in ws["taus"] if r["tau"] in (2, 30, 300, 600)] xs = np.arange(len(sel)); w = 0.38 - ax2.bar(xs - w/2, [r["peak_blocks"]*bgb for r in sel], w, label="peak", color="#1f77b4") - ax2.bar(xs + w/2, [r["p50_blocks"]*bgb for r in sel], w, label="median", color="#aec7e8") - ax2.axhline(pool, ls="--", color="#2ca02c", lw=2, label=f"1 replica KV pool ({pool:.0f} GB)") - ax2.axhline(oracle_gb, ls=":", color="#d62728", lw=2, label=f"oracle full-ceiling ({oracle_gb:.0f} GB)") + ax2.bar(xs - w/2, [r["peak_blocks"]*bgb/pool for r in sel], w, label="peak", color="#1f77b4") + ax2.bar(xs + w/2, [r["p50_blocks"]*bgb/pool for r in sel], w, label="median", color="#aec7e8") + ax2.axhline(1, ls="--", color="#2ca02c", lw=2, label="your budget: 1 node") + ax2.axhline(oracle_nodes, ls=":", color="#d62728", lw=2, + label=f"oracle full-ceiling ({oracle_nodes:.0f} nodes)") ax2.set_xticks(xs); ax2.set_xticklabels([f"T={r['tau']:g}s\nAPC={r['apc']*100:.0f}%" for r in sel]) - ax2.set_ylabel("KV footprint (GB)") + ax2.set_ylabel("# nodes of HBM needed") ax2.set_yscale("log") - ax2.set_title("Footprint by retention window vs pool") + ax2.set_title("Cluster size by retention window") ax2.grid(alpha=.3, axis="y", which="both"); ax2.legend(loc="upper left", fontsize=9) fig.suptitle(label, fontsize=13, fontweight="bold") @@ -230,7 +231,7 @@ def main(): gpus_per_replica = a.tp * a.pp total_hbm = gpus_per_replica * GPU_HBM_GB[a.gpu] kv_pool_gb = total_hbm - a.weight_gb - a.activation_gb - hw = {"gpus_per_replica": gpus_per_replica, "kv_pool_gb": kv_pool_gb} + hw = {"gpus_per_replica": gpus_per_replica, "kv_pool_gb": kv_pool_gb, "gpu": a.gpu} taus = [1, 2, 5, 10, 30, 60, 300, 600, 1800] n, ids, ts = load_trace(a.trace, a.min_ts, a.max_ts)