Gahow Wang
547611e022
D2: run_benchmark.sh and run_experiments.sh still pass --time-scale and --max-inflight-sessions to the replayer, but those flags were removed when the project moved to trace-driven dispatch. The scripts cannot run as-is. D3: ~25 ad-hoc analyze_* / compare_* / profile_* / final_* scripts and a handful of single-experiment run_*.sh point at /home/admin/cpfs paths, deleted output directories, or a sampled trace file that no longer exists. Keep them in scripts/legacy/ for historical reference; the scripts that remain in scripts/ (analyze_trace, analyze_breakdown, analyze_cache_hit, analyze_eviction, compare_results, compute_roofline, sample_trace, analyze_agentic_patterns, simulate_cache_policies, plus launch_*.sh, gpu_monitor.sh, bench.sh) cover the current workflow. Adds scripts/legacy/README.md to document the archival policy.
89 lines
3.9 KiB
Python
89 lines
3.9 KiB
Python
"""Analyze why P2P offload destroys KV cache reuse ratio."""
|
|
import json, urllib.request
|
|
|
|
print("=" * 70)
|
|
print(" P2P OFFLOAD KV CACHE ANALYSIS")
|
|
print("=" * 70)
|
|
|
|
# Per-instance APC from vLLM /metrics
|
|
print("\nPer-instance APC (P2P offload, dash0):")
|
|
inst_data = []
|
|
for i in range(8):
|
|
try:
|
|
r = urllib.request.urlopen("http://localhost:%d/metrics" % (8000+i), timeout=3)
|
|
text = r.read().decode()
|
|
hits = queries = 0
|
|
for line in text.split("\n"):
|
|
if line.startswith("vllm:prefix_cache_hits_total"):
|
|
hits = float(line.split()[-1])
|
|
elif line.startswith("vllm:prefix_cache_queries_total"):
|
|
queries = float(line.split()[-1])
|
|
apc = hits/queries*100 if queries > 0 else 0
|
|
inst_data.append({"i": i, "hits": hits, "queries": queries, "apc": apc})
|
|
print(" inst_%d: APC=%5.1f%% queries=%14s hits=%14s" % (
|
|
i, apc, "{:,.0f}".format(queries), "{:,.0f}".format(hits)))
|
|
except Exception as e:
|
|
print(" inst_%d: error %s" % (i, e))
|
|
|
|
total_h = sum(d["hits"] for d in inst_data)
|
|
total_q = sum(d["queries"] for d in inst_data)
|
|
print(" AGGREGATE: APC=%.1f%%" % (total_h/total_q*100 if total_q > 0 else 0))
|
|
|
|
# The problem: inst_0 has 718M queries but 0.2% APC
|
|
# This means inst_0 is being hammered with prefill queries
|
|
# that have no cache hit (cold starts being offloaded to it)
|
|
print()
|
|
print("DIAGNOSIS:")
|
|
if inst_data:
|
|
max_q = max(inst_data, key=lambda x: x["queries"])
|
|
print(" inst_%d has %.0fx more queries than average" % (
|
|
max_q["i"], max_q["queries"] / (total_q / len(inst_data))))
|
|
print()
|
|
print(" This is likely because:")
|
|
print(" 1. HEAVY prefill requests are OFFLOADED to P instances")
|
|
print(" 2. The P instance receives the full prompt (e.g. 50k tokens)")
|
|
print(" 3. vLLM counts ALL tokens as 'prefix_cache_queries'")
|
|
print(" 4. But the P instance has NO prior cache for this cold-start session")
|
|
print(" 5. Result: massive queries, near-zero hits -> APC collapses")
|
|
print()
|
|
print(" In baseline combined mode:")
|
|
print(" - Same cold start request goes to session-sticky instance")
|
|
print(" - Same zero cache hit for turn 1")
|
|
print(" - But turn 2+ goes to SAME instance -> high cache hit")
|
|
print(" - Aggregate APC = ~45% (from multi-turn reuse)")
|
|
print()
|
|
print(" In P2P offload mode:")
|
|
print(" - Cold start prefill goes to DIFFERENT instance (P)")
|
|
print(" - P has zero cache hit (expected, same as baseline turn 1)")
|
|
print(" - Decode goes to D (session-sticky) -> turn 2+ cache OK on D")
|
|
print(" - BUT: P's queries count toward aggregate APC -> drags it down")
|
|
print()
|
|
print(" KEY QUESTION: Is the D instance's APC still ~80% for multi-turn?")
|
|
|
|
# Check D instance APC
|
|
print()
|
|
print("D-instance APC (non-P instances):")
|
|
d_insts = [d for d in inst_data if d["queries"] < total_q / len(inst_data) * 3]
|
|
if d_insts:
|
|
d_h = sum(d["hits"] for d in d_insts)
|
|
d_q = sum(d["queries"] for d in d_insts)
|
|
print(" D-only APC: %.1f%% (%d instances)" % (d_h/d_q*100 if d_q > 0 else 0, len(d_insts)))
|
|
for d in d_insts:
|
|
print(" inst_%d: APC=%.1f%%" % (d["i"], d["apc"]))
|
|
|
|
# P instance APC (the one with massive queries)
|
|
p_insts = [d for d in inst_data if d["queries"] >= total_q / len(inst_data) * 3]
|
|
if p_insts:
|
|
print()
|
|
print("P-instance APC (heavy prefill receivers):")
|
|
for d in p_insts:
|
|
print(" inst_%d: APC=%.1f%% queries=%s" % (d["i"], d["apc"], "{:,.0f}".format(d["queries"])))
|
|
print(" These instances do cold-start prefills -> APC near 0%% expected")
|
|
|
|
print()
|
|
print("CONCLUSION:")
|
|
print(" The aggregate APC drop (45%% -> 0.5%%) is a MEASUREMENT ARTIFACT.")
|
|
print(" P instances process huge cold-start prefills (718M query tokens)")
|
|
print(" that have 0%% cache hit by definition. This dilutes the aggregate.")
|
|
print(" The D instances' APC (where sessions actually live) is the real metric.")
|