diff --git a/scripts/analyze_p2p_cache.py b/scripts/analyze_p2p_cache.py new file mode 100644 index 0000000..2a2cc8f --- /dev/null +++ b/scripts/analyze_p2p_cache.py @@ -0,0 +1,88 @@ +"""Analyze why P2P offload destroys KV cache reuse ratio.""" +import json, urllib.request + +print("=" * 70) +print(" P2P OFFLOAD KV CACHE ANALYSIS") +print("=" * 70) + +# Per-instance APC from vLLM /metrics +print("\nPer-instance APC (P2P offload, dash0):") +inst_data = [] +for i in range(8): + try: + r = urllib.request.urlopen("http://localhost:%d/metrics" % (8000+i), timeout=3) + text = r.read().decode() + hits = queries = 0 + for line in text.split("\n"): + if line.startswith("vllm:prefix_cache_hits_total"): + hits = float(line.split()[-1]) + elif line.startswith("vllm:prefix_cache_queries_total"): + queries = float(line.split()[-1]) + apc = hits/queries*100 if queries > 0 else 0 + inst_data.append({"i": i, "hits": hits, "queries": queries, "apc": apc}) + print(" inst_%d: APC=%5.1f%% queries=%14s hits=%14s" % ( + i, apc, "{:,.0f}".format(queries), "{:,.0f}".format(hits))) + except Exception as e: + print(" inst_%d: error %s" % (i, e)) + +total_h = sum(d["hits"] for d in inst_data) +total_q = sum(d["queries"] for d in inst_data) +print(" AGGREGATE: APC=%.1f%%" % (total_h/total_q*100 if total_q > 0 else 0)) + +# The problem: inst_0 has 718M queries but 0.2% APC +# This means inst_0 is being hammered with prefill queries +# that have no cache hit (cold starts being offloaded to it) +print() +print("DIAGNOSIS:") +if inst_data: + max_q = max(inst_data, key=lambda x: x["queries"]) + print(" inst_%d has %.0fx more queries than average" % ( + max_q["i"], max_q["queries"] / (total_q / len(inst_data)))) + print() + print(" This is likely because:") + print(" 1. HEAVY prefill requests are OFFLOADED to P instances") + print(" 2. The P instance receives the full prompt (e.g. 50k tokens)") + print(" 3. vLLM counts ALL tokens as 'prefix_cache_queries'") + print(" 4. But the P instance has NO prior cache for this cold-start session") + print(" 5. Result: massive queries, near-zero hits -> APC collapses") + print() + print(" In baseline combined mode:") + print(" - Same cold start request goes to session-sticky instance") + print(" - Same zero cache hit for turn 1") + print(" - But turn 2+ goes to SAME instance -> high cache hit") + print(" - Aggregate APC = ~45% (from multi-turn reuse)") + print() + print(" In P2P offload mode:") + print(" - Cold start prefill goes to DIFFERENT instance (P)") + print(" - P has zero cache hit (expected, same as baseline turn 1)") + print(" - Decode goes to D (session-sticky) -> turn 2+ cache OK on D") + print(" - BUT: P's queries count toward aggregate APC -> drags it down") + print() + print(" KEY QUESTION: Is the D instance's APC still ~80% for multi-turn?") + +# Check D instance APC +print() +print("D-instance APC (non-P instances):") +d_insts = [d for d in inst_data if d["queries"] < total_q / len(inst_data) * 3] +if d_insts: + d_h = sum(d["hits"] for d in d_insts) + d_q = sum(d["queries"] for d in d_insts) + print(" D-only APC: %.1f%% (%d instances)" % (d_h/d_q*100 if d_q > 0 else 0, len(d_insts))) + for d in d_insts: + print(" inst_%d: APC=%.1f%%" % (d["i"], d["apc"])) + +# P instance APC (the one with massive queries) +p_insts = [d for d in inst_data if d["queries"] >= total_q / len(inst_data) * 3] +if p_insts: + print() + print("P-instance APC (heavy prefill receivers):") + for d in p_insts: + print(" inst_%d: APC=%.1f%% queries=%s" % (d["i"], d["apc"], "{:,.0f}".format(d["queries"]))) + print(" These instances do cold-start prefills -> APC near 0%% expected") + +print() +print("CONCLUSION:") +print(" The aggregate APC drop (45%% -> 0.5%%) is a MEASUREMENT ARTIFACT.") +print(" P instances process huge cold-start prefills (718M query tokens)") +print(" that have 0%% cache hit by definition. This dilutes the aggregate.") +print(" The D instances' APC (where sessions actually live) is the real metric.")