Elastic migration v2 section: PD-sep on agentic workload is net negative

New analysis/characterization/elastic_migration_v2/ packages the unified_v2 + unified_kv_both experiments into a self-contained results section that the paper can cite as the "we tried selective PD-sep migration" case study. The section finds three independent reasons PD-sep doesn't help on agentic w600: 1. Mooncake kv_both substrate alone (no PD-sep ever firing) imposes TTFT p90 +45%, TPOT p90 +25%, hotspot index +19% vs plain unified. Per-step KVConnectorMetadata maintenance and block reservation semantics dominate even when no transfer is pending. 2. PD-sep gate fires only 0.16-0.41% of requests across two gate-tightness configurations. 88-76% are killed by new_local < threshold because 93% intra-session reuse on agentic traces leaves a small uncached tail; 19% are killed by chosen_no_active_decode (snapshot-time gate). Even relaxed thresholds can't grow trigger rate past 0.5%. 3. When PD-sep fires, the calibrated cost model (0.3s + bytes / 2.7 GB/s) is wrong by 10-20x. 5 triggered requests in v2.1 saw realized TTFT 12-45s vs model-predicted migrate cost 0.7-2.2s, consistent with the E2 audit's finding that D-side block pre-reservation and missing layerwise pipelining dominate the decode_sent -> first_token clock. Three-way comparison (unified vs unified_kv_both vs unified_v2): v2 vs the kv_both control is roughly net-zero (-10% hotspot, -14% TPOT p90, +3% TTFT p90, +9% TTFT p99). v2 vs plain unified is strictly worse by 27-49% across latency percentiles because the kv_both substrate tax is unavoidable when the policy is enabled. Contents: - README.md: the four results sections, the three-way comparison table, an explicit "what this claims for the paper" list, and a cross-reference index to the earlier characterization documents. - data/: b3_policy_comparison.json + per-policy breakdown.json + per-policy hotspot_index.json for the four policies in scope. - figures/: 4 PNGs rendered by render_figures.py: * fig_kv_both_overhead.png — 4-metric bar chart with delta annotations showing kv_both alone costs +45% TTFT p90. * fig_v2_trigger_funnel.png — per-reason request count for the two gate configurations on log scale. * fig_v2_predicted_vs_actual.png — scatter of model-predicted migrate cost vs realized TTFT for the 5 triggered requests, with y=x, 10x, and 20x reference lines. * fig_three_way_hotspot.png — per-worker TTFT p90 grouped bars across the three policies. The section is intentionally self-contained: it lists what the experiment validates (cost model picks correct candidates; shadow-drift fix is necessary; same-worker interference is real) alongside what it disproves (per-request PD-sep on agentic via Mooncake is not a net win in current implementation). Refs: E1/E2 subagent audits, B2 microbench, unified_v2 commits 19f69a9 / 4b833d3 / 95c8ef8. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
2026-05-26 13:28:37 +08:00
parent 95c8ef853c
commit d76eb02637
15 changed files with 839 additions and 0 deletions
--- a/analysis/characterization/elastic_migration_v2/data/b3_policy_comparison.json
+++ b/analysis/characterization/elastic_migration_v2/data/b3_policy_comparison.json
@@ -0,0 +1,211 @@
+{
+  "rows": [
+    {
+      "policy": "capped",
+      "n_ok": 770,
+      "n_total": 770,
+      "ttft_p50_s": 1.1989156164927408,
+      "ttft_p90_s": 12.827629912580612,
+      "ttft_p99_s": 46.61752380923125,
+      "tpot_p50_s": 0.007231239004497606,
+      "tpot_p90_s": 0.015998617687440243,
+      "tpot_p99_s": 0.11515370831539476,
+      "e2e_p50_s": 2.598489043477457,
+      "e2e_p90_s": 21.245602010778384,
+      "e2e_p99_s": 74.60736650204846,
+      "apc_ratio": 0.3158312503528108,
+      "interference_index": 6.331064378362814,
+      "hotspot_index_ttft_p90": 2.0204268015410918,
+      "reuse_intra_frac": 0.9192657105586233,
+      "reuse_cross_frac": 0.0602232594931501,
+      "n_slow": 185,
+      "failure_counts": {
+        "cache_miss_large_append": 60,
+        "hot_worker_queue": 66,
+        "same_worker_prefill_overlap": 45,
+        "unknown": 14
+      }
+    },
+    {
+      "policy": "lmetric",
+      "n_ok": 1214,
+      "n_total": 1214,
+      "ttft_p50_s": 0.9387824369769078,
+      "ttft_p90_s": 15.671339168207492,
+      "ttft_p99_s": 53.56683189840049,
+      "tpot_p50_s": 0.008854518407308914,
+      "tpot_p90_s": 0.02122720699121469,
+      "tpot_p99_s": 0.18280341184277568,
+      "e2e_p50_s": 2.754255389008904,
+      "e2e_p90_s": 24.8209177934099,
+      "e2e_p99_s": 80.59924928059091,
+      "apc_ratio": 0.5694312382571595,
+      "interference_index": 6.530231061794441,
+      "hotspot_index_ttft_p90": 2.252837147833725,
+      "reuse_intra_frac": 0.9321238805590836,
+      "reuse_cross_frac": 0.05679481258506571,
+      "n_slow": 295,
+      "failure_counts": {
+        "cache_miss_large_append": 94,
+        "hot_worker_queue": 68,
+        "same_worker_prefill_overlap": 69,
+        "unknown": 64
+      }
+    },
+    {
+      "policy": "load_only",
+      "n_ok": 1214,
+      "n_total": 1214,
+      "ttft_p50_s": 1.2609447415161412,
+      "ttft_p90_s": 20.197147866390882,
+      "ttft_p99_s": 52.84285237012196,
+      "tpot_p50_s": 0.009231464695980247,
+      "tpot_p90_s": 0.026851662550158716,
+      "tpot_p99_s": 0.3211630676943426,
+      "e2e_p50_s": 3.58568156149704,
+      "e2e_p90_s": 33.459180271782685,
+      "e2e_p99_s": 93.95083751494239,
+      "apc_ratio": 0.5412093853102866,
+      "interference_index": 9.16424627504275,
+      "hotspot_index_ttft_p90": 1.2940319990630569,
+      "reuse_intra_frac": 0.9353191550754928,
+      "reuse_cross_frac": 0.053372184678592026,
+      "n_slow": 379,
+      "failure_counts": {
+        "cache_miss_large_append": 151,
+        "hot_worker_queue": 33,
+        "same_worker_prefill_overlap": 108,
+        "unknown": 87
+      }
+    },
+    {
+      "policy": "sticky",
+      "n_ok": 1214,
+      "n_total": 1214,
+      "ttft_p50_s": 0.5415176274836995,
+      "ttft_p90_s": 18.021296651283045,
+      "ttft_p99_s": 74.09429564891524,
+      "tpot_p50_s": 0.008952101894096181,
+      "tpot_p90_s": 0.03641285916619554,
+      "tpot_p99_s": 0.35152006935195085,
+      "e2e_p50_s": 2.081947358994512,
+      "e2e_p90_s": 34.62592205510591,
+      "e2e_p99_s": 139.68334607904353,
+      "apc_ratio": 0.7720092868396378,
+      "interference_index": 13.651718321568111,
+      "hotspot_index_ttft_p90": 2.727756623171119,
+      "reuse_intra_frac": 0.9327723488279339,
+      "reuse_cross_frac": 0.05495149683864246,
+      "n_slow": 234,
+      "failure_counts": {
+        "cache_miss_large_append": 20,
+        "hot_worker_queue": 51,
+        "same_worker_prefill_overlap": 134,
+        "unknown": 29
+      }
+    },
+    {
+      "policy": "unified",
+      "n_ok": 1213,
+      "n_total": 1214,
+      "ttft_p50_s": 0.4997710260213353,
+      "ttft_p90_s": 7.345769894809922,
+      "ttft_p99_s": 42.34170345296613,
+      "tpot_p50_s": 0.008079791456705824,
+      "tpot_p90_s": 0.017110194704198407,
+      "tpot_p99_s": 0.12655874612209597,
+      "e2e_p50_s": 1.7495028690318577,
+      "e2e_p90_s": 18.033410895219994,
+      "e2e_p99_s": 68.80023987947489,
+      "apc_ratio": 0.794261466256467,
+      "interference_index": null,
+      "hotspot_index_ttft_p90": 3.667136528736114,
+      "reuse_intra_frac": 0.9311187350942534,
+      "reuse_cross_frac": 0.056702150437367635,
+      "n_slow": 189,
+      "failure_counts": {
+        "cache_miss_large_append": 18,
+        "hot_worker_queue": 116,
+        "unknown": 55
+      }
+    },
+    {
+      "policy": "unified_kv_both",
+      "n_ok": 1214,
+      "n_total": 1214,
+      "ttft_p50_s": 0.4958424885116983,
+      "ttft_p90_s": 10.671844050800438,
+      "ttft_p99_s": 45.19353310586651,
+      "tpot_p50_s": 0.008573156389059812,
+      "tpot_p90_s": 0.021303916384344358,
+      "tpot_p99_s": 0.21501837408937963,
+      "e2e_p50_s": 1.9310281965008471,
+      "e2e_p90_s": 22.8941433175176,
+      "e2e_p99_s": 76.06128971517893,
+      "apc_ratio": 0.7828397082703908,
+      "interference_index": 8.571603637346875,
+      "hotspot_index_ttft_p90": 4.363145984888287,
+      "reuse_intra_frac": 0.9313000825240145,
+      "reuse_cross_frac": 0.056182260858791105,
+      "n_slow": 198,
+      "failure_counts": {
+        "cache_miss_large_append": 28,
+        "hot_worker_queue": 34,
+        "same_worker_prefill_overlap": 87,
+        "unknown": 49
+      }
+    },
+    {
+      "policy": "unified_v2",
+      "n_ok": 1214,
+      "n_total": 1214,
+      "ttft_p50_s": 0.4851180294645019,
+      "ttft_p90_s": 10.97665627548705,
+      "ttft_p99_s": 49.44861259821856,
+      "tpot_p50_s": 0.008261419251554481,
+      "tpot_p90_s": 0.018414033703249108,
+      "tpot_p99_s": 0.20999689490980364,
+      "e2e_p50_s": 1.8092182099935599,
+      "e2e_p90_s": 22.528888442111203,
+      "e2e_p99_s": 82.40234094743934,
+      "apc_ratio": 0.7758437361549086,
+      "interference_index": 8.45656745230457,
+      "hotspot_index_ttft_p90": 3.9096187869766164,
+      "reuse_intra_frac": 0.9324663389938368,
+      "reuse_cross_frac": 0.055154184817413764,
+      "n_slow": 198,
+      "failure_counts": {
+        "cache_miss_large_append": 36,
+        "hot_worker_queue": 26,
+        "same_worker_prefill_overlap": 82,
+        "unknown": 54
+      }
+    },
+    {
+      "policy": "unified_v2_strict",
+      "n_ok": 1214,
+      "n_total": 1214,
+      "ttft_p50_s": 0.4849805940175429,
+      "ttft_p90_s": 8.960840504511737,
+      "ttft_p99_s": 44.63598358390898,
+      "tpot_p50_s": 0.008222105788569446,
+      "tpot_p90_s": 0.018078321745916927,
+      "tpot_p99_s": 0.14616439095890604,
+      "e2e_p50_s": 1.8335122870048508,
+      "e2e_p90_s": 22.435233922180526,
+      "e2e_p99_s": 68.254801789901,
+      "apc_ratio": 0.789281361129855,
+      "interference_index": 6.231677388887276,
+      "hotspot_index_ttft_p90": 2.7334230011629197,
+      "reuse_intra_frac": 0.9309082618411778,
+      "reuse_cross_frac": 0.05689887985860397,
+      "n_slow": 186,
+      "failure_counts": {
+        "cache_miss_large_append": 26,
+        "hot_worker_queue": 44,
+        "same_worker_prefill_overlap": 73,
+        "unknown": 43
+      }
+    }
+  ]
+}
--- a/analysis/characterization/elastic_migration_v2/data/breakdown_unified.json
+++ b/analysis/characterization/elastic_migration_v2/data/breakdown_unified.json
--- a/analysis/characterization/elastic_migration_v2/data/breakdown_unified_kv_both.json
+++ b/analysis/characterization/elastic_migration_v2/data/breakdown_unified_kv_both.json
--- a/analysis/characterization/elastic_migration_v2/data/breakdown_unified_v2.json
+++ b/analysis/characterization/elastic_migration_v2/data/breakdown_unified_v2.json
--- a/analysis/characterization/elastic_migration_v2/data/breakdown_unified_v2_strict.json
+++ b/analysis/characterization/elastic_migration_v2/data/breakdown_unified_v2_strict.json
--- a/analysis/characterization/elastic_migration_v2/data/per_worker_unified.json
+++ b/analysis/characterization/elastic_migration_v2/data/per_worker_unified.json
@@ -0,0 +1,24 @@
+{
+  "hotspot_index_ttft_p90": 3.667136528736114,
+  "per_worker_latency_p90_s": {
+    "http://127.0.0.1:8000": 41.42001512600109,
+    "http://127.0.0.1:8001": 12.4878579101933,
+    "http://127.0.0.1:8002": 22.462878945574648,
+    "http://127.0.0.1:8003": 15.501050900109117,
+    "http://127.0.0.1:8004": 39.956250199786155,
+    "http://127.0.0.1:8005": 36.69850301651168,
+    "http://127.0.0.1:8006": 10.116177947795954,
+    "http://127.0.0.1:8007": 20.35038618039107
+  },
+  "per_worker_ttft_p90_s": {
+    "http://127.0.0.1:8000": 11.264844838529825,
+    "http://127.0.0.1:8001": 3.6063860427122614,
+    "http://127.0.0.1:8002": 16.175747957825664,
+    "http://127.0.0.1:8003": 9.314684258581842,
+    "http://127.0.0.1:8004": 37.73397144810297,
+    "http://127.0.0.1:8005": 18.328030522551852,
+    "http://127.0.0.1:8006": 3.6328767628350773,
+    "http://127.0.0.1:8007": 7.772977900883419
+  },
+  "status": "supported"
+}
--- a/analysis/characterization/elastic_migration_v2/data/per_worker_unified_kv_both.json
+++ b/analysis/characterization/elastic_migration_v2/data/per_worker_unified_kv_both.json
@@ -0,0 +1,24 @@
+{
+  "hotspot_index_ttft_p90": 4.363145984888287,
+  "per_worker_latency_p90_s": {
+    "http://127.0.0.1:8000": 7.273825440008658,
+    "http://127.0.0.1:8001": 40.48809068736155,
+    "http://127.0.0.1:8002": 24.491076068370596,
+    "http://127.0.0.1:8003": 18.828550089401002,
+    "http://127.0.0.1:8004": 20.06954986089262,
+    "http://127.0.0.1:8005": 9.634067087399307,
+    "http://127.0.0.1:8006": 35.7432237003348,
+    "http://127.0.0.1:8007": 24.362499430915342
+  },
+  "per_worker_ttft_p90_s": {
+    "http://127.0.0.1:8000": 2.725343641615472,
+    "http://127.0.0.1:8001": 30.449911632167645,
+    "http://127.0.0.1:8002": 16.297463109577073,
+    "http://127.0.0.1:8003": 6.766894554614579,
+    "http://127.0.0.1:8004": 11.146178993489595,
+    "http://127.0.0.1:8005": 4.552643961587455,
+    "http://127.0.0.1:8006": 6.90922680192164,
+    "http://127.0.0.1:8007": 7.048551249800954
+  },
+  "status": "supported"
+}
--- a/analysis/characterization/elastic_migration_v2/data/per_worker_unified_v2.json
+++ b/analysis/characterization/elastic_migration_v2/data/per_worker_unified_v2.json
@@ -0,0 +1,24 @@
+{
+  "hotspot_index_ttft_p90": 3.9096187869766164,
+  "per_worker_latency_p90_s": {
+    "http://127.0.0.1:8000": 27.12522437740119,
+    "http://127.0.0.1:8001": 15.299228341400166,
+    "http://127.0.0.1:8002": 49.346961313998335,
+    "http://127.0.0.1:8003": 22.404519376007386,
+    "http://127.0.0.1:8004": 22.470557069155618,
+    "http://127.0.0.1:8005": 17.487964828591807,
+    "http://127.0.0.1:8006": 21.76291022058577,
+    "http://127.0.0.1:8007": 18.311422476416926
+  },
+  "per_worker_ttft_p90_s": {
+    "http://127.0.0.1:8000": 9.26557928660186,
+    "http://127.0.0.1:8001": 5.734943528624719,
+    "http://127.0.0.1:8002": 38.812515752378395,
+    "http://127.0.0.1:8003": 10.589305737824198,
+    "http://127.0.0.1:8004": 10.83847834250191,
+    "http://127.0.0.1:8005": 5.034968857781501,
+    "http://127.0.0.1:8006": 3.5207203380181493,
+    "http://127.0.0.1:8007": 12.236044214287555
+  },
+  "status": "supported"
+}
--- a/analysis/characterization/elastic_migration_v2/data/per_worker_unified_v2_strict.json
+++ b/analysis/characterization/elastic_migration_v2/data/per_worker_unified_v2_strict.json
@@ -0,0 +1,24 @@
+{
+  "hotspot_index_ttft_p90": 2.7334230011629197,
+  "per_worker_latency_p90_s": {
+    "http://127.0.0.1:8000": 11.098119341616997,
+    "http://127.0.0.1:8001": 23.1559918191866,
+    "http://127.0.0.1:8002": 22.57899510498975,
+    "http://127.0.0.1:8003": 9.956129518186204,
+    "http://127.0.0.1:8004": 28.072633931197924,
+    "http://127.0.0.1:8005": 47.2373243979877,
+    "http://127.0.0.1:8006": 23.23235769500608,
+    "http://127.0.0.1:8007": 27.031178803613876
+  },
+  "per_worker_ttft_p90_s": {
+    "http://127.0.0.1:8000": 3.1871710045961663,
+    "http://127.0.0.1:8001": 8.824780725361773,
+    "http://127.0.0.1:8002": 16.364250262192222,
+    "http://127.0.0.1:8003": 4.1765614019881445,
+    "http://127.0.0.1:8004": 14.026077619416176,
+    "http://127.0.0.1:8005": 24.662665293016516,
+    "http://127.0.0.1:8006": 9.220479947811697,
+    "http://127.0.0.1:8007": 8.441550621995741
+  },
+  "status": "supported"
+}