gahow/agentic-pd-hybrid
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

docs: v4 final results, error analysis, and updated journey

Browse Source 
Add v4 sweep results and post-mortem analysis showing:

- direct-to-D path: 54.3% (1P7D) / 58.0% (2P6D) of requests now use
  KVC cleanly. P50=0.5s and TTFT P50=0.043s; this path beats baseline
  8DP across the board (P50 -24%, TTFT P50 -54%, TTFT P90 -79%).

- Overall vs baseline (errors+truncated excluded):
  v4 2P6D P50=0.85s vs baseline 0.66s (28% slower).
  Reason is not errors -- 35% of requests still hit
  fallback-large-append-session-cap, where capacity-based
  cap = usable_tokens / target_tokens evaluates to 1-2 (not 16)
  for large agentic inputs.

- 9-10% errors on KVC variants are mooncake TCP transfer timeouts,
  not SGLang logic bugs. Prefill log shows
  "Failed to send kv chunk ... 32s timeout ... session not alive".
  Errors concentrate in turn>=31 (large inputs) after run >44.8%.

Track:
- docs/KVC_DEBUG_JOURNEY_V1_TO_V4.md: append v4 results table,
  per-mode breakdown, and error root cause.
- scripts/analysis/{analyze_v3,analyze_v4,analyze_errors,compare_no_error}.py
- outputs/qwen3-30b-tp1-v{3,4}*/exp*_summary.json (force-added,
  small JSON; metrics.jsonl excluded due to size).
- outputs/qwen3-30b-tp1-v{3,4}*/sweep_results.txt

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
kzlin
2026-04-28 23:34:01 +08:00
parent c9d350b372
commit 74194e660a
11 changed files with 1162 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

88
outputs/qwen3-30b-tp1-v3-kvaware/exp1_1p7d_kvc_kvaware_summary.json Normal file
View File

@@ -0,0 +1,88 @@
{
"actual_output_tokens_stats": {
"count": 4086.0,
"mean": 213.95105237395987,
"p50": 83.0,
"p90": 562.0,
"p99": 1346.0
},
"cache_hit_request_count": 3929,
"cached_tokens_stats": {
"count": 4449.0,
"mean": 22635.924702180266,
"p50": 20010.0,
"p90": 48002.0,
"p99": 65424.0
},
"decode_request_priorities": {},
"error_count": 363,
"execution_modes": {
"kvcache-centric": 363,
"kvcache-direct-to-d-session": 1716,
"pd-router-d-session-reseed": 23,
"pd-router-fallback-d-backpressure": 12,
"pd-router-fallback-large-append": 5,
"pd-router-fallback-large-append-seed-filter-early-turn": 51,
"pd-router-fallback-large-append-session-cap": 2148,
"pd-router-fallback-no-d-capacity": 7,
"pd-router-fallback-session-cap": 32,
"pd-router-large-append-reseed": 39,
"pd-router-large-append-reseed-after-eviction": 2,
"pd-router-turn1-d-backpressure": 1,
"pd-router-turn1-no-d-capacity": 3,
"pd-router-turn1-seed": 34,
"pd-router-turn1-session-cap": 13
},
"latency_stats_s": {
"count": 4086.0,
"mean": 4.8753733304192455,
"p50": 1.754677688702941,
"p90": 12.66968655679375,
"p99": 28.717210091650486
},
"mechanisms": {
"kvcache-centric": 4449
},
"per_decode_load": {
"decode-0": 616,
"decode-1": 658,
"decode-2": 674,
"decode-3": 582,
"decode-4": 656,
"decode-5": 662,
"decode-6": 601
},
"per_prefill_load": {
"prefill-0": 4449
},
"prefill_request_priorities": {
"-100": 98,
"100": 2272
},
"re_prefill_count": 0,
"request_count": 4449,
"reuse_expected_count": 4397,
"reuse_observed_count": 4397,
"router_url": "http://127.0.0.1:8000",
"session_reset_count": 0,
"session_reused_count": 1716,
"total_actual_kv_transfer_blocks": 62123,
"total_cached_tokens": 100707229,
"total_kv_transfer_blocks": 105235,
"tpot_stats_s": {
"count": 4086.0,
"mean": 0.005829451223571163,
"p50": 0.005684156496173296,
"p90": 0.007143743503740225,
"p99": 0.008634991403068266
},
"trace_path": "outputs/qwen3-30b-tp1-v3-kvaware/kvcache-centric-kv-aware-worker-admission-20260428T095141Z/sampled-trace.jsonl",
"truncated_request_count": 42,
"ttft_stats_s": {
"count": 4086.0,
"mean": 3.5955862397812597,
"p50": 0.36274072993546724,
"p90": 10.972254231572151,
"p99": 27.433656523004174
}
}

85
outputs/qwen3-30b-tp1-v3-kvaware/exp2_2p6d_kvc_kvaware_summary.json Normal file
View File

@@ -0,0 +1,85 @@
{
"actual_output_tokens_stats": {
"count": 4440.0,
"mean": 225.87972972972972,
"p50": 86.0,
"p90": 576.0,
"p99": 1347.0
},
"cache_hit_request_count": 4201,
"cached_tokens_stats": {
"count": 4449.0,
"mean": 24345.55787817487,
"p50": 21504.0,
"p90": 48792.0,
"p99": 69120.0
},
"decode_request_priorities": {},
"error_count": 9,
"execution_modes": {
"kvcache-centric": 9,
"kvcache-direct-to-d-session": 1358,
"pd-router-d-session-reseed": 12,
"pd-router-fallback-d-backpressure": 2,
"pd-router-fallback-large-append-seed-filter-early-turn": 52,
"pd-router-fallback-large-append-session-cap": 2902,
"pd-router-fallback-session-cap": 25,
"pd-router-large-append-reseed": 34,
"pd-router-large-append-reseed-after-eviction": 4,
"pd-router-turn1-d-backpressure": 1,
"pd-router-turn1-seed": 30,
"pd-router-turn1-session-cap": 20
},
"latency_stats_s": {
"count": 4440.0,
"mean": 3.582334662846558,
"p50": 1.517257746309042,
"p90": 9.225348330102861,
"p99": 18.70269925892353
},
"mechanisms": {
"kvcache-centric": 4449
},
"per_decode_load": {
"decode-0": 710,
"decode-1": 630,
"decode-2": 763,
"decode-3": 737,
"decode-4": 879,
"decode-5": 730
},
"per_prefill_load": {
"prefill-0": 2225,
"prefill-1": 2224
},
"prefill_request_priorities": {
"-100": 80,
"100": 3002
},
"re_prefill_count": 0,
"request_count": 4449,
"reuse_expected_count": 4397,
"reuse_observed_count": 4397,
"router_url": "http://127.0.0.1:8000",
"session_reset_count": 0,
"session_reused_count": 1358,
"total_actual_kv_transfer_blocks": 78979,
"total_cached_tokens": 108313387,
"total_kv_transfer_blocks": 105235,
"tpot_stats_s": {
"count": 4440.0,
"mean": 0.005882534704321737,
"p50": 0.005807478777200416,
"p90": 0.00712956755887717,
"p99": 0.008372141476720572
},
"trace_path": "outputs/qwen3-30b-tp1-v3-kvaware/kvcache-centric-kv-aware-worker-admission-20260428T104343Z/sampled-trace.jsonl",
"truncated_request_count": 42,
"ttft_stats_s": {
"count": 4440.0,
"mean": 2.2045287611873334,
"p50": 0.32809355948120356,
"p90": 6.947275545448065,
"p99": 16.705802395939827
}
}

189
outputs/qwen3-30b-tp1-v3-kvaware/sweep_results.txt Normal file
View File

@@ -0,0 +1,189 @@
[2026-04-28 17:51:41] Starting TP1 v3 sweep (KVC with kv-aware policy)
[2026-04-28 17:51:41] Model: /mnt/kzlin/workflow/pd-hybrid/simm-swe-bench/models/Qwen3-30B-A3B-Instruct-2507
[2026-04-28 17:51:41] Trace: outputs/qwen35-swebench-50sess.jsonl (4449 requests, 52 sessions)
[2026-04-28 17:51:41] Key change: --policy kv-aware for KVC (was --policy default in v2)
[2026-04-28 17:51:41]
[2026-04-28 17:51:41] === [EXP1] 1P7D KVC kv-aware ===
[2026-04-28 18:43:43] === exp1_1p7d_kvc_kvaware COMPLETED ===
[2026-04-28 18:43:43] Summary:
{
"actual_output_tokens_stats": {
"count": 4086.0,
"mean": 213.95105237395987,
"p50": 83.0,
"p90": 562.0,
"p99": 1346.0
},
"cache_hit_request_count": 3929,
"cached_tokens_stats": {
"count": 4449.0,
"mean": 22635.924702180266,
"p50": 20010.0,
"p90": 48002.0,
"p99": 65424.0
},
"decode_request_priorities": {},
"error_count": 363,
"execution_modes": {
"kvcache-centric": 363,
"kvcache-direct-to-d-session": 1716,
"pd-router-d-session-reseed": 23,
"pd-router-fallback-d-backpressure": 12,
"pd-router-fallback-large-append": 5,
"pd-router-fallback-large-append-seed-filter-early-turn": 51,
"pd-router-fallback-large-append-session-cap": 2148,
"pd-router-fallback-no-d-capacity": 7,
"pd-router-fallback-session-cap": 32,
"pd-router-large-append-reseed": 39,
"pd-router-large-append-reseed-after-eviction": 2,
"pd-router-turn1-d-backpressure": 1,
"pd-router-turn1-no-d-capacity": 3,
"pd-router-turn1-seed": 34,
"pd-router-turn1-session-cap": 13
},
"latency_stats_s": {
"count": 4086.0,
"mean": 4.8753733304192455,
"p50": 1.754677688702941,
"p90": 12.66968655679375,
"p99": 28.717210091650486
},
"mechanisms": {
"kvcache-centric": 4449
},
"per_decode_load": {
"decode-0": 616,
"decode-1": 658,
"decode-2": 674,
"decode-3": 582,
"decode-4": 656,
"decode-5": 662,
"decode-6": 601
},
"per_prefill_load": {
"prefill-0": 4449
},
"prefill_request_priorities": {
"-100": 98,
"100": 2272
},
"re_prefill_count": 0,
"request_count": 4449,
"reuse_expected_count": 4397,
"reuse_observed_count": 4397,
"router_url": "http://127.0.0.1:8000",
"session_reset_count": 0,
"session_reused_count": 1716,
"total_actual_kv_transfer_blocks": 62123,
"total_cached_tokens": 100707229,
"total_kv_transfer_blocks": 105235,
"tpot_stats_s": {
"count": 4086.0,
"mean": 0.005829451223571163,
"p50": 0.005684156496173296,
"p90": 0.007143743503740225,
"p99": 0.008634991403068266
},
"trace_path": "outputs/qwen3-30b-tp1-v3-kvaware/kvcache-centric-kv-aware-worker-admission-20260428T095141Z/sampled-trace.jsonl",
"truncated_request_count": 42,
"ttft_stats_s": {
"count": 4086.0,
"mean": 3.5955862397812597,
"p50": 0.36274072993546724,
"p90": 10.972254231572151,
"p99": 27.433656523004174
}
}
[2026-04-28 18:43:43] Saved to outputs/qwen3-30b-tp1-v3-kvaware/exp1_1p7d_kvc_kvaware_summary.json + exp1_1p7d_kvc_kvaware_metrics.jsonl
[2026-04-28 18:43:43]
[2026-04-28 18:43:43] === [EXP2] 2P6D KVC kv-aware ===
[2026-04-28 19:30:38] === exp2_2p6d_kvc_kvaware COMPLETED ===
[2026-04-28 19:30:38] Summary:
{
"actual_output_tokens_stats": {
"count": 4440.0,
"mean": 225.87972972972972,
"p50": 86.0,
"p90": 576.0,
"p99": 1347.0
},
"cache_hit_request_count": 4201,
"cached_tokens_stats": {
"count": 4449.0,
"mean": 24345.55787817487,
"p50": 21504.0,
"p90": 48792.0,
"p99": 69120.0
},
"decode_request_priorities": {},
"error_count": 9,
"execution_modes": {
"kvcache-centric": 9,
"kvcache-direct-to-d-session": 1358,
"pd-router-d-session-reseed": 12,
"pd-router-fallback-d-backpressure": 2,
"pd-router-fallback-large-append-seed-filter-early-turn": 52,
"pd-router-fallback-large-append-session-cap": 2902,
"pd-router-fallback-session-cap": 25,
"pd-router-large-append-reseed": 34,
"pd-router-large-append-reseed-after-eviction": 4,
"pd-router-turn1-d-backpressure": 1,
"pd-router-turn1-seed": 30,
"pd-router-turn1-session-cap": 20
},
"latency_stats_s": {
"count": 4440.0,
"mean": 3.582334662846558,
"p50": 1.517257746309042,
"p90": 9.225348330102861,
"p99": 18.70269925892353
},
"mechanisms": {
"kvcache-centric": 4449
},
"per_decode_load": {
"decode-0": 710,
"decode-1": 630,
"decode-2": 763,
"decode-3": 737,
"decode-4": 879,
"decode-5": 730
},
"per_prefill_load": {
"prefill-0": 2225,
"prefill-1": 2224
},
"prefill_request_priorities": {
"-100": 80,
"100": 3002
},
"re_prefill_count": 0,
"request_count": 4449,
"reuse_expected_count": 4397,
"reuse_observed_count": 4397,
"router_url": "http://127.0.0.1:8000",
"session_reset_count": 0,
"session_reused_count": 1358,
"total_actual_kv_transfer_blocks": 78979,
"total_cached_tokens": 108313387,
"total_kv_transfer_blocks": 105235,
"tpot_stats_s": {
"count": 4440.0,
"mean": 0.005882534704321737,
"p50": 0.005807478777200416,
"p90": 0.00712956755887717,
"p99": 0.008372141476720572
},
"trace_path": "outputs/qwen3-30b-tp1-v3-kvaware/kvcache-centric-kv-aware-worker-admission-20260428T104343Z/sampled-trace.jsonl",
"truncated_request_count": 42,
"ttft_stats_s": {
"count": 4440.0,
"mean": 2.2045287611873334,
"p50": 0.32809355948120356,
"p90": 6.947275545448065,
"p99": 16.705802395939827
}
}
[2026-04-28 19:30:38] Saved to outputs/qwen3-30b-tp1-v3-kvaware/exp2_2p6d_kvc_kvaware_summary.json + exp2_2p6d_kvc_kvaware_metrics.jsonl
[2026-04-28 19:30:38]
[2026-04-28 19:30:38] === ALL TP1 V3 SWEEP EXPERIMENTS DONE ===