diff --git a/docs/KVC_DEBUG_JOURNEY_V1_TO_V4.md b/docs/KVC_DEBUG_JOURNEY_V1_TO_V4.md index ff19e0c..9e76a81 100644 --- a/docs/KVC_DEBUG_JOURNEY_V1_TO_V4.md +++ b/docs/KVC_DEBUG_JOURNEY_V1_TO_V4.md @@ -174,11 +174,84 @@ def _decode_session_soft_cap(...) -> int: + return max(1, min(16, usable_capacity_tokens // target_tokens)) ``` -7 D × 16 = 112 个 slot,远超 52 个 session 需求。预期 session-cap fallback 占比降到 <10%,整体 P50 向 direct-to-D 的 0.46s 收敛。 +7 D × 16 = 112 个 slot,远超 52 个 session 需求。 -实际数据见 `outputs/qwen3-30b-tp1-v4-cap16/`。 +### v4 实际结果(vs v3 1P7D / 2P6D) -## 后续可以考虑的更深方案:让 D 自己决定 admission +| 指标 | v3 1P7D | **v4 1P7D** | v3 2P6D | **v4 2P6D** | baseline 8DP | +|------|:---:|:---:|:---:|:---:|:---:| +| Errors | 363 (8%) | 435 (10%) | 9 (0%) | **403 (9%)** | 0 | +| 截断 | 42 | 43 | 42 | 36 | 68 | +| **direct-to-D** | 38.6% | **54.3%** | 30.5% | **58.0%** ⭐ | - | +| **session-cap fallback** | 48.3% | 37.4% | 65.4% | **34.7%** | - | +| Session reused | 1716 | 2180 | 1358 | **2348** | - | +| KV transfer blocks | 62K | 53K | 79K | **51K** | - | +| Mean | 4.88s | 4.21s | 3.58s | **2.51s** | 1.43s | +| **P50** | 1.75s | 1.08s | 1.52s | **0.84s** | **0.65s** | +| P90 | 12.67s | 13.38s | 9.23s | **6.51s** | 3.61s | +| P99 | 28.72s | 24.45s | 18.70s | 18.34s | 8.38s | +| **TTFT P50** | 0.36s | 0.056s | 0.33s | **0.051s** ⭐ | 0.094s | +| TTFT P90 | 10.97s | 11.90s | 6.95s | **2.64s** | 0.26s | + +✓ direct-to-D 占比从 v3 的 30-38% 涨到 v4 的 54-58% +✓ session 复用 +27% (1P7D) / +73% (2P6D) +✓ KV transfer 量 -15% (1P7D) / -36% (2P6D) +✓ TTFT P50 反超 baseline 46%(0.051s vs 0.094s) + +### Direct-to-D 路径全面碾压 baseline(KVC 真实价值) + +| Config | n | Lat P50 | Lat P90 | TTFT P50 | TTFT P90 | +|--------|:---:|:---:|:---:|:---:|:---:| +| baseline 8DP | 4381 | 0.66s | 3.65s | 0.094s | 0.256s | +| v4 1P7D direct-to-D | 2179 | 0.495s | 3.03s | 0.044s | 0.055s | +| **v4 2P6D direct-to-D** | **2348** | **0.499s** | **2.86s** | **0.043s** | **0.054s** | + +direct-to-D 子集相对 baseline: +- P50 快 24-30% +- P90 快 16-22% +- TTFT P50 快 54% +- TTFT P90 快 79% + +### 整体性能(去掉 errors 和 truncated)vs baseline + +| Config | clean | Mean | P50 | P90 | P99 | +|--------|:---:|:---:|:---:|:---:|:---:| +| baseline 8DP | 4381 | 1.45s | 0.66s | 3.65s | 8.38s | +| v4 2P6D | 4010 | 2.53s | 0.85s | 6.55s | 18.33s | + +vs baseline:P50 慢 28%、P90 慢 80%、P99 慢 119%。即使错误率为 0,整体仍输 baseline——根因是 35% 请求被推到 fallback 路径。 + +### 新瓶颈 1:35% 请求仍走 session-cap fallback + +抬到 16 后真实瓶颈是 capacity-based 计算:`min(16, usable_capacity_tokens // target_tokens)`。 +- `target_tokens = input + output`,agentic 里常见 50-100K +- D 的 KV pool ≈ 100-150K tokens(80GB H100, mem_fraction=0.835) +- `usable / target` = 1-2,远没到 16 → 真实 cap 是 capacity 算出来的小数字 + +要解决必须改 capacity-based 估算逻辑(或上方案 D,让 D 自己决定)。 + +### 新瓶颈 2:9-10% errors(mooncake 传输超时) + +P-side log 显示: + +``` +KVTransferError: Failed to send kv chunk of to 10.45.7.165:40319 +Sync batch data transfer timeout after 32722558107ns (32 秒超时) +Decode instance could be dead, remote mooncake session ... is not alive +``` + +特征: +- 所有 errors 在 run 的 44.8% 之后出现(系统压力累积) +- 98% errors 集中在 turn ≥ 31(大 input 的请求) +- v3 cap=4 时 1P7D 已有 363 errors(仅 1 个 D 集中受冲击),v4 cap=16 把压力均匀分布但量级更大 + +是 mooncake TCP loopback 在并发上去后撞超时,**不是 SGLang 逻辑 bug**。修复方向: +1. 加长 mooncake transfer timeout(现在 32s) +2. 限制并发 inflight transfer 数量 +3. 改用 RDMA(loopback 是单机模拟,生产环境换真 RDMA) +4. chunked KV transfer + +## 后续可以考虑的更深方案:让 D 自己决定 admission(方案 D) v4 的硬 cap 抬高只是把数字调大,实际容量管理还是 replay 自己估算。代码里 `replay.py:_decode_session_soft_cap` 用 `target_tokens = input + output`(基于当前请求的 size)估算每个 session footprint,但: - agentic context 越攒越长,target_tokens 动态增长,cap 会随之缩小 diff --git a/outputs/qwen3-30b-tp1-v3-kvaware/exp1_1p7d_kvc_kvaware_summary.json b/outputs/qwen3-30b-tp1-v3-kvaware/exp1_1p7d_kvc_kvaware_summary.json new file mode 100644 index 0000000..cfdaf94 --- /dev/null +++ b/outputs/qwen3-30b-tp1-v3-kvaware/exp1_1p7d_kvc_kvaware_summary.json @@ -0,0 +1,88 @@ +{ + "actual_output_tokens_stats": { + "count": 4086.0, + "mean": 213.95105237395987, + "p50": 83.0, + "p90": 562.0, + "p99": 1346.0 + }, + "cache_hit_request_count": 3929, + "cached_tokens_stats": { + "count": 4449.0, + "mean": 22635.924702180266, + "p50": 20010.0, + "p90": 48002.0, + "p99": 65424.0 + }, + "decode_request_priorities": {}, + "error_count": 363, + "execution_modes": { + "kvcache-centric": 363, + "kvcache-direct-to-d-session": 1716, + "pd-router-d-session-reseed": 23, + "pd-router-fallback-d-backpressure": 12, + "pd-router-fallback-large-append": 5, + "pd-router-fallback-large-append-seed-filter-early-turn": 51, + "pd-router-fallback-large-append-session-cap": 2148, + "pd-router-fallback-no-d-capacity": 7, + "pd-router-fallback-session-cap": 32, + "pd-router-large-append-reseed": 39, + "pd-router-large-append-reseed-after-eviction": 2, + "pd-router-turn1-d-backpressure": 1, + "pd-router-turn1-no-d-capacity": 3, + "pd-router-turn1-seed": 34, + "pd-router-turn1-session-cap": 13 + }, + "latency_stats_s": { + "count": 4086.0, + "mean": 4.8753733304192455, + "p50": 1.754677688702941, + "p90": 12.66968655679375, + "p99": 28.717210091650486 + }, + "mechanisms": { + "kvcache-centric": 4449 + }, + "per_decode_load": { + "decode-0": 616, + "decode-1": 658, + "decode-2": 674, + "decode-3": 582, + "decode-4": 656, + "decode-5": 662, + "decode-6": 601 + }, + "per_prefill_load": { + "prefill-0": 4449 + }, + "prefill_request_priorities": { + "-100": 98, + "100": 2272 + }, + "re_prefill_count": 0, + "request_count": 4449, + "reuse_expected_count": 4397, + "reuse_observed_count": 4397, + "router_url": "http://127.0.0.1:8000", + "session_reset_count": 0, + "session_reused_count": 1716, + "total_actual_kv_transfer_blocks": 62123, + "total_cached_tokens": 100707229, + "total_kv_transfer_blocks": 105235, + "tpot_stats_s": { + "count": 4086.0, + "mean": 0.005829451223571163, + "p50": 0.005684156496173296, + "p90": 0.007143743503740225, + "p99": 0.008634991403068266 + }, + "trace_path": "outputs/qwen3-30b-tp1-v3-kvaware/kvcache-centric-kv-aware-worker-admission-20260428T095141Z/sampled-trace.jsonl", + "truncated_request_count": 42, + "ttft_stats_s": { + "count": 4086.0, + "mean": 3.5955862397812597, + "p50": 0.36274072993546724, + "p90": 10.972254231572151, + "p99": 27.433656523004174 + } +} \ No newline at end of file diff --git a/outputs/qwen3-30b-tp1-v3-kvaware/exp2_2p6d_kvc_kvaware_summary.json b/outputs/qwen3-30b-tp1-v3-kvaware/exp2_2p6d_kvc_kvaware_summary.json new file mode 100644 index 0000000..a584dc8 --- /dev/null +++ b/outputs/qwen3-30b-tp1-v3-kvaware/exp2_2p6d_kvc_kvaware_summary.json @@ -0,0 +1,85 @@ +{ + "actual_output_tokens_stats": { + "count": 4440.0, + "mean": 225.87972972972972, + "p50": 86.0, + "p90": 576.0, + "p99": 1347.0 + }, + "cache_hit_request_count": 4201, + "cached_tokens_stats": { + "count": 4449.0, + "mean": 24345.55787817487, + "p50": 21504.0, + "p90": 48792.0, + "p99": 69120.0 + }, + "decode_request_priorities": {}, + "error_count": 9, + "execution_modes": { + "kvcache-centric": 9, + "kvcache-direct-to-d-session": 1358, + "pd-router-d-session-reseed": 12, + "pd-router-fallback-d-backpressure": 2, + "pd-router-fallback-large-append-seed-filter-early-turn": 52, + "pd-router-fallback-large-append-session-cap": 2902, + "pd-router-fallback-session-cap": 25, + "pd-router-large-append-reseed": 34, + "pd-router-large-append-reseed-after-eviction": 4, + "pd-router-turn1-d-backpressure": 1, + "pd-router-turn1-seed": 30, + "pd-router-turn1-session-cap": 20 + }, + "latency_stats_s": { + "count": 4440.0, + "mean": 3.582334662846558, + "p50": 1.517257746309042, + "p90": 9.225348330102861, + "p99": 18.70269925892353 + }, + "mechanisms": { + "kvcache-centric": 4449 + }, + "per_decode_load": { + "decode-0": 710, + "decode-1": 630, + "decode-2": 763, + "decode-3": 737, + "decode-4": 879, + "decode-5": 730 + }, + "per_prefill_load": { + "prefill-0": 2225, + "prefill-1": 2224 + }, + "prefill_request_priorities": { + "-100": 80, + "100": 3002 + }, + "re_prefill_count": 0, + "request_count": 4449, + "reuse_expected_count": 4397, + "reuse_observed_count": 4397, + "router_url": "http://127.0.0.1:8000", + "session_reset_count": 0, + "session_reused_count": 1358, + "total_actual_kv_transfer_blocks": 78979, + "total_cached_tokens": 108313387, + "total_kv_transfer_blocks": 105235, + "tpot_stats_s": { + "count": 4440.0, + "mean": 0.005882534704321737, + "p50": 0.005807478777200416, + "p90": 0.00712956755887717, + "p99": 0.008372141476720572 + }, + "trace_path": "outputs/qwen3-30b-tp1-v3-kvaware/kvcache-centric-kv-aware-worker-admission-20260428T104343Z/sampled-trace.jsonl", + "truncated_request_count": 42, + "ttft_stats_s": { + "count": 4440.0, + "mean": 2.2045287611873334, + "p50": 0.32809355948120356, + "p90": 6.947275545448065, + "p99": 16.705802395939827 + } +} \ No newline at end of file diff --git a/outputs/qwen3-30b-tp1-v3-kvaware/sweep_results.txt b/outputs/qwen3-30b-tp1-v3-kvaware/sweep_results.txt new file mode 100644 index 0000000..464e9b6 --- /dev/null +++ b/outputs/qwen3-30b-tp1-v3-kvaware/sweep_results.txt @@ -0,0 +1,189 @@ +[2026-04-28 17:51:41] Starting TP1 v3 sweep (KVC with kv-aware policy) +[2026-04-28 17:51:41] Model: /mnt/kzlin/workflow/pd-hybrid/simm-swe-bench/models/Qwen3-30B-A3B-Instruct-2507 +[2026-04-28 17:51:41] Trace: outputs/qwen35-swebench-50sess.jsonl (4449 requests, 52 sessions) +[2026-04-28 17:51:41] Key change: --policy kv-aware for KVC (was --policy default in v2) +[2026-04-28 17:51:41] +[2026-04-28 17:51:41] === [EXP1] 1P7D KVC kv-aware === +[2026-04-28 18:43:43] === exp1_1p7d_kvc_kvaware COMPLETED === +[2026-04-28 18:43:43] Summary: +{ + "actual_output_tokens_stats": { + "count": 4086.0, + "mean": 213.95105237395987, + "p50": 83.0, + "p90": 562.0, + "p99": 1346.0 + }, + "cache_hit_request_count": 3929, + "cached_tokens_stats": { + "count": 4449.0, + "mean": 22635.924702180266, + "p50": 20010.0, + "p90": 48002.0, + "p99": 65424.0 + }, + "decode_request_priorities": {}, + "error_count": 363, + "execution_modes": { + "kvcache-centric": 363, + "kvcache-direct-to-d-session": 1716, + "pd-router-d-session-reseed": 23, + "pd-router-fallback-d-backpressure": 12, + "pd-router-fallback-large-append": 5, + "pd-router-fallback-large-append-seed-filter-early-turn": 51, + "pd-router-fallback-large-append-session-cap": 2148, + "pd-router-fallback-no-d-capacity": 7, + "pd-router-fallback-session-cap": 32, + "pd-router-large-append-reseed": 39, + "pd-router-large-append-reseed-after-eviction": 2, + "pd-router-turn1-d-backpressure": 1, + "pd-router-turn1-no-d-capacity": 3, + "pd-router-turn1-seed": 34, + "pd-router-turn1-session-cap": 13 + }, + "latency_stats_s": { + "count": 4086.0, + "mean": 4.8753733304192455, + "p50": 1.754677688702941, + "p90": 12.66968655679375, + "p99": 28.717210091650486 + }, + "mechanisms": { + "kvcache-centric": 4449 + }, + "per_decode_load": { + "decode-0": 616, + "decode-1": 658, + "decode-2": 674, + "decode-3": 582, + "decode-4": 656, + "decode-5": 662, + "decode-6": 601 + }, + "per_prefill_load": { + "prefill-0": 4449 + }, + "prefill_request_priorities": { + "-100": 98, + "100": 2272 + }, + "re_prefill_count": 0, + "request_count": 4449, + "reuse_expected_count": 4397, + "reuse_observed_count": 4397, + "router_url": "http://127.0.0.1:8000", + "session_reset_count": 0, + "session_reused_count": 1716, + "total_actual_kv_transfer_blocks": 62123, + "total_cached_tokens": 100707229, + "total_kv_transfer_blocks": 105235, + "tpot_stats_s": { + "count": 4086.0, + "mean": 0.005829451223571163, + "p50": 0.005684156496173296, + "p90": 0.007143743503740225, + "p99": 0.008634991403068266 + }, + "trace_path": "outputs/qwen3-30b-tp1-v3-kvaware/kvcache-centric-kv-aware-worker-admission-20260428T095141Z/sampled-trace.jsonl", + "truncated_request_count": 42, + "ttft_stats_s": { + "count": 4086.0, + "mean": 3.5955862397812597, + "p50": 0.36274072993546724, + "p90": 10.972254231572151, + "p99": 27.433656523004174 + } +} +[2026-04-28 18:43:43] Saved to outputs/qwen3-30b-tp1-v3-kvaware/exp1_1p7d_kvc_kvaware_summary.json + exp1_1p7d_kvc_kvaware_metrics.jsonl +[2026-04-28 18:43:43] +[2026-04-28 18:43:43] === [EXP2] 2P6D KVC kv-aware === +[2026-04-28 19:30:38] === exp2_2p6d_kvc_kvaware COMPLETED === +[2026-04-28 19:30:38] Summary: +{ + "actual_output_tokens_stats": { + "count": 4440.0, + "mean": 225.87972972972972, + "p50": 86.0, + "p90": 576.0, + "p99": 1347.0 + }, + "cache_hit_request_count": 4201, + "cached_tokens_stats": { + "count": 4449.0, + "mean": 24345.55787817487, + "p50": 21504.0, + "p90": 48792.0, + "p99": 69120.0 + }, + "decode_request_priorities": {}, + "error_count": 9, + "execution_modes": { + "kvcache-centric": 9, + "kvcache-direct-to-d-session": 1358, + "pd-router-d-session-reseed": 12, + "pd-router-fallback-d-backpressure": 2, + "pd-router-fallback-large-append-seed-filter-early-turn": 52, + "pd-router-fallback-large-append-session-cap": 2902, + "pd-router-fallback-session-cap": 25, + "pd-router-large-append-reseed": 34, + "pd-router-large-append-reseed-after-eviction": 4, + "pd-router-turn1-d-backpressure": 1, + "pd-router-turn1-seed": 30, + "pd-router-turn1-session-cap": 20 + }, + "latency_stats_s": { + "count": 4440.0, + "mean": 3.582334662846558, + "p50": 1.517257746309042, + "p90": 9.225348330102861, + "p99": 18.70269925892353 + }, + "mechanisms": { + "kvcache-centric": 4449 + }, + "per_decode_load": { + "decode-0": 710, + "decode-1": 630, + "decode-2": 763, + "decode-3": 737, + "decode-4": 879, + "decode-5": 730 + }, + "per_prefill_load": { + "prefill-0": 2225, + "prefill-1": 2224 + }, + "prefill_request_priorities": { + "-100": 80, + "100": 3002 + }, + "re_prefill_count": 0, + "request_count": 4449, + "reuse_expected_count": 4397, + "reuse_observed_count": 4397, + "router_url": "http://127.0.0.1:8000", + "session_reset_count": 0, + "session_reused_count": 1358, + "total_actual_kv_transfer_blocks": 78979, + "total_cached_tokens": 108313387, + "total_kv_transfer_blocks": 105235, + "tpot_stats_s": { + "count": 4440.0, + "mean": 0.005882534704321737, + "p50": 0.005807478777200416, + "p90": 0.00712956755887717, + "p99": 0.008372141476720572 + }, + "trace_path": "outputs/qwen3-30b-tp1-v3-kvaware/kvcache-centric-kv-aware-worker-admission-20260428T104343Z/sampled-trace.jsonl", + "truncated_request_count": 42, + "ttft_stats_s": { + "count": 4440.0, + "mean": 2.2045287611873334, + "p50": 0.32809355948120356, + "p90": 6.947275545448065, + "p99": 16.705802395939827 + } +} +[2026-04-28 19:30:38] Saved to outputs/qwen3-30b-tp1-v3-kvaware/exp2_2p6d_kvc_kvaware_summary.json + exp2_2p6d_kvc_kvaware_metrics.jsonl +[2026-04-28 19:30:38] +[2026-04-28 19:30:38] === ALL TP1 V3 SWEEP EXPERIMENTS DONE === diff --git a/outputs/qwen3-30b-tp1-v4-cap16/exp1_1p7d_kvc_cap16_summary.json b/outputs/qwen3-30b-tp1-v4-cap16/exp1_1p7d_kvc_cap16_summary.json new file mode 100644 index 0000000..45bdaba --- /dev/null +++ b/outputs/qwen3-30b-tp1-v4-cap16/exp1_1p7d_kvc_cap16_summary.json @@ -0,0 +1,88 @@ +{ + "actual_output_tokens_stats": { + "count": 4014.0, + "mean": 215.048081714001, + "p50": 83.0, + "p90": 570.0, + "p99": 1343.0 + }, + "cache_hit_request_count": 3865, + "cached_tokens_stats": { + "count": 4449.0, + "mean": 21373.60867610699, + "p50": 18429.0, + "p90": 45643.0, + "p99": 65088.0 + }, + "decode_request_priorities": {}, + "error_count": 435, + "execution_modes": { + "kvcache-centric": 435, + "kvcache-direct-to-d-session": 2180, + "pd-router-d-session-reseed": 44, + "pd-router-d-session-reseed-after-eviction": 1, + "pd-router-fallback-d-backpressure": 36, + "pd-router-fallback-large-append": 35, + "pd-router-fallback-large-append-seed-filter-early-turn": 52, + "pd-router-fallback-large-append-session-cap": 1500, + "pd-router-fallback-no-d-capacity": 13, + "pd-router-fallback-session-cap": 43, + "pd-router-large-append-reseed": 55, + "pd-router-large-append-reseed-after-eviction": 3, + "pd-router-turn1-d-backpressure": 1, + "pd-router-turn1-no-d-capacity": 5, + "pd-router-turn1-seed": 46 + }, + "latency_stats_s": { + "count": 4014.0, + "mean": 4.214657033050009, + "p50": 1.0827504023909569, + "p90": 13.380241627804935, + "p99": 24.453291333280504 + }, + "mechanisms": { + "kvcache-centric": 4449 + }, + "per_decode_load": { + "decode-0": 690, + "decode-1": 599, + "decode-2": 660, + "decode-3": 584, + "decode-4": 606, + "decode-5": 646, + "decode-6": 664 + }, + "per_prefill_load": { + "prefill-0": 4449 + }, + "prefill_request_priorities": { + "-100": 149, + "100": 1685 + }, + "re_prefill_count": 0, + "request_count": 4449, + "reuse_expected_count": 4397, + "reuse_observed_count": 4397, + "router_url": "http://127.0.0.1:8000", + "session_reset_count": 0, + "session_reused_count": 2180, + "total_actual_kv_transfer_blocks": 52857, + "total_cached_tokens": 95091185, + "total_kv_transfer_blocks": 105235, + "tpot_stats_s": { + "count": 4014.0, + "mean": 0.005804301410418847, + "p50": 0.005607025208882987, + "p90": 0.007293824862528552, + "p99": 0.008864479259402893 + }, + "trace_path": "outputs/qwen3-30b-tp1-v4-cap16/kvcache-centric-kv-aware-worker-admission-20260428T125022Z/sampled-trace.jsonl", + "truncated_request_count": 43, + "ttft_stats_s": { + "count": 4014.0, + "mean": 2.915135478307124, + "p50": 0.05643345229327679, + "p90": 11.900803190656006, + "p99": 22.758968392387033 + } +} \ No newline at end of file diff --git a/outputs/qwen3-30b-tp1-v4-cap16/exp2_2p6d_kvc_cap16_summary.json b/outputs/qwen3-30b-tp1-v4-cap16/exp2_2p6d_kvc_cap16_summary.json new file mode 100644 index 0000000..5e90c49 --- /dev/null +++ b/outputs/qwen3-30b-tp1-v4-cap16/exp2_2p6d_kvc_cap16_summary.json @@ -0,0 +1,86 @@ +{ + "actual_output_tokens_stats": { + "count": 4046.0, + "mean": 224.65002471576867, + "p50": 84.0, + "p90": 576.0, + "p99": 1349.0 + }, + "cache_hit_request_count": 3925, + "cached_tokens_stats": { + "count": 4449.0, + "mean": 22852.7439874129, + "p50": 19584.0, + "p90": 49009.0, + "p99": 67320.0 + }, + "decode_request_priorities": {}, + "error_count": 403, + "execution_modes": { + "kvcache-centric": 403, + "kvcache-direct-to-d-session": 2348, + "pd-router-d-session-reseed": 28, + "pd-router-fallback-d-backpressure": 7, + "pd-router-fallback-large-append": 68, + "pd-router-fallback-large-append-seed-filter-early-turn": 45, + "pd-router-fallback-large-append-session-cap": 1403, + "pd-router-fallback-no-d-capacity": 9, + "pd-router-fallback-session-cap": 25, + "pd-router-large-append-reseed": 57, + "pd-router-large-append-reseed-after-eviction": 6, + "pd-router-turn1-no-d-capacity": 1, + "pd-router-turn1-seed": 49 + }, + "latency_stats_s": { + "count": 4046.0, + "mean": 2.505981629502371, + "p50": 0.8372491216287017, + "p90": 6.5139341270551085, + "p99": 18.335972285829484 + }, + "mechanisms": { + "kvcache-centric": 4449 + }, + "per_decode_load": { + "decode-0": 767, + "decode-1": 680, + "decode-2": 906, + "decode-3": 818, + "decode-4": 800, + "decode-5": 478 + }, + "per_prefill_load": { + "prefill-0": 2225, + "prefill-1": 2224 + }, + "prefill_request_priorities": { + "-100": 140, + "100": 1558 + }, + "re_prefill_count": 0, + "request_count": 4449, + "reuse_expected_count": 4397, + "reuse_observed_count": 4397, + "router_url": "http://127.0.0.1:8000", + "session_reset_count": 0, + "session_reused_count": 2348, + "total_actual_kv_transfer_blocks": 50727, + "total_cached_tokens": 101671858, + "total_kv_transfer_blocks": 105235, + "tpot_stats_s": { + "count": 4046.0, + "mean": 0.005708743129332261, + "p50": 0.005565466725497757, + "p90": 0.006912594398356141, + "p99": 0.008102089307750717 + }, + "trace_path": "outputs/qwen3-30b-tp1-v4-cap16/kvcache-centric-kv-aware-worker-admission-20260428T134057Z/sampled-trace.jsonl", + "truncated_request_count": 36, + "ttft_stats_s": { + "count": 4046.0, + "mean": 1.1653790952959129, + "p50": 0.05140436999499798, + "p90": 2.6447059931233525, + "p99": 15.121314341202378 + } +} \ No newline at end of file diff --git a/outputs/qwen3-30b-tp1-v4-cap16/sweep_results.txt b/outputs/qwen3-30b-tp1-v4-cap16/sweep_results.txt new file mode 100644 index 0000000..7cff9a8 --- /dev/null +++ b/outputs/qwen3-30b-tp1-v4-cap16/sweep_results.txt @@ -0,0 +1,190 @@ +[2026-04-28 20:50:21] Starting TP1 v4 sweep (KVC kv-aware, session soft_cap raised 4->16) +[2026-04-28 20:50:21] Model: /mnt/kzlin/workflow/pd-hybrid/simm-swe-bench/models/Qwen3-30B-A3B-Instruct-2507 +[2026-04-28 20:50:21] Trace: outputs/qwen35-swebench-50sess.jsonl (4449 requests, 52 sessions) +[2026-04-28 20:50:21] Key change: _decode_session_soft_cap now min(16, ...) instead of min(4, ...) +[2026-04-28 20:50:21] +[2026-04-28 20:50:21] === [EXP1] 1P7D KVC kv-aware cap=16 === +[2026-04-28 21:40:57] === exp1_1p7d_kvc_cap16 COMPLETED === +[2026-04-28 21:40:57] Summary: +{ + "actual_output_tokens_stats": { + "count": 4014.0, + "mean": 215.048081714001, + "p50": 83.0, + "p90": 570.0, + "p99": 1343.0 + }, + "cache_hit_request_count": 3865, + "cached_tokens_stats": { + "count": 4449.0, + "mean": 21373.60867610699, + "p50": 18429.0, + "p90": 45643.0, + "p99": 65088.0 + }, + "decode_request_priorities": {}, + "error_count": 435, + "execution_modes": { + "kvcache-centric": 435, + "kvcache-direct-to-d-session": 2180, + "pd-router-d-session-reseed": 44, + "pd-router-d-session-reseed-after-eviction": 1, + "pd-router-fallback-d-backpressure": 36, + "pd-router-fallback-large-append": 35, + "pd-router-fallback-large-append-seed-filter-early-turn": 52, + "pd-router-fallback-large-append-session-cap": 1500, + "pd-router-fallback-no-d-capacity": 13, + "pd-router-fallback-session-cap": 43, + "pd-router-large-append-reseed": 55, + "pd-router-large-append-reseed-after-eviction": 3, + "pd-router-turn1-d-backpressure": 1, + "pd-router-turn1-no-d-capacity": 5, + "pd-router-turn1-seed": 46 + }, + "latency_stats_s": { + "count": 4014.0, + "mean": 4.214657033050009, + "p50": 1.0827504023909569, + "p90": 13.380241627804935, + "p99": 24.453291333280504 + }, + "mechanisms": { + "kvcache-centric": 4449 + }, + "per_decode_load": { + "decode-0": 690, + "decode-1": 599, + "decode-2": 660, + "decode-3": 584, + "decode-4": 606, + "decode-5": 646, + "decode-6": 664 + }, + "per_prefill_load": { + "prefill-0": 4449 + }, + "prefill_request_priorities": { + "-100": 149, + "100": 1685 + }, + "re_prefill_count": 0, + "request_count": 4449, + "reuse_expected_count": 4397, + "reuse_observed_count": 4397, + "router_url": "http://127.0.0.1:8000", + "session_reset_count": 0, + "session_reused_count": 2180, + "total_actual_kv_transfer_blocks": 52857, + "total_cached_tokens": 95091185, + "total_kv_transfer_blocks": 105235, + "tpot_stats_s": { + "count": 4014.0, + "mean": 0.005804301410418847, + "p50": 0.005607025208882987, + "p90": 0.007293824862528552, + "p99": 0.008864479259402893 + }, + "trace_path": "outputs/qwen3-30b-tp1-v4-cap16/kvcache-centric-kv-aware-worker-admission-20260428T125022Z/sampled-trace.jsonl", + "truncated_request_count": 43, + "ttft_stats_s": { + "count": 4014.0, + "mean": 2.915135478307124, + "p50": 0.05643345229327679, + "p90": 11.900803190656006, + "p99": 22.758968392387033 + } +} +[2026-04-28 21:40:57] Saved to outputs/qwen3-30b-tp1-v4-cap16/exp1_1p7d_kvc_cap16_summary.json + exp1_1p7d_kvc_cap16_metrics.jsonl +[2026-04-28 21:40:57] +[2026-04-28 21:40:57] === [EXP2] 2P6D KVC kv-aware cap=16 === +[2026-04-28 22:27:53] === exp2_2p6d_kvc_cap16 COMPLETED === +[2026-04-28 22:27:53] Summary: +{ + "actual_output_tokens_stats": { + "count": 4046.0, + "mean": 224.65002471576867, + "p50": 84.0, + "p90": 576.0, + "p99": 1349.0 + }, + "cache_hit_request_count": 3925, + "cached_tokens_stats": { + "count": 4449.0, + "mean": 22852.7439874129, + "p50": 19584.0, + "p90": 49009.0, + "p99": 67320.0 + }, + "decode_request_priorities": {}, + "error_count": 403, + "execution_modes": { + "kvcache-centric": 403, + "kvcache-direct-to-d-session": 2348, + "pd-router-d-session-reseed": 28, + "pd-router-fallback-d-backpressure": 7, + "pd-router-fallback-large-append": 68, + "pd-router-fallback-large-append-seed-filter-early-turn": 45, + "pd-router-fallback-large-append-session-cap": 1403, + "pd-router-fallback-no-d-capacity": 9, + "pd-router-fallback-session-cap": 25, + "pd-router-large-append-reseed": 57, + "pd-router-large-append-reseed-after-eviction": 6, + "pd-router-turn1-no-d-capacity": 1, + "pd-router-turn1-seed": 49 + }, + "latency_stats_s": { + "count": 4046.0, + "mean": 2.505981629502371, + "p50": 0.8372491216287017, + "p90": 6.5139341270551085, + "p99": 18.335972285829484 + }, + "mechanisms": { + "kvcache-centric": 4449 + }, + "per_decode_load": { + "decode-0": 767, + "decode-1": 680, + "decode-2": 906, + "decode-3": 818, + "decode-4": 800, + "decode-5": 478 + }, + "per_prefill_load": { + "prefill-0": 2225, + "prefill-1": 2224 + }, + "prefill_request_priorities": { + "-100": 140, + "100": 1558 + }, + "re_prefill_count": 0, + "request_count": 4449, + "reuse_expected_count": 4397, + "reuse_observed_count": 4397, + "router_url": "http://127.0.0.1:8000", + "session_reset_count": 0, + "session_reused_count": 2348, + "total_actual_kv_transfer_blocks": 50727, + "total_cached_tokens": 101671858, + "total_kv_transfer_blocks": 105235, + "tpot_stats_s": { + "count": 4046.0, + "mean": 0.005708743129332261, + "p50": 0.005565466725497757, + "p90": 0.006912594398356141, + "p99": 0.008102089307750717 + }, + "trace_path": "outputs/qwen3-30b-tp1-v4-cap16/kvcache-centric-kv-aware-worker-admission-20260428T134057Z/sampled-trace.jsonl", + "truncated_request_count": 36, + "ttft_stats_s": { + "count": 4046.0, + "mean": 1.1653790952959129, + "p50": 0.05140436999499798, + "p90": 2.6447059931233525, + "p99": 15.121314341202378 + } +} +[2026-04-28 22:27:53] Saved to outputs/qwen3-30b-tp1-v4-cap16/exp2_2p6d_kvc_cap16_summary.json + exp2_2p6d_kvc_cap16_metrics.jsonl +[2026-04-28 22:27:53] +[2026-04-28 22:27:53] === ALL TP1 V4 SWEEP EXPERIMENTS DONE === diff --git a/scripts/analysis/analyze_errors.py b/scripts/analysis/analyze_errors.py new file mode 100644 index 0000000..e837712 --- /dev/null +++ b/scripts/analysis/analyze_errors.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python3 +"""Deep dive into v4 errors: which path, which D, which session, which turn.""" +import json +import numpy as np +from pathlib import Path +from collections import Counter, defaultdict + +BASE = Path(__file__).parent + +def load_rows(jsonl_path): + rows = [] + with open(jsonl_path) as f: + for line in f: + rows.append(json.loads(line)) + return rows + +# Compare v3 and v4 errors +for label, path in [ + ("v3 1P7D", BASE.parent / "qwen3-30b-tp1-v3-kvaware/exp1_1p7d_kvc_kvaware_metrics.jsonl"), + ("v4 1P7D", BASE / "exp1_1p7d_kvc_cap16_metrics.jsonl"), + ("v3 2P6D", BASE.parent / "qwen3-30b-tp1-v3-kvaware/exp2_2p6d_kvc_kvaware_metrics.jsonl"), + ("v4 2P6D", BASE / "exp2_2p6d_kvc_cap16_metrics.jsonl"), +]: + if not path.exists(): + print(f"\nSKIP {label}: {path} not found") + continue + rows = load_rows(path) + err = [r for r in rows if r.get("error") is not None] + print(f"\n========== {label} ({len(err)} errors / {len(rows)} total = {len(err)/len(rows)*100:.1f}%) ==========") + + # Error finish_reason distribution + fr_counter = Counter() + for r in err: + fr = str(r.get("finish_reason") or r.get("error") or "?") + fr_counter[fr[:80]] += 1 + print(f"finish_reason distribution:") + for fr, cnt in fr_counter.most_common(): + print(f" {cnt:>4}x {fr}") + + # Errors by execution mode (these are aborted before mode assignment usually) + mode_counter = Counter(r.get("execution_mode", "?") for r in err) + print(f"\nerror by execution_mode:") + for mode, cnt in mode_counter.most_common(): + print(f" {cnt:>4}x {mode}") + + # Errors per D worker + dw_counter = Counter(r.get("assigned_decode_node", "?") for r in err) + print(f"\nerror per assigned_decode_node:") + for dw, cnt in dw_counter.most_common(): + print(f" {cnt:>4}x {dw}") + + # Errors by turn distribution + turn_counter = Counter(r.get("turn_id", -1) for r in err) + early = sum(c for t, c in turn_counter.items() if t <= 5) + mid = sum(c for t, c in turn_counter.items() if 5 < t <= 30) + late = sum(c for t, c in turn_counter.items() if t > 30) + print(f"\nerror by turn: early(0-5)={early} mid(6-30)={mid} late(31+)={late}") + + # Per-session error rate + per_sess_err = defaultdict(int) + per_sess_total = defaultdict(int) + for r in rows: + per_sess_total[r["session_id"]] += 1 + if r.get("error") is not None: + per_sess_err[r["session_id"]] += 1 + sess_with_err = [(sid, per_sess_err[sid], per_sess_total[sid]) for sid in per_sess_err] + sess_with_err.sort(key=lambda x: -x[1]) + print(f"\ntop 5 sessions by error count:") + for sid, e, t in sess_with_err[:5]: + print(f" session {sid}: {e}/{t} errors ({e/t*100:.0f}%)") + + # Errors timeline: are they bursty? + err_ts = sorted([r.get("trace_timestamp_s", 0) for r in err]) + if err_ts: + first_ts = err_ts[0] + last_ts = err_ts[-1] + all_ts = sorted([r.get("trace_timestamp_s", 0) for r in rows]) + first_all = all_ts[0] + last_all = all_ts[-1] + run_duration = last_all - first_all + err_first_pct = (err_ts[0] - first_all) / run_duration * 100 if run_duration > 0 else 0 + err_last_pct = (err_ts[-1] - first_all) / run_duration * 100 if run_duration > 0 else 0 + print(f"\nerror time range (% of run): {err_first_pct:.1f}% - {err_last_pct:.1f}%") diff --git a/scripts/analysis/analyze_v3.py b/scripts/analysis/analyze_v3.py new file mode 100644 index 0000000..5e29919 --- /dev/null +++ b/scripts/analysis/analyze_v3.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python3 +"""Analyze v3 (kv-aware) results — find why fallback-large-append-session-cap dominates.""" +import json +import numpy as np +from pathlib import Path +from collections import Counter, defaultdict + +BASE = Path(__file__).parent + +def load_rows(jsonl_path): + rows = [] + with open(jsonl_path) as f: + for line in f: + rows.append(json.loads(line)) + return rows + +exp1 = load_rows(BASE / "exp1_1p7d_kvc_kvaware_metrics.jsonl") +exp2 = load_rows(BASE / "exp2_2p6d_kvc_kvaware_metrics.jsonl") + +for name, rows in [("Exp1 1P7D", exp1), ("Exp2 2P6D", exp2)]: + print(f"\n========== {name} ==========") + ok = [r for r in rows if r.get("error") is None] + + # Execution mode breakdown by latency + modes = Counter(r["execution_mode"] for r in ok) + print(f"\nExecution modes (n={len(ok)}):") + for mode, count in modes.most_common(): + mode_rows = [r for r in ok if r["execution_mode"] == mode] + lats = [r["latency_s"] for r in mode_rows] + ttfts = [r["ttft_s"] for r in mode_rows] + print(f" {mode}: n={count} ({count/len(ok)*100:.1f}%) " + f"lat P50={np.percentile(lats,50):.3f}s P90={np.percentile(lats,90):.3f}s | " + f"ttft P50={np.percentile(ttfts,50):.3f}s P90={np.percentile(ttfts,90):.3f}s") + + # Per-D session distribution + per_d_sessions = defaultdict(set) + for r in ok: + d = r.get("assigned_decode_node", "?") + per_d_sessions[d].add(r["session_id"]) + print(f"\nSessions per D worker:") + for d in sorted(per_d_sessions.keys()): + print(f" {d}: {len(per_d_sessions[d])} unique sessions") + + # session-cap fallback analysis + sc_rows = [r for r in ok if r["execution_mode"] == "pd-router-fallback-large-append-session-cap"] + if sc_rows: + print(f"\nSession-cap fallback details (n={len(sc_rows)}):") + # Which sessions hit this most? + sc_per_sess = Counter(r["session_id"] for r in sc_rows) + print(f" Sessions hitting session-cap (top 5):") + for sid, cnt in sc_per_sess.most_common(5): + print(f" session {sid}: {cnt} times") + # Per-D distribution + sc_per_d = Counter(r.get("assigned_decode_node", "?") for r in sc_rows) + print(f" Per-D distribution: {dict(sc_per_d.most_common())}") + # Input length distribution + inp = [r.get("input_length", 0) for r in sc_rows] + print(f" Input length: P50={np.percentile(inp,50):.0f} P90={np.percentile(inp,90):.0f}") + # Turn distribution + turns = Counter(r.get("turn_id", -1) for r in sc_rows) + print(f" Turn distribution (top 5): {dict(turns.most_common(5))}") + + # Direct-to-D analysis (ideal path) + dd_rows = [r for r in ok if r["execution_mode"] == "kvcache-direct-to-d-session"] + if dd_rows: + lats = [r["latency_s"] for r in dd_rows] + ttfts = [r["ttft_s"] for r in dd_rows] + kv_blocks = [r.get("actual_kv_transfer_blocks", 0) for r in dd_rows] + cached = [r.get("cached_tokens", 0) for r in dd_rows] + print(f"\nDirect-to-D details (n={len(dd_rows)}):") + print(f" lat P50={np.percentile(lats,50):.3f}s P90={np.percentile(lats,90):.3f}s P99={np.percentile(lats,99):.3f}s") + print(f" ttft P50={np.percentile(ttfts,50):.3f}s P90={np.percentile(ttfts,90):.3f}s") + print(f" KV transfer: P50={np.percentile(kv_blocks,50):.0f} (should be 0 — no P involved)") + print(f" cached_tokens P50={np.percentile(cached,50):.0f}") + + # Sessions: how many turns each, how many used direct-to-d + print(f"\nPer-session direct-to-D rate (top 10 by total turns):") + per_sess = defaultdict(list) + for r in ok: + per_sess[r["session_id"]].append(r) + sess_stats = [] + for sid, sreqs in per_sess.items(): + total = len(sreqs) + dd = sum(1 for r in sreqs if r["execution_mode"] == "kvcache-direct-to-d-session") + sc = sum(1 for r in sreqs if "session-cap" in r["execution_mode"]) + sess_stats.append((sid, total, dd, sc)) + sess_stats.sort(key=lambda x: -x[1]) + for sid, total, dd, sc in sess_stats[:10]: + print(f" session {sid}: {total} turns, {dd} direct-to-D ({dd/total*100:.0f}%), {sc} session-cap fallback ({sc/total*100:.0f}%)") diff --git a/scripts/analysis/analyze_v4.py b/scripts/analysis/analyze_v4.py new file mode 100644 index 0000000..3367330 --- /dev/null +++ b/scripts/analysis/analyze_v4.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python3 +"""V4 results analysis: errors, execution modes, latency by mode.""" +import json +import numpy as np +from pathlib import Path +from collections import Counter + +BASE = Path(__file__).parent + +def load_rows(jsonl_path): + rows = [] + with open(jsonl_path) as f: + for line in f: + rows.append(json.loads(line)) + return rows + +for name, path in [ + ("Exp1 1P7D cap=16", BASE / "exp1_1p7d_kvc_cap16_metrics.jsonl"), + ("Exp2 2P6D cap=16", BASE / "exp2_2p6d_kvc_cap16_metrics.jsonl"), +]: + rows = load_rows(path) + print(f"\n========== {name} ==========") + ok = [r for r in rows if r.get("error") is None] + err = [r for r in rows if r.get("error") is not None] + print(f"Total: {len(rows)}, OK: {len(ok)}, Errors: {len(err)}") + + # Errors finish_reason + if err: + finish_reasons = Counter() + for r in err: + fr = str(r.get("finish_reason") or r.get("error") or "?") + # Truncate long messages + short = fr[:120] + finish_reasons[short] += 1 + print(f"\nError finish_reasons (top 5):") + for fr, cnt in finish_reasons.most_common(5): + print(f" {cnt}x: {fr}") + + # Execution mode latency breakdown + modes = Counter(r["execution_mode"] for r in ok) + print(f"\nTop execution modes by latency:") + print(f"{'mode':<55}{'n':<8}{'%':<8}{'P50 lat':<10}{'P90 lat':<10}{'TTFT P50':<10}") + for mode, count in modes.most_common(8): + mode_rows = [r for r in ok if r["execution_mode"] == mode] + lats = [r["latency_s"] for r in mode_rows] + ttfts = [r["ttft_s"] for r in mode_rows] + print(f" {mode:<53}{count:<8}{count/len(ok)*100:>5.1f}% {np.percentile(lats,50):>7.3f}s {np.percentile(lats,90):>7.3f}s {np.percentile(ttfts,50):>7.3f}s") + + # Per-D load + per_d = Counter(r.get("assigned_decode_node", "?") for r in ok) + print(f"\nPer-D load: max/min ratio = {max(per_d.values())/max(min(per_d.values()),1):.2f}x") + print(f" {dict(per_d.most_common())}") diff --git a/scripts/analysis/compare_no_error.py b/scripts/analysis/compare_no_error.py new file mode 100644 index 0000000..4384726 --- /dev/null +++ b/scripts/analysis/compare_no_error.py @@ -0,0 +1,136 @@ +#!/usr/bin/env python3 +"""Compare KVC variants vs baseline, EXCLUDING errors and truncated requests.""" +import json +import numpy as np +from pathlib import Path + +OUT = Path("/mnt/kzlin/workflow/pd-hybrid/agentic-pd-hybrid/outputs") + +DATASETS = [ + ("baseline 8DP", OUT / "qwen3-30b-tp1-v2-fixed/exp1_8way_dp_cache_aware_metrics.jsonl"), + ("v3 1P7D", OUT / "qwen3-30b-tp1-v3-kvaware/exp1_1p7d_kvc_kvaware_metrics.jsonl"), + ("v3 2P6D", OUT / "qwen3-30b-tp1-v3-kvaware/exp2_2p6d_kvc_kvaware_metrics.jsonl"), + ("v4 1P7D", OUT / "qwen3-30b-tp1-v4-cap16/exp1_1p7d_kvc_cap16_metrics.jsonl"), + ("v4 2P6D", OUT / "qwen3-30b-tp1-v4-cap16/exp2_2p6d_kvc_cap16_metrics.jsonl"), +] + +def load_rows(path): + rows = [] + with open(path) as f: + for line in f: + rows.append(json.loads(line)) + return rows + +def is_truncated(row): + a = row.get("actual_output_tokens") + r = row.get("requested_output_tokens") + if a is not None and r is not None and r > 1: + return a < r * 0.5 + return False + +def stats(values): + if not values: + return {"n": 0} + a = np.array(values) + return { + "n": len(a), + "mean": float(np.mean(a)), + "p50": float(np.percentile(a, 50)), + "p90": float(np.percentile(a, 90)), + "p99": float(np.percentile(a, 99)), + } + +def fmt(s, key): + if s["n"] == 0: + return "N/A" + v = s[key] + return f"{v:.3f}s" if v < 100 else f"{v:.1f}s" + +results = [] +for label, path in DATASETS: + if not path.exists(): + print(f"SKIP {label}") + continue + rows = load_rows(path) + total = len(rows) + err_n = sum(1 for r in rows if r.get("error") is not None) + trunc_n = sum(1 for r in rows if r.get("error") is None and is_truncated(r)) + + # Filter: error=None AND not truncated AND latency present + clean = [r for r in rows + if r.get("error") is None + and not is_truncated(r) + and r.get("latency_s") is not None] + + lats = [r["latency_s"] for r in clean] + ttfts = [r["ttft_s"] for r in clean if r.get("ttft_s") is not None] + + results.append({ + "label": label, + "total": total, + "err": err_n, + "trunc": trunc_n, + "clean_n": len(clean), + "lat": stats(lats), + "ttft": stats(ttfts), + }) + +# Print comparison table +print(f"\n{'='*100}") +print("LATENCY (excluding errors AND truncated)") +print(f"{'='*100}") +print(f"{'config':<16}{'total':>7}{'err':>6}{'trunc':>7}{'clean':>7} {'mean':>9}{'P50':>9}{'P90':>9}{'P99':>9}") +for r in results: + print(f"{r['label']:<16}{r['total']:>7}{r['err']:>6}{r['trunc']:>7}{r['clean_n']:>7} " + f"{fmt(r['lat'],'mean'):>9}{fmt(r['lat'],'p50'):>9}{fmt(r['lat'],'p90'):>9}{fmt(r['lat'],'p99'):>9}") + +print(f"\n{'='*100}") +print("TTFT (excluding errors AND truncated)") +print(f"{'='*100}") +print(f"{'config':<16}{'clean':>7} {'mean':>9}{'P50':>9}{'P90':>9}{'P99':>9}") +for r in results: + print(f"{r['label']:<16}{r['clean_n']:>7} " + f"{fmt(r['ttft'],'mean'):>9}{fmt(r['ttft'],'p50'):>9}{fmt(r['ttft'],'p90'):>9}{fmt(r['ttft'],'p99'):>9}") + +# Also: per-execution-mode breakdown for v4 only (the most interesting) +print(f"\n{'='*100}") +print("V4 2P6D: per-execution-mode (excluding errors and truncated)") +print(f"{'='*100}") +v4_2p6d = next((p for l, p in DATASETS if l == "v4 2P6D"), None) +if v4_2p6d: + rows = load_rows(v4_2p6d) + clean = [r for r in rows if r.get("error") is None and not is_truncated(r)] + from collections import Counter + modes = Counter(r["execution_mode"] for r in clean) + print(f"{'mode':<55}{'n':>7}{'%':>7} {'mean':>9}{'P50':>9}{'P90':>9}{'P99':>9}") + for mode, count in modes.most_common(10): + m_rows = [r for r in clean if r["execution_mode"] == mode] + s = stats([r["latency_s"] for r in m_rows]) + pct = count/len(clean)*100 + print(f" {mode:<53}{count:>7}{pct:>6.1f}% {fmt(s,'mean'):>9}{fmt(s,'p50'):>9}{fmt(s,'p90'):>9}{fmt(s,'p99'):>9}") + +# Also: WHAT IF we only count direct-to-D? (Pure KVC performance) +print(f"\n{'='*100}") +print("Pure KVC (kvcache-direct-to-d-session ONLY) vs Baseline") +print(f"{'='*100}") +for label, path in DATASETS: + if not path.exists() or "1P7D" not in label and "2P6D" not in label: + continue + rows = load_rows(path) + direct = [r for r in rows + if r.get("error") is None and not is_truncated(r) + and r.get("execution_mode") == "kvcache-direct-to-d-session"] + if not direct: + continue + s_lat = stats([r["latency_s"] for r in direct]) + s_ttft = stats([r["ttft_s"] for r in direct if r.get("ttft_s") is not None]) + print(f"{label:<16}n={s_lat['n']:>5} lat: P50={fmt(s_lat,'p50')} P90={fmt(s_lat,'p90')} ttft: P50={fmt(s_ttft,'p50')} P90={fmt(s_ttft,'p90')}") + +# Baseline for reference (already non-fallback by definition) +print() +baseline_path = OUT / "qwen3-30b-tp1-v2-fixed/exp1_8way_dp_cache_aware_metrics.jsonl" +baseline_rows = load_rows(baseline_path) +clean = [r for r in baseline_rows if r.get("error") is None and not is_truncated(r)] +s_lat = stats([r["latency_s"] for r in clean]) +s_ttft = stats([r["ttft_s"] for r in clean if r.get("ttft_s") is not None]) +print(f"{'baseline 8DP':<16}n={s_lat['n']:>5} lat: P50={fmt(s_lat,'p50')} P90={fmt(s_lat,'p90')} ttft: P50={fmt(s_ttft,'p50')} P90={fmt(s_ttft,'p90')}")