From 74194e660a2c56be824950190e39d6931a8be0a5 Mon Sep 17 00:00:00 2001 From: kzlin Date: Tue, 28 Apr 2026 23:34:01 +0800 Subject: [PATCH] docs: v4 final results, error analysis, and updated journey Add v4 sweep results and post-mortem analysis showing: - direct-to-D path: 54.3% (1P7D) / 58.0% (2P6D) of requests now use KVC cleanly. P50=0.5s and TTFT P50=0.043s; this path beats baseline 8DP across the board (P50 -24%, TTFT P50 -54%, TTFT P90 -79%). - Overall vs baseline (errors+truncated excluded): v4 2P6D P50=0.85s vs baseline 0.66s (28% slower). Reason is not errors -- 35% of requests still hit fallback-large-append-session-cap, where capacity-based cap = usable_tokens / target_tokens evaluates to 1-2 (not 16) for large agentic inputs. - 9-10% errors on KVC variants are mooncake TCP transfer timeouts, not SGLang logic bugs. Prefill log shows "Failed to send kv chunk ... 32s timeout ... session not alive". Errors concentrate in turn>=31 (large inputs) after run >44.8%. Track: - docs/KVC_DEBUG_JOURNEY_V1_TO_V4.md: append v4 results table, per-mode breakdown, and error root cause. - scripts/analysis/{analyze_v3,analyze_v4,analyze_errors,compare_no_error}.py - outputs/qwen3-30b-tp1-v{3,4}*/exp*_summary.json (force-added, small JSON; metrics.jsonl excluded due to size). - outputs/qwen3-30b-tp1-v{3,4}*/sweep_results.txt Co-Authored-By: Claude Opus 4.7 (1M context) --- docs/KVC_DEBUG_JOURNEY_V1_TO_V4.md | 79 +++++++- .../exp1_1p7d_kvc_kvaware_summary.json | 88 ++++++++ .../exp2_2p6d_kvc_kvaware_summary.json | 85 ++++++++ .../sweep_results.txt | 189 +++++++++++++++++ .../exp1_1p7d_kvc_cap16_summary.json | 88 ++++++++ .../exp2_2p6d_kvc_cap16_summary.json | 86 ++++++++ .../qwen3-30b-tp1-v4-cap16/sweep_results.txt | 190 ++++++++++++++++++ scripts/analysis/analyze_errors.py | 83 ++++++++ scripts/analysis/analyze_v3.py | 89 ++++++++ scripts/analysis/analyze_v4.py | 52 +++++ scripts/analysis/compare_no_error.py | 136 +++++++++++++ 11 files changed, 1162 insertions(+), 3 deletions(-) create mode 100644 outputs/qwen3-30b-tp1-v3-kvaware/exp1_1p7d_kvc_kvaware_summary.json create mode 100644 outputs/qwen3-30b-tp1-v3-kvaware/exp2_2p6d_kvc_kvaware_summary.json create mode 100644 outputs/qwen3-30b-tp1-v3-kvaware/sweep_results.txt create mode 100644 outputs/qwen3-30b-tp1-v4-cap16/exp1_1p7d_kvc_cap16_summary.json create mode 100644 outputs/qwen3-30b-tp1-v4-cap16/exp2_2p6d_kvc_cap16_summary.json create mode 100644 outputs/qwen3-30b-tp1-v4-cap16/sweep_results.txt create mode 100644 scripts/analysis/analyze_errors.py create mode 100644 scripts/analysis/analyze_v3.py create mode 100644 scripts/analysis/analyze_v4.py create mode 100644 scripts/analysis/compare_no_error.py diff --git a/docs/KVC_DEBUG_JOURNEY_V1_TO_V4.md b/docs/KVC_DEBUG_JOURNEY_V1_TO_V4.md index ff19e0c..9e76a81 100644 --- a/docs/KVC_DEBUG_JOURNEY_V1_TO_V4.md +++ b/docs/KVC_DEBUG_JOURNEY_V1_TO_V4.md @@ -174,11 +174,84 @@ def _decode_session_soft_cap(...) -> int: + return max(1, min(16, usable_capacity_tokens // target_tokens)) ``` -7 D × 16 = 112 个 slot，远超 52 个 session 需求。预期 session-cap fallback 占比降到 <10%，整体 P50 向 direct-to-D 的 0.46s 收敛。 +7 D × 16 = 112 个 slot，远超 52 个 session 需求。 -实际数据见 `outputs/qwen3-30b-tp1-v4-cap16/`。 +### v4 实际结果（vs v3 1P7D / 2P6D） -## 后续可以考虑的更深方案：让 D 自己决定 admission +| 指标 | v3 1P7D | **v4 1P7D** | v3 2P6D | **v4 2P6D** | baseline 8DP | +|------|:---:|:---:|:---:|:---:|:---:| +| Errors | 363 (8%) | 435 (10%) | 9 (0%) | **403 (9%)** | 0 | +| 截断 | 42 | 43 | 42 | 36 | 68 | +| **direct-to-D** | 38.6% | **54.3%** | 30.5% | **58.0%** ⭐ | - | +| **session-cap fallback** | 48.3% | 37.4% | 65.4% | **34.7%** | - | +| Session reused | 1716 | 2180 | 1358 | **2348** | - | +| KV transfer blocks | 62K | 53K | 79K | **51K** | - | +| Mean | 4.88s | 4.21s | 3.58s | **2.51s** | 1.43s | +| **P50** | 1.75s | 1.08s | 1.52s | **0.84s** | **0.65s** | +| P90 | 12.67s | 13.38s | 9.23s | **6.51s** | 3.61s | +| P99 | 28.72s | 24.45s | 18.70s | 18.34s | 8.38s | +| **TTFT P50** | 0.36s | 0.056s | 0.33s | **0.051s** ⭐ | 0.094s | +| TTFT P90 | 10.97s | 11.90s | 6.95s | **2.64s** | 0.26s | + +✓ direct-to-D 占比从 v3 的 30-38% 涨到 v4 的 54-58% +✓ session 复用 +27% (1P7D) / +73% (2P6D) +✓ KV transfer 量 -15% (1P7D) / -36% (2P6D) +✓ TTFT P50 反超 baseline 46%（0.051s vs 0.094s） + +### Direct-to-D 路径全面碾压 baseline（KVC 真实价值） + +| Config | n | Lat P50 | Lat P90 | TTFT P50 | TTFT P90 | +|--------|:---:|:---:|:---:|:---:|:---:| +| baseline 8DP | 4381 | 0.66s | 3.65s | 0.094s | 0.256s | +| v4 1P7D direct-to-D | 2179 | 0.495s | 3.03s | 0.044s | 0.055s | +| **v4 2P6D direct-to-D** | **2348** | **0.499s** | **2.86s** | **0.043s** | **0.054s** | + +direct-to-D 子集相对 baseline： +- P50 快 24-30% +- P90 快 16-22% +- TTFT P50 快 54% +- TTFT P90 快 79% + +### 整体性能（去掉 errors 和 truncated）vs baseline + +| Config | clean | Mean | P50 | P90 | P99 | +|--------|:---:|:---:|:---:|:---:|:---:| +| baseline 8DP | 4381 | 1.45s | 0.66s | 3.65s | 8.38s | +| v4 2P6D | 4010 | 2.53s | 0.85s | 6.55s | 18.33s | + +vs baseline：P50 慢 28%、P90 慢 80%、P99 慢 119%。即使错误率为 0，整体仍输 baseline——根因是 35% 请求被推到 fallback 路径。 + +### 新瓶颈 1：35% 请求仍走 session-cap fallback + +抬到 16 后真实瓶颈是 capacity-based 计算：`min(16, usable_capacity_tokens // target_tokens)`。 +- `target_tokens = input + output`，agentic 里常见 50-100K +- D 的 KV pool ≈ 100-150K tokens（80GB H100, mem_fraction=0.835） +- `usable / target` = 1-2，远没到 16 → 真实 cap 是 capacity 算出来的小数字 + +要解决必须改 capacity-based 估算逻辑（或上方案 D，让 D 自己决定）。 + +### 新瓶颈 2：9-10% errors（mooncake 传输超时） + +P-side log 显示： + +``` +KVTransferError: Failed to send kv chunk of to 10.45.7.165:40319 +Sync batch data transfer timeout after 32722558107ns (32 秒超时) +Decode instance could be dead, remote mooncake session ... is not alive +``` + +特征： +- 所有 errors 在 run 的 44.8% 之后出现（系统压力累积） +- 98% errors 集中在 turn ≥ 31（大 input 的请求） +- v3 cap=4 时 1P7D 已有 363 errors（仅 1 个 D 集中受冲击），v4 cap=16 把压力均匀分布但量级更大 + +是 mooncake TCP loopback 在并发上去后撞超时，**不是 SGLang 逻辑 bug**。修复方向： +1. 加长 mooncake transfer timeout（现在 32s） +2. 限制并发 inflight transfer 数量 +3. 改用 RDMA（loopback 是单机模拟，生产环境换真 RDMA） +4. chunked KV transfer + +## 后续可以考虑的更深方案：让 D 自己决定 admission（方案 D） v4 的硬 cap 抬高只是把数字调大，实际容量管理还是 replay 自己估算。代码里 `replay.py:_decode_session_soft_cap` 用 `target_tokens = input + output`（基于当前请求的 size）估算每个 session footprint，但： - agentic context 越攒越长，target_tokens 动态增长，cap 会随之缩小 diff --git a/outputs/qwen3-30b-tp1-v3-kvaware/exp1_1p7d_kvc_kvaware_summary.json b/outputs/qwen3-30b-tp1-v3-kvaware/exp1_1p7d_kvc_kvaware_summary.json new file mode 100644 index 0000000..cfdaf94 --- /dev/null +++ b/outputs/qwen3-30b-tp1-v3-kvaware/exp1_1p7d_kvc_kvaware_summary.json @@ -0,0 +1,88 @@ +{ + "actual_output_tokens_stats": { + "count": 4086.0, + "mean": 213.95105237395987, + "p50": 83.0, + "p90": 562.0, + "p99": 1346.0 + }, + "cache_hit_request_count": 3929, + "cached_tokens_stats": { + "count": 4449.0, + "mean": 22635.924702180266, + "p50": 20010.0, + "p90": 48002.0, + "p99": 65424.0 + }, + "decode_request_priorities": {}, + "error_count": 363, + "execution_modes": { + "kvcache-centric": 363, + "kvcache-direct-to-d-session": 1716, + "pd-router-d-session-reseed": 23, + "pd-router-fallback-d-backpressure": 12, + "pd-router-fallback-large-append": 5, + "pd-router-fallback-large-append-seed-filter-early-turn": 51, + "pd-router-fallback-large-append-session-cap": 2148, + "pd-router-fallback-no-d-capacity": 7, + "pd-router-fallback-session-cap": 32, + "pd-router-large-append-reseed": 39, + "pd-router-large-append-reseed-after-eviction": 2, + "pd-router-turn1-d-backpressure": 1, + "pd-router-turn1-no-d-capacity": 3, + "pd-router-turn1-seed": 34, + "pd-router-turn1-session-cap": 13 + }, + "latency_stats_s": { + "count": 4086.0, + "mean": 4.8753733304192455, + "p50": 1.754677688702941, + "p90": 12.66968655679375, + "p99": 28.717210091650486 + }, + "mechanisms": { + "kvcache-centric": 4449 + }, + "per_decode_load": { + "decode-0": 616, + "decode-1": 658, + "decode-2": 674, + "decode-3": 582, + "decode-4": 656, + "decode-5": 662, + "decode-6": 601 + }, + "per_prefill_load": { + "prefill-0": 4449 + }, + "prefill_request_priorities": { + "-100": 98, + "100": 2272 + }, + "re_prefill_count": 0, + "request_count": 4449, + "reuse_expected_count": 4397, + "reuse_observed_count": 4397, + "router_url": "http://127.0.0.1:8000", + "session_reset_count": 0, + "session_reused_count": 1716, + "total_actual_kv_transfer_blocks": 62123, + "total_cached_tokens": 100707229, + "total_kv_transfer_blocks": 105235, + "tpot_stats_s": { + "count": 4086.0, + "mean": 0.005829451223571163, + "p50": 0.005684156496173296, + "p90": 0.007143743503740225, + "p99": 0.008634991403068266 + }, + "trace_path": "outputs/qwen3-30b-tp1-v3-kvaware/kvcache-centric-kv-aware-worker-admission-20260428T095141Z/sampled-trace.jsonl", + "truncated_request_count": 42, + "ttft_stats_s": { + "count": 4086.0, + "mean": 3.5955862397812597, + "p50": 0.36274072993546724, + "p90": 10.972254231572151, + "p99": 27.433656523004174 + } +} \ No newline at end of file diff --git a/outputs/qwen3-30b-tp1-v3-kvaware/exp2_2p6d_kvc_kvaware_summary.json b/outputs/qwen3-30b-tp1-v3-kvaware/exp2_2p6d_kvc_kvaware_summary.json new file mode 100644 index 0000000..a584dc8 --- /dev/null +++ b/outputs/qwen3-30b-tp1-v3-kvaware/exp2_2p6d_kvc_kvaware_summary.json @@ -0,0 +1,85 @@ +{ + "actual_output_tokens_stats": { + "count": 4440.0, + "mean": 225.87972972972972, + "p50": 86.0, + "p90": 576.0, + "p99": 1347.0 + }, + "cache_hit_request_count": 4201, + "cached_tokens_stats": { + "count": 4449.0, + "mean": 24345.55787817487, + "p50": 21504.0, + "p90": 48792.0, + "p99": 69120.0 + }, + "decode_request_priorities": {}, + "error_count": 9, + "execution_modes": { + "kvcache-centric": 9, + "kvcache-direct-to-d-session": 1358, + "pd-router-d-session-reseed": 12, + "pd-router-fallback-d-backpressure": 2, + "pd-router-fallback-large-append-seed-filter-early-turn": 52, + "pd-router-fallback-large-append-session-cap": 2902, + "pd-router-fallback-session-cap": 25, + "pd-router-large-append-reseed": 34, + "pd-router-large-append-reseed-after-eviction": 4, + "pd-router-turn1-d-backpressure": 1, + "pd-router-turn1-seed": 30, + "pd-router-turn1-session-cap": 20 + }, + "latency_stats_s": { + "count": 4440.0, + "mean": 3.582334662846558, + "p50": 1.517257746309042, + "p90": 9.225348330102861, + "p99": 18.70269925892353 + }, + "mechanisms": { + "kvcache-centric": 4449 + }, + "per_decode_load": { + "decode-0": 710, + "decode-1": 630, + "decode-2": 763, + "decode-3": 737, + "decode-4": 879, + "decode-5": 730 + }, + "per_prefill_load": { + "prefill-0": 2225, + "prefill-1": 2224 + }, + "prefill_request_priorities": { + "-100": 80, + "100": 3002 + }, + "re_prefill_count": 0, + "request_count": 4449, + "reuse_expected_count": 4397, + "reuse_observed_count": 4397, + "router_url": "http://127.0.0.1:8000", + "session_reset_count": 0, + "session_reused_count": 1358, + "total_actual_kv_transfer_blocks": 78979, + "total_cached_tokens": 108313387, + "total_kv_transfer_blocks": 105235, + "tpot_stats_s": { + "count": 4440.0, + "mean": 0.005882534704321737, + "p50": 0.005807478777200416, + "p90": 0.00712956755887717, + "p99": 0.008372141476720572 + }, + "trace_path": "outputs/qwen3-30b-tp1-v3-kvaware/kvcache-centric-kv-aware-worker-admission-20260428T104343Z/sampled-trace.jsonl", + "truncated_request_count": 42, + "ttft_stats_s": { + "count": 4440.0, + "mean": 2.2045287611873334, + "p50": 0.32809355948120356, + "p90": 6.947275545448065, + "p99": 16.705802395939827 + } +} \ No newline at end of file diff --git a/outputs/qwen3-30b-tp1-v3-kvaware/sweep_results.txt b/outputs/qwen3-30b-tp1-v3-kvaware/sweep_results.txt new file mode 100644 index 0000000..464e9b6 --- /dev/null +++ b/outputs/qwen3-30b-tp1-v3-kvaware/sweep_results.txt @@ -0,0 +1,189 @@ +[2026-04-28 17:51:41] Starting TP1 v3 sweep (KVC with kv-aware policy) +[2026-04-28 17:51:41] Model: /mnt/kzlin/workflow/pd-hybrid/simm-swe-bench/models/Qwen3-30B-A3B-Instruct-2507 +[2026-04-28 17:51:41] Trace: outputs/qwen35-swebench-50sess.jsonl (4449 requests, 52 sessions) +[2026-04-28 17:51:41] Key change: --policy kv-aware for KVC (was --policy default in v2) +[2026-04-28 17:51:41] +[2026-04-28 17:51:41] === [EXP1] 1P7D KVC kv-aware === +[2026-04-28 18:43:43] === exp1_1p7d_kvc_kvaware COMPLETED === +[2026-04-28 18:43:43] Summary: +{ + "actual_output_tokens_stats": { + "count": 4086.0, + "mean": 213.95105237395987, + "p50": 83.0, + "p90": 562.0, + "p99": 1346.0 + }, + "cache_hit_request_count": 3929, + "cached_tokens_stats": { + "count": 4449.0, + "mean": 22635.924702180266, + "p50": 20010.0, + "p90": 48002.0, + "p99": 65424.0 + }, + "decode_request_priorities": {}, + "error_count": 363, + "execution_modes": { + "kvcache-centric": 363, + "kvcache-direct-to-d-session": 1716, + "pd-router-d-session-reseed": 23, + "pd-router-fallback-d-backpressure": 12, + "pd-router-fallback-large-append": 5, + "pd-router-fallback-large-append-seed-filter-early-turn": 51, + "pd-router-fallback-large-append-session-cap": 2148, + "pd-router-fallback-no-d-capacity": 7, + "pd-router-fallback-session-cap": 32, + "pd-router-large-append-reseed": 39, + "pd-router-large-append-reseed-after-eviction": 2, + "pd-router-turn1-d-backpressure": 1, + "pd-router-turn1-no-d-capacity": 3, + "pd-router-turn1-seed": 34, + "pd-router-turn1-session-cap": 13 + }, + "latency_stats_s": { + "count": 4086.0, + "mean": 4.8753733304192455, + "p50": 1.754677688702941, + "p90": 12.66968655679375, + "p99": 28.717210091650486 + }, + "mechanisms": { + "kvcache-centric": 4449 + }, + "per_decode_load": { + "decode-0": 616, + "decode-1": 658, + "decode-2": 674, + "decode-3": 582, + "decode-4": 656, + "decode-5": 662, + "decode-6": 601 + }, + "per_prefill_load": { + "prefill-0": 4449 + }, + "prefill_request_priorities": { + "-100": 98, + "100": 2272 + }, + "re_prefill_count": 0, + "request_count": 4449, + "reuse_expected_count": 4397, + "reuse_observed_count": 4397, + "router_url": "http://127.0.0.1:8000", + "session_reset_count": 0, + "session_reused_count": 1716, + "total_actual_kv_transfer_blocks": 62123, + "total_cached_tokens": 100707229, + "total_kv_transfer_blocks": 105235, + "tpot_stats_s": { + "count": 4086.0, + "mean": 0.005829451223571163, + "p50": 0.005684156496173296, + "p90": 0.007143743503740225, + "p99": 0.008634991403068266 + }, + "trace_path": "outputs/qwen3-30b-tp1-v3-kvaware/kvcache-centric-kv-aware-worker-admission-20260428T095141Z/sampled-trace.jsonl", + "truncated_request_count": 42, + "ttft_stats_s": { + "count": 4086.0, + "mean": 3.5955862397812597, + "p50": 0.36274072993546724, + "p90": 10.972254231572151, + "p99": 27.433656523004174 + } +} +[2026-04-28 18:43:43] Saved to outputs/qwen3-30b-tp1-v3-kvaware/exp1_1p7d_kvc_kvaware_summary.json + exp1_1p7d_kvc_kvaware_metrics.jsonl +[2026-04-28 18:43:43] +[2026-04-28 18:43:43] === [EXP2] 2P6D KVC kv-aware === +[2026-04-28 19:30:38] === exp2_2p6d_kvc_kvaware COMPLETED === +[2026-04-28 19:30:38] Summary: +{ + "actual_output_tokens_stats": { + "count": 4440.0, + "mean": 225.87972972972972, + "p50": 86.0, + "p90": 576.0, + "p99": 1347.0 + }, + "cache_hit_request_count": 4201, + "cached_tokens_stats": { + "count": 4449.0, + "mean": 24345.55787817487, + "p50": 21504.0, + "p90": 48792.0, + "p99": 69120.0 + }, + "decode_request_priorities": {}, + "error_count": 9, + "execution_modes": { + "kvcache-centric": 9, + "kvcache-direct-to-d-session": 1358, + "pd-router-d-session-reseed": 12, + "pd-router-fallback-d-backpressure": 2, + "pd-router-fallback-large-append-seed-filter-early-turn": 52, + "pd-router-fallback-large-append-session-cap": 2902, + "pd-router-fallback-session-cap": 25, + "pd-router-large-append-reseed": 34, + "pd-router-large-append-reseed-after-eviction": 4, + "pd-router-turn1-d-backpressure": 1, + "pd-router-turn1-seed": 30, + "pd-router-turn1-session-cap": 20 + }, + "latency_stats_s": { + "count": 4440.0, + "mean": 3.582334662846558, + "p50": 1.517257746309042, + "p90": 9.225348330102861, + "p99": 18.70269925892353 + }, + "mechanisms": { + "kvcache-centric": 4449 + }, + "per_decode_load": { + "decode-0": 710, + "decode-1": 630, + "decode-2": 763, + "decode-3": 737, + "decode-4": 879, + "decode-5": 730 + }, + "per_prefill_load": { + "prefill-0": 2225, + "prefill-1": 2224 + }, + "prefill_request_priorities": { + "-100": 80, + "100": 3002 + }, + "re_prefill_count": 0, + "request_count": 4449, + "reuse_expected_count": 4397, + "reuse_observed_count": 4397, + "router_url": "http://127.0.0.1:8000", + "session_reset_count": 0, + "session_reused_count": 1358, + "total_actual_kv_transfer_blocks": 78979, + "total_cached_tokens": 108313387, + "total_kv_transfer_blocks": 105235, + "tpot_stats_s": { + "count": 4440.0, + "mean": 0.005882534704321737, + "p50": 0.005807478777200416, + "p90": 0.00712956755887717, + "p99": 0.008372141476720572 + }, + "trace_path": "outputs/qwen3-30b-tp1-v3-kvaware/kvcache-centric-kv-aware-worker-admission-20260428T104343Z/sampled-trace.jsonl", + "truncated_request_count": 42, + "ttft_stats_s": { + "count": 4440.0, + "mean": 2.2045287611873334, + "p50": 0.32809355948120356, + "p90": 6.947275545448065, + "p99": 16.705802395939827 + } +} +[2026-04-28 19:30:38] Saved to outputs/qwen3-30b-tp1-v3-kvaware/exp2_2p6d_kvc_kvaware_summary.json + exp2_2p6d_kvc_kvaware_metrics.jsonl +[2026-04-28 19:30:38] +[2026-04-28 19:30:38] === ALL TP1 V3 SWEEP EXPERIMENTS DONE === diff --git a/outputs/qwen3-30b-tp1-v4-cap16/exp1_1p7d_kvc_cap16_summary.json b/outputs/qwen3-30b-tp1-v4-cap16/exp1_1p7d_kvc_cap16_summary.json new file mode 100644 index 0000000..45bdaba --- /dev/null +++ b/outputs/qwen3-30b-tp1-v4-cap16/exp1_1p7d_kvc_cap16_summary.json @@ -0,0 +1,88 @@ +{ + "actual_output_tokens_stats": { + "count": 4014.0, + "mean": 215.048081714001, + "p50": 83.0, + "p90": 570.0, + "p99": 1343.0 + }, + "cache_hit_request_count": 3865, + "cached_tokens_stats": { + "count": 4449.0, + "mean": 21373.60867610699, + "p50": 18429.0, + "p90": 45643.0, + "p99": 65088.0 + }, + "decode_request_priorities": {}, + "error_count": 435, + "execution_modes": { + "kvcache-centric": 435, + "kvcache-direct-to-d-session": 2180, + "pd-router-d-session-reseed": 44, + "pd-router-d-session-reseed-after-eviction": 1, + "pd-router-fallback-d-backpressure": 36, + "pd-router-fallback-large-append": 35, + "pd-router-fallback-large-append-seed-filter-early-turn": 52, + "pd-router-fallback-large-append-session-cap": 1500, + "pd-router-fallback-no-d-capacity": 13, + "pd-router-fallback-session-cap": 43, + "pd-router-large-append-reseed": 55, + "pd-router-large-append-reseed-after-eviction": 3, + "pd-router-turn1-d-backpressure": 1, + "pd-router-turn1-no-d-capacity": 5, + "pd-router-turn1-seed": 46 + }, + "latency_stats_s": { + "count": 4014.0, + "mean": 4.214657033050009, + "p50": 1.0827504023909569, + "p90": 13.380241627804935, + "p99": 24.453291333280504 + }, + "mechanisms": { + "kvcache-centric": 4449 + }, + "per_decode_load": { + "decode-0": 690, + "decode-1": 599, + "decode-2": 660, + "decode-3": 584, + "decode-4": 606, + "decode-5": 646, + "decode-6": 664 + }, + "per_prefill_load": { + "prefill-0": 4449 + }, + "prefill_request_priorities": { + "-100": 149, + "100": 1685 + }, + "re_prefill_count": 0, + "request_count": 4449, + "reuse_expected_count": 4397, + "reuse_observed_count": 4397, + "router_url": "http://127.0.0.1:8000", + "session_reset_count": 0, + "session_reused_count": 2180, + "total_actual_kv_transfer_blocks": 52857, + "total_cached_tokens": 95091185, + "total_kv_transfer_blocks": 105235, + "tpot_stats_s": { + "count": 4014.0, + "mean": 0.005804301410418847, + "p50": 0.005607025208882987, + "p90": 0.007293824862528552, + "p99": 0.008864479259402893 + }, + "trace_path": "outputs/qwen3-30b-tp1-v4-cap16/kvcache-centric-kv-aware-worker-admission-20260428T125022Z/sampled-trace.jsonl", + "truncated_request_count": 43, + "ttft_stats_s": { + "count": 4014.0, + "mean": 2.915135478307124, + "p50": 0.05643345229327679, + "p90": 11.900803190656006, + "p99": 22.758968392387033 + } +} \ No newline at end of file diff --git a/outputs/qwen3-30b-tp1-v4-cap16/exp2_2p6d_kvc_cap16_summary.json b/outputs/qwen3-30b-tp1-v4-cap16/exp2_2p6d_kvc_cap16_summary.json new file mode 100644 index 0000000..5e90c49 --- /dev/null +++ b/outputs/qwen3-30b-tp1-v4-cap16/exp2_2p6d_kvc_cap16_summary.json @@ -0,0 +1,86 @@ +{ + "actual_output_tokens_stats": { + "count": 4046.0, + "mean": 224.65002471576867, + "p50": 84.0, + "p90": 576.0, + "p99": 1349.0 + }, + "cache_hit_request_count": 3925, + "cached_tokens_stats": { + "count": 4449.0, + "mean": 22852.7439874129, + "p50": 19584.0, + "p90": 49009.0, + "p99": 67320.0 + }, + "decode_request_priorities": {}, + "error_count": 403, + "execution_modes": { + "kvcache-centric": 403, + "kvcache-direct-to-d-session": 2348, + "pd-router-d-session-reseed": 28, + "pd-router-fallback-d-backpressure": 7, + "pd-router-fallback-large-append": 68, + "pd-router-fallback-large-append-seed-filter-early-turn": 45, + "pd-router-fallback-large-append-session-cap": 1403, + "pd-router-fallback-no-d-capacity": 9, + "pd-router-fallback-session-cap": 25, + "pd-router-large-append-reseed": 57, + "pd-router-large-append-reseed-after-eviction": 6, + "pd-router-turn1-no-d-capacity": 1, + "pd-router-turn1-seed": 49 + }, + "latency_stats_s": { + "count": 4046.0, + "mean": 2.505981629502371, + "p50": 0.8372491216287017, + "p90": 6.5139341270551085, + "p99": 18.335972285829484 + }, + "mechanisms": { + "kvcache-centric": 4449 + }, + "per_decode_load": { + "decode-0": 767, + "decode-1": 680, + "decode-2": 906, + "decode-3": 818, + "decode-4": 800, + "decode-5": 478 + }, + "per_prefill_load": { + "prefill-0": 2225, + "prefill-1": 2224 + }, + "prefill_request_priorities": { + "-100": 140, + "100": 1558 + }, + "re_prefill_count": 0, + "request_count": 4449, + "reuse_expected_count": 4397, + "reuse_observed_count": 4397, + "router_url": "http://127.0.0.1:8000", + "session_reset_count": 0, + "session_reused_count": 2348, + "total_actual_kv_transfer_blocks": 50727, + "total_cached_tokens": 101671858, + "total_kv_transfer_blocks": 105235, + "tpot_stats_s": { + "count": 4046.0, + "mean": 0.005708743129332261, + "p50": 0.005565466725497757, + "p90": 0.006912594398356141, + "p99": 0.008102089307750717 + }, + "trace_path": "outputs/qwen3-30b-tp1-v4-cap16/kvcache-centric-kv-aware-worker-admission-20260428T134057Z/sampled-trace.jsonl", + "truncated_request_count": 36, + "ttft_stats_s": { + "count": 4046.0, + "mean": 1.1653790952959129, + "p50": 0.05140436999499798, + "p90": 2.6447059931233525, + "p99": 15.121314341202378 + } +} \ No newline at end of file diff --git a/outputs/qwen3-30b-tp1-v4-cap16/sweep_results.txt b/outputs/qwen3-30b-tp1-v4-cap16/sweep_results.txt new file mode 100644 index 0000000..7cff9a8 --- /dev/null +++ b/outputs/qwen3-30b-tp1-v4-cap16/sweep_results.txt @@ -0,0 +1,190 @@ +[2026-04-28 20:50:21] Starting TP1 v4 sweep (KVC kv-aware, session soft_cap raised 4->16) +[2026-04-28 20:50:21] Model: /mnt/kzlin/workflow/pd-hybrid/simm-swe-bench/models/Qwen3-30B-A3B-Instruct-2507 +[2026-04-28 20:50:21] Trace: outputs/qwen35-swebench-50sess.jsonl (4449 requests, 52 sessions) +[2026-04-28 20:50:21] Key change: _decode_session_soft_cap now min(16, ...) instead of min(4, ...) +[2026-04-28 20:50:21] +[2026-04-28 20:50:21] === [EXP1] 1P7D KVC kv-aware cap=16 === +[2026-04-28 21:40:57] === exp1_1p7d_kvc_cap16 COMPLETED === +[2026-04-28 21:40:57] Summary: +{ + "actual_output_tokens_stats": { + "count": 4014.0, + "mean": 215.048081714001, + "p50": 83.0, + "p90": 570.0, + "p99": 1343.0 + }, + "cache_hit_request_count": 3865, + "cached_tokens_stats": { + "count": 4449.0, + "mean": 21373.60867610699, + "p50": 18429.0, + "p90": 45643.0, + "p99": 65088.0 + }, + "decode_request_priorities": {}, + "error_count": 435, + "execution_modes": { + "kvcache-centric": 435, + "kvcache-direct-to-d-session": 2180, + "pd-router-d-session-reseed": 44, + "pd-router-d-session-reseed-after-eviction": 1, + "pd-router-fallback-d-backpressure": 36, + "pd-router-fallback-large-append": 35, + "pd-router-fallback-large-append-seed-filter-early-turn": 52, + "pd-router-fallback-large-append-session-cap": 1500, + "pd-router-fallback-no-d-capacity": 13, + "pd-router-fallback-session-cap": 43, + "pd-router-large-append-reseed": 55, + "pd-router-large-append-reseed-after-eviction": 3, + "pd-router-turn1-d-backpressure": 1, + "pd-router-turn1-no-d-capacity": 5, + "pd-router-turn1-seed": 46 + }, + "latency_stats_s": { + "count": 4014.0, + "mean": 4.214657033050009, + "p50": 1.0827504023909569, + "p90": 13.380241627804935, + "p99": 24.453291333280504 + }, + "mechanisms": { + "kvcache-centric": 4449 + }, + "per_decode_load": { + "decode-0": 690, + "decode-1": 599, + "decode-2": 660, + "decode-3": 584, + "decode-4": 606, + "decode-5": 646, + "decode-6": 664 + }, + "per_prefill_load": { + "prefill-0": 4449 + }, + "prefill_request_priorities": { + "-100": 149, + "100": 1685 + }, + "re_prefill_count": 0, + "request_count": 4449, + "reuse_expected_count": 4397, + "reuse_observed_count": 4397, + "router_url": "http://127.0.0.1:8000", + "session_reset_count": 0, + "session_reused_count": 2180, + "total_actual_kv_transfer_blocks": 52857, + "total_cached_tokens": 95091185, + "total_kv_transfer_blocks": 105235, + "tpot_stats_s": { + "count": 4014.0, + "mean": 0.005804301410418847, + "p50": 0.005607025208882987, + "p90": 0.007293824862528552, + "p99": 0.008864479259402893 + }, + "trace_path": "outputs/qwen3-30b-tp1-v4-cap16/kvcache-centric-kv-aware-worker-admission-20260428T125022Z/sampled-trace.jsonl", + "truncated_request_count": 43, + "ttft_stats_s": { + "count": 4014.0, + "mean": 2.915135478307124, + "p50": 0.05643345229327679, + "p90": 11.900803190656006, + "p99": 22.758968392387033 + } +} +[2026-04-28 21:40:57] Saved to outputs/qwen3-30b-tp1-v4-cap16/exp1_1p7d_kvc_cap16_summary.json + exp1_1p7d_kvc_cap16_metrics.jsonl +[2026-04-28 21:40:57] +[2026-04-28 21:40:57] === [EXP2] 2P6D KVC kv-aware cap=16 === +[2026-04-28 22:27:53] === exp2_2p6d_kvc_cap16 COMPLETED === +[2026-04-28 22:27:53] Summary: +{ + "actual_output_tokens_stats": { + "count": 4046.0, + "mean": 224.65002471576867, + "p50": 84.0, + "p90": 576.0, + "p99": 1349.0 + }, + "cache_hit_request_count": 3925, + "cached_tokens_stats": { + "count": 4449.0, + "mean": 22852.7439874129, + "p50": 19584.0, + "p90": 49009.0, + "p99": 67320.0 + }, + "decode_request_priorities": {}, + "error_count": 403, + "execution_modes": { + "kvcache-centric": 403, + "kvcache-direct-to-d-session": 2348, + "pd-router-d-session-reseed": 28, + "pd-router-fallback-d-backpressure": 7, + "pd-router-fallback-large-append": 68, + "pd-router-fallback-large-append-seed-filter-early-turn": 45, + "pd-router-fallback-large-append-session-cap": 1403, + "pd-router-fallback-no-d-capacity": 9, + "pd-router-fallback-session-cap": 25, + "pd-router-large-append-reseed": 57, + "pd-router-large-append-reseed-after-eviction": 6, + "pd-router-turn1-no-d-capacity": 1, + "pd-router-turn1-seed": 49 + }, + "latency_stats_s": { + "count": 4046.0, + "mean": 2.505981629502371, + "p50": 0.8372491216287017, + "p90": 6.5139341270551085, + "p99": 18.335972285829484 + }, + "mechanisms": { + "kvcache-centric": 4449 + }, + "per_decode_load": { + "decode-0": 767, + "decode-1": 680, + "decode-2": 906, + "decode-3": 818, + "decode-4": 800, + "decode-5": 478 + }, + "per_prefill_load": { + "prefill-0": 2225, + "prefill-1": 2224 + }, + "prefill_request_priorities": { + "-100": 140, + "100": 1558 + }, + "re_prefill_count": 0, + "request_count": 4449, + "reuse_expected_count": 4397, + "reuse_observed_count": 4397, + "router_url": "http://127.0.0.1:8000", + "session_reset_count": 0, + "session_reused_count": 2348, + "total_actual_kv_transfer_blocks": 50727, + "total_cached_tokens": 101671858, + "total_kv_transfer_blocks": 105235, + "tpot_stats_s": { + "count": 4046.0, + "mean": 0.005708743129332261, + "p50": 0.005565466725497757, + "p90": 0.006912594398356141, + "p99": 0.008102089307750717 + }, + "trace_path": "outputs/qwen3-30b-tp1-v4-cap16/kvcache-centric-kv-aware-worker-admission-20260428T134057Z/sampled-trace.jsonl", + "truncated_request_count": 36, + "ttft_stats_s": { + "count": 4046.0, + "mean": 1.1653790952959129, + "p50": 0.05140436999499798, + "p90": 2.6447059931233525, + "p99": 15.121314341202378 + } +} +[2026-04-28 22:27:53] Saved to outputs/qwen3-30b-tp1-v4-cap16/exp2_2p6d_kvc_cap16_summary.json + exp2_2p6d_kvc_cap16_metrics.jsonl +[2026-04-28 22:27:53] +[2026-04-28 22:27:53] === ALL TP1 V4 SWEEP EXPERIMENTS DONE === diff --git a/scripts/analysis/analyze_errors.py b/scripts/analysis/analyze_errors.py new file mode 100644 index 0000000..e837712 --- /dev/null +++ b/scripts/analysis/analyze_errors.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python3 +"""Deep dive into v4 errors: which path, which D, which session, which turn.""" +import json +import numpy as np +from pathlib import Path +from collections import Counter, defaultdict + +BASE = Path(__file__).parent + +def load_rows(jsonl_path): + rows = [] + with open(jsonl_path) as f: + for line in f: + rows.append(json.loads(line)) + return rows + +# Compare v3 and v4 errors +for label, path in [ + ("v3 1P7D", BASE.parent / "qwen3-30b-tp1-v3-kvaware/exp1_1p7d_kvc_kvaware_metrics.jsonl"), + ("v4 1P7D", BASE / "exp1_1p7d_kvc_cap16_metrics.jsonl"), + ("v3 2P6D", BASE.parent / "qwen3-30b-tp1-v3-kvaware/exp2_2p6d_kvc_kvaware_metrics.jsonl"), + ("v4 2P6D", BASE / "exp2_2p6d_kvc_cap16_metrics.jsonl"), +]: + if not path.exists(): + print(f"\nSKIP {label}: {path} not found") + continue + rows = load_rows(path) + err = [r for r in rows if r.get("error") is not None] + print(f"\n========== {label} ({len(err)} errors / {len(rows)} total = {len(err)/len(rows)*100:.1f}%) ==========") + + # Error finish_reason distribution + fr_counter = Counter() + for r in err: + fr = str(r.get("finish_reason") or r.get("error") or "?") + fr_counter[fr[:80]] += 1 + print(f"finish_reason distribution:") + for fr, cnt in fr_counter.most_common(): + print(f" {cnt:>4}x {fr}") + + # Errors by execution mode (these are aborted before mode assignment usually) + mode_counter = Counter(r.get("execution_mode", "?") for r in err) + print(f"\nerror by execution_mode:") + for mode, cnt in mode_counter.most_common(): + print(f" {cnt:>4}x {mode}") + + # Errors per D worker + dw_counter = Counter(r.get("assigned_decode_node", "?") for r in err) + print(f"\nerror per assigned_decode_node:") + for dw, cnt in dw_counter.most_common(): + print(f" {cnt:>4}x {dw}") + + # Errors by turn distribution + turn_counter = Counter(r.get("turn_id", -1) for r in err) + early = sum(c for t, c in turn_counter.items() if t <= 5) + mid = sum(c for t, c in turn_counter.items() if 5 < t <= 30) + late = sum(c for t, c in turn_counter.items() if t > 30) + print(f"\nerror by turn: early(0-5)={early} mid(6-30)={mid} late(31+)={late}") + + # Per-session error rate + per_sess_err = defaultdict(int) + per_sess_total = defaultdict(int) + for r in rows: + per_sess_total[r["session_id"]] += 1 + if r.get("error") is not None: + per_sess_err[r["session_id"]] += 1 + sess_with_err = [(sid, per_sess_err[sid], per_sess_total[sid]) for sid in per_sess_err] + sess_with_err.sort(key=lambda x: -x[1]) + print(f"\ntop 5 sessions by error count:") + for sid, e, t in sess_with_err[:5]: + print(f" session {sid}: {e}/{t} errors ({e/t*100:.0f}%)") + + # Errors timeline: are they bursty? + err_ts = sorted([r.get("trace_timestamp_s", 0) for r in err]) + if err_ts: + first_ts = err_ts[0] + last_ts = err_ts[-1] + all_ts = sorted([r.get("trace_timestamp_s", 0) for r in rows]) + first_all = all_ts[0] + last_all = all_ts[-1] + run_duration = last_all - first_all + err_first_pct = (err_ts[0] - first_all) / run_duration * 100 if run_duration > 0 else 0 + err_last_pct = (err_ts[-1] - first_all) / run_duration * 100 if run_duration > 0 else 0 + print(f"\nerror time range (% of run): {err_first_pct:.1f}% - {err_last_pct:.1f}%") diff --git a/scripts/analysis/analyze_v3.py b/scripts/analysis/analyze_v3.py new file mode 100644 index 0000000..5e29919 --- /dev/null +++ b/scripts/analysis/analyze_v3.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python3 +"""Analyze v3 (kv-aware) results — find why fallback-large-append-session-cap dominates.""" +import json +import numpy as np +from pathlib import Path +from collections import Counter, defaultdict + +BASE = Path(__file__).parent + +def load_rows(jsonl_path): + rows = [] + with open(jsonl_path) as f: + for line in f: + rows.append(json.loads(line)) + return rows + +exp1 = load_rows(BASE / "exp1_1p7d_kvc_kvaware_metrics.jsonl") +exp2 = load_rows(BASE / "exp2_2p6d_kvc_kvaware_metrics.jsonl") + +for name, rows in [("Exp1 1P7D", exp1), ("Exp2 2P6D", exp2)]: + print(f"\n========== {name} ==========") + ok = [r for r in rows if r.get("error") is None] + + # Execution mode breakdown by latency + modes = Counter(r["execution_mode"] for r in ok) + print(f"\nExecution modes (n={len(ok)}):") + for mode, count in modes.most_common(): + mode_rows = [r for r in ok if r["execution_mode"] == mode] + lats = [r["latency_s"] for r in mode_rows] + ttfts = [r["ttft_s"] for r in mode_rows] + print(f" {mode}: n={count} ({count/len(ok)*100:.1f}%) " + f"lat P50={np.percentile(lats,50):.3f}s P90={np.percentile(lats,90):.3f}s | " + f"ttft P50={np.percentile(ttfts,50):.3f}s P90={np.percentile(ttfts,90):.3f}s") + + # Per-D session distribution + per_d_sessions = defaultdict(set) + for r in ok: + d = r.get("assigned_decode_node", "?") + per_d_sessions[d].add(r["session_id"]) + print(f"\nSessions per D worker:") + for d in sorted(per_d_sessions.keys()): + print(f" {d}: {len(per_d_sessions[d])} unique sessions") + + # session-cap fallback analysis + sc_rows = [r for r in ok if r["execution_mode"] == "pd-router-fallback-large-append-session-cap"] + if sc_rows: + print(f"\nSession-cap fallback details (n={len(sc_rows)}):") + # Which sessions hit this most? + sc_per_sess = Counter(r["session_id"] for r in sc_rows) + print(f" Sessions hitting session-cap (top 5):") + for sid, cnt in sc_per_sess.most_common(5): + print(f" session {sid}: {cnt} times") + # Per-D distribution + sc_per_d = Counter(r.get("assigned_decode_node", "?") for r in sc_rows) + print(f" Per-D distribution: {dict(sc_per_d.most_common())}") + # Input length distribution + inp = [r.get("input_length", 0) for r in sc_rows] + print(f" Input length: P50={np.percentile(inp,50):.0f} P90={np.percentile(inp,90):.0f}") + # Turn distribution + turns = Counter(r.get("turn_id", -1) for r in sc_rows) + print(f" Turn distribution (top 5): {dict(turns.most_common(5))}") + + # Direct-to-D analysis (ideal path) + dd_rows = [r for r in ok if r["execution_mode"] == "kvcache-direct-to-d-session"] + if dd_rows: + lats = [r["latency_s"] for r in dd_rows] + ttfts = [r["ttft_s"] for r in dd_rows] + kv_blocks = [r.get("actual_kv_transfer_blocks", 0) for r in dd_rows] + cached = [r.get("cached_tokens", 0) for r in dd_rows] + print(f"\nDirect-to-D details (n={len(dd_rows)}):") + print(f" lat P50={np.percentile(lats,50):.3f}s P90={np.percentile(lats,90):.3f}s P99={np.percentile(lats,99):.3f}s") + print(f" ttft P50={np.percentile(ttfts,50):.3f}s P90={np.percentile(ttfts,90):.3f}s") + print(f" KV transfer: P50={np.percentile(kv_blocks,50):.0f} (should be 0 — no P involved)") + print(f" cached_tokens P50={np.percentile(cached,50):.0f}") + + # Sessions: how many turns each, how many used direct-to-d + print(f"\nPer-session direct-to-D rate (top 10 by total turns):") + per_sess = defaultdict(list) + for r in ok: + per_sess[r["session_id"]].append(r) + sess_stats = [] + for sid, sreqs in per_sess.items(): + total = len(sreqs) + dd = sum(1 for r in sreqs if r["execution_mode"] == "kvcache-direct-to-d-session") + sc = sum(1 for r in sreqs if "session-cap" in r["execution_mode"]) + sess_stats.append((sid, total, dd, sc)) + sess_stats.sort(key=lambda x: -x[1]) + for sid, total, dd, sc in sess_stats[:10]: + print(f" session {sid}: {total} turns, {dd} direct-to-D ({dd/total*100:.0f}%), {sc} session-cap fallback ({sc/total*100:.0f}%)") diff --git a/scripts/analysis/analyze_v4.py b/scripts/analysis/analyze_v4.py new file mode 100644 index 0000000..3367330 --- /dev/null +++ b/scripts/analysis/analyze_v4.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python3 +"""V4 results analysis: errors, execution modes, latency by mode.""" +import json +import numpy as np +from pathlib import Path +from collections import Counter + +BASE = Path(__file__).parent + +def load_rows(jsonl_path): + rows = [] + with open(jsonl_path) as f: + for line in f: + rows.append(json.loads(line)) + return rows + +for name, path in [ + ("Exp1 1P7D cap=16", BASE / "exp1_1p7d_kvc_cap16_metrics.jsonl"), + ("Exp2 2P6D cap=16", BASE / "exp2_2p6d_kvc_cap16_metrics.jsonl"), +]: + rows = load_rows(path) + print(f"\n========== {name} ==========") + ok = [r for r in rows if r.get("error") is None] + err = [r for r in rows if r.get("error") is not None] + print(f"Total: {len(rows)}, OK: {len(ok)}, Errors: {len(err)}") + + # Errors finish_reason + if err: + finish_reasons = Counter() + for r in err: + fr = str(r.get("finish_reason") or r.get("error") or "?") + # Truncate long messages + short = fr[:120] + finish_reasons[short] += 1 + print(f"\nError finish_reasons (top 5):") + for fr, cnt in finish_reasons.most_common(5): + print(f" {cnt}x: {fr}") + + # Execution mode latency breakdown + modes = Counter(r["execution_mode"] for r in ok) + print(f"\nTop execution modes by latency:") + print(f"{'mode':<55}{'n':<8}{'%':<8}{'P50 lat':<10}{'P90 lat':<10}{'TTFT P50':<10}") + for mode, count in modes.most_common(8): + mode_rows = [r for r in ok if r["execution_mode"] == mode] + lats = [r["latency_s"] for r in mode_rows] + ttfts = [r["ttft_s"] for r in mode_rows] + print(f" {mode:<53}{count:<8}{count/len(ok)*100:>5.1f}% {np.percentile(lats,50):>7.3f}s {np.percentile(lats,90):>7.3f}s {np.percentile(ttfts,50):>7.3f}s") + + # Per-D load + per_d = Counter(r.get("assigned_decode_node", "?") for r in ok) + print(f"\nPer-D load: max/min ratio = {max(per_d.values())/max(min(per_d.values()),1):.2f}x") + print(f" {dict(per_d.most_common())}") diff --git a/scripts/analysis/compare_no_error.py b/scripts/analysis/compare_no_error.py new file mode 100644 index 0000000..4384726 --- /dev/null +++ b/scripts/analysis/compare_no_error.py @@ -0,0 +1,136 @@ +#!/usr/bin/env python3 +"""Compare KVC variants vs baseline, EXCLUDING errors and truncated requests.""" +import json +import numpy as np +from pathlib import Path + +OUT = Path("/mnt/kzlin/workflow/pd-hybrid/agentic-pd-hybrid/outputs") + +DATASETS = [ + ("baseline 8DP", OUT / "qwen3-30b-tp1-v2-fixed/exp1_8way_dp_cache_aware_metrics.jsonl"), + ("v3 1P7D", OUT / "qwen3-30b-tp1-v3-kvaware/exp1_1p7d_kvc_kvaware_metrics.jsonl"), + ("v3 2P6D", OUT / "qwen3-30b-tp1-v3-kvaware/exp2_2p6d_kvc_kvaware_metrics.jsonl"), + ("v4 1P7D", OUT / "qwen3-30b-tp1-v4-cap16/exp1_1p7d_kvc_cap16_metrics.jsonl"), + ("v4 2P6D", OUT / "qwen3-30b-tp1-v4-cap16/exp2_2p6d_kvc_cap16_metrics.jsonl"), +] + +def load_rows(path): + rows = [] + with open(path) as f: + for line in f: + rows.append(json.loads(line)) + return rows + +def is_truncated(row): + a = row.get("actual_output_tokens") + r = row.get("requested_output_tokens") + if a is not None and r is not None and r > 1: + return a < r * 0.5 + return False + +def stats(values): + if not values: + return {"n": 0} + a = np.array(values) + return { + "n": len(a), + "mean": float(np.mean(a)), + "p50": float(np.percentile(a, 50)), + "p90": float(np.percentile(a, 90)), + "p99": float(np.percentile(a, 99)), + } + +def fmt(s, key): + if s["n"] == 0: + return "N/A" + v = s[key] + return f"{v:.3f}s" if v < 100 else f"{v:.1f}s" + +results = [] +for label, path in DATASETS: + if not path.exists(): + print(f"SKIP {label}") + continue + rows = load_rows(path) + total = len(rows) + err_n = sum(1 for r in rows if r.get("error") is not None) + trunc_n = sum(1 for r in rows if r.get("error") is None and is_truncated(r)) + + # Filter: error=None AND not truncated AND latency present + clean = [r for r in rows + if r.get("error") is None + and not is_truncated(r) + and r.get("latency_s") is not None] + + lats = [r["latency_s"] for r in clean] + ttfts = [r["ttft_s"] for r in clean if r.get("ttft_s") is not None] + + results.append({ + "label": label, + "total": total, + "err": err_n, + "trunc": trunc_n, + "clean_n": len(clean), + "lat": stats(lats), + "ttft": stats(ttfts), + }) + +# Print comparison table +print(f"\n{'='*100}") +print("LATENCY (excluding errors AND truncated)") +print(f"{'='*100}") +print(f"{'config':<16}{'total':>7}{'err':>6}{'trunc':>7}{'clean':>7} {'mean':>9}{'P50':>9}{'P90':>9}{'P99':>9}") +for r in results: + print(f"{r['label']:<16}{r['total']:>7}{r['err']:>6}{r['trunc']:>7}{r['clean_n']:>7} " + f"{fmt(r['lat'],'mean'):>9}{fmt(r['lat'],'p50'):>9}{fmt(r['lat'],'p90'):>9}{fmt(r['lat'],'p99'):>9}") + +print(f"\n{'='*100}") +print("TTFT (excluding errors AND truncated)") +print(f"{'='*100}") +print(f"{'config':<16}{'clean':>7} {'mean':>9}{'P50':>9}{'P90':>9}{'P99':>9}") +for r in results: + print(f"{r['label']:<16}{r['clean_n']:>7} " + f"{fmt(r['ttft'],'mean'):>9}{fmt(r['ttft'],'p50'):>9}{fmt(r['ttft'],'p90'):>9}{fmt(r['ttft'],'p99'):>9}") + +# Also: per-execution-mode breakdown for v4 only (the most interesting) +print(f"\n{'='*100}") +print("V4 2P6D: per-execution-mode (excluding errors and truncated)") +print(f"{'='*100}") +v4_2p6d = next((p for l, p in DATASETS if l == "v4 2P6D"), None) +if v4_2p6d: + rows = load_rows(v4_2p6d) + clean = [r for r in rows if r.get("error") is None and not is_truncated(r)] + from collections import Counter + modes = Counter(r["execution_mode"] for r in clean) + print(f"{'mode':<55}{'n':>7}{'%':>7} {'mean':>9}{'P50':>9}{'P90':>9}{'P99':>9}") + for mode, count in modes.most_common(10): + m_rows = [r for r in clean if r["execution_mode"] == mode] + s = stats([r["latency_s"] for r in m_rows]) + pct = count/len(clean)*100 + print(f" {mode:<53}{count:>7}{pct:>6.1f}% {fmt(s,'mean'):>9}{fmt(s,'p50'):>9}{fmt(s,'p90'):>9}{fmt(s,'p99'):>9}") + +# Also: WHAT IF we only count direct-to-D? (Pure KVC performance) +print(f"\n{'='*100}") +print("Pure KVC (kvcache-direct-to-d-session ONLY) vs Baseline") +print(f"{'='*100}") +for label, path in DATASETS: + if not path.exists() or "1P7D" not in label and "2P6D" not in label: + continue + rows = load_rows(path) + direct = [r for r in rows + if r.get("error") is None and not is_truncated(r) + and r.get("execution_mode") == "kvcache-direct-to-d-session"] + if not direct: + continue + s_lat = stats([r["latency_s"] for r in direct]) + s_ttft = stats([r["ttft_s"] for r in direct if r.get("ttft_s") is not None]) + print(f"{label:<16}n={s_lat['n']:>5} lat: P50={fmt(s_lat,'p50')} P90={fmt(s_lat,'p90')} ttft: P50={fmt(s_ttft,'p50')} P90={fmt(s_ttft,'p90')}") + +# Baseline for reference (already non-fallback by definition) +print() +baseline_path = OUT / "qwen3-30b-tp1-v2-fixed/exp1_8way_dp_cache_aware_metrics.jsonl" +baseline_rows = load_rows(baseline_path) +clean = [r for r in baseline_rows if r.get("error") is None and not is_truncated(r)] +s_lat = stats([r["latency_s"] for r in clean]) +s_ttft = stats([r["ttft_s"] for r in clean if r.get("ttft_s") is not None]) +print(f"{'baseline 8DP':<16}n={s_lat['n']:>5} lat: P50={fmt(s_lat,'p50')} P90={fmt(s_lat,'p90')} ttft: P50={fmt(s_ttft,'p50')} P90={fmt(s_ttft,'p90')}")