8d422c4301
Ran unified vs unified_v4 A/B on dash2 (8×H20, kv_both+DR-fix substrate, w600_r0.0015_st30_first600s trace). Key findings: - At 1x QPS (~1.3 req/s): zero migrations. pending_prefill_tokens is 0 for 95% of routing decisions because instances complete prefill before the next request arrives. The relative arm (src_pp > fleet_median*1.5) never fires. - At 2x QPS (~2.7 req/s): 4 migrations (0.5%). src_pp>0 rises to 24% of eligible decisions. Trigger correctly identifies genuinely overloaded instances (src_pp 13k–73k vs fleet median 3.8k–33k). Conclusion: mechanism is correct but migration benefit requires higher concurrency (scale-out or >3x QPS) where queue pressure makes the signal non-trivial. At single-node 8-instance scale, Pillar 1 (affinity routing) is sufficient and Pillar 2 gracefully degrades to no-op. Next: scale-out validation (16+ GPU) where session skew naturally concentrates load and triggers migration. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
110 lines
2.9 KiB
JSON
110 lines
2.9 KiB
JSON
{
|
|
"experiment": "migration_trigger_validation",
|
|
"date": "2026-05-30",
|
|
"hardware": "dash2, 8xH20, Qwen3-Coder-30B-A3B, TP=1",
|
|
"substrate": "kv_both + DR-fix (delay_free_blocks + VLLM_EVICT_SENT_BLOCKS gate)",
|
|
"arms": {
|
|
"unified_1x": {
|
|
"metrics": {
|
|
"ok": 807,
|
|
"total": 807,
|
|
"ttft_mean": 3.99,
|
|
"ttft_p50": 0.719,
|
|
"ttft_p90": 11.499,
|
|
"ttft_p99": 45.982,
|
|
"tpot_p90": 0.0239,
|
|
"e2e_p50": 2.265,
|
|
"e2e_p90": 24.507,
|
|
"e2e_p99": 71.233
|
|
},
|
|
"migrations": 0
|
|
},
|
|
"unified_v4_1x": {
|
|
"metrics": {
|
|
"ok": 807,
|
|
"total": 807,
|
|
"ttft_mean": 4.142,
|
|
"ttft_p50": 0.711,
|
|
"ttft_p90": 12.293,
|
|
"ttft_p99": 46.148,
|
|
"tpot_p90": 0.0217,
|
|
"e2e_p50": 2.293,
|
|
"e2e_p90": 23.955,
|
|
"e2e_p99": 75.915
|
|
},
|
|
"trigger_summary": {
|
|
"trace": "w600_r0.0015_st30_first600s.jsonl",
|
|
"qps_factor": 1,
|
|
"total_requests": 807,
|
|
"migrations_triggered": 13,
|
|
"size_floor_filtered": 552,
|
|
"eligible_requests": 255,
|
|
"slo_arm_true": 100,
|
|
"src_pp_nonzero": 22
|
|
}
|
|
},
|
|
"unified_v4_2x": {
|
|
"metrics": {
|
|
"ok": 807,
|
|
"total": 807,
|
|
"ttft_mean": 5.227,
|
|
"ttft_p50": 0.942,
|
|
"ttft_p90": 15.0,
|
|
"ttft_p99": 59.227,
|
|
"tpot_p90": 0.1087,
|
|
"e2e_p50": 5.035,
|
|
"e2e_p90": 39.401,
|
|
"e2e_p99": 163.032
|
|
},
|
|
"trigger_summary": {
|
|
"trace": "trace_2x.jsonl (timestamps / 2)",
|
|
"qps_factor": 2,
|
|
"total_requests": 807,
|
|
"migrations_triggered": 4,
|
|
"size_floor_filtered": 550,
|
|
"eligible_requests": 257,
|
|
"slo_arm_true": 133,
|
|
"src_pp_nonzero": 62,
|
|
"pending_prefill_p90_when_nonzero": 68131,
|
|
"migrated_details": [
|
|
{
|
|
"session_id": "1313181",
|
|
"input_length": 22686,
|
|
"new_local": 22686,
|
|
"src_pending_prefill": 13360,
|
|
"fleet_median_pp": 6680.0,
|
|
"proj_prefill_s": 5.15,
|
|
"target_idx": 5
|
|
},
|
|
{
|
|
"session_id": "1310590",
|
|
"input_length": 32440,
|
|
"new_local": 14520,
|
|
"src_pending_prefill": 57051,
|
|
"fleet_median_pp": 12630.5,
|
|
"proj_prefill_s": 10.22,
|
|
"target_idx": 4
|
|
},
|
|
{
|
|
"session_id": "1373431",
|
|
"input_length": 126340,
|
|
"new_local": 126340,
|
|
"src_pending_prefill": 73385,
|
|
"fleet_median_pp": 33294.5,
|
|
"proj_prefill_s": 28.53,
|
|
"target_idx": 4
|
|
},
|
|
{
|
|
"session_id": "1313181",
|
|
"input_length": 60004,
|
|
"new_local": 17508,
|
|
"src_pending_prefill": 19503,
|
|
"fleet_median_pp": 3806.5,
|
|
"proj_prefill_s": 5.29,
|
|
"target_idx": 5
|
|
}
|
|
]
|
|
}
|
|
}
|
|
}
|
|
} |