825 lines
24 KiB
JSON
825 lines
24 KiB
JSON
{
|
|
"environment": [
|
|
{
|
|
"backend_env": {
|
|
"VLLM_ALLREDUCE_USE_FLASHINFER": "1",
|
|
"VLLM_FLASHINFER_ALLREDUCE_BACKEND": "trtllm"
|
|
},
|
|
"gpu": "NVIDIA H20",
|
|
"model": "/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B",
|
|
"torch_cuda": "12.9",
|
|
"torch_version": "2.11.0+cu129",
|
|
"vllm_source_commit": "88d34c6409e9fb3c7b8ca0c04756f061d2099eb1",
|
|
"vllm_version": "0.20.0"
|
|
},
|
|
{
|
|
"backend_env": {
|
|
"VLLM_ALLREDUCE_USE_FLASHINFER": "1",
|
|
"VLLM_FLASHINFER_ALLREDUCE_BACKEND": "trtllm"
|
|
},
|
|
"gpu": "NVIDIA H20",
|
|
"model": "/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B",
|
|
"torch_cuda": "12.9",
|
|
"torch_version": "2.11.0+cu129",
|
|
"vllm_source_commit": "88d34c6409e9fb3c7b8ca0c04756f061d2099eb1",
|
|
"vllm_version": "0.20.0"
|
|
}
|
|
],
|
|
"frontier_consumption": "diagnostic_only_in_base_profile_only_run; measured lookup requires a separate CC-backend injection ablation",
|
|
"rows": [
|
|
{
|
|
"communicator": "vllm.tensor_model_parallel_all_reduce",
|
|
"critical_path_median_ms": 0.08288000151515007,
|
|
"dtype": "bfloat16",
|
|
"hidden_dim": 2048,
|
|
"num_tokens": 1,
|
|
"payload_bytes": 4096,
|
|
"per_rank_time_ms": [
|
|
{
|
|
"max": 0.4872959852218628,
|
|
"mean": 0.1310015980154276,
|
|
"median": 0.07679999992251396,
|
|
"min": 0.06217600032687187,
|
|
"std": 0.12790721677293843
|
|
},
|
|
{
|
|
"max": 0.4402880072593689,
|
|
"mean": 0.12842560112476348,
|
|
"median": 0.08288000151515007,
|
|
"min": 0.0655680000782013,
|
|
"std": 0.11220176524616535
|
|
}
|
|
],
|
|
"selected_backend": "flashinfer_trtllm",
|
|
"tensor_parallel_size": 2
|
|
},
|
|
{
|
|
"communicator": "vllm.tensor_model_parallel_all_reduce",
|
|
"critical_path_median_ms": 0.0793600007891655,
|
|
"dtype": "bfloat16",
|
|
"hidden_dim": 2048,
|
|
"num_tokens": 8,
|
|
"payload_bytes": 32768,
|
|
"per_rank_time_ms": [
|
|
{
|
|
"max": 0.12716799974441528,
|
|
"mean": 0.07871360033750534,
|
|
"median": 0.0759200006723404,
|
|
"min": 0.06032000109553337,
|
|
"std": 0.019331314939874535
|
|
},
|
|
{
|
|
"max": 0.12380799651145935,
|
|
"mean": 0.08059840016067028,
|
|
"median": 0.0793600007891655,
|
|
"min": 0.06217600032687187,
|
|
"std": 0.01687088356254092
|
|
}
|
|
],
|
|
"selected_backend": "flashinfer_trtllm",
|
|
"tensor_parallel_size": 2
|
|
},
|
|
{
|
|
"communicator": "vllm.tensor_model_parallel_all_reduce",
|
|
"critical_path_median_ms": 0.0713919997215271,
|
|
"dtype": "bfloat16",
|
|
"hidden_dim": 2048,
|
|
"num_tokens": 16,
|
|
"payload_bytes": 65536,
|
|
"per_rank_time_ms": [
|
|
{
|
|
"max": 0.12697599828243256,
|
|
"mean": 0.0767391998320818,
|
|
"median": 0.07078400254249573,
|
|
"min": 0.05910399928689003,
|
|
"std": 0.018775178979463278
|
|
},
|
|
{
|
|
"max": 0.11430399864912033,
|
|
"mean": 0.07594559974968433,
|
|
"median": 0.0713919997215271,
|
|
"min": 0.06124800071120262,
|
|
"std": 0.015719922150631036
|
|
}
|
|
],
|
|
"selected_backend": "flashinfer_trtllm",
|
|
"tensor_parallel_size": 2
|
|
},
|
|
{
|
|
"communicator": "vllm.tensor_model_parallel_all_reduce",
|
|
"critical_path_median_ms": 0.08056000247597694,
|
|
"dtype": "bfloat16",
|
|
"hidden_dim": 2048,
|
|
"num_tokens": 32,
|
|
"payload_bytes": 131072,
|
|
"per_rank_time_ms": [
|
|
{
|
|
"max": 0.1037760004401207,
|
|
"mean": 0.07954559996724128,
|
|
"median": 0.08056000247597694,
|
|
"min": 0.05955199897289276,
|
|
"std": 0.0135697420393132
|
|
},
|
|
{
|
|
"max": 0.10608000308275223,
|
|
"mean": 0.07971520014107228,
|
|
"median": 0.07593599706888199,
|
|
"min": 0.06028800085186958,
|
|
"std": 0.015319849772775456
|
|
}
|
|
],
|
|
"selected_backend": "flashinfer_trtllm",
|
|
"tensor_parallel_size": 2
|
|
},
|
|
{
|
|
"communicator": "vllm.tensor_model_parallel_all_reduce",
|
|
"critical_path_median_ms": 0.0865279994904995,
|
|
"dtype": "bfloat16",
|
|
"hidden_dim": 2048,
|
|
"num_tokens": 64,
|
|
"payload_bytes": 262144,
|
|
"per_rank_time_ms": [
|
|
{
|
|
"max": 0.14470399916172028,
|
|
"mean": 0.09121599942445754,
|
|
"median": 0.0865279994904995,
|
|
"min": 0.06441599875688553,
|
|
"std": 0.024893837894277456
|
|
},
|
|
{
|
|
"max": 0.12438400089740753,
|
|
"mean": 0.08531199917197227,
|
|
"median": 0.08031999692320824,
|
|
"min": 0.06364800035953522,
|
|
"std": 0.01878029830059533
|
|
}
|
|
],
|
|
"selected_backend": "flashinfer_trtllm",
|
|
"tensor_parallel_size": 2
|
|
},
|
|
{
|
|
"communicator": "vllm.tensor_model_parallel_all_reduce",
|
|
"critical_path_median_ms": 0.07135999947786331,
|
|
"dtype": "bfloat16",
|
|
"hidden_dim": 2048,
|
|
"num_tokens": 128,
|
|
"payload_bytes": 524288,
|
|
"per_rank_time_ms": [
|
|
{
|
|
"max": 0.11753600090742111,
|
|
"mean": 0.07606079950928687,
|
|
"median": 0.07135999947786331,
|
|
"min": 0.05843200162053108,
|
|
"std": 0.01755519771639284
|
|
},
|
|
{
|
|
"max": 0.1103999987244606,
|
|
"mean": 0.07607359997928143,
|
|
"median": 0.07073600217700005,
|
|
"min": 0.05721599981188774,
|
|
"std": 0.016904445220949783
|
|
}
|
|
],
|
|
"selected_backend": "flashinfer_trtllm",
|
|
"tensor_parallel_size": 2
|
|
},
|
|
{
|
|
"communicator": "vllm.tensor_model_parallel_all_reduce",
|
|
"critical_path_median_ms": 0.07321599870920181,
|
|
"dtype": "bfloat16",
|
|
"hidden_dim": 2048,
|
|
"num_tokens": 256,
|
|
"payload_bytes": 1048576,
|
|
"per_rank_time_ms": [
|
|
{
|
|
"max": 0.11740799993276596,
|
|
"mean": 0.07749119997024537,
|
|
"median": 0.07203200086951256,
|
|
"min": 0.05862399935722351,
|
|
"std": 0.017701381594822835
|
|
},
|
|
{
|
|
"max": 0.11382400244474411,
|
|
"mean": 0.07733759954571724,
|
|
"median": 0.07321599870920181,
|
|
"min": 0.059039998799562454,
|
|
"std": 0.017188890882557036
|
|
}
|
|
],
|
|
"selected_backend": "flashinfer_trtllm",
|
|
"tensor_parallel_size": 2
|
|
},
|
|
{
|
|
"communicator": "vllm.tensor_model_parallel_all_reduce",
|
|
"critical_path_median_ms": 0.09025600180029869,
|
|
"dtype": "bfloat16",
|
|
"hidden_dim": 2048,
|
|
"num_tokens": 512,
|
|
"payload_bytes": 2097152,
|
|
"per_rank_time_ms": [
|
|
{
|
|
"max": 0.13980799913406372,
|
|
"mean": 0.0950367994606495,
|
|
"median": 0.09025600180029869,
|
|
"min": 0.06815999746322632,
|
|
"std": 0.022829835127539378
|
|
},
|
|
{
|
|
"max": 0.14764800667762756,
|
|
"mean": 0.09710080176591873,
|
|
"median": 0.08720000088214874,
|
|
"min": 0.07152000069618225,
|
|
"std": 0.02500574954063789
|
|
}
|
|
],
|
|
"selected_backend": "flashinfer_trtllm",
|
|
"tensor_parallel_size": 2
|
|
},
|
|
{
|
|
"communicator": "vllm.tensor_model_parallel_all_reduce",
|
|
"critical_path_median_ms": 0.08083200082182884,
|
|
"dtype": "bfloat16",
|
|
"hidden_dim": 2048,
|
|
"num_tokens": 1024,
|
|
"payload_bytes": 4194304,
|
|
"per_rank_time_ms": [
|
|
{
|
|
"max": 0.15574400126934052,
|
|
"mean": 0.08760640025138855,
|
|
"median": 0.07846399769186974,
|
|
"min": 0.07097599655389786,
|
|
"std": 0.024487311423551025
|
|
},
|
|
{
|
|
"max": 0.16284799575805664,
|
|
"mean": 0.08963519930839539,
|
|
"median": 0.08083200082182884,
|
|
"min": 0.07100799679756165,
|
|
"std": 0.026047320562445356
|
|
}
|
|
],
|
|
"selected_backend": "flashinfer_trtllm",
|
|
"tensor_parallel_size": 2
|
|
},
|
|
{
|
|
"communicator": "vllm.tensor_model_parallel_all_reduce",
|
|
"critical_path_median_ms": 0.10891199856996536,
|
|
"dtype": "bfloat16",
|
|
"hidden_dim": 2048,
|
|
"num_tokens": 2048,
|
|
"payload_bytes": 8388608,
|
|
"per_rank_time_ms": [
|
|
{
|
|
"max": 0.1582079976797104,
|
|
"mean": 0.11541439890861512,
|
|
"median": 0.10836799815297127,
|
|
"min": 0.09062399715185165,
|
|
"std": 0.018285136316576037
|
|
},
|
|
{
|
|
"max": 0.1578879952430725,
|
|
"mean": 0.11537599861621857,
|
|
"median": 0.10891199856996536,
|
|
"min": 0.0960640013217926,
|
|
"std": 0.018246538626977286
|
|
}
|
|
],
|
|
"selected_backend": "flashinfer_trtllm",
|
|
"tensor_parallel_size": 2
|
|
},
|
|
{
|
|
"communicator": "vllm.tensor_model_parallel_all_reduce",
|
|
"critical_path_median_ms": 0.1703840047121048,
|
|
"dtype": "bfloat16",
|
|
"hidden_dim": 2048,
|
|
"num_tokens": 4096,
|
|
"payload_bytes": 16777216,
|
|
"per_rank_time_ms": [
|
|
{
|
|
"max": 0.19327999651432037,
|
|
"mean": 0.1707327976822853,
|
|
"median": 0.1703840047121048,
|
|
"min": 0.14815999567508698,
|
|
"std": 0.014211056022719618
|
|
},
|
|
{
|
|
"max": 0.19276799261569977,
|
|
"mean": 0.1658592015504837,
|
|
"median": 0.16379200667142868,
|
|
"min": 0.14560000598430634,
|
|
"std": 0.013840249648693638
|
|
}
|
|
],
|
|
"selected_backend": "flashinfer_trtllm",
|
|
"tensor_parallel_size": 2
|
|
},
|
|
{
|
|
"communicator": "vllm.tensor_model_parallel_all_reduce",
|
|
"critical_path_median_ms": 0.25539200007915497,
|
|
"dtype": "bfloat16",
|
|
"hidden_dim": 2048,
|
|
"num_tokens": 8192,
|
|
"payload_bytes": 33554432,
|
|
"per_rank_time_ms": [
|
|
{
|
|
"max": 0.2807359993457794,
|
|
"mean": 0.25750079900026324,
|
|
"median": 0.25539200007915497,
|
|
"min": 0.24624000489711761,
|
|
"std": 0.008925204570802302
|
|
},
|
|
{
|
|
"max": 0.2863999903202057,
|
|
"mean": 0.2585055992007256,
|
|
"median": 0.255280002951622,
|
|
"min": 0.24371199309825897,
|
|
"std": 0.012059738582656496
|
|
}
|
|
],
|
|
"selected_backend": "flashinfer_trtllm",
|
|
"tensor_parallel_size": 2
|
|
},
|
|
{
|
|
"communicator": "vllm.tensor_model_parallel_all_reduce",
|
|
"critical_path_median_ms": 0.1021759994328022,
|
|
"dtype": "bfloat16",
|
|
"hidden_dim": 2048,
|
|
"num_tokens": 1,
|
|
"payload_bytes": 4096,
|
|
"per_rank_time_ms": [
|
|
{
|
|
"max": 0.9770879745483398,
|
|
"mean": 0.1974783968180418,
|
|
"median": 0.10099200159311295,
|
|
"min": 0.05913599953055382,
|
|
"std": 0.2660581738745569
|
|
},
|
|
{
|
|
"max": 0.892799973487854,
|
|
"mean": 0.18164799660444259,
|
|
"median": 0.1021759994328022,
|
|
"min": 0.06435199826955795,
|
|
"std": 0.23947520188197013
|
|
},
|
|
{
|
|
"max": 0.6467199921607971,
|
|
"mean": 0.15839359983801843,
|
|
"median": 0.10100800171494484,
|
|
"min": 0.06800000369548798,
|
|
"std": 0.16617013141866102
|
|
},
|
|
{
|
|
"max": 0.6725760102272034,
|
|
"mean": 0.15686400160193442,
|
|
"median": 0.10044800117611885,
|
|
"min": 0.06063999980688095,
|
|
"std": 0.17523222161300497
|
|
}
|
|
],
|
|
"selected_backend": "flashinfer_trtllm",
|
|
"tensor_parallel_size": 4
|
|
},
|
|
{
|
|
"communicator": "vllm.tensor_model_parallel_all_reduce",
|
|
"critical_path_median_ms": 0.12694399803876877,
|
|
"dtype": "bfloat16",
|
|
"hidden_dim": 2048,
|
|
"num_tokens": 8,
|
|
"payload_bytes": 32768,
|
|
"per_rank_time_ms": [
|
|
{
|
|
"max": 0.6659520268440247,
|
|
"mean": 0.1927648030221462,
|
|
"median": 0.12694399803876877,
|
|
"min": 0.07609599828720093,
|
|
"std": 0.16697784531907736
|
|
},
|
|
{
|
|
"max": 0.695360004901886,
|
|
"mean": 0.19356480240821838,
|
|
"median": 0.11726400256156921,
|
|
"min": 0.0796160027384758,
|
|
"std": 0.17588096678867862
|
|
},
|
|
{
|
|
"max": 0.5939840078353882,
|
|
"mean": 0.1868800014257431,
|
|
"median": 0.12379200011491776,
|
|
"min": 0.07427199929952621,
|
|
"std": 0.1463231714943902
|
|
},
|
|
{
|
|
"max": 0.6635839939117432,
|
|
"mean": 0.1874335989356041,
|
|
"median": 0.12014400213956833,
|
|
"min": 0.07526399940252304,
|
|
"std": 0.16680959677760304
|
|
}
|
|
],
|
|
"selected_backend": "flashinfer_trtllm",
|
|
"tensor_parallel_size": 4
|
|
},
|
|
{
|
|
"communicator": "vllm.tensor_model_parallel_all_reduce",
|
|
"critical_path_median_ms": 0.09161599725484848,
|
|
"dtype": "bfloat16",
|
|
"hidden_dim": 2048,
|
|
"num_tokens": 16,
|
|
"payload_bytes": 65536,
|
|
"per_rank_time_ms": [
|
|
{
|
|
"max": 0.12665599584579468,
|
|
"mean": 0.09443839862942696,
|
|
"median": 0.09124799817800522,
|
|
"min": 0.06431999802589417,
|
|
"std": 0.0205690155775906
|
|
},
|
|
{
|
|
"max": 0.1303039938211441,
|
|
"mean": 0.09712959825992584,
|
|
"median": 0.09161599725484848,
|
|
"min": 0.07648000121116638,
|
|
"std": 0.019072048129173236
|
|
},
|
|
{
|
|
"max": 0.13836799561977386,
|
|
"mean": 0.09821119979023933,
|
|
"median": 0.09148800000548363,
|
|
"min": 0.0727040022611618,
|
|
"std": 0.021314066545189116
|
|
},
|
|
{
|
|
"max": 0.12992000579833984,
|
|
"mean": 0.09272959977388381,
|
|
"median": 0.08931199833750725,
|
|
"min": 0.06406400352716446,
|
|
"std": 0.021360803830354765
|
|
}
|
|
],
|
|
"selected_backend": "flashinfer_trtllm",
|
|
"tensor_parallel_size": 4
|
|
},
|
|
{
|
|
"communicator": "vllm.tensor_model_parallel_all_reduce",
|
|
"critical_path_median_ms": 0.08580800145864487,
|
|
"dtype": "bfloat16",
|
|
"hidden_dim": 2048,
|
|
"num_tokens": 32,
|
|
"payload_bytes": 131072,
|
|
"per_rank_time_ms": [
|
|
{
|
|
"max": 0.13449600338935852,
|
|
"mean": 0.08960640132427215,
|
|
"median": 0.08299200236797333,
|
|
"min": 0.06796800345182419,
|
|
"std": 0.020633597898445998
|
|
},
|
|
{
|
|
"max": 0.14735999703407288,
|
|
"mean": 0.09248319901525974,
|
|
"median": 0.08580800145864487,
|
|
"min": 0.05913599953055382,
|
|
"std": 0.025960234928829564
|
|
},
|
|
{
|
|
"max": 0.13705599308013916,
|
|
"mean": 0.08947199806571007,
|
|
"median": 0.08460799977183342,
|
|
"min": 0.0634239986538887,
|
|
"std": 0.02122838946992339
|
|
},
|
|
{
|
|
"max": 0.13846400380134583,
|
|
"mean": 0.0859104000031948,
|
|
"median": 0.08308799937367439,
|
|
"min": 0.05974400043487549,
|
|
"std": 0.022373631424433445
|
|
}
|
|
],
|
|
"selected_backend": "flashinfer_trtllm",
|
|
"tensor_parallel_size": 4
|
|
},
|
|
{
|
|
"communicator": "vllm.tensor_model_parallel_all_reduce",
|
|
"critical_path_median_ms": 0.09867199882864952,
|
|
"dtype": "bfloat16",
|
|
"hidden_dim": 2048,
|
|
"num_tokens": 64,
|
|
"payload_bytes": 262144,
|
|
"per_rank_time_ms": [
|
|
{
|
|
"max": 0.1279039978981018,
|
|
"mean": 0.09748799875378608,
|
|
"median": 0.09678399935364723,
|
|
"min": 0.0655359998345375,
|
|
"std": 0.02315719144614622
|
|
},
|
|
{
|
|
"max": 0.1356479972600937,
|
|
"mean": 0.10018239840865135,
|
|
"median": 0.09532799944281578,
|
|
"min": 0.06185600161552429,
|
|
"std": 0.02330700253650042
|
|
},
|
|
{
|
|
"max": 0.13142399489879608,
|
|
"mean": 0.09778879955410957,
|
|
"median": 0.09492799639701843,
|
|
"min": 0.06560000032186508,
|
|
"std": 0.02307579212430045
|
|
},
|
|
{
|
|
"max": 0.1276479959487915,
|
|
"mean": 0.09611519873142242,
|
|
"median": 0.09867199882864952,
|
|
"min": 0.0642239972949028,
|
|
"std": 0.02273612181973431
|
|
}
|
|
],
|
|
"selected_backend": "flashinfer_trtllm",
|
|
"tensor_parallel_size": 4
|
|
},
|
|
{
|
|
"communicator": "vllm.tensor_model_parallel_all_reduce",
|
|
"critical_path_median_ms": 0.09646400064229965,
|
|
"dtype": "bfloat16",
|
|
"hidden_dim": 2048,
|
|
"num_tokens": 128,
|
|
"payload_bytes": 524288,
|
|
"per_rank_time_ms": [
|
|
{
|
|
"max": 0.14060799777507782,
|
|
"mean": 0.09770880043506622,
|
|
"median": 0.09115200117230415,
|
|
"min": 0.06950400024652481,
|
|
"std": 0.024752645089849798
|
|
},
|
|
{
|
|
"max": 0.14377599954605103,
|
|
"mean": 0.09824960008263588,
|
|
"median": 0.08999999985098839,
|
|
"min": 0.07103999704122543,
|
|
"std": 0.025403407389046027
|
|
},
|
|
{
|
|
"max": 0.13680000603199005,
|
|
"mean": 0.09993600100278854,
|
|
"median": 0.09646400064229965,
|
|
"min": 0.06790400296449661,
|
|
"std": 0.022297424273985882
|
|
},
|
|
{
|
|
"max": 0.1391039937734604,
|
|
"mean": 0.09769919961690902,
|
|
"median": 0.09601600095629692,
|
|
"min": 0.06835199892520905,
|
|
"std": 0.023899922100804445
|
|
}
|
|
],
|
|
"selected_backend": "flashinfer_trtllm",
|
|
"tensor_parallel_size": 4
|
|
},
|
|
{
|
|
"communicator": "vllm.tensor_model_parallel_all_reduce",
|
|
"critical_path_median_ms": 0.08377600088715553,
|
|
"dtype": "bfloat16",
|
|
"hidden_dim": 2048,
|
|
"num_tokens": 256,
|
|
"payload_bytes": 1048576,
|
|
"per_rank_time_ms": [
|
|
{
|
|
"max": 0.1464959979057312,
|
|
"mean": 0.09276160076260567,
|
|
"median": 0.08377600088715553,
|
|
"min": 0.06777600198984146,
|
|
"std": 0.024751086465295658
|
|
},
|
|
{
|
|
"max": 0.14319999516010284,
|
|
"mean": 0.09080640003085136,
|
|
"median": 0.080400001257658,
|
|
"min": 0.06796800345182419,
|
|
"std": 0.023365718881708488
|
|
},
|
|
{
|
|
"max": 0.1382399946451187,
|
|
"mean": 0.09063360020518303,
|
|
"median": 0.08193599805235863,
|
|
"min": 0.06627199798822403,
|
|
"std": 0.02372618650854302
|
|
},
|
|
{
|
|
"max": 0.14313599467277527,
|
|
"mean": 0.09044799953699112,
|
|
"median": 0.08128000050783157,
|
|
"min": 0.06652799993753433,
|
|
"std": 0.02462486862033686
|
|
}
|
|
],
|
|
"selected_backend": "flashinfer_trtllm",
|
|
"tensor_parallel_size": 4
|
|
},
|
|
{
|
|
"communicator": "vllm.tensor_model_parallel_all_reduce",
|
|
"critical_path_median_ms": 0.1128000020980835,
|
|
"dtype": "bfloat16",
|
|
"hidden_dim": 2048,
|
|
"num_tokens": 512,
|
|
"payload_bytes": 2097152,
|
|
"per_rank_time_ms": [
|
|
{
|
|
"max": 0.1600320041179657,
|
|
"mean": 0.11426239982247352,
|
|
"median": 0.1128000020980835,
|
|
"min": 0.07657600194215775,
|
|
"std": 0.030240087702350687
|
|
},
|
|
{
|
|
"max": 0.15881599485874176,
|
|
"mean": 0.11206399947404862,
|
|
"median": 0.10628800094127655,
|
|
"min": 0.0772159993648529,
|
|
"std": 0.029933135345483627
|
|
},
|
|
{
|
|
"max": 0.15612800419330597,
|
|
"mean": 0.10761600062251091,
|
|
"median": 0.09860799834132195,
|
|
"min": 0.07689599692821503,
|
|
"std": 0.02634237020678647
|
|
},
|
|
{
|
|
"max": 0.15865600109100342,
|
|
"mean": 0.11094079986214637,
|
|
"median": 0.10979199782013893,
|
|
"min": 0.07583999633789062,
|
|
"std": 0.028619217028542445
|
|
}
|
|
],
|
|
"selected_backend": "flashinfer_trtllm",
|
|
"tensor_parallel_size": 4
|
|
},
|
|
{
|
|
"communicator": "vllm.tensor_model_parallel_all_reduce",
|
|
"critical_path_median_ms": 0.08755199983716011,
|
|
"dtype": "bfloat16",
|
|
"hidden_dim": 2048,
|
|
"num_tokens": 1024,
|
|
"payload_bytes": 4194304,
|
|
"per_rank_time_ms": [
|
|
{
|
|
"max": 0.11999999731779099,
|
|
"mean": 0.0843871995806694,
|
|
"median": 0.08755199983716011,
|
|
"min": 0.06332799792289734,
|
|
"std": 0.016638543890716024
|
|
},
|
|
{
|
|
"max": 0.1218239963054657,
|
|
"mean": 0.08518079966306687,
|
|
"median": 0.08032000064849854,
|
|
"min": 0.06393600255250931,
|
|
"std": 0.019900899429956945
|
|
},
|
|
{
|
|
"max": 0.11849600076675415,
|
|
"mean": 0.0843968003988266,
|
|
"median": 0.08702399954199791,
|
|
"min": 0.06297600269317627,
|
|
"std": 0.017031908773433545
|
|
},
|
|
{
|
|
"max": 0.12300799787044525,
|
|
"mean": 0.0846304003149271,
|
|
"median": 0.08139199763536453,
|
|
"min": 0.06195199862122536,
|
|
"std": 0.020106523698622265
|
|
}
|
|
],
|
|
"selected_backend": "nccl_fallback",
|
|
"tensor_parallel_size": 4
|
|
},
|
|
{
|
|
"communicator": "vllm.tensor_model_parallel_all_reduce",
|
|
"critical_path_median_ms": 0.12361599877476692,
|
|
"dtype": "bfloat16",
|
|
"hidden_dim": 2048,
|
|
"num_tokens": 2048,
|
|
"payload_bytes": 8388608,
|
|
"per_rank_time_ms": [
|
|
{
|
|
"max": 0.17155200242996216,
|
|
"mean": 0.12168959975242614,
|
|
"median": 0.12055999785661697,
|
|
"min": 0.09609600156545639,
|
|
"std": 0.022565485532483928
|
|
},
|
|
{
|
|
"max": 0.9246399998664856,
|
|
"mean": 0.19978560134768486,
|
|
"median": 0.12361599877476692,
|
|
"min": 0.09715200215578079,
|
|
"std": 0.24230694305662265
|
|
},
|
|
{
|
|
"max": 0.9317439794540405,
|
|
"mean": 0.20037759989500045,
|
|
"median": 0.12327999994158745,
|
|
"min": 0.09603200107812881,
|
|
"std": 0.24450903278000383
|
|
},
|
|
{
|
|
"max": 0.9321280121803284,
|
|
"mean": 0.19875840097665787,
|
|
"median": 0.12230399996042252,
|
|
"min": 0.0950080007314682,
|
|
"std": 0.24519252220785093
|
|
}
|
|
],
|
|
"selected_backend": "nccl_fallback",
|
|
"tensor_parallel_size": 4
|
|
},
|
|
{
|
|
"communicator": "vllm.tensor_model_parallel_all_reduce",
|
|
"critical_path_median_ms": 0.20030399411916733,
|
|
"dtype": "bfloat16",
|
|
"hidden_dim": 2048,
|
|
"num_tokens": 4096,
|
|
"payload_bytes": 16777216,
|
|
"per_rank_time_ms": [
|
|
{
|
|
"max": 0.3261440098285675,
|
|
"mean": 0.20826880186796187,
|
|
"median": 0.19974400103092194,
|
|
"min": 0.1409280002117157,
|
|
"std": 0.051397264855718945
|
|
},
|
|
{
|
|
"max": 0.3248000144958496,
|
|
"mean": 0.20548800230026246,
|
|
"median": 0.1979840025305748,
|
|
"min": 0.141184002161026,
|
|
"std": 0.04980002399656717
|
|
},
|
|
{
|
|
"max": 0.32547199726104736,
|
|
"mean": 0.21280319690704347,
|
|
"median": 0.20030399411916733,
|
|
"min": 0.14127999544143677,
|
|
"std": 0.05304243085685509
|
|
},
|
|
{
|
|
"max": 0.26047998666763306,
|
|
"mean": 0.1969312012195587,
|
|
"median": 0.18079999834299088,
|
|
"min": 0.14057600498199463,
|
|
"std": 0.042174123638424224
|
|
}
|
|
],
|
|
"selected_backend": "nccl_fallback",
|
|
"tensor_parallel_size": 4
|
|
},
|
|
{
|
|
"communicator": "vllm.tensor_model_parallel_all_reduce",
|
|
"critical_path_median_ms": 0.2924960106611252,
|
|
"dtype": "bfloat16",
|
|
"hidden_dim": 2048,
|
|
"num_tokens": 8192,
|
|
"payload_bytes": 33554432,
|
|
"per_rank_time_ms": [
|
|
{
|
|
"max": 0.45052799582481384,
|
|
"mean": 0.3013375997543335,
|
|
"median": 0.28273600339889526,
|
|
"min": 0.21334399282932281,
|
|
"std": 0.07214223764527236
|
|
},
|
|
{
|
|
"max": 0.4466240108013153,
|
|
"mean": 0.2968191936612129,
|
|
"median": 0.27796798944473267,
|
|
"min": 0.21241599321365356,
|
|
"std": 0.07400678240849741
|
|
},
|
|
{
|
|
"max": 0.3830080032348633,
|
|
"mean": 0.2946112036705017,
|
|
"median": 0.2924960106611252,
|
|
"min": 0.21084800362586975,
|
|
"std": 0.05323627615746332
|
|
},
|
|
{
|
|
"max": 0.4609600007534027,
|
|
"mean": 0.3054272010922432,
|
|
"median": 0.289792001247406,
|
|
"min": 0.21062399446964264,
|
|
"std": 0.07608482904865672
|
|
}
|
|
],
|
|
"selected_backend": "nccl_fallback",
|
|
"tensor_parallel_size": 4
|
|
}
|
|
],
|
|
"schema_version": "qwen30_vllm020_allreduce_frozen.v1"
|
|
}
|