565 lines
17 KiB
JSON
565 lines
17 KiB
JSON
{
|
|
"environment": {
|
|
"attention_backend": "FLASH_ATTN",
|
|
"block_size": 16,
|
|
"dtype": "bfloat16",
|
|
"gpu": "NVIDIA H20",
|
|
"model": "/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B",
|
|
"profile_kv_update": true,
|
|
"profile_method": "cuda_event",
|
|
"torch_cuda": "12.9",
|
|
"torch_version": "2.11.0+cu129",
|
|
"vllm_source_commit": "88d34c6409e9fb3c7b8ca0c04756f061d2099eb1",
|
|
"vllm_version": "0.20.0"
|
|
},
|
|
"rows": [
|
|
{
|
|
"attention_core_excludes_kv_cache_update": true,
|
|
"config": {
|
|
"backend": "FLASH_ATTN",
|
|
"batch_spec": "q8ks48k",
|
|
"block_size": 16,
|
|
"device": "cuda:0",
|
|
"dtype": "torch.bfloat16",
|
|
"head_dim": 128,
|
|
"kv_cache_dtype": "auto",
|
|
"kv_lora_rank": null,
|
|
"num_kv_heads": 1,
|
|
"num_kv_splits": null,
|
|
"num_layers": 1,
|
|
"num_q_heads": 8,
|
|
"prefill_backend": null,
|
|
"profile_memory": true,
|
|
"qk_nope_head_dim": null,
|
|
"qk_rope_head_dim": null,
|
|
"reorder_batch_threshold": null,
|
|
"repeats": 10,
|
|
"use_cuda_graphs": false,
|
|
"v_head_dim": null,
|
|
"warmup_iters": 5
|
|
},
|
|
"error": null,
|
|
"kv_cache_update_time": {
|
|
"max_ms": 0.039583999663591385,
|
|
"mean_ms": 0.027286400087177753,
|
|
"median_ms": 0.025200000032782555,
|
|
"min_ms": 0.024191999807953835,
|
|
"std_ms": 0.004620118437533604
|
|
},
|
|
"max_time": 0.011496224403381348,
|
|
"mean_time": 0.011354345512390137,
|
|
"memory_allocated_mb": 60.0849609375,
|
|
"memory_reserved_mb": 62.0,
|
|
"min_time": 0.01132140827178955,
|
|
"std_time": 5.011776621543997e-05,
|
|
"tensor_parallel_size": 4,
|
|
"throughput_tokens_per_sec": 721485.8831854897
|
|
},
|
|
{
|
|
"attention_core_excludes_kv_cache_update": true,
|
|
"config": {
|
|
"backend": "FLASH_ATTN",
|
|
"batch_spec": "q8ks64k",
|
|
"block_size": 16,
|
|
"device": "cuda:0",
|
|
"dtype": "torch.bfloat16",
|
|
"head_dim": 128,
|
|
"kv_cache_dtype": "auto",
|
|
"kv_lora_rank": null,
|
|
"num_kv_heads": 1,
|
|
"num_kv_splits": null,
|
|
"num_layers": 1,
|
|
"num_q_heads": 8,
|
|
"prefill_backend": null,
|
|
"profile_memory": true,
|
|
"qk_nope_head_dim": null,
|
|
"qk_rope_head_dim": null,
|
|
"reorder_batch_threshold": null,
|
|
"repeats": 10,
|
|
"use_cuda_graphs": false,
|
|
"v_head_dim": null,
|
|
"warmup_iters": 5
|
|
},
|
|
"error": null,
|
|
"kv_cache_update_time": {
|
|
"max_ms": 0.03232000023126602,
|
|
"mean_ms": 0.026406400091946124,
|
|
"median_ms": 0.02518399991095066,
|
|
"min_ms": 0.024032000452280045,
|
|
"std_ms": 0.002792696117638337
|
|
},
|
|
"max_time": 0.015639776229858397,
|
|
"mean_time": 0.015451993656158448,
|
|
"memory_allocated_mb": 68.0888671875,
|
|
"memory_reserved_mb": 94.0,
|
|
"min_time": 0.01542249584197998,
|
|
"std_time": 6.304729308178388e-05,
|
|
"tensor_parallel_size": 4,
|
|
"throughput_tokens_per_sec": 530158.1260185833
|
|
},
|
|
{
|
|
"attention_core_excludes_kv_cache_update": true,
|
|
"config": {
|
|
"backend": "FLASH_ATTN",
|
|
"batch_spec": "q8ks80k",
|
|
"block_size": 16,
|
|
"device": "cuda:0",
|
|
"dtype": "torch.bfloat16",
|
|
"head_dim": 128,
|
|
"kv_cache_dtype": "auto",
|
|
"kv_lora_rank": null,
|
|
"num_kv_heads": 1,
|
|
"num_kv_splits": null,
|
|
"num_layers": 1,
|
|
"num_q_heads": 8,
|
|
"prefill_backend": null,
|
|
"profile_memory": true,
|
|
"qk_nope_head_dim": null,
|
|
"qk_rope_head_dim": null,
|
|
"reorder_batch_threshold": null,
|
|
"repeats": 10,
|
|
"use_cuda_graphs": false,
|
|
"v_head_dim": null,
|
|
"warmup_iters": 5
|
|
},
|
|
"error": null,
|
|
"kv_cache_update_time": {
|
|
"max_ms": 0.03488000109791756,
|
|
"mean_ms": 0.026147199980914592,
|
|
"median_ms": 0.02459200005978346,
|
|
"min_ms": 0.024224000051617622,
|
|
"std_ms": 0.003303059079404486
|
|
},
|
|
"max_time": 0.01970569610595703,
|
|
"mean_time": 0.01954985942840576,
|
|
"memory_allocated_mb": 76.0927734375,
|
|
"memory_reserved_mb": 134.0,
|
|
"min_time": 0.019525152206420898,
|
|
"std_time": 5.224891496533832e-05,
|
|
"tensor_parallel_size": 4,
|
|
"throughput_tokens_per_sec": 419031.1459783236
|
|
},
|
|
{
|
|
"attention_core_excludes_kv_cache_update": true,
|
|
"config": {
|
|
"backend": "FLASH_ATTN",
|
|
"batch_spec": "q8ks96k",
|
|
"block_size": 16,
|
|
"device": "cuda:0",
|
|
"dtype": "torch.bfloat16",
|
|
"head_dim": 128,
|
|
"kv_cache_dtype": "auto",
|
|
"kv_lora_rank": null,
|
|
"num_kv_heads": 1,
|
|
"num_kv_splits": null,
|
|
"num_layers": 1,
|
|
"num_q_heads": 8,
|
|
"prefill_backend": null,
|
|
"profile_memory": true,
|
|
"qk_nope_head_dim": null,
|
|
"qk_rope_head_dim": null,
|
|
"reorder_batch_threshold": null,
|
|
"repeats": 10,
|
|
"use_cuda_graphs": false,
|
|
"v_head_dim": null,
|
|
"warmup_iters": 5
|
|
},
|
|
"error": null,
|
|
"kv_cache_update_time": {
|
|
"max_ms": 0.03046399913728237,
|
|
"mean_ms": 0.0255103999748826,
|
|
"median_ms": 0.024656000547111034,
|
|
"min_ms": 0.024159999564290047,
|
|
"std_ms": 0.00195717586616911
|
|
},
|
|
"max_time": 0.023927040100097656,
|
|
"mean_time": 0.02367282257080078,
|
|
"memory_allocated_mb": 84.0966796875,
|
|
"memory_reserved_mb": 182.0,
|
|
"min_time": 0.023630016326904296,
|
|
"std_time": 8.715594728873713e-05,
|
|
"tensor_parallel_size": 4,
|
|
"throughput_tokens_per_sec": 346050.83426360885
|
|
},
|
|
{
|
|
"attention_core_excludes_kv_cache_update": true,
|
|
"config": {
|
|
"backend": "FLASH_ATTN",
|
|
"batch_spec": "q8ks112k",
|
|
"block_size": 16,
|
|
"device": "cuda:0",
|
|
"dtype": "torch.bfloat16",
|
|
"head_dim": 128,
|
|
"kv_cache_dtype": "auto",
|
|
"kv_lora_rank": null,
|
|
"num_kv_heads": 1,
|
|
"num_kv_splits": null,
|
|
"num_layers": 1,
|
|
"num_q_heads": 8,
|
|
"prefill_backend": null,
|
|
"profile_memory": true,
|
|
"qk_nope_head_dim": null,
|
|
"qk_rope_head_dim": null,
|
|
"reorder_batch_threshold": null,
|
|
"repeats": 10,
|
|
"use_cuda_graphs": false,
|
|
"v_head_dim": null,
|
|
"warmup_iters": 5
|
|
},
|
|
"error": null,
|
|
"kv_cache_update_time": {
|
|
"max_ms": 0.03574400022625923,
|
|
"mean_ms": 0.02678720001131296,
|
|
"median_ms": 0.024720000103116035,
|
|
"min_ms": 0.02380800060927868,
|
|
"std_ms": 0.003678231234048846
|
|
},
|
|
"max_time": 0.02779840087890625,
|
|
"mean_time": 0.027747158622741696,
|
|
"memory_allocated_mb": 92.1005859375,
|
|
"memory_reserved_mb": 238.0,
|
|
"min_time": 0.027736671447753908,
|
|
"std_time": 1.8952179343788658e-05,
|
|
"tensor_parallel_size": 4,
|
|
"throughput_tokens_per_sec": 295237.4371509809
|
|
},
|
|
{
|
|
"attention_core_excludes_kv_cache_update": true,
|
|
"config": {
|
|
"backend": "FLASH_ATTN",
|
|
"batch_spec": "q8ks128k",
|
|
"block_size": 16,
|
|
"device": "cuda:0",
|
|
"dtype": "torch.bfloat16",
|
|
"head_dim": 128,
|
|
"kv_cache_dtype": "auto",
|
|
"kv_lora_rank": null,
|
|
"num_kv_heads": 1,
|
|
"num_kv_splits": null,
|
|
"num_layers": 1,
|
|
"num_q_heads": 8,
|
|
"prefill_backend": null,
|
|
"profile_memory": true,
|
|
"qk_nope_head_dim": null,
|
|
"qk_rope_head_dim": null,
|
|
"reorder_batch_threshold": null,
|
|
"repeats": 10,
|
|
"use_cuda_graphs": false,
|
|
"v_head_dim": null,
|
|
"warmup_iters": 5
|
|
},
|
|
"error": null,
|
|
"kv_cache_update_time": {
|
|
"max_ms": 0.042080000042915344,
|
|
"mean_ms": 0.02844479978084564,
|
|
"median_ms": 0.02527999971061945,
|
|
"min_ms": 0.024224000051617622,
|
|
"std_ms": 0.006562862750709003
|
|
},
|
|
"max_time": 0.03208835220336914,
|
|
"mean_time": 0.03186994876861572,
|
|
"memory_allocated_mb": 100.1044921875,
|
|
"memory_reserved_mb": 302.0,
|
|
"min_time": 0.03183427238464356,
|
|
"std_time": 7.444245241186317e-05,
|
|
"tensor_parallel_size": 4,
|
|
"throughput_tokens_per_sec": 257044.65543625728
|
|
},
|
|
{
|
|
"attention_core_excludes_kv_cache_update": true,
|
|
"config": {
|
|
"backend": "FLASH_ATTN",
|
|
"batch_spec": "q8ks136k",
|
|
"block_size": 16,
|
|
"device": "cuda:0",
|
|
"dtype": "torch.bfloat16",
|
|
"head_dim": 128,
|
|
"kv_cache_dtype": "auto",
|
|
"kv_lora_rank": null,
|
|
"num_kv_heads": 1,
|
|
"num_kv_splits": null,
|
|
"num_layers": 1,
|
|
"num_q_heads": 8,
|
|
"prefill_backend": null,
|
|
"profile_memory": true,
|
|
"qk_nope_head_dim": null,
|
|
"qk_rope_head_dim": null,
|
|
"reorder_batch_threshold": null,
|
|
"repeats": 10,
|
|
"use_cuda_graphs": false,
|
|
"v_head_dim": null,
|
|
"warmup_iters": 5
|
|
},
|
|
"error": null,
|
|
"kv_cache_update_time": {
|
|
"max_ms": 0.03139200061559677,
|
|
"mean_ms": 0.026396799832582474,
|
|
"median_ms": 0.025071999989449978,
|
|
"min_ms": 0.02377600036561489,
|
|
"std_ms": 0.0025962810796740263
|
|
},
|
|
"max_time": 0.033969856262207034,
|
|
"mean_time": 0.033897081375122075,
|
|
"memory_allocated_mb": 104.1064453125,
|
|
"memory_reserved_mb": 370.0,
|
|
"min_time": 0.03388313674926758,
|
|
"std_time": 2.518493598134421e-05,
|
|
"tensor_parallel_size": 4,
|
|
"throughput_tokens_per_sec": 241672.72424853416
|
|
},
|
|
{
|
|
"attention_core_excludes_kv_cache_update": true,
|
|
"config": {
|
|
"backend": "FLASH_ATTN",
|
|
"batch_spec": "q512s128k",
|
|
"block_size": 16,
|
|
"device": "cuda:0",
|
|
"dtype": "torch.bfloat16",
|
|
"head_dim": 128,
|
|
"kv_cache_dtype": "auto",
|
|
"kv_lora_rank": null,
|
|
"num_kv_heads": 1,
|
|
"num_kv_splits": null,
|
|
"num_layers": 1,
|
|
"num_q_heads": 8,
|
|
"prefill_backend": null,
|
|
"profile_memory": true,
|
|
"qk_nope_head_dim": null,
|
|
"qk_rope_head_dim": null,
|
|
"reorder_batch_threshold": null,
|
|
"repeats": 10,
|
|
"use_cuda_graphs": false,
|
|
"v_head_dim": null,
|
|
"warmup_iters": 5
|
|
},
|
|
"error": null,
|
|
"kv_cache_update_time": {
|
|
"max_ms": 0.022624000906944275,
|
|
"mean_ms": 0.017174400109797715,
|
|
"median_ms": 0.015536000020802021,
|
|
"min_ms": 0.014944000169634819,
|
|
"std_ms": 0.0028331860349340154
|
|
},
|
|
"max_time": 0.003192960023880005,
|
|
"mean_time": 0.003182867193222046,
|
|
"memory_allocated_mb": 66.2958984375,
|
|
"memory_reserved_mb": 372.0,
|
|
"min_time": 0.003179744005203247,
|
|
"std_time": 3.5239767128429387e-06,
|
|
"tensor_parallel_size": 4,
|
|
"throughput_tokens_per_sec": 160861.25148115202
|
|
},
|
|
{
|
|
"attention_core_excludes_kv_cache_update": true,
|
|
"config": {
|
|
"backend": "FLASH_ATTN",
|
|
"batch_spec": "q2ks66k",
|
|
"block_size": 16,
|
|
"device": "cuda:0",
|
|
"dtype": "torch.bfloat16",
|
|
"head_dim": 128,
|
|
"kv_cache_dtype": "auto",
|
|
"kv_lora_rank": null,
|
|
"num_kv_heads": 1,
|
|
"num_kv_splits": null,
|
|
"num_layers": 1,
|
|
"num_q_heads": 8,
|
|
"prefill_backend": null,
|
|
"profile_memory": true,
|
|
"qk_nope_head_dim": null,
|
|
"qk_rope_head_dim": null,
|
|
"reorder_batch_threshold": null,
|
|
"repeats": 10,
|
|
"use_cuda_graphs": false,
|
|
"v_head_dim": null,
|
|
"warmup_iters": 5
|
|
},
|
|
"error": null,
|
|
"kv_cache_update_time": {
|
|
"max_ms": 0.02985600009560585,
|
|
"mean_ms": 0.017318400088697672,
|
|
"median_ms": 0.015568000264465809,
|
|
"min_ms": 0.014655999839305878,
|
|
"std_ms": 0.004479309643844885
|
|
},
|
|
"max_time": 0.004826848030090332,
|
|
"mean_time": 0.004817302370071412,
|
|
"memory_allocated_mb": 42.04248046875,
|
|
"memory_reserved_mb": 372.0,
|
|
"min_time": 0.004811935901641846,
|
|
"std_time": 5.94674080057053e-06,
|
|
"tensor_parallel_size": 4,
|
|
"throughput_tokens_per_sec": 425134.20222979283
|
|
},
|
|
{
|
|
"attention_core_excludes_kv_cache_update": true,
|
|
"config": {
|
|
"backend": "FLASH_ATTN",
|
|
"batch_spec": "q4ks100k",
|
|
"block_size": 16,
|
|
"device": "cuda:0",
|
|
"dtype": "torch.bfloat16",
|
|
"head_dim": 128,
|
|
"kv_cache_dtype": "auto",
|
|
"kv_lora_rank": null,
|
|
"num_kv_heads": 1,
|
|
"num_kv_splits": null,
|
|
"num_layers": 1,
|
|
"num_q_heads": 8,
|
|
"prefill_backend": null,
|
|
"profile_memory": true,
|
|
"qk_nope_head_dim": null,
|
|
"qk_rope_head_dim": null,
|
|
"reorder_batch_threshold": null,
|
|
"repeats": 10,
|
|
"use_cuda_graphs": false,
|
|
"v_head_dim": null,
|
|
"warmup_iters": 5
|
|
},
|
|
"error": null,
|
|
"kv_cache_update_time": {
|
|
"max_ms": 0.03916800022125244,
|
|
"mean_ms": 0.021356799826025962,
|
|
"median_ms": 0.01935999933630228,
|
|
"min_ms": 0.017184000462293625,
|
|
"std_ms": 0.0062801287963799276
|
|
},
|
|
"max_time": 0.014635007858276367,
|
|
"mean_time": 0.014412275123596191,
|
|
"memory_allocated_mb": 68.06640625,
|
|
"memory_reserved_mb": 372.0,
|
|
"min_time": 0.014374591827392579,
|
|
"std_time": 7.569270440104603e-05,
|
|
"tensor_parallel_size": 4,
|
|
"throughput_tokens_per_sec": 284202.17938345566
|
|
},
|
|
{
|
|
"attention_core_excludes_kv_cache_update": true,
|
|
"config": {
|
|
"backend": "FLASH_ATTN",
|
|
"batch_spec": "q6ks134k",
|
|
"block_size": 16,
|
|
"device": "cuda:0",
|
|
"dtype": "torch.bfloat16",
|
|
"head_dim": 128,
|
|
"kv_cache_dtype": "auto",
|
|
"kv_lora_rank": null,
|
|
"num_kv_heads": 1,
|
|
"num_kv_splits": null,
|
|
"num_layers": 1,
|
|
"num_q_heads": 8,
|
|
"prefill_backend": null,
|
|
"profile_memory": true,
|
|
"qk_nope_head_dim": null,
|
|
"qk_rope_head_dim": null,
|
|
"reorder_batch_threshold": null,
|
|
"repeats": 10,
|
|
"use_cuda_graphs": false,
|
|
"v_head_dim": null,
|
|
"warmup_iters": 5
|
|
},
|
|
"error": null,
|
|
"kv_cache_update_time": {
|
|
"max_ms": 0.033215999603271484,
|
|
"mean_ms": 0.023340800032019614,
|
|
"median_ms": 0.02092800009995699,
|
|
"min_ms": 0.020479999482631683,
|
|
"std_ms": 0.004304974035052475
|
|
},
|
|
"max_time": 0.02418492889404297,
|
|
"mean_time": 0.02411243553161621,
|
|
"memory_allocated_mb": 96.09033203125,
|
|
"memory_reserved_mb": 372.0,
|
|
"min_time": 0.024088096618652344,
|
|
"std_time": 2.797845886790172e-05,
|
|
"tensor_parallel_size": 4,
|
|
"throughput_tokens_per_sec": 254806.28001862322
|
|
},
|
|
{
|
|
"attention_core_excludes_kv_cache_update": true,
|
|
"config": {
|
|
"backend": "FLASH_ATTN",
|
|
"batch_spec": "q1ks8k",
|
|
"block_size": 16,
|
|
"device": "cuda:0",
|
|
"dtype": "torch.bfloat16",
|
|
"head_dim": 128,
|
|
"kv_cache_dtype": "auto",
|
|
"kv_lora_rank": null,
|
|
"num_kv_heads": 1,
|
|
"num_kv_splits": null,
|
|
"num_layers": 1,
|
|
"num_q_heads": 8,
|
|
"prefill_backend": null,
|
|
"profile_memory": true,
|
|
"qk_nope_head_dim": null,
|
|
"qk_rope_head_dim": null,
|
|
"reorder_batch_threshold": null,
|
|
"repeats": 10,
|
|
"use_cuda_graphs": false,
|
|
"v_head_dim": null,
|
|
"warmup_iters": 5
|
|
},
|
|
"error": null,
|
|
"kv_cache_update_time": {
|
|
"max_ms": 0.04992000013589859,
|
|
"mean_ms": 0.019958400167524815,
|
|
"median_ms": 0.015552000608295202,
|
|
"min_ms": 0.014688000082969666,
|
|
"std_ms": 0.010290953685254347
|
|
},
|
|
"max_time": 0.00036675199866294863,
|
|
"mean_time": 0.0003469599992036819,
|
|
"memory_allocated_mb": 8.5205078125,
|
|
"memory_reserved_mb": 372.0,
|
|
"min_time": 0.0003364480137825012,
|
|
"std_time": 7.875187259592915e-06,
|
|
"tensor_parallel_size": 4,
|
|
"throughput_tokens_per_sec": 2951348.865431786
|
|
},
|
|
{
|
|
"attention_core_excludes_kv_cache_update": true,
|
|
"config": {
|
|
"backend": "FLASH_ATTN",
|
|
"batch_spec": "q512s4k",
|
|
"block_size": 16,
|
|
"device": "cuda:0",
|
|
"dtype": "torch.bfloat16",
|
|
"head_dim": 128,
|
|
"kv_cache_dtype": "auto",
|
|
"kv_lora_rank": null,
|
|
"num_kv_heads": 1,
|
|
"num_kv_splits": null,
|
|
"num_layers": 1,
|
|
"num_q_heads": 8,
|
|
"prefill_backend": null,
|
|
"profile_memory": true,
|
|
"qk_nope_head_dim": null,
|
|
"qk_rope_head_dim": null,
|
|
"reorder_batch_threshold": null,
|
|
"repeats": 10,
|
|
"use_cuda_graphs": false,
|
|
"v_head_dim": null,
|
|
"warmup_iters": 5
|
|
},
|
|
"error": null,
|
|
"kv_cache_update_time": {
|
|
"max_ms": 0.022336000576615334,
|
|
"mean_ms": 0.016252800077199935,
|
|
"median_ms": 0.014895999804139137,
|
|
"min_ms": 0.014271999709308147,
|
|
"std_ms": 0.0028271219653727385
|
|
},
|
|
"max_time": 0.00016867199540138245,
|
|
"mean_time": 0.00015103680044412614,
|
|
"memory_allocated_mb": 4.265625,
|
|
"memory_reserved_mb": 372.0,
|
|
"min_time": 0.00014329600334167482,
|
|
"std_time": 7.872199103975806e-06,
|
|
"tensor_parallel_size": 4,
|
|
"throughput_tokens_per_sec": 3389902.3184710997
|
|
}
|
|
],
|
|
"schema_version": "qwen30_vllm020_flashattn_raw.v1"
|
|
}
|