565 lines
17 KiB
JSON
565 lines
17 KiB
JSON
{
|
|
"environment": {
|
|
"attention_backend": "FLASH_ATTN",
|
|
"block_size": 16,
|
|
"dtype": "bfloat16",
|
|
"gpu": "NVIDIA H20",
|
|
"model": "/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B",
|
|
"profile_kv_update": true,
|
|
"profile_method": "cuda_event",
|
|
"torch_cuda": "12.9",
|
|
"torch_version": "2.11.0+cu129",
|
|
"vllm_source_commit": "88d34c6409e9fb3c7b8ca0c04756f061d2099eb1",
|
|
"vllm_version": "0.20.0"
|
|
},
|
|
"rows": [
|
|
{
|
|
"attention_core_excludes_kv_cache_update": true,
|
|
"config": {
|
|
"backend": "FLASH_ATTN",
|
|
"batch_spec": "q8ks48k",
|
|
"block_size": 16,
|
|
"device": "cuda:0",
|
|
"dtype": "torch.bfloat16",
|
|
"head_dim": 128,
|
|
"kv_cache_dtype": "auto",
|
|
"kv_lora_rank": null,
|
|
"num_kv_heads": 2,
|
|
"num_kv_splits": null,
|
|
"num_layers": 1,
|
|
"num_q_heads": 16,
|
|
"prefill_backend": null,
|
|
"profile_memory": true,
|
|
"qk_nope_head_dim": null,
|
|
"qk_rope_head_dim": null,
|
|
"reorder_batch_threshold": null,
|
|
"repeats": 10,
|
|
"use_cuda_graphs": false,
|
|
"v_head_dim": null,
|
|
"warmup_iters": 5
|
|
},
|
|
"error": null,
|
|
"kv_cache_update_time": {
|
|
"max_ms": 0.047520000487565994,
|
|
"mean_ms": 0.04390720054507256,
|
|
"median_ms": 0.043088000267744064,
|
|
"min_ms": 0.041760001331567764,
|
|
"std_ms": 0.0020753894539536763
|
|
},
|
|
"max_time": 0.0226856632232666,
|
|
"mean_time": 0.022550320053100585,
|
|
"memory_allocated_mb": 120.0849609375,
|
|
"memory_reserved_mb": 134.0,
|
|
"min_time": 0.02251683235168457,
|
|
"std_time": 5.03848780842368e-05,
|
|
"tensor_parallel_size": 2,
|
|
"throughput_tokens_per_sec": 363276.440454495
|
|
},
|
|
{
|
|
"attention_core_excludes_kv_cache_update": true,
|
|
"config": {
|
|
"backend": "FLASH_ATTN",
|
|
"batch_spec": "q8ks64k",
|
|
"block_size": 16,
|
|
"device": "cuda:0",
|
|
"dtype": "torch.bfloat16",
|
|
"head_dim": 128,
|
|
"kv_cache_dtype": "auto",
|
|
"kv_lora_rank": null,
|
|
"num_kv_heads": 2,
|
|
"num_kv_splits": null,
|
|
"num_layers": 1,
|
|
"num_q_heads": 16,
|
|
"prefill_backend": null,
|
|
"profile_memory": true,
|
|
"qk_nope_head_dim": null,
|
|
"qk_rope_head_dim": null,
|
|
"reorder_batch_threshold": null,
|
|
"repeats": 10,
|
|
"use_cuda_graphs": false,
|
|
"v_head_dim": null,
|
|
"warmup_iters": 5
|
|
},
|
|
"error": null,
|
|
"kv_cache_update_time": {
|
|
"max_ms": 0.05392000079154968,
|
|
"mean_ms": 0.04477120004594326,
|
|
"median_ms": 0.04323200136423111,
|
|
"min_ms": 0.04150399938225746,
|
|
"std_ms": 0.003602053060896233
|
|
},
|
|
"max_time": 0.03088857650756836,
|
|
"mean_time": 0.030766559982299803,
|
|
"memory_allocated_mb": 136.0888671875,
|
|
"memory_reserved_mb": 198.0,
|
|
"min_time": 0.030721696853637695,
|
|
"std_time": 5.739627345668801e-05,
|
|
"tensor_parallel_size": 2,
|
|
"throughput_tokens_per_sec": 266263.11179127306
|
|
},
|
|
{
|
|
"attention_core_excludes_kv_cache_update": true,
|
|
"config": {
|
|
"backend": "FLASH_ATTN",
|
|
"batch_spec": "q8ks80k",
|
|
"block_size": 16,
|
|
"device": "cuda:0",
|
|
"dtype": "torch.bfloat16",
|
|
"head_dim": 128,
|
|
"kv_cache_dtype": "auto",
|
|
"kv_lora_rank": null,
|
|
"num_kv_heads": 2,
|
|
"num_kv_splits": null,
|
|
"num_layers": 1,
|
|
"num_q_heads": 16,
|
|
"prefill_backend": null,
|
|
"profile_memory": true,
|
|
"qk_nope_head_dim": null,
|
|
"qk_rope_head_dim": null,
|
|
"reorder_batch_threshold": null,
|
|
"repeats": 10,
|
|
"use_cuda_graphs": false,
|
|
"v_head_dim": null,
|
|
"warmup_iters": 5
|
|
},
|
|
"error": null,
|
|
"kv_cache_update_time": {
|
|
"max_ms": 0.04835199937224388,
|
|
"mean_ms": 0.04382400028407574,
|
|
"median_ms": 0.0427200011909008,
|
|
"min_ms": 0.041600000113248825,
|
|
"std_ms": 0.002316465883417635
|
|
},
|
|
"max_time": 0.03911993789672852,
|
|
"mean_time": 0.03901147232055664,
|
|
"memory_allocated_mb": 152.0927734375,
|
|
"memory_reserved_mb": 278.0,
|
|
"min_time": 0.03894742584228516,
|
|
"std_time": 5.231496369987919e-05,
|
|
"tensor_parallel_size": 2,
|
|
"throughput_tokens_per_sec": 209989.51110295113
|
|
},
|
|
{
|
|
"attention_core_excludes_kv_cache_update": true,
|
|
"config": {
|
|
"backend": "FLASH_ATTN",
|
|
"batch_spec": "q8ks96k",
|
|
"block_size": 16,
|
|
"device": "cuda:0",
|
|
"dtype": "torch.bfloat16",
|
|
"head_dim": 128,
|
|
"kv_cache_dtype": "auto",
|
|
"kv_lora_rank": null,
|
|
"num_kv_heads": 2,
|
|
"num_kv_splits": null,
|
|
"num_layers": 1,
|
|
"num_q_heads": 16,
|
|
"prefill_backend": null,
|
|
"profile_memory": true,
|
|
"qk_nope_head_dim": null,
|
|
"qk_rope_head_dim": null,
|
|
"reorder_batch_threshold": null,
|
|
"repeats": 10,
|
|
"use_cuda_graphs": false,
|
|
"v_head_dim": null,
|
|
"warmup_iters": 5
|
|
},
|
|
"error": null,
|
|
"kv_cache_update_time": {
|
|
"max_ms": 0.04902400076389313,
|
|
"mean_ms": 0.04363839998841286,
|
|
"median_ms": 0.04273599945008755,
|
|
"min_ms": 0.04153599962592125,
|
|
"std_ms": 0.002234876899495284
|
|
},
|
|
"max_time": 0.047580448150634766,
|
|
"mean_time": 0.0472187198638916,
|
|
"memory_allocated_mb": 168.0966796875,
|
|
"memory_reserved_mb": 374.0,
|
|
"min_time": 0.04714566421508789,
|
|
"std_time": 0.00012505866815108255,
|
|
"tensor_parallel_size": 2,
|
|
"throughput_tokens_per_sec": 173490.51443185066
|
|
},
|
|
{
|
|
"attention_core_excludes_kv_cache_update": true,
|
|
"config": {
|
|
"backend": "FLASH_ATTN",
|
|
"batch_spec": "q8ks112k",
|
|
"block_size": 16,
|
|
"device": "cuda:0",
|
|
"dtype": "torch.bfloat16",
|
|
"head_dim": 128,
|
|
"kv_cache_dtype": "auto",
|
|
"kv_lora_rank": null,
|
|
"num_kv_heads": 2,
|
|
"num_kv_splits": null,
|
|
"num_layers": 1,
|
|
"num_q_heads": 16,
|
|
"prefill_backend": null,
|
|
"profile_memory": true,
|
|
"qk_nope_head_dim": null,
|
|
"qk_rope_head_dim": null,
|
|
"reorder_batch_threshold": null,
|
|
"repeats": 10,
|
|
"use_cuda_graphs": false,
|
|
"v_head_dim": null,
|
|
"warmup_iters": 5
|
|
},
|
|
"error": null,
|
|
"kv_cache_update_time": {
|
|
"max_ms": 0.04848000034689903,
|
|
"mean_ms": 0.04378239996731281,
|
|
"median_ms": 0.042527999728918076,
|
|
"min_ms": 0.04163200035691261,
|
|
"std_ms": 0.002414242667964844
|
|
},
|
|
"max_time": 0.05583651351928711,
|
|
"mean_time": 0.055455712127685554,
|
|
"memory_allocated_mb": 184.1005859375,
|
|
"memory_reserved_mb": 486.0,
|
|
"min_time": 0.05535052871704101,
|
|
"std_time": 0.00014272069779462445,
|
|
"tensor_parallel_size": 2,
|
|
"throughput_tokens_per_sec": 147721.48234501254
|
|
},
|
|
{
|
|
"attention_core_excludes_kv_cache_update": true,
|
|
"config": {
|
|
"backend": "FLASH_ATTN",
|
|
"batch_spec": "q8ks128k",
|
|
"block_size": 16,
|
|
"device": "cuda:0",
|
|
"dtype": "torch.bfloat16",
|
|
"head_dim": 128,
|
|
"kv_cache_dtype": "auto",
|
|
"kv_lora_rank": null,
|
|
"num_kv_heads": 2,
|
|
"num_kv_splits": null,
|
|
"num_layers": 1,
|
|
"num_q_heads": 16,
|
|
"prefill_backend": null,
|
|
"profile_memory": true,
|
|
"qk_nope_head_dim": null,
|
|
"qk_rope_head_dim": null,
|
|
"reorder_batch_threshold": null,
|
|
"repeats": 10,
|
|
"use_cuda_graphs": false,
|
|
"v_head_dim": null,
|
|
"warmup_iters": 5
|
|
},
|
|
"error": null,
|
|
"kv_cache_update_time": {
|
|
"max_ms": 0.04851200059056282,
|
|
"mean_ms": 0.04361280016601086,
|
|
"median_ms": 0.042128000408411026,
|
|
"min_ms": 0.04179200157523155,
|
|
"std_ms": 0.0023557503035821635
|
|
},
|
|
"max_time": 0.06379235076904297,
|
|
"mean_time": 0.06362125053405762,
|
|
"memory_allocated_mb": 200.1044921875,
|
|
"memory_reserved_mb": 614.0,
|
|
"min_time": 0.06355436706542969,
|
|
"std_time": 7.081883106781061e-05,
|
|
"tensor_parallel_size": 2,
|
|
"throughput_tokens_per_sec": 128762.00846782589
|
|
},
|
|
{
|
|
"attention_core_excludes_kv_cache_update": true,
|
|
"config": {
|
|
"backend": "FLASH_ATTN",
|
|
"batch_spec": "q8ks136k",
|
|
"block_size": 16,
|
|
"device": "cuda:0",
|
|
"dtype": "torch.bfloat16",
|
|
"head_dim": 128,
|
|
"kv_cache_dtype": "auto",
|
|
"kv_lora_rank": null,
|
|
"num_kv_heads": 2,
|
|
"num_kv_splits": null,
|
|
"num_layers": 1,
|
|
"num_q_heads": 16,
|
|
"prefill_backend": null,
|
|
"profile_memory": true,
|
|
"qk_nope_head_dim": null,
|
|
"qk_rope_head_dim": null,
|
|
"reorder_batch_threshold": null,
|
|
"repeats": 10,
|
|
"use_cuda_graphs": false,
|
|
"v_head_dim": null,
|
|
"warmup_iters": 5
|
|
},
|
|
"error": null,
|
|
"kv_cache_update_time": {
|
|
"max_ms": 0.05718399956822395,
|
|
"mean_ms": 0.04485439993441105,
|
|
"median_ms": 0.042527999728918076,
|
|
"min_ms": 0.04156799986958504,
|
|
"std_ms": 0.004775190073319638
|
|
},
|
|
"max_time": 0.06783235168457032,
|
|
"mean_time": 0.06771669235229492,
|
|
"memory_allocated_mb": 208.1064453125,
|
|
"memory_reserved_mb": 750.0,
|
|
"min_time": 0.06765017700195312,
|
|
"std_time": 5.8650788028093655e-05,
|
|
"tensor_parallel_size": 2,
|
|
"throughput_tokens_per_sec": 120974.603387024
|
|
},
|
|
{
|
|
"attention_core_excludes_kv_cache_update": true,
|
|
"config": {
|
|
"backend": "FLASH_ATTN",
|
|
"batch_spec": "q512s128k",
|
|
"block_size": 16,
|
|
"device": "cuda:0",
|
|
"dtype": "torch.bfloat16",
|
|
"head_dim": 128,
|
|
"kv_cache_dtype": "auto",
|
|
"kv_lora_rank": null,
|
|
"num_kv_heads": 2,
|
|
"num_kv_splits": null,
|
|
"num_layers": 1,
|
|
"num_q_heads": 16,
|
|
"prefill_backend": null,
|
|
"profile_memory": true,
|
|
"qk_nope_head_dim": null,
|
|
"qk_rope_head_dim": null,
|
|
"reorder_batch_threshold": null,
|
|
"repeats": 10,
|
|
"use_cuda_graphs": false,
|
|
"v_head_dim": null,
|
|
"warmup_iters": 5
|
|
},
|
|
"error": null,
|
|
"kv_cache_update_time": {
|
|
"max_ms": 0.019807999953627586,
|
|
"mean_ms": 0.015823999978601934,
|
|
"median_ms": 0.015039999969303608,
|
|
"min_ms": 0.014303999952971935,
|
|
"std_ms": 0.001913656346272671
|
|
},
|
|
"max_time": 0.004766176223754883,
|
|
"mean_time": 0.0047575551986694335,
|
|
"memory_allocated_mb": 132.5458984375,
|
|
"memory_reserved_mb": 750.0,
|
|
"min_time": 0.004751776218414307,
|
|
"std_time": 4.190866412679079e-06,
|
|
"tensor_parallel_size": 2,
|
|
"throughput_tokens_per_sec": 107618.29944573072
|
|
},
|
|
{
|
|
"attention_core_excludes_kv_cache_update": true,
|
|
"config": {
|
|
"backend": "FLASH_ATTN",
|
|
"batch_spec": "q2ks66k",
|
|
"block_size": 16,
|
|
"device": "cuda:0",
|
|
"dtype": "torch.bfloat16",
|
|
"head_dim": 128,
|
|
"kv_cache_dtype": "auto",
|
|
"kv_lora_rank": null,
|
|
"num_kv_heads": 2,
|
|
"num_kv_splits": null,
|
|
"num_layers": 1,
|
|
"num_q_heads": 16,
|
|
"prefill_backend": null,
|
|
"profile_memory": true,
|
|
"qk_nope_head_dim": null,
|
|
"qk_rope_head_dim": null,
|
|
"reorder_batch_threshold": null,
|
|
"repeats": 10,
|
|
"use_cuda_graphs": false,
|
|
"v_head_dim": null,
|
|
"warmup_iters": 5
|
|
},
|
|
"error": null,
|
|
"kv_cache_update_time": {
|
|
"max_ms": 0.02687999978661537,
|
|
"mean_ms": 0.01938559990376234,
|
|
"median_ms": 0.018383999355137348,
|
|
"min_ms": 0.018015999346971512,
|
|
"std_ms": 0.0025744710494052777
|
|
},
|
|
"max_time": 0.009582624435424805,
|
|
"mean_time": 0.009561939239501951,
|
|
"memory_allocated_mb": 84.04248046875,
|
|
"memory_reserved_mb": 752.0,
|
|
"min_time": 0.009554431915283204,
|
|
"std_time": 8.291579854680458e-06,
|
|
"tensor_parallel_size": 2,
|
|
"throughput_tokens_per_sec": 214182.49464913702
|
|
},
|
|
{
|
|
"attention_core_excludes_kv_cache_update": true,
|
|
"config": {
|
|
"backend": "FLASH_ATTN",
|
|
"batch_spec": "q4ks100k",
|
|
"block_size": 16,
|
|
"device": "cuda:0",
|
|
"dtype": "torch.bfloat16",
|
|
"head_dim": 128,
|
|
"kv_cache_dtype": "auto",
|
|
"kv_lora_rank": null,
|
|
"num_kv_heads": 2,
|
|
"num_kv_splits": null,
|
|
"num_layers": 1,
|
|
"num_q_heads": 16,
|
|
"prefill_backend": null,
|
|
"profile_memory": true,
|
|
"qk_nope_head_dim": null,
|
|
"qk_rope_head_dim": null,
|
|
"reorder_batch_threshold": null,
|
|
"repeats": 10,
|
|
"use_cuda_graphs": false,
|
|
"v_head_dim": null,
|
|
"warmup_iters": 5
|
|
},
|
|
"error": null,
|
|
"kv_cache_update_time": {
|
|
"max_ms": 0.03206399828195572,
|
|
"mean_ms": 0.028105599991977214,
|
|
"median_ms": 0.02675200067460537,
|
|
"min_ms": 0.025728000327944756,
|
|
"std_ms": 0.0022390414558466644
|
|
},
|
|
"max_time": 0.02538035202026367,
|
|
"mean_time": 0.0252856897354126,
|
|
"memory_allocated_mb": 136.06640625,
|
|
"memory_reserved_mb": 752.0,
|
|
"min_time": 0.025221471786499024,
|
|
"std_time": 4.676405035080222e-05,
|
|
"tensor_parallel_size": 2,
|
|
"throughput_tokens_per_sec": 161988.85784252718
|
|
},
|
|
{
|
|
"attention_core_excludes_kv_cache_update": true,
|
|
"config": {
|
|
"backend": "FLASH_ATTN",
|
|
"batch_spec": "q6ks134k",
|
|
"block_size": 16,
|
|
"device": "cuda:0",
|
|
"dtype": "torch.bfloat16",
|
|
"head_dim": 128,
|
|
"kv_cache_dtype": "auto",
|
|
"kv_lora_rank": null,
|
|
"num_kv_heads": 2,
|
|
"num_kv_splits": null,
|
|
"num_layers": 1,
|
|
"num_q_heads": 16,
|
|
"prefill_backend": null,
|
|
"profile_memory": true,
|
|
"qk_nope_head_dim": null,
|
|
"qk_rope_head_dim": null,
|
|
"reorder_batch_threshold": null,
|
|
"repeats": 10,
|
|
"use_cuda_graphs": false,
|
|
"v_head_dim": null,
|
|
"warmup_iters": 5
|
|
},
|
|
"error": null,
|
|
"kv_cache_update_time": {
|
|
"max_ms": 0.04089599847793579,
|
|
"mean_ms": 0.03524160049855709,
|
|
"median_ms": 0.03411199897527695,
|
|
"min_ms": 0.03385600075125694,
|
|
"std_ms": 0.0022532652248236193
|
|
},
|
|
"max_time": 0.048259136199951175,
|
|
"mean_time": 0.048143408584594725,
|
|
"memory_allocated_mb": 188.09033203125,
|
|
"memory_reserved_mb": 752.0,
|
|
"min_time": 0.048105857849121095,
|
|
"std_time": 5.1879732806325464e-05,
|
|
"tensor_parallel_size": 2,
|
|
"throughput_tokens_per_sec": 127618.7162611083
|
|
},
|
|
{
|
|
"attention_core_excludes_kv_cache_update": true,
|
|
"config": {
|
|
"backend": "FLASH_ATTN",
|
|
"batch_spec": "q1ks8k",
|
|
"block_size": 16,
|
|
"device": "cuda:0",
|
|
"dtype": "torch.bfloat16",
|
|
"head_dim": 128,
|
|
"kv_cache_dtype": "auto",
|
|
"kv_lora_rank": null,
|
|
"num_kv_heads": 2,
|
|
"num_kv_splits": null,
|
|
"num_layers": 1,
|
|
"num_q_heads": 16,
|
|
"prefill_backend": null,
|
|
"profile_memory": true,
|
|
"qk_nope_head_dim": null,
|
|
"qk_rope_head_dim": null,
|
|
"reorder_batch_threshold": null,
|
|
"repeats": 10,
|
|
"use_cuda_graphs": false,
|
|
"v_head_dim": null,
|
|
"warmup_iters": 5
|
|
},
|
|
"error": null,
|
|
"kv_cache_update_time": {
|
|
"max_ms": 0.018783999606966972,
|
|
"mean_ms": 0.015311999991536141,
|
|
"median_ms": 0.014943999703973532,
|
|
"min_ms": 0.014271999709308147,
|
|
"std_ms": 0.0012243293501672883
|
|
},
|
|
"max_time": 0.0008861759901046753,
|
|
"mean_time": 0.0006353183984756471,
|
|
"memory_allocated_mb": 17.0205078125,
|
|
"memory_reserved_mb": 752.0,
|
|
"min_time": 0.0006019840240478516,
|
|
"std_time": 8.386037831062328e-05,
|
|
"tensor_parallel_size": 2,
|
|
"throughput_tokens_per_sec": 1611790.2495141604
|
|
},
|
|
{
|
|
"attention_core_excludes_kv_cache_update": true,
|
|
"config": {
|
|
"backend": "FLASH_ATTN",
|
|
"batch_spec": "q512s4k",
|
|
"block_size": 16,
|
|
"device": "cuda:0",
|
|
"dtype": "torch.bfloat16",
|
|
"head_dim": 128,
|
|
"kv_cache_dtype": "auto",
|
|
"kv_lora_rank": null,
|
|
"num_kv_heads": 2,
|
|
"num_kv_splits": null,
|
|
"num_layers": 1,
|
|
"num_q_heads": 16,
|
|
"prefill_backend": null,
|
|
"profile_memory": true,
|
|
"qk_nope_head_dim": null,
|
|
"qk_rope_head_dim": null,
|
|
"reorder_batch_threshold": null,
|
|
"repeats": 10,
|
|
"use_cuda_graphs": false,
|
|
"v_head_dim": null,
|
|
"warmup_iters": 5
|
|
},
|
|
"error": null,
|
|
"kv_cache_update_time": {
|
|
"max_ms": 0.019840000197291374,
|
|
"mean_ms": 0.015875199995934963,
|
|
"median_ms": 0.01497599994763732,
|
|
"min_ms": 0.014431999996304512,
|
|
"std_ms": 0.001896142444948042
|
|
},
|
|
"max_time": 0.00022230400145053863,
|
|
"mean_time": 0.00020609600096940998,
|
|
"memory_allocated_mb": 8.515625,
|
|
"memory_reserved_mb": 752.0,
|
|
"min_time": 0.00019760000705718993,
|
|
"std_time": 7.520956759601396e-06,
|
|
"tensor_parallel_size": 2,
|
|
"throughput_tokens_per_sec": 2484279.1591865686
|
|
}
|
|
],
|
|
"schema_version": "qwen30_vllm020_flashattn_raw.v1"
|
|
}
|