gahow/aituner
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Evaluate vLLM 0.20 profiles against Frontier

Browse Source
This commit is contained in:
Gahow Wang
2026-07-16 23:59:10 +08:00
parent 008324e70c
commit 76107d3e87
33 changed files with 13973 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

824
runs/frontier-qwen30-vllm020-profile-v1/frozen/profile-v1/allreduce.json Normal file
View File

@@ -0,0 +1,824 @@
{
"environment": [
{
"backend_env": {
"VLLM_ALLREDUCE_USE_FLASHINFER": "1",
"VLLM_FLASHINFER_ALLREDUCE_BACKEND": "trtllm"
},
"gpu": "NVIDIA H20",
"model": "/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B",
"torch_cuda": "12.9",
"torch_version": "2.11.0+cu129",
"vllm_source_commit": "88d34c6409e9fb3c7b8ca0c04756f061d2099eb1",
"vllm_version": "0.20.0"
},
{
"backend_env": {
"VLLM_ALLREDUCE_USE_FLASHINFER": "1",
"VLLM_FLASHINFER_ALLREDUCE_BACKEND": "trtllm"
},
"gpu": "NVIDIA H20",
"model": "/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B",
"torch_cuda": "12.9",
"torch_version": "2.11.0+cu129",
"vllm_source_commit": "88d34c6409e9fb3c7b8ca0c04756f061d2099eb1",
"vllm_version": "0.20.0"
}
],
"frontier_consumption": "diagnostic_only_in_base_profile_only_run; measured lookup requires a separate CC-backend injection ablation",
"rows": [
{
"communicator": "vllm.tensor_model_parallel_all_reduce",
"critical_path_median_ms": 0.08288000151515007,
"dtype": "bfloat16",
"hidden_dim": 2048,
"num_tokens": 1,
"payload_bytes": 4096,
"per_rank_time_ms": [
{
"max": 0.4872959852218628,
"mean": 0.1310015980154276,
"median": 0.07679999992251396,
"min": 0.06217600032687187,
"std": 0.12790721677293843
},
{
"max": 0.4402880072593689,
"mean": 0.12842560112476348,
"median": 0.08288000151515007,
"min": 0.0655680000782013,
"std": 0.11220176524616535
}
],
"selected_backend": "flashinfer_trtllm",
"tensor_parallel_size": 2
},
{
"communicator": "vllm.tensor_model_parallel_all_reduce",
"critical_path_median_ms": 0.0793600007891655,
"dtype": "bfloat16",
"hidden_dim": 2048,
"num_tokens": 8,
"payload_bytes": 32768,
"per_rank_time_ms": [
{
"max": 0.12716799974441528,
"mean": 0.07871360033750534,
"median": 0.0759200006723404,
"min": 0.06032000109553337,
"std": 0.019331314939874535
},
{
"max": 0.12380799651145935,
"mean": 0.08059840016067028,
"median": 0.0793600007891655,
"min": 0.06217600032687187,
"std": 0.01687088356254092
}
],
"selected_backend": "flashinfer_trtllm",
"tensor_parallel_size": 2
},
{
"communicator": "vllm.tensor_model_parallel_all_reduce",
"critical_path_median_ms": 0.0713919997215271,
"dtype": "bfloat16",
"hidden_dim": 2048,
"num_tokens": 16,
"payload_bytes": 65536,
"per_rank_time_ms": [
{
"max": 0.12697599828243256,
"mean": 0.0767391998320818,
"median": 0.07078400254249573,
"min": 0.05910399928689003,
"std": 0.018775178979463278
},
{
"max": 0.11430399864912033,
"mean": 0.07594559974968433,
"median": 0.0713919997215271,
"min": 0.06124800071120262,
"std": 0.015719922150631036
}
],
"selected_backend": "flashinfer_trtllm",
"tensor_parallel_size": 2
},
{
"communicator": "vllm.tensor_model_parallel_all_reduce",
"critical_path_median_ms": 0.08056000247597694,
"dtype": "bfloat16",
"hidden_dim": 2048,
"num_tokens": 32,
"payload_bytes": 131072,
"per_rank_time_ms": [
{
"max": 0.1037760004401207,
"mean": 0.07954559996724128,
"median": 0.08056000247597694,
"min": 0.05955199897289276,
"std": 0.0135697420393132
},
{
"max": 0.10608000308275223,
"mean": 0.07971520014107228,
"median": 0.07593599706888199,
"min": 0.06028800085186958,
"std": 0.015319849772775456
}
],
"selected_backend": "flashinfer_trtllm",
"tensor_parallel_size": 2
},
{
"communicator": "vllm.tensor_model_parallel_all_reduce",
"critical_path_median_ms": 0.0865279994904995,
"dtype": "bfloat16",
"hidden_dim": 2048,
"num_tokens": 64,
"payload_bytes": 262144,
"per_rank_time_ms": [
{
"max": 0.14470399916172028,
"mean": 0.09121599942445754,
"median": 0.0865279994904995,
"min": 0.06441599875688553,
"std": 0.024893837894277456
},
{
"max": 0.12438400089740753,
"mean": 0.08531199917197227,
"median": 0.08031999692320824,
"min": 0.06364800035953522,
"std": 0.01878029830059533
}
],
"selected_backend": "flashinfer_trtllm",
"tensor_parallel_size": 2
},
{
"communicator": "vllm.tensor_model_parallel_all_reduce",
"critical_path_median_ms": 0.07135999947786331,
"dtype": "bfloat16",
"hidden_dim": 2048,
"num_tokens": 128,
"payload_bytes": 524288,
"per_rank_time_ms": [
{
"max": 0.11753600090742111,
"mean": 0.07606079950928687,
"median": 0.07135999947786331,
"min": 0.05843200162053108,
"std": 0.01755519771639284
},
{
"max": 0.1103999987244606,
"mean": 0.07607359997928143,
"median": 0.07073600217700005,
"min": 0.05721599981188774,
"std": 0.016904445220949783
}
],
"selected_backend": "flashinfer_trtllm",
"tensor_parallel_size": 2
},
{
"communicator": "vllm.tensor_model_parallel_all_reduce",
"critical_path_median_ms": 0.07321599870920181,
"dtype": "bfloat16",
"hidden_dim": 2048,
"num_tokens": 256,
"payload_bytes": 1048576,
"per_rank_time_ms": [
{
"max": 0.11740799993276596,
"mean": 0.07749119997024537,
"median": 0.07203200086951256,
"min": 0.05862399935722351,
"std": 0.017701381594822835
},
{
"max": 0.11382400244474411,
"mean": 0.07733759954571724,
"median": 0.07321599870920181,
"min": 0.059039998799562454,
"std": 0.017188890882557036
}
],
"selected_backend": "flashinfer_trtllm",
"tensor_parallel_size": 2
},
{
"communicator": "vllm.tensor_model_parallel_all_reduce",
"critical_path_median_ms": 0.09025600180029869,
"dtype": "bfloat16",
"hidden_dim": 2048,
"num_tokens": 512,
"payload_bytes": 2097152,
"per_rank_time_ms": [
{
"max": 0.13980799913406372,
"mean": 0.0950367994606495,
"median": 0.09025600180029869,
"min": 0.06815999746322632,
"std": 0.022829835127539378
},
{
"max": 0.14764800667762756,
"mean": 0.09710080176591873,
"median": 0.08720000088214874,
"min": 0.07152000069618225,
"std": 0.02500574954063789
}
],
"selected_backend": "flashinfer_trtllm",
"tensor_parallel_size": 2
},
{
"communicator": "vllm.tensor_model_parallel_all_reduce",
"critical_path_median_ms": 0.08083200082182884,
"dtype": "bfloat16",
"hidden_dim": 2048,
"num_tokens": 1024,
"payload_bytes": 4194304,
"per_rank_time_ms": [
{
"max": 0.15574400126934052,
"mean": 0.08760640025138855,
"median": 0.07846399769186974,
"min": 0.07097599655389786,
"std": 0.024487311423551025
},
{
"max": 0.16284799575805664,
"mean": 0.08963519930839539,
"median": 0.08083200082182884,
"min": 0.07100799679756165,
"std": 0.026047320562445356
}
],
"selected_backend": "flashinfer_trtllm",
"tensor_parallel_size": 2
},
{
"communicator": "vllm.tensor_model_parallel_all_reduce",
"critical_path_median_ms": 0.10891199856996536,
"dtype": "bfloat16",
"hidden_dim": 2048,
"num_tokens": 2048,
"payload_bytes": 8388608,
"per_rank_time_ms": [
{
"max": 0.1582079976797104,
"mean": 0.11541439890861512,
"median": 0.10836799815297127,
"min": 0.09062399715185165,
"std": 0.018285136316576037
},
{
"max": 0.1578879952430725,
"mean": 0.11537599861621857,
"median": 0.10891199856996536,
"min": 0.0960640013217926,
"std": 0.018246538626977286
}
],
"selected_backend": "flashinfer_trtllm",
"tensor_parallel_size": 2
},
{
"communicator": "vllm.tensor_model_parallel_all_reduce",
"critical_path_median_ms": 0.1703840047121048,
"dtype": "bfloat16",
"hidden_dim": 2048,
"num_tokens": 4096,
"payload_bytes": 16777216,
"per_rank_time_ms": [
{
"max": 0.19327999651432037,
"mean": 0.1707327976822853,
"median": 0.1703840047121048,
"min": 0.14815999567508698,
"std": 0.014211056022719618
},
{
"max": 0.19276799261569977,
"mean": 0.1658592015504837,
"median": 0.16379200667142868,
"min": 0.14560000598430634,
"std": 0.013840249648693638
}
],
"selected_backend": "flashinfer_trtllm",
"tensor_parallel_size": 2
},
{
"communicator": "vllm.tensor_model_parallel_all_reduce",
"critical_path_median_ms": 0.25539200007915497,
"dtype": "bfloat16",
"hidden_dim": 2048,
"num_tokens": 8192,
"payload_bytes": 33554432,
"per_rank_time_ms": [
{
"max": 0.2807359993457794,
"mean": 0.25750079900026324,
"median": 0.25539200007915497,
"min": 0.24624000489711761,
"std": 0.008925204570802302
},
{
"max": 0.2863999903202057,
"mean": 0.2585055992007256,
"median": 0.255280002951622,
"min": 0.24371199309825897,
"std": 0.012059738582656496
}
],
"selected_backend": "flashinfer_trtllm",
"tensor_parallel_size": 2
},
{
"communicator": "vllm.tensor_model_parallel_all_reduce",
"critical_path_median_ms": 0.1021759994328022,
"dtype": "bfloat16",
"hidden_dim": 2048,
"num_tokens": 1,
"payload_bytes": 4096,
"per_rank_time_ms": [
{
"max": 0.9770879745483398,
"mean": 0.1974783968180418,
"median": 0.10099200159311295,
"min": 0.05913599953055382,
"std": 0.2660581738745569
},
{
"max": 0.892799973487854,
"mean": 0.18164799660444259,
"median": 0.1021759994328022,
"min": 0.06435199826955795,
"std": 0.23947520188197013
},
{
"max": 0.6467199921607971,
"mean": 0.15839359983801843,
"median": 0.10100800171494484,
"min": 0.06800000369548798,
"std": 0.16617013141866102
},
{
"max": 0.6725760102272034,
"mean": 0.15686400160193442,
"median": 0.10044800117611885,
"min": 0.06063999980688095,
"std": 0.17523222161300497
}
],
"selected_backend": "flashinfer_trtllm",
"tensor_parallel_size": 4
},
{
"communicator": "vllm.tensor_model_parallel_all_reduce",
"critical_path_median_ms": 0.12694399803876877,
"dtype": "bfloat16",
"hidden_dim": 2048,
"num_tokens": 8,
"payload_bytes": 32768,
"per_rank_time_ms": [
{
"max": 0.6659520268440247,
"mean": 0.1927648030221462,
"median": 0.12694399803876877,
"min": 0.07609599828720093,
"std": 0.16697784531907736
},
{
"max": 0.695360004901886,
"mean": 0.19356480240821838,
"median": 0.11726400256156921,
"min": 0.0796160027384758,
"std": 0.17588096678867862
},
{
"max": 0.5939840078353882,
"mean": 0.1868800014257431,
"median": 0.12379200011491776,
"min": 0.07427199929952621,
"std": 0.1463231714943902
},
{
"max": 0.6635839939117432,
"mean": 0.1874335989356041,
"median": 0.12014400213956833,
"min": 0.07526399940252304,
"std": 0.16680959677760304
}
],
"selected_backend": "flashinfer_trtllm",
"tensor_parallel_size": 4
},
{
"communicator": "vllm.tensor_model_parallel_all_reduce",
"critical_path_median_ms": 0.09161599725484848,
"dtype": "bfloat16",
"hidden_dim": 2048,
"num_tokens": 16,
"payload_bytes": 65536,
"per_rank_time_ms": [
{
"max": 0.12665599584579468,
"mean": 0.09443839862942696,
"median": 0.09124799817800522,
"min": 0.06431999802589417,
"std": 0.0205690155775906
},
{
"max": 0.1303039938211441,
"mean": 0.09712959825992584,
"median": 0.09161599725484848,
"min": 0.07648000121116638,
"std": 0.019072048129173236
},
{
"max": 0.13836799561977386,
"mean": 0.09821119979023933,
"median": 0.09148800000548363,
"min": 0.0727040022611618,
"std": 0.021314066545189116
},
{
"max": 0.12992000579833984,
"mean": 0.09272959977388381,
"median": 0.08931199833750725,
"min": 0.06406400352716446,
"std": 0.021360803830354765
}
],
"selected_backend": "flashinfer_trtllm",
"tensor_parallel_size": 4
},
{
"communicator": "vllm.tensor_model_parallel_all_reduce",
"critical_path_median_ms": 0.08580800145864487,
"dtype": "bfloat16",
"hidden_dim": 2048,
"num_tokens": 32,
"payload_bytes": 131072,
"per_rank_time_ms": [
{
"max": 0.13449600338935852,
"mean": 0.08960640132427215,
"median": 0.08299200236797333,
"min": 0.06796800345182419,
"std": 0.020633597898445998
},
{
"max": 0.14735999703407288,
"mean": 0.09248319901525974,
"median": 0.08580800145864487,
"min": 0.05913599953055382,
"std": 0.025960234928829564
},
{
"max": 0.13705599308013916,
"mean": 0.08947199806571007,
"median": 0.08460799977183342,
"min": 0.0634239986538887,
"std": 0.02122838946992339
},
{
"max": 0.13846400380134583,
"mean": 0.0859104000031948,
"median": 0.08308799937367439,
"min": 0.05974400043487549,
"std": 0.022373631424433445
}
],
"selected_backend": "flashinfer_trtllm",
"tensor_parallel_size": 4
},
{
"communicator": "vllm.tensor_model_parallel_all_reduce",
"critical_path_median_ms": 0.09867199882864952,
"dtype": "bfloat16",
"hidden_dim": 2048,
"num_tokens": 64,
"payload_bytes": 262144,
"per_rank_time_ms": [
{
"max": 0.1279039978981018,
"mean": 0.09748799875378608,
"median": 0.09678399935364723,
"min": 0.0655359998345375,
"std": 0.02315719144614622
},
{
"max": 0.1356479972600937,
"mean": 0.10018239840865135,
"median": 0.09532799944281578,
"min": 0.06185600161552429,
"std": 0.02330700253650042
},
{
"max": 0.13142399489879608,
"mean": 0.09778879955410957,
"median": 0.09492799639701843,
"min": 0.06560000032186508,
"std": 0.02307579212430045
},
{
"max": 0.1276479959487915,
"mean": 0.09611519873142242,
"median": 0.09867199882864952,
"min": 0.0642239972949028,
"std": 0.02273612181973431
}
],
"selected_backend": "flashinfer_trtllm",
"tensor_parallel_size": 4
},
{
"communicator": "vllm.tensor_model_parallel_all_reduce",
"critical_path_median_ms": 0.09646400064229965,
"dtype": "bfloat16",
"hidden_dim": 2048,
"num_tokens": 128,
"payload_bytes": 524288,
"per_rank_time_ms": [
{
"max": 0.14060799777507782,
"mean": 0.09770880043506622,
"median": 0.09115200117230415,
"min": 0.06950400024652481,
"std": 0.024752645089849798
},
{
"max": 0.14377599954605103,
"mean": 0.09824960008263588,
"median": 0.08999999985098839,
"min": 0.07103999704122543,
"std": 0.025403407389046027
},
{
"max": 0.13680000603199005,
"mean": 0.09993600100278854,
"median": 0.09646400064229965,
"min": 0.06790400296449661,
"std": 0.022297424273985882
},
{
"max": 0.1391039937734604,
"mean": 0.09769919961690902,
"median": 0.09601600095629692,
"min": 0.06835199892520905,
"std": 0.023899922100804445
}
],
"selected_backend": "flashinfer_trtllm",
"tensor_parallel_size": 4
},
{
"communicator": "vllm.tensor_model_parallel_all_reduce",
"critical_path_median_ms": 0.08377600088715553,
"dtype": "bfloat16",
"hidden_dim": 2048,
"num_tokens": 256,
"payload_bytes": 1048576,
"per_rank_time_ms": [
{
"max": 0.1464959979057312,
"mean": 0.09276160076260567,
"median": 0.08377600088715553,
"min": 0.06777600198984146,
"std": 0.024751086465295658
},
{
"max": 0.14319999516010284,
"mean": 0.09080640003085136,
"median": 0.080400001257658,
"min": 0.06796800345182419,
"std": 0.023365718881708488
},
{
"max": 0.1382399946451187,
"mean": 0.09063360020518303,
"median": 0.08193599805235863,
"min": 0.06627199798822403,
"std": 0.02372618650854302
},
{
"max": 0.14313599467277527,
"mean": 0.09044799953699112,
"median": 0.08128000050783157,
"min": 0.06652799993753433,
"std": 0.02462486862033686
}
],
"selected_backend": "flashinfer_trtllm",
"tensor_parallel_size": 4
},
{
"communicator": "vllm.tensor_model_parallel_all_reduce",
"critical_path_median_ms": 0.1128000020980835,
"dtype": "bfloat16",
"hidden_dim": 2048,
"num_tokens": 512,
"payload_bytes": 2097152,
"per_rank_time_ms": [
{
"max": 0.1600320041179657,
"mean": 0.11426239982247352,
"median": 0.1128000020980835,
"min": 0.07657600194215775,
"std": 0.030240087702350687
},
{
"max": 0.15881599485874176,
"mean": 0.11206399947404862,
"median": 0.10628800094127655,
"min": 0.0772159993648529,
"std": 0.029933135345483627
},
{
"max": 0.15612800419330597,
"mean": 0.10761600062251091,
"median": 0.09860799834132195,
"min": 0.07689599692821503,
"std": 0.02634237020678647
},
{
"max": 0.15865600109100342,
"mean": 0.11094079986214637,
"median": 0.10979199782013893,
"min": 0.07583999633789062,
"std": 0.028619217028542445
}
],
"selected_backend": "flashinfer_trtllm",
"tensor_parallel_size": 4
},
{
"communicator": "vllm.tensor_model_parallel_all_reduce",
"critical_path_median_ms": 0.08755199983716011,
"dtype": "bfloat16",
"hidden_dim": 2048,
"num_tokens": 1024,
"payload_bytes": 4194304,
"per_rank_time_ms": [
{
"max": 0.11999999731779099,
"mean": 0.0843871995806694,
"median": 0.08755199983716011,
"min": 0.06332799792289734,
"std": 0.016638543890716024
},
{
"max": 0.1218239963054657,
"mean": 0.08518079966306687,
"median": 0.08032000064849854,
"min": 0.06393600255250931,
"std": 0.019900899429956945
},
{
"max": 0.11849600076675415,
"mean": 0.0843968003988266,
"median": 0.08702399954199791,
"min": 0.06297600269317627,
"std": 0.017031908773433545
},
{
"max": 0.12300799787044525,
"mean": 0.0846304003149271,
"median": 0.08139199763536453,
"min": 0.06195199862122536,
"std": 0.020106523698622265
}
],
"selected_backend": "nccl_fallback",
"tensor_parallel_size": 4
},
{
"communicator": "vllm.tensor_model_parallel_all_reduce",
"critical_path_median_ms": 0.12361599877476692,
"dtype": "bfloat16",
"hidden_dim": 2048,
"num_tokens": 2048,
"payload_bytes": 8388608,
"per_rank_time_ms": [
{
"max": 0.17155200242996216,
"mean": 0.12168959975242614,
"median": 0.12055999785661697,
"min": 0.09609600156545639,
"std": 0.022565485532483928
},
{
"max": 0.9246399998664856,
"mean": 0.19978560134768486,
"median": 0.12361599877476692,
"min": 0.09715200215578079,
"std": 0.24230694305662265
},
{
"max": 0.9317439794540405,
"mean": 0.20037759989500045,
"median": 0.12327999994158745,
"min": 0.09603200107812881,
"std": 0.24450903278000383
},
{
"max": 0.9321280121803284,
"mean": 0.19875840097665787,
"median": 0.12230399996042252,
"min": 0.0950080007314682,
"std": 0.24519252220785093
}
],
"selected_backend": "nccl_fallback",
"tensor_parallel_size": 4
},
{
"communicator": "vllm.tensor_model_parallel_all_reduce",
"critical_path_median_ms": 0.20030399411916733,
"dtype": "bfloat16",
"hidden_dim": 2048,
"num_tokens": 4096,
"payload_bytes": 16777216,
"per_rank_time_ms": [
{
"max": 0.3261440098285675,
"mean": 0.20826880186796187,
"median": 0.19974400103092194,
"min": 0.1409280002117157,
"std": 0.051397264855718945
},
{
"max": 0.3248000144958496,
"mean": 0.20548800230026246,
"median": 0.1979840025305748,
"min": 0.141184002161026,
"std": 0.04980002399656717
},
{
"max": 0.32547199726104736,
"mean": 0.21280319690704347,
"median": 0.20030399411916733,
"min": 0.14127999544143677,
"std": 0.05304243085685509
},
{
"max": 0.26047998666763306,
"mean": 0.1969312012195587,
"median": 0.18079999834299088,
"min": 0.14057600498199463,
"std": 0.042174123638424224
}
],
"selected_backend": "nccl_fallback",
"tensor_parallel_size": 4
},
{
"communicator": "vllm.tensor_model_parallel_all_reduce",
"critical_path_median_ms": 0.2924960106611252,
"dtype": "bfloat16",
"hidden_dim": 2048,
"num_tokens": 8192,
"payload_bytes": 33554432,
"per_rank_time_ms": [
{
"max": 0.45052799582481384,
"mean": 0.3013375997543335,
"median": 0.28273600339889526,
"min": 0.21334399282932281,
"std": 0.07214223764527236
},
{
"max": 0.4466240108013153,
"mean": 0.2968191936612129,
"median": 0.27796798944473267,
"min": 0.21241599321365356,
"std": 0.07400678240849741
},
{
"max": 0.3830080032348633,
"mean": 0.2946112036705017,
"median": 0.2924960106611252,
"min": 0.21084800362586975,
"std": 0.05323627615746332
},
{
"max": 0.4609600007534027,
"mean": 0.3054272010922432,
"median": 0.289792001247406,
"min": 0.21062399446964264,
"std": 0.07608482904865672
}
],
"selected_backend": "nccl_fallback",
"tensor_parallel_size": 4
}
],
"schema_version": "qwen30_vllm020_allreduce_frozen.v1"
}

103
runs/frontier-qwen30-vllm020-profile-v1/frozen/profile-v1/attention.csv Normal file
View File

@@ -0,0 +1,103 @@
time_stats.attn_input_reshape.min,time_stats.attn_input_reshape.max,time_stats.attn_input_reshape.mean,time_stats.attn_input_reshape.median,time_stats.attn_input_reshape.std,time_stats.attn_kv_cache_save.min,time_stats.attn_kv_cache_save.max,time_stats.attn_kv_cache_save.mean,time_stats.attn_kv_cache_save.median,time_stats.attn_kv_cache_save.std,time_stats.attn_prefill.min,time_stats.attn_prefill.max,time_stats.attn_prefill.mean,time_stats.attn_prefill.median,time_stats.attn_prefill.std,time_stats.attn_decode.min,time_stats.attn_decode.max,time_stats.attn_decode.mean,time_stats.attn_decode.median,time_stats.attn_decode.std,time_stats.attn_output_reshape.min,time_stats.attn_output_reshape.max,time_stats.attn_output_reshape.mean,time_stats.attn_output_reshape.median,time_stats.attn_output_reshape.std,n_embd,n_q_head,n_kv_head,block_size,num_tensor_parallel_workers,max_model_len,batch_size,prefill_chunk_size,kv_cache_size,is_prefill,attention_backend,is_mixed_batch,mode,seq_lens,total_tokens,max_seq_len,min_seq_len,avg_seq_len,equal_seq_len,seq_len_variance,seq_len_std,seq_len_cv,is_chunked_prefill_sample,chunk_start_token,chunk_end_token,total_prefill_tokens,profiling_precision,model_arch,quant_signature,measurement_type,is_true_mixed_batch,prefill_seq_lens,prefill_kv_cache_sizes,decode_kv_cache_sizes,num_prefill_seqs,num_decode_seqs,decode_batch_size,total_batch_size,total_decode_tokens,decode_avg_kv_cache_size,batch_composition_ratio,batch_spec,projection_policy
0.0,0.0,0.0,0.0,0.0,0.01414399966597557,0.028863999992609024,0.019705599918961526,0.01771199982613325,0.005157200849836681,0.047968000173568726,0.07046400010585785,0.05810240097343922,0.05810240097343922,0.007477463486041561,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,64,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[64],64,64,64,64.0,True,0.0,0.0,0.0,False,0.0,64.0,64,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q64,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.014399999752640724,0.04947200044989586,0.020412799902260303,0.01635199971497059,0.010107497379722417,0.046560000628232956,0.08323200047016144,0.05587520003318787,0.05587520003318787,0.011126758739503428,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,128,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[128],128,128,128,128.0,True,0.0,0.0,0.0,False,0.0,128.0,128,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.01484800036996603,0.022207999601960182,0.017033600155264138,0.015312000177800655,0.002819235991970241,0.05104000121355057,0.07692799717187881,0.056396800279617305,0.056396800279617305,0.007481982178637539,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,256,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[256],256,256,256,256.0,True,0.0,0.0,0.0,False,0.0,256.0,256,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q256,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.015072000212967396,0.022272000089287758,0.01706880023702979,0.01616000011563301,0.002460889579319197,0.06931199878454208,0.0838719978928566,0.07432000041007995,0.07432000041007995,0.004777766433175866,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,512,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[512],512,512,512,512.0,True,0.0,0.0,0.0,False,0.0,512.0,512,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q512,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.018592000007629395,0.028543999418616295,0.02095999978482723,0.019183999858796597,0.003198175496053494,0.12179200351238251,0.15408000349998474,0.1307712011039257,0.1307712011039257,0.00858807797538298,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,1024,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[1024],1024,1024,1024,1024.0,True,0.0,0.0,0.0,False,0.0,1024.0,1024,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.027775999158620834,0.03385600075125694,0.030131200328469276,0.029680000618100166,0.0021152558103575215,0.32678401470184326,0.3450239896774292,0.33396480381488797,0.33396480381488797,0.0045872424917606375,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,2048,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[2048],2048,2048,2048,2048.0,True,0.0,0.0,0.0,False,0.0,2048.0,2048,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.04438399896025658,0.05084799975156784,0.046540799736976626,0.04531199857592583,0.002277905811237223,1.0959680080413818,1.1151360273361206,1.0999775886535645,1.0999775886535645,0.005694403246120485,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,4096,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[4096],4096,4096,4096,4096.0,True,0.0,0.0,0.0,False,0.0,4096.0,4096,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.078015998005867,0.08691199868917465,0.08114239946007729,0.08019199967384338,0.00292795706334475,4.070400238037109,4.113152027130127,4.087088012695312,4.087088012695312,0.013660567012509554,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,8192,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[8192],8192,8192,8192,8192.0,True,0.0,0.0,0.0,False,0.0,8192.0,8192,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.016063999384641647,0.05167999863624573,0.022115200012922286,0.017583999782800674,0.010340094822340818,0.05196800082921982,0.09011200070381165,0.06328320093452933,0.06328320093452933,0.012557341255467452,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,64,448.0,True,FLASH_ATTN,False,vllm020_batch_spec,[64],64,64,64,64.0,True,0.0,0.0,0.0,True,448.0,512.0,64,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q64s512,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.01583999954164028,0.026623999699950218,0.018927999772131443,0.017376000061631203,0.003514650316260619,0.06681600213050842,0.07993599772453308,0.0725280001759529,0.0725280001759529,0.004343502558613716,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,128,896.0,True,FLASH_ATTN,False,vllm020_batch_spec,[128],128,128,128,128.0,True,0.0,0.0,0.0,True,896.0,1024.0,128,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q128s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.01648000068962574,0.030880000442266464,0.01945280022919178,0.017967999912798405,0.004096211183007485,0.1311360001564026,0.1546880006790161,0.13908160030841826,0.13908160030841826,0.007511906874366178,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,256,1792.0,True,FLASH_ATTN,False,vllm020_batch_spec,[256],256,256,256,256.0,True,0.0,0.0,0.0,True,1792.0,2048.0,256,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q256s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.017855999991297722,0.03558399900794029,0.020851199887692927,0.018559999763965607,0.005235911594130716,0.32950401306152344,0.350271999835968,0.33912960588932034,0.33912960588932034,0.006027400986663648,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,512,3584.0,True,FLASH_ATTN,False,vllm020_batch_spec,[512],512,512,512,512.0,True,0.0,0.0,0.0,True,3584.0,4096.0,512,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q512s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.019328000023961067,0.040608000010252,0.022790400311350822,0.020704000256955624,0.006113051965778337,1.1415679454803467,1.1518720388412476,1.144483208656311,1.144483208656311,0.0032332311374389127,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,1024,7168.0,True,FLASH_ATTN,False,vllm020_batch_spec,[1024],1024,1024,1024,1024.0,True,0.0,0.0,0.0,True,7168.0,8192.0,1024,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q1ks8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.015807999297976494,0.030688000842928886,0.019596799835562707,0.01774400006979704,0.004343384771033462,0.0,0.0,0.0,0.0,0.0,0.049056001007556915,0.07580800354480743,0.05948160067200661,0.05948160067200661,0.009031541471446955,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,0,127.0,False,FLASH_ATTN,False,vllm020_batch_spec,[1],1,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.01603199914097786,0.02486399933695793,0.01923839971423149,0.018079999834299088,0.0032282528537266424,0.0,0.0,0.0,0.0,0.0,0.05142400041222572,0.07353600114583969,0.059328000620007516,0.059328000620007516,0.0073307807735143084,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,8,0,127.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.016543999314308167,0.03977600112557411,0.021379199624061585,0.018511999398469925,0.006593576176246171,0.0,0.0,0.0,0.0,0.0,0.0488319993019104,0.06435199826955795,0.05479039996862411,0.05479039996862411,0.005672522998491864,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,16,0,127.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.01635199971497059,0.02844800055027008,0.019267200119793416,0.017952000722289085,0.0035068687666949577,0.0,0.0,0.0,0.0,0.0,0.049855999648571014,0.07798399776220322,0.05986879989504815,0.05986879989504815,0.01043914754878828,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,32,0,127.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.016383999958634377,0.026079999282956123,0.01923519968986511,0.017791999503970146,0.0032161974331284568,0.0,0.0,0.0,0.0,0.0,0.058111999183893204,0.1045759990811348,0.06708480007946492,0.06708480007946492,0.013479022462646494,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,64,0,127.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.014720000326633453,0.04057599976658821,0.019100800156593323,0.015455999877303839,0.007512281243011577,0.0,0.0,0.0,0.0,0.0,0.05363199859857559,0.07782399654388428,0.06090559959411622,0.06090559959411622,0.007544620176348091,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,8,0,1023.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.014431999996304512,0.02191999927163124,0.016684799920767546,0.01563199982047081,0.0024293118621811216,0.0,0.0,0.0,0.0,0.0,0.0629120022058487,0.07891199737787247,0.06891520097851753,0.06891520097851753,0.005472695665695425,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,16,0,1023.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.014560000039637089,0.038943998515605927,0.018313600029796363,0.01561600062996149,0.007127270260115769,0.0,0.0,0.0,0.0,0.0,0.08675199747085571,0.10662399977445602,0.09391999915242194,0.09391999915242194,0.006988099589086635,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,32,0,1023.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.014527999795973301,0.054687999188899994,0.021439999900758268,0.01539199985563755,0.012052764849597775,0.0,0.0,0.0,0.0,0.0,0.13488000631332397,0.1528639942407608,0.1431359991431236,0.1431359991431236,0.005436271464033599,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,64,0,1023.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.014399999752640724,0.041919998824596405,0.01899839974939823,0.015343999955803156,0.007989843526623287,0.0,0.0,0.0,0.0,0.0,0.06176000088453293,0.08374399691820145,0.06747519969940186,0.06747519969940186,0.0066067747128778,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,8,0,2047.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.016256000846624374,0.11353600025177002,0.042761600017547606,0.028960000723600388,0.029104301538020762,0.0,0.0,0.0,0.0,0.0,0.09734400361776352,0.14422400295734406,0.11392960175871848,0.11392960175871848,0.013198594600417867,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,16,0,2047.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.014688000082969666,0.034143999218940735,0.018918400071561335,0.01643200032413006,0.005500943993080684,0.0,0.0,0.0,0.0,0.0,0.12918399274349213,0.15087999403476715,0.13807999789714814,0.13807999789714814,0.007658330538677587,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,32,0,2047.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.016063999384641647,0.03641600161790848,0.0198208000510931,0.01780799962580204,0.0057264128169845765,0.0,0.0,0.0,0.0,0.0,0.22099199891090393,0.23904000222682953,0.2293503984808922,0.2293503984808922,0.004861342907006028,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,64,0,2047.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.014751999638974667,0.035840000957250595,0.018908800091594458,0.015792000107467175,0.0064374817924757475,0.0,0.0,0.0,0.0,0.0,0.10134399682283401,0.12201599776744843,0.10896319895982742,0.10896319895982742,0.006336330809165179,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,8,0,4095.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.013856000266969204,0.03417599946260452,0.017846399918198586,0.014800000004470348,0.006495007539635255,0.0,0.0,0.0,0.0,0.0,0.13126400113105774,0.15561600029468536,0.1389280006289482,0.1389280006289482,0.008381472811075022,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,16,0,4095.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.014431999996304512,0.03519999980926514,0.019168000388890504,0.01600000075995922,0.005995522477654695,0.0,0.0,0.0,0.0,0.0,0.21728000044822693,0.2343679964542389,0.2231455981731415,0.2231455981731415,0.004720730646739123,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,32,0,4095.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.014047999866306782,0.03670400008559227,0.018441599886864425,0.015023999847471714,0.006596535162793127,0.0,0.0,0.0,0.0,0.0,0.39321601390838623,0.4524799883365631,0.4058080047369003,0.4058080047369003,0.01578349755088941,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,64,0,4095.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.014336000196635723,0.022143999114632607,0.016912000067532063,0.015168000012636185,0.0028156089295136347,0.0,0.0,0.0,0.0,0.0,0.15587200224399567,0.3079040050506592,0.17838079929351805,0.17838079929351805,0.04355865575265927,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,8,0,8191.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.014271999709308147,0.02112000063061714,0.015516800060868263,0.01488000014796853,0.001940870731593904,0.0,0.0,0.0,0.0,0.0,0.21587200462818146,0.23561599850654602,0.22250880002975468,0.22250880002975468,0.006181951170646666,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,16,0,8191.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.015552000142633915,0.039264000952243805,0.023609600123018028,0.02131200022995472,0.007236979625711548,0.0,0.0,0.0,0.0,0.0,0.408735990524292,0.470335990190506,0.4336863994598388,0.4336863994598388,0.01844662383160074,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,32,0,8191.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.014399999752640724,0.025407999753952026,0.016227199975401164,0.014960000291466713,0.0031617375441736185,0.0,0.0,0.0,0.0,0.0,0.7412800192832947,0.7627840042114258,0.7464000046253203,0.7464000046253203,0.006112167448837547,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,64,0,8191.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.014720000326633453,0.02364799939095974,0.01809599995613098,0.016352000646293163,0.0035481127058959038,0.04822399839758873,0.08566399663686752,0.05961279980838299,0.05961279980838299,0.011445665413968877,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,64,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[64],64,64,64,64.0,True,0.0,0.0,0.0,False,0.0,64.0,64,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q64,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.014175999909639359,0.020864000543951988,0.01668160008266568,0.015664000064134598,0.0025769063833097584,0.049695998430252075,0.08057600259780884,0.05882879942655563,0.05882879942655563,0.009515126108519331,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,128,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[128],128,128,128,128.0,True,0.0,0.0,0.0,False,0.0,128.0,128,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.014399999752640724,0.028704000636935234,0.017430400010198355,0.015056000091135502,0.004349294613335555,0.049855999648571014,0.07366400212049484,0.05459520071744919,0.05459520071744919,0.0069610920757925574,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,256,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[256],256,256,256,256.0,True,0.0,0.0,0.0,False,0.0,256.0,256,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q256,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.014336000196635723,0.03161599859595299,0.017788799852132796,0.015711999963968992,0.005005385723318659,0.06102399900555611,0.08179199695587158,0.06650560013949873,0.06650560013949873,0.006947995595801105,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,512,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[512],512,512,512,512.0,True,0.0,0.0,0.0,False,0.0,512.0,512,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q512,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.014655999839305878,0.04416000097990036,0.019200000166893005,0.015488000120967627,0.008561241323364038,0.08054400235414505,0.09196799993515015,0.08607039973139763,0.08607039973139763,0.004145329035483754,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,1024,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[1024],1024,1024,1024,1024.0,True,0.0,0.0,0.0,False,0.0,1024.0,1024,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.017855999991297722,0.023391999304294586,0.019667199812829494,0.018463999964296818,0.0022577669687832585,0.18729600310325623,0.20585599541664124,0.19546559900045393,0.19546559900045393,0.006339068824303663,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,2048,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[2048],2048,2048,2048,2048.0,True,0.0,0.0,0.0,False,0.0,2048.0,2048,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.026240000501275063,0.03446400165557861,0.028297600522637367,0.027088000439107418,0.0025148441522922374,0.5754240155220032,0.5889919996261597,0.5800191938877105,0.5800191938877105,0.003858869273829596,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,4096,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[4096],4096,4096,4096,4096.0,True,0.0,0.0,0.0,False,0.0,4096.0,4096,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.04182400181889534,0.047200001776218414,0.043036799877882004,0.0423360001295805,0.0017073405772076728,2.063199996948242,2.0787200927734375,2.067151999473572,2.067151999473572,0.004959651271127835,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,8192,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[8192],8192,8192,8192,8192.0,True,0.0,0.0,0.0,False,0.0,8192.0,8192,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.014399999752640724,0.05648000165820122,0.02270399993285537,0.017935999669134617,0.012377915150727689,0.049536000937223434,0.07196799665689468,0.056015999615192415,0.056015999615192415,0.0070476637552742884,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,64,448.0,True,FLASH_ATTN,False,vllm020_batch_spec,[64],64,64,64,64.0,True,0.0,0.0,0.0,True,448.0,512.0,64,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q64s512,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.01408000010997057,0.021344000473618507,0.01611520005390048,0.014928000047802925,0.0025499884993961702,0.07072000205516815,0.2642880082130432,0.1588256008923054,0.1588256008923054,0.053220347086102376,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,128,896.0,True,FLASH_ATTN,False,vllm020_batch_spec,[128],128,128,128,128.0,True,0.0,0.0,0.0,True,896.0,1024.0,128,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q128s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.01360000018030405,0.03014400042593479,0.016975999902933837,0.015056000091135502,0.004715159697364111,0.08393599838018417,0.11036799848079681,0.09160000011324881,0.09160000011324881,0.007911669434472792,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,256,1792.0,True,FLASH_ATTN,False,vllm020_batch_spec,[256],256,256,256,256.0,True,0.0,0.0,0.0,True,1792.0,2048.0,256,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q256s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.014495999552309513,0.020767999812960625,0.016233599931001663,0.015392000321298838,0.002202615907995427,0.1998399943113327,0.22070400416851044,0.20855360180139543,0.20855360180139543,0.00689307200230667,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,512,3584.0,True,FLASH_ATTN,False,vllm020_batch_spec,[512],512,512,512,512.0,True,0.0,0.0,0.0,True,3584.0,4096.0,512,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q512s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.01484800036996603,0.033440001308918,0.018684800155460833,0.016048000194132328,0.00553991005639174,0.6043199896812439,0.635807991027832,0.6126143991947173,0.6126143991947173,0.008745933953408096,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,1024,7168.0,True,FLASH_ATTN,False,vllm020_batch_spec,[1024],1024,1024,1024,1024.0,True,0.0,0.0,0.0,True,7168.0,8192.0,1024,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q1ks8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.013824000023305416,0.03359999880194664,0.017811199743300678,0.014944000169634819,0.005926140483565472,0.0,0.0,0.0,0.0,0.0,0.045471999794244766,0.07539200037717819,0.054758400097489356,0.054758400097489356,0.010253548506101549,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,0,127.0,False,FLASH_ATTN,False,vllm020_batch_spec,[1],1,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.014047999866306782,0.02409599907696247,0.01641279999166727,0.01473599998280406,0.003347158638713987,0.0,0.0,0.0,0.0,0.0,0.0461760014295578,0.07529599964618683,0.05460800044238568,0.05460800044238568,0.009748937798340135,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,8,0,127.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.014271999709308147,0.028416000306606293,0.018611199874430894,0.01598400017246604,0.005209875093863647,0.0,0.0,0.0,0.0,0.0,0.048128001391887665,0.07897599786520004,0.061353600397706036,0.061353600397706036,0.010488153655157845,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,16,0,127.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.014112000353634357,0.025728000327944756,0.016092800162732603,0.01512000011280179,0.003300888123052605,0.0,0.0,0.0,0.0,0.0,0.04864000156521797,0.07648000121116638,0.05810560062527656,0.05810560062527656,0.009473544218404408,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,32,0,127.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.014655999839305878,0.03551999852061272,0.018588799890130757,0.016064000315964222,0.006071057437992447,0.0,0.0,0.0,0.0,0.0,0.04822399839758873,0.07862400263547897,0.05660480037331582,0.05660480037331582,0.009565394213730401,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,64,0,127.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.014303999952971935,0.028831999748945236,0.01699519995599985,0.015024000313133001,0.004285000744868188,0.0,0.0,0.0,0.0,0.0,0.04854400083422661,0.06719999760389328,0.05778240002691746,0.05778240002691746,0.006852125554679805,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,8,0,1023.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.0144640002399683,0.024927999824285507,0.0161183999851346,0.01521599991247058,0.0029774808524673907,0.0,0.0,0.0,0.0,0.0,0.05379199981689453,0.08966399729251862,0.06270079985260964,0.06270079985260964,0.009983591277092696,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,16,0,1023.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.014431999996304512,0.03001599945127964,0.017648000083863736,0.01547200046479702,0.004585126309484848,0.0,0.0,0.0,0.0,0.0,0.061664000153541565,0.07692799717187881,0.06704320013523103,0.06704320013523103,0.005500191798490629,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,32,0,1023.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.014175999909639359,0.026367999613285065,0.016947199776768684,0.015039999969303608,0.0037951566103550205,0.0,0.0,0.0,0.0,0.0,0.08799999952316284,0.111455999314785,0.0964031994342804,0.0964031994342804,0.007397541615558088,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,64,0,1023.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.01369599997997284,0.021247999742627144,0.015359999984502793,0.014640000183135271,0.0020942770950814317,0.0,0.0,0.0,0.0,0.0,0.051711998879909515,0.07065600156784058,0.058054400235414506,0.058054400235414506,0.006633034815910025,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,8,0,2047.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.014208000153303146,0.0307839997112751,0.017148799914866685,0.015039999969303608,0.004882374686312284,0.0,0.0,0.0,0.0,0.0,0.061919998377561576,0.07843200117349625,0.06628479920327664,0.06628479920327664,0.004962852689801192,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,16,0,2047.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.013887999579310417,0.02969600073993206,0.017500799987465142,0.015232000034302473,0.00467273319705314,0.0,0.0,0.0,0.0,0.0,0.08819200098514557,0.11097600311040878,0.09493440166115762,0.09493440166115762,0.007509235042985577,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,32,0,2047.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.014399999752640724,0.022112000733613968,0.01751680001616478,0.016047999262809753,0.0030306621792915785,0.0,0.0,0.0,0.0,0.0,0.13065600395202637,0.15110400319099426,0.13857279866933822,0.13857279866933822,0.00750137841249771,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,64,0,2047.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.013919999822974205,0.04012800008058548,0.01892479993402958,0.01550400024279952,0.007459545267816961,0.0,0.0,0.0,0.0,0.0,0.06278400123119354,0.08259200304746628,0.0704512007534504,0.0704512007534504,0.005979055984382744,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,8,0,4095.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.014336000196635723,0.02236800082027912,0.016332800220698118,0.014928000047802925,0.002784044363186731,0.0,0.0,0.0,0.0,0.0,0.1003199964761734,0.1279360055923462,0.10921279862523078,0.10921279862523078,0.00862769617046716,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,16,0,4095.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.013919999822974205,0.0225600004196167,0.016128000058233737,0.014800000004470348,0.002958953332547708,0.0,0.0,0.0,0.0,0.0,0.13116799294948578,0.14812800288200378,0.13783999979496003,0.13783999979496003,0.005696384361148053,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,32,0,4095.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.014208000153303146,0.029983999207615852,0.017468800116330386,0.01508800033479929,0.0047379550962483065,0.0,0.0,0.0,0.0,0.0,0.217631995677948,0.2447360008955002,0.22715839892625808,0.22715839892625808,0.008831828221847138,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,64,0,4095.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.014047999866306782,0.02304000034928322,0.016512000095099212,0.014512000139802694,0.0035026774828624254,0.0,0.0,0.0,0.0,0.0,0.11020799726247787,0.12307199835777283,0.11600959971547126,0.11600959971547126,0.004667637266950902,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,8,0,8191.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.014112000353634357,0.042080000042915344,0.017510399967432023,0.014607999939471483,0.00823117883530167,0.0,0.0,0.0,0.0,0.0,0.15702399611473083,0.17587199807167053,0.16399359852075576,0.16399359852075576,0.006588299074676393,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,16,0,8191.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.013919999822974205,0.0208320003002882,0.015299199987202883,0.01462399959564209,0.001916768086505386,0.0,0.0,0.0,0.0,0.0,0.21779200434684753,0.2415360063314438,0.2260768011212349,0.2260768011212349,0.007251352080685236,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,32,0,8191.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.014015999622642994,0.021088000386953354,0.015619200188666582,0.01473599998280406,0.002013227452302141,0.0,0.0,0.0,0.0,0.0,0.3959999978542328,0.4152640104293823,0.40332479774951924,0.40332479774951924,0.006942401431914052,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,64,0,8191.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.014560000039637089,0.022784000262618065,0.016435200069099664,0.015519999898970127,0.0023688301421469523,0.04879999905824661,0.09139200299978256,0.06054079942405224,0.06054079942405224,0.012152608702448775,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,64,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[64],64,64,64,64.0,True,0.0,0.0,0.0,False,0.0,64.0,64,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q64,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.014816000126302242,0.02844800055027008,0.017008000146597625,0.015696000307798386,0.003941466294662679,0.047807998955249786,0.07356800138950348,0.05626560002565384,0.05626560002565384,0.00842179125412236,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,128,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[128],128,128,128,128.0,True,0.0,0.0,0.0,False,0.0,128.0,128,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.014688000082969666,0.053408000618219376,0.019353600218892097,0.01532800029963255,0.01138302400841626,0.048448000103235245,0.0785600021481514,0.0556256003677845,0.0556256003677845,0.009348274502616908,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,256,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[256],256,256,256,256.0,True,0.0,0.0,0.0,False,0.0,256.0,256,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q256,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.015039999969303608,0.03139200061559677,0.01823679991066456,0.016159999649971724,0.004860071238302512,0.055424001067876816,0.08505599945783615,0.0640383992344141,0.0640383992344141,0.00921448636178623,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,512,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[512],512,512,512,512.0,True,0.0,0.0,0.0,False,0.0,512.0,512,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q512,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.014751999638974667,0.026528000831604004,0.017324799951165915,0.01536000007763505,0.0036004822686428305,0.07660800218582153,0.0942080020904541,0.08209280073642732,0.08209280073642732,0.00515895587669752,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,1024,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[1024],1024,1024,1024,1024.0,True,0.0,0.0,0.0,False,0.0,1024.0,1024,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.014879999682307243,0.021247999742627144,0.016403199825435876,0.01532800029963255,0.001995897022647934,0.11395200341939926,0.15113599598407745,0.12431039959192276,0.12431039959192276,0.011164431123683732,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,2048,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[2048],2048,2048,2048,2048.0,True,0.0,0.0,0.0,False,0.0,2048.0,2048,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.017184000462293625,0.028672000393271446,0.019865600019693376,0.017823999747633934,0.0035798491315929977,0.3171840012073517,0.3341119885444641,0.3261695951223373,0.3261695951223373,0.005111046452878918,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,4096,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[4096],4096,4096,4096,4096.0,True,0.0,0.0,0.0,False,0.0,4096.0,4096,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.024639999493956566,0.030400000512599945,0.02656640000641346,0.02556800004094839,0.0020189846603237303,1.0648640394210815,1.0828479528427124,1.0715327858924866,1.0715327858924866,0.005558639263049851,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,8192,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[8192],8192,8192,8192,8192.0,True,0.0,0.0,0.0,False,0.0,8192.0,8192,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.014976000413298607,0.021695999428629875,0.017305599898099898,0.01593599934130907,0.0025718046517268054,0.04956800118088722,0.07932800054550171,0.06228480041027069,0.06228480041027069,0.01027160349757827,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,64,448.0,True,FLASH_ATTN,False,vllm020_batch_spec,[64],64,64,64,64.0,True,0.0,0.0,0.0,True,448.0,512.0,64,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q64s512,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.01539199985563755,0.03580800071358681,0.019686400331556796,0.017136000096797943,0.005918387811304003,0.05158400163054466,0.10713600367307663,0.06364160068333148,0.06364160068333148,0.015832957809696766,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,128,896.0,True,FLASH_ATTN,False,vllm020_batch_spec,[128],128,128,128,128.0,True,0.0,0.0,0.0,True,896.0,1024.0,128,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q128s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.015168000012636185,0.02223999984562397,0.016672000009566545,0.015887999907135963,0.0020934945946034563,0.06521599739789963,0.08902399986982346,0.07520959973335266,0.07520959973335266,0.007913840684102929,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,256,1792.0,True,FLASH_ATTN,False,vllm020_batch_spec,[256],256,256,256,256.0,True,0.0,0.0,0.0,True,1792.0,2048.0,256,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q256s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.015231999568641186,0.04028800129890442,0.019971200078725816,0.01646399963647127,0.007351305886577286,0.14467200636863708,0.16844800114631653,0.15363519936800005,0.15363519936800005,0.008174435899956223,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,512,3584.0,True,FLASH_ATTN,False,vllm020_batch_spec,[512],512,512,512,512.0,True,0.0,0.0,0.0,True,3584.0,4096.0,512,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q512s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.015519999898970127,0.02304000034928322,0.01775679988786578,0.016784000210464,0.0025077243712082584,0.33740800619125366,0.35343998670578003,0.3445120006799698,0.3445120006799698,0.00445648463590358,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,1024,7168.0,True,FLASH_ATTN,False,vllm020_batch_spec,[1024],1024,1024,1024,1024.0,True,0.0,0.0,0.0,True,7168.0,8192.0,1024,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q1ks8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.015647999942302704,0.02393599972128868,0.01809599995613098,0.01654400024563074,0.002998393102466254,0.0,0.0,0.0,0.0,0.0,0.0504320003092289,0.0843840017914772,0.060083200410008426,0.060083200410008426,0.00986959318572296,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,0,127.0,False,FLASH_ATTN,False,vllm020_batch_spec,[1],1,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.01603199914097786,0.030047999694943428,0.019510399922728537,0.017680000513792038,0.004065185265963332,0.0,0.0,0.0,0.0,0.0,0.05004800111055374,0.07036799937486649,0.059315200522542,0.059315200522542,0.006537768821329647,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,8,0,127.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.01484800036996603,0.02393599972128868,0.018035199772566558,0.016512000001966953,0.0030667689116777724,0.0,0.0,0.0,0.0,0.0,0.05100800096988678,0.06735999882221222,0.058387200161814694,0.058387200161814694,0.005832787739241381,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,16,0,127.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.01500799972563982,0.026208000257611275,0.017500799987465142,0.01646399963647127,0.0030666103306165714,0.0,0.0,0.0,0.0,0.0,0.05023999884724617,0.07254400104284286,0.057254400476813315,0.057254400476813315,0.006890853653068151,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,32,0,127.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.014688000082969666,0.027327999472618103,0.0179776000790298,0.01648000068962574,0.003783417447531392,0.0,0.0,0.0,0.0,0.0,0.04819199815392494,0.07100799679756165,0.05621119923889638,0.05621119923889638,0.007824391484124725,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,64,0,127.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.015647999942302704,0.036448001861572266,0.019136000238358975,0.016704000532627106,0.006076530095624803,0.0,0.0,0.0,0.0,0.0,0.047488000243902206,0.08441600203514099,0.0588383998721838,0.0588383998721838,0.011275940726332522,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,8,0,1023.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.015135999768972397,0.033952001482248306,0.02119360016658902,0.018000000156462193,0.007136646614305304,0.0,0.0,0.0,0.0,0.0,0.04931199923157692,0.06992000341415405,0.05755840018391609,0.05755840018391609,0.007297587694315226,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,16,0,1023.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.014879999682307243,0.034272000193595886,0.0204415999352932,0.017311999574303627,0.00695404701803013,0.0,0.0,0.0,0.0,0.0,0.05215999856591225,0.0735040009021759,0.06117440015077591,0.06117440015077591,0.007381118436894922,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,32,0,1023.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.015359999611973763,0.03743999823927879,0.02012479966506362,0.01775999926030636,0.006181045509079057,0.0,0.0,0.0,0.0,0.0,0.06355199962854385,0.08508799970149994,0.07019200026988984,0.07019200026988984,0.0062246580442117845,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,64,0,1023.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.015552000142633915,0.032607998698949814,0.01959999995306134,0.017487999983131886,0.004882475068460749,0.0,0.0,0.0,0.0,0.0,0.04918399825692177,0.0865280032157898,0.05973760038614274,0.05973760038614274,0.011546847942113974,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,8,0,2047.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.015168000012636185,0.02675200067460537,0.017430400010198355,0.016560000367462635,0.003243135094239819,0.0,0.0,0.0,0.0,0.0,0.052671998739242554,0.08367999643087387,0.06238719932734965,0.06238719932734965,0.011207824780630104,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,16,0,2047.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.01500799972563982,0.03612799942493439,0.019327999837696553,0.01688000001013279,0.006058251425153085,0.0,0.0,0.0,0.0,0.0,0.06364800035953522,0.07878399640321732,0.06935679838061332,0.06935679838061332,0.004515588328677643,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,32,0,2047.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.015104000456631184,0.03711999952793121,0.019670399930328132,0.017152000218629837,0.006165994646522264,0.0,0.0,0.0,0.0,0.0,0.09071999788284302,0.11507199704647064,0.10252480059862136,0.10252480059862136,0.008782544051535657,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,64,0,2047.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.015039999969303608,0.026528000831604004,0.018396800104528665,0.01601599995046854,0.004279788048986283,0.0,0.0,0.0,0.0,0.0,0.05331199988722801,0.07977599650621414,0.06076480001211167,0.06076480001211167,0.008761579122069606,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,8,0,4095.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.015359999611973763,0.02643200010061264,0.01726400004699826,0.015855999663472176,0.003210008417242029,0.0,0.0,0.0,0.0,0.0,0.062431998550891876,0.08246400207281113,0.06970879957079888,0.06970879957079888,0.007016341676068233,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,16,0,4095.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.014911999925971031,0.02223999984562397,0.0166015999391675,0.015887999907135963,0.00204913969129354,0.0,0.0,0.0,0.0,0.0,0.10127999633550644,0.1141119971871376,0.10621120035648347,0.10621120035648347,0.004029318311754605,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,32,0,4095.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.015359999611973763,0.03363199904561043,0.019305599946528675,0.016671999357640743,0.0055445582307981234,0.0,0.0,0.0,0.0,0.0,0.1319040060043335,0.15839999914169312,0.14040640145540234,0.14040640145540234,0.007998041073596942,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,64,0,4095.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.014368000440299511,0.04447999969124794,0.018908800091594458,0.0157279996201396,0.0086814937461721,0.0,0.0,0.0,0.0,0.0,0.06428799778223038,0.08982399851083755,0.07349760085344315,0.07349760085344315,0.008605284512197258,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,8,0,8191.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.015104000456631184,0.028672000393271446,0.01890560006722808,0.016080000437796116,0.004797584627815021,0.0,0.0,0.0,0.0,0.0,0.11123199760913849,0.14115199446678162,0.11942399889230729,0.11942399889230729,0.008513394706791027,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,16,0,8191.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.014271999709308147,0.024927999824285507,0.016915200091898442,0.015216000378131866,0.003374147499442635,0.0,0.0,0.0,0.0,0.0,0.15887999534606934,0.18111999332904816,0.16934399753808976,0.16934399753808976,0.007415181260761123,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,32,0,8191.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.014336000196635723,0.026944000273942947,0.017737600207328796,0.015199999790638685,0.00415598714375255,0.0,0.0,0.0,0.0,0.0,0.2192319929599762,0.23472000658512115,0.22809920012950893,0.22809920012950893,0.004730841376327335,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,64,0,8191.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
1 time_stats.attn_input_reshape.min time_stats.attn_input_reshape.max time_stats.attn_input_reshape.mean time_stats.attn_input_reshape.median time_stats.attn_input_reshape.std time_stats.attn_kv_cache_save.min time_stats.attn_kv_cache_save.max time_stats.attn_kv_cache_save.mean time_stats.attn_kv_cache_save.median time_stats.attn_kv_cache_save.std time_stats.attn_prefill.min time_stats.attn_prefill.max time_stats.attn_prefill.mean time_stats.attn_prefill.median time_stats.attn_prefill.std time_stats.attn_decode.min time_stats.attn_decode.max time_stats.attn_decode.mean time_stats.attn_decode.median time_stats.attn_decode.std time_stats.attn_output_reshape.min time_stats.attn_output_reshape.max time_stats.attn_output_reshape.mean time_stats.attn_output_reshape.median time_stats.attn_output_reshape.std n_embd n_q_head n_kv_head block_size num_tensor_parallel_workers max_model_len batch_size prefill_chunk_size kv_cache_size is_prefill attention_backend is_mixed_batch mode seq_lens total_tokens max_seq_len min_seq_len avg_seq_len equal_seq_len seq_len_variance seq_len_std seq_len_cv is_chunked_prefill_sample chunk_start_token chunk_end_token total_prefill_tokens profiling_precision model_arch quant_signature measurement_type is_true_mixed_batch prefill_seq_lens prefill_kv_cache_sizes decode_kv_cache_sizes num_prefill_seqs num_decode_seqs decode_batch_size total_batch_size total_decode_tokens decode_avg_kv_cache_size batch_composition_ratio batch_spec projection_policy
2 0.0 0.0 0.0 0.0 0.0 0.01414399966597557 0.028863999992609024 0.019705599918961526 0.01771199982613325 0.005157200849836681 0.047968000173568726 0.07046400010585785 0.05810240097343922 0.05810240097343922 0.007477463486041561 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 1 40960 1 64 0.0 True FLASH_ATTN False vllm020_batch_spec [64] 64 64 64 64.0 True 0.0 0.0 0.0 False 0.0 64.0 64 BF16 generic none CUDA_EVENT False q64 measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
3 0.0 0.0 0.0 0.0 0.0 0.014399999752640724 0.04947200044989586 0.020412799902260303 0.01635199971497059 0.010107497379722417 0.046560000628232956 0.08323200047016144 0.05587520003318787 0.05587520003318787 0.011126758739503428 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 1 40960 1 128 0.0 True FLASH_ATTN False vllm020_batch_spec [128] 128 128 128 128.0 True 0.0 0.0 0.0 False 0.0 128.0 128 BF16 generic none CUDA_EVENT False q128 measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
4 0.0 0.0 0.0 0.0 0.0 0.01484800036996603 0.022207999601960182 0.017033600155264138 0.015312000177800655 0.002819235991970241 0.05104000121355057 0.07692799717187881 0.056396800279617305 0.056396800279617305 0.007481982178637539 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 1 40960 1 256 0.0 True FLASH_ATTN False vllm020_batch_spec [256] 256 256 256 256.0 True 0.0 0.0 0.0 False 0.0 256.0 256 BF16 generic none CUDA_EVENT False q256 measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
5 0.0 0.0 0.0 0.0 0.0 0.015072000212967396 0.022272000089287758 0.01706880023702979 0.01616000011563301 0.002460889579319197 0.06931199878454208 0.0838719978928566 0.07432000041007995 0.07432000041007995 0.004777766433175866 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 1 40960 1 512 0.0 True FLASH_ATTN False vllm020_batch_spec [512] 512 512 512 512.0 True 0.0 0.0 0.0 False 0.0 512.0 512 BF16 generic none CUDA_EVENT False q512 measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
6 0.0 0.0 0.0 0.0 0.0 0.018592000007629395 0.028543999418616295 0.02095999978482723 0.019183999858796597 0.003198175496053494 0.12179200351238251 0.15408000349998474 0.1307712011039257 0.1307712011039257 0.00858807797538298 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 1 40960 1 1024 0.0 True FLASH_ATTN False vllm020_batch_spec [1024] 1024 1024 1024 1024.0 True 0.0 0.0 0.0 False 0.0 1024.0 1024 BF16 generic none CUDA_EVENT False q1k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
7 0.0 0.0 0.0 0.0 0.0 0.027775999158620834 0.03385600075125694 0.030131200328469276 0.029680000618100166 0.0021152558103575215 0.32678401470184326 0.3450239896774292 0.33396480381488797 0.33396480381488797 0.0045872424917606375 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 1 40960 1 2048 0.0 True FLASH_ATTN False vllm020_batch_spec [2048] 2048 2048 2048 2048.0 True 0.0 0.0 0.0 False 0.0 2048.0 2048 BF16 generic none CUDA_EVENT False q2k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
8 0.0 0.0 0.0 0.0 0.0 0.04438399896025658 0.05084799975156784 0.046540799736976626 0.04531199857592583 0.002277905811237223 1.0959680080413818 1.1151360273361206 1.0999775886535645 1.0999775886535645 0.005694403246120485 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 1 40960 1 4096 0.0 True FLASH_ATTN False vllm020_batch_spec [4096] 4096 4096 4096 4096.0 True 0.0 0.0 0.0 False 0.0 4096.0 4096 BF16 generic none CUDA_EVENT False q4k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
9 0.0 0.0 0.0 0.0 0.0 0.078015998005867 0.08691199868917465 0.08114239946007729 0.08019199967384338 0.00292795706334475 4.070400238037109 4.113152027130127 4.087088012695312 4.087088012695312 0.013660567012509554 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 1 40960 1 8192 0.0 True FLASH_ATTN False vllm020_batch_spec [8192] 8192 8192 8192 8192.0 True 0.0 0.0 0.0 False 0.0 8192.0 8192 BF16 generic none CUDA_EVENT False q8k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
10 0.0 0.0 0.0 0.0 0.0 0.016063999384641647 0.05167999863624573 0.022115200012922286 0.017583999782800674 0.010340094822340818 0.05196800082921982 0.09011200070381165 0.06328320093452933 0.06328320093452933 0.012557341255467452 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 1 40960 1 64 448.0 True FLASH_ATTN False vllm020_batch_spec [64] 64 64 64 64.0 True 0.0 0.0 0.0 True 448.0 512.0 64 BF16 generic none CUDA_EVENT False q64s512 measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
11 0.0 0.0 0.0 0.0 0.0 0.01583999954164028 0.026623999699950218 0.018927999772131443 0.017376000061631203 0.003514650316260619 0.06681600213050842 0.07993599772453308 0.0725280001759529 0.0725280001759529 0.004343502558613716 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 1 40960 1 128 896.0 True FLASH_ATTN False vllm020_batch_spec [128] 128 128 128 128.0 True 0.0 0.0 0.0 True 896.0 1024.0 128 BF16 generic none CUDA_EVENT False q128s1k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
12 0.0 0.0 0.0 0.0 0.0 0.01648000068962574 0.030880000442266464 0.01945280022919178 0.017967999912798405 0.004096211183007485 0.1311360001564026 0.1546880006790161 0.13908160030841826 0.13908160030841826 0.007511906874366178 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 1 40960 1 256 1792.0 True FLASH_ATTN False vllm020_batch_spec [256] 256 256 256 256.0 True 0.0 0.0 0.0 True 1792.0 2048.0 256 BF16 generic none CUDA_EVENT False q256s2k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
13 0.0 0.0 0.0 0.0 0.0 0.017855999991297722 0.03558399900794029 0.020851199887692927 0.018559999763965607 0.005235911594130716 0.32950401306152344 0.350271999835968 0.33912960588932034 0.33912960588932034 0.006027400986663648 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 1 40960 1 512 3584.0 True FLASH_ATTN False vllm020_batch_spec [512] 512 512 512 512.0 True 0.0 0.0 0.0 True 3584.0 4096.0 512 BF16 generic none CUDA_EVENT False q512s4k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
14 0.0 0.0 0.0 0.0 0.0 0.019328000023961067 0.040608000010252 0.022790400311350822 0.020704000256955624 0.006113051965778337 1.1415679454803467 1.1518720388412476 1.144483208656311 1.144483208656311 0.0032332311374389127 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 1 40960 1 1024 7168.0 True FLASH_ATTN False vllm020_batch_spec [1024] 1024 1024 1024 1024.0 True 0.0 0.0 0.0 True 7168.0 8192.0 1024 BF16 generic none CUDA_EVENT False q1ks8k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
15 0.0 0.0 0.0 0.0 0.0 0.015807999297976494 0.030688000842928886 0.019596799835562707 0.01774400006979704 0.004343384771033462 0.0 0.0 0.0 0.0 0.0 0.049056001007556915 0.07580800354480743 0.05948160067200661 0.05948160067200661 0.009031541471446955 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 1 40960 1 0 127.0 False FLASH_ATTN False vllm020_batch_spec [1] 1 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False q1s128 measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
16 0.0 0.0 0.0 0.0 0.0 0.01603199914097786 0.02486399933695793 0.01923839971423149 0.018079999834299088 0.0032282528537266424 0.0 0.0 0.0 0.0 0.0 0.05142400041222572 0.07353600114583969 0.059328000620007516 0.059328000620007516 0.0073307807735143084 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 1 40960 8 0 127.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1] 8 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 8q1s128 measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
17 0.0 0.0 0.0 0.0 0.0 0.016543999314308167 0.03977600112557411 0.021379199624061585 0.018511999398469925 0.006593576176246171 0.0 0.0 0.0 0.0 0.0 0.0488319993019104 0.06435199826955795 0.05479039996862411 0.05479039996862411 0.005672522998491864 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 1 40960 16 0 127.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] 16 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 16q1s128 measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
18 0.0 0.0 0.0 0.0 0.0 0.01635199971497059 0.02844800055027008 0.019267200119793416 0.017952000722289085 0.0035068687666949577 0.0 0.0 0.0 0.0 0.0 0.049855999648571014 0.07798399776220322 0.05986879989504815 0.05986879989504815 0.01043914754878828 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 1 40960 32 0 127.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] 32 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 32q1s128 measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
19 0.0 0.0 0.0 0.0 0.0 0.016383999958634377 0.026079999282956123 0.01923519968986511 0.017791999503970146 0.0032161974331284568 0.0 0.0 0.0 0.0 0.0 0.058111999183893204 0.1045759990811348 0.06708480007946492 0.06708480007946492 0.013479022462646494 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 1 40960 64 0 127.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] 64 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 64q1s128 measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
20 0.0 0.0 0.0 0.0 0.0 0.014720000326633453 0.04057599976658821 0.019100800156593323 0.015455999877303839 0.007512281243011577 0.0 0.0 0.0 0.0 0.0 0.05363199859857559 0.07782399654388428 0.06090559959411622 0.06090559959411622 0.007544620176348091 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 1 40960 8 0 1023.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1] 8 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 8q1s1k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
21 0.0 0.0 0.0 0.0 0.0 0.014431999996304512 0.02191999927163124 0.016684799920767546 0.01563199982047081 0.0024293118621811216 0.0 0.0 0.0 0.0 0.0 0.0629120022058487 0.07891199737787247 0.06891520097851753 0.06891520097851753 0.005472695665695425 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 1 40960 16 0 1023.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] 16 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 16q1s1k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
22 0.0 0.0 0.0 0.0 0.0 0.014560000039637089 0.038943998515605927 0.018313600029796363 0.01561600062996149 0.007127270260115769 0.0 0.0 0.0 0.0 0.0 0.08675199747085571 0.10662399977445602 0.09391999915242194 0.09391999915242194 0.006988099589086635 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 1 40960 32 0 1023.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] 32 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 32q1s1k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
23 0.0 0.0 0.0 0.0 0.0 0.014527999795973301 0.054687999188899994 0.021439999900758268 0.01539199985563755 0.012052764849597775 0.0 0.0 0.0 0.0 0.0 0.13488000631332397 0.1528639942407608 0.1431359991431236 0.1431359991431236 0.005436271464033599 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 1 40960 64 0 1023.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] 64 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 64q1s1k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
24 0.0 0.0 0.0 0.0 0.0 0.014399999752640724 0.041919998824596405 0.01899839974939823 0.015343999955803156 0.007989843526623287 0.0 0.0 0.0 0.0 0.0 0.06176000088453293 0.08374399691820145 0.06747519969940186 0.06747519969940186 0.0066067747128778 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 1 40960 8 0 2047.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1] 8 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 8q1s2k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
25 0.0 0.0 0.0 0.0 0.0 0.016256000846624374 0.11353600025177002 0.042761600017547606 0.028960000723600388 0.029104301538020762 0.0 0.0 0.0 0.0 0.0 0.09734400361776352 0.14422400295734406 0.11392960175871848 0.11392960175871848 0.013198594600417867 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 1 40960 16 0 2047.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] 16 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 16q1s2k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
26 0.0 0.0 0.0 0.0 0.0 0.014688000082969666 0.034143999218940735 0.018918400071561335 0.01643200032413006 0.005500943993080684 0.0 0.0 0.0 0.0 0.0 0.12918399274349213 0.15087999403476715 0.13807999789714814 0.13807999789714814 0.007658330538677587 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 1 40960 32 0 2047.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] 32 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 32q1s2k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
27 0.0 0.0 0.0 0.0 0.0 0.016063999384641647 0.03641600161790848 0.0198208000510931 0.01780799962580204 0.0057264128169845765 0.0 0.0 0.0 0.0 0.0 0.22099199891090393 0.23904000222682953 0.2293503984808922 0.2293503984808922 0.004861342907006028 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 1 40960 64 0 2047.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] 64 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 64q1s2k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
28 0.0 0.0 0.0 0.0 0.0 0.014751999638974667 0.035840000957250595 0.018908800091594458 0.015792000107467175 0.0064374817924757475 0.0 0.0 0.0 0.0 0.0 0.10134399682283401 0.12201599776744843 0.10896319895982742 0.10896319895982742 0.006336330809165179 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 1 40960 8 0 4095.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1] 8 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 8q1s4k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
29 0.0 0.0 0.0 0.0 0.0 0.013856000266969204 0.03417599946260452 0.017846399918198586 0.014800000004470348 0.006495007539635255 0.0 0.0 0.0 0.0 0.0 0.13126400113105774 0.15561600029468536 0.1389280006289482 0.1389280006289482 0.008381472811075022 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 1 40960 16 0 4095.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] 16 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 16q1s4k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
30 0.0 0.0 0.0 0.0 0.0 0.014431999996304512 0.03519999980926514 0.019168000388890504 0.01600000075995922 0.005995522477654695 0.0 0.0 0.0 0.0 0.0 0.21728000044822693 0.2343679964542389 0.2231455981731415 0.2231455981731415 0.004720730646739123 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 1 40960 32 0 4095.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] 32 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 32q1s4k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
31 0.0 0.0 0.0 0.0 0.0 0.014047999866306782 0.03670400008559227 0.018441599886864425 0.015023999847471714 0.006596535162793127 0.0 0.0 0.0 0.0 0.0 0.39321601390838623 0.4524799883365631 0.4058080047369003 0.4058080047369003 0.01578349755088941 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 1 40960 64 0 4095.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] 64 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 64q1s4k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
32 0.0 0.0 0.0 0.0 0.0 0.014336000196635723 0.022143999114632607 0.016912000067532063 0.015168000012636185 0.0028156089295136347 0.0 0.0 0.0 0.0 0.0 0.15587200224399567 0.3079040050506592 0.17838079929351805 0.17838079929351805 0.04355865575265927 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 1 40960 8 0 8191.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1] 8 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 8q1s8k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
33 0.0 0.0 0.0 0.0 0.0 0.014271999709308147 0.02112000063061714 0.015516800060868263 0.01488000014796853 0.001940870731593904 0.0 0.0 0.0 0.0 0.0 0.21587200462818146 0.23561599850654602 0.22250880002975468 0.22250880002975468 0.006181951170646666 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 1 40960 16 0 8191.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] 16 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 16q1s8k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
34 0.0 0.0 0.0 0.0 0.0 0.015552000142633915 0.039264000952243805 0.023609600123018028 0.02131200022995472 0.007236979625711548 0.0 0.0 0.0 0.0 0.0 0.408735990524292 0.470335990190506 0.4336863994598388 0.4336863994598388 0.01844662383160074 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 1 40960 32 0 8191.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] 32 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 32q1s8k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
35 0.0 0.0 0.0 0.0 0.0 0.014399999752640724 0.025407999753952026 0.016227199975401164 0.014960000291466713 0.0031617375441736185 0.0 0.0 0.0 0.0 0.0 0.7412800192832947 0.7627840042114258 0.7464000046253203 0.7464000046253203 0.006112167448837547 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 1 40960 64 0 8191.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] 64 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 64q1s8k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
36 0.0 0.0 0.0 0.0 0.0 0.014720000326633453 0.02364799939095974 0.01809599995613098 0.016352000646293163 0.0035481127058959038 0.04822399839758873 0.08566399663686752 0.05961279980838299 0.05961279980838299 0.011445665413968877 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 2 40960 1 64 0.0 True FLASH_ATTN False vllm020_batch_spec [64] 64 64 64 64.0 True 0.0 0.0 0.0 False 0.0 64.0 64 BF16 generic none CUDA_EVENT False q64 measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
37 0.0 0.0 0.0 0.0 0.0 0.014175999909639359 0.020864000543951988 0.01668160008266568 0.015664000064134598 0.0025769063833097584 0.049695998430252075 0.08057600259780884 0.05882879942655563 0.05882879942655563 0.009515126108519331 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 2 40960 1 128 0.0 True FLASH_ATTN False vllm020_batch_spec [128] 128 128 128 128.0 True 0.0 0.0 0.0 False 0.0 128.0 128 BF16 generic none CUDA_EVENT False q128 measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
38 0.0 0.0 0.0 0.0 0.0 0.014399999752640724 0.028704000636935234 0.017430400010198355 0.015056000091135502 0.004349294613335555 0.049855999648571014 0.07366400212049484 0.05459520071744919 0.05459520071744919 0.0069610920757925574 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 2 40960 1 256 0.0 True FLASH_ATTN False vllm020_batch_spec [256] 256 256 256 256.0 True 0.0 0.0 0.0 False 0.0 256.0 256 BF16 generic none CUDA_EVENT False q256 measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
39 0.0 0.0 0.0 0.0 0.0 0.014336000196635723 0.03161599859595299 0.017788799852132796 0.015711999963968992 0.005005385723318659 0.06102399900555611 0.08179199695587158 0.06650560013949873 0.06650560013949873 0.006947995595801105 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 2 40960 1 512 0.0 True FLASH_ATTN False vllm020_batch_spec [512] 512 512 512 512.0 True 0.0 0.0 0.0 False 0.0 512.0 512 BF16 generic none CUDA_EVENT False q512 measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
40 0.0 0.0 0.0 0.0 0.0 0.014655999839305878 0.04416000097990036 0.019200000166893005 0.015488000120967627 0.008561241323364038 0.08054400235414505 0.09196799993515015 0.08607039973139763 0.08607039973139763 0.004145329035483754 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 2 40960 1 1024 0.0 True FLASH_ATTN False vllm020_batch_spec [1024] 1024 1024 1024 1024.0 True 0.0 0.0 0.0 False 0.0 1024.0 1024 BF16 generic none CUDA_EVENT False q1k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
41 0.0 0.0 0.0 0.0 0.0 0.017855999991297722 0.023391999304294586 0.019667199812829494 0.018463999964296818 0.0022577669687832585 0.18729600310325623 0.20585599541664124 0.19546559900045393 0.19546559900045393 0.006339068824303663 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 2 40960 1 2048 0.0 True FLASH_ATTN False vllm020_batch_spec [2048] 2048 2048 2048 2048.0 True 0.0 0.0 0.0 False 0.0 2048.0 2048 BF16 generic none CUDA_EVENT False q2k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
42 0.0 0.0 0.0 0.0 0.0 0.026240000501275063 0.03446400165557861 0.028297600522637367 0.027088000439107418 0.0025148441522922374 0.5754240155220032 0.5889919996261597 0.5800191938877105 0.5800191938877105 0.003858869273829596 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 2 40960 1 4096 0.0 True FLASH_ATTN False vllm020_batch_spec [4096] 4096 4096 4096 4096.0 True 0.0 0.0 0.0 False 0.0 4096.0 4096 BF16 generic none CUDA_EVENT False q4k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
43 0.0 0.0 0.0 0.0 0.0 0.04182400181889534 0.047200001776218414 0.043036799877882004 0.0423360001295805 0.0017073405772076728 2.063199996948242 2.0787200927734375 2.067151999473572 2.067151999473572 0.004959651271127835 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 2 40960 1 8192 0.0 True FLASH_ATTN False vllm020_batch_spec [8192] 8192 8192 8192 8192.0 True 0.0 0.0 0.0 False 0.0 8192.0 8192 BF16 generic none CUDA_EVENT False q8k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
44 0.0 0.0 0.0 0.0 0.0 0.014399999752640724 0.05648000165820122 0.02270399993285537 0.017935999669134617 0.012377915150727689 0.049536000937223434 0.07196799665689468 0.056015999615192415 0.056015999615192415 0.0070476637552742884 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 2 40960 1 64 448.0 True FLASH_ATTN False vllm020_batch_spec [64] 64 64 64 64.0 True 0.0 0.0 0.0 True 448.0 512.0 64 BF16 generic none CUDA_EVENT False q64s512 measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
45 0.0 0.0 0.0 0.0 0.0 0.01408000010997057 0.021344000473618507 0.01611520005390048 0.014928000047802925 0.0025499884993961702 0.07072000205516815 0.2642880082130432 0.1588256008923054 0.1588256008923054 0.053220347086102376 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 2 40960 1 128 896.0 True FLASH_ATTN False vllm020_batch_spec [128] 128 128 128 128.0 True 0.0 0.0 0.0 True 896.0 1024.0 128 BF16 generic none CUDA_EVENT False q128s1k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
46 0.0 0.0 0.0 0.0 0.0 0.01360000018030405 0.03014400042593479 0.016975999902933837 0.015056000091135502 0.004715159697364111 0.08393599838018417 0.11036799848079681 0.09160000011324881 0.09160000011324881 0.007911669434472792 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 2 40960 1 256 1792.0 True FLASH_ATTN False vllm020_batch_spec [256] 256 256 256 256.0 True 0.0 0.0 0.0 True 1792.0 2048.0 256 BF16 generic none CUDA_EVENT False q256s2k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
47 0.0 0.0 0.0 0.0 0.0 0.014495999552309513 0.020767999812960625 0.016233599931001663 0.015392000321298838 0.002202615907995427 0.1998399943113327 0.22070400416851044 0.20855360180139543 0.20855360180139543 0.00689307200230667 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 2 40960 1 512 3584.0 True FLASH_ATTN False vllm020_batch_spec [512] 512 512 512 512.0 True 0.0 0.0 0.0 True 3584.0 4096.0 512 BF16 generic none CUDA_EVENT False q512s4k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
48 0.0 0.0 0.0 0.0 0.0 0.01484800036996603 0.033440001308918 0.018684800155460833 0.016048000194132328 0.00553991005639174 0.6043199896812439 0.635807991027832 0.6126143991947173 0.6126143991947173 0.008745933953408096 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 2 40960 1 1024 7168.0 True FLASH_ATTN False vllm020_batch_spec [1024] 1024 1024 1024 1024.0 True 0.0 0.0 0.0 True 7168.0 8192.0 1024 BF16 generic none CUDA_EVENT False q1ks8k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
49 0.0 0.0 0.0 0.0 0.0 0.013824000023305416 0.03359999880194664 0.017811199743300678 0.014944000169634819 0.005926140483565472 0.0 0.0 0.0 0.0 0.0 0.045471999794244766 0.07539200037717819 0.054758400097489356 0.054758400097489356 0.010253548506101549 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 2 40960 1 0 127.0 False FLASH_ATTN False vllm020_batch_spec [1] 1 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False q1s128 measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
50 0.0 0.0 0.0 0.0 0.0 0.014047999866306782 0.02409599907696247 0.01641279999166727 0.01473599998280406 0.003347158638713987 0.0 0.0 0.0 0.0 0.0 0.0461760014295578 0.07529599964618683 0.05460800044238568 0.05460800044238568 0.009748937798340135 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 2 40960 8 0 127.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1] 8 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 8q1s128 measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
51 0.0 0.0 0.0 0.0 0.0 0.014271999709308147 0.028416000306606293 0.018611199874430894 0.01598400017246604 0.005209875093863647 0.0 0.0 0.0 0.0 0.0 0.048128001391887665 0.07897599786520004 0.061353600397706036 0.061353600397706036 0.010488153655157845 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 2 40960 16 0 127.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] 16 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 16q1s128 measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
52 0.0 0.0 0.0 0.0 0.0 0.014112000353634357 0.025728000327944756 0.016092800162732603 0.01512000011280179 0.003300888123052605 0.0 0.0 0.0 0.0 0.0 0.04864000156521797 0.07648000121116638 0.05810560062527656 0.05810560062527656 0.009473544218404408 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 2 40960 32 0 127.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] 32 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 32q1s128 measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
53 0.0 0.0 0.0 0.0 0.0 0.014655999839305878 0.03551999852061272 0.018588799890130757 0.016064000315964222 0.006071057437992447 0.0 0.0 0.0 0.0 0.0 0.04822399839758873 0.07862400263547897 0.05660480037331582 0.05660480037331582 0.009565394213730401 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 2 40960 64 0 127.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] 64 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 64q1s128 measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
54 0.0 0.0 0.0 0.0 0.0 0.014303999952971935 0.028831999748945236 0.01699519995599985 0.015024000313133001 0.004285000744868188 0.0 0.0 0.0 0.0 0.0 0.04854400083422661 0.06719999760389328 0.05778240002691746 0.05778240002691746 0.006852125554679805 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 2 40960 8 0 1023.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1] 8 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 8q1s1k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
55 0.0 0.0 0.0 0.0 0.0 0.0144640002399683 0.024927999824285507 0.0161183999851346 0.01521599991247058 0.0029774808524673907 0.0 0.0 0.0 0.0 0.0 0.05379199981689453 0.08966399729251862 0.06270079985260964 0.06270079985260964 0.009983591277092696 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 2 40960 16 0 1023.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] 16 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 16q1s1k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
56 0.0 0.0 0.0 0.0 0.0 0.014431999996304512 0.03001599945127964 0.017648000083863736 0.01547200046479702 0.004585126309484848 0.0 0.0 0.0 0.0 0.0 0.061664000153541565 0.07692799717187881 0.06704320013523103 0.06704320013523103 0.005500191798490629 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 2 40960 32 0 1023.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] 32 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 32q1s1k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
57 0.0 0.0 0.0 0.0 0.0 0.014175999909639359 0.026367999613285065 0.016947199776768684 0.015039999969303608 0.0037951566103550205 0.0 0.0 0.0 0.0 0.0 0.08799999952316284 0.111455999314785 0.0964031994342804 0.0964031994342804 0.007397541615558088 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 2 40960 64 0 1023.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] 64 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 64q1s1k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
58 0.0 0.0 0.0 0.0 0.0 0.01369599997997284 0.021247999742627144 0.015359999984502793 0.014640000183135271 0.0020942770950814317 0.0 0.0 0.0 0.0 0.0 0.051711998879909515 0.07065600156784058 0.058054400235414506 0.058054400235414506 0.006633034815910025 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 2 40960 8 0 2047.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1] 8 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 8q1s2k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
59 0.0 0.0 0.0 0.0 0.0 0.014208000153303146 0.0307839997112751 0.017148799914866685 0.015039999969303608 0.004882374686312284 0.0 0.0 0.0 0.0 0.0 0.061919998377561576 0.07843200117349625 0.06628479920327664 0.06628479920327664 0.004962852689801192 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 2 40960 16 0 2047.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] 16 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 16q1s2k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
60 0.0 0.0 0.0 0.0 0.0 0.013887999579310417 0.02969600073993206 0.017500799987465142 0.015232000034302473 0.00467273319705314 0.0 0.0 0.0 0.0 0.0 0.08819200098514557 0.11097600311040878 0.09493440166115762 0.09493440166115762 0.007509235042985577 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 2 40960 32 0 2047.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] 32 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 32q1s2k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
61 0.0 0.0 0.0 0.0 0.0 0.014399999752640724 0.022112000733613968 0.01751680001616478 0.016047999262809753 0.0030306621792915785 0.0 0.0 0.0 0.0 0.0 0.13065600395202637 0.15110400319099426 0.13857279866933822 0.13857279866933822 0.00750137841249771 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 2 40960 64 0 2047.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] 64 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 64q1s2k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
62 0.0 0.0 0.0 0.0 0.0 0.013919999822974205 0.04012800008058548 0.01892479993402958 0.01550400024279952 0.007459545267816961 0.0 0.0 0.0 0.0 0.0 0.06278400123119354 0.08259200304746628 0.0704512007534504 0.0704512007534504 0.005979055984382744 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 2 40960 8 0 4095.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1] 8 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 8q1s4k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
63 0.0 0.0 0.0 0.0 0.0 0.014336000196635723 0.02236800082027912 0.016332800220698118 0.014928000047802925 0.002784044363186731 0.0 0.0 0.0 0.0 0.0 0.1003199964761734 0.1279360055923462 0.10921279862523078 0.10921279862523078 0.00862769617046716 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 2 40960 16 0 4095.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] 16 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 16q1s4k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
64 0.0 0.0 0.0 0.0 0.0 0.013919999822974205 0.0225600004196167 0.016128000058233737 0.014800000004470348 0.002958953332547708 0.0 0.0 0.0 0.0 0.0 0.13116799294948578 0.14812800288200378 0.13783999979496003 0.13783999979496003 0.005696384361148053 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 2 40960 32 0 4095.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] 32 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 32q1s4k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
65 0.0 0.0 0.0 0.0 0.0 0.014208000153303146 0.029983999207615852 0.017468800116330386 0.01508800033479929 0.0047379550962483065 0.0 0.0 0.0 0.0 0.0 0.217631995677948 0.2447360008955002 0.22715839892625808 0.22715839892625808 0.008831828221847138 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 2 40960 64 0 4095.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] 64 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 64q1s4k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
66 0.0 0.0 0.0 0.0 0.0 0.014047999866306782 0.02304000034928322 0.016512000095099212 0.014512000139802694 0.0035026774828624254 0.0 0.0 0.0 0.0 0.0 0.11020799726247787 0.12307199835777283 0.11600959971547126 0.11600959971547126 0.004667637266950902 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 2 40960 8 0 8191.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1] 8 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 8q1s8k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
67 0.0 0.0 0.0 0.0 0.0 0.014112000353634357 0.042080000042915344 0.017510399967432023 0.014607999939471483 0.00823117883530167 0.0 0.0 0.0 0.0 0.0 0.15702399611473083 0.17587199807167053 0.16399359852075576 0.16399359852075576 0.006588299074676393 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 2 40960 16 0 8191.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] 16 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 16q1s8k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
68 0.0 0.0 0.0 0.0 0.0 0.013919999822974205 0.0208320003002882 0.015299199987202883 0.01462399959564209 0.001916768086505386 0.0 0.0 0.0 0.0 0.0 0.21779200434684753 0.2415360063314438 0.2260768011212349 0.2260768011212349 0.007251352080685236 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 2 40960 32 0 8191.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] 32 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 32q1s8k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
69 0.0 0.0 0.0 0.0 0.0 0.014015999622642994 0.021088000386953354 0.015619200188666582 0.01473599998280406 0.002013227452302141 0.0 0.0 0.0 0.0 0.0 0.3959999978542328 0.4152640104293823 0.40332479774951924 0.40332479774951924 0.006942401431914052 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 2 40960 64 0 8191.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] 64 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 64q1s8k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
70 0.0 0.0 0.0 0.0 0.0 0.014560000039637089 0.022784000262618065 0.016435200069099664 0.015519999898970127 0.0023688301421469523 0.04879999905824661 0.09139200299978256 0.06054079942405224 0.06054079942405224 0.012152608702448775 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 4 40960 1 64 0.0 True FLASH_ATTN False vllm020_batch_spec [64] 64 64 64 64.0 True 0.0 0.0 0.0 False 0.0 64.0 64 BF16 generic none CUDA_EVENT False q64 measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
71 0.0 0.0 0.0 0.0 0.0 0.014816000126302242 0.02844800055027008 0.017008000146597625 0.015696000307798386 0.003941466294662679 0.047807998955249786 0.07356800138950348 0.05626560002565384 0.05626560002565384 0.00842179125412236 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 4 40960 1 128 0.0 True FLASH_ATTN False vllm020_batch_spec [128] 128 128 128 128.0 True 0.0 0.0 0.0 False 0.0 128.0 128 BF16 generic none CUDA_EVENT False q128 measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
72 0.0 0.0 0.0 0.0 0.0 0.014688000082969666 0.053408000618219376 0.019353600218892097 0.01532800029963255 0.01138302400841626 0.048448000103235245 0.0785600021481514 0.0556256003677845 0.0556256003677845 0.009348274502616908 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 4 40960 1 256 0.0 True FLASH_ATTN False vllm020_batch_spec [256] 256 256 256 256.0 True 0.0 0.0 0.0 False 0.0 256.0 256 BF16 generic none CUDA_EVENT False q256 measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
73 0.0 0.0 0.0 0.0 0.0 0.015039999969303608 0.03139200061559677 0.01823679991066456 0.016159999649971724 0.004860071238302512 0.055424001067876816 0.08505599945783615 0.0640383992344141 0.0640383992344141 0.00921448636178623 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 4 40960 1 512 0.0 True FLASH_ATTN False vllm020_batch_spec [512] 512 512 512 512.0 True 0.0 0.0 0.0 False 0.0 512.0 512 BF16 generic none CUDA_EVENT False q512 measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
74 0.0 0.0 0.0 0.0 0.0 0.014751999638974667 0.026528000831604004 0.017324799951165915 0.01536000007763505 0.0036004822686428305 0.07660800218582153 0.0942080020904541 0.08209280073642732 0.08209280073642732 0.00515895587669752 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 4 40960 1 1024 0.0 True FLASH_ATTN False vllm020_batch_spec [1024] 1024 1024 1024 1024.0 True 0.0 0.0 0.0 False 0.0 1024.0 1024 BF16 generic none CUDA_EVENT False q1k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
75 0.0 0.0 0.0 0.0 0.0 0.014879999682307243 0.021247999742627144 0.016403199825435876 0.01532800029963255 0.001995897022647934 0.11395200341939926 0.15113599598407745 0.12431039959192276 0.12431039959192276 0.011164431123683732 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 4 40960 1 2048 0.0 True FLASH_ATTN False vllm020_batch_spec [2048] 2048 2048 2048 2048.0 True 0.0 0.0 0.0 False 0.0 2048.0 2048 BF16 generic none CUDA_EVENT False q2k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
76 0.0 0.0 0.0 0.0 0.0 0.017184000462293625 0.028672000393271446 0.019865600019693376 0.017823999747633934 0.0035798491315929977 0.3171840012073517 0.3341119885444641 0.3261695951223373 0.3261695951223373 0.005111046452878918 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 4 40960 1 4096 0.0 True FLASH_ATTN False vllm020_batch_spec [4096] 4096 4096 4096 4096.0 True 0.0 0.0 0.0 False 0.0 4096.0 4096 BF16 generic none CUDA_EVENT False q4k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
77 0.0 0.0 0.0 0.0 0.0 0.024639999493956566 0.030400000512599945 0.02656640000641346 0.02556800004094839 0.0020189846603237303 1.0648640394210815 1.0828479528427124 1.0715327858924866 1.0715327858924866 0.005558639263049851 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 4 40960 1 8192 0.0 True FLASH_ATTN False vllm020_batch_spec [8192] 8192 8192 8192 8192.0 True 0.0 0.0 0.0 False 0.0 8192.0 8192 BF16 generic none CUDA_EVENT False q8k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
78 0.0 0.0 0.0 0.0 0.0 0.014976000413298607 0.021695999428629875 0.017305599898099898 0.01593599934130907 0.0025718046517268054 0.04956800118088722 0.07932800054550171 0.06228480041027069 0.06228480041027069 0.01027160349757827 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 4 40960 1 64 448.0 True FLASH_ATTN False vllm020_batch_spec [64] 64 64 64 64.0 True 0.0 0.0 0.0 True 448.0 512.0 64 BF16 generic none CUDA_EVENT False q64s512 measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
79 0.0 0.0 0.0 0.0 0.0 0.01539199985563755 0.03580800071358681 0.019686400331556796 0.017136000096797943 0.005918387811304003 0.05158400163054466 0.10713600367307663 0.06364160068333148 0.06364160068333148 0.015832957809696766 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 4 40960 1 128 896.0 True FLASH_ATTN False vllm020_batch_spec [128] 128 128 128 128.0 True 0.0 0.0 0.0 True 896.0 1024.0 128 BF16 generic none CUDA_EVENT False q128s1k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
80 0.0 0.0 0.0 0.0 0.0 0.015168000012636185 0.02223999984562397 0.016672000009566545 0.015887999907135963 0.0020934945946034563 0.06521599739789963 0.08902399986982346 0.07520959973335266 0.07520959973335266 0.007913840684102929 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 4 40960 1 256 1792.0 True FLASH_ATTN False vllm020_batch_spec [256] 256 256 256 256.0 True 0.0 0.0 0.0 True 1792.0 2048.0 256 BF16 generic none CUDA_EVENT False q256s2k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
81 0.0 0.0 0.0 0.0 0.0 0.015231999568641186 0.04028800129890442 0.019971200078725816 0.01646399963647127 0.007351305886577286 0.14467200636863708 0.16844800114631653 0.15363519936800005 0.15363519936800005 0.008174435899956223 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 4 40960 1 512 3584.0 True FLASH_ATTN False vllm020_batch_spec [512] 512 512 512 512.0 True 0.0 0.0 0.0 True 3584.0 4096.0 512 BF16 generic none CUDA_EVENT False q512s4k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
82 0.0 0.0 0.0 0.0 0.0 0.015519999898970127 0.02304000034928322 0.01775679988786578 0.016784000210464 0.0025077243712082584 0.33740800619125366 0.35343998670578003 0.3445120006799698 0.3445120006799698 0.00445648463590358 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 4 40960 1 1024 7168.0 True FLASH_ATTN False vllm020_batch_spec [1024] 1024 1024 1024 1024.0 True 0.0 0.0 0.0 True 7168.0 8192.0 1024 BF16 generic none CUDA_EVENT False q1ks8k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
83 0.0 0.0 0.0 0.0 0.0 0.015647999942302704 0.02393599972128868 0.01809599995613098 0.01654400024563074 0.002998393102466254 0.0 0.0 0.0 0.0 0.0 0.0504320003092289 0.0843840017914772 0.060083200410008426 0.060083200410008426 0.00986959318572296 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 4 40960 1 0 127.0 False FLASH_ATTN False vllm020_batch_spec [1] 1 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False q1s128 measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
84 0.0 0.0 0.0 0.0 0.0 0.01603199914097786 0.030047999694943428 0.019510399922728537 0.017680000513792038 0.004065185265963332 0.0 0.0 0.0 0.0 0.0 0.05004800111055374 0.07036799937486649 0.059315200522542 0.059315200522542 0.006537768821329647 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 4 40960 8 0 127.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1] 8 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 8q1s128 measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
85 0.0 0.0 0.0 0.0 0.0 0.01484800036996603 0.02393599972128868 0.018035199772566558 0.016512000001966953 0.0030667689116777724 0.0 0.0 0.0 0.0 0.0 0.05100800096988678 0.06735999882221222 0.058387200161814694 0.058387200161814694 0.005832787739241381 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 4 40960 16 0 127.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] 16 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 16q1s128 measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
86 0.0 0.0 0.0 0.0 0.0 0.01500799972563982 0.026208000257611275 0.017500799987465142 0.01646399963647127 0.0030666103306165714 0.0 0.0 0.0 0.0 0.0 0.05023999884724617 0.07254400104284286 0.057254400476813315 0.057254400476813315 0.006890853653068151 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 4 40960 32 0 127.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] 32 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 32q1s128 measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
87 0.0 0.0 0.0 0.0 0.0 0.014688000082969666 0.027327999472618103 0.0179776000790298 0.01648000068962574 0.003783417447531392 0.0 0.0 0.0 0.0 0.0 0.04819199815392494 0.07100799679756165 0.05621119923889638 0.05621119923889638 0.007824391484124725 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 4 40960 64 0 127.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] 64 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 64q1s128 measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
88 0.0 0.0 0.0 0.0 0.0 0.015647999942302704 0.036448001861572266 0.019136000238358975 0.016704000532627106 0.006076530095624803 0.0 0.0 0.0 0.0 0.0 0.047488000243902206 0.08441600203514099 0.0588383998721838 0.0588383998721838 0.011275940726332522 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 4 40960 8 0 1023.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1] 8 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 8q1s1k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
89 0.0 0.0 0.0 0.0 0.0 0.015135999768972397 0.033952001482248306 0.02119360016658902 0.018000000156462193 0.007136646614305304 0.0 0.0 0.0 0.0 0.0 0.04931199923157692 0.06992000341415405 0.05755840018391609 0.05755840018391609 0.007297587694315226 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 4 40960 16 0 1023.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] 16 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 16q1s1k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
90 0.0 0.0 0.0 0.0 0.0 0.014879999682307243 0.034272000193595886 0.0204415999352932 0.017311999574303627 0.00695404701803013 0.0 0.0 0.0 0.0 0.0 0.05215999856591225 0.0735040009021759 0.06117440015077591 0.06117440015077591 0.007381118436894922 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 4 40960 32 0 1023.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] 32 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 32q1s1k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
91 0.0 0.0 0.0 0.0 0.0 0.015359999611973763 0.03743999823927879 0.02012479966506362 0.01775999926030636 0.006181045509079057 0.0 0.0 0.0 0.0 0.0 0.06355199962854385 0.08508799970149994 0.07019200026988984 0.07019200026988984 0.0062246580442117845 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 4 40960 64 0 1023.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] 64 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 64q1s1k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
92 0.0 0.0 0.0 0.0 0.0 0.015552000142633915 0.032607998698949814 0.01959999995306134 0.017487999983131886 0.004882475068460749 0.0 0.0 0.0 0.0 0.0 0.04918399825692177 0.0865280032157898 0.05973760038614274 0.05973760038614274 0.011546847942113974 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 4 40960 8 0 2047.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1] 8 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 8q1s2k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
93 0.0 0.0 0.0 0.0 0.0 0.015168000012636185 0.02675200067460537 0.017430400010198355 0.016560000367462635 0.003243135094239819 0.0 0.0 0.0 0.0 0.0 0.052671998739242554 0.08367999643087387 0.06238719932734965 0.06238719932734965 0.011207824780630104 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 4 40960 16 0 2047.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] 16 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 16q1s2k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
94 0.0 0.0 0.0 0.0 0.0 0.01500799972563982 0.03612799942493439 0.019327999837696553 0.01688000001013279 0.006058251425153085 0.0 0.0 0.0 0.0 0.0 0.06364800035953522 0.07878399640321732 0.06935679838061332 0.06935679838061332 0.004515588328677643 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 4 40960 32 0 2047.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] 32 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 32q1s2k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
95 0.0 0.0 0.0 0.0 0.0 0.015104000456631184 0.03711999952793121 0.019670399930328132 0.017152000218629837 0.006165994646522264 0.0 0.0 0.0 0.0 0.0 0.09071999788284302 0.11507199704647064 0.10252480059862136 0.10252480059862136 0.008782544051535657 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 4 40960 64 0 2047.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] 64 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 64q1s2k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
96 0.0 0.0 0.0 0.0 0.0 0.015039999969303608 0.026528000831604004 0.018396800104528665 0.01601599995046854 0.004279788048986283 0.0 0.0 0.0 0.0 0.0 0.05331199988722801 0.07977599650621414 0.06076480001211167 0.06076480001211167 0.008761579122069606 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 4 40960 8 0 4095.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1] 8 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 8q1s4k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
97 0.0 0.0 0.0 0.0 0.0 0.015359999611973763 0.02643200010061264 0.01726400004699826 0.015855999663472176 0.003210008417242029 0.0 0.0 0.0 0.0 0.0 0.062431998550891876 0.08246400207281113 0.06970879957079888 0.06970879957079888 0.007016341676068233 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 4 40960 16 0 4095.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] 16 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 16q1s4k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
98 0.0 0.0 0.0 0.0 0.0 0.014911999925971031 0.02223999984562397 0.0166015999391675 0.015887999907135963 0.00204913969129354 0.0 0.0 0.0 0.0 0.0 0.10127999633550644 0.1141119971871376 0.10621120035648347 0.10621120035648347 0.004029318311754605 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 4 40960 32 0 4095.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] 32 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 32q1s4k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
99 0.0 0.0 0.0 0.0 0.0 0.015359999611973763 0.03363199904561043 0.019305599946528675 0.016671999357640743 0.0055445582307981234 0.0 0.0 0.0 0.0 0.0 0.1319040060043335 0.15839999914169312 0.14040640145540234 0.14040640145540234 0.007998041073596942 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 4 40960 64 0 4095.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] 64 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 64q1s4k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
100 0.0 0.0 0.0 0.0 0.0 0.014368000440299511 0.04447999969124794 0.018908800091594458 0.0157279996201396 0.0086814937461721 0.0 0.0 0.0 0.0 0.0 0.06428799778223038 0.08982399851083755 0.07349760085344315 0.07349760085344315 0.008605284512197258 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 4 40960 8 0 8191.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1] 8 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 8q1s8k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
101 0.0 0.0 0.0 0.0 0.0 0.015104000456631184 0.028672000393271446 0.01890560006722808 0.016080000437796116 0.004797584627815021 0.0 0.0 0.0 0.0 0.0 0.11123199760913849 0.14115199446678162 0.11942399889230729 0.11942399889230729 0.008513394706791027 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 4 40960 16 0 8191.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] 16 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 16q1s8k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
102 0.0 0.0 0.0 0.0 0.0 0.014271999709308147 0.024927999824285507 0.016915200091898442 0.015216000378131866 0.003374147499442635 0.0 0.0 0.0 0.0 0.0 0.15887999534606934 0.18111999332904816 0.16934399753808976 0.16934399753808976 0.007415181260761123 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 4 40960 32 0 8191.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] 32 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 32q1s8k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
103 0.0 0.0 0.0 0.0 0.0 0.014336000196635723 0.026944000273942947 0.017737600207328796 0.015199999790638685 0.00415598714375255 0.0 0.0 0.0 0.0 0.0 0.2192319929599762 0.23472000658512115 0.22809920012950893 0.22809920012950893 0.004730841376327335 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 4 40960 64 0 8191.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] 64 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 64q1s8k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median

31
runs/frontier-qwen30-vllm020-profile-v1/frozen/profile-v1/attention_true_mixed_fused.csv Normal file
View File

@@ -0,0 +1,31 @@
num_tensor_parallel_workers,batch_spec,num_prefill_seqs,num_decode_seqs,total_prefill_tokens,total_decode_tokens,decode_avg_kv_cache_size,attention_core_mean_ms,attention_core_mean_as_median_ms,kv_cache_update_median_ms,representation
1,q64_8q1s512,1,8,64,8,511.0,0.06604800000786781,0.06604800000786781,0.021359999664127827,one_fused_FA3_call_not_split_into_prefill_decode
1,q128_8q1s1k,1,8,128,8,1023.0,0.07483199909329416,0.07483199909329416,0.024255999363958836,one_fused_FA3_call_not_split_into_prefill_decode
1,q128_16q1s1k,1,16,128,16,1023.0,0.10597119852900506,0.10597119852900506,0.018112000077962875,one_fused_FA3_call_not_split_into_prefill_decode
1,q256_16q1s2k,1,16,256,16,2047.0,0.13697599917650224,0.13697599917650224,0.01673599984496832,one_fused_FA3_call_not_split_into_prefill_decode
1,q256_32q1s2k,1,32,256,32,2047.0,0.2163648009300232,0.2163648009300232,0.01515199989080429,one_fused_FA3_call_not_split_into_prefill_decode
1,q512_32q1s4k,1,32,512,32,4095.0,0.3746495962142944,0.3746495962142944,0.016191999427974224,one_fused_FA3_call_not_split_into_prefill_decode
1,q512_64q1s4k,1,64,512,64,4095.0,0.6718560099601746,0.6718560099601746,0.024848000146448612,one_fused_FA3_call_not_split_into_prefill_decode
1,q1k_64q1s8k,1,64,1024,64,8191.0,1.3472000002861022,1.3472000002861022,0.019600000232458115,one_fused_FA3_call_not_split_into_prefill_decode
1,q2k_32q1s4k,1,32,2048,32,4095.0,0.6351647913455962,0.6351647913455962,0.028256000019609928,one_fused_FA3_call_not_split_into_prefill_decode
1,q4k_16q1s4k,1,16,4096,16,4095.0,1.2515872120857237,1.2515872120857237,0.04395199939608574,one_fused_FA3_call_not_split_into_prefill_decode
2,q64_8q1s512,1,8,64,8,511.0,0.05652800053358078,0.05652800053358078,0.019600000232458115,one_fused_FA3_call_not_split_into_prefill_decode
2,q128_8q1s1k,1,8,128,8,1023.0,0.07362559959292413,0.07362559959292413,0.014431999996304512,one_fused_FA3_call_not_split_into_prefill_decode
2,q128_16q1s1k,1,16,128,16,1023.0,0.0736224003136158,0.0736224003136158,0.014479999896138906,one_fused_FA3_call_not_split_into_prefill_decode
2,q256_16q1s2k,1,16,256,16,2047.0,0.10047360062599181,0.10047360062599181,0.01425600005313754,one_fused_FA3_call_not_split_into_prefill_decode
2,q256_32q1s2k,1,32,256,32,2047.0,0.14032640159130094,0.14032640159130094,0.01566399959847331,one_fused_FA3_call_not_split_into_prefill_decode
2,q512_32q1s4k,1,32,512,32,4095.0,0.22105600088834762,0.22105600088834762,0.016032000072300434,one_fused_FA3_call_not_split_into_prefill_decode
2,q512_64q1s4k,1,64,512,64,4095.0,0.36867519915103913,0.36867519915103913,0.015263999812304974,one_fused_FA3_call_not_split_into_prefill_decode
2,q1k_64q1s8k,1,64,1024,64,8191.0,0.6853824079036711,0.6853824079036711,0.015856000129133463,one_fused_FA3_call_not_split_into_prefill_decode
2,q2k_32q1s4k,1,32,2048,32,4095.0,0.3503839999437332,0.3503839999437332,0.018432000651955605,one_fused_FA3_call_not_split_into_prefill_decode
2,q4k_16q1s4k,1,16,4096,16,4095.0,0.7185311973094941,0.7185311973094941,0.026575999334454536,one_fused_FA3_call_not_split_into_prefill_decode
4,q64_8q1s512,1,8,64,8,511.0,0.0562208004295826,0.0562208004295826,0.015520000364631414,one_fused_FA3_call_not_split_into_prefill_decode
4,q128_8q1s1k,1,8,128,8,1023.0,0.061199999228119854,0.061199999228119854,0.018240000121295452,one_fused_FA3_call_not_split_into_prefill_decode
4,q128_16q1s1k,1,16,128,16,1023.0,0.06104319989681244,0.06104319989681244,0.016207999549806118,one_fused_FA3_call_not_split_into_prefill_decode
4,q256_16q1s2k,1,16,256,16,2047.0,0.07356479987502099,0.07356479987502099,0.015584000386297703,one_fused_FA3_call_not_split_into_prefill_decode
4,q256_32q1s2k,1,32,256,32,2047.0,0.09806400015950203,0.09806400015950203,0.015008000191301107,one_fused_FA3_call_not_split_into_prefill_decode
4,q512_32q1s4k,1,32,512,32,4095.0,0.1509471982717514,0.1509471982717514,0.014864000026136637,one_fused_FA3_call_not_split_into_prefill_decode
4,q512_64q1s4k,1,64,512,64,4095.0,0.21687040030956264,0.21687040030956264,0.015343999955803156,one_fused_FA3_call_not_split_into_prefill_decode
4,q1k_64q1s8k,1,64,1024,64,8191.0,0.3848575979471207,0.3848575979471207,0.015647999942302704,one_fused_FA3_call_not_split_into_prefill_decode
4,q2k_32q1s4k,1,32,2048,32,4095.0,0.2785151988267898,0.2785151988267898,0.01726400014013052,one_fused_FA3_call_not_split_into_prefill_decode
4,q4k_16q1s4k,1,16,4096,16,4095.0,0.456512001156807,0.456512001156807,0.017680000513792038,one_fused_FA3_call_not_split_into_prefill_decode
1 num_tensor_parallel_workers batch_spec num_prefill_seqs num_decode_seqs total_prefill_tokens total_decode_tokens decode_avg_kv_cache_size attention_core_mean_ms attention_core_mean_as_median_ms kv_cache_update_median_ms representation
2 1 q64_8q1s512 1 8 64 8 511.0 0.06604800000786781 0.06604800000786781 0.021359999664127827 one_fused_FA3_call_not_split_into_prefill_decode
3 1 q128_8q1s1k 1 8 128 8 1023.0 0.07483199909329416 0.07483199909329416 0.024255999363958836 one_fused_FA3_call_not_split_into_prefill_decode
4 1 q128_16q1s1k 1 16 128 16 1023.0 0.10597119852900506 0.10597119852900506 0.018112000077962875 one_fused_FA3_call_not_split_into_prefill_decode
5 1 q256_16q1s2k 1 16 256 16 2047.0 0.13697599917650224 0.13697599917650224 0.01673599984496832 one_fused_FA3_call_not_split_into_prefill_decode
6 1 q256_32q1s2k 1 32 256 32 2047.0 0.2163648009300232 0.2163648009300232 0.01515199989080429 one_fused_FA3_call_not_split_into_prefill_decode
7 1 q512_32q1s4k 1 32 512 32 4095.0 0.3746495962142944 0.3746495962142944 0.016191999427974224 one_fused_FA3_call_not_split_into_prefill_decode
8 1 q512_64q1s4k 1 64 512 64 4095.0 0.6718560099601746 0.6718560099601746 0.024848000146448612 one_fused_FA3_call_not_split_into_prefill_decode
9 1 q1k_64q1s8k 1 64 1024 64 8191.0 1.3472000002861022 1.3472000002861022 0.019600000232458115 one_fused_FA3_call_not_split_into_prefill_decode
10 1 q2k_32q1s4k 1 32 2048 32 4095.0 0.6351647913455962 0.6351647913455962 0.028256000019609928 one_fused_FA3_call_not_split_into_prefill_decode
11 1 q4k_16q1s4k 1 16 4096 16 4095.0 1.2515872120857237 1.2515872120857237 0.04395199939608574 one_fused_FA3_call_not_split_into_prefill_decode
12 2 q64_8q1s512 1 8 64 8 511.0 0.05652800053358078 0.05652800053358078 0.019600000232458115 one_fused_FA3_call_not_split_into_prefill_decode
13 2 q128_8q1s1k 1 8 128 8 1023.0 0.07362559959292413 0.07362559959292413 0.014431999996304512 one_fused_FA3_call_not_split_into_prefill_decode
14 2 q128_16q1s1k 1 16 128 16 1023.0 0.0736224003136158 0.0736224003136158 0.014479999896138906 one_fused_FA3_call_not_split_into_prefill_decode
15 2 q256_16q1s2k 1 16 256 16 2047.0 0.10047360062599181 0.10047360062599181 0.01425600005313754 one_fused_FA3_call_not_split_into_prefill_decode
16 2 q256_32q1s2k 1 32 256 32 2047.0 0.14032640159130094 0.14032640159130094 0.01566399959847331 one_fused_FA3_call_not_split_into_prefill_decode
17 2 q512_32q1s4k 1 32 512 32 4095.0 0.22105600088834762 0.22105600088834762 0.016032000072300434 one_fused_FA3_call_not_split_into_prefill_decode
18 2 q512_64q1s4k 1 64 512 64 4095.0 0.36867519915103913 0.36867519915103913 0.015263999812304974 one_fused_FA3_call_not_split_into_prefill_decode
19 2 q1k_64q1s8k 1 64 1024 64 8191.0 0.6853824079036711 0.6853824079036711 0.015856000129133463 one_fused_FA3_call_not_split_into_prefill_decode
20 2 q2k_32q1s4k 1 32 2048 32 4095.0 0.3503839999437332 0.3503839999437332 0.018432000651955605 one_fused_FA3_call_not_split_into_prefill_decode
21 2 q4k_16q1s4k 1 16 4096 16 4095.0 0.7185311973094941 0.7185311973094941 0.026575999334454536 one_fused_FA3_call_not_split_into_prefill_decode
22 4 q64_8q1s512 1 8 64 8 511.0 0.0562208004295826 0.0562208004295826 0.015520000364631414 one_fused_FA3_call_not_split_into_prefill_decode
23 4 q128_8q1s1k 1 8 128 8 1023.0 0.061199999228119854 0.061199999228119854 0.018240000121295452 one_fused_FA3_call_not_split_into_prefill_decode
24 4 q128_16q1s1k 1 16 128 16 1023.0 0.06104319989681244 0.06104319989681244 0.016207999549806118 one_fused_FA3_call_not_split_into_prefill_decode
25 4 q256_16q1s2k 1 16 256 16 2047.0 0.07356479987502099 0.07356479987502099 0.015584000386297703 one_fused_FA3_call_not_split_into_prefill_decode
26 4 q256_32q1s2k 1 32 256 32 2047.0 0.09806400015950203 0.09806400015950203 0.015008000191301107 one_fused_FA3_call_not_split_into_prefill_decode
27 4 q512_32q1s4k 1 32 512 32 4095.0 0.1509471982717514 0.1509471982717514 0.014864000026136637 one_fused_FA3_call_not_split_into_prefill_decode
28 4 q512_64q1s4k 1 64 512 64 4095.0 0.21687040030956264 0.21687040030956264 0.015343999955803156 one_fused_FA3_call_not_split_into_prefill_decode
29 4 q1k_64q1s8k 1 64 1024 64 8191.0 0.3848575979471207 0.3848575979471207 0.015647999942302704 one_fused_FA3_call_not_split_into_prefill_decode
30 4 q2k_32q1s4k 1 32 2048 32 4095.0 0.2785151988267898 0.2785151988267898 0.01726400014013052 one_fused_FA3_call_not_split_into_prefill_decode
31 4 q4k_16q1s4k 1 16 4096 16 4095.0 0.456512001156807 0.456512001156807 0.017680000513792038 one_fused_FA3_call_not_split_into_prefill_decode

37
runs/frontier-qwen30-vllm020-profile-v1/frozen/profile-v1/linear_op.csv Normal file
View File

@@ -0,0 +1,37 @@
time_stats.emb.min,time_stats.emb.max,time_stats.emb.mean,time_stats.emb.median,time_stats.emb.std,time_stats.input_layernorm.min,time_stats.input_layernorm.max,time_stats.input_layernorm.mean,time_stats.input_layernorm.median,time_stats.input_layernorm.std,time_stats.attn_pre_proj.min,time_stats.attn_pre_proj.max,time_stats.attn_pre_proj.mean,time_stats.attn_pre_proj.median,time_stats.attn_pre_proj.std,time_stats.attn_rope.min,time_stats.attn_rope.max,time_stats.attn_rope.mean,time_stats.attn_rope.median,time_stats.attn_rope.std,time_stats.attn_post_proj.min,time_stats.attn_post_proj.max,time_stats.attn_post_proj.mean,time_stats.attn_post_proj.median,time_stats.attn_post_proj.std,time_stats.post_attention_layernorm.min,time_stats.post_attention_layernorm.max,time_stats.post_attention_layernorm.mean,time_stats.post_attention_layernorm.median,time_stats.post_attention_layernorm.std,n_head,n_kv_head,n_embd,n_expanded_embd,vocab_size,use_gated_mlp,use_qk_norm,attn_output_gate,num_tokens,num_tensor_parallel_workers,padded_n_embd,padded_n_expanded_embd,model_arch,is_step2_mini,share_expert_dim,share_q_dim,measurement_type,profiling_precision,quant_signature
0.029184000566601753,0.06780800223350525,0.03157280012965202,0.030736000277101994,0.005874173435341216,0.033215999603271484,0.04825599864125252,0.03443359974771738,0.0337119996547699,0.0031825678429048183,1.438431978225708,1.505568027496338,1.446228802204132,1.4429279565811157,0.014128607642643025,0.538752019405365,0.5440319776535034,0.5416463971138,0.5420799851417542,0.0016099306164432847,1.0073280334472656,1.0163840055465698,1.0098415970802308,1.0081279873847961,0.0032980457197329728,0.04022400081157684,0.041728001087903976,0.04091359991580248,0.04081599973142147,0.0004728505416767937,32,4,2048,768,151936,True,True,False,8192,1,2048,768,generic,False,,,CUDA_EVENT,BF16,none
0.01360000018030405,0.06784000247716904,0.017755200061947106,0.0179840000346303,0.00837681421401443,0.01836800016462803,0.03651199862360954,0.019606399815529585,0.018719999119639397,0.0038821958848767424,0.7512000203132629,0.8208960294723511,0.7566704005002975,0.75382399559021,0.014793092586577971,0.28995200991630554,0.29337599873542786,0.29135999977588656,0.29150401055812836,0.000998381071258815,0.5149760246276855,0.5169600248336792,0.5160208016633987,0.5158880054950714,0.0005038652783291977,0.02051199972629547,0.021503999829292297,0.021067200042307378,0.021104000508785248,0.00023542855306902367,32,4,2048,768,151936,True,True,False,4096,1,2048,768,generic,False,,,CUDA_EVENT,BF16,none
0.0080960001796484,0.04281599819660187,0.011092799971811474,0.011039999779313803,0.005489135752949302,0.012223999947309494,0.026335999369621277,0.013187199970707298,0.01247999956831336,0.0030236510562153375,0.3928639888763428,0.45372799038887024,0.3976895987987518,0.39528000354766846,0.012905176716136006,0.1547199934720993,0.15884800255298615,0.15712319910526276,0.1573439985513687,0.001165121945135037,0.26633599400520325,0.268095999956131,0.26719200164079665,0.2671840041875839,0.0004242740199415328,0.013024000450968742,0.013887999579310417,0.013489600038155913,0.013520000036805868,0.0002382817554057738,32,4,2048,768,151936,True,True,False,2048,1,2048,768,generic,False,,,CUDA_EVENT,BF16,none
0.00825599953532219,0.04755200073122978,0.02398160002194345,0.01961600035429001,0.00874683840888523,0.018880000337958336,0.03868800029158592,0.021590400114655496,0.0208320003002882,0.004057015751746236,0.24316799640655518,0.2710399925708771,0.2521967992186546,0.25065599381923676,0.007998418528894075,0.09644799679517746,0.19120000302791595,0.10407840013504029,0.09963199868798256,0.020043722414992166,0.13600000739097595,0.18892799317836761,0.15760480016469955,0.15760000050067902,0.0112223677907762,0.009664000011980534,0.010015999898314476,0.009836799977347255,0.009824000298976898,9.016971952948177e-05,32,4,2048,768,151936,True,True,False,1024,1,2048,768,generic,False,,,CUDA_EVENT,BF16,none
0.017376000061631203,0.044704001396894455,0.026318399980664254,0.027312000282108784,0.007565344034680866,0.01836800016462803,0.03014400042593479,0.021276800055056812,0.020655999891459942,0.002632977855097951,0.1438719928264618,0.17132799327373505,0.152497598528862,0.15012799948453903,0.007947150319625347,0.1430719941854477,0.19305600225925446,0.16630879789590836,0.1685439944267273,0.014628024163894684,0.08899199962615967,0.10467199981212616,0.0943599995225668,0.09374399855732918,0.003472669494074113,0.007615999784320593,0.007935999892652035,0.007769599952735007,0.0077760000713169575,7.680004540222077e-05,32,4,2048,768,151936,True,True,False,512,1,2048,768,generic,False,,,CUDA_EVENT,BF16,none
0.016992000862956047,0.0544000007212162,0.02573199989274144,0.026016000658273697,0.00803323401720434,0.018432000651955605,0.02348800003528595,0.020648000109940768,0.02062400057911873,0.0013939985047930988,0.10220800340175629,0.1361600011587143,0.11850560046732425,0.11684799939393997,0.010637754898360304,0.16710400581359863,0.21110400557518005,0.19078560024499894,0.19409599900245667,0.013714352646558832,0.06265600025653839,0.0740479975938797,0.06842879951000214,0.0690080001950264,0.0031292240446560557,0.00979200005531311,0.033440001308918,0.01864320016466081,0.017280000261962414,0.006957998188876383,32,4,2048,768,151936,True,True,False,256,1,2048,768,generic,False,,,CUDA_EVENT,BF16,none
0.017152000218629837,0.04396799951791763,0.025907999789342284,0.02598400041460991,0.007714554737915267,0.018400000408291817,0.03667199984192848,0.024132800102233887,0.02112000063061714,0.006120348733354326,0.10678400099277496,0.1363839954137802,0.1193264003843069,0.11583999916911125,0.009838647443214228,0.17017599940299988,0.22748799622058868,0.18853759989142418,0.18433599919080734,0.015728078443174653,0.04569600149989128,0.06652799993753433,0.05192639995366335,0.05151999928057194,0.004516605694976449,0.021856000646948814,0.026335999369621277,0.02384479995816946,0.023599999956786633,0.0011301153013314744,32,4,2048,768,151936,True,True,False,128,1,2048,768,generic,False,,,CUDA_EVENT,BF16,none
0.017343999817967415,1.0683200359344482,0.05748240072280168,0.029504000209271908,0.16366824814254827,0.018688000738620758,0.3317759931087494,0.03685439983382821,0.021151999942958355,0.06767422466731164,0.10255999863147736,0.9434880018234253,0.16412640027701855,0.11956800147891045,0.17964606281728834,0.1714559942483902,2.1306240558624268,0.3011296011507511,0.1926399990916252,0.42310127734378766,0.03574400022625923,0.6859520077705383,0.08389280084520578,0.04279999993741512,0.14321647071615612,0.020479999482631683,0.1831360012292862,0.033024000097066165,0.023856000043451786,0.03470987082429836,32,4,2048,768,151936,True,True,False,64,1,2048,768,generic,False,,,CUDA_EVENT,BF16,none
0.016095999628305435,0.05142400041222572,0.026363200135529043,0.02676799986511469,0.008637725852473854,0.01849599927663803,0.03577600046992302,0.022193600237369538,0.020848000422120094,0.004615266181181815,0.10540799796581268,0.15014399588108063,0.12211520001292228,0.11896000057458878,0.012772013396624768,0.17315199971199036,0.21478399634361267,0.1881632000207901,0.1873439997434616,0.011761164657572015,0.03481600061058998,0.058079998940229416,0.042200000025331974,0.03969600051641464,0.006461281521769989,0.02143999934196472,0.038816001266241074,0.02466559996828437,0.023856000043451786,0.0035418033748569927,32,4,2048,768,151936,True,True,False,32,1,2048,768,generic,False,,,CUDA_EVENT,BF16,none
0.015904000028967857,0.03753599897027016,0.023127999808639287,0.02527999971061945,0.0054230027890305385,0.018400000408291817,0.03001599945127964,0.020488000102341176,0.020096000283956528,0.002585688394137697,0.10355199873447418,0.1438400000333786,0.11536479964852334,0.11124800145626068,0.011136028294919255,0.16502399742603302,0.2072959989309311,0.18646399974822997,0.19075199961662292,0.013296437761247597,0.03142400085926056,0.05215999856591225,0.03888959977775812,0.03750399872660637,0.0057399302067536314,0.020128000527620316,0.04064000025391579,0.023937600292265417,0.023648000322282314,0.004144197400898248,32,4,2048,768,151936,True,True,False,16,1,2048,768,generic,False,,,CUDA_EVENT,BF16,none
0.015200000256299973,0.037151999771595,0.02144480012357235,0.02195199951529503,0.005576364091933697,0.017952000722289085,0.0226879995316267,0.019934400077909233,0.019952000118792057,0.0011812499470458758,0.10063999891281128,0.13468800485134125,0.11595199964940547,0.1207519993185997,0.011974301621092394,0.16332800686359406,0.20748800039291382,0.18162400051951408,0.1767839938402176,0.01474120743720334,0.03222399950027466,0.043455999344587326,0.03829439990222454,0.03859200142323971,0.0031378131240041122,0.020959999412298203,0.03587200120091438,0.023795200139284135,0.023423999547958374,0.0031339489695198443,32,4,2048,768,151936,True,True,False,8,1,2048,768,generic,False,,,CUDA_EVENT,BF16,none
0.014944000169634819,0.04022400081157684,0.02162720002233982,0.022463999688625336,0.006017219317026815,0.018464000895619392,0.026528000831604004,0.021264000236988066,0.020896000787615776,0.001934095017586082,0.10156799852848053,0.1703999936580658,0.12565439902245998,0.1244799979031086,0.01641776722785409,0.1653759926557541,0.23865599930286407,0.1969360001385212,0.19223999977111816,0.02058903050274278,0.03254399821162224,0.06752000004053116,0.0443536002188921,0.041519999504089355,0.00984829071098989,0.020096000283956528,0.040031999349594116,0.026934400014579297,0.02478400059044361,0.00562071702648593,32,4,2048,768,151936,True,True,False,1,1,2048,768,generic,False,,,CUDA_EVENT,BF16,none
,,,,,,,,,,0.7531200051307678,0.8461440205574036,0.7618160009384155,0.7576479911804199,0.019464233617982506,0.2922559976577759,0.2985599935054779,0.2953856036067009,0.29576000571250916,0.001575901077689139,0.510047972202301,0.5140479803085327,0.5121696025133133,0.5123839974403381,0.0012263137313476844,,,,,,32,4,2048,768,151936,True,True,False,8192,2,2048,768,generic,False,,,CUDA_EVENT,BF16,none
,,,,,,,,,,0.39529600739479065,0.47279998660087585,0.40216960161924364,0.39825600385665894,0.0162749796240819,0.15625600516796112,0.16211199760437012,0.15959519892930984,0.15988799929618835,0.0011493418942396922,0.2635200023651123,0.2642880082130432,0.2639120012521744,0.26392000913619995,0.0001903593401384135,,,,,,32,4,2048,768,151936,True,True,False,4096,2,2048,768,generic,False,,,CUDA_EVENT,BF16,none
,,,,,,,,,,0.2531839907169342,0.32419198751449585,0.26924319565296173,0.26049599051475525,0.01657194262368116,0.10051199793815613,0.2375359982252121,0.12411200068891048,0.10311999917030334,0.039466165428540506,0.15881599485874176,0.19289599359035492,0.16896959990262986,0.1640480011701584,0.009162416086418127,,,,,,32,4,2048,768,151936,True,True,False,2048,2,2048,768,generic,False,,,CUDA_EVENT,BF16,none
,,,,,,,,,,0.15302400290966034,0.1780800074338913,0.16284480094909667,0.16113600134849548,0.00740355661356144,0.14521600306034088,0.20233599841594696,0.17807039842009545,0.1796799972653389,0.013908448560094403,0.0907519981265068,0.09750399738550186,0.09460479989647866,0.09478399902582169,0.0017965487667361475,,,,,,32,4,2048,768,151936,True,True,False,1024,2,2048,768,generic,False,,,CUDA_EVENT,BF16,none
,,,,,,,,,,0.11740799993276596,0.17958399653434753,0.13262080028653145,0.12878400087356567,0.014146060532423056,0.17468799650669098,0.21161599457263947,0.1910431995987892,0.18966399878263474,0.01270903647700971,0.05926400050520897,0.07577600330114365,0.06530559975653887,0.06404799968004227,0.00479458462514625,,,,,,32,4,2048,768,151936,True,True,False,512,2,2048,768,generic,False,,,CUDA_EVENT,BF16,none
,,,,,,,,,,0.11260800063610077,0.17209599912166595,0.13610880002379416,0.13809599727392197,0.017675922458993677,0.18729600310325623,0.26822400093078613,0.20815680101513861,0.2078079953789711,0.01700718593658771,0.04499199986457825,0.06537599861621857,0.0507551996037364,0.04787199944257736,0.005727123232692158,,,,,,32,4,2048,768,151936,True,True,False,256,2,2048,768,generic,False,,,CUDA_EVENT,BF16,none
,,,,,,,,,,0.11257600039243698,0.23452800512313843,0.13866880126297473,0.13964799791574478,0.026590047697062115,0.1828799992799759,0.24751999974250793,0.20735519900918006,0.20670399814844131,0.01500438131586336,0.03654399886727333,0.05593600124120712,0.0408239996060729,0.03892800025641918,0.004864409450710354,,,,,,32,4,2048,768,151936,True,True,False,128,2,2048,768,generic,False,,,CUDA_EVENT,BF16,none
,,,,,,,,,,0.11734399944543839,0.18726399540901184,0.13890240006148816,0.13308800011873245,0.019086167340006257,0.16991999745368958,0.2443840056657791,0.19185120090842248,0.19257599860429764,0.018017536159219673,0.03033600002527237,0.04956800118088722,0.038387199863791466,0.03750400058925152,0.00500897018702887,,,,,,32,4,2048,768,151936,True,True,False,64,2,2048,768,generic,False,,,CUDA_EVENT,BF16,none
,,,,,,,,,,0.11420799791812897,0.18115200102329254,0.13938880078494548,0.1393439993262291,0.018141313962922536,0.1693439930677414,0.221343994140625,0.19187839925289155,0.19438399374485016,0.01430752121272052,0.03017600066959858,0.06019200012087822,0.03866560012102127,0.0364960003644228,0.0074199790031205266,,,,,,32,4,2048,768,151936,True,True,False,32,2,2048,768,generic,False,,,CUDA_EVENT,BF16,none
,,,,,,,,,,0.11276800185441971,0.16223999857902527,0.1331360016018152,0.13305599987506866,0.01432493740801146,0.17187200486660004,0.23625600337982178,0.19287680014967917,0.1913280040025711,0.01754628831589704,0.029823999851942062,0.07574400305747986,0.03900959976017475,0.036927999928593636,0.009277465083962879,,,,,,32,4,2048,768,151936,True,True,False,16,2,2048,768,generic,False,,,CUDA_EVENT,BF16,none
,,,,,,,,,,0.1130559965968132,0.19814400374889374,0.13379519879817964,0.12531199678778648,0.020074427302248836,0.16841599345207214,0.21139200031757355,0.1893615983426571,0.19092799723148346,0.012418720676762711,0.029503999277949333,0.07558400183916092,0.040144000016152856,0.03728000074625015,0.010223239196498205,,,,,,32,4,2048,768,151936,True,True,False,8,2,2048,768,generic,False,,,CUDA_EVENT,BF16,none
,,,,,,,,,,0.115167997777462,0.2640640139579773,0.13805920109152794,0.13118399679660797,0.03132849683515479,0.17103999853134155,0.2977280020713806,0.19899839907884598,0.1976960003376007,0.028928046763067948,0.0297279991209507,0.05990400165319443,0.03834720011800528,0.0363520011305809,0.0070885330713495905,,,,,,32,4,2048,768,151936,True,True,False,1,2,2048,768,generic,False,,,CUDA_EVENT,BF16,none
,,,,,,,,,,0.3940800130367279,0.46483200788497925,0.39923040121793746,0.3957759886980057,0.015108903906233569,0.16022400557994843,0.1634880006313324,0.16203359961509706,0.16228799521923065,0.0010220720912414007,0.26073598861694336,0.2627840042114258,0.26138080209493636,0.2613760083913803,0.00041640363494172775,,,,,,32,4,2048,768,151936,True,True,False,8192,4,2048,768,generic,False,,,CUDA_EVENT,BF16,none
,,,,,,,,,,0.2536959946155548,0.27161601185798645,0.25960480123758317,0.2577280104160309,0.005142017404245015,0.09849599748849869,0.10281600058078766,0.10057279989123344,0.10063999891281128,0.0008836232237268523,0.13913600146770477,0.17606399953365326,0.15875840038061143,0.15988799929618835,0.010058952112389172,,,,,,32,4,2048,768,151936,True,True,False,4096,4,2048,768,generic,False,,,CUDA_EVENT,BF16,none
,,,,,,,,,,0.15142400562763214,0.1780479997396469,0.16139679849147798,0.1602879986166954,0.0070656055731385115,0.14601600170135498,0.23343999683856964,0.17913119941949845,0.179967999458313,0.023065274309176566,0.09388799965381622,0.12108799815177917,0.10317599996924401,0.10073599964380264,0.007364345618470178,,,,,,32,4,2048,768,151936,True,True,False,2048,4,2048,768,generic,False,,,CUDA_EVENT,BF16,none
,,,,,,,,,,0.12198399752378464,0.18406400084495544,0.13770400024950505,0.13232000172138214,0.01623164885687898,0.1773120015859604,0.20688000321388245,0.1870912007987499,0.18535999953746796,0.00825628058448234,0.05766399949789047,0.06739199906587601,0.06187200043350458,0.061216000467538834,0.003004312467037863,,,,,,32,4,2048,768,151936,True,True,False,1024,4,2048,768,generic,False,,,CUDA_EVENT,BF16,none
,,,,,,,,,,0.11737599968910217,0.18614399433135986,0.13109439946711063,0.12771200388669968,0.01372168725934724,0.17587199807167053,0.2699519991874695,0.19926720038056372,0.19075199961662292,0.02327478982490538,0.041728001087903976,0.06224000081419945,0.05000480003654957,0.049375999718904495,0.006185850944273961,,,,,,32,4,2048,768,151936,True,True,False,512,4,2048,768,generic,False,,,CUDA_EVENT,BF16,none
,,,,,,,,,,0.1159679964184761,0.1610880047082901,0.12988320142030715,0.12494400516152382,0.011865375783184065,0.1767680048942566,0.27379199862480164,0.21035519987344742,0.21275199949741364,0.023206964017550125,0.037087999284267426,0.062144000083208084,0.04439679980278015,0.04283200018107891,0.006155226392511492,,,,,,32,4,2048,768,151936,True,True,False,256,4,2048,768,generic,False,,,CUDA_EVENT,BF16,none
,,,,,,,,,,0.12015999853610992,0.158720001578331,0.13573280088603495,0.1343199983239174,0.010666830836014414,0.1737920045852661,0.22127999365329742,0.20261440128087999,0.20321600139141083,0.011153965849067301,0.03587200120091438,0.04944000020623207,0.038265600241720675,0.037328001111745834,0.0030431580113575636,,,,,,32,4,2048,768,151936,True,True,False,128,4,2048,768,generic,False,,,CUDA_EVENT,BF16,none
,,,,,,,,,,0.1212799996137619,0.15139199793338776,0.13183839991688728,0.13014400005340576,0.00851207547022104,0.17257599532604218,0.2250880002975464,0.1872655987739563,0.18193599581718445,0.013638284975248818,0.02969600073993206,0.0525440014898777,0.03840640028938651,0.03444799967110157,0.007598511754254174,,,,,,32,4,2048,768,151936,True,True,False,64,4,2048,768,generic,False,,,CUDA_EVENT,BF16,none
,,,,,,,,,,0.11270400136709213,0.1624639928340912,0.13508000001311302,0.13180799782276154,0.014243564451606775,0.17052799463272095,0.23715199530124664,0.1953311987221241,0.1966560035943985,0.01500670718978398,0.030239999294281006,0.05270399898290634,0.03807039987295866,0.03742399998009205,0.004906165602021684,,,,,,32,4,2048,768,151936,True,True,False,32,4,2048,768,generic,False,,,CUDA_EVENT,BF16,none
,,,,,,,,,,0.12319999933242798,0.17919999361038208,0.14059039913117885,0.14156799763441086,0.016217662982107223,0.1701119989156723,0.21987199783325195,0.18903039917349815,0.1870879977941513,0.014117854507841239,0.030368000268936157,0.06406400352716446,0.04045119984075427,0.03710399940609932,0.00878942205983628,,,,,,32,4,2048,768,151936,True,True,False,16,4,2048,768,generic,False,,,CUDA_EVENT,BF16,none
,,,,,,,,,,0.11296000331640244,0.16761599481105804,0.13843199908733367,0.13814399391412735,0.014925286935582404,0.1711679995059967,0.21561600267887115,0.1906527981162071,0.1876479983329773,0.011803992622137974,0.0306560005992651,0.09216000139713287,0.041129599791020155,0.036847999319434166,0.013739721468154687,,,,,,32,4,2048,768,151936,True,True,False,8,4,2048,768,generic,False,,,CUDA_EVENT,BF16,none
,,,,,,,,,,0.11929599940776825,0.2008959949016571,0.14535359852015972,0.13971199840307236,0.024769473061576282,0.16803200542926788,0.25123199820518494,0.19573760256171227,0.19366399943828583,0.023196011081705884,0.03049599938094616,0.08089599758386612,0.04739360017701984,0.0453919991850853,0.01286389846889463,,,,,,32,4,2048,768,151936,True,True,False,1,4,2048,768,generic,False,,,CUDA_EVENT,BF16,none
1 time_stats.emb.min time_stats.emb.max time_stats.emb.mean time_stats.emb.median time_stats.emb.std time_stats.input_layernorm.min time_stats.input_layernorm.max time_stats.input_layernorm.mean time_stats.input_layernorm.median time_stats.input_layernorm.std time_stats.attn_pre_proj.min time_stats.attn_pre_proj.max time_stats.attn_pre_proj.mean time_stats.attn_pre_proj.median time_stats.attn_pre_proj.std time_stats.attn_rope.min time_stats.attn_rope.max time_stats.attn_rope.mean time_stats.attn_rope.median time_stats.attn_rope.std time_stats.attn_post_proj.min time_stats.attn_post_proj.max time_stats.attn_post_proj.mean time_stats.attn_post_proj.median time_stats.attn_post_proj.std time_stats.post_attention_layernorm.min time_stats.post_attention_layernorm.max time_stats.post_attention_layernorm.mean time_stats.post_attention_layernorm.median time_stats.post_attention_layernorm.std n_head n_kv_head n_embd n_expanded_embd vocab_size use_gated_mlp use_qk_norm attn_output_gate num_tokens num_tensor_parallel_workers padded_n_embd padded_n_expanded_embd model_arch is_step2_mini share_expert_dim share_q_dim measurement_type profiling_precision quant_signature
2 0.029184000566601753 0.06780800223350525 0.03157280012965202 0.030736000277101994 0.005874173435341216 0.033215999603271484 0.04825599864125252 0.03443359974771738 0.0337119996547699 0.0031825678429048183 1.438431978225708 1.505568027496338 1.446228802204132 1.4429279565811157 0.014128607642643025 0.538752019405365 0.5440319776535034 0.5416463971138 0.5420799851417542 0.0016099306164432847 1.0073280334472656 1.0163840055465698 1.0098415970802308 1.0081279873847961 0.0032980457197329728 0.04022400081157684 0.041728001087903976 0.04091359991580248 0.04081599973142147 0.0004728505416767937 32 4 2048 768 151936 True True False 8192 1 2048 768 generic False CUDA_EVENT BF16 none
3 0.01360000018030405 0.06784000247716904 0.017755200061947106 0.0179840000346303 0.00837681421401443 0.01836800016462803 0.03651199862360954 0.019606399815529585 0.018719999119639397 0.0038821958848767424 0.7512000203132629 0.8208960294723511 0.7566704005002975 0.75382399559021 0.014793092586577971 0.28995200991630554 0.29337599873542786 0.29135999977588656 0.29150401055812836 0.000998381071258815 0.5149760246276855 0.5169600248336792 0.5160208016633987 0.5158880054950714 0.0005038652783291977 0.02051199972629547 0.021503999829292297 0.021067200042307378 0.021104000508785248 0.00023542855306902367 32 4 2048 768 151936 True True False 4096 1 2048 768 generic False CUDA_EVENT BF16 none
4 0.0080960001796484 0.04281599819660187 0.011092799971811474 0.011039999779313803 0.005489135752949302 0.012223999947309494 0.026335999369621277 0.013187199970707298 0.01247999956831336 0.0030236510562153375 0.3928639888763428 0.45372799038887024 0.3976895987987518 0.39528000354766846 0.012905176716136006 0.1547199934720993 0.15884800255298615 0.15712319910526276 0.1573439985513687 0.001165121945135037 0.26633599400520325 0.268095999956131 0.26719200164079665 0.2671840041875839 0.0004242740199415328 0.013024000450968742 0.013887999579310417 0.013489600038155913 0.013520000036805868 0.0002382817554057738 32 4 2048 768 151936 True True False 2048 1 2048 768 generic False CUDA_EVENT BF16 none
5 0.00825599953532219 0.04755200073122978 0.02398160002194345 0.01961600035429001 0.00874683840888523 0.018880000337958336 0.03868800029158592 0.021590400114655496 0.0208320003002882 0.004057015751746236 0.24316799640655518 0.2710399925708771 0.2521967992186546 0.25065599381923676 0.007998418528894075 0.09644799679517746 0.19120000302791595 0.10407840013504029 0.09963199868798256 0.020043722414992166 0.13600000739097595 0.18892799317836761 0.15760480016469955 0.15760000050067902 0.0112223677907762 0.009664000011980534 0.010015999898314476 0.009836799977347255 0.009824000298976898 9.016971952948177e-05 32 4 2048 768 151936 True True False 1024 1 2048 768 generic False CUDA_EVENT BF16 none
6 0.017376000061631203 0.044704001396894455 0.026318399980664254 0.027312000282108784 0.007565344034680866 0.01836800016462803 0.03014400042593479 0.021276800055056812 0.020655999891459942 0.002632977855097951 0.1438719928264618 0.17132799327373505 0.152497598528862 0.15012799948453903 0.007947150319625347 0.1430719941854477 0.19305600225925446 0.16630879789590836 0.1685439944267273 0.014628024163894684 0.08899199962615967 0.10467199981212616 0.0943599995225668 0.09374399855732918 0.003472669494074113 0.007615999784320593 0.007935999892652035 0.007769599952735007 0.0077760000713169575 7.680004540222077e-05 32 4 2048 768 151936 True True False 512 1 2048 768 generic False CUDA_EVENT BF16 none
7 0.016992000862956047 0.0544000007212162 0.02573199989274144 0.026016000658273697 0.00803323401720434 0.018432000651955605 0.02348800003528595 0.020648000109940768 0.02062400057911873 0.0013939985047930988 0.10220800340175629 0.1361600011587143 0.11850560046732425 0.11684799939393997 0.010637754898360304 0.16710400581359863 0.21110400557518005 0.19078560024499894 0.19409599900245667 0.013714352646558832 0.06265600025653839 0.0740479975938797 0.06842879951000214 0.0690080001950264 0.0031292240446560557 0.00979200005531311 0.033440001308918 0.01864320016466081 0.017280000261962414 0.006957998188876383 32 4 2048 768 151936 True True False 256 1 2048 768 generic False CUDA_EVENT BF16 none
8 0.017152000218629837 0.04396799951791763 0.025907999789342284 0.02598400041460991 0.007714554737915267 0.018400000408291817 0.03667199984192848 0.024132800102233887 0.02112000063061714 0.006120348733354326 0.10678400099277496 0.1363839954137802 0.1193264003843069 0.11583999916911125 0.009838647443214228 0.17017599940299988 0.22748799622058868 0.18853759989142418 0.18433599919080734 0.015728078443174653 0.04569600149989128 0.06652799993753433 0.05192639995366335 0.05151999928057194 0.004516605694976449 0.021856000646948814 0.026335999369621277 0.02384479995816946 0.023599999956786633 0.0011301153013314744 32 4 2048 768 151936 True True False 128 1 2048 768 generic False CUDA_EVENT BF16 none
9 0.017343999817967415 1.0683200359344482 0.05748240072280168 0.029504000209271908 0.16366824814254827 0.018688000738620758 0.3317759931087494 0.03685439983382821 0.021151999942958355 0.06767422466731164 0.10255999863147736 0.9434880018234253 0.16412640027701855 0.11956800147891045 0.17964606281728834 0.1714559942483902 2.1306240558624268 0.3011296011507511 0.1926399990916252 0.42310127734378766 0.03574400022625923 0.6859520077705383 0.08389280084520578 0.04279999993741512 0.14321647071615612 0.020479999482631683 0.1831360012292862 0.033024000097066165 0.023856000043451786 0.03470987082429836 32 4 2048 768 151936 True True False 64 1 2048 768 generic False CUDA_EVENT BF16 none
10 0.016095999628305435 0.05142400041222572 0.026363200135529043 0.02676799986511469 0.008637725852473854 0.01849599927663803 0.03577600046992302 0.022193600237369538 0.020848000422120094 0.004615266181181815 0.10540799796581268 0.15014399588108063 0.12211520001292228 0.11896000057458878 0.012772013396624768 0.17315199971199036 0.21478399634361267 0.1881632000207901 0.1873439997434616 0.011761164657572015 0.03481600061058998 0.058079998940229416 0.042200000025331974 0.03969600051641464 0.006461281521769989 0.02143999934196472 0.038816001266241074 0.02466559996828437 0.023856000043451786 0.0035418033748569927 32 4 2048 768 151936 True True False 32 1 2048 768 generic False CUDA_EVENT BF16 none
11 0.015904000028967857 0.03753599897027016 0.023127999808639287 0.02527999971061945 0.0054230027890305385 0.018400000408291817 0.03001599945127964 0.020488000102341176 0.020096000283956528 0.002585688394137697 0.10355199873447418 0.1438400000333786 0.11536479964852334 0.11124800145626068 0.011136028294919255 0.16502399742603302 0.2072959989309311 0.18646399974822997 0.19075199961662292 0.013296437761247597 0.03142400085926056 0.05215999856591225 0.03888959977775812 0.03750399872660637 0.0057399302067536314 0.020128000527620316 0.04064000025391579 0.023937600292265417 0.023648000322282314 0.004144197400898248 32 4 2048 768 151936 True True False 16 1 2048 768 generic False CUDA_EVENT BF16 none
12 0.015200000256299973 0.037151999771595 0.02144480012357235 0.02195199951529503 0.005576364091933697 0.017952000722289085 0.0226879995316267 0.019934400077909233 0.019952000118792057 0.0011812499470458758 0.10063999891281128 0.13468800485134125 0.11595199964940547 0.1207519993185997 0.011974301621092394 0.16332800686359406 0.20748800039291382 0.18162400051951408 0.1767839938402176 0.01474120743720334 0.03222399950027466 0.043455999344587326 0.03829439990222454 0.03859200142323971 0.0031378131240041122 0.020959999412298203 0.03587200120091438 0.023795200139284135 0.023423999547958374 0.0031339489695198443 32 4 2048 768 151936 True True False 8 1 2048 768 generic False CUDA_EVENT BF16 none
13 0.014944000169634819 0.04022400081157684 0.02162720002233982 0.022463999688625336 0.006017219317026815 0.018464000895619392 0.026528000831604004 0.021264000236988066 0.020896000787615776 0.001934095017586082 0.10156799852848053 0.1703999936580658 0.12565439902245998 0.1244799979031086 0.01641776722785409 0.1653759926557541 0.23865599930286407 0.1969360001385212 0.19223999977111816 0.02058903050274278 0.03254399821162224 0.06752000004053116 0.0443536002188921 0.041519999504089355 0.00984829071098989 0.020096000283956528 0.040031999349594116 0.026934400014579297 0.02478400059044361 0.00562071702648593 32 4 2048 768 151936 True True False 1 1 2048 768 generic False CUDA_EVENT BF16 none
14 0.7531200051307678 0.8461440205574036 0.7618160009384155 0.7576479911804199 0.019464233617982506 0.2922559976577759 0.2985599935054779 0.2953856036067009 0.29576000571250916 0.001575901077689139 0.510047972202301 0.5140479803085327 0.5121696025133133 0.5123839974403381 0.0012263137313476844 32 4 2048 768 151936 True True False 8192 2 2048 768 generic False CUDA_EVENT BF16 none
15 0.39529600739479065 0.47279998660087585 0.40216960161924364 0.39825600385665894 0.0162749796240819 0.15625600516796112 0.16211199760437012 0.15959519892930984 0.15988799929618835 0.0011493418942396922 0.2635200023651123 0.2642880082130432 0.2639120012521744 0.26392000913619995 0.0001903593401384135 32 4 2048 768 151936 True True False 4096 2 2048 768 generic False CUDA_EVENT BF16 none
16 0.2531839907169342 0.32419198751449585 0.26924319565296173 0.26049599051475525 0.01657194262368116 0.10051199793815613 0.2375359982252121 0.12411200068891048 0.10311999917030334 0.039466165428540506 0.15881599485874176 0.19289599359035492 0.16896959990262986 0.1640480011701584 0.009162416086418127 32 4 2048 768 151936 True True False 2048 2 2048 768 generic False CUDA_EVENT BF16 none
17 0.15302400290966034 0.1780800074338913 0.16284480094909667 0.16113600134849548 0.00740355661356144 0.14521600306034088 0.20233599841594696 0.17807039842009545 0.1796799972653389 0.013908448560094403 0.0907519981265068 0.09750399738550186 0.09460479989647866 0.09478399902582169 0.0017965487667361475 32 4 2048 768 151936 True True False 1024 2 2048 768 generic False CUDA_EVENT BF16 none
18 0.11740799993276596 0.17958399653434753 0.13262080028653145 0.12878400087356567 0.014146060532423056 0.17468799650669098 0.21161599457263947 0.1910431995987892 0.18966399878263474 0.01270903647700971 0.05926400050520897 0.07577600330114365 0.06530559975653887 0.06404799968004227 0.00479458462514625 32 4 2048 768 151936 True True False 512 2 2048 768 generic False CUDA_EVENT BF16 none
19 0.11260800063610077 0.17209599912166595 0.13610880002379416 0.13809599727392197 0.017675922458993677 0.18729600310325623 0.26822400093078613 0.20815680101513861 0.2078079953789711 0.01700718593658771 0.04499199986457825 0.06537599861621857 0.0507551996037364 0.04787199944257736 0.005727123232692158 32 4 2048 768 151936 True True False 256 2 2048 768 generic False CUDA_EVENT BF16 none
20 0.11257600039243698 0.23452800512313843 0.13866880126297473 0.13964799791574478 0.026590047697062115 0.1828799992799759 0.24751999974250793 0.20735519900918006 0.20670399814844131 0.01500438131586336 0.03654399886727333 0.05593600124120712 0.0408239996060729 0.03892800025641918 0.004864409450710354 32 4 2048 768 151936 True True False 128 2 2048 768 generic False CUDA_EVENT BF16 none
21 0.11734399944543839 0.18726399540901184 0.13890240006148816 0.13308800011873245 0.019086167340006257 0.16991999745368958 0.2443840056657791 0.19185120090842248 0.19257599860429764 0.018017536159219673 0.03033600002527237 0.04956800118088722 0.038387199863791466 0.03750400058925152 0.00500897018702887 32 4 2048 768 151936 True True False 64 2 2048 768 generic False CUDA_EVENT BF16 none
22 0.11420799791812897 0.18115200102329254 0.13938880078494548 0.1393439993262291 0.018141313962922536 0.1693439930677414 0.221343994140625 0.19187839925289155 0.19438399374485016 0.01430752121272052 0.03017600066959858 0.06019200012087822 0.03866560012102127 0.0364960003644228 0.0074199790031205266 32 4 2048 768 151936 True True False 32 2 2048 768 generic False CUDA_EVENT BF16 none
23 0.11276800185441971 0.16223999857902527 0.1331360016018152 0.13305599987506866 0.01432493740801146 0.17187200486660004 0.23625600337982178 0.19287680014967917 0.1913280040025711 0.01754628831589704 0.029823999851942062 0.07574400305747986 0.03900959976017475 0.036927999928593636 0.009277465083962879 32 4 2048 768 151936 True True False 16 2 2048 768 generic False CUDA_EVENT BF16 none
24 0.1130559965968132 0.19814400374889374 0.13379519879817964 0.12531199678778648 0.020074427302248836 0.16841599345207214 0.21139200031757355 0.1893615983426571 0.19092799723148346 0.012418720676762711 0.029503999277949333 0.07558400183916092 0.040144000016152856 0.03728000074625015 0.010223239196498205 32 4 2048 768 151936 True True False 8 2 2048 768 generic False CUDA_EVENT BF16 none
25 0.115167997777462 0.2640640139579773 0.13805920109152794 0.13118399679660797 0.03132849683515479 0.17103999853134155 0.2977280020713806 0.19899839907884598 0.1976960003376007 0.028928046763067948 0.0297279991209507 0.05990400165319443 0.03834720011800528 0.0363520011305809 0.0070885330713495905 32 4 2048 768 151936 True True False 1 2 2048 768 generic False CUDA_EVENT BF16 none
26 0.3940800130367279 0.46483200788497925 0.39923040121793746 0.3957759886980057 0.015108903906233569 0.16022400557994843 0.1634880006313324 0.16203359961509706 0.16228799521923065 0.0010220720912414007 0.26073598861694336 0.2627840042114258 0.26138080209493636 0.2613760083913803 0.00041640363494172775 32 4 2048 768 151936 True True False 8192 4 2048 768 generic False CUDA_EVENT BF16 none
27 0.2536959946155548 0.27161601185798645 0.25960480123758317 0.2577280104160309 0.005142017404245015 0.09849599748849869 0.10281600058078766 0.10057279989123344 0.10063999891281128 0.0008836232237268523 0.13913600146770477 0.17606399953365326 0.15875840038061143 0.15988799929618835 0.010058952112389172 32 4 2048 768 151936 True True False 4096 4 2048 768 generic False CUDA_EVENT BF16 none
28 0.15142400562763214 0.1780479997396469 0.16139679849147798 0.1602879986166954 0.0070656055731385115 0.14601600170135498 0.23343999683856964 0.17913119941949845 0.179967999458313 0.023065274309176566 0.09388799965381622 0.12108799815177917 0.10317599996924401 0.10073599964380264 0.007364345618470178 32 4 2048 768 151936 True True False 2048 4 2048 768 generic False CUDA_EVENT BF16 none
29 0.12198399752378464 0.18406400084495544 0.13770400024950505 0.13232000172138214 0.01623164885687898 0.1773120015859604 0.20688000321388245 0.1870912007987499 0.18535999953746796 0.00825628058448234 0.05766399949789047 0.06739199906587601 0.06187200043350458 0.061216000467538834 0.003004312467037863 32 4 2048 768 151936 True True False 1024 4 2048 768 generic False CUDA_EVENT BF16 none
30 0.11737599968910217 0.18614399433135986 0.13109439946711063 0.12771200388669968 0.01372168725934724 0.17587199807167053 0.2699519991874695 0.19926720038056372 0.19075199961662292 0.02327478982490538 0.041728001087903976 0.06224000081419945 0.05000480003654957 0.049375999718904495 0.006185850944273961 32 4 2048 768 151936 True True False 512 4 2048 768 generic False CUDA_EVENT BF16 none
31 0.1159679964184761 0.1610880047082901 0.12988320142030715 0.12494400516152382 0.011865375783184065 0.1767680048942566 0.27379199862480164 0.21035519987344742 0.21275199949741364 0.023206964017550125 0.037087999284267426 0.062144000083208084 0.04439679980278015 0.04283200018107891 0.006155226392511492 32 4 2048 768 151936 True True False 256 4 2048 768 generic False CUDA_EVENT BF16 none
32 0.12015999853610992 0.158720001578331 0.13573280088603495 0.1343199983239174 0.010666830836014414 0.1737920045852661 0.22127999365329742 0.20261440128087999 0.20321600139141083 0.011153965849067301 0.03587200120091438 0.04944000020623207 0.038265600241720675 0.037328001111745834 0.0030431580113575636 32 4 2048 768 151936 True True False 128 4 2048 768 generic False CUDA_EVENT BF16 none
33 0.1212799996137619 0.15139199793338776 0.13183839991688728 0.13014400005340576 0.00851207547022104 0.17257599532604218 0.2250880002975464 0.1872655987739563 0.18193599581718445 0.013638284975248818 0.02969600073993206 0.0525440014898777 0.03840640028938651 0.03444799967110157 0.007598511754254174 32 4 2048 768 151936 True True False 64 4 2048 768 generic False CUDA_EVENT BF16 none
34 0.11270400136709213 0.1624639928340912 0.13508000001311302 0.13180799782276154 0.014243564451606775 0.17052799463272095 0.23715199530124664 0.1953311987221241 0.1966560035943985 0.01500670718978398 0.030239999294281006 0.05270399898290634 0.03807039987295866 0.03742399998009205 0.004906165602021684 32 4 2048 768 151936 True True False 32 4 2048 768 generic False CUDA_EVENT BF16 none
35 0.12319999933242798 0.17919999361038208 0.14059039913117885 0.14156799763441086 0.016217662982107223 0.1701119989156723 0.21987199783325195 0.18903039917349815 0.1870879977941513 0.014117854507841239 0.030368000268936157 0.06406400352716446 0.04045119984075427 0.03710399940609932 0.00878942205983628 32 4 2048 768 151936 True True False 16 4 2048 768 generic False CUDA_EVENT BF16 none
36 0.11296000331640244 0.16761599481105804 0.13843199908733367 0.13814399391412735 0.014925286935582404 0.1711679995059967 0.21561600267887115 0.1906527981162071 0.1876479983329773 0.011803992622137974 0.0306560005992651 0.09216000139713287 0.041129599791020155 0.036847999319434166 0.013739721468154687 32 4 2048 768 151936 True True False 8 4 2048 768 generic False CUDA_EVENT BF16 none
37 0.11929599940776825 0.2008959949016571 0.14535359852015972 0.13971199840307236 0.024769473061576282 0.16803200542926788 0.25123199820518494 0.19573760256171227 0.19366399943828583 0.023196011081705884 0.03049599938094616 0.08089599758386612 0.04739360017701984 0.0453919991850853 0.01286389846889463 32 4 2048 768 151936 True True False 1 4 2048 768 generic False CUDA_EVENT BF16 none

53
runs/frontier-qwen30-vllm020-profile-v1/frozen/profile-v1/manifest.json Normal file
View File

@@ -0,0 +1,53 @@
{
"attention_tp_coverage": [
1,
2,
4
],
"environment_contract": {
"dtype": "bfloat16",
"frontier_commit": "d9cfeb6d8791fbf2f295dd9744c56a666171776e",
"hardware": "NVIDIA H20",
"model": "Qwen3-30B-A3B",
"tensor_parallel_sizes": [
1,
2,
4
],
"vllm_source_commit": "88d34c6409e9fb3c7b8ca0c04756f061d2099eb1",
"vllm_version": "0.20.0"
},
"inputs": {
"/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-allreduce-full-tp2-20260716-v1-dispatch-aware-20260716T140743025781Z/artifacts/artifacts/allreduce-full-tp2-v1/raw/allreduce-tp2.json": "97c3c76b5a04e95bd9192423c2b891667c668f39cc0dfecbd097d749939f2d0a",
"/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-allreduce-full-tp4-20260716-v1-dispatch-aware-20260716T141106009788Z/artifacts/artifacts/allreduce-full-tp4-v1/raw/allreduce-tp4.json": "809df9baa6f468cf12bf0c99827475acc67894dd9f3f948976590b665fac0e76",
"/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-flashattn-kv-full-tp1-20260716-v2-20260716T135132587012Z/artifacts/artifacts/flashattn-kv-full-v2-tp1/raw/flashattn-tp1.json": "dcb4c1bf7e76b9c765f78ddd2b8a734f2d7ba2adac13ce017689a8a77fe69a27",
"/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-flashattn-kv-full-tp2-20260716-v2-20260716T135134194295Z/artifacts/artifacts/flashattn-kv-full-v2-tp2/raw/flashattn-tp2.json": "43ce042556ba887c8860614b43ccf0f564e5cebc1a0cffbce299d0acb9fa8d07",
"/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-flashattn-kv-full-tp4-20260716-v2-20260716T135135197200Z/artifacts/artifacts/flashattn-kv-full-v2-tp4/raw/flashattn-tp4.json": "84eef31bcad0f556907a093318a420959d14fdc94474823d11f659704bdfec73",
"/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-frontier-linear-full-20260716-v2-max-tokens-20260716T144444676943Z/artifacts/artifacts/frontier-linear-full-v2/profiles/compute/h20/qwen3-a3b-30b-moe/linear_op.csv": "67666cb0a4901b74599d468df2e31bcaa2a11a7842cc0cefba24ffce62508e0c",
"/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-moe-full-20260716-v1-local-shard-20260716T141334565164Z/artifacts/artifacts/moe-full-v1/raw/moe-full.json": "588f6ad0d69c9636d1b852e3df0a12d13cfe731f050ea7ec7aea457cceefbde8",
"/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-router-full-20260716-v3-tp-context-20260716T145446098505Z/artifacts/artifacts/router-full-v3/raw/router.json": "1962972e983bff3e06a721ef4ae4ec65728ff669681497a4a7e7f769b88b4931"
},
"outputs": {
"allreduce.json": "b38d14f990578d668523d25b107aceed433da5020d8ada3b6e44d3562261a3b3",
"attention.csv": "674e010598eea4c06bdee04ccba413f4c34194fbe2d70b143c6ec2169b638cc5",
"attention_true_mixed_fused.csv": "92cd22f8b669abc7af488cf3d067ed880a6cced2dba1a3213a93a4f02a0001a8",
"linear_op.csv": "67666cb0a4901b74599d468df2e31bcaa2a11a7842cc0cefba24ffce62508e0c",
"moe.csv": "0e4dcba72918a1c4cf4e96ced31ee3829248a19ad54553cebef14417725808b0"
},
"profile_id": "qwen3-30b-a3b-bf16-vllm020-h20-tp1-2-4",
"projection_contract": {
"allreduce": "Frozen exact runtime measurements; base profile-only comparison keeps the historical Frontier CC backend fixed to isolate compute profile fidelity",
"attention": "Pure prefill/extend/decode FA3 core plus separately measured KV update; input/output reshape assumed zero; exported mean is used as median target",
"attention_true_mixed": "Preserved as fused-total diagnostics and excluded from attention.csv because Frontier requires an unobservable prefill/decode split",
"linear": "Frontier profiler using vLLM 0.20 CUDA operators",
"moe": "Replicated gate and fused top-k plus TP-local modular expert kernel; expert measurement already includes prepare/finalize so shuffling is zero"
},
"row_counts": {
"allreduce": 24,
"attention_frontier_compatible": 102,
"attention_true_mixed_fused_diagnostic": 30,
"linear": 36,
"moe": 72
},
"schema_version": "frontier_qwen30_vllm020_frozen_profile.v1"
}

73
runs/frontier-qwen30-vllm020-profile-v1/frozen/profile-v1/moe.csv Normal file
View File

@@ -0,0 +1,73 @@
time_stats.moe_gating_linear.min,time_stats.moe_gating_linear.max,time_stats.moe_gating_linear.mean,time_stats.moe_gating_linear.median,time_stats.moe_gating_linear.std,time_stats.moe_gating_routing_topk.min,time_stats.moe_gating_routing_topk.max,time_stats.moe_gating_routing_topk.mean,time_stats.moe_gating_routing_topk.median,time_stats.moe_gating_routing_topk.std,time_stats.moe_shuffling.min,time_stats.moe_shuffling.max,time_stats.moe_shuffling.mean,time_stats.moe_shuffling.median,time_stats.moe_shuffling.std,time_stats.moe_grouped_gemm.min,time_stats.moe_grouped_gemm.max,time_stats.moe_grouped_gemm.mean,time_stats.moe_grouped_gemm.median,time_stats.moe_grouped_gemm.std,num_tokens,num_experts,num_experts_per_device,expert_parallel_size,routing_runtime_path,routing_assignment_policy,routing_weight_policy,routing_uses_router_logits,gating_runtime_context,gating_runtime_context_impl,router_topk,hidden_dim,expert_hidden_dim,use_gated,num_tensor_parallel_workers,total_routed_tokens,model_expansion_ratio,tokens_per_expert_avg,tokens_to_experts_ratio,expert_utilization,min_load_ratio,load_imbalance_cv,max_load_ratio,load_entropy,load_gini_coefficient,load_distribution,seed,moe_grouped_gemm_backend,measurement_type,profiling_precision,model_arch,quant_signature,router_median_nonadditivity_ratio,projection_policy
0.02502400055527687,0.06092799827456474,0.033839999698102474,0.028672000393271446,0.010315255343709818,0.019360000267624855,0.0352960005402565,0.023424000293016434,0.022064000368118286,0.0038898102289194572,0.0,0.0,0.0,0.0,0.0,0.33926400542259216,0.405023992061615,0.36780479848384856,0.36507199704647064,0.01690507644474779,1,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,8,0.375,0.0625,0.0625,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0233364439829928,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.025280000641942024,0.05132799968123436,0.030641599837690593,0.027343999594449997,0.007562409232763304,0.020160000771284103,0.052960000932216644,0.024145600199699403,0.021424000151455402,0.007450198645599985,0.0,0.0,0.0,0.0,0.0,1.1943039894104004,1.286784052848816,1.228384006023407,1.2273280024528503,0.02832547242381263,8,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,64,0.375,0.5,0.5,0.3984375,0.0,1.346291201783626,4.0,5.59375,0.661865234375,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9806430689981738,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.024831999093294144,0.046560000628232956,0.030371200107038022,0.02763199992477894,0.005878487205028602,0.020320000126957893,0.04560000076889992,0.02601920012384653,0.02270400058478117,0.00751129965906799,0.0,0.0,0.0,0.0,0.0,1.679744005203247,1.766144037246704,1.7095808148384095,1.7015680074691772,0.02438921262998535,16,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,128,0.375,1.0,1.0,0.625,0.0,1.015504800579495,5.0,6.15516433212955,0.529052734375,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9103623678483975,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.024288000538945198,0.049375999718904495,0.03086080001667142,0.0267359996214509,0.0070864531384997225,0.020479999482631683,0.030912000685930252,0.02274719988927245,0.021359999664127827,0.0029966813650672505,0.0,0.0,0.0,0.0,0.0,2.1576640605926514,2.2921600341796875,2.2097824096679686,2.188944101333618,0.045572321842012986,32,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,256,0.375,2.0,2.0,0.875,0.0,0.6343057228182637,2.5,6.64370748444639,0.35369873046875,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9610778571819444,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.02393599972128868,0.04342399910092354,0.029380799923092126,0.026688000187277794,0.0057374782000526574,0.020031999796628952,0.036607999354600906,0.022487999964505435,0.020911999978125095,0.0038135203062103235,0.0,0.0,0.0,0.0,0.0,2.422368049621582,2.5130879878997803,2.4516672134399413,2.434159994125366,0.03278900287381846,64,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,512,0.375,4.0,4.0,0.984375,0.0,0.4921254921257382,2.25,6.817190042344769,0.272369384765625,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9952941013961014,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.025407999753952026,0.05510399863123894,0.031430399790406224,0.02798399981111288,0.007335050273982725,0.020479999482631683,0.03561599925160408,0.02275839988142252,0.021551999263465405,0.003545718365165811,0.0,0.0,0.0,0.0,0.0,2.2217600345611572,2.289599895477295,2.2571327924728393,2.263375997543335,0.021660416089449488,128,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,1024,0.375,8.0,8.0,1.0,0.125,0.3486861500690843,1.875,6.908192310183997,0.197662353515625,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9273256282883522,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.0244159996509552,0.05395200103521347,0.03188959984108806,0.026031999848783016,0.00943995927387483,0.02051199972629547,0.036607999354600906,0.023247999791055917,0.02147199958562851,0.003910623837069648,0.0,0.0,0.0,0.0,0.0,2.18668794631958,2.318079948425293,2.2218016147613526,2.211087942123413,0.035380897213135316,256,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,2048,0.375,16.0,16.0,1.0,0.4375,0.2525504668006971,1.875,6.953347743053017,0.1410369873046875,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0380599882396606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.024512000381946564,0.04620800167322159,0.02884640023112297,0.026320000179111958,0.005516557583355925,0.02054399996995926,0.03846399858593941,0.023401600029319524,0.021263999864459038,0.0044742944198265374,0.0,0.0,0.0,0.0,0.0,2.2291839122772217,2.3929600715637207,2.2908096313476562,2.2804640531539917,0.04479348924786221,512,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,4096,0.375,32.0,32.0,1.0,0.65625,0.15765965680164504,1.5625,6.98229848728205,0.08779525756835938,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9569603278386984,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.02412799932062626,0.06940799951553345,0.030817600432783365,0.026800000108778477,0.010047085187652939,0.020128000527620316,0.044096000492572784,0.024606400076299904,0.022304000332951546,0.00622857166823714,0.0,0.0,0.0,0.0,0.0,2.0678720474243164,2.1297600269317627,2.0837119817733765,2.0779199600219727,0.017880044357986735,1024,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,8192,0.375,64.0,64.0,1.0,0.625,0.12169081635504074,1.3125,6.9892029662356325,0.06879425048828125,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9524274993623046,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.02831999957561493,0.043487999588251114,0.031744000129401685,0.02991999965161085,0.003973415836428909,0.02070399932563305,0.029343999922275543,0.022886400017887353,0.021743999794125557,0.0026873065045088873,0.0,0.0,0.0,0.0,0.0,2.916032075881958,3.0819520950317383,2.9805248022079467,2.9656319618225098,0.05482195799019572,2048,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,16384,0.375,128.0,128.0,1.0,0.796875,0.07935434147688751,1.1796875,6.9954297964750305,0.044734954833984375,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8971818172100244,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.03830400109291077,0.06268800050020218,0.043337599746882914,0.040511999279260635,0.005823016247946116,0.023135999217629433,0.03747199848294258,0.025206399988383053,0.02393599972128868,0.003271381256231161,0.0,0.0,0.0,0.0,0.0,4.421599864959717,4.535359859466553,4.486294317245483,4.497056007385254,0.036990243787549344,4096,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,32768,0.375,256.0,256.0,1.0,0.8203125,0.060849326483103046,1.17578125,6.9973188375859685,0.033740997314453125,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8113207890716184,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.05660799890756607,0.07932800054550171,0.06249920018017292,0.06039999984204769,0.005636461691480845,0.02956799976527691,0.03747199848294258,0.031126399897038935,0.030287999659776688,0.002157571006631541,0.0,0.0,0.0,0.0,0.0,7.302591800689697,7.402751922607422,7.354758310317993,7.3464319705963135,0.032142662400335566,8192,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,65536,0.375,512.0,512.0,1.0,0.890625,0.0412323087266341,1.08984375,6.998772433185578,0.02334284782409668,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.883909666885606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.02502400055527687,0.06092799827456474,0.033839999698102474,0.028672000393271446,0.010315255343709818,0.019360000267624855,0.0352960005402565,0.023424000293016434,0.022064000368118286,0.0038898102289194572,0.0,0.0,0.0,0.0,0.0,0.35280001163482666,0.39692801237106323,0.37662720382213594,0.37196800112724304,0.013401318050665304,1,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,8,0.375,0.0625,0.0625,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0233364439829928,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.025280000641942024,0.05132799968123436,0.030641599837690593,0.027343999594449997,0.007562409232763304,0.020160000771284103,0.052960000932216644,0.024145600199699403,0.021424000151455402,0.007450198645599985,0.0,0.0,0.0,0.0,0.0,0.4692479968070984,0.5523840188980103,0.5134752035140991,0.5100640058517456,0.02291433464135784,8,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,64,0.375,0.5,0.5,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9806430689981738,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.024831999093294144,0.046560000628232956,0.030371200107038022,0.02763199992477894,0.005878487205028602,0.020320000126957893,0.04560000076889992,0.02601920012384653,0.02270400058478117,0.00751129965906799,0.0,0.0,0.0,0.0,0.0,0.34652799367904663,0.4119040071964264,0.3789471983909607,0.38550400733947754,0.02073945105803335,16,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,128,0.375,1.0,1.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9103623678483975,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.024288000538945198,0.049375999718904495,0.03086080001667142,0.0267359996214509,0.0070864531384997225,0.020479999482631683,0.030912000685930252,0.02274719988927245,0.021359999664127827,0.0029966813650672505,0.0,0.0,0.0,0.0,0.0,0.31462401151657104,0.7456960082054138,0.38617280423641204,0.34545600414276123,0.12230201266253077,32,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,256,0.375,2.0,2.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9610778571819444,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.02393599972128868,0.04342399910092354,0.029380799923092126,0.026688000187277794,0.0057374782000526574,0.020031999796628952,0.036607999354600906,0.022487999964505435,0.020911999978125095,0.0038135203062103235,0.0,0.0,0.0,0.0,0.0,0.32521599531173706,0.419871985912323,0.36325119733810424,0.34968000650405884,0.03161798848223672,64,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,512,0.375,4.0,4.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9952941013961014,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.025407999753952026,0.05510399863123894,0.031430399790406224,0.02798399981111288,0.007335050273982725,0.020479999482631683,0.03561599925160408,0.02275839988142252,0.021551999263465405,0.003545718365165811,0.0,0.0,0.0,0.0,0.0,0.289792001247406,0.4663360118865967,0.4091839998960495,0.41655999422073364,0.0446001986506615,128,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,1024,0.375,8.0,8.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9273256282883522,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.0244159996509552,0.05395200103521347,0.03188959984108806,0.026031999848783016,0.00943995927387483,0.02051199972629547,0.036607999354600906,0.023247999791055917,0.02147199958562851,0.003910623837069648,0.0,0.0,0.0,0.0,0.0,0.3761279881000519,0.4416320025920868,0.40686399936676027,0.40540799498558044,0.02257778769899645,256,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,2048,0.375,16.0,16.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0380599882396606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.024512000381946564,0.04620800167322159,0.02884640023112297,0.026320000179111958,0.005516557583355925,0.02054399996995926,0.03846399858593941,0.023401600029319524,0.021263999864459038,0.0044742944198265374,0.0,0.0,0.0,0.0,0.0,0.7172480225563049,0.8663039803504944,0.7723807990550995,0.7591840028762817,0.04164772379451242,512,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,4096,0.375,32.0,32.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9569603278386984,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.02412799932062626,0.06940799951553345,0.030817600432783365,0.026800000108778477,0.010047085187652939,0.020128000527620316,0.044096000492572784,0.024606400076299904,0.022304000332951546,0.00622857166823714,0.0,0.0,0.0,0.0,0.0,1.0195519924163818,1.2216639518737793,1.1253888130187988,1.1453600525856018,0.06548005322243594,1024,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,8192,0.375,64.0,64.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9524274993623046,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.02831999957561493,0.043487999588251114,0.031744000129401685,0.02991999965161085,0.003973415836428909,0.02070399932563305,0.029343999922275543,0.022886400017887353,0.021743999794125557,0.0026873065045088873,0.0,0.0,0.0,0.0,0.0,1.7490559816360474,1.9644800424575806,1.8529024004936219,1.814303994178772,0.08042565617327288,2048,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,16384,0.375,128.0,128.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8971818172100244,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.03830400109291077,0.06268800050020218,0.043337599746882914,0.040511999279260635,0.005823016247946116,0.023135999217629433,0.03747199848294258,0.025206399988383053,0.02393599972128868,0.003271381256231161,0.0,0.0,0.0,0.0,0.0,3.2479360103607178,3.385279893875122,3.296070408821106,3.2800960540771484,0.04525529026442046,4096,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,32768,0.375,256.0,256.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8113207890716184,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.05660799890756607,0.07932800054550171,0.06249920018017292,0.06039999984204769,0.005636461691480845,0.02956799976527691,0.03747199848294258,0.031126399897038935,0.030287999659776688,0.002157571006631541,0.0,0.0,0.0,0.0,0.0,6.344799995422363,6.517856121063232,6.464438438415527,6.478623867034912,0.05116674443145098,8192,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,65536,0.375,512.0,512.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.883909666885606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.02502400055527687,0.06092799827456474,0.033839999698102474,0.028672000393271446,0.010315255343709818,0.019360000267624855,0.0352960005402565,0.023424000293016434,0.022064000368118286,0.0038898102289194572,0.0,0.0,0.0,0.0,0.0,0.2648000121116638,0.325439989566803,0.28852800130844114,0.28390398621559143,0.01933778377635077,1,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,8,0.375,0.0625,0.0625,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0233364439829928,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.025280000641942024,0.05132799968123436,0.030641599837690593,0.027343999594449997,0.007562409232763304,0.020160000771284103,0.052960000932216644,0.024145600199699403,0.021424000151455402,0.007450198645599985,0.0,0.0,0.0,0.0,0.0,0.7347840070724487,0.862496018409729,0.7769344031810761,0.769216001033783,0.03290485328796285,8,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,64,0.375,0.5,0.5,0.421875,0.0,1.346291201783626,6.0,5.652114648336087,0.636962890625,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9806430689981738,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.024831999093294144,0.046560000628232956,0.030371200107038022,0.02763199992477894,0.005878487205028602,0.020320000126957893,0.04560000076889992,0.02601920012384653,0.02270400058478117,0.00751129965906799,0.0,0.0,0.0,0.0,0.0,0.9198399782180786,0.9646080136299133,0.9412063956260681,0.9411839842796326,0.014939365085478117,16,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,128,0.375,1.0,1.0,0.5703125,0.0,1.118033988749895,5.0,6.008641773518898,0.580810546875,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9103623678483975,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.024288000538945198,0.049375999718904495,0.03086080001667142,0.0267359996214509,0.0070864531384997225,0.020479999482631683,0.030912000685930252,0.02274719988927245,0.021359999664127827,0.0029966813650672505,0.0,0.0,0.0,0.0,0.0,1.2796800136566162,1.3484159708023071,1.3006976008415223,1.2929120063781738,0.020807998177176254,32,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,256,0.375,2.0,2.0,0.8828125,0.0,0.6959705453537527,3.0,6.60872850615583,0.38055419921875,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9610778571819444,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.02393599972128868,0.04342399910092354,0.029380799923092126,0.026688000187277794,0.0057374782000526574,0.020031999796628952,0.036607999354600906,0.022487999964505435,0.020911999978125095,0.0038135203062103235,0.0,0.0,0.0,0.0,0.0,1.3630399703979492,1.4430400133132935,1.3909215927124023,1.3892319798469543,0.022335744492366926,64,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,512,0.375,4.0,4.0,0.984375,0.0,0.5201036555341637,3.0,6.798826509158851,0.28302001953125,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9952941013961014,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.025407999753952026,0.05510399863123894,0.031430399790406224,0.02798399981111288,0.007335050273982725,0.020479999482631683,0.03561599925160408,0.02275839988142252,0.021551999263465405,0.003545718365165811,0.0,0.0,0.0,0.0,0.0,1.27948796749115,1.3904000520706177,1.3176063895225525,1.309440016746521,0.038060887827312775,128,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,1024,0.375,8.0,8.0,1.0,0.25,0.3511282039725661,1.875,6.91002266305238,0.1970977783203125,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9273256282883522,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.0244159996509552,0.05395200103521347,0.03188959984108806,0.026031999848783016,0.00943995927387483,0.02051199972629547,0.036607999354600906,0.023247999791055917,0.02147199958562851,0.003910623837069648,0.0,0.0,0.0,0.0,0.0,1.264415979385376,1.3145920038223267,1.2791999936103822,1.2753440141677856,0.014130605249568332,256,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,2048,0.375,16.0,16.0,1.0,0.375,0.24692938483248605,1.6875,6.955481130775285,0.13909912109375,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0380599882396606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.024512000381946564,0.04620800167322159,0.02884640023112297,0.026320000179111958,0.005516557583355925,0.02054399996995926,0.03846399858593941,0.023401600029319524,0.021263999864459038,0.0044742944198265374,0.0,0.0,0.0,0.0,0.0,1.3081920146942139,1.347648024559021,1.3292255997657776,1.329967975616455,0.014558863679016933,512,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,4096,0.375,32.0,32.0,1.0,0.625,0.17143053326165383,1.5625,6.9786675275754035,0.09520339965820312,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9569603278386984,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.02412799932062626,0.06940799951553345,0.030817600432783365,0.026800000108778477,0.010047085187652939,0.020128000527620316,0.044096000492572784,0.024606400076299904,0.022304000332951546,0.00622857166823714,0.0,0.0,0.0,0.0,0.0,1.242751955986023,1.3112000226974487,1.2747935891151427,1.266207993030548,0.021093073517695057,1024,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,8192,0.375,64.0,64.0,1.0,0.78125,0.11000099875256815,1.296875,6.991308871213679,0.062183380126953125,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9524274993623046,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.02831999957561493,0.043487999588251114,0.031744000129401685,0.02991999965161085,0.003973415836428909,0.02070399932563305,0.029343999922275543,0.022886400017887353,0.021743999794125557,0.0026873065045088873,0.0,0.0,0.0,0.0,0.0,1.7388160228729248,1.8077759742736816,1.772764801979065,1.772704005241394,0.021056644077284283,2048,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,16384,0.375,128.0,128.0,1.0,0.78125,0.0864630150197678,1.1796875,6.994552526394139,0.048796653747558594,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8971818172100244,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.03830400109291077,0.06268800050020218,0.043337599746882914,0.040511999279260635,0.005823016247946116,0.023135999217629433,0.03747199848294258,0.025206399988383053,0.02393599972128868,0.003271381256231161,0.0,0.0,0.0,0.0,0.0,2.6563520431518555,2.7063679695129395,2.6785055875778196,2.6791679859161377,0.01639963463052944,4096,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,32768,0.375,256.0,256.0,1.0,0.8671875,0.06127686514721937,1.16015625,6.997291583027146,0.03497934341430664,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8113207890716184,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.05660799890756607,0.07932800054550171,0.06249920018017292,0.06039999984204769,0.005636461691480845,0.02956799976527691,0.03747199848294258,0.031126399897038935,0.030287999659776688,0.002157571006631541,0.0,0.0,0.0,0.0,0.0,4.386879920959473,4.452256202697754,4.4108480453491214,4.406303882598877,0.019768161791937636,8192,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,65536,0.375,512.0,512.0,1.0,0.884765625,0.041723768525324195,1.1171875,6.998746434318934,0.02298593521118164,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.883909666885606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.02502400055527687,0.06092799827456474,0.033839999698102474,0.028672000393271446,0.010315255343709818,0.019360000267624855,0.0352960005402565,0.023424000293016434,0.022064000368118286,0.0038898102289194572,0.0,0.0,0.0,0.0,0.0,0.24208000302314758,0.4028480052947998,0.3041536003351212,0.277103990316391,0.05660721484881584,1,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,8,0.375,0.0625,0.0625,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0233364439829928,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.025280000641942024,0.05132799968123436,0.030641599837690593,0.027343999594449997,0.007562409232763304,0.020160000771284103,0.052960000932216644,0.024145600199699403,0.021424000151455402,0.007450198645599985,0.0,0.0,0.0,0.0,0.0,0.2447039932012558,0.30502399802207947,0.26446720361709597,0.26265600323677063,0.016744548364435372,8,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,64,0.375,0.5,0.5,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9806430689981738,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.024831999093294144,0.046560000628232956,0.030371200107038022,0.02763199992477894,0.005878487205028602,0.020320000126957893,0.04560000076889992,0.02601920012384653,0.02270400058478117,0.00751129965906799,0.0,0.0,0.0,0.0,0.0,0.2337920069694519,0.2881599962711334,0.26074880361557007,0.264384001493454,0.016469850143940968,16,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,128,0.375,1.0,1.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9103623678483975,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.024288000538945198,0.049375999718904495,0.03086080001667142,0.0267359996214509,0.0070864531384997225,0.020479999482631683,0.030912000685930252,0.02274719988927245,0.021359999664127827,0.0029966813650672505,0.0,0.0,0.0,0.0,0.0,0.23369599878787994,0.28591999411582947,0.25465920120477675,0.25385600328445435,0.01593204652619194,32,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,256,0.375,2.0,2.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9610778571819444,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.02393599972128868,0.04342399910092354,0.029380799923092126,0.026688000187277794,0.0057374782000526574,0.020031999796628952,0.036607999354600906,0.022487999964505435,0.020911999978125095,0.0038135203062103235,0.0,0.0,0.0,0.0,0.0,0.2295999974012375,0.26556798815727234,0.24674240052700042,0.2497600018978119,0.010345732066199003,64,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,512,0.375,4.0,4.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9952941013961014,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.025407999753952026,0.05510399863123894,0.031430399790406224,0.02798399981111288,0.007335050273982725,0.020479999482631683,0.03561599925160408,0.02275839988142252,0.021551999263465405,0.003545718365165811,0.0,0.0,0.0,0.0,0.0,0.21721599996089935,0.29020801186561584,0.2394208014011383,0.2346400022506714,0.018747330357768585,128,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,1024,0.375,8.0,8.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9273256282883522,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.0244159996509552,0.05395200103521347,0.03188959984108806,0.026031999848783016,0.00943995927387483,0.02051199972629547,0.036607999354600906,0.023247999791055917,0.02147199958562851,0.003910623837069648,0.0,0.0,0.0,0.0,0.0,0.2717759907245636,0.305184006690979,0.28813759982585907,0.28809599578380585,0.01183422600545854,256,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,2048,0.375,16.0,16.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0380599882396606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.024512000381946564,0.04620800167322159,0.02884640023112297,0.026320000179111958,0.005516557583355925,0.02054399996995926,0.03846399858593941,0.023401600029319524,0.021263999864459038,0.0044742944198265374,0.0,0.0,0.0,0.0,0.0,0.3917759954929352,0.43772798776626587,0.41130879521369934,0.4131519943475723,0.012992473640805227,512,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,4096,0.375,32.0,32.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9569603278386984,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.02412799932062626,0.06940799951553345,0.030817600432783365,0.026800000108778477,0.010047085187652939,0.020128000527620316,0.044096000492572784,0.024606400076299904,0.022304000332951546,0.00622857166823714,0.0,0.0,0.0,0.0,0.0,0.6176639795303345,0.7009919881820679,0.642767995595932,0.6330719888210297,0.024074084919938756,1024,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,8192,0.375,64.0,64.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9524274993623046,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.02831999957561493,0.043487999588251114,0.031744000129401685,0.02991999965161085,0.003973415836428909,0.02070399932563305,0.029343999922275543,0.022886400017887353,0.021743999794125557,0.0026873065045088873,0.0,0.0,0.0,0.0,0.0,1.0820800065994263,1.1674879789352417,1.1034304022789,1.0977439880371094,0.0233067292981566,2048,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,16384,0.375,128.0,128.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8971818172100244,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.03830400109291077,0.06268800050020218,0.043337599746882914,0.040511999279260635,0.005823016247946116,0.023135999217629433,0.03747199848294258,0.025206399988383053,0.02393599972128868,0.003271381256231161,0.0,0.0,0.0,0.0,0.0,1.9809919595718384,2.0415360927581787,2.003715181350708,1.992751955986023,0.022066076434645737,4096,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,32768,0.375,256.0,256.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8113207890716184,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.05660799890756607,0.07932800054550171,0.06249920018017292,0.06039999984204769,0.005636461691480845,0.02956799976527691,0.03747199848294258,0.031126399897038935,0.030287999659776688,0.002157571006631541,0.0,0.0,0.0,0.0,0.0,3.790112018585205,3.8651199340820312,3.829139161109924,3.8230879306793213,0.025177464160110564,8192,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,65536,0.375,512.0,512.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.883909666885606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.02502400055527687,0.06092799827456474,0.033839999698102474,0.028672000393271446,0.010315255343709818,0.019360000267624855,0.0352960005402565,0.023424000293016434,0.022064000368118286,0.0038898102289194572,0.0,0.0,0.0,0.0,0.0,0.212351992726326,0.24383999407291412,0.22760000079870224,0.22723200172185898,0.01050568575837594,1,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,8,0.375,0.0625,0.0625,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0233364439829928,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.025280000641942024,0.05132799968123436,0.030641599837690593,0.027343999594449997,0.007562409232763304,0.020160000771284103,0.052960000932216644,0.024145600199699403,0.021424000151455402,0.007450198645599985,0.0,0.0,0.0,0.0,0.0,0.47494399547576904,0.5184000134468079,0.4920704007148743,0.49169600009918213,0.011991064701471855,8,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,64,0.375,0.5,0.5,0.3984375,0.0,1.3919410907075054,6.0,5.570159765557392,0.667236328125,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9806430689981738,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.024831999093294144,0.046560000628232956,0.030371200107038022,0.02763199992477894,0.005878487205028602,0.020320000126957893,0.04560000076889992,0.02601920012384653,0.02270400058478117,0.00751129965906799,0.0,0.0,0.0,0.0,0.0,0.6360960006713867,0.7004479765892029,0.6608384013175964,0.6572319865226746,0.020416877242438597,16,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,128,0.375,1.0,1.0,0.625,0.0,1.0307764064044151,4.0,6.138251855282827,0.5382080078125,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9103623678483975,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.024288000538945198,0.049375999718904495,0.03086080001667142,0.0267359996214509,0.0070864531384997225,0.020479999482631683,0.030912000685930252,0.02274719988927245,0.021359999664127827,0.0029966813650672505,0.0,0.0,0.0,0.0,0.0,0.780896008014679,0.8301439881324768,0.80346559882164,0.8030399978160858,0.016801230312128875,32,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,256,0.375,2.0,2.0,0.859375,0.0,0.6903350635742038,3.0,6.5943747091218174,0.38067626953125,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9610778571819444,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.02393599972128868,0.04342399910092354,0.029380799923092126,0.026688000187277794,0.0057374782000526574,0.020031999796628952,0.036607999354600906,0.022487999964505435,0.020911999978125095,0.0038135203062103235,0.0,0.0,0.0,0.0,0.0,0.8607040047645569,0.9195200204849243,0.8783008038997651,0.8751039803028107,0.01719059253115595,64,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,512,0.375,4.0,4.0,0.9765625,0.0,0.49410588440130926,2.75,6.814452474347134,0.271270751953125,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9952941013961014,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.025407999753952026,0.05510399863123894,0.031430399790406224,0.02798399981111288,0.007335050273982725,0.020479999482631683,0.03561599925160408,0.02275839988142252,0.021551999263465405,0.003545718365165811,0.0,0.0,0.0,0.0,0.0,0.833952009677887,0.894752025604248,0.8619967997074127,0.863215982913971,0.018716378851797198,128,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,1024,0.375,8.0,8.0,1.0,0.25,0.33693529145074724,2.125,6.9186075263155535,0.1867218017578125,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9273256282883522,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.0244159996509552,0.05395200103521347,0.03188959984108806,0.026031999848783016,0.00943995927387483,0.02051199972629547,0.036607999354600906,0.023247999791055917,0.02147199958562851,0.003910623837069648,0.0,0.0,0.0,0.0,0.0,0.834879994392395,0.8871039748191833,0.8651552021503448,0.8638879954814911,0.015262430894400969,256,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,2048,0.375,16.0,16.0,1.0,0.375,0.25567294018677456,1.8125,6.952441049154937,0.14349365234375,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0380599882396606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.024512000381946564,0.04620800167322159,0.02884640023112297,0.026320000179111958,0.005516557583355925,0.02054399996995926,0.03846399858593941,0.023401600029319524,0.021263999864459038,0.0044742944198265374,0.0,0.0,0.0,0.0,0.0,0.8518080115318298,0.9097599983215332,0.8810272097587586,0.8751040101051331,0.017005819633271906,512,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,4096,0.375,32.0,32.0,1.0,0.625,0.1747801353218523,1.53125,6.978069554482723,0.09820938110351562,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9569603278386984,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.02412799932062626,0.06940799951553345,0.030817600432783365,0.026800000108778477,0.010047085187652939,0.020128000527620316,0.044096000492572784,0.024606400076299904,0.022304000332951546,0.00622857166823714,0.0,0.0,0.0,0.0,0.0,0.8470079898834229,0.9010239839553833,0.8694015920162201,0.8716959953308105,0.016138881171190216,1024,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,8192,0.375,64.0,64.0,1.0,0.65625,0.1158122428154187,1.3125,6.9901908183358845,0.06445503234863281,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9524274993623046,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.02831999957561493,0.043487999588251114,0.031744000129401685,0.02991999965161085,0.003973415836428909,0.02070399932563305,0.029343999922275543,0.022886400017887353,0.021743999794125557,0.0026873065045088873,0.0,0.0,0.0,0.0,0.0,1.1698240041732788,1.2311359643936157,1.1888479948043824,1.1890720129013062,0.017978797794492758,2048,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,16384,0.375,128.0,128.0,1.0,0.78125,0.08347181893108634,1.1796875,6.994921772573154,0.046871185302734375,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8971818172100244,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.03830400109291077,0.06268800050020218,0.043337599746882914,0.040511999279260635,0.005823016247946116,0.023135999217629433,0.03747199848294258,0.025206399988383053,0.02393599972128868,0.003271381256231161,0.0,0.0,0.0,0.0,0.0,1.7702720165252686,1.8097599744796753,1.7919103980064393,1.7956640124320984,0.012641295676021557,4096,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,32768,0.375,256.0,256.0,1.0,0.78125,0.06866734477822484,1.20703125,6.996602562938728,0.03801727294921875,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8113207890716184,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.05660799890756607,0.07932800054550171,0.06249920018017292,0.06039999984204769,0.005636461691480845,0.02956799976527691,0.03747199848294258,0.031126399897038935,0.030287999659776688,0.002157571006631541,0.0,0.0,0.0,0.0,0.0,2.968672037124634,3.0278079509735107,2.9899007797241213,2.9824799299240112,0.01816414122631667,8192,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,65536,0.375,512.0,512.0,1.0,0.8984375,0.04399546833977376,1.126953125,6.998607314922362,0.024699926376342773,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.883909666885606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.02502400055527687,0.06092799827456474,0.033839999698102474,0.028672000393271446,0.010315255343709818,0.019360000267624855,0.0352960005402565,0.023424000293016434,0.022064000368118286,0.0038898102289194572,0.0,0.0,0.0,0.0,0.0,0.19574399292469025,0.2512960135936737,0.21939200013875962,0.2199999988079071,0.017532156418212565,1,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,8,0.375,0.0625,0.0625,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0233364439829928,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.025280000641942024,0.05132799968123436,0.030641599837690593,0.027343999594449997,0.007562409232763304,0.020160000771284103,0.052960000932216644,0.024145600199699403,0.021424000151455402,0.007450198645599985,0.0,0.0,0.0,0.0,0.0,0.20483200252056122,0.24006399512290955,0.22215040028095245,0.22433599829673767,0.00969639786132892,8,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,64,0.375,0.5,0.5,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9806430689981738,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.024831999093294144,0.046560000628232956,0.030371200107038022,0.02763199992477894,0.005878487205028602,0.020320000126957893,0.04560000076889992,0.02601920012384653,0.02270400058478117,0.00751129965906799,0.0,0.0,0.0,0.0,0.0,0.20559999346733093,0.24726399779319763,0.22126719802618028,0.22207999974489212,0.01328093478485499,16,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,128,0.375,1.0,1.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9103623678483975,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.024288000538945198,0.049375999718904495,0.03086080001667142,0.0267359996214509,0.0070864531384997225,0.020479999482631683,0.030912000685930252,0.02274719988927245,0.021359999664127827,0.0029966813650672505,0.0,0.0,0.0,0.0,0.0,0.20003199577331543,0.2301120012998581,0.21453119963407516,0.21598400175571442,0.010402239855151332,32,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,256,0.375,2.0,2.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9610778571819444,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.02393599972128868,0.04342399910092354,0.029380799923092126,0.026688000187277794,0.0057374782000526574,0.020031999796628952,0.036607999354600906,0.022487999964505435,0.020911999978125095,0.0038135203062103235,0.0,0.0,0.0,0.0,0.0,0.19551999866962433,0.22972799837589264,0.21238719969987868,0.21488000452518463,0.010706095835489097,64,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,512,0.375,4.0,4.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9952941013961014,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.025407999753952026,0.05510399863123894,0.031430399790406224,0.02798399981111288,0.007335050273982725,0.020479999482631683,0.03561599925160408,0.02275839988142252,0.021551999263465405,0.003545718365165811,0.0,0.0,0.0,0.0,0.0,0.19420799612998962,0.2903999984264374,0.2211231991648674,0.21476799994707108,0.025955008886196004,128,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,1024,0.375,8.0,8.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9273256282883522,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.0244159996509552,0.05395200103521347,0.03188959984108806,0.026031999848783016,0.00943995927387483,0.02051199972629547,0.036607999354600906,0.023247999791055917,0.02147199958562851,0.003910623837069648,0.0,0.0,0.0,0.0,0.0,0.2290560007095337,0.289247989654541,0.2543327987194061,0.24939200282096863,0.01663236753503115,256,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,2048,0.375,16.0,16.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0380599882396606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.024512000381946564,0.04620800167322159,0.02884640023112297,0.026320000179111958,0.005516557583355925,0.02054399996995926,0.03846399858593941,0.023401600029319524,0.021263999864459038,0.0044742944198265374,0.0,0.0,0.0,0.0,0.0,0.30831998586654663,0.36953601241111755,0.3324000000953674,0.3288639932870865,0.018617999572156707,512,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,4096,0.375,32.0,32.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9569603278386984,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.02412799932062626,0.06940799951553345,0.030817600432783365,0.026800000108778477,0.010047085187652939,0.020128000527620316,0.044096000492572784,0.024606400076299904,0.022304000332951546,0.00622857166823714,0.0,0.0,0.0,0.0,0.0,0.462911993265152,0.5497919917106628,0.4893856018781662,0.4816960096359253,0.023348152887178286,1024,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,8192,0.375,64.0,64.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9524274993623046,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.02831999957561493,0.043487999588251114,0.031744000129401685,0.02991999965161085,0.003973415836428909,0.02070399932563305,0.029343999922275543,0.022886400017887353,0.021743999794125557,0.0026873065045088873,0.0,0.0,0.0,0.0,0.0,0.7662079930305481,0.8717759847640991,0.788454395532608,0.7744799852371216,0.03174746482604812,2048,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,16384,0.375,128.0,128.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8971818172100244,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.03830400109291077,0.06268800050020218,0.043337599746882914,0.040511999279260635,0.005823016247946116,0.023135999217629433,0.03747199848294258,0.025206399988383053,0.02393599972128868,0.003271381256231161,0.0,0.0,0.0,0.0,0.0,1.363935947418213,1.4143040180206299,1.3812703967094422,1.3798720240592957,0.014770450075530007,4096,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,32768,0.375,256.0,256.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8113207890716184,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.05660799890756607,0.07932800054550171,0.06249920018017292,0.06039999984204769,0.005636461691480845,0.02956799976527691,0.03747199848294258,0.031126399897038935,0.030287999659776688,0.002157571006631541,0.0,0.0,0.0,0.0,0.0,2.5507519245147705,2.680704116821289,2.579859209060669,2.566223978996277,0.03673283558365955,8192,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,65536,0.375,512.0,512.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.883909666885606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
1 time_stats.moe_gating_linear.min time_stats.moe_gating_linear.max time_stats.moe_gating_linear.mean time_stats.moe_gating_linear.median time_stats.moe_gating_linear.std time_stats.moe_gating_routing_topk.min time_stats.moe_gating_routing_topk.max time_stats.moe_gating_routing_topk.mean time_stats.moe_gating_routing_topk.median time_stats.moe_gating_routing_topk.std time_stats.moe_shuffling.min time_stats.moe_shuffling.max time_stats.moe_shuffling.mean time_stats.moe_shuffling.median time_stats.moe_shuffling.std time_stats.moe_grouped_gemm.min time_stats.moe_grouped_gemm.max time_stats.moe_grouped_gemm.mean time_stats.moe_grouped_gemm.median time_stats.moe_grouped_gemm.std num_tokens num_experts num_experts_per_device expert_parallel_size routing_runtime_path routing_assignment_policy routing_weight_policy routing_uses_router_logits gating_runtime_context gating_runtime_context_impl router_topk hidden_dim expert_hidden_dim use_gated num_tensor_parallel_workers total_routed_tokens model_expansion_ratio tokens_per_expert_avg tokens_to_experts_ratio expert_utilization min_load_ratio load_imbalance_cv max_load_ratio load_entropy load_gini_coefficient load_distribution seed moe_grouped_gemm_backend measurement_type profiling_precision model_arch quant_signature router_median_nonadditivity_ratio projection_policy
2 0.02502400055527687 0.06092799827456474 0.033839999698102474 0.028672000393271446 0.010315255343709818 0.019360000267624855 0.0352960005402565 0.023424000293016434 0.022064000368118286 0.0038898102289194572 0.0 0.0 0.0 0.0 0.0 0.33926400542259216 0.405023992061615 0.36780479848384856 0.36507199704647064 0.01690507644474779 1 128 128 1 standard_fused_topk logit_topk softmax_renorm True standalone_legacy vllm020_replicated_linear 8 2048 768 True 1 8 0.375 0.0625 0.0625 0.0625 0.0 3.872983346207417 16.0 3.0 0.9375 uniform_random_logits 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 1.0233364439829928 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
3 0.025280000641942024 0.05132799968123436 0.030641599837690593 0.027343999594449997 0.007562409232763304 0.020160000771284103 0.052960000932216644 0.024145600199699403 0.021424000151455402 0.007450198645599985 0.0 0.0 0.0 0.0 0.0 1.1943039894104004 1.286784052848816 1.228384006023407 1.2273280024528503 0.02832547242381263 8 128 128 1 standard_fused_topk logit_topk softmax_renorm True standalone_legacy vllm020_replicated_linear 8 2048 768 True 1 64 0.375 0.5 0.5 0.3984375 0.0 1.346291201783626 4.0 5.59375 0.661865234375 uniform_random_logits 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 0.9806430689981738 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
4 0.024831999093294144 0.046560000628232956 0.030371200107038022 0.02763199992477894 0.005878487205028602 0.020320000126957893 0.04560000076889992 0.02601920012384653 0.02270400058478117 0.00751129965906799 0.0 0.0 0.0 0.0 0.0 1.679744005203247 1.766144037246704 1.7095808148384095 1.7015680074691772 0.02438921262998535 16 128 128 1 standard_fused_topk logit_topk softmax_renorm True standalone_legacy vllm020_replicated_linear 8 2048 768 True 1 128 0.375 1.0 1.0 0.625 0.0 1.015504800579495 5.0 6.15516433212955 0.529052734375 uniform_random_logits 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 0.9103623678483975 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
5 0.024288000538945198 0.049375999718904495 0.03086080001667142 0.0267359996214509 0.0070864531384997225 0.020479999482631683 0.030912000685930252 0.02274719988927245 0.021359999664127827 0.0029966813650672505 0.0 0.0 0.0 0.0 0.0 2.1576640605926514 2.2921600341796875 2.2097824096679686 2.188944101333618 0.045572321842012986 32 128 128 1 standard_fused_topk logit_topk softmax_renorm True standalone_legacy vllm020_replicated_linear 8 2048 768 True 1 256 0.375 2.0 2.0 0.875 0.0 0.6343057228182637 2.5 6.64370748444639 0.35369873046875 uniform_random_logits 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 0.9610778571819444 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
6 0.02393599972128868 0.04342399910092354 0.029380799923092126 0.026688000187277794 0.0057374782000526574 0.020031999796628952 0.036607999354600906 0.022487999964505435 0.020911999978125095 0.0038135203062103235 0.0 0.0 0.0 0.0 0.0 2.422368049621582 2.5130879878997803 2.4516672134399413 2.434159994125366 0.03278900287381846 64 128 128 1 standard_fused_topk logit_topk softmax_renorm True standalone_legacy vllm020_replicated_linear 8 2048 768 True 1 512 0.375 4.0 4.0 0.984375 0.0 0.4921254921257382 2.25 6.817190042344769 0.272369384765625 uniform_random_logits 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 0.9952941013961014 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
7 0.025407999753952026 0.05510399863123894 0.031430399790406224 0.02798399981111288 0.007335050273982725 0.020479999482631683 0.03561599925160408 0.02275839988142252 0.021551999263465405 0.003545718365165811 0.0 0.0 0.0 0.0 0.0 2.2217600345611572 2.289599895477295 2.2571327924728393 2.263375997543335 0.021660416089449488 128 128 128 1 standard_fused_topk logit_topk softmax_renorm True standalone_legacy vllm020_replicated_linear 8 2048 768 True 1 1024 0.375 8.0 8.0 1.0 0.125 0.3486861500690843 1.875 6.908192310183997 0.197662353515625 uniform_random_logits 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 0.9273256282883522 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
8 0.0244159996509552 0.05395200103521347 0.03188959984108806 0.026031999848783016 0.00943995927387483 0.02051199972629547 0.036607999354600906 0.023247999791055917 0.02147199958562851 0.003910623837069648 0.0 0.0 0.0 0.0 0.0 2.18668794631958 2.318079948425293 2.2218016147613526 2.211087942123413 0.035380897213135316 256 128 128 1 standard_fused_topk logit_topk softmax_renorm True standalone_legacy vllm020_replicated_linear 8 2048 768 True 1 2048 0.375 16.0 16.0 1.0 0.4375 0.2525504668006971 1.875 6.953347743053017 0.1410369873046875 uniform_random_logits 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 1.0380599882396606 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
9 0.024512000381946564 0.04620800167322159 0.02884640023112297 0.026320000179111958 0.005516557583355925 0.02054399996995926 0.03846399858593941 0.023401600029319524 0.021263999864459038 0.0044742944198265374 0.0 0.0 0.0 0.0 0.0 2.2291839122772217 2.3929600715637207 2.2908096313476562 2.2804640531539917 0.04479348924786221 512 128 128 1 standard_fused_topk logit_topk softmax_renorm True standalone_legacy vllm020_replicated_linear 8 2048 768 True 1 4096 0.375 32.0 32.0 1.0 0.65625 0.15765965680164504 1.5625 6.98229848728205 0.08779525756835938 uniform_random_logits 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 0.9569603278386984 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
10 0.02412799932062626 0.06940799951553345 0.030817600432783365 0.026800000108778477 0.010047085187652939 0.020128000527620316 0.044096000492572784 0.024606400076299904 0.022304000332951546 0.00622857166823714 0.0 0.0 0.0 0.0 0.0 2.0678720474243164 2.1297600269317627 2.0837119817733765 2.0779199600219727 0.017880044357986735 1024 128 128 1 standard_fused_topk logit_topk softmax_renorm True standalone_legacy vllm020_replicated_linear 8 2048 768 True 1 8192 0.375 64.0 64.0 1.0 0.625 0.12169081635504074 1.3125 6.9892029662356325 0.06879425048828125 uniform_random_logits 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 0.9524274993623046 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
11 0.02831999957561493 0.043487999588251114 0.031744000129401685 0.02991999965161085 0.003973415836428909 0.02070399932563305 0.029343999922275543 0.022886400017887353 0.021743999794125557 0.0026873065045088873 0.0 0.0 0.0 0.0 0.0 2.916032075881958 3.0819520950317383 2.9805248022079467 2.9656319618225098 0.05482195799019572 2048 128 128 1 standard_fused_topk logit_topk softmax_renorm True standalone_legacy vllm020_replicated_linear 8 2048 768 True 1 16384 0.375 128.0 128.0 1.0 0.796875 0.07935434147688751 1.1796875 6.9954297964750305 0.044734954833984375 uniform_random_logits 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 0.8971818172100244 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
12 0.03830400109291077 0.06268800050020218 0.043337599746882914 0.040511999279260635 0.005823016247946116 0.023135999217629433 0.03747199848294258 0.025206399988383053 0.02393599972128868 0.003271381256231161 0.0 0.0 0.0 0.0 0.0 4.421599864959717 4.535359859466553 4.486294317245483 4.497056007385254 0.036990243787549344 4096 128 128 1 standard_fused_topk logit_topk softmax_renorm True standalone_legacy vllm020_replicated_linear 8 2048 768 True 1 32768 0.375 256.0 256.0 1.0 0.8203125 0.060849326483103046 1.17578125 6.9973188375859685 0.033740997314453125 uniform_random_logits 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 0.8113207890716184 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
13 0.05660799890756607 0.07932800054550171 0.06249920018017292 0.06039999984204769 0.005636461691480845 0.02956799976527691 0.03747199848294258 0.031126399897038935 0.030287999659776688 0.002157571006631541 0.0 0.0 0.0 0.0 0.0 7.302591800689697 7.402751922607422 7.354758310317993 7.3464319705963135 0.032142662400335566 8192 128 128 1 standard_fused_topk logit_topk softmax_renorm True standalone_legacy vllm020_replicated_linear 8 2048 768 True 1 65536 0.375 512.0 512.0 1.0 0.890625 0.0412323087266341 1.08984375 6.998772433185578 0.02334284782409668 uniform_random_logits 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 0.883909666885606 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
14 0.02502400055527687 0.06092799827456474 0.033839999698102474 0.028672000393271446 0.010315255343709818 0.019360000267624855 0.0352960005402565 0.023424000293016434 0.022064000368118286 0.0038898102289194572 0.0 0.0 0.0 0.0 0.0 0.35280001163482666 0.39692801237106323 0.37662720382213594 0.37196800112724304 0.013401318050665304 1 128 128 1 standard_fused_topk fixed_hotset8 softmax_renorm False standalone_legacy vllm020_replicated_linear 8 2048 768 True 1 8 0.375 0.0625 0.0625 0.0625 0.0 3.872983346207417 16.0 3.0 0.9375 hotset8 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 1.0233364439829928 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
15 0.025280000641942024 0.05132799968123436 0.030641599837690593 0.027343999594449997 0.007562409232763304 0.020160000771284103 0.052960000932216644 0.024145600199699403 0.021424000151455402 0.007450198645599985 0.0 0.0 0.0 0.0 0.0 0.4692479968070984 0.5523840188980103 0.5134752035140991 0.5100640058517456 0.02291433464135784 8 128 128 1 standard_fused_topk fixed_hotset8 softmax_renorm False standalone_legacy vllm020_replicated_linear 8 2048 768 True 1 64 0.375 0.5 0.5 0.0625 0.0 3.872983346207417 16.0 3.0 0.9375 hotset8 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 0.9806430689981738 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
16 0.024831999093294144 0.046560000628232956 0.030371200107038022 0.02763199992477894 0.005878487205028602 0.020320000126957893 0.04560000076889992 0.02601920012384653 0.02270400058478117 0.00751129965906799 0.0 0.0 0.0 0.0 0.0 0.34652799367904663 0.4119040071964264 0.3789471983909607 0.38550400733947754 0.02073945105803335 16 128 128 1 standard_fused_topk fixed_hotset8 softmax_renorm False standalone_legacy vllm020_replicated_linear 8 2048 768 True 1 128 0.375 1.0 1.0 0.0625 0.0 3.872983346207417 16.0 3.0 0.9375 hotset8 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 0.9103623678483975 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
17 0.024288000538945198 0.049375999718904495 0.03086080001667142 0.0267359996214509 0.0070864531384997225 0.020479999482631683 0.030912000685930252 0.02274719988927245 0.021359999664127827 0.0029966813650672505 0.0 0.0 0.0 0.0 0.0 0.31462401151657104 0.7456960082054138 0.38617280423641204 0.34545600414276123 0.12230201266253077 32 128 128 1 standard_fused_topk fixed_hotset8 softmax_renorm False standalone_legacy vllm020_replicated_linear 8 2048 768 True 1 256 0.375 2.0 2.0 0.0625 0.0 3.872983346207417 16.0 3.0 0.9375 hotset8 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 0.9610778571819444 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
18 0.02393599972128868 0.04342399910092354 0.029380799923092126 0.026688000187277794 0.0057374782000526574 0.020031999796628952 0.036607999354600906 0.022487999964505435 0.020911999978125095 0.0038135203062103235 0.0 0.0 0.0 0.0 0.0 0.32521599531173706 0.419871985912323 0.36325119733810424 0.34968000650405884 0.03161798848223672 64 128 128 1 standard_fused_topk fixed_hotset8 softmax_renorm False standalone_legacy vllm020_replicated_linear 8 2048 768 True 1 512 0.375 4.0 4.0 0.0625 0.0 3.872983346207417 16.0 3.0 0.9375 hotset8 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 0.9952941013961014 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
19 0.025407999753952026 0.05510399863123894 0.031430399790406224 0.02798399981111288 0.007335050273982725 0.020479999482631683 0.03561599925160408 0.02275839988142252 0.021551999263465405 0.003545718365165811 0.0 0.0 0.0 0.0 0.0 0.289792001247406 0.4663360118865967 0.4091839998960495 0.41655999422073364 0.0446001986506615 128 128 128 1 standard_fused_topk fixed_hotset8 softmax_renorm False standalone_legacy vllm020_replicated_linear 8 2048 768 True 1 1024 0.375 8.0 8.0 0.0625 0.0 3.872983346207417 16.0 3.0 0.9375 hotset8 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 0.9273256282883522 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
20 0.0244159996509552 0.05395200103521347 0.03188959984108806 0.026031999848783016 0.00943995927387483 0.02051199972629547 0.036607999354600906 0.023247999791055917 0.02147199958562851 0.003910623837069648 0.0 0.0 0.0 0.0 0.0 0.3761279881000519 0.4416320025920868 0.40686399936676027 0.40540799498558044 0.02257778769899645 256 128 128 1 standard_fused_topk fixed_hotset8 softmax_renorm False standalone_legacy vllm020_replicated_linear 8 2048 768 True 1 2048 0.375 16.0 16.0 0.0625 0.0 3.872983346207417 16.0 3.0 0.9375 hotset8 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 1.0380599882396606 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
21 0.024512000381946564 0.04620800167322159 0.02884640023112297 0.026320000179111958 0.005516557583355925 0.02054399996995926 0.03846399858593941 0.023401600029319524 0.021263999864459038 0.0044742944198265374 0.0 0.0 0.0 0.0 0.0 0.7172480225563049 0.8663039803504944 0.7723807990550995 0.7591840028762817 0.04164772379451242 512 128 128 1 standard_fused_topk fixed_hotset8 softmax_renorm False standalone_legacy vllm020_replicated_linear 8 2048 768 True 1 4096 0.375 32.0 32.0 0.0625 0.0 3.872983346207417 16.0 3.0 0.9375 hotset8 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 0.9569603278386984 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
22 0.02412799932062626 0.06940799951553345 0.030817600432783365 0.026800000108778477 0.010047085187652939 0.020128000527620316 0.044096000492572784 0.024606400076299904 0.022304000332951546 0.00622857166823714 0.0 0.0 0.0 0.0 0.0 1.0195519924163818 1.2216639518737793 1.1253888130187988 1.1453600525856018 0.06548005322243594 1024 128 128 1 standard_fused_topk fixed_hotset8 softmax_renorm False standalone_legacy vllm020_replicated_linear 8 2048 768 True 1 8192 0.375 64.0 64.0 0.0625 0.0 3.872983346207417 16.0 3.0 0.9375 hotset8 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 0.9524274993623046 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
23 0.02831999957561493 0.043487999588251114 0.031744000129401685 0.02991999965161085 0.003973415836428909 0.02070399932563305 0.029343999922275543 0.022886400017887353 0.021743999794125557 0.0026873065045088873 0.0 0.0 0.0 0.0 0.0 1.7490559816360474 1.9644800424575806 1.8529024004936219 1.814303994178772 0.08042565617327288 2048 128 128 1 standard_fused_topk fixed_hotset8 softmax_renorm False standalone_legacy vllm020_replicated_linear 8 2048 768 True 1 16384 0.375 128.0 128.0 0.0625 0.0 3.872983346207417 16.0 3.0 0.9375 hotset8 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 0.8971818172100244 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
24 0.03830400109291077 0.06268800050020218 0.043337599746882914 0.040511999279260635 0.005823016247946116 0.023135999217629433 0.03747199848294258 0.025206399988383053 0.02393599972128868 0.003271381256231161 0.0 0.0 0.0 0.0 0.0 3.2479360103607178 3.385279893875122 3.296070408821106 3.2800960540771484 0.04525529026442046 4096 128 128 1 standard_fused_topk fixed_hotset8 softmax_renorm False standalone_legacy vllm020_replicated_linear 8 2048 768 True 1 32768 0.375 256.0 256.0 0.0625 0.0 3.872983346207417 16.0 3.0 0.9375 hotset8 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 0.8113207890716184 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
25 0.05660799890756607 0.07932800054550171 0.06249920018017292 0.06039999984204769 0.005636461691480845 0.02956799976527691 0.03747199848294258 0.031126399897038935 0.030287999659776688 0.002157571006631541 0.0 0.0 0.0 0.0 0.0 6.344799995422363 6.517856121063232 6.464438438415527 6.478623867034912 0.05116674443145098 8192 128 128 1 standard_fused_topk fixed_hotset8 softmax_renorm False standalone_legacy vllm020_replicated_linear 8 2048 768 True 1 65536 0.375 512.0 512.0 0.0625 0.0 3.872983346207417 16.0 3.0 0.9375 hotset8 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 0.883909666885606 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
26 0.02502400055527687 0.06092799827456474 0.033839999698102474 0.028672000393271446 0.010315255343709818 0.019360000267624855 0.0352960005402565 0.023424000293016434 0.022064000368118286 0.0038898102289194572 0.0 0.0 0.0 0.0 0.0 0.2648000121116638 0.325439989566803 0.28852800130844114 0.28390398621559143 0.01933778377635077 1 128 128 1 standard_fused_topk logit_topk softmax_renorm True standalone_legacy vllm020_replicated_linear 8 2048 768 True 2 8 0.375 0.0625 0.0625 0.0625 0.0 3.872983346207417 16.0 3.0 0.9375 uniform_random_logits 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 1.0233364439829928 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
27 0.025280000641942024 0.05132799968123436 0.030641599837690593 0.027343999594449997 0.007562409232763304 0.020160000771284103 0.052960000932216644 0.024145600199699403 0.021424000151455402 0.007450198645599985 0.0 0.0 0.0 0.0 0.0 0.7347840070724487 0.862496018409729 0.7769344031810761 0.769216001033783 0.03290485328796285 8 128 128 1 standard_fused_topk logit_topk softmax_renorm True standalone_legacy vllm020_replicated_linear 8 2048 768 True 2 64 0.375 0.5 0.5 0.421875 0.0 1.346291201783626 6.0 5.652114648336087 0.636962890625 uniform_random_logits 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 0.9806430689981738 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
28 0.024831999093294144 0.046560000628232956 0.030371200107038022 0.02763199992477894 0.005878487205028602 0.020320000126957893 0.04560000076889992 0.02601920012384653 0.02270400058478117 0.00751129965906799 0.0 0.0 0.0 0.0 0.0 0.9198399782180786 0.9646080136299133 0.9412063956260681 0.9411839842796326 0.014939365085478117 16 128 128 1 standard_fused_topk logit_topk softmax_renorm True standalone_legacy vllm020_replicated_linear 8 2048 768 True 2 128 0.375 1.0 1.0 0.5703125 0.0 1.118033988749895 5.0 6.008641773518898 0.580810546875 uniform_random_logits 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 0.9103623678483975 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
29 0.024288000538945198 0.049375999718904495 0.03086080001667142 0.0267359996214509 0.0070864531384997225 0.020479999482631683 0.030912000685930252 0.02274719988927245 0.021359999664127827 0.0029966813650672505 0.0 0.0 0.0 0.0 0.0 1.2796800136566162 1.3484159708023071 1.3006976008415223 1.2929120063781738 0.020807998177176254 32 128 128 1 standard_fused_topk logit_topk softmax_renorm True standalone_legacy vllm020_replicated_linear 8 2048 768 True 2 256 0.375 2.0 2.0 0.8828125 0.0 0.6959705453537527 3.0 6.60872850615583 0.38055419921875 uniform_random_logits 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 0.9610778571819444 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
30 0.02393599972128868 0.04342399910092354 0.029380799923092126 0.026688000187277794 0.0057374782000526574 0.020031999796628952 0.036607999354600906 0.022487999964505435 0.020911999978125095 0.0038135203062103235 0.0 0.0 0.0 0.0 0.0 1.3630399703979492 1.4430400133132935 1.3909215927124023 1.3892319798469543 0.022335744492366926 64 128 128 1 standard_fused_topk logit_topk softmax_renorm True standalone_legacy vllm020_replicated_linear 8 2048 768 True 2 512 0.375 4.0 4.0 0.984375 0.0 0.5201036555341637 3.0 6.798826509158851 0.28302001953125 uniform_random_logits 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 0.9952941013961014 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
31 0.025407999753952026 0.05510399863123894 0.031430399790406224 0.02798399981111288 0.007335050273982725 0.020479999482631683 0.03561599925160408 0.02275839988142252 0.021551999263465405 0.003545718365165811 0.0 0.0 0.0 0.0 0.0 1.27948796749115 1.3904000520706177 1.3176063895225525 1.309440016746521 0.038060887827312775 128 128 128 1 standard_fused_topk logit_topk softmax_renorm True standalone_legacy vllm020_replicated_linear 8 2048 768 True 2 1024 0.375 8.0 8.0 1.0 0.25 0.3511282039725661 1.875 6.91002266305238 0.1970977783203125 uniform_random_logits 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 0.9273256282883522 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
32 0.0244159996509552 0.05395200103521347 0.03188959984108806 0.026031999848783016 0.00943995927387483 0.02051199972629547 0.036607999354600906 0.023247999791055917 0.02147199958562851 0.003910623837069648 0.0 0.0 0.0 0.0 0.0 1.264415979385376 1.3145920038223267 1.2791999936103822 1.2753440141677856 0.014130605249568332 256 128 128 1 standard_fused_topk logit_topk softmax_renorm True standalone_legacy vllm020_replicated_linear 8 2048 768 True 2 2048 0.375 16.0 16.0 1.0 0.375 0.24692938483248605 1.6875 6.955481130775285 0.13909912109375 uniform_random_logits 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 1.0380599882396606 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
33 0.024512000381946564 0.04620800167322159 0.02884640023112297 0.026320000179111958 0.005516557583355925 0.02054399996995926 0.03846399858593941 0.023401600029319524 0.021263999864459038 0.0044742944198265374 0.0 0.0 0.0 0.0 0.0 1.3081920146942139 1.347648024559021 1.3292255997657776 1.329967975616455 0.014558863679016933 512 128 128 1 standard_fused_topk logit_topk softmax_renorm True standalone_legacy vllm020_replicated_linear 8 2048 768 True 2 4096 0.375 32.0 32.0 1.0 0.625 0.17143053326165383 1.5625 6.9786675275754035 0.09520339965820312 uniform_random_logits 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 0.9569603278386984 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
34 0.02412799932062626 0.06940799951553345 0.030817600432783365 0.026800000108778477 0.010047085187652939 0.020128000527620316 0.044096000492572784 0.024606400076299904 0.022304000332951546 0.00622857166823714 0.0 0.0 0.0 0.0 0.0 1.242751955986023 1.3112000226974487 1.2747935891151427 1.266207993030548 0.021093073517695057 1024 128 128 1 standard_fused_topk logit_topk softmax_renorm True standalone_legacy vllm020_replicated_linear 8 2048 768 True 2 8192 0.375 64.0 64.0 1.0 0.78125 0.11000099875256815 1.296875 6.991308871213679 0.062183380126953125 uniform_random_logits 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 0.9524274993623046 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
35 0.02831999957561493 0.043487999588251114 0.031744000129401685 0.02991999965161085 0.003973415836428909 0.02070399932563305 0.029343999922275543 0.022886400017887353 0.021743999794125557 0.0026873065045088873 0.0 0.0 0.0 0.0 0.0 1.7388160228729248 1.8077759742736816 1.772764801979065 1.772704005241394 0.021056644077284283 2048 128 128 1 standard_fused_topk logit_topk softmax_renorm True standalone_legacy vllm020_replicated_linear 8 2048 768 True 2 16384 0.375 128.0 128.0 1.0 0.78125 0.0864630150197678 1.1796875 6.994552526394139 0.048796653747558594 uniform_random_logits 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 0.8971818172100244 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
36 0.03830400109291077 0.06268800050020218 0.043337599746882914 0.040511999279260635 0.005823016247946116 0.023135999217629433 0.03747199848294258 0.025206399988383053 0.02393599972128868 0.003271381256231161 0.0 0.0 0.0 0.0 0.0 2.6563520431518555 2.7063679695129395 2.6785055875778196 2.6791679859161377 0.01639963463052944 4096 128 128 1 standard_fused_topk logit_topk softmax_renorm True standalone_legacy vllm020_replicated_linear 8 2048 768 True 2 32768 0.375 256.0 256.0 1.0 0.8671875 0.06127686514721937 1.16015625 6.997291583027146 0.03497934341430664 uniform_random_logits 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 0.8113207890716184 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
37 0.05660799890756607 0.07932800054550171 0.06249920018017292 0.06039999984204769 0.005636461691480845 0.02956799976527691 0.03747199848294258 0.031126399897038935 0.030287999659776688 0.002157571006631541 0.0 0.0 0.0 0.0 0.0 4.386879920959473 4.452256202697754 4.4108480453491214 4.406303882598877 0.019768161791937636 8192 128 128 1 standard_fused_topk logit_topk softmax_renorm True standalone_legacy vllm020_replicated_linear 8 2048 768 True 2 65536 0.375 512.0 512.0 1.0 0.884765625 0.041723768525324195 1.1171875 6.998746434318934 0.02298593521118164 uniform_random_logits 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 0.883909666885606 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
38 0.02502400055527687 0.06092799827456474 0.033839999698102474 0.028672000393271446 0.010315255343709818 0.019360000267624855 0.0352960005402565 0.023424000293016434 0.022064000368118286 0.0038898102289194572 0.0 0.0 0.0 0.0 0.0 0.24208000302314758 0.4028480052947998 0.3041536003351212 0.277103990316391 0.05660721484881584 1 128 128 1 standard_fused_topk fixed_hotset8 softmax_renorm False standalone_legacy vllm020_replicated_linear 8 2048 768 True 2 8 0.375 0.0625 0.0625 0.0625 0.0 3.872983346207417 16.0 3.0 0.9375 hotset8 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 1.0233364439829928 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
39 0.025280000641942024 0.05132799968123436 0.030641599837690593 0.027343999594449997 0.007562409232763304 0.020160000771284103 0.052960000932216644 0.024145600199699403 0.021424000151455402 0.007450198645599985 0.0 0.0 0.0 0.0 0.0 0.2447039932012558 0.30502399802207947 0.26446720361709597 0.26265600323677063 0.016744548364435372 8 128 128 1 standard_fused_topk fixed_hotset8 softmax_renorm False standalone_legacy vllm020_replicated_linear 8 2048 768 True 2 64 0.375 0.5 0.5 0.0625 0.0 3.872983346207417 16.0 3.0 0.9375 hotset8 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 0.9806430689981738 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
40 0.024831999093294144 0.046560000628232956 0.030371200107038022 0.02763199992477894 0.005878487205028602 0.020320000126957893 0.04560000076889992 0.02601920012384653 0.02270400058478117 0.00751129965906799 0.0 0.0 0.0 0.0 0.0 0.2337920069694519 0.2881599962711334 0.26074880361557007 0.264384001493454 0.016469850143940968 16 128 128 1 standard_fused_topk fixed_hotset8 softmax_renorm False standalone_legacy vllm020_replicated_linear 8 2048 768 True 2 128 0.375 1.0 1.0 0.0625 0.0 3.872983346207417 16.0 3.0 0.9375 hotset8 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 0.9103623678483975 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
41 0.024288000538945198 0.049375999718904495 0.03086080001667142 0.0267359996214509 0.0070864531384997225 0.020479999482631683 0.030912000685930252 0.02274719988927245 0.021359999664127827 0.0029966813650672505 0.0 0.0 0.0 0.0 0.0 0.23369599878787994 0.28591999411582947 0.25465920120477675 0.25385600328445435 0.01593204652619194 32 128 128 1 standard_fused_topk fixed_hotset8 softmax_renorm False standalone_legacy vllm020_replicated_linear 8 2048 768 True 2 256 0.375 2.0 2.0 0.0625 0.0 3.872983346207417 16.0 3.0 0.9375 hotset8 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 0.9610778571819444 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
42 0.02393599972128868 0.04342399910092354 0.029380799923092126 0.026688000187277794 0.0057374782000526574 0.020031999796628952 0.036607999354600906 0.022487999964505435 0.020911999978125095 0.0038135203062103235 0.0 0.0 0.0 0.0 0.0 0.2295999974012375 0.26556798815727234 0.24674240052700042 0.2497600018978119 0.010345732066199003 64 128 128 1 standard_fused_topk fixed_hotset8 softmax_renorm False standalone_legacy vllm020_replicated_linear 8 2048 768 True 2 512 0.375 4.0 4.0 0.0625 0.0 3.872983346207417 16.0 3.0 0.9375 hotset8 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 0.9952941013961014 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
43 0.025407999753952026 0.05510399863123894 0.031430399790406224 0.02798399981111288 0.007335050273982725 0.020479999482631683 0.03561599925160408 0.02275839988142252 0.021551999263465405 0.003545718365165811 0.0 0.0 0.0 0.0 0.0 0.21721599996089935 0.29020801186561584 0.2394208014011383 0.2346400022506714 0.018747330357768585 128 128 128 1 standard_fused_topk fixed_hotset8 softmax_renorm False standalone_legacy vllm020_replicated_linear 8 2048 768 True 2 1024 0.375 8.0 8.0 0.0625 0.0 3.872983346207417 16.0 3.0 0.9375 hotset8 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 0.9273256282883522 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
44 0.0244159996509552 0.05395200103521347 0.03188959984108806 0.026031999848783016 0.00943995927387483 0.02051199972629547 0.036607999354600906 0.023247999791055917 0.02147199958562851 0.003910623837069648 0.0 0.0 0.0 0.0 0.0 0.2717759907245636 0.305184006690979 0.28813759982585907 0.28809599578380585 0.01183422600545854 256 128 128 1 standard_fused_topk fixed_hotset8 softmax_renorm False standalone_legacy vllm020_replicated_linear 8 2048 768 True 2 2048 0.375 16.0 16.0 0.0625 0.0 3.872983346207417 16.0 3.0 0.9375 hotset8 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 1.0380599882396606 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
45 0.024512000381946564 0.04620800167322159 0.02884640023112297 0.026320000179111958 0.005516557583355925 0.02054399996995926 0.03846399858593941 0.023401600029319524 0.021263999864459038 0.0044742944198265374 0.0 0.0 0.0 0.0 0.0 0.3917759954929352 0.43772798776626587 0.41130879521369934 0.4131519943475723 0.012992473640805227 512 128 128 1 standard_fused_topk fixed_hotset8 softmax_renorm False standalone_legacy vllm020_replicated_linear 8 2048 768 True 2 4096 0.375 32.0 32.0 0.0625 0.0 3.872983346207417 16.0 3.0 0.9375 hotset8 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 0.9569603278386984 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
46 0.02412799932062626 0.06940799951553345 0.030817600432783365 0.026800000108778477 0.010047085187652939 0.020128000527620316 0.044096000492572784 0.024606400076299904 0.022304000332951546 0.00622857166823714 0.0 0.0 0.0 0.0 0.0 0.6176639795303345 0.7009919881820679 0.642767995595932 0.6330719888210297 0.024074084919938756 1024 128 128 1 standard_fused_topk fixed_hotset8 softmax_renorm False standalone_legacy vllm020_replicated_linear 8 2048 768 True 2 8192 0.375 64.0 64.0 0.0625 0.0 3.872983346207417 16.0 3.0 0.9375 hotset8 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 0.9524274993623046 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
47 0.02831999957561493 0.043487999588251114 0.031744000129401685 0.02991999965161085 0.003973415836428909 0.02070399932563305 0.029343999922275543 0.022886400017887353 0.021743999794125557 0.0026873065045088873 0.0 0.0 0.0 0.0 0.0 1.0820800065994263 1.1674879789352417 1.1034304022789 1.0977439880371094 0.0233067292981566 2048 128 128 1 standard_fused_topk fixed_hotset8 softmax_renorm False standalone_legacy vllm020_replicated_linear 8 2048 768 True 2 16384 0.375 128.0 128.0 0.0625 0.0 3.872983346207417 16.0 3.0 0.9375 hotset8 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 0.8971818172100244 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
48 0.03830400109291077 0.06268800050020218 0.043337599746882914 0.040511999279260635 0.005823016247946116 0.023135999217629433 0.03747199848294258 0.025206399988383053 0.02393599972128868 0.003271381256231161 0.0 0.0 0.0 0.0 0.0 1.9809919595718384 2.0415360927581787 2.003715181350708 1.992751955986023 0.022066076434645737 4096 128 128 1 standard_fused_topk fixed_hotset8 softmax_renorm False standalone_legacy vllm020_replicated_linear 8 2048 768 True 2 32768 0.375 256.0 256.0 0.0625 0.0 3.872983346207417 16.0 3.0 0.9375 hotset8 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 0.8113207890716184 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
49 0.05660799890756607 0.07932800054550171 0.06249920018017292 0.06039999984204769 0.005636461691480845 0.02956799976527691 0.03747199848294258 0.031126399897038935 0.030287999659776688 0.002157571006631541 0.0 0.0 0.0 0.0 0.0 3.790112018585205 3.8651199340820312 3.829139161109924 3.8230879306793213 0.025177464160110564 8192 128 128 1 standard_fused_topk fixed_hotset8 softmax_renorm False standalone_legacy vllm020_replicated_linear 8 2048 768 True 2 65536 0.375 512.0 512.0 0.0625 0.0 3.872983346207417 16.0 3.0 0.9375 hotset8 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 0.883909666885606 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
50 0.02502400055527687 0.06092799827456474 0.033839999698102474 0.028672000393271446 0.010315255343709818 0.019360000267624855 0.0352960005402565 0.023424000293016434 0.022064000368118286 0.0038898102289194572 0.0 0.0 0.0 0.0 0.0 0.212351992726326 0.24383999407291412 0.22760000079870224 0.22723200172185898 0.01050568575837594 1 128 128 1 standard_fused_topk logit_topk softmax_renorm True standalone_legacy vllm020_replicated_linear 8 2048 768 True 4 8 0.375 0.0625 0.0625 0.0625 0.0 3.872983346207417 16.0 3.0 0.9375 uniform_random_logits 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 1.0233364439829928 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
51 0.025280000641942024 0.05132799968123436 0.030641599837690593 0.027343999594449997 0.007562409232763304 0.020160000771284103 0.052960000932216644 0.024145600199699403 0.021424000151455402 0.007450198645599985 0.0 0.0 0.0 0.0 0.0 0.47494399547576904 0.5184000134468079 0.4920704007148743 0.49169600009918213 0.011991064701471855 8 128 128 1 standard_fused_topk logit_topk softmax_renorm True standalone_legacy vllm020_replicated_linear 8 2048 768 True 4 64 0.375 0.5 0.5 0.3984375 0.0 1.3919410907075054 6.0 5.570159765557392 0.667236328125 uniform_random_logits 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 0.9806430689981738 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
52 0.024831999093294144 0.046560000628232956 0.030371200107038022 0.02763199992477894 0.005878487205028602 0.020320000126957893 0.04560000076889992 0.02601920012384653 0.02270400058478117 0.00751129965906799 0.0 0.0 0.0 0.0 0.0 0.6360960006713867 0.7004479765892029 0.6608384013175964 0.6572319865226746 0.020416877242438597 16 128 128 1 standard_fused_topk logit_topk softmax_renorm True standalone_legacy vllm020_replicated_linear 8 2048 768 True 4 128 0.375 1.0 1.0 0.625 0.0 1.0307764064044151 4.0 6.138251855282827 0.5382080078125 uniform_random_logits 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 0.9103623678483975 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
53 0.024288000538945198 0.049375999718904495 0.03086080001667142 0.0267359996214509 0.0070864531384997225 0.020479999482631683 0.030912000685930252 0.02274719988927245 0.021359999664127827 0.0029966813650672505 0.0 0.0 0.0 0.0 0.0 0.780896008014679 0.8301439881324768 0.80346559882164 0.8030399978160858 0.016801230312128875 32 128 128 1 standard_fused_topk logit_topk softmax_renorm True standalone_legacy vllm020_replicated_linear 8 2048 768 True 4 256 0.375 2.0 2.0 0.859375 0.0 0.6903350635742038 3.0 6.5943747091218174 0.38067626953125 uniform_random_logits 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 0.9610778571819444 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
54 0.02393599972128868 0.04342399910092354 0.029380799923092126 0.026688000187277794 0.0057374782000526574 0.020031999796628952 0.036607999354600906 0.022487999964505435 0.020911999978125095 0.0038135203062103235 0.0 0.0 0.0 0.0 0.0 0.8607040047645569 0.9195200204849243 0.8783008038997651 0.8751039803028107 0.01719059253115595 64 128 128 1 standard_fused_topk logit_topk softmax_renorm True standalone_legacy vllm020_replicated_linear 8 2048 768 True 4 512 0.375 4.0 4.0 0.9765625 0.0 0.49410588440130926 2.75 6.814452474347134 0.271270751953125 uniform_random_logits 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 0.9952941013961014 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
55 0.025407999753952026 0.05510399863123894 0.031430399790406224 0.02798399981111288 0.007335050273982725 0.020479999482631683 0.03561599925160408 0.02275839988142252 0.021551999263465405 0.003545718365165811 0.0 0.0 0.0 0.0 0.0 0.833952009677887 0.894752025604248 0.8619967997074127 0.863215982913971 0.018716378851797198 128 128 128 1 standard_fused_topk logit_topk softmax_renorm True standalone_legacy vllm020_replicated_linear 8 2048 768 True 4 1024 0.375 8.0 8.0 1.0 0.25 0.33693529145074724 2.125 6.9186075263155535 0.1867218017578125 uniform_random_logits 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 0.9273256282883522 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
56 0.0244159996509552 0.05395200103521347 0.03188959984108806 0.026031999848783016 0.00943995927387483 0.02051199972629547 0.036607999354600906 0.023247999791055917 0.02147199958562851 0.003910623837069648 0.0 0.0 0.0 0.0 0.0 0.834879994392395 0.8871039748191833 0.8651552021503448 0.8638879954814911 0.015262430894400969 256 128 128 1 standard_fused_topk logit_topk softmax_renorm True standalone_legacy vllm020_replicated_linear 8 2048 768 True 4 2048 0.375 16.0 16.0 1.0 0.375 0.25567294018677456 1.8125 6.952441049154937 0.14349365234375 uniform_random_logits 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 1.0380599882396606 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
57 0.024512000381946564 0.04620800167322159 0.02884640023112297 0.026320000179111958 0.005516557583355925 0.02054399996995926 0.03846399858593941 0.023401600029319524 0.021263999864459038 0.0044742944198265374 0.0 0.0 0.0 0.0 0.0 0.8518080115318298 0.9097599983215332 0.8810272097587586 0.8751040101051331 0.017005819633271906 512 128 128 1 standard_fused_topk logit_topk softmax_renorm True standalone_legacy vllm020_replicated_linear 8 2048 768 True 4 4096 0.375 32.0 32.0 1.0 0.625 0.1747801353218523 1.53125 6.978069554482723 0.09820938110351562 uniform_random_logits 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 0.9569603278386984 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
58 0.02412799932062626 0.06940799951553345 0.030817600432783365 0.026800000108778477 0.010047085187652939 0.020128000527620316 0.044096000492572784 0.024606400076299904 0.022304000332951546 0.00622857166823714 0.0 0.0 0.0 0.0 0.0 0.8470079898834229 0.9010239839553833 0.8694015920162201 0.8716959953308105 0.016138881171190216 1024 128 128 1 standard_fused_topk logit_topk softmax_renorm True standalone_legacy vllm020_replicated_linear 8 2048 768 True 4 8192 0.375 64.0 64.0 1.0 0.65625 0.1158122428154187 1.3125 6.9901908183358845 0.06445503234863281 uniform_random_logits 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 0.9524274993623046 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
59 0.02831999957561493 0.043487999588251114 0.031744000129401685 0.02991999965161085 0.003973415836428909 0.02070399932563305 0.029343999922275543 0.022886400017887353 0.021743999794125557 0.0026873065045088873 0.0 0.0 0.0 0.0 0.0 1.1698240041732788 1.2311359643936157 1.1888479948043824 1.1890720129013062 0.017978797794492758 2048 128 128 1 standard_fused_topk logit_topk softmax_renorm True standalone_legacy vllm020_replicated_linear 8 2048 768 True 4 16384 0.375 128.0 128.0 1.0 0.78125 0.08347181893108634 1.1796875 6.994921772573154 0.046871185302734375 uniform_random_logits 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 0.8971818172100244 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
60 0.03830400109291077 0.06268800050020218 0.043337599746882914 0.040511999279260635 0.005823016247946116 0.023135999217629433 0.03747199848294258 0.025206399988383053 0.02393599972128868 0.003271381256231161 0.0 0.0 0.0 0.0 0.0 1.7702720165252686 1.8097599744796753 1.7919103980064393 1.7956640124320984 0.012641295676021557 4096 128 128 1 standard_fused_topk logit_topk softmax_renorm True standalone_legacy vllm020_replicated_linear 8 2048 768 True 4 32768 0.375 256.0 256.0 1.0 0.78125 0.06866734477822484 1.20703125 6.996602562938728 0.03801727294921875 uniform_random_logits 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 0.8113207890716184 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
61 0.05660799890756607 0.07932800054550171 0.06249920018017292 0.06039999984204769 0.005636461691480845 0.02956799976527691 0.03747199848294258 0.031126399897038935 0.030287999659776688 0.002157571006631541 0.0 0.0 0.0 0.0 0.0 2.968672037124634 3.0278079509735107 2.9899007797241213 2.9824799299240112 0.01816414122631667 8192 128 128 1 standard_fused_topk logit_topk softmax_renorm True standalone_legacy vllm020_replicated_linear 8 2048 768 True 4 65536 0.375 512.0 512.0 1.0 0.8984375 0.04399546833977376 1.126953125 6.998607314922362 0.024699926376342773 uniform_random_logits 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 0.883909666885606 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
62 0.02502400055527687 0.06092799827456474 0.033839999698102474 0.028672000393271446 0.010315255343709818 0.019360000267624855 0.0352960005402565 0.023424000293016434 0.022064000368118286 0.0038898102289194572 0.0 0.0 0.0 0.0 0.0 0.19574399292469025 0.2512960135936737 0.21939200013875962 0.2199999988079071 0.017532156418212565 1 128 128 1 standard_fused_topk fixed_hotset8 softmax_renorm False standalone_legacy vllm020_replicated_linear 8 2048 768 True 4 8 0.375 0.0625 0.0625 0.0625 0.0 3.872983346207417 16.0 3.0 0.9375 hotset8 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 1.0233364439829928 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
63 0.025280000641942024 0.05132799968123436 0.030641599837690593 0.027343999594449997 0.007562409232763304 0.020160000771284103 0.052960000932216644 0.024145600199699403 0.021424000151455402 0.007450198645599985 0.0 0.0 0.0 0.0 0.0 0.20483200252056122 0.24006399512290955 0.22215040028095245 0.22433599829673767 0.00969639786132892 8 128 128 1 standard_fused_topk fixed_hotset8 softmax_renorm False standalone_legacy vllm020_replicated_linear 8 2048 768 True 4 64 0.375 0.5 0.5 0.0625 0.0 3.872983346207417 16.0 3.0 0.9375 hotset8 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 0.9806430689981738 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
64 0.024831999093294144 0.046560000628232956 0.030371200107038022 0.02763199992477894 0.005878487205028602 0.020320000126957893 0.04560000076889992 0.02601920012384653 0.02270400058478117 0.00751129965906799 0.0 0.0 0.0 0.0 0.0 0.20559999346733093 0.24726399779319763 0.22126719802618028 0.22207999974489212 0.01328093478485499 16 128 128 1 standard_fused_topk fixed_hotset8 softmax_renorm False standalone_legacy vllm020_replicated_linear 8 2048 768 True 4 128 0.375 1.0 1.0 0.0625 0.0 3.872983346207417 16.0 3.0 0.9375 hotset8 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 0.9103623678483975 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
65 0.024288000538945198 0.049375999718904495 0.03086080001667142 0.0267359996214509 0.0070864531384997225 0.020479999482631683 0.030912000685930252 0.02274719988927245 0.021359999664127827 0.0029966813650672505 0.0 0.0 0.0 0.0 0.0 0.20003199577331543 0.2301120012998581 0.21453119963407516 0.21598400175571442 0.010402239855151332 32 128 128 1 standard_fused_topk fixed_hotset8 softmax_renorm False standalone_legacy vllm020_replicated_linear 8 2048 768 True 4 256 0.375 2.0 2.0 0.0625 0.0 3.872983346207417 16.0 3.0 0.9375 hotset8 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 0.9610778571819444 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
66 0.02393599972128868 0.04342399910092354 0.029380799923092126 0.026688000187277794 0.0057374782000526574 0.020031999796628952 0.036607999354600906 0.022487999964505435 0.020911999978125095 0.0038135203062103235 0.0 0.0 0.0 0.0 0.0 0.19551999866962433 0.22972799837589264 0.21238719969987868 0.21488000452518463 0.010706095835489097 64 128 128 1 standard_fused_topk fixed_hotset8 softmax_renorm False standalone_legacy vllm020_replicated_linear 8 2048 768 True 4 512 0.375 4.0 4.0 0.0625 0.0 3.872983346207417 16.0 3.0 0.9375 hotset8 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 0.9952941013961014 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
67 0.025407999753952026 0.05510399863123894 0.031430399790406224 0.02798399981111288 0.007335050273982725 0.020479999482631683 0.03561599925160408 0.02275839988142252 0.021551999263465405 0.003545718365165811 0.0 0.0 0.0 0.0 0.0 0.19420799612998962 0.2903999984264374 0.2211231991648674 0.21476799994707108 0.025955008886196004 128 128 128 1 standard_fused_topk fixed_hotset8 softmax_renorm False standalone_legacy vllm020_replicated_linear 8 2048 768 True 4 1024 0.375 8.0 8.0 0.0625 0.0 3.872983346207417 16.0 3.0 0.9375 hotset8 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 0.9273256282883522 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
68 0.0244159996509552 0.05395200103521347 0.03188959984108806 0.026031999848783016 0.00943995927387483 0.02051199972629547 0.036607999354600906 0.023247999791055917 0.02147199958562851 0.003910623837069648 0.0 0.0 0.0 0.0 0.0 0.2290560007095337 0.289247989654541 0.2543327987194061 0.24939200282096863 0.01663236753503115 256 128 128 1 standard_fused_topk fixed_hotset8 softmax_renorm False standalone_legacy vllm020_replicated_linear 8 2048 768 True 4 2048 0.375 16.0 16.0 0.0625 0.0 3.872983346207417 16.0 3.0 0.9375 hotset8 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 1.0380599882396606 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
69 0.024512000381946564 0.04620800167322159 0.02884640023112297 0.026320000179111958 0.005516557583355925 0.02054399996995926 0.03846399858593941 0.023401600029319524 0.021263999864459038 0.0044742944198265374 0.0 0.0 0.0 0.0 0.0 0.30831998586654663 0.36953601241111755 0.3324000000953674 0.3288639932870865 0.018617999572156707 512 128 128 1 standard_fused_topk fixed_hotset8 softmax_renorm False standalone_legacy vllm020_replicated_linear 8 2048 768 True 4 4096 0.375 32.0 32.0 0.0625 0.0 3.872983346207417 16.0 3.0 0.9375 hotset8 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 0.9569603278386984 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
70 0.02412799932062626 0.06940799951553345 0.030817600432783365 0.026800000108778477 0.010047085187652939 0.020128000527620316 0.044096000492572784 0.024606400076299904 0.022304000332951546 0.00622857166823714 0.0 0.0 0.0 0.0 0.0 0.462911993265152 0.5497919917106628 0.4893856018781662 0.4816960096359253 0.023348152887178286 1024 128 128 1 standard_fused_topk fixed_hotset8 softmax_renorm False standalone_legacy vllm020_replicated_linear 8 2048 768 True 4 8192 0.375 64.0 64.0 0.0625 0.0 3.872983346207417 16.0 3.0 0.9375 hotset8 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 0.9524274993623046 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
71 0.02831999957561493 0.043487999588251114 0.031744000129401685 0.02991999965161085 0.003973415836428909 0.02070399932563305 0.029343999922275543 0.022886400017887353 0.021743999794125557 0.0026873065045088873 0.0 0.0 0.0 0.0 0.0 0.7662079930305481 0.8717759847640991 0.788454395532608 0.7744799852371216 0.03174746482604812 2048 128 128 1 standard_fused_topk fixed_hotset8 softmax_renorm False standalone_legacy vllm020_replicated_linear 8 2048 768 True 4 16384 0.375 128.0 128.0 0.0625 0.0 3.872983346207417 16.0 3.0 0.9375 hotset8 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 0.8971818172100244 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
72 0.03830400109291077 0.06268800050020218 0.043337599746882914 0.040511999279260635 0.005823016247946116 0.023135999217629433 0.03747199848294258 0.025206399988383053 0.02393599972128868 0.003271381256231161 0.0 0.0 0.0 0.0 0.0 1.363935947418213 1.4143040180206299 1.3812703967094422 1.3798720240592957 0.014770450075530007 4096 128 128 1 standard_fused_topk fixed_hotset8 softmax_renorm False standalone_legacy vllm020_replicated_linear 8 2048 768 True 4 32768 0.375 256.0 256.0 0.0625 0.0 3.872983346207417 16.0 3.0 0.9375 hotset8 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 0.8113207890716184 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
73 0.05660799890756607 0.07932800054550171 0.06249920018017292 0.06039999984204769 0.005636461691480845 0.02956799976527691 0.03747199848294258 0.031126399897038935 0.030287999659776688 0.002157571006631541 0.0 0.0 0.0 0.0 0.0 2.5507519245147705 2.680704116821289 2.579859209060669 2.566223978996277 0.03673283558365955 8192 128 128 1 standard_fused_topk fixed_hotset8 softmax_renorm False standalone_legacy vllm020_replicated_linear 8 2048 768 True 4 65536 0.375 512.0 512.0 0.0625 0.0 3.872983346207417 16.0 3.0 0.9375 hotset8 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 0.883909666885606 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize