gahow/aituner
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Freeze long-context Frontier profile

Browse Source
This commit is contained in:
Gahow Wang
2026-07-17 10:30:37 +08:00
parent b6ef6eeae7
commit 10afe4e2e8
7 changed files with 1372 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

824
runs/frontier-fidelity-envelope-v1/profiles/profile-v4-trace-final/allreduce.json Normal file
View File

@@ -0,0 +1,824 @@
{
"environment": [
{
"backend_env": {
"VLLM_ALLREDUCE_USE_FLASHINFER": "1",
"VLLM_FLASHINFER_ALLREDUCE_BACKEND": "trtllm"
},
"gpu": "NVIDIA H20",
"model": "/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B",
"torch_cuda": "12.9",
"torch_version": "2.11.0+cu129",
"vllm_source_commit": "88d34c6409e9fb3c7b8ca0c04756f061d2099eb1",
"vllm_version": "0.20.0"
},
{
"backend_env": {
"VLLM_ALLREDUCE_USE_FLASHINFER": "1",
"VLLM_FLASHINFER_ALLREDUCE_BACKEND": "trtllm"
},
"gpu": "NVIDIA H20",
"model": "/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B",
"torch_cuda": "12.9",
"torch_version": "2.11.0+cu129",
"vllm_source_commit": "88d34c6409e9fb3c7b8ca0c04756f061d2099eb1",
"vllm_version": "0.20.0"
}
],
"frontier_consumption": "diagnostic_only_in_base_profile_only_run; measured lookup requires a separate CC-backend injection ablation",
"rows": [
{
"communicator": "vllm.tensor_model_parallel_all_reduce",
"critical_path_median_ms": 0.08288000151515007,
"dtype": "bfloat16",
"hidden_dim": 2048,
"num_tokens": 1,
"payload_bytes": 4096,
"per_rank_time_ms": [
{
"max": 0.4872959852218628,
"mean": 0.1310015980154276,
"median": 0.07679999992251396,
"min": 0.06217600032687187,
"std": 0.12790721677293843
},
{
"max": 0.4402880072593689,
"mean": 0.12842560112476348,
"median": 0.08288000151515007,
"min": 0.0655680000782013,
"std": 0.11220176524616535
}
],
"selected_backend": "flashinfer_trtllm",
"tensor_parallel_size": 2
},
{
"communicator": "vllm.tensor_model_parallel_all_reduce",
"critical_path_median_ms": 0.0793600007891655,
"dtype": "bfloat16",
"hidden_dim": 2048,
"num_tokens": 8,
"payload_bytes": 32768,
"per_rank_time_ms": [
{
"max": 0.12716799974441528,
"mean": 0.07871360033750534,
"median": 0.0759200006723404,
"min": 0.06032000109553337,
"std": 0.019331314939874535
},
{
"max": 0.12380799651145935,
"mean": 0.08059840016067028,
"median": 0.0793600007891655,
"min": 0.06217600032687187,
"std": 0.01687088356254092
}
],
"selected_backend": "flashinfer_trtllm",
"tensor_parallel_size": 2
},
{
"communicator": "vllm.tensor_model_parallel_all_reduce",
"critical_path_median_ms": 0.0713919997215271,
"dtype": "bfloat16",
"hidden_dim": 2048,
"num_tokens": 16,
"payload_bytes": 65536,
"per_rank_time_ms": [
{
"max": 0.12697599828243256,
"mean": 0.0767391998320818,
"median": 0.07078400254249573,
"min": 0.05910399928689003,
"std": 0.018775178979463278
},
{
"max": 0.11430399864912033,
"mean": 0.07594559974968433,
"median": 0.0713919997215271,
"min": 0.06124800071120262,
"std": 0.015719922150631036
}
],
"selected_backend": "flashinfer_trtllm",
"tensor_parallel_size": 2
},
{
"communicator": "vllm.tensor_model_parallel_all_reduce",
"critical_path_median_ms": 0.08056000247597694,
"dtype": "bfloat16",
"hidden_dim": 2048,
"num_tokens": 32,
"payload_bytes": 131072,
"per_rank_time_ms": [
{
"max": 0.1037760004401207,
"mean": 0.07954559996724128,
"median": 0.08056000247597694,
"min": 0.05955199897289276,
"std": 0.0135697420393132
},
{
"max": 0.10608000308275223,
"mean": 0.07971520014107228,
"median": 0.07593599706888199,
"min": 0.06028800085186958,
"std": 0.015319849772775456
}
],
"selected_backend": "flashinfer_trtllm",
"tensor_parallel_size": 2
},
{
"communicator": "vllm.tensor_model_parallel_all_reduce",
"critical_path_median_ms": 0.0865279994904995,
"dtype": "bfloat16",
"hidden_dim": 2048,
"num_tokens": 64,
"payload_bytes": 262144,
"per_rank_time_ms": [
{
"max": 0.14470399916172028,
"mean": 0.09121599942445754,
"median": 0.0865279994904995,
"min": 0.06441599875688553,
"std": 0.024893837894277456
},
{
"max": 0.12438400089740753,
"mean": 0.08531199917197227,
"median": 0.08031999692320824,
"min": 0.06364800035953522,
"std": 0.01878029830059533
}
],
"selected_backend": "flashinfer_trtllm",
"tensor_parallel_size": 2
},
{
"communicator": "vllm.tensor_model_parallel_all_reduce",
"critical_path_median_ms": 0.07135999947786331,
"dtype": "bfloat16",
"hidden_dim": 2048,
"num_tokens": 128,
"payload_bytes": 524288,
"per_rank_time_ms": [
{
"max": 0.11753600090742111,
"mean": 0.07606079950928687,
"median": 0.07135999947786331,
"min": 0.05843200162053108,
"std": 0.01755519771639284
},
{
"max": 0.1103999987244606,
"mean": 0.07607359997928143,
"median": 0.07073600217700005,
"min": 0.05721599981188774,
"std": 0.016904445220949783
}
],
"selected_backend": "flashinfer_trtllm",
"tensor_parallel_size": 2
},
{
"communicator": "vllm.tensor_model_parallel_all_reduce",
"critical_path_median_ms": 0.07321599870920181,
"dtype": "bfloat16",
"hidden_dim": 2048,
"num_tokens": 256,
"payload_bytes": 1048576,
"per_rank_time_ms": [
{
"max": 0.11740799993276596,
"mean": 0.07749119997024537,
"median": 0.07203200086951256,
"min": 0.05862399935722351,
"std": 0.017701381594822835
},
{
"max": 0.11382400244474411,
"mean": 0.07733759954571724,
"median": 0.07321599870920181,
"min": 0.059039998799562454,
"std": 0.017188890882557036
}
],
"selected_backend": "flashinfer_trtllm",
"tensor_parallel_size": 2
},
{
"communicator": "vllm.tensor_model_parallel_all_reduce",
"critical_path_median_ms": 0.09025600180029869,
"dtype": "bfloat16",
"hidden_dim": 2048,
"num_tokens": 512,
"payload_bytes": 2097152,
"per_rank_time_ms": [
{
"max": 0.13980799913406372,
"mean": 0.0950367994606495,
"median": 0.09025600180029869,
"min": 0.06815999746322632,
"std": 0.022829835127539378
},
{
"max": 0.14764800667762756,
"mean": 0.09710080176591873,
"median": 0.08720000088214874,
"min": 0.07152000069618225,
"std": 0.02500574954063789
}
],
"selected_backend": "flashinfer_trtllm",
"tensor_parallel_size": 2
},
{
"communicator": "vllm.tensor_model_parallel_all_reduce",
"critical_path_median_ms": 0.08083200082182884,
"dtype": "bfloat16",
"hidden_dim": 2048,
"num_tokens": 1024,
"payload_bytes": 4194304,
"per_rank_time_ms": [
{
"max": 0.15574400126934052,
"mean": 0.08760640025138855,
"median": 0.07846399769186974,
"min": 0.07097599655389786,
"std": 0.024487311423551025
},
{
"max": 0.16284799575805664,
"mean": 0.08963519930839539,
"median": 0.08083200082182884,
"min": 0.07100799679756165,
"std": 0.026047320562445356
}
],
"selected_backend": "flashinfer_trtllm",
"tensor_parallel_size": 2
},
{
"communicator": "vllm.tensor_model_parallel_all_reduce",
"critical_path_median_ms": 0.10891199856996536,
"dtype": "bfloat16",
"hidden_dim": 2048,
"num_tokens": 2048,
"payload_bytes": 8388608,
"per_rank_time_ms": [
{
"max": 0.1582079976797104,
"mean": 0.11541439890861512,
"median": 0.10836799815297127,
"min": 0.09062399715185165,
"std": 0.018285136316576037
},
{
"max": 0.1578879952430725,
"mean": 0.11537599861621857,
"median": 0.10891199856996536,
"min": 0.0960640013217926,
"std": 0.018246538626977286
}
],
"selected_backend": "flashinfer_trtllm",
"tensor_parallel_size": 2
},
{
"communicator": "vllm.tensor_model_parallel_all_reduce",
"critical_path_median_ms": 0.1703840047121048,
"dtype": "bfloat16",
"hidden_dim": 2048,
"num_tokens": 4096,
"payload_bytes": 16777216,
"per_rank_time_ms": [
{
"max": 0.19327999651432037,
"mean": 0.1707327976822853,
"median": 0.1703840047121048,
"min": 0.14815999567508698,
"std": 0.014211056022719618
},
{
"max": 0.19276799261569977,
"mean": 0.1658592015504837,
"median": 0.16379200667142868,
"min": 0.14560000598430634,
"std": 0.013840249648693638
}
],
"selected_backend": "flashinfer_trtllm",
"tensor_parallel_size": 2
},
{
"communicator": "vllm.tensor_model_parallel_all_reduce",
"critical_path_median_ms": 0.25539200007915497,
"dtype": "bfloat16",
"hidden_dim": 2048,
"num_tokens": 8192,
"payload_bytes": 33554432,
"per_rank_time_ms": [
{
"max": 0.2807359993457794,
"mean": 0.25750079900026324,
"median": 0.25539200007915497,
"min": 0.24624000489711761,
"std": 0.008925204570802302
},
{
"max": 0.2863999903202057,
"mean": 0.2585055992007256,
"median": 0.255280002951622,
"min": 0.24371199309825897,
"std": 0.012059738582656496
}
],
"selected_backend": "flashinfer_trtllm",
"tensor_parallel_size": 2
},
{
"communicator": "vllm.tensor_model_parallel_all_reduce",
"critical_path_median_ms": 0.1021759994328022,
"dtype": "bfloat16",
"hidden_dim": 2048,
"num_tokens": 1,
"payload_bytes": 4096,
"per_rank_time_ms": [
{
"max": 0.9770879745483398,
"mean": 0.1974783968180418,
"median": 0.10099200159311295,
"min": 0.05913599953055382,
"std": 0.2660581738745569
},
{
"max": 0.892799973487854,
"mean": 0.18164799660444259,
"median": 0.1021759994328022,
"min": 0.06435199826955795,
"std": 0.23947520188197013
},
{
"max": 0.6467199921607971,
"mean": 0.15839359983801843,
"median": 0.10100800171494484,
"min": 0.06800000369548798,
"std": 0.16617013141866102
},
{
"max": 0.6725760102272034,
"mean": 0.15686400160193442,
"median": 0.10044800117611885,
"min": 0.06063999980688095,
"std": 0.17523222161300497
}
],
"selected_backend": "flashinfer_trtllm",
"tensor_parallel_size": 4
},
{
"communicator": "vllm.tensor_model_parallel_all_reduce",
"critical_path_median_ms": 0.12694399803876877,
"dtype": "bfloat16",
"hidden_dim": 2048,
"num_tokens": 8,
"payload_bytes": 32768,
"per_rank_time_ms": [
{
"max": 0.6659520268440247,
"mean": 0.1927648030221462,
"median": 0.12694399803876877,
"min": 0.07609599828720093,
"std": 0.16697784531907736
},
{
"max": 0.695360004901886,
"mean": 0.19356480240821838,
"median": 0.11726400256156921,
"min": 0.0796160027384758,
"std": 0.17588096678867862
},
{
"max": 0.5939840078353882,
"mean": 0.1868800014257431,
"median": 0.12379200011491776,
"min": 0.07427199929952621,
"std": 0.1463231714943902
},
{
"max": 0.6635839939117432,
"mean": 0.1874335989356041,
"median": 0.12014400213956833,
"min": 0.07526399940252304,
"std": 0.16680959677760304
}
],
"selected_backend": "flashinfer_trtllm",
"tensor_parallel_size": 4
},
{
"communicator": "vllm.tensor_model_parallel_all_reduce",
"critical_path_median_ms": 0.09161599725484848,
"dtype": "bfloat16",
"hidden_dim": 2048,
"num_tokens": 16,
"payload_bytes": 65536,
"per_rank_time_ms": [
{
"max": 0.12665599584579468,
"mean": 0.09443839862942696,
"median": 0.09124799817800522,
"min": 0.06431999802589417,
"std": 0.0205690155775906
},
{
"max": 0.1303039938211441,
"mean": 0.09712959825992584,
"median": 0.09161599725484848,
"min": 0.07648000121116638,
"std": 0.019072048129173236
},
{
"max": 0.13836799561977386,
"mean": 0.09821119979023933,
"median": 0.09148800000548363,
"min": 0.0727040022611618,
"std": 0.021314066545189116
},
{
"max": 0.12992000579833984,
"mean": 0.09272959977388381,
"median": 0.08931199833750725,
"min": 0.06406400352716446,
"std": 0.021360803830354765
}
],
"selected_backend": "flashinfer_trtllm",
"tensor_parallel_size": 4
},
{
"communicator": "vllm.tensor_model_parallel_all_reduce",
"critical_path_median_ms": 0.08580800145864487,
"dtype": "bfloat16",
"hidden_dim": 2048,
"num_tokens": 32,
"payload_bytes": 131072,
"per_rank_time_ms": [
{
"max": 0.13449600338935852,
"mean": 0.08960640132427215,
"median": 0.08299200236797333,
"min": 0.06796800345182419,
"std": 0.020633597898445998
},
{
"max": 0.14735999703407288,
"mean": 0.09248319901525974,
"median": 0.08580800145864487,
"min": 0.05913599953055382,
"std": 0.025960234928829564
},
{
"max": 0.13705599308013916,
"mean": 0.08947199806571007,
"median": 0.08460799977183342,
"min": 0.0634239986538887,
"std": 0.02122838946992339
},
{
"max": 0.13846400380134583,
"mean": 0.0859104000031948,
"median": 0.08308799937367439,
"min": 0.05974400043487549,
"std": 0.022373631424433445
}
],
"selected_backend": "flashinfer_trtllm",
"tensor_parallel_size": 4
},
{
"communicator": "vllm.tensor_model_parallel_all_reduce",
"critical_path_median_ms": 0.09867199882864952,
"dtype": "bfloat16",
"hidden_dim": 2048,
"num_tokens": 64,
"payload_bytes": 262144,
"per_rank_time_ms": [
{
"max": 0.1279039978981018,
"mean": 0.09748799875378608,
"median": 0.09678399935364723,
"min": 0.0655359998345375,
"std": 0.02315719144614622
},
{
"max": 0.1356479972600937,
"mean": 0.10018239840865135,
"median": 0.09532799944281578,
"min": 0.06185600161552429,
"std": 0.02330700253650042
},
{
"max": 0.13142399489879608,
"mean": 0.09778879955410957,
"median": 0.09492799639701843,
"min": 0.06560000032186508,
"std": 0.02307579212430045
},
{
"max": 0.1276479959487915,
"mean": 0.09611519873142242,
"median": 0.09867199882864952,
"min": 0.0642239972949028,
"std": 0.02273612181973431
}
],
"selected_backend": "flashinfer_trtllm",
"tensor_parallel_size": 4
},
{
"communicator": "vllm.tensor_model_parallel_all_reduce",
"critical_path_median_ms": 0.09646400064229965,
"dtype": "bfloat16",
"hidden_dim": 2048,
"num_tokens": 128,
"payload_bytes": 524288,
"per_rank_time_ms": [
{
"max": 0.14060799777507782,
"mean": 0.09770880043506622,
"median": 0.09115200117230415,
"min": 0.06950400024652481,
"std": 0.024752645089849798
},
{
"max": 0.14377599954605103,
"mean": 0.09824960008263588,
"median": 0.08999999985098839,
"min": 0.07103999704122543,
"std": 0.025403407389046027
},
{
"max": 0.13680000603199005,
"mean": 0.09993600100278854,
"median": 0.09646400064229965,
"min": 0.06790400296449661,
"std": 0.022297424273985882
},
{
"max": 0.1391039937734604,
"mean": 0.09769919961690902,
"median": 0.09601600095629692,
"min": 0.06835199892520905,
"std": 0.023899922100804445
}
],
"selected_backend": "flashinfer_trtllm",
"tensor_parallel_size": 4
},
{
"communicator": "vllm.tensor_model_parallel_all_reduce",
"critical_path_median_ms": 0.08377600088715553,
"dtype": "bfloat16",
"hidden_dim": 2048,
"num_tokens": 256,
"payload_bytes": 1048576,
"per_rank_time_ms": [
{
"max": 0.1464959979057312,
"mean": 0.09276160076260567,
"median": 0.08377600088715553,
"min": 0.06777600198984146,
"std": 0.024751086465295658
},
{
"max": 0.14319999516010284,
"mean": 0.09080640003085136,
"median": 0.080400001257658,
"min": 0.06796800345182419,
"std": 0.023365718881708488
},
{
"max": 0.1382399946451187,
"mean": 0.09063360020518303,
"median": 0.08193599805235863,
"min": 0.06627199798822403,
"std": 0.02372618650854302
},
{
"max": 0.14313599467277527,
"mean": 0.09044799953699112,
"median": 0.08128000050783157,
"min": 0.06652799993753433,
"std": 0.02462486862033686
}
],
"selected_backend": "flashinfer_trtllm",
"tensor_parallel_size": 4
},
{
"communicator": "vllm.tensor_model_parallel_all_reduce",
"critical_path_median_ms": 0.1128000020980835,
"dtype": "bfloat16",
"hidden_dim": 2048,
"num_tokens": 512,
"payload_bytes": 2097152,
"per_rank_time_ms": [
{
"max": 0.1600320041179657,
"mean": 0.11426239982247352,
"median": 0.1128000020980835,
"min": 0.07657600194215775,
"std": 0.030240087702350687
},
{
"max": 0.15881599485874176,
"mean": 0.11206399947404862,
"median": 0.10628800094127655,
"min": 0.0772159993648529,
"std": 0.029933135345483627
},
{
"max": 0.15612800419330597,
"mean": 0.10761600062251091,
"median": 0.09860799834132195,
"min": 0.07689599692821503,
"std": 0.02634237020678647
},
{
"max": 0.15865600109100342,
"mean": 0.11094079986214637,
"median": 0.10979199782013893,
"min": 0.07583999633789062,
"std": 0.028619217028542445
}
],
"selected_backend": "flashinfer_trtllm",
"tensor_parallel_size": 4
},
{
"communicator": "vllm.tensor_model_parallel_all_reduce",
"critical_path_median_ms": 0.08755199983716011,
"dtype": "bfloat16",
"hidden_dim": 2048,
"num_tokens": 1024,
"payload_bytes": 4194304,
"per_rank_time_ms": [
{
"max": 0.11999999731779099,
"mean": 0.0843871995806694,
"median": 0.08755199983716011,
"min": 0.06332799792289734,
"std": 0.016638543890716024
},
{
"max": 0.1218239963054657,
"mean": 0.08518079966306687,
"median": 0.08032000064849854,
"min": 0.06393600255250931,
"std": 0.019900899429956945
},
{
"max": 0.11849600076675415,
"mean": 0.0843968003988266,
"median": 0.08702399954199791,
"min": 0.06297600269317627,
"std": 0.017031908773433545
},
{
"max": 0.12300799787044525,
"mean": 0.0846304003149271,
"median": 0.08139199763536453,
"min": 0.06195199862122536,
"std": 0.020106523698622265
}
],
"selected_backend": "nccl_fallback",
"tensor_parallel_size": 4
},
{
"communicator": "vllm.tensor_model_parallel_all_reduce",
"critical_path_median_ms": 0.12361599877476692,
"dtype": "bfloat16",
"hidden_dim": 2048,
"num_tokens": 2048,
"payload_bytes": 8388608,
"per_rank_time_ms": [
{
"max": 0.17155200242996216,
"mean": 0.12168959975242614,
"median": 0.12055999785661697,
"min": 0.09609600156545639,
"std": 0.022565485532483928
},
{
"max": 0.9246399998664856,
"mean": 0.19978560134768486,
"median": 0.12361599877476692,
"min": 0.09715200215578079,
"std": 0.24230694305662265
},
{
"max": 0.9317439794540405,
"mean": 0.20037759989500045,
"median": 0.12327999994158745,
"min": 0.09603200107812881,
"std": 0.24450903278000383
},
{
"max": 0.9321280121803284,
"mean": 0.19875840097665787,
"median": 0.12230399996042252,
"min": 0.0950080007314682,
"std": 0.24519252220785093
}
],
"selected_backend": "nccl_fallback",
"tensor_parallel_size": 4
},
{
"communicator": "vllm.tensor_model_parallel_all_reduce",
"critical_path_median_ms": 0.20030399411916733,
"dtype": "bfloat16",
"hidden_dim": 2048,
"num_tokens": 4096,
"payload_bytes": 16777216,
"per_rank_time_ms": [
{
"max": 0.3261440098285675,
"mean": 0.20826880186796187,
"median": 0.19974400103092194,
"min": 0.1409280002117157,
"std": 0.051397264855718945
},
{
"max": 0.3248000144958496,
"mean": 0.20548800230026246,
"median": 0.1979840025305748,
"min": 0.141184002161026,
"std": 0.04980002399656717
},
{
"max": 0.32547199726104736,
"mean": 0.21280319690704347,
"median": 0.20030399411916733,
"min": 0.14127999544143677,
"std": 0.05304243085685509
},
{
"max": 0.26047998666763306,
"mean": 0.1969312012195587,
"median": 0.18079999834299088,
"min": 0.14057600498199463,
"std": 0.042174123638424224
}
],
"selected_backend": "nccl_fallback",
"tensor_parallel_size": 4
},
{
"communicator": "vllm.tensor_model_parallel_all_reduce",
"critical_path_median_ms": 0.2924960106611252,
"dtype": "bfloat16",
"hidden_dim": 2048,
"num_tokens": 8192,
"payload_bytes": 33554432,
"per_rank_time_ms": [
{
"max": 0.45052799582481384,
"mean": 0.3013375997543335,
"median": 0.28273600339889526,
"min": 0.21334399282932281,
"std": 0.07214223764527236
},
{
"max": 0.4466240108013153,
"mean": 0.2968191936612129,
"median": 0.27796798944473267,
"min": 0.21241599321365356,
"std": 0.07400678240849741
},
{
"max": 0.3830080032348633,
"mean": 0.2946112036705017,
"median": 0.2924960106611252,
"min": 0.21084800362586975,
"std": 0.05323627615746332
},
{
"max": 0.4609600007534027,
"mean": 0.3054272010922432,
"median": 0.289792001247406,
"min": 0.21062399446964264,
"std": 0.07608482904865672
}
],
"selected_backend": "nccl_fallback",
"tensor_parallel_size": 4
}
],
"schema_version": "qwen30_vllm020_allreduce_frozen.v1"
}

214
runs/frontier-fidelity-envelope-v1/profiles/profile-v4-trace-final/attention.csv Normal file
View File

@@ -0,0 +1,214 @@
time_stats.attn_input_reshape.min,time_stats.attn_input_reshape.max,time_stats.attn_input_reshape.mean,time_stats.attn_input_reshape.median,time_stats.attn_input_reshape.std,time_stats.attn_kv_cache_save.min,time_stats.attn_kv_cache_save.max,time_stats.attn_kv_cache_save.mean,time_stats.attn_kv_cache_save.median,time_stats.attn_kv_cache_save.std,time_stats.attn_prefill.min,time_stats.attn_prefill.max,time_stats.attn_prefill.mean,time_stats.attn_prefill.median,time_stats.attn_prefill.std,time_stats.attn_decode.min,time_stats.attn_decode.max,time_stats.attn_decode.mean,time_stats.attn_decode.median,time_stats.attn_decode.std,time_stats.attn_output_reshape.min,time_stats.attn_output_reshape.max,time_stats.attn_output_reshape.mean,time_stats.attn_output_reshape.median,time_stats.attn_output_reshape.std,n_embd,n_q_head,n_kv_head,block_size,num_tensor_parallel_workers,max_model_len,batch_size,prefill_chunk_size,kv_cache_size,is_prefill,attention_backend,is_mixed_batch,mode,seq_lens,total_tokens,max_seq_len,min_seq_len,avg_seq_len,equal_seq_len,seq_len_variance,seq_len_std,seq_len_cv,is_chunked_prefill_sample,chunk_start_token,chunk_end_token,total_prefill_tokens,profiling_precision,model_arch,quant_signature,measurement_type,is_true_mixed_batch,prefill_seq_lens,prefill_kv_cache_sizes,decode_kv_cache_sizes,num_prefill_seqs,num_decode_seqs,decode_batch_size,total_batch_size,total_decode_tokens,decode_avg_kv_cache_size,batch_composition_ratio,batch_spec,projection_policy
0.0,0.0,0.0,0.0,0.0,0.01414399966597557,0.028863999992609024,0.019705599918961526,0.01771199982613325,0.005157200849836681,0.047968000173568726,0.07046400010585785,0.05810240097343922,0.05810240097343922,0.007477463486041561,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,64,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[64],64,64,64,64.0,True,0.0,0.0,0.0,False,0.0,64.0,64,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q64,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.014399999752640724,0.04947200044989586,0.020412799902260303,0.01635199971497059,0.010107497379722417,0.046560000628232956,0.08323200047016144,0.05587520003318787,0.05587520003318787,0.011126758739503428,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,128,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[128],128,128,128,128.0,True,0.0,0.0,0.0,False,0.0,128.0,128,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.01484800036996603,0.022207999601960182,0.017033600155264138,0.015312000177800655,0.002819235991970241,0.05104000121355057,0.07692799717187881,0.056396800279617305,0.056396800279617305,0.007481982178637539,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,256,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[256],256,256,256,256.0,True,0.0,0.0,0.0,False,0.0,256.0,256,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q256,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.015072000212967396,0.022272000089287758,0.01706880023702979,0.01616000011563301,0.002460889579319197,0.06931199878454208,0.0838719978928566,0.07432000041007995,0.07432000041007995,0.004777766433175866,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,512,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[512],512,512,512,512.0,True,0.0,0.0,0.0,False,0.0,512.0,512,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q512,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.018592000007629395,0.028543999418616295,0.02095999978482723,0.019183999858796597,0.003198175496053494,0.12179200351238251,0.15408000349998474,0.1307712011039257,0.1307712011039257,0.00858807797538298,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,1024,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[1024],1024,1024,1024,1024.0,True,0.0,0.0,0.0,False,0.0,1024.0,1024,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.027775999158620834,0.03385600075125694,0.030131200328469276,0.029680000618100166,0.0021152558103575215,0.32678401470184326,0.3450239896774292,0.33396480381488797,0.33396480381488797,0.0045872424917606375,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,2048,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[2048],2048,2048,2048,2048.0,True,0.0,0.0,0.0,False,0.0,2048.0,2048,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.04438399896025658,0.05084799975156784,0.046540799736976626,0.04531199857592583,0.002277905811237223,1.0959680080413818,1.1151360273361206,1.0999775886535645,1.0999775886535645,0.005694403246120485,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,4096,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[4096],4096,4096,4096,4096.0,True,0.0,0.0,0.0,False,0.0,4096.0,4096,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.078015998005867,0.08691199868917465,0.08114239946007729,0.08019199967384338,0.00292795706334475,4.070400238037109,4.113152027130127,4.087088012695312,4.087088012695312,0.013660567012509554,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,8192,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[8192],8192,8192,8192,8192.0,True,0.0,0.0,0.0,False,0.0,8192.0,8192,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.016063999384641647,0.05167999863624573,0.022115200012922286,0.017583999782800674,0.010340094822340818,0.05196800082921982,0.09011200070381165,0.06328320093452933,0.06328320093452933,0.012557341255467452,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,64,448.0,True,FLASH_ATTN,False,vllm020_batch_spec,[64],64,64,64,64.0,True,0.0,0.0,0.0,True,448.0,512.0,64,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q64s512,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.01583999954164028,0.026623999699950218,0.018927999772131443,0.017376000061631203,0.003514650316260619,0.06681600213050842,0.07993599772453308,0.0725280001759529,0.0725280001759529,0.004343502558613716,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,128,896.0,True,FLASH_ATTN,False,vllm020_batch_spec,[128],128,128,128,128.0,True,0.0,0.0,0.0,True,896.0,1024.0,128,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q128s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.01648000068962574,0.030880000442266464,0.01945280022919178,0.017967999912798405,0.004096211183007485,0.1311360001564026,0.1546880006790161,0.13908160030841826,0.13908160030841826,0.007511906874366178,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,256,1792.0,True,FLASH_ATTN,False,vllm020_batch_spec,[256],256,256,256,256.0,True,0.0,0.0,0.0,True,1792.0,2048.0,256,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q256s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.017855999991297722,0.03558399900794029,0.020851199887692927,0.018559999763965607,0.005235911594130716,0.32950401306152344,0.350271999835968,0.33912960588932034,0.33912960588932034,0.006027400986663648,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,512,3584.0,True,FLASH_ATTN,False,vllm020_batch_spec,[512],512,512,512,512.0,True,0.0,0.0,0.0,True,3584.0,4096.0,512,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q512s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.019328000023961067,0.040608000010252,0.022790400311350822,0.020704000256955624,0.006113051965778337,1.1415679454803467,1.1518720388412476,1.144483208656311,1.144483208656311,0.0032332311374389127,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,1024,7168.0,True,FLASH_ATTN,False,vllm020_batch_spec,[1024],1024,1024,1024,1024.0,True,0.0,0.0,0.0,True,7168.0,8192.0,1024,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q1ks8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.015807999297976494,0.030688000842928886,0.019596799835562707,0.01774400006979704,0.004343384771033462,0.0,0.0,0.0,0.0,0.0,0.049056001007556915,0.07580800354480743,0.05948160067200661,0.05948160067200661,0.009031541471446955,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,0,128.0,False,FLASH_ATTN,False,vllm020_batch_spec,[1],1,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.01603199914097786,0.02486399933695793,0.01923839971423149,0.018079999834299088,0.0032282528537266424,0.0,0.0,0.0,0.0,0.0,0.05142400041222572,0.07353600114583969,0.059328000620007516,0.059328000620007516,0.0073307807735143084,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,8,0,128.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.016543999314308167,0.03977600112557411,0.021379199624061585,0.018511999398469925,0.006593576176246171,0.0,0.0,0.0,0.0,0.0,0.0488319993019104,0.06435199826955795,0.05479039996862411,0.05479039996862411,0.005672522998491864,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,16,0,128.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.01635199971497059,0.02844800055027008,0.019267200119793416,0.017952000722289085,0.0035068687666949577,0.0,0.0,0.0,0.0,0.0,0.049855999648571014,0.07798399776220322,0.05986879989504815,0.05986879989504815,0.01043914754878828,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,32,0,128.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.016383999958634377,0.026079999282956123,0.01923519968986511,0.017791999503970146,0.0032161974331284568,0.0,0.0,0.0,0.0,0.0,0.058111999183893204,0.1045759990811348,0.06708480007946492,0.06708480007946492,0.013479022462646494,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,64,0,128.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.014720000326633453,0.04057599976658821,0.019100800156593323,0.015455999877303839,0.007512281243011577,0.0,0.0,0.0,0.0,0.0,0.05363199859857559,0.07782399654388428,0.06090559959411622,0.06090559959411622,0.007544620176348091,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,8,0,1024.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.014431999996304512,0.02191999927163124,0.016684799920767546,0.01563199982047081,0.0024293118621811216,0.0,0.0,0.0,0.0,0.0,0.0629120022058487,0.07891199737787247,0.06891520097851753,0.06891520097851753,0.005472695665695425,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,16,0,1024.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.014560000039637089,0.038943998515605927,0.018313600029796363,0.01561600062996149,0.007127270260115769,0.0,0.0,0.0,0.0,0.0,0.08675199747085571,0.10662399977445602,0.09391999915242194,0.09391999915242194,0.006988099589086635,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,32,0,1024.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.014527999795973301,0.054687999188899994,0.021439999900758268,0.01539199985563755,0.012052764849597775,0.0,0.0,0.0,0.0,0.0,0.13488000631332397,0.1528639942407608,0.1431359991431236,0.1431359991431236,0.005436271464033599,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,64,0,1024.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.014399999752640724,0.041919998824596405,0.01899839974939823,0.015343999955803156,0.007989843526623287,0.0,0.0,0.0,0.0,0.0,0.06176000088453293,0.08374399691820145,0.06747519969940186,0.06747519969940186,0.0066067747128778,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,8,0,2048.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.016256000846624374,0.11353600025177002,0.042761600017547606,0.028960000723600388,0.029104301538020762,0.0,0.0,0.0,0.0,0.0,0.09734400361776352,0.14422400295734406,0.11392960175871848,0.11392960175871848,0.013198594600417867,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,16,0,2048.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.014688000082969666,0.034143999218940735,0.018918400071561335,0.01643200032413006,0.005500943993080684,0.0,0.0,0.0,0.0,0.0,0.12918399274349213,0.15087999403476715,0.13807999789714814,0.13807999789714814,0.007658330538677587,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,32,0,2048.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.016063999384641647,0.03641600161790848,0.0198208000510931,0.01780799962580204,0.0057264128169845765,0.0,0.0,0.0,0.0,0.0,0.22099199891090393,0.23904000222682953,0.2293503984808922,0.2293503984808922,0.004861342907006028,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,64,0,2048.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.014751999638974667,0.035840000957250595,0.018908800091594458,0.015792000107467175,0.0064374817924757475,0.0,0.0,0.0,0.0,0.0,0.10134399682283401,0.12201599776744843,0.10896319895982742,0.10896319895982742,0.006336330809165179,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,8,0,4096.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.013856000266969204,0.03417599946260452,0.017846399918198586,0.014800000004470348,0.006495007539635255,0.0,0.0,0.0,0.0,0.0,0.13126400113105774,0.15561600029468536,0.1389280006289482,0.1389280006289482,0.008381472811075022,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,16,0,4096.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.014431999996304512,0.03519999980926514,0.019168000388890504,0.01600000075995922,0.005995522477654695,0.0,0.0,0.0,0.0,0.0,0.21728000044822693,0.2343679964542389,0.2231455981731415,0.2231455981731415,0.004720730646739123,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,32,0,4096.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.014047999866306782,0.03670400008559227,0.018441599886864425,0.015023999847471714,0.006596535162793127,0.0,0.0,0.0,0.0,0.0,0.39321601390838623,0.4524799883365631,0.4058080047369003,0.4058080047369003,0.01578349755088941,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,64,0,4096.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.014336000196635723,0.022143999114632607,0.016912000067532063,0.015168000012636185,0.0028156089295136347,0.0,0.0,0.0,0.0,0.0,0.15587200224399567,0.3079040050506592,0.17838079929351805,0.17838079929351805,0.04355865575265927,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,8,0,8192.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.014271999709308147,0.02112000063061714,0.015516800060868263,0.01488000014796853,0.001940870731593904,0.0,0.0,0.0,0.0,0.0,0.21587200462818146,0.23561599850654602,0.22250880002975468,0.22250880002975468,0.006181951170646666,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,16,0,8192.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.015552000142633915,0.039264000952243805,0.023609600123018028,0.02131200022995472,0.007236979625711548,0.0,0.0,0.0,0.0,0.0,0.408735990524292,0.470335990190506,0.4336863994598388,0.4336863994598388,0.01844662383160074,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,32,0,8192.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.014399999752640724,0.025407999753952026,0.016227199975401164,0.014960000291466713,0.0031617375441736185,0.0,0.0,0.0,0.0,0.0,0.7412800192832947,0.7627840042114258,0.7464000046253203,0.7464000046253203,0.006112167448837547,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,64,0,8192.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.01462399959564209,0.02812799997627735,0.020652799773961304,0.021359999664127827,0.004308706957613102,0.028383498565450627,0.039859687970646644,0.032492258074592426,0.032492258074592426,0.00453597266208597,0.029312501176103633,0.04116431058787463,0.03355574193327539,0.03355574193327539,0.004684436757701648,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,9,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,72,,,,,,,,False,,,64,BF16,generic,none,CUDA_EVENT,True,[64],[0],"[512, 512, 512, 512, 512, 512, 512, 512]",1,8,8,9,8,512.0,0.1111111111111111,q64_8q1s512,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
0.0,0.0,0.0,0.0,0.0,0.015552000142633915,0.034143999218940735,0.023171199765056372,0.024255999363958836,0.005908614918096971,0.03333159243114438,0.038935341782478095,0.03580428402241854,0.03580428402241854,0.002082270297044095,0.03633240903369937,0.04244065945634484,0.03902771507087562,0.03902771507087562,0.0022697354261490147,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,9,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,136,,,,,,,,False,,,128,BF16,generic,none,CUDA_EVENT,True,[128],[0],"[1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024]",1,8,8,9,8,1024.0,0.1111111111111111,q128_8q1s1k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
0.0,0.0,0.0,0.0,0.0,0.014655999839305878,0.02611199952661991,0.019635199941694735,0.018112000077962875,0.0038298050749257795,0.04189529417991216,0.057484239920526384,0.04744885718421094,0.04744885718421094,0.004779830748455743,0.051672703037266184,0.07089975418195164,0.05852234134479412,0.05852234134479412,0.005895334539786378,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,17,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,144,,,,,,,,False,,,128,BF16,generic,none,CUDA_EVENT,True,[128],[0],"[1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024]",1,16,16,17,16,1024.0,0.058823529411764705,q128_16q1s1k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
0.0,0.0,0.0,0.0,0.0,0.014720000326633453,0.029311999678611755,0.018662399891763926,0.01673599984496832,0.0044017112162725355,0.04322973959325901,0.05049827064705393,0.04535414343408448,0.04535414343408448,0.0022944156000240697,0.08733025617719538,0.10201372836398578,0.09162185574241775,0.09162185574241775,0.004635047631846023,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,17,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,272,,,,,,,,False,,,256,BF16,generic,none,CUDA_EVENT,True,[256],[0],"[2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048]",1,16,16,17,16,2048.0,0.058823529411764705,q256_16q1s2k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
0.0,0.0,0.0,0.0,0.0,0.014592000283300877,0.026367999613285065,0.016336000058799982,0.01515199989080429,0.003415353455946402,0.06031842775160765,0.06618323188375198,0.06274415549817247,0.06274415549817247,0.001985647289644255,0.14768157653992678,0.16204076249052324,0.15362064543185072,0.15362064543185072,0.004861590945216247,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,33,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,288,,,,,,,,False,,,256,BF16,generic,none,CUDA_EVENT,True,[256],[0],"[2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048]",1,32,32,33,32,2048.0,0.030303030303030304,q256_32q1s2k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
0.0,0.0,0.0,0.0,0.0,0.014751999638974667,0.02454400062561035,0.017167999967932702,0.016191999427974224,0.0028685016454498436,0.09128700688359712,0.09689150775996329,0.09360396051475776,0.09360396051475776,0.0013477542744916764,0.2740889887523892,0.2909164873209846,0.2810456356995366,0.2810456356995366,0.004046628526808561,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,33,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,544,,,,,,,,False,,,512,BF16,generic,none,CUDA_EVENT,True,[512],[0],"[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096]",1,32,32,33,32,4096.0,0.030303030303030304,q512_32q1s4k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
0.0,0.0,0.0,0.0,0.0,0.01881599985063076,0.03097599931061268,0.024598400108516216,0.024848000146448612,0.0037539437391565975,0.1035249255866932,0.10603132147437412,0.10399797220840973,0.10399797220840973,0.0007025109429056814,0.5652750707893444,0.5789606938874117,0.5678580377517648,0.5678580377517648,0.003835906384194897,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,65,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,576,,,,,,,,False,,,512,BF16,generic,none,CUDA_EVENT,True,[512],[0],"[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096]",1,64,64,65,64,4096.0,0.015384615384615385,q512_64q1s4k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
0.0,0.0,0.0,0.0,0.0,0.018624000251293182,0.043487999588251114,0.024460799992084503,0.019600000232458115,0.008922081629781394,0.196169204945307,0.2076140047945799,0.2008444429250931,0.2008444429250931,0.00394350953292805,1.1196708009293268,1.1849940417370974,1.1463555573610091,1.1463555573610091,0.022508285530529783,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,65,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,1088,,,,,,,,False,,,1024,BF16,generic,none,CUDA_EVENT,True,[1024],[0],"[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192]",1,64,64,65,64,8192.0,0.015384615384615385,q1k_64q1s8k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
0.0,0.0,0.0,0.0,0.0,0.027135999873280525,0.03667199984192848,0.02945920005440712,0.028256000019609928,0.0029446678408169553,0.37757279619664613,0.3898113624476372,0.38075520430942184,0.38075520430942184,0.0036600203831042254,0.2522831942132049,0.2604606493092597,0.25440958703617433,0.25440958703617433,0.0024455194930253126,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,33,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,2080,,,,,,,,False,,,2048,BF16,generic,none,CUDA_EVENT,True,[2048],[0],"[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096]",1,32,32,33,32,4096.0,0.030303030303030304,q2k_32q1s4k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
0.0,0.0,0.0,0.0,0.0,0.0435199998319149,0.049695998430252075,0.04502719938755036,0.04395199939608574,0.002035785660169308,1.1048984388245497,1.1189053886476108,1.1112371236754128,1.1112371236754128,0.0050220696724624985,0.1395495077239122,0.14131859607071193,0.14035008841031085,0.14035008841031085,0.0006342911944856293,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,17,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,4112,,,,,,,,False,,,4096,BF16,generic,none,CUDA_EVENT,True,[4096],[0],"[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096]",1,16,16,17,16,4096.0,0.058823529411764705,q4k_16q1s4k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
0.0,0.0,0.0,0.0,0.0,0.018783999606966972,0.024288000538945198,0.020627199858427047,0.019567999988794327,0.0019457052717059358,0.09455999732017517,0.12185599654912949,0.10207359939813614,0.10207359939813614,0.00753346544014261,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,2,1024,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,"[512, 512]",1024,512,512,512.0,True,0.0,0.0,0.0,False,0.0,1024.0,1024,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,2q512,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.027295999228954315,0.034752000123262405,0.02943360023200512,0.028528000228106976,0.0023159160681123767,0.14416000247001648,0.15904000401496887,0.14979200065135959,0.14979200065135959,0.00480624675866005,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,4,2048,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,"[512, 512, 512, 512]",2048,512,512,512.0,True,0.0,0.0,0.0,False,0.0,2048.0,2048,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,4q512,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.043455999344587326,0.05084799975156784,0.045500800386071204,0.04411200061440468,0.002490556062831049,0.24454399943351746,0.25865599513053894,0.2504959970712662,0.2504959970712662,0.003778494140301353,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,8,4096,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,"[512, 512, 512, 512, 512, 512, 512, 512]",4096,512,512,512.0,True,0.0,0.0,0.0,False,0.0,4096.0,4096,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q512,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.07673600316047668,0.08374399691820145,0.07970559895038605,0.07873599976301193,0.0022907976135004057,0.445248007774353,0.4758400022983551,0.45409599840641024,0.45409599840641024,0.009133230340383306,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,16,8192,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,"[512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512]",8192,512,512,512.0,True,0.0,0.0,0.0,False,0.0,8192.0,8192,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q512,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.043296001851558685,0.051072001457214355,0.044972800090909,0.04399999976158142,0.002451216824441962,0.6147199869155884,0.6290879845619202,0.6204927921295166,0.6204927921295166,0.004344846739788608,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,2,4096,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,"[2048, 2048]",4096,2048,2048,2048.0,True,0.0,0.0,0.0,False,0.0,4096.0,4096,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,2q2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.07737600058317184,0.09123200178146362,0.08094720020890236,0.07980800047516823,0.004065264798368611,1.1744320392608643,1.1887680292129517,1.178323209285736,1.178323209285736,0.004395876469319665,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,4,8192,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,"[2048, 2048, 2048, 2048]",8192,2048,2048,2048.0,True,0.0,0.0,0.0,False,0.0,8192.0,8192,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,4q2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.014688000082969666,0.03254399821162224,0.01810879958793521,0.01611199975013733,0.005058478889747962,0.0,0.0,0.0,0.0,0.0,0.06339199841022491,0.0841279998421669,0.07019519805908202,0.07019519805908202,0.006070126182471763,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,0,16384.0,False,FLASH_ATTN,False,vllm020_batch_spec,[1],1,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q1s16k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.01484800036996603,0.030271999537944794,0.018124799989163876,0.015711999498307705,0.004696400814632773,0.0,0.0,0.0,0.0,0.0,0.2739199995994568,0.2922239899635315,0.28281279802322384,0.28281279802322384,0.00633037978599564,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,8,0,16384.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s16k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.014495999552309513,0.022431999444961548,0.016844799742102623,0.015471999999135733,0.0027624803153841917,0.0,0.0,0.0,0.0,0.0,0.3909119963645935,0.4160960018634796,0.39996159672737125,0.39996159672737125,0.007037174636804898,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,16,0,16384.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s16k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.014240000396966934,0.035071998834609985,0.018764800019562246,0.015583999920636415,0.006305594700523817,0.0,0.0,0.0,0.0,0.0,0.7383679747581482,0.7597119808197021,0.7430047929286957,0.7430047929286957,0.006192645960911466,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,32,0,16384.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s16k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.014399999752640724,0.028991999104619026,0.019401599932461978,0.01665600063279271,0.005434953793780546,0.0,0.0,0.0,0.0,0.0,1.427008032798767,1.4517120122909546,1.4361984014511109,1.4361984014511109,0.008802755821028187,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,64,0,16384.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s16k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.014271999709308147,0.02844800055027008,0.01820160010829568,0.014944000169634819,0.004942002036909087,0.0,0.0,0.0,0.0,0.0,0.08361600339412689,0.11027199774980545,0.09058240056037903,0.09058240056037903,0.00872938604423622,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,0,32768.0,False,FLASH_ATTN,False,vllm020_batch_spec,[1],1,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q1s32k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.014112000353634357,0.023104000836610794,0.01648960020393133,0.015632000286132097,0.0027628828340349578,0.0,0.0,0.0,0.0,0.0,0.5063040256500244,0.5497599840164185,0.5168287932872773,0.5168287932872773,0.013557229606313293,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,8,0,32768.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s32k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.01616000011563301,0.029055999591946602,0.018780799955129622,0.017215999774634838,0.003794434947431941,0.0,0.0,0.0,0.0,0.0,0.7439360022544861,0.7719680070877075,0.7502080142498017,0.7502080142498017,0.00839205598262567,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,16,0,32768.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s32k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.015968000516295433,0.03046399913728237,0.018822400271892546,0.01726400014013052,0.004212536831927695,0.0,0.0,0.0,0.0,0.0,1.4256000518798828,1.449504017829895,1.4325888037681578,1.4325888037681578,0.0065448028108711165,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,32,0,32768.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s32k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.014271999709308147,0.027712000533938408,0.01633920017629862,0.014944000169634819,0.0038488220071983326,0.0,0.0,0.0,0.0,0.0,2.798719882965088,3.0184640884399414,2.8293471813201903,2.8293471813201903,0.06375662767704417,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,64,0,32768.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s32k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.014720000326633453,0.021407999098300934,0.01637439979240298,0.014992000069469213,0.0024463594774459265,0.0,0.0,0.0,0.0,0.0,0.09347199648618698,0.10473600029945374,0.09820479974150656,0.09820479974150656,0.004061668684999473,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,0,40960.0,False,FLASH_ATTN,False,vllm020_batch_spec,[1],1,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q1s40k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.014303999952971935,0.04790399968624115,0.020179199893027543,0.016447999514639378,0.009635010283506967,0.0,0.0,0.0,0.0,0.0,0.6221439838409424,0.6444799900054932,0.6277984082698822,0.6277984082698822,0.007234140179397069,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,8,0,40960.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s40k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.014271999709308147,0.03276799991726875,0.01921279989182949,0.015232000034302473,0.0065506148511265076,0.0,0.0,0.0,0.0,0.0,0.9111359715461731,0.9307839870452881,0.9151648044586183,0.9151648044586183,0.005528712062927265,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,16,0,40960.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s40k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.014175999909639359,0.02707199938595295,0.016512000095099212,0.014688000082969666,0.0038989080209321414,0.0,0.0,0.0,0.0,0.0,1.7645119428634644,1.7849279642105103,1.7684095859527589,1.7684095859527589,0.005876963340184193,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,32,0,40960.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s40k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.014240000396966934,0.029184000566601753,0.016825600154697896,0.014752000104635954,0.00457910514148248,0.0,0.0,0.0,0.0,0.0,3.4781761169433594,3.5388801097869873,3.490892815589905,3.490892815589905,0.01818910600692522,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,64,0,40960.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s40k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.014976000413298607,0.03139200061559677,0.01898880014196038,0.016848000697791576,0.005083715524393418,0.13752702814163098,0.14117629917511842,0.13848196486571557,0.13848196486571557,0.0009880564964713527,0.523336967410432,0.5372236808930884,0.5269708253945994,0.5269708253945994,0.00375988994658546,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,9,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,520,,,,,,,,False,,,512,BF16,generic,none,CUDA_EVENT,True,[512],[0],"[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384]",1,8,8,9,8,16384.0,0.1111111111111111,q512_8q1s16k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
0.0,0.0,0.0,0.0,0.0,0.01833599992096424,0.02707199938595295,0.020995199866592883,0.01976000051945448,0.0028392656352050185,0.17817885890237703,0.18622915300900206,0.18109068484526028,0.18109068484526028,0.0027040006080457624,0.5449571488834201,0.5695788427872015,0.5538629212834322,0.5538629212834322,0.008270141985514729,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,17,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,1040,,,,,,,,False,,,1024,BF16,generic,none,CUDA_EVENT,True,[1024],[0],"[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384]",1,16,16,17,16,16384.0,0.058823529411764705,q1k_16q1s16k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
0.0,0.0,0.0,0.0,0.0,0.02703999914228916,0.03308799862861633,0.028883199393749236,0.02759999968111515,0.002285758578932753,0.4688266550410815,0.48301665772804186,0.4727750380198454,0.4727750380198454,0.004396396389258256,1.0430453981052807,1.0746153117238624,1.051829759343436,1.051829759343436,0.009781101336187197,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,33,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,2080,,,,,,,,False,,,2048,BF16,generic,none,CUDA_EVENT,True,[2048],[0],"[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384]",1,32,32,33,32,16384.0,0.030303030303030304,q2k_32q1s16k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
0.0,0.0,0.0,0.0,0.0,0.04368000105023384,0.059328000992536545,0.04809600040316582,0.04617599956691265,0.005158792915002645,1.3509461459747514,1.3759066409666496,1.3589365122072,1.3589365122072,0.008008877716183384,0.9213738861449998,0.9383974724214122,0.9268234851606594,0.9268234851606594,0.005462224239661061,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,17,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,4112,,,,,,,,False,,,4096,BF16,generic,none,CUDA_EVENT,True,[4096],[0],"[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768]",1,16,16,17,16,32768.0,0.058823529411764705,q4k_16q1s32k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
0.0,0.0,0.0,0.0,0.0,0.026944000273942947,0.040511999279260635,0.030291200056672095,0.02817599941045046,0.004386386489305873,0.5071830964059985,0.5142792136001758,0.5096392092471257,0.5096392092471257,0.0022059272685869546,2.175632932188972,2.2060727208328075,2.186168772243963,2.186168772243963,0.009462633998570079,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,33,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,2080,,,,,,,,False,,,2048,BF16,generic,none,CUDA_EVENT,True,[2048],[0],"[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768]",1,32,32,33,32,32768.0,0.030303030303030304,q2k_32q1s32k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
0.0,0.0,0.0,0.0,0.0,0.018303999677300453,0.03017600066959858,0.02039040010422468,0.018943999893963337,0.0034612051983613614,0.21341429693945616,0.21552776300295579,0.21393496609766816,0.21393496609766816,0.0005736773262009222,4.617401495144284,4.663128147488988,4.628666619290515,4.628666619290515,0.012412001359412127,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,65,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,1088,,,,,,,,False,,,1024,BF16,generic,none,CUDA_EVENT,True,[1024],[0],"[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768]",1,64,64,65,64,32768.0,0.015384615384615385,q1k_64q1s32k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
0.0,0.0,0.0,0.0,0.0,0.014720000326633453,0.02364799939095974,0.01809599995613098,0.016352000646293163,0.0035481127058959038,0.04822399839758873,0.08566399663686752,0.05961279980838299,0.05961279980838299,0.011445665413968877,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,64,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[64],64,64,64,64.0,True,0.0,0.0,0.0,False,0.0,64.0,64,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q64,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.014175999909639359,0.020864000543951988,0.01668160008266568,0.015664000064134598,0.0025769063833097584,0.049695998430252075,0.08057600259780884,0.05882879942655563,0.05882879942655563,0.009515126108519331,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,128,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[128],128,128,128,128.0,True,0.0,0.0,0.0,False,0.0,128.0,128,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.014399999752640724,0.028704000636935234,0.017430400010198355,0.015056000091135502,0.004349294613335555,0.049855999648571014,0.07366400212049484,0.05459520071744919,0.05459520071744919,0.0069610920757925574,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,256,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[256],256,256,256,256.0,True,0.0,0.0,0.0,False,0.0,256.0,256,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q256,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.014336000196635723,0.03161599859595299,0.017788799852132796,0.015711999963968992,0.005005385723318659,0.06102399900555611,0.08179199695587158,0.06650560013949873,0.06650560013949873,0.006947995595801105,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,512,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[512],512,512,512,512.0,True,0.0,0.0,0.0,False,0.0,512.0,512,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q512,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.014655999839305878,0.04416000097990036,0.019200000166893005,0.015488000120967627,0.008561241323364038,0.08054400235414505,0.09196799993515015,0.08607039973139763,0.08607039973139763,0.004145329035483754,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,1024,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[1024],1024,1024,1024,1024.0,True,0.0,0.0,0.0,False,0.0,1024.0,1024,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.017855999991297722,0.023391999304294586,0.019667199812829494,0.018463999964296818,0.0022577669687832585,0.18729600310325623,0.20585599541664124,0.19546559900045393,0.19546559900045393,0.006339068824303663,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,2048,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[2048],2048,2048,2048,2048.0,True,0.0,0.0,0.0,False,0.0,2048.0,2048,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.026240000501275063,0.03446400165557861,0.028297600522637367,0.027088000439107418,0.0025148441522922374,0.5754240155220032,0.5889919996261597,0.5800191938877105,0.5800191938877105,0.003858869273829596,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,4096,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[4096],4096,4096,4096,4096.0,True,0.0,0.0,0.0,False,0.0,4096.0,4096,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.04182400181889534,0.047200001776218414,0.043036799877882004,0.0423360001295805,0.0017073405772076728,2.063199996948242,2.0787200927734375,2.067151999473572,2.067151999473572,0.004959651271127835,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,8192,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[8192],8192,8192,8192,8192.0,True,0.0,0.0,0.0,False,0.0,8192.0,8192,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.014399999752640724,0.05648000165820122,0.02270399993285537,0.017935999669134617,0.012377915150727689,0.049536000937223434,0.07196799665689468,0.056015999615192415,0.056015999615192415,0.0070476637552742884,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,64,448.0,True,FLASH_ATTN,False,vllm020_batch_spec,[64],64,64,64,64.0,True,0.0,0.0,0.0,True,448.0,512.0,64,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q64s512,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.01408000010997057,0.021344000473618507,0.01611520005390048,0.014928000047802925,0.0025499884993961702,0.07072000205516815,0.2642880082130432,0.1588256008923054,0.1588256008923054,0.053220347086102376,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,128,896.0,True,FLASH_ATTN,False,vllm020_batch_spec,[128],128,128,128,128.0,True,0.0,0.0,0.0,True,896.0,1024.0,128,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q128s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.01360000018030405,0.03014400042593479,0.016975999902933837,0.015056000091135502,0.004715159697364111,0.08393599838018417,0.11036799848079681,0.09160000011324881,0.09160000011324881,0.007911669434472792,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,256,1792.0,True,FLASH_ATTN,False,vllm020_batch_spec,[256],256,256,256,256.0,True,0.0,0.0,0.0,True,1792.0,2048.0,256,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q256s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.014495999552309513,0.020767999812960625,0.016233599931001663,0.015392000321298838,0.002202615907995427,0.1998399943113327,0.22070400416851044,0.20855360180139543,0.20855360180139543,0.00689307200230667,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,512,3584.0,True,FLASH_ATTN,False,vllm020_batch_spec,[512],512,512,512,512.0,True,0.0,0.0,0.0,True,3584.0,4096.0,512,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q512s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.01484800036996603,0.033440001308918,0.018684800155460833,0.016048000194132328,0.00553991005639174,0.6043199896812439,0.635807991027832,0.6126143991947173,0.6126143991947173,0.008745933953408096,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,1024,7168.0,True,FLASH_ATTN,False,vllm020_batch_spec,[1024],1024,1024,1024,1024.0,True,0.0,0.0,0.0,True,7168.0,8192.0,1024,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q1ks8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.013824000023305416,0.03359999880194664,0.017811199743300678,0.014944000169634819,0.005926140483565472,0.0,0.0,0.0,0.0,0.0,0.045471999794244766,0.07539200037717819,0.054758400097489356,0.054758400097489356,0.010253548506101549,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,0,128.0,False,FLASH_ATTN,False,vllm020_batch_spec,[1],1,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.014047999866306782,0.02409599907696247,0.01641279999166727,0.01473599998280406,0.003347158638713987,0.0,0.0,0.0,0.0,0.0,0.0461760014295578,0.07529599964618683,0.05460800044238568,0.05460800044238568,0.009748937798340135,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,8,0,128.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.014271999709308147,0.028416000306606293,0.018611199874430894,0.01598400017246604,0.005209875093863647,0.0,0.0,0.0,0.0,0.0,0.048128001391887665,0.07897599786520004,0.061353600397706036,0.061353600397706036,0.010488153655157845,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,16,0,128.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.014112000353634357,0.025728000327944756,0.016092800162732603,0.01512000011280179,0.003300888123052605,0.0,0.0,0.0,0.0,0.0,0.04864000156521797,0.07648000121116638,0.05810560062527656,0.05810560062527656,0.009473544218404408,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,32,0,128.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.014655999839305878,0.03551999852061272,0.018588799890130757,0.016064000315964222,0.006071057437992447,0.0,0.0,0.0,0.0,0.0,0.04822399839758873,0.07862400263547897,0.05660480037331582,0.05660480037331582,0.009565394213730401,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,64,0,128.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.014303999952971935,0.028831999748945236,0.01699519995599985,0.015024000313133001,0.004285000744868188,0.0,0.0,0.0,0.0,0.0,0.04854400083422661,0.06719999760389328,0.05778240002691746,0.05778240002691746,0.006852125554679805,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,8,0,1024.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.0144640002399683,0.024927999824285507,0.0161183999851346,0.01521599991247058,0.0029774808524673907,0.0,0.0,0.0,0.0,0.0,0.05379199981689453,0.08966399729251862,0.06270079985260964,0.06270079985260964,0.009983591277092696,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,16,0,1024.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.014431999996304512,0.03001599945127964,0.017648000083863736,0.01547200046479702,0.004585126309484848,0.0,0.0,0.0,0.0,0.0,0.061664000153541565,0.07692799717187881,0.06704320013523103,0.06704320013523103,0.005500191798490629,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,32,0,1024.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.014175999909639359,0.026367999613285065,0.016947199776768684,0.015039999969303608,0.0037951566103550205,0.0,0.0,0.0,0.0,0.0,0.08799999952316284,0.111455999314785,0.0964031994342804,0.0964031994342804,0.007397541615558088,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,64,0,1024.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.01369599997997284,0.021247999742627144,0.015359999984502793,0.014640000183135271,0.0020942770950814317,0.0,0.0,0.0,0.0,0.0,0.051711998879909515,0.07065600156784058,0.058054400235414506,0.058054400235414506,0.006633034815910025,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,8,0,2048.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.014208000153303146,0.0307839997112751,0.017148799914866685,0.015039999969303608,0.004882374686312284,0.0,0.0,0.0,0.0,0.0,0.061919998377561576,0.07843200117349625,0.06628479920327664,0.06628479920327664,0.004962852689801192,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,16,0,2048.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.013887999579310417,0.02969600073993206,0.017500799987465142,0.015232000034302473,0.00467273319705314,0.0,0.0,0.0,0.0,0.0,0.08819200098514557,0.11097600311040878,0.09493440166115762,0.09493440166115762,0.007509235042985577,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,32,0,2048.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.014399999752640724,0.022112000733613968,0.01751680001616478,0.016047999262809753,0.0030306621792915785,0.0,0.0,0.0,0.0,0.0,0.13065600395202637,0.15110400319099426,0.13857279866933822,0.13857279866933822,0.00750137841249771,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,64,0,2048.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.013919999822974205,0.04012800008058548,0.01892479993402958,0.01550400024279952,0.007459545267816961,0.0,0.0,0.0,0.0,0.0,0.06278400123119354,0.08259200304746628,0.0704512007534504,0.0704512007534504,0.005979055984382744,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,8,0,4096.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.014336000196635723,0.02236800082027912,0.016332800220698118,0.014928000047802925,0.002784044363186731,0.0,0.0,0.0,0.0,0.0,0.1003199964761734,0.1279360055923462,0.10921279862523078,0.10921279862523078,0.00862769617046716,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,16,0,4096.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.013919999822974205,0.0225600004196167,0.016128000058233737,0.014800000004470348,0.002958953332547708,0.0,0.0,0.0,0.0,0.0,0.13116799294948578,0.14812800288200378,0.13783999979496003,0.13783999979496003,0.005696384361148053,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,32,0,4096.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.014208000153303146,0.029983999207615852,0.017468800116330386,0.01508800033479929,0.0047379550962483065,0.0,0.0,0.0,0.0,0.0,0.217631995677948,0.2447360008955002,0.22715839892625808,0.22715839892625808,0.008831828221847138,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,64,0,4096.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.014047999866306782,0.02304000034928322,0.016512000095099212,0.014512000139802694,0.0035026774828624254,0.0,0.0,0.0,0.0,0.0,0.11020799726247787,0.12307199835777283,0.11600959971547126,0.11600959971547126,0.004667637266950902,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,8,0,8192.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.014112000353634357,0.042080000042915344,0.017510399967432023,0.014607999939471483,0.00823117883530167,0.0,0.0,0.0,0.0,0.0,0.15702399611473083,0.17587199807167053,0.16399359852075576,0.16399359852075576,0.006588299074676393,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,16,0,8192.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.013919999822974205,0.0208320003002882,0.015299199987202883,0.01462399959564209,0.001916768086505386,0.0,0.0,0.0,0.0,0.0,0.21779200434684753,0.2415360063314438,0.2260768011212349,0.2260768011212349,0.007251352080685236,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,32,0,8192.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.014015999622642994,0.021088000386953354,0.015619200188666582,0.01473599998280406,0.002013227452302141,0.0,0.0,0.0,0.0,0.0,0.3959999978542328,0.4152640104293823,0.40332479774951924,0.40332479774951924,0.006942401431914052,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,64,0,8192.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.014175999909639359,0.03379200026392937,0.020595200080424547,0.019600000232458115,0.006548754248881344,0.026192623739694512,0.040006882507168426,0.029155180178492036,0.029155180178492036,0.00408828748438241,0.024591375524545756,0.037561119537986146,0.027372820355088746,0.027372820355088746,0.0038383559348575944,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,9,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,72,,,,,,,,False,,,64,BF16,generic,none,CUDA_EVENT,True,[64],[0],"[512, 512, 512, 512, 512, 512, 512, 512]",1,8,8,9,8,512.0,0.1111111111111111,q64_8q1s512,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
0.0,0.0,0.0,0.0,0.0,0.013856000266969204,0.020896000787615776,0.015318400040268899,0.014431999996304512,0.0019640312960926966,0.029349018208693862,0.06236262941356679,0.03714313592014963,0.03714313592014963,0.009618313791872569,0.028826981462526914,0.06125337308649041,0.036482463672774496,0.036482463672774496,0.009447230957011863,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,9,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,136,,,,,,,,False,,,128,BF16,generic,none,CUDA_EVENT,True,[128],[0],"[1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024]",1,8,8,9,8,1024.0,0.1111111111111111,q128_8q1s1k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
0.0,0.0,0.0,0.0,0.0,0.013856000266969204,0.032127998769283295,0.017286399938166143,0.014479999896138906,0.005536495446636193,0.03273085874558354,0.04394578491937082,0.03563837490653034,0.03563837490653034,0.003429601730256567,0.03488514202593899,0.04683821345079978,0.037984025407085474,0.037984025407085474,0.0036553316361902684,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,17,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,144,,,,,,,,False,,,128,BF16,generic,none,CUDA_EVENT,True,[128],[0],"[1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024]",1,16,16,17,16,1024.0,0.058823529411764705,q128_16q1s1k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
0.0,0.0,0.0,0.0,0.0,0.013856000266969204,0.021503999829292297,0.015078400075435639,0.01425600005313754,0.002238435975478849,0.042100813549974185,0.051784144690147756,0.045378692890289625,0.045378692890289625,0.003184109940650555,0.05111518843748309,0.06287185663927425,0.05509490773570219,0.05509490773570219,0.0038658725544299132,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,17,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,272,,,,,,,,False,,,256,BF16,generic,none,CUDA_EVENT,True,[256],[0],"[2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048]",1,16,16,17,16,2048.0,0.058823529411764705,q256_16q1s2k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
0.0,0.0,0.0,0.0,0.0,0.014399999752640724,0.02687999978661537,0.017667199857532977,0.01566399959847331,0.003864811846387173,0.048475323773821306,0.05599956978723733,0.05123499252968345,0.05123499252968345,0.002427379820423941,0.08429268135885387,0.09737642835214408,0.0890914090616175,0.0890914090616175,0.0042209177332077005,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,33,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,288,,,,,,,,False,,,256,BF16,generic,none,CUDA_EVENT,True,[256],[0],"[2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048]",1,32,32,33,32,2048.0,0.030303030303030304,q256_32q1s2k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
0.0,0.0,0.0,0.0,0.0,0.01462399959564209,0.03359999880194664,0.019804799742996693,0.016032000072300434,0.006750519908145474,0.07121508474579985,0.07292308367136396,0.07194410845270183,0.07194410845270183,0.0005500065915943844,0.14760091249712767,0.15114092373010238,0.14911189243564577,0.14911189243564577,0.0011399477384397005,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,33,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,544,,,,,,,,False,,,512,BF16,generic,none,CUDA_EVENT,True,[512],[0],"[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096]",1,32,32,33,32,4096.0,0.030303030303030304,q512_32q1s4k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
0.0,0.0,0.0,0.0,0.0,0.014399999752640724,0.021856000646948814,0.016835200227797033,0.015263999812304974,0.00283854448975065,0.08146338272142935,0.08560865714384096,0.0834932625520734,0.0834932625520734,0.0012259635905347874,0.2782486219401307,0.29240733788179385,0.2851819365989658,0.2851819365989658,0.004187435731481665,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,65,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,576,,,,,,,,False,,,512,BF16,generic,none,CUDA_EVENT,True,[512],[0],"[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096]",1,64,64,65,64,4096.0,0.015384615384615385,q512_64q1s4k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
0.0,0.0,0.0,0.0,0.0,0.014399999752640724,0.04495999962091446,0.02143679987639189,0.015856000129133463,0.009709987872683342,0.1194459208702178,0.12302524755181463,0.12053885718696096,0.12053885718696096,0.001005118119341339,0.5597220650459199,0.5764947443228802,0.5648435507167102,0.5648435507167102,0.004709970715400788,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,65,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,1088,,,,,,,,False,,,1024,BF16,generic,none,CUDA_EVENT,True,[1024],[0],"[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192]",1,64,64,65,64,8192.0,0.015384615384615385,q1k_64q1s8k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
0.0,0.0,0.0,0.0,0.0,0.018112000077962875,0.026496000587940216,0.019318400137126445,0.018432000651955605,0.0024249605112359905,0.19770253574610402,0.222811797868348,0.2054811520619412,0.2054811520619412,0.008000944254868043,0.13941746080159492,0.157124211777114,0.14490284788179197,0.14490284788179197,0.005642170080515992,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,33,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,2080,,,,,,,,False,,,2048,BF16,generic,none,CUDA_EVENT,True,[2048],[0],"[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096]",1,32,32,33,32,4096.0,0.030303030303030304,q2k_32q1s4k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
0.0,0.0,0.0,0.0,0.0,0.02595200017094612,0.03155200183391571,0.02727359998971224,0.026575999334454536,0.0016260948764843166,0.6014684881116659,0.6095472988212,0.6046757700946376,0.6046757700946376,0.0023537435563177303,0.11325152264578045,0.11477269562841545,0.11385542721485654,0.11385542721485654,0.0004431903698023628,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,17,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,4112,,,,,,,,False,,,4096,BF16,generic,none,CUDA_EVENT,True,[4096],[0],"[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096]",1,16,16,17,16,4096.0,0.058823529411764705,q4k_16q1s4k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
0.0,0.0,0.0,0.0,0.0,0.014303999952971935,0.03359999880194664,0.01814719969406724,0.015375999733805656,0.005678506740662529,0.06774400174617767,0.07891199737787247,0.07312640026211739,0.07312640026211739,0.003826583051422731,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,2,1024,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,"[512, 512]",1024,512,512,512.0,True,0.0,0.0,0.0,False,0.0,1024.0,1024,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,2q512,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.01788800023496151,0.028543999418616295,0.021340799890458582,0.019504000432789326,0.003718627037219618,0.09548799693584442,0.1327359974384308,0.10823359936475753,0.10823359936475753,0.013230635292043864,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,4,2048,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,"[512, 512, 512, 512]",2048,512,512,512.0,True,0.0,0.0,0.0,False,0.0,2048.0,2048,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,4q512,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.02627200074493885,0.030368000268936157,0.027184000052511693,0.02643200010061264,0.0013545679205210022,0.14364799857139587,0.1597760021686554,0.15008639842271806,0.15008639842271806,0.005367723415171222,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,8,4096,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,"[512, 512, 512, 512, 512, 512, 512, 512]",4096,512,512,512.0,True,0.0,0.0,0.0,False,0.0,4096.0,4096,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q512,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.04150399938225746,0.04726399853825569,0.042950399965047834,0.04224000126123428,0.001720147501765378,0.2433920055627823,0.28963199257850647,0.2549152016639709,0.2549152016639709,0.013450991019687407,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,16,8192,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,"[512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512]",8192,512,512,512.0,True,0.0,0.0,0.0,False,0.0,8192.0,8192,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q512,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.025887999683618546,0.03407999873161316,0.028303999826312064,0.026367999613285065,0.0028877495368841042,0.325439989566803,0.3441599905490875,0.33442879617214205,0.33442879617214205,0.005693597811441922,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,2,4096,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,"[2048, 2048]",4096,2048,2048,2048.0,True,0.0,0.0,0.0,False,0.0,4096.0,4096,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,2q2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.04137599840760231,0.05084799975156784,0.044828799366950986,0.043087998405098915,0.003560363865797613,0.6110399961471558,0.6421759724617004,0.6204223990440368,0.6204223990440368,0.011214490370794758,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,4,8192,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,"[2048, 2048, 2048, 2048]",8192,2048,2048,2048.0,True,0.0,0.0,0.0,False,0.0,8192.0,8192,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,4q2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.014816000126302242,0.02364799939095974,0.016668799985200166,0.01592000015079975,0.0025515238179941247,0.0,0.0,0.0,0.0,0.0,0.0544000007212162,0.09014400094747543,0.06511679962277411,0.06511679962277411,0.013005726711295521,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,0,16384.0,False,FLASH_ATTN,False,vllm020_batch_spec,[1],1,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q1s16k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.014527999795973301,0.0208320003002882,0.016604799963533878,0.01566399959847331,0.0021353594266203244,0.0,0.0,0.0,0.0,0.0,0.17871999740600586,0.1961279958486557,0.18568639904260634,0.18568639904260634,0.004566142885274747,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,8,0,16384.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s16k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.014431999996304512,0.02006400004029274,0.015619200002402068,0.01508800033479929,0.001572984783715635,0.0,0.0,0.0,0.0,0.0,0.2730880081653595,0.29721599817276,0.2815328001976013,0.2815328001976013,0.007016534727616923,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,16,0,16384.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s16k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.014879999682307243,0.02755199931561947,0.018678399827331306,0.015919999685138464,0.004323555372382858,0.0,0.0,0.0,0.0,0.0,0.3893119990825653,0.44041600823402405,0.40225600004196166,0.40225600004196166,0.013442998165884852,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,32,0,16384.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s16k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.014303999952971935,0.02659199945628643,0.016748800035566093,0.015343999955803156,0.0036412341868394082,0.0,0.0,0.0,0.0,0.0,0.7404800057411194,0.9689919948577881,0.8451807916164398,0.8451807916164398,0.08637455920036795,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,64,0,16384.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s16k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.014751999638974667,0.02160000056028366,0.016752000153064727,0.01521599991247058,0.0025167650350367246,0.0,0.0,0.0,0.0,0.0,0.06412799656391144,0.08956799656152725,0.07242240011692047,0.07242240011692047,0.009797120588901621,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,0,32768.0,False,FLASH_ATTN,False,vllm020_batch_spec,[1],1,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q1s32k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.014495999552309513,0.022911999374628067,0.016137600038200618,0.015343999955803156,0.002330911555470513,0.0,0.0,0.0,0.0,0.0,0.3128319978713989,0.3282879889011383,0.3203647971153259,0.3203647971153259,0.004598568238907996,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,8,0,32768.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s32k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.014655999839305878,0.03574400022625923,0.01919359974563122,0.01583999954164028,0.006642521913489215,0.0,0.0,0.0,0.0,0.0,0.5050879716873169,0.5311999917030334,0.511932796239853,0.511932796239853,0.007150929647317677,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,16,0,32768.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s32k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.014655999839305878,0.02595200017094612,0.016992000117897987,0.015456000342965126,0.003390731937723684,0.0,0.0,0.0,0.0,0.0,0.7385600209236145,0.7681919932365417,0.7483008027076722,0.7483008027076722,0.010214373356208012,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,32,0,32768.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s32k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.014527999795973301,0.038975998759269714,0.01952639976516366,0.015327999833971262,0.007695927285621061,0.0,0.0,0.0,0.0,0.0,1.4228800535202026,1.5237760543823242,1.4435008168220522,1.4435008168220522,0.0342955291725742,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,64,0,32768.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s32k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.014495999552309513,0.03907199949026108,0.01793599994853139,0.015248000156134367,0.007167103363234667,0.0,0.0,0.0,0.0,0.0,0.06947200000286102,0.08857599645853043,0.07469440028071403,0.07469440028071403,0.00583249952929747,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,0,40960.0,False,FLASH_ATTN,False,vllm020_batch_spec,[1],1,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q1s40k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.014495999552309513,0.02175999991595745,0.016710399929434062,0.015327999833971262,0.002705383977079183,0.0,0.0,0.0,0.0,0.0,0.388480007648468,0.4073280096054077,0.3966591984033584,0.3966591984033584,0.004487315516095964,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,8,0,40960.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s40k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.0144640002399683,0.022048000246286392,0.017011200170964004,0.015392000321298838,0.002776899649845722,0.0,0.0,0.0,0.0,0.0,0.622048020362854,0.6347839832305908,0.6262047946453094,0.6262047946453094,0.004593360122958751,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,16,0,40960.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s40k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.014592000283300877,0.02937600016593933,0.01847040019929409,0.015344000421464443,0.005043809142122341,0.0,0.0,0.0,0.0,0.0,0.9105280041694641,0.9304640293121338,0.914108806848526,0.914108806848526,0.00563919268816644,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,32,0,40960.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s40k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.01462399959564209,0.039903998374938965,0.01791359977796674,0.015440000221133232,0.007367547649297217,0.0,0.0,0.0,0.0,0.0,1.764799952507019,1.7965760231018066,1.770739197731018,1.770739197731018,0.009082122770320404,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,64,0,40960.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s40k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.015168000012636185,0.03187200054526329,0.018553599901497363,0.016127999871969223,0.0050628774268006264,0.16963527081512533,0.1737871320906266,0.17088926838108623,0.17088926838108623,0.0011703114258121128,0.47362872483230506,0.48522089403242025,0.47712993814280913,0.47712993814280913,0.0032675581298664976,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,9,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,520,,,,,,,,False,,,512,BF16,generic,none,CUDA_EVENT,True,[512],[0],"[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384]",1,8,8,9,8,16384.0,0.1111111111111111,q512_8q1s16k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
0.0,0.0,0.0,0.0,0.0,0.014879999682307243,0.03868800029158592,0.01921919994056225,0.016959999687969685,0.006691309533030663,0.1589600576212269,0.16117782913137854,0.15969057520605917,0.15969057520605917,0.0006212984448748182,0.5199519263456005,0.5272061673552852,0.5223414198520004,0.5223414198520004,0.002032242112153396,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,17,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,1040,,,,,,,,False,,,1024,BF16,generic,none,CUDA_EVENT,True,[1024],[0],"[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384]",1,16,16,17,16,16384.0,0.058823529411764705,q1k_16q1s16k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
0.0,0.0,0.0,0.0,0.0,0.018400000408291817,0.024320000782608986,0.019910399988293647,0.0191040001809597,0.001841532763984352,0.25650752966102763,0.27066608538463055,0.259736894547936,0.259736894547936,0.00462927315947332,0.5278764825612624,0.5570139063374764,0.5345223138928448,0.5345223138928448,0.009526755161797178,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,33,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,2080,,,,,,,,False,,,2048,BF16,generic,none,CUDA_EVENT,True,[2048],[0],"[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384]",1,32,32,33,32,16384.0,0.030303030303030304,q2k_32q1s16k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
0.0,0.0,0.0,0.0,0.0,0.026048000901937485,0.03728000074625015,0.028636799938976765,0.02705600019544363,0.003313953649011259,0.9270006318443208,0.9712965120641314,0.93480767601568,0.93480767601568,0.012612551450284608,0.8181833128578277,0.8572794566782395,0.8250739157811953,0.8250739157811953,0.01113200873299586,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,17,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,4112,,,,,,,,False,,,4096,BF16,generic,none,CUDA_EVENT,True,[4096],[0],"[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768]",1,16,16,17,16,32768.0,0.058823529411764705,q4k_16q1s32k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
0.0,0.0,0.0,0.0,0.0,0.018079999834299088,0.037696000188589096,0.022092800214886667,0.019024000503122807,0.005986278786633071,0.2839857165542317,0.2876110048757805,0.2849506939696605,0.2849506939696605,0.0009995178425916847,1.0871823008331585,1.101060989333509,1.0908765231323903,1.0908765231323903,0.0038264533900426207,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,33,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,2080,,,,,,,,False,,,2048,BF16,generic,none,CUDA_EVENT,True,[2048],[0],"[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768]",1,32,32,33,32,32768.0,0.030303030303030304,q2k_32q1s32k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
0.0,0.0,0.0,0.0,0.0,0.015072000212967396,0.025567999109625816,0.01726400014013052,0.015728000551462173,0.0031324465940990903,0.137270464802061,0.13799613818579368,0.13751499486424038,0.13751499486424038,0.00025221761530472904,2.3021855212208884,2.314355908780759,2.3062865750744197,2.3062865750744197,0.004229983070201486,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,65,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,1088,,,,,,,,False,,,1024,BF16,generic,none,CUDA_EVENT,True,[1024],[0],"[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768]",1,64,64,65,64,32768.0,0.015384615384615385,q1k_64q1s32k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
0.0,0.0,0.0,0.0,0.0,0.014560000039637089,0.022784000262618065,0.016435200069099664,0.015519999898970127,0.0023688301421469523,0.04879999905824661,0.09139200299978256,0.06054079942405224,0.06054079942405224,0.012152608702448775,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,64,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[64],64,64,64,64.0,True,0.0,0.0,0.0,False,0.0,64.0,64,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q64,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.014816000126302242,0.02844800055027008,0.017008000146597625,0.015696000307798386,0.003941466294662679,0.047807998955249786,0.07356800138950348,0.05626560002565384,0.05626560002565384,0.00842179125412236,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,128,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[128],128,128,128,128.0,True,0.0,0.0,0.0,False,0.0,128.0,128,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.014688000082969666,0.053408000618219376,0.019353600218892097,0.01532800029963255,0.01138302400841626,0.048448000103235245,0.0785600021481514,0.0556256003677845,0.0556256003677845,0.009348274502616908,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,256,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[256],256,256,256,256.0,True,0.0,0.0,0.0,False,0.0,256.0,256,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q256,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.015039999969303608,0.03139200061559677,0.01823679991066456,0.016159999649971724,0.004860071238302512,0.055424001067876816,0.08505599945783615,0.0640383992344141,0.0640383992344141,0.00921448636178623,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,512,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[512],512,512,512,512.0,True,0.0,0.0,0.0,False,0.0,512.0,512,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q512,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.014751999638974667,0.026528000831604004,0.017324799951165915,0.01536000007763505,0.0036004822686428305,0.07660800218582153,0.0942080020904541,0.08209280073642732,0.08209280073642732,0.00515895587669752,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,1024,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[1024],1024,1024,1024,1024.0,True,0.0,0.0,0.0,False,0.0,1024.0,1024,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.014879999682307243,0.021247999742627144,0.016403199825435876,0.01532800029963255,0.001995897022647934,0.11395200341939926,0.15113599598407745,0.12431039959192276,0.12431039959192276,0.011164431123683732,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,2048,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[2048],2048,2048,2048,2048.0,True,0.0,0.0,0.0,False,0.0,2048.0,2048,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.017184000462293625,0.028672000393271446,0.019865600019693376,0.017823999747633934,0.0035798491315929977,0.3171840012073517,0.3341119885444641,0.3261695951223373,0.3261695951223373,0.005111046452878918,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,4096,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[4096],4096,4096,4096,4096.0,True,0.0,0.0,0.0,False,0.0,4096.0,4096,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.024639999493956566,0.030400000512599945,0.02656640000641346,0.02556800004094839,0.0020189846603237303,1.0648640394210815,1.0828479528427124,1.0715327858924866,1.0715327858924866,0.005558639263049851,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,8192,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[8192],8192,8192,8192,8192.0,True,0.0,0.0,0.0,False,0.0,8192.0,8192,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.014976000413298607,0.021695999428629875,0.017305599898099898,0.01593599934130907,0.0025718046517268054,0.04956800118088722,0.07932800054550171,0.06228480041027069,0.06228480041027069,0.01027160349757827,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,64,448.0,True,FLASH_ATTN,False,vllm020_batch_spec,[64],64,64,64,64.0,True,0.0,0.0,0.0,True,448.0,512.0,64,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q64s512,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.01539199985563755,0.03580800071358681,0.019686400331556796,0.017136000096797943,0.005918387811304003,0.05158400163054466,0.10713600367307663,0.06364160068333148,0.06364160068333148,0.015832957809696766,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,128,896.0,True,FLASH_ATTN,False,vllm020_batch_spec,[128],128,128,128,128.0,True,0.0,0.0,0.0,True,896.0,1024.0,128,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q128s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.015168000012636185,0.02223999984562397,0.016672000009566545,0.015887999907135963,0.0020934945946034563,0.06521599739789963,0.08902399986982346,0.07520959973335266,0.07520959973335266,0.007913840684102929,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,256,1792.0,True,FLASH_ATTN,False,vllm020_batch_spec,[256],256,256,256,256.0,True,0.0,0.0,0.0,True,1792.0,2048.0,256,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q256s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.015231999568641186,0.04028800129890442,0.019971200078725816,0.01646399963647127,0.007351305886577286,0.14467200636863708,0.16844800114631653,0.15363519936800005,0.15363519936800005,0.008174435899956223,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,512,3584.0,True,FLASH_ATTN,False,vllm020_batch_spec,[512],512,512,512,512.0,True,0.0,0.0,0.0,True,3584.0,4096.0,512,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q512s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.015519999898970127,0.02304000034928322,0.01775679988786578,0.016784000210464,0.0025077243712082584,0.33740800619125366,0.35343998670578003,0.3445120006799698,0.3445120006799698,0.00445648463590358,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,1024,7168.0,True,FLASH_ATTN,False,vllm020_batch_spec,[1024],1024,1024,1024,1024.0,True,0.0,0.0,0.0,True,7168.0,8192.0,1024,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q1ks8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.015647999942302704,0.02393599972128868,0.01809599995613098,0.01654400024563074,0.002998393102466254,0.0,0.0,0.0,0.0,0.0,0.0504320003092289,0.0843840017914772,0.060083200410008426,0.060083200410008426,0.00986959318572296,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,0,128.0,False,FLASH_ATTN,False,vllm020_batch_spec,[1],1,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.01603199914097786,0.030047999694943428,0.019510399922728537,0.017680000513792038,0.004065185265963332,0.0,0.0,0.0,0.0,0.0,0.05004800111055374,0.07036799937486649,0.059315200522542,0.059315200522542,0.006537768821329647,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,8,0,128.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.01484800036996603,0.02393599972128868,0.018035199772566558,0.016512000001966953,0.0030667689116777724,0.0,0.0,0.0,0.0,0.0,0.05100800096988678,0.06735999882221222,0.058387200161814694,0.058387200161814694,0.005832787739241381,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,16,0,128.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.01500799972563982,0.026208000257611275,0.017500799987465142,0.01646399963647127,0.0030666103306165714,0.0,0.0,0.0,0.0,0.0,0.05023999884724617,0.07254400104284286,0.057254400476813315,0.057254400476813315,0.006890853653068151,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,32,0,128.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.014688000082969666,0.027327999472618103,0.0179776000790298,0.01648000068962574,0.003783417447531392,0.0,0.0,0.0,0.0,0.0,0.04819199815392494,0.07100799679756165,0.05621119923889638,0.05621119923889638,0.007824391484124725,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,64,0,128.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.015647999942302704,0.036448001861572266,0.019136000238358975,0.016704000532627106,0.006076530095624803,0.0,0.0,0.0,0.0,0.0,0.047488000243902206,0.08441600203514099,0.0588383998721838,0.0588383998721838,0.011275940726332522,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,8,0,1024.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.015135999768972397,0.033952001482248306,0.02119360016658902,0.018000000156462193,0.007136646614305304,0.0,0.0,0.0,0.0,0.0,0.04931199923157692,0.06992000341415405,0.05755840018391609,0.05755840018391609,0.007297587694315226,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,16,0,1024.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.014879999682307243,0.034272000193595886,0.0204415999352932,0.017311999574303627,0.00695404701803013,0.0,0.0,0.0,0.0,0.0,0.05215999856591225,0.0735040009021759,0.06117440015077591,0.06117440015077591,0.007381118436894922,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,32,0,1024.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.015359999611973763,0.03743999823927879,0.02012479966506362,0.01775999926030636,0.006181045509079057,0.0,0.0,0.0,0.0,0.0,0.06355199962854385,0.08508799970149994,0.07019200026988984,0.07019200026988984,0.0062246580442117845,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,64,0,1024.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.015552000142633915,0.032607998698949814,0.01959999995306134,0.017487999983131886,0.004882475068460749,0.0,0.0,0.0,0.0,0.0,0.04918399825692177,0.0865280032157898,0.05973760038614274,0.05973760038614274,0.011546847942113974,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,8,0,2048.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.015168000012636185,0.02675200067460537,0.017430400010198355,0.016560000367462635,0.003243135094239819,0.0,0.0,0.0,0.0,0.0,0.052671998739242554,0.08367999643087387,0.06238719932734965,0.06238719932734965,0.011207824780630104,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,16,0,2048.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.01500799972563982,0.03612799942493439,0.019327999837696553,0.01688000001013279,0.006058251425153085,0.0,0.0,0.0,0.0,0.0,0.06364800035953522,0.07878399640321732,0.06935679838061332,0.06935679838061332,0.004515588328677643,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,32,0,2048.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.015104000456631184,0.03711999952793121,0.019670399930328132,0.017152000218629837,0.006165994646522264,0.0,0.0,0.0,0.0,0.0,0.09071999788284302,0.11507199704647064,0.10252480059862136,0.10252480059862136,0.008782544051535657,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,64,0,2048.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.015039999969303608,0.026528000831604004,0.018396800104528665,0.01601599995046854,0.004279788048986283,0.0,0.0,0.0,0.0,0.0,0.05331199988722801,0.07977599650621414,0.06076480001211167,0.06076480001211167,0.008761579122069606,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,8,0,4096.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.015359999611973763,0.02643200010061264,0.01726400004699826,0.015855999663472176,0.003210008417242029,0.0,0.0,0.0,0.0,0.0,0.062431998550891876,0.08246400207281113,0.06970879957079888,0.06970879957079888,0.007016341676068233,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,16,0,4096.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.014911999925971031,0.02223999984562397,0.0166015999391675,0.015887999907135963,0.00204913969129354,0.0,0.0,0.0,0.0,0.0,0.10127999633550644,0.1141119971871376,0.10621120035648347,0.10621120035648347,0.004029318311754605,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,32,0,4096.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.015359999611973763,0.03363199904561043,0.019305599946528675,0.016671999357640743,0.0055445582307981234,0.0,0.0,0.0,0.0,0.0,0.1319040060043335,0.15839999914169312,0.14040640145540234,0.14040640145540234,0.007998041073596942,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,64,0,4096.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.014368000440299511,0.04447999969124794,0.018908800091594458,0.0157279996201396,0.0086814937461721,0.0,0.0,0.0,0.0,0.0,0.06428799778223038,0.08982399851083755,0.07349760085344315,0.07349760085344315,0.008605284512197258,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,8,0,8192.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.015104000456631184,0.028672000393271446,0.01890560006722808,0.016080000437796116,0.004797584627815021,0.0,0.0,0.0,0.0,0.0,0.11123199760913849,0.14115199446678162,0.11942399889230729,0.11942399889230729,0.008513394706791027,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,16,0,8192.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.014271999709308147,0.024927999824285507,0.016915200091898442,0.015216000378131866,0.003374147499442635,0.0,0.0,0.0,0.0,0.0,0.15887999534606934,0.18111999332904816,0.16934399753808976,0.16934399753808976,0.007415181260761123,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,32,0,8192.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.014336000196635723,0.026944000273942947,0.017737600207328796,0.015199999790638685,0.00415598714375255,0.0,0.0,0.0,0.0,0.0,0.2192319929599762,0.23472000658512115,0.22809920012950893,0.22809920012950893,0.004730841376327335,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,64,0,8192.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.014688000082969666,0.05052800104022026,0.020320000313222408,0.015520000364631414,0.010604522177924678,0.024270629882498958,0.03340247625954076,0.028446344104128624,0.028446344104128624,0.0032330369099793834,0.023697370291069768,0.03261352722995356,0.027774456325453972,0.027774456325453972,0.0031566742680923386,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,9,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,72,,,,,,,,False,,,64,BF16,generic,none,CUDA_EVENT,True,[64],[0],"[512, 512, 512, 512, 512, 512, 512, 512]",1,8,8,9,8,512.0,0.1111111111111111,q64_8q1s512,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
0.0,0.0,0.0,0.0,0.0,0.015359999611973763,0.033663999289274216,0.019987199828028678,0.018240000121295452,0.005522929139253732,0.0259194055660947,0.03719755183990719,0.029916029687899703,0.029916029687899703,0.003966666590554318,0.027104595853449518,0.03889844644729374,0.03128396954022015,0.03128396954022015,0.004148046317967881,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,9,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,136,,,,,,,,False,,,128,BF16,generic,none,CUDA_EVENT,True,[128],[0],"[1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024]",1,8,8,9,8,1024.0,0.1111111111111111,q128_8q1s1k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
0.0,0.0,0.0,0.0,0.0,0.014592000283300877,0.03299200162291527,0.019840000104159115,0.016207999549806118,0.006546343008726814,0.02587869595769926,0.03763167265431482,0.030174939058162802,0.030174939058162802,0.0037303581782277364,0.026473304070195713,0.03849632587654989,0.03086826083864964,0.03086826083864964,0.003816069654529222,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,17,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,144,,,,,,,,False,,,128,BF16,generic,none,CUDA_EVENT,True,[128],[0],"[1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024]",1,16,16,17,16,1024.0,0.058823529411764705,q128_16q1s1k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
0.0,0.0,0.0,0.0,0.0,0.013824000023305416,0.021023999899625778,0.016304000187665223,0.015584000386297703,0.0023236165806545476,0.031810621525966996,0.04156949936878106,0.0346749350032807,0.0346749350032807,0.003130209181143985,0.035677378270900374,0.04662250161636451,0.038889864871740294,0.038889864871740294,0.0035107033960828727,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,17,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,272,,,,,,,,False,,,256,BF16,generic,none,CUDA_EVENT,True,[256],[0],"[2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048]",1,16,16,17,16,2048.0,0.058823529411764705,q256_16q1s2k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
0.0,0.0,0.0,0.0,0.0,0.014527999795973301,0.02175999991595745,0.01569600012153387,0.015008000191301107,0.0020882666168036863,0.038211712107062853,0.07212229256520057,0.04364509673334097,0.04364509673334097,0.009671953074025085,0.0476442859917874,0.08992570455184198,0.05441890342616107,0.05441890342616107,0.01205947791784038,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,33,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,288,,,,,,,,False,,,256,BF16,generic,none,CUDA_EVENT,True,[256],[0],"[2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048]",1,32,32,33,32,2048.0,0.030303030303030304,q256_32q1s2k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
0.0,0.0,0.0,0.0,0.0,0.014368000440299511,0.033824000507593155,0.017628800217062236,0.014864000026136637,0.005791495403857738,0.05214261250030033,0.06266261508706669,0.05677791295527661,0.05677791295527661,0.0030298223079651514,0.08648138506878382,0.10392938682791132,0.09416928531647478,0.09416928531647478,0.005025126612204489,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,33,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,544,,,,,,,,False,,,512,BF16,generic,none,CUDA_EVENT,True,[512],[0],"[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096]",1,32,32,33,32,4096.0,0.030303030303030304,q512_32q1s4k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
0.0,0.0,0.0,0.0,0.0,0.014527999795973301,0.031199999153614044,0.017900799959897996,0.015343999955803156,0.004974475661316249,0.06411958891421111,0.07405276123263956,0.06793047918211377,0.06793047918211377,0.0033918658556700006,0.14058441263169497,0.16236323591492058,0.1489399211274489,0.1489399211274489,0.00743678300375353,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,65,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,576,,,,,,,,False,,,512,BF16,generic,none,CUDA_EVENT,True,[512],[0],"[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096]",1,64,64,65,64,4096.0,0.015384615384615385,q512_64q1s4k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
0.0,0.0,0.0,0.0,0.0,0.014720000326633453,0.03855999931693077,0.018495999928563833,0.015647999942302704,0.006918530056761627,0.09872985549401277,0.10683454583043753,0.10185316839884552,0.10185316839884552,0.0020802254037042074,0.2743261390166379,0.2968454508984596,0.2830044295482752,0.2830044295482752,0.005780016596065092,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,65,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,1088,,,,,,,,False,,,1024,BF16,generic,none,CUDA_EVENT,True,[1024],[0],"[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192]",1,64,64,65,64,8192.0,0.015384615384615385,q1k_64q1s8k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
0.0,0.0,0.0,0.0,0.0,0.014911999925971031,0.030400000512599945,0.01977920001372695,0.01726400014013052,0.005350883464206169,0.13918872472233365,0.16279522855335207,0.1501912864839173,0.1501912864839173,0.008992185529011513,0.11892328861766266,0.13909276049083735,0.1283239123428725,0.1283239123428725,0.007682951884956939,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,33,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,2080,,,,,,,,False,,,2048,BF16,generic,none,CUDA_EVENT,True,[2048],[0],"[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096]",1,32,32,33,32,4096.0,0.030303030303030304,q2k_32q1s4k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
0.0,0.0,0.0,0.0,0.0,0.01727999933063984,0.02223999984562397,0.01819519978016615,0.017680000513792038,0.0014397298270620873,0.3728044181625443,0.38295504353701676,0.3761264483787333,0.3761264483787333,0.0033660272332490977,0.07967557017401883,0.08184495665371805,0.08038555277807365,0.08038555277807365,0.0007193856241088455,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,17,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,4112,,,,,,,,False,,,4096,BF16,generic,none,CUDA_EVENT,True,[4096],[0],"[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096]",1,16,16,17,16,4096.0,0.058823529411764705,q4k_16q1s4k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
0.0,0.0,0.0,0.0,0.0,0.01500799972563982,0.034591998904943466,0.01941439984366298,0.01756799966096878,0.005541380584373771,0.05951999872922897,0.08268799632787704,0.06715519949793816,0.06715519949793816,0.006657802116288364,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,2,1024,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,"[512, 512]",1024,512,512,512.0,True,0.0,0.0,0.0,False,0.0,1024.0,1024,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,2q512,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.015104000456631184,0.03283200040459633,0.019180799927562477,0.016543999314308167,0.0052187911233635975,0.07199999690055847,0.08675199747085571,0.07749439924955369,0.07749439924955369,0.004849874741156772,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,4,2048,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,"[512, 512, 512, 512]",2048,512,512,512.0,True,0.0,0.0,0.0,False,0.0,2048.0,2048,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,4q512,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.01724799908697605,0.02284800074994564,0.01928640007972717,0.018400000408291817,0.0020641390157565697,0.09676799923181534,0.12310399860143663,0.10618879944086074,0.10618879944086074,0.008982738207839057,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,8,4096,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,"[512, 512, 512, 512, 512, 512, 512, 512]",4096,512,512,512.0,True,0.0,0.0,0.0,False,0.0,4096.0,4096,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q512,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.024224000051617622,0.03315199911594391,0.027436799928545953,0.027328000403940678,0.002679154874546328,0.14716799557209015,0.16412800550460815,0.15470399856567385,0.15470399856567385,0.005423454433987419,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,16,8192,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,"[512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512]",8192,512,512,512.0,True,0.0,0.0,0.0,False,0.0,8192.0,8192,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q512,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.01833599992096424,0.04806400090456009,0.026252799853682517,0.022672000341117382,0.00878438440429033,0.1844799965620041,0.2072959989309311,0.19359359890222552,0.19359359890222552,0.007663539820967659,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,2,4096,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,"[2048, 2048]",4096,2048,2048,2048.0,True,0.0,0.0,0.0,False,0.0,4096.0,4096,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,2q2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.024480000138282776,0.04979199916124344,0.029081599973142146,0.025679999962449074,0.007227422044689197,0.34147199988365173,0.37968000769615173,0.3583200007677078,0.3583200007677078,0.011804422100726231,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,4,8192,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,"[2048, 2048, 2048, 2048]",8192,2048,2048,2048.0,True,0.0,0.0,0.0,False,0.0,8192.0,8192,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,4q2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.014720000326633453,0.023744000121951103,0.017753600236028434,0.016368000768125057,0.003061040281616705,0.0,0.0,0.0,0.0,0.0,0.05686400085687637,0.1090880036354065,0.06629760004580021,0.06629760004580021,0.014803727026812366,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,0,16384.0,False,FLASH_ATTN,False,vllm020_batch_spec,[1],1,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q1s16k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.014879999682307243,0.021088000386953354,0.01647359998896718,0.015840000472962856,0.0017964402433206,0.0,0.0,0.0,0.0,0.0,0.08540800213813782,0.09961599856615067,0.09146559983491898,0.09146559983491898,0.00447423706371901,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,8,0,16384.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s16k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.014560000039637089,0.04156799986958504,0.018512000143527985,0.015584000386297703,0.007849701681877904,0.0,0.0,0.0,0.0,0.0,0.17948800325393677,0.194815993309021,0.1866239994764328,0.1866239994764328,0.004156328241374654,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,16,0,16384.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s16k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.014879999682307243,0.029632000252604485,0.019449600111693145,0.016736000776290894,0.005126811425136928,0.0,0.0,0.0,0.0,0.0,0.27529600262641907,0.2898879945278168,0.2825664013624191,0.2825664013624191,0.004542801609674093,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,32,0,16384.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s16k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.014399999752640724,0.031328000128269196,0.019759999960660933,0.016367999836802483,0.006266303740589241,0.0,0.0,0.0,0.0,0.0,0.39190399646759033,0.4079039990901947,0.3987520009279252,0.3987520009279252,0.0037367727509362725,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,64,0,16384.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s16k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.01539199985563755,0.040031999349594116,0.0212032001465559,0.01649599988013506,0.008068547381446682,0.0,0.0,0.0,0.0,0.0,0.06588800251483917,0.07897599786520004,0.0703904002904892,0.0703904002904892,0.004214826869769361,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,0,32768.0,False,FLASH_ATTN,False,vllm020_batch_spec,[1],1,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q1s32k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.014495999552309513,0.022495999932289124,0.016006399970501663,0.01515199989080429,0.002265581884342885,0.0,0.0,0.0,0.0,0.0,0.1231679990887642,0.13526399433612823,0.1288223996758461,0.1288223996758461,0.004328976434897094,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,8,0,32768.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s32k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.014592000283300877,0.023231999948620796,0.01705600004643202,0.016048000194132328,0.002810904312746391,0.0,0.0,0.0,0.0,0.0,0.3158079981803894,0.33129599690437317,0.32348800003528594,0.32348800003528594,0.00417599076136664,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,16,0,32768.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s32k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.014527999795973301,0.021888000890612602,0.01708160014823079,0.01609600055962801,0.0025289234621475062,0.0,0.0,0.0,0.0,0.0,0.5103679895401001,0.5200319886207581,0.5132320046424866,0.5132320046424866,0.0035759433157749533,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,32,0,32768.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s32k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.014688000082969666,0.032416000962257385,0.019353600032627583,0.016176000237464905,0.0061458798960684425,0.0,0.0,0.0,0.0,0.0,0.7400320172309875,0.7516480088233948,0.7449311971664427,0.7449311971664427,0.004573461776168607,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,64,0,32768.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s32k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.0144640002399683,0.02006400004029274,0.01573119992390275,0.014992000069469213,0.001654446341262991,0.0,0.0,0.0,0.0,0.0,0.07097599655389786,0.09932799637317657,0.07828159928321837,0.07828159928321837,0.009464602895232642,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,0,40960.0,False,FLASH_ATTN,False,vllm020_batch_spec,[1],1,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q1s40k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.01484800036996603,0.02319999970495701,0.016883199848234654,0.015775999519973993,0.0026693906156048403,0.0,0.0,0.0,0.0,0.0,0.14176000654697418,0.1598079949617386,0.15063679963350293,0.15063679963350293,0.005741662969679433,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,8,0,40960.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s40k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.01484800036996603,0.03203200176358223,0.022224000189453363,0.02247999981045723,0.005996176183124981,0.0,0.0,0.0,0.0,0.0,0.39180800318717957,0.41046398878097534,0.4003200054168701,0.4003200054168701,0.005552433764307601,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,16,0,40960.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s40k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.015072000212967396,0.028192000463604927,0.017152000125497578,0.015536000020802021,0.003847486121602065,0.0,0.0,0.0,0.0,0.0,0.6239359974861145,0.6367359757423401,0.6285343945026398,0.6285343945026398,0.004006084286606002,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,32,0,40960.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s40k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.014879999682307243,0.026688000187277794,0.01718079997226596,0.015488000120967627,0.0037132536813398722,0.0,0.0,0.0,0.0,0.0,0.9141119718551636,0.9721279740333557,0.927455997467041,0.927455997467041,0.02148767779232235,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,64,0,40960.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s40k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
0.0,0.0,0.0,0.0,0.0,0.014751999638974667,0.047520000487565994,0.028710400220006704,0.0266720000654459,0.010774688401598903,0.06094816381288764,0.06816969726785131,0.0644060660218149,0.0644060660218149,0.002203106589073183,0.087051838094461,0.09736630405680231,0.09199073574791852,0.09199073574791852,0.0031466818046499644,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,9,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,520,,,,,,,,False,,,512,BF16,generic,none,CUDA_EVENT,True,[512],[0],"[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384]",1,8,8,9,8,16384.0,0.1111111111111111,q512_8q1s16k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
0.0,0.0,0.0,0.0,0.0,0.014911999925971031,0.02489599958062172,0.017427200078964235,0.01595200039446354,0.003071906489592143,0.19733789497223914,0.2016295616652644,0.19860388994013833,0.19860388994013833,0.001512389485439305,0.44861409134062713,0.45837046456077934,0.45149211526120153,0.45149211526120153,0.0034381598874302305,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,17,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,1040,,,,,,,,False,,,1024,BF16,generic,none,CUDA_EVENT,True,[1024],[0],"[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384]",1,16,16,17,16,16384.0,0.058823529411764705,q1k_16q1s16k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
0.0,0.0,0.0,0.0,0.0,0.015039999969303608,0.03046399913728237,0.020643199887126686,0.016944000497460365,0.006320148675595729,0.2157924314537054,0.22215709640166995,0.21769400765743155,0.21769400765743155,0.0019679921844645526,0.4905115822753901,0.5049789194903589,0.4948340005651485,0.4948340005651485,0.0044733865493072344,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,33,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,2080,,,,,,,,False,,,2048,BF16,generic,none,CUDA_EVENT,True,[2048],[0],"[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384]",1,32,32,33,32,16384.0,0.030303030303030304,q2k_32q1s16k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
0.0,0.0,0.0,0.0,0.0,0.017152000218629837,0.02425600029528141,0.01865920014679432,0.018112000077962875,0.0019454553198986453,0.7430866512973927,0.7578834738720781,0.7455351005236347,0.7455351005236347,0.00416692487556653,0.7369773831645824,0.7516525540362471,0.7394057025273603,0.7394057025273603,0.004132666607961175,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,17,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,4112,,,,,,,,False,,,4096,BF16,generic,none,CUDA_EVENT,True,[4096],[0],"[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768]",1,16,16,17,16,32768.0,0.058823529411764705,q4k_16q1s32k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
0.0,0.0,0.0,0.0,0.0,0.015263999812304974,0.02470399998128414,0.018009600043296815,0.01657600048929453,0.0031544532774534346,0.25251173919752334,0.25566890792461106,0.2536729054481981,0.2536729054481981,0.0010279562245342983,1.0425282722084215,1.0555630628624373,1.0473223013846877,1.0473223013846877,0.004244053880724073,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,33,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,2080,,,,,,,,False,,,2048,BF16,generic,none,CUDA_EVENT,True,[2048],[0],"[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768]",1,32,32,33,32,32768.0,0.030303030303030304,q2k_32q1s32k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
0.0,0.0,0.0,0.0,0.0,0.01500799972563982,0.027807999402284622,0.01912960009649396,0.01643200032413006,0.004688164695934997,0.1251379565220268,0.14341821167748953,0.12742497577885914,0.12742497577885914,0.005353812167343766,1.1355340167064276,1.3014137556763312,1.1562870179154463,1.1562870179154463,0.048581869194943325,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,65,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,1088,,,,,,,,False,,,1024,BF16,generic,none,CUDA_EVENT,True,[1024],[0],"[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768]",1,64,64,65,64,32768.0,0.015384615384615385,q1k_64q1s32k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
1 time_stats.attn_input_reshape.min time_stats.attn_input_reshape.max time_stats.attn_input_reshape.mean time_stats.attn_input_reshape.median time_stats.attn_input_reshape.std time_stats.attn_kv_cache_save.min time_stats.attn_kv_cache_save.max time_stats.attn_kv_cache_save.mean time_stats.attn_kv_cache_save.median time_stats.attn_kv_cache_save.std time_stats.attn_prefill.min time_stats.attn_prefill.max time_stats.attn_prefill.mean time_stats.attn_prefill.median time_stats.attn_prefill.std time_stats.attn_decode.min time_stats.attn_decode.max time_stats.attn_decode.mean time_stats.attn_decode.median time_stats.attn_decode.std time_stats.attn_output_reshape.min time_stats.attn_output_reshape.max time_stats.attn_output_reshape.mean time_stats.attn_output_reshape.median time_stats.attn_output_reshape.std n_embd n_q_head n_kv_head block_size num_tensor_parallel_workers max_model_len batch_size prefill_chunk_size kv_cache_size is_prefill attention_backend is_mixed_batch mode seq_lens total_tokens max_seq_len min_seq_len avg_seq_len equal_seq_len seq_len_variance seq_len_std seq_len_cv is_chunked_prefill_sample chunk_start_token chunk_end_token total_prefill_tokens profiling_precision model_arch quant_signature measurement_type is_true_mixed_batch prefill_seq_lens prefill_kv_cache_sizes decode_kv_cache_sizes num_prefill_seqs num_decode_seqs decode_batch_size total_batch_size total_decode_tokens decode_avg_kv_cache_size batch_composition_ratio batch_spec projection_policy
2 0.0 0.0 0.0 0.0 0.0 0.01414399966597557 0.028863999992609024 0.019705599918961526 0.01771199982613325 0.005157200849836681 0.047968000173568726 0.07046400010585785 0.05810240097343922 0.05810240097343922 0.007477463486041561 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 1 40960 1 64 0.0 True FLASH_ATTN False vllm020_batch_spec [64] 64 64 64 64.0 True 0.0 0.0 0.0 False 0.0 64.0 64 BF16 generic none CUDA_EVENT False q64 measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
3 0.0 0.0 0.0 0.0 0.0 0.014399999752640724 0.04947200044989586 0.020412799902260303 0.01635199971497059 0.010107497379722417 0.046560000628232956 0.08323200047016144 0.05587520003318787 0.05587520003318787 0.011126758739503428 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 1 40960 1 128 0.0 True FLASH_ATTN False vllm020_batch_spec [128] 128 128 128 128.0 True 0.0 0.0 0.0 False 0.0 128.0 128 BF16 generic none CUDA_EVENT False q128 measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
4 0.0 0.0 0.0 0.0 0.0 0.01484800036996603 0.022207999601960182 0.017033600155264138 0.015312000177800655 0.002819235991970241 0.05104000121355057 0.07692799717187881 0.056396800279617305 0.056396800279617305 0.007481982178637539 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 1 40960 1 256 0.0 True FLASH_ATTN False vllm020_batch_spec [256] 256 256 256 256.0 True 0.0 0.0 0.0 False 0.0 256.0 256 BF16 generic none CUDA_EVENT False q256 measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
5 0.0 0.0 0.0 0.0 0.0 0.015072000212967396 0.022272000089287758 0.01706880023702979 0.01616000011563301 0.002460889579319197 0.06931199878454208 0.0838719978928566 0.07432000041007995 0.07432000041007995 0.004777766433175866 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 1 40960 1 512 0.0 True FLASH_ATTN False vllm020_batch_spec [512] 512 512 512 512.0 True 0.0 0.0 0.0 False 0.0 512.0 512 BF16 generic none CUDA_EVENT False q512 measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
6 0.0 0.0 0.0 0.0 0.0 0.018592000007629395 0.028543999418616295 0.02095999978482723 0.019183999858796597 0.003198175496053494 0.12179200351238251 0.15408000349998474 0.1307712011039257 0.1307712011039257 0.00858807797538298 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 1 40960 1 1024 0.0 True FLASH_ATTN False vllm020_batch_spec [1024] 1024 1024 1024 1024.0 True 0.0 0.0 0.0 False 0.0 1024.0 1024 BF16 generic none CUDA_EVENT False q1k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
7 0.0 0.0 0.0 0.0 0.0 0.027775999158620834 0.03385600075125694 0.030131200328469276 0.029680000618100166 0.0021152558103575215 0.32678401470184326 0.3450239896774292 0.33396480381488797 0.33396480381488797 0.0045872424917606375 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 1 40960 1 2048 0.0 True FLASH_ATTN False vllm020_batch_spec [2048] 2048 2048 2048 2048.0 True 0.0 0.0 0.0 False 0.0 2048.0 2048 BF16 generic none CUDA_EVENT False q2k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
8 0.0 0.0 0.0 0.0 0.0 0.04438399896025658 0.05084799975156784 0.046540799736976626 0.04531199857592583 0.002277905811237223 1.0959680080413818 1.1151360273361206 1.0999775886535645 1.0999775886535645 0.005694403246120485 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 1 40960 1 4096 0.0 True FLASH_ATTN False vllm020_batch_spec [4096] 4096 4096 4096 4096.0 True 0.0 0.0 0.0 False 0.0 4096.0 4096 BF16 generic none CUDA_EVENT False q4k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
9 0.0 0.0 0.0 0.0 0.0 0.078015998005867 0.08691199868917465 0.08114239946007729 0.08019199967384338 0.00292795706334475 4.070400238037109 4.113152027130127 4.087088012695312 4.087088012695312 0.013660567012509554 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 1 40960 1 8192 0.0 True FLASH_ATTN False vllm020_batch_spec [8192] 8192 8192 8192 8192.0 True 0.0 0.0 0.0 False 0.0 8192.0 8192 BF16 generic none CUDA_EVENT False q8k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
10 0.0 0.0 0.0 0.0 0.0 0.016063999384641647 0.05167999863624573 0.022115200012922286 0.017583999782800674 0.010340094822340818 0.05196800082921982 0.09011200070381165 0.06328320093452933 0.06328320093452933 0.012557341255467452 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 1 40960 1 64 448.0 True FLASH_ATTN False vllm020_batch_spec [64] 64 64 64 64.0 True 0.0 0.0 0.0 True 448.0 512.0 64 BF16 generic none CUDA_EVENT False q64s512 measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
11 0.0 0.0 0.0 0.0 0.0 0.01583999954164028 0.026623999699950218 0.018927999772131443 0.017376000061631203 0.003514650316260619 0.06681600213050842 0.07993599772453308 0.0725280001759529 0.0725280001759529 0.004343502558613716 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 1 40960 1 128 896.0 True FLASH_ATTN False vllm020_batch_spec [128] 128 128 128 128.0 True 0.0 0.0 0.0 True 896.0 1024.0 128 BF16 generic none CUDA_EVENT False q128s1k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
12 0.0 0.0 0.0 0.0 0.0 0.01648000068962574 0.030880000442266464 0.01945280022919178 0.017967999912798405 0.004096211183007485 0.1311360001564026 0.1546880006790161 0.13908160030841826 0.13908160030841826 0.007511906874366178 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 1 40960 1 256 1792.0 True FLASH_ATTN False vllm020_batch_spec [256] 256 256 256 256.0 True 0.0 0.0 0.0 True 1792.0 2048.0 256 BF16 generic none CUDA_EVENT False q256s2k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
13 0.0 0.0 0.0 0.0 0.0 0.017855999991297722 0.03558399900794029 0.020851199887692927 0.018559999763965607 0.005235911594130716 0.32950401306152344 0.350271999835968 0.33912960588932034 0.33912960588932034 0.006027400986663648 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 1 40960 1 512 3584.0 True FLASH_ATTN False vllm020_batch_spec [512] 512 512 512 512.0 True 0.0 0.0 0.0 True 3584.0 4096.0 512 BF16 generic none CUDA_EVENT False q512s4k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
14 0.0 0.0 0.0 0.0 0.0 0.019328000023961067 0.040608000010252 0.022790400311350822 0.020704000256955624 0.006113051965778337 1.1415679454803467 1.1518720388412476 1.144483208656311 1.144483208656311 0.0032332311374389127 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 1 40960 1 1024 7168.0 True FLASH_ATTN False vllm020_batch_spec [1024] 1024 1024 1024 1024.0 True 0.0 0.0 0.0 True 7168.0 8192.0 1024 BF16 generic none CUDA_EVENT False q1ks8k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
15 0.0 0.0 0.0 0.0 0.0 0.015807999297976494 0.030688000842928886 0.019596799835562707 0.01774400006979704 0.004343384771033462 0.0 0.0 0.0 0.0 0.0 0.049056001007556915 0.07580800354480743 0.05948160067200661 0.05948160067200661 0.009031541471446955 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 1 40960 1 0 128.0 False FLASH_ATTN False vllm020_batch_spec [1] 1 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False q1s128 measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
16 0.0 0.0 0.0 0.0 0.0 0.01603199914097786 0.02486399933695793 0.01923839971423149 0.018079999834299088 0.0032282528537266424 0.0 0.0 0.0 0.0 0.0 0.05142400041222572 0.07353600114583969 0.059328000620007516 0.059328000620007516 0.0073307807735143084 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 1 40960 8 0 128.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1] 8 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 8q1s128 measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
17 0.0 0.0 0.0 0.0 0.0 0.016543999314308167 0.03977600112557411 0.021379199624061585 0.018511999398469925 0.006593576176246171 0.0 0.0 0.0 0.0 0.0 0.0488319993019104 0.06435199826955795 0.05479039996862411 0.05479039996862411 0.005672522998491864 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 1 40960 16 0 128.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] 16 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 16q1s128 measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
18 0.0 0.0 0.0 0.0 0.0 0.01635199971497059 0.02844800055027008 0.019267200119793416 0.017952000722289085 0.0035068687666949577 0.0 0.0 0.0 0.0 0.0 0.049855999648571014 0.07798399776220322 0.05986879989504815 0.05986879989504815 0.01043914754878828 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 1 40960 32 0 128.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] 32 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 32q1s128 measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
19 0.0 0.0 0.0 0.0 0.0 0.016383999958634377 0.026079999282956123 0.01923519968986511 0.017791999503970146 0.0032161974331284568 0.0 0.0 0.0 0.0 0.0 0.058111999183893204 0.1045759990811348 0.06708480007946492 0.06708480007946492 0.013479022462646494 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 1 40960 64 0 128.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] 64 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 64q1s128 measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
20 0.0 0.0 0.0 0.0 0.0 0.014720000326633453 0.04057599976658821 0.019100800156593323 0.015455999877303839 0.007512281243011577 0.0 0.0 0.0 0.0 0.0 0.05363199859857559 0.07782399654388428 0.06090559959411622 0.06090559959411622 0.007544620176348091 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 1 40960 8 0 1024.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1] 8 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 8q1s1k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
21 0.0 0.0 0.0 0.0 0.0 0.014431999996304512 0.02191999927163124 0.016684799920767546 0.01563199982047081 0.0024293118621811216 0.0 0.0 0.0 0.0 0.0 0.0629120022058487 0.07891199737787247 0.06891520097851753 0.06891520097851753 0.005472695665695425 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 1 40960 16 0 1024.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] 16 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 16q1s1k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
22 0.0 0.0 0.0 0.0 0.0 0.014560000039637089 0.038943998515605927 0.018313600029796363 0.01561600062996149 0.007127270260115769 0.0 0.0 0.0 0.0 0.0 0.08675199747085571 0.10662399977445602 0.09391999915242194 0.09391999915242194 0.006988099589086635 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 1 40960 32 0 1024.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] 32 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 32q1s1k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
23 0.0 0.0 0.0 0.0 0.0 0.014527999795973301 0.054687999188899994 0.021439999900758268 0.01539199985563755 0.012052764849597775 0.0 0.0 0.0 0.0 0.0 0.13488000631332397 0.1528639942407608 0.1431359991431236 0.1431359991431236 0.005436271464033599 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 1 40960 64 0 1024.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] 64 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 64q1s1k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
24 0.0 0.0 0.0 0.0 0.0 0.014399999752640724 0.041919998824596405 0.01899839974939823 0.015343999955803156 0.007989843526623287 0.0 0.0 0.0 0.0 0.0 0.06176000088453293 0.08374399691820145 0.06747519969940186 0.06747519969940186 0.0066067747128778 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 1 40960 8 0 2048.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1] 8 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 8q1s2k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
25 0.0 0.0 0.0 0.0 0.0 0.016256000846624374 0.11353600025177002 0.042761600017547606 0.028960000723600388 0.029104301538020762 0.0 0.0 0.0 0.0 0.0 0.09734400361776352 0.14422400295734406 0.11392960175871848 0.11392960175871848 0.013198594600417867 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 1 40960 16 0 2048.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] 16 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 16q1s2k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
26 0.0 0.0 0.0 0.0 0.0 0.014688000082969666 0.034143999218940735 0.018918400071561335 0.01643200032413006 0.005500943993080684 0.0 0.0 0.0 0.0 0.0 0.12918399274349213 0.15087999403476715 0.13807999789714814 0.13807999789714814 0.007658330538677587 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 1 40960 32 0 2048.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] 32 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 32q1s2k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
27 0.0 0.0 0.0 0.0 0.0 0.016063999384641647 0.03641600161790848 0.0198208000510931 0.01780799962580204 0.0057264128169845765 0.0 0.0 0.0 0.0 0.0 0.22099199891090393 0.23904000222682953 0.2293503984808922 0.2293503984808922 0.004861342907006028 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 1 40960 64 0 2048.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] 64 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 64q1s2k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
28 0.0 0.0 0.0 0.0 0.0 0.014751999638974667 0.035840000957250595 0.018908800091594458 0.015792000107467175 0.0064374817924757475 0.0 0.0 0.0 0.0 0.0 0.10134399682283401 0.12201599776744843 0.10896319895982742 0.10896319895982742 0.006336330809165179 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 1 40960 8 0 4096.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1] 8 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 8q1s4k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
29 0.0 0.0 0.0 0.0 0.0 0.013856000266969204 0.03417599946260452 0.017846399918198586 0.014800000004470348 0.006495007539635255 0.0 0.0 0.0 0.0 0.0 0.13126400113105774 0.15561600029468536 0.1389280006289482 0.1389280006289482 0.008381472811075022 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 1 40960 16 0 4096.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] 16 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 16q1s4k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
30 0.0 0.0 0.0 0.0 0.0 0.014431999996304512 0.03519999980926514 0.019168000388890504 0.01600000075995922 0.005995522477654695 0.0 0.0 0.0 0.0 0.0 0.21728000044822693 0.2343679964542389 0.2231455981731415 0.2231455981731415 0.004720730646739123 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 1 40960 32 0 4096.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] 32 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 32q1s4k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
31 0.0 0.0 0.0 0.0 0.0 0.014047999866306782 0.03670400008559227 0.018441599886864425 0.015023999847471714 0.006596535162793127 0.0 0.0 0.0 0.0 0.0 0.39321601390838623 0.4524799883365631 0.4058080047369003 0.4058080047369003 0.01578349755088941 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 1 40960 64 0 4096.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] 64 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 64q1s4k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
32 0.0 0.0 0.0 0.0 0.0 0.014336000196635723 0.022143999114632607 0.016912000067532063 0.015168000012636185 0.0028156089295136347 0.0 0.0 0.0 0.0 0.0 0.15587200224399567 0.3079040050506592 0.17838079929351805 0.17838079929351805 0.04355865575265927 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 1 40960 8 0 8192.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1] 8 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 8q1s8k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
33 0.0 0.0 0.0 0.0 0.0 0.014271999709308147 0.02112000063061714 0.015516800060868263 0.01488000014796853 0.001940870731593904 0.0 0.0 0.0 0.0 0.0 0.21587200462818146 0.23561599850654602 0.22250880002975468 0.22250880002975468 0.006181951170646666 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 1 40960 16 0 8192.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] 16 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 16q1s8k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
34 0.0 0.0 0.0 0.0 0.0 0.015552000142633915 0.039264000952243805 0.023609600123018028 0.02131200022995472 0.007236979625711548 0.0 0.0 0.0 0.0 0.0 0.408735990524292 0.470335990190506 0.4336863994598388 0.4336863994598388 0.01844662383160074 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 1 40960 32 0 8192.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] 32 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 32q1s8k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
35 0.0 0.0 0.0 0.0 0.0 0.014399999752640724 0.025407999753952026 0.016227199975401164 0.014960000291466713 0.0031617375441736185 0.0 0.0 0.0 0.0 0.0 0.7412800192832947 0.7627840042114258 0.7464000046253203 0.7464000046253203 0.006112167448837547 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 1 40960 64 0 8192.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] 64 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 64q1s8k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
36 0.0 0.0 0.0 0.0 0.0 0.01462399959564209 0.02812799997627735 0.020652799773961304 0.021359999664127827 0.004308706957613102 0.028383498565450627 0.039859687970646644 0.032492258074592426 0.032492258074592426 0.00453597266208597 0.029312501176103633 0.04116431058787463 0.03355574193327539 0.03355574193327539 0.004684436757701648 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 1 40960 9 0 0 True FLASH_ATTN False true_mixed_fused_projected 72 False 64 BF16 generic none CUDA_EVENT True [64] [0] [512, 512, 512, 512, 512, 512, 512, 512] 1 8 8 9 8 512.0 0.1111111111111111 q64_8q1s512 fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
37 0.0 0.0 0.0 0.0 0.0 0.015552000142633915 0.034143999218940735 0.023171199765056372 0.024255999363958836 0.005908614918096971 0.03333159243114438 0.038935341782478095 0.03580428402241854 0.03580428402241854 0.002082270297044095 0.03633240903369937 0.04244065945634484 0.03902771507087562 0.03902771507087562 0.0022697354261490147 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 1 40960 9 0 0 True FLASH_ATTN False true_mixed_fused_projected 136 False 128 BF16 generic none CUDA_EVENT True [128] [0] [1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024] 1 8 8 9 8 1024.0 0.1111111111111111 q128_8q1s1k fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
38 0.0 0.0 0.0 0.0 0.0 0.014655999839305878 0.02611199952661991 0.019635199941694735 0.018112000077962875 0.0038298050749257795 0.04189529417991216 0.057484239920526384 0.04744885718421094 0.04744885718421094 0.004779830748455743 0.051672703037266184 0.07089975418195164 0.05852234134479412 0.05852234134479412 0.005895334539786378 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 1 40960 17 0 0 True FLASH_ATTN False true_mixed_fused_projected 144 False 128 BF16 generic none CUDA_EVENT True [128] [0] [1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024] 1 16 16 17 16 1024.0 0.058823529411764705 q128_16q1s1k fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
39 0.0 0.0 0.0 0.0 0.0 0.014720000326633453 0.029311999678611755 0.018662399891763926 0.01673599984496832 0.0044017112162725355 0.04322973959325901 0.05049827064705393 0.04535414343408448 0.04535414343408448 0.0022944156000240697 0.08733025617719538 0.10201372836398578 0.09162185574241775 0.09162185574241775 0.004635047631846023 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 1 40960 17 0 0 True FLASH_ATTN False true_mixed_fused_projected 272 False 256 BF16 generic none CUDA_EVENT True [256] [0] [2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048] 1 16 16 17 16 2048.0 0.058823529411764705 q256_16q1s2k fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
40 0.0 0.0 0.0 0.0 0.0 0.014592000283300877 0.026367999613285065 0.016336000058799982 0.01515199989080429 0.003415353455946402 0.06031842775160765 0.06618323188375198 0.06274415549817247 0.06274415549817247 0.001985647289644255 0.14768157653992678 0.16204076249052324 0.15362064543185072 0.15362064543185072 0.004861590945216247 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 1 40960 33 0 0 True FLASH_ATTN False true_mixed_fused_projected 288 False 256 BF16 generic none CUDA_EVENT True [256] [0] [2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048] 1 32 32 33 32 2048.0 0.030303030303030304 q256_32q1s2k fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
41 0.0 0.0 0.0 0.0 0.0 0.014751999638974667 0.02454400062561035 0.017167999967932702 0.016191999427974224 0.0028685016454498436 0.09128700688359712 0.09689150775996329 0.09360396051475776 0.09360396051475776 0.0013477542744916764 0.2740889887523892 0.2909164873209846 0.2810456356995366 0.2810456356995366 0.004046628526808561 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 1 40960 33 0 0 True FLASH_ATTN False true_mixed_fused_projected 544 False 512 BF16 generic none CUDA_EVENT True [512] [0] [4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096] 1 32 32 33 32 4096.0 0.030303030303030304 q512_32q1s4k fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
42 0.0 0.0 0.0 0.0 0.0 0.01881599985063076 0.03097599931061268 0.024598400108516216 0.024848000146448612 0.0037539437391565975 0.1035249255866932 0.10603132147437412 0.10399797220840973 0.10399797220840973 0.0007025109429056814 0.5652750707893444 0.5789606938874117 0.5678580377517648 0.5678580377517648 0.003835906384194897 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 1 40960 65 0 0 True FLASH_ATTN False true_mixed_fused_projected 576 False 512 BF16 generic none CUDA_EVENT True [512] [0] [4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096] 1 64 64 65 64 4096.0 0.015384615384615385 q512_64q1s4k fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
43 0.0 0.0 0.0 0.0 0.0 0.018624000251293182 0.043487999588251114 0.024460799992084503 0.019600000232458115 0.008922081629781394 0.196169204945307 0.2076140047945799 0.2008444429250931 0.2008444429250931 0.00394350953292805 1.1196708009293268 1.1849940417370974 1.1463555573610091 1.1463555573610091 0.022508285530529783 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 1 40960 65 0 0 True FLASH_ATTN False true_mixed_fused_projected 1088 False 1024 BF16 generic none CUDA_EVENT True [1024] [0] [8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] 1 64 64 65 64 8192.0 0.015384615384615385 q1k_64q1s8k fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
44 0.0 0.0 0.0 0.0 0.0 0.027135999873280525 0.03667199984192848 0.02945920005440712 0.028256000019609928 0.0029446678408169553 0.37757279619664613 0.3898113624476372 0.38075520430942184 0.38075520430942184 0.0036600203831042254 0.2522831942132049 0.2604606493092597 0.25440958703617433 0.25440958703617433 0.0024455194930253126 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 1 40960 33 0 0 True FLASH_ATTN False true_mixed_fused_projected 2080 False 2048 BF16 generic none CUDA_EVENT True [2048] [0] [4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096] 1 32 32 33 32 4096.0 0.030303030303030304 q2k_32q1s4k fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
45 0.0 0.0 0.0 0.0 0.0 0.0435199998319149 0.049695998430252075 0.04502719938755036 0.04395199939608574 0.002035785660169308 1.1048984388245497 1.1189053886476108 1.1112371236754128 1.1112371236754128 0.0050220696724624985 0.1395495077239122 0.14131859607071193 0.14035008841031085 0.14035008841031085 0.0006342911944856293 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 1 40960 17 0 0 True FLASH_ATTN False true_mixed_fused_projected 4112 False 4096 BF16 generic none CUDA_EVENT True [4096] [0] [4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096] 1 16 16 17 16 4096.0 0.058823529411764705 q4k_16q1s4k fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
46 0.0 0.0 0.0 0.0 0.0 0.018783999606966972 0.024288000538945198 0.020627199858427047 0.019567999988794327 0.0019457052717059358 0.09455999732017517 0.12185599654912949 0.10207359939813614 0.10207359939813614 0.00753346544014261 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 1 40960 2 1024 0.0 True FLASH_ATTN False vllm020_batch_spec [512, 512] 1024 512 512 512.0 True 0.0 0.0 0.0 False 0.0 1024.0 1024 BF16 generic none CUDA_EVENT False 2q512 measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
47 0.0 0.0 0.0 0.0 0.0 0.027295999228954315 0.034752000123262405 0.02943360023200512 0.028528000228106976 0.0023159160681123767 0.14416000247001648 0.15904000401496887 0.14979200065135959 0.14979200065135959 0.00480624675866005 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 1 40960 4 2048 0.0 True FLASH_ATTN False vllm020_batch_spec [512, 512, 512, 512] 2048 512 512 512.0 True 0.0 0.0 0.0 False 0.0 2048.0 2048 BF16 generic none CUDA_EVENT False 4q512 measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
48 0.0 0.0 0.0 0.0 0.0 0.043455999344587326 0.05084799975156784 0.045500800386071204 0.04411200061440468 0.002490556062831049 0.24454399943351746 0.25865599513053894 0.2504959970712662 0.2504959970712662 0.003778494140301353 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 1 40960 8 4096 0.0 True FLASH_ATTN False vllm020_batch_spec [512, 512, 512, 512, 512, 512, 512, 512] 4096 512 512 512.0 True 0.0 0.0 0.0 False 0.0 4096.0 4096 BF16 generic none CUDA_EVENT False 8q512 measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
49 0.0 0.0 0.0 0.0 0.0 0.07673600316047668 0.08374399691820145 0.07970559895038605 0.07873599976301193 0.0022907976135004057 0.445248007774353 0.4758400022983551 0.45409599840641024 0.45409599840641024 0.009133230340383306 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 1 40960 16 8192 0.0 True FLASH_ATTN False vllm020_batch_spec [512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512] 8192 512 512 512.0 True 0.0 0.0 0.0 False 0.0 8192.0 8192 BF16 generic none CUDA_EVENT False 16q512 measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
50 0.0 0.0 0.0 0.0 0.0 0.043296001851558685 0.051072001457214355 0.044972800090909 0.04399999976158142 0.002451216824441962 0.6147199869155884 0.6290879845619202 0.6204927921295166 0.6204927921295166 0.004344846739788608 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 1 40960 2 4096 0.0 True FLASH_ATTN False vllm020_batch_spec [2048, 2048] 4096 2048 2048 2048.0 True 0.0 0.0 0.0 False 0.0 4096.0 4096 BF16 generic none CUDA_EVENT False 2q2k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
51 0.0 0.0 0.0 0.0 0.0 0.07737600058317184 0.09123200178146362 0.08094720020890236 0.07980800047516823 0.004065264798368611 1.1744320392608643 1.1887680292129517 1.178323209285736 1.178323209285736 0.004395876469319665 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 1 40960 4 8192 0.0 True FLASH_ATTN False vllm020_batch_spec [2048, 2048, 2048, 2048] 8192 2048 2048 2048.0 True 0.0 0.0 0.0 False 0.0 8192.0 8192 BF16 generic none CUDA_EVENT False 4q2k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
52 0.0 0.0 0.0 0.0 0.0 0.014688000082969666 0.03254399821162224 0.01810879958793521 0.01611199975013733 0.005058478889747962 0.0 0.0 0.0 0.0 0.0 0.06339199841022491 0.0841279998421669 0.07019519805908202 0.07019519805908202 0.006070126182471763 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 1 40960 1 0 16384.0 False FLASH_ATTN False vllm020_batch_spec [1] 1 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False q1s16k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
53 0.0 0.0 0.0 0.0 0.0 0.01484800036996603 0.030271999537944794 0.018124799989163876 0.015711999498307705 0.004696400814632773 0.0 0.0 0.0 0.0 0.0 0.2739199995994568 0.2922239899635315 0.28281279802322384 0.28281279802322384 0.00633037978599564 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 1 40960 8 0 16384.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1] 8 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 8q1s16k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
54 0.0 0.0 0.0 0.0 0.0 0.014495999552309513 0.022431999444961548 0.016844799742102623 0.015471999999135733 0.0027624803153841917 0.0 0.0 0.0 0.0 0.0 0.3909119963645935 0.4160960018634796 0.39996159672737125 0.39996159672737125 0.007037174636804898 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 1 40960 16 0 16384.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] 16 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 16q1s16k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
55 0.0 0.0 0.0 0.0 0.0 0.014240000396966934 0.035071998834609985 0.018764800019562246 0.015583999920636415 0.006305594700523817 0.0 0.0 0.0 0.0 0.0 0.7383679747581482 0.7597119808197021 0.7430047929286957 0.7430047929286957 0.006192645960911466 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 1 40960 32 0 16384.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] 32 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 32q1s16k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
56 0.0 0.0 0.0 0.0 0.0 0.014399999752640724 0.028991999104619026 0.019401599932461978 0.01665600063279271 0.005434953793780546 0.0 0.0 0.0 0.0 0.0 1.427008032798767 1.4517120122909546 1.4361984014511109 1.4361984014511109 0.008802755821028187 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 1 40960 64 0 16384.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] 64 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 64q1s16k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
57 0.0 0.0 0.0 0.0 0.0 0.014271999709308147 0.02844800055027008 0.01820160010829568 0.014944000169634819 0.004942002036909087 0.0 0.0 0.0 0.0 0.0 0.08361600339412689 0.11027199774980545 0.09058240056037903 0.09058240056037903 0.00872938604423622 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 1 40960 1 0 32768.0 False FLASH_ATTN False vllm020_batch_spec [1] 1 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False q1s32k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
58 0.0 0.0 0.0 0.0 0.0 0.014112000353634357 0.023104000836610794 0.01648960020393133 0.015632000286132097 0.0027628828340349578 0.0 0.0 0.0 0.0 0.0 0.5063040256500244 0.5497599840164185 0.5168287932872773 0.5168287932872773 0.013557229606313293 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 1 40960 8 0 32768.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1] 8 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 8q1s32k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
59 0.0 0.0 0.0 0.0 0.0 0.01616000011563301 0.029055999591946602 0.018780799955129622 0.017215999774634838 0.003794434947431941 0.0 0.0 0.0 0.0 0.0 0.7439360022544861 0.7719680070877075 0.7502080142498017 0.7502080142498017 0.00839205598262567 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 1 40960 16 0 32768.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] 16 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 16q1s32k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
60 0.0 0.0 0.0 0.0 0.0 0.015968000516295433 0.03046399913728237 0.018822400271892546 0.01726400014013052 0.004212536831927695 0.0 0.0 0.0 0.0 0.0 1.4256000518798828 1.449504017829895 1.4325888037681578 1.4325888037681578 0.0065448028108711165 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 1 40960 32 0 32768.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] 32 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 32q1s32k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
61 0.0 0.0 0.0 0.0 0.0 0.014271999709308147 0.027712000533938408 0.01633920017629862 0.014944000169634819 0.0038488220071983326 0.0 0.0 0.0 0.0 0.0 2.798719882965088 3.0184640884399414 2.8293471813201903 2.8293471813201903 0.06375662767704417 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 1 40960 64 0 32768.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] 64 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 64q1s32k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
62 0.0 0.0 0.0 0.0 0.0 0.014720000326633453 0.021407999098300934 0.01637439979240298 0.014992000069469213 0.0024463594774459265 0.0 0.0 0.0 0.0 0.0 0.09347199648618698 0.10473600029945374 0.09820479974150656 0.09820479974150656 0.004061668684999473 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 1 40960 1 0 40960.0 False FLASH_ATTN False vllm020_batch_spec [1] 1 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False q1s40k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
63 0.0 0.0 0.0 0.0 0.0 0.014303999952971935 0.04790399968624115 0.020179199893027543 0.016447999514639378 0.009635010283506967 0.0 0.0 0.0 0.0 0.0 0.6221439838409424 0.6444799900054932 0.6277984082698822 0.6277984082698822 0.007234140179397069 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 1 40960 8 0 40960.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1] 8 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 8q1s40k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
64 0.0 0.0 0.0 0.0 0.0 0.014271999709308147 0.03276799991726875 0.01921279989182949 0.015232000034302473 0.0065506148511265076 0.0 0.0 0.0 0.0 0.0 0.9111359715461731 0.9307839870452881 0.9151648044586183 0.9151648044586183 0.005528712062927265 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 1 40960 16 0 40960.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] 16 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 16q1s40k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
65 0.0 0.0 0.0 0.0 0.0 0.014175999909639359 0.02707199938595295 0.016512000095099212 0.014688000082969666 0.0038989080209321414 0.0 0.0 0.0 0.0 0.0 1.7645119428634644 1.7849279642105103 1.7684095859527589 1.7684095859527589 0.005876963340184193 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 1 40960 32 0 40960.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] 32 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 32q1s40k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
66 0.0 0.0 0.0 0.0 0.0 0.014240000396966934 0.029184000566601753 0.016825600154697896 0.014752000104635954 0.00457910514148248 0.0 0.0 0.0 0.0 0.0 3.4781761169433594 3.5388801097869873 3.490892815589905 3.490892815589905 0.01818910600692522 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 1 40960 64 0 40960.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] 64 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 64q1s40k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
67 0.0 0.0 0.0 0.0 0.0 0.014976000413298607 0.03139200061559677 0.01898880014196038 0.016848000697791576 0.005083715524393418 0.13752702814163098 0.14117629917511842 0.13848196486571557 0.13848196486571557 0.0009880564964713527 0.523336967410432 0.5372236808930884 0.5269708253945994 0.5269708253945994 0.00375988994658546 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 1 40960 9 0 0 True FLASH_ATTN False true_mixed_fused_projected 520 False 512 BF16 generic none CUDA_EVENT True [512] [0] [16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] 1 8 8 9 8 16384.0 0.1111111111111111 q512_8q1s16k fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
68 0.0 0.0 0.0 0.0 0.0 0.01833599992096424 0.02707199938595295 0.020995199866592883 0.01976000051945448 0.0028392656352050185 0.17817885890237703 0.18622915300900206 0.18109068484526028 0.18109068484526028 0.0027040006080457624 0.5449571488834201 0.5695788427872015 0.5538629212834322 0.5538629212834322 0.008270141985514729 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 1 40960 17 0 0 True FLASH_ATTN False true_mixed_fused_projected 1040 False 1024 BF16 generic none CUDA_EVENT True [1024] [0] [16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] 1 16 16 17 16 16384.0 0.058823529411764705 q1k_16q1s16k fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
69 0.0 0.0 0.0 0.0 0.0 0.02703999914228916 0.03308799862861633 0.028883199393749236 0.02759999968111515 0.002285758578932753 0.4688266550410815 0.48301665772804186 0.4727750380198454 0.4727750380198454 0.004396396389258256 1.0430453981052807 1.0746153117238624 1.051829759343436 1.051829759343436 0.009781101336187197 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 1 40960 33 0 0 True FLASH_ATTN False true_mixed_fused_projected 2080 False 2048 BF16 generic none CUDA_EVENT True [2048] [0] [16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] 1 32 32 33 32 16384.0 0.030303030303030304 q2k_32q1s16k fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
70 0.0 0.0 0.0 0.0 0.0 0.04368000105023384 0.059328000992536545 0.04809600040316582 0.04617599956691265 0.005158792915002645 1.3509461459747514 1.3759066409666496 1.3589365122072 1.3589365122072 0.008008877716183384 0.9213738861449998 0.9383974724214122 0.9268234851606594 0.9268234851606594 0.005462224239661061 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 1 40960 17 0 0 True FLASH_ATTN False true_mixed_fused_projected 4112 False 4096 BF16 generic none CUDA_EVENT True [4096] [0] [32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] 1 16 16 17 16 32768.0 0.058823529411764705 q4k_16q1s32k fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
71 0.0 0.0 0.0 0.0 0.0 0.026944000273942947 0.040511999279260635 0.030291200056672095 0.02817599941045046 0.004386386489305873 0.5071830964059985 0.5142792136001758 0.5096392092471257 0.5096392092471257 0.0022059272685869546 2.175632932188972 2.2060727208328075 2.186168772243963 2.186168772243963 0.009462633998570079 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 1 40960 33 0 0 True FLASH_ATTN False true_mixed_fused_projected 2080 False 2048 BF16 generic none CUDA_EVENT True [2048] [0] [32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] 1 32 32 33 32 32768.0 0.030303030303030304 q2k_32q1s32k fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
72 0.0 0.0 0.0 0.0 0.0 0.018303999677300453 0.03017600066959858 0.02039040010422468 0.018943999893963337 0.0034612051983613614 0.21341429693945616 0.21552776300295579 0.21393496609766816 0.21393496609766816 0.0005736773262009222 4.617401495144284 4.663128147488988 4.628666619290515 4.628666619290515 0.012412001359412127 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 1 40960 65 0 0 True FLASH_ATTN False true_mixed_fused_projected 1088 False 1024 BF16 generic none CUDA_EVENT True [1024] [0] [32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] 1 64 64 65 64 32768.0 0.015384615384615385 q1k_64q1s32k fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
73 0.0 0.0 0.0 0.0 0.0 0.014720000326633453 0.02364799939095974 0.01809599995613098 0.016352000646293163 0.0035481127058959038 0.04822399839758873 0.08566399663686752 0.05961279980838299 0.05961279980838299 0.011445665413968877 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 2 40960 1 64 0.0 True FLASH_ATTN False vllm020_batch_spec [64] 64 64 64 64.0 True 0.0 0.0 0.0 False 0.0 64.0 64 BF16 generic none CUDA_EVENT False q64 measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
74 0.0 0.0 0.0 0.0 0.0 0.014175999909639359 0.020864000543951988 0.01668160008266568 0.015664000064134598 0.0025769063833097584 0.049695998430252075 0.08057600259780884 0.05882879942655563 0.05882879942655563 0.009515126108519331 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 2 40960 1 128 0.0 True FLASH_ATTN False vllm020_batch_spec [128] 128 128 128 128.0 True 0.0 0.0 0.0 False 0.0 128.0 128 BF16 generic none CUDA_EVENT False q128 measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
75 0.0 0.0 0.0 0.0 0.0 0.014399999752640724 0.028704000636935234 0.017430400010198355 0.015056000091135502 0.004349294613335555 0.049855999648571014 0.07366400212049484 0.05459520071744919 0.05459520071744919 0.0069610920757925574 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 2 40960 1 256 0.0 True FLASH_ATTN False vllm020_batch_spec [256] 256 256 256 256.0 True 0.0 0.0 0.0 False 0.0 256.0 256 BF16 generic none CUDA_EVENT False q256 measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
76 0.0 0.0 0.0 0.0 0.0 0.014336000196635723 0.03161599859595299 0.017788799852132796 0.015711999963968992 0.005005385723318659 0.06102399900555611 0.08179199695587158 0.06650560013949873 0.06650560013949873 0.006947995595801105 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 2 40960 1 512 0.0 True FLASH_ATTN False vllm020_batch_spec [512] 512 512 512 512.0 True 0.0 0.0 0.0 False 0.0 512.0 512 BF16 generic none CUDA_EVENT False q512 measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
77 0.0 0.0 0.0 0.0 0.0 0.014655999839305878 0.04416000097990036 0.019200000166893005 0.015488000120967627 0.008561241323364038 0.08054400235414505 0.09196799993515015 0.08607039973139763 0.08607039973139763 0.004145329035483754 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 2 40960 1 1024 0.0 True FLASH_ATTN False vllm020_batch_spec [1024] 1024 1024 1024 1024.0 True 0.0 0.0 0.0 False 0.0 1024.0 1024 BF16 generic none CUDA_EVENT False q1k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
78 0.0 0.0 0.0 0.0 0.0 0.017855999991297722 0.023391999304294586 0.019667199812829494 0.018463999964296818 0.0022577669687832585 0.18729600310325623 0.20585599541664124 0.19546559900045393 0.19546559900045393 0.006339068824303663 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 2 40960 1 2048 0.0 True FLASH_ATTN False vllm020_batch_spec [2048] 2048 2048 2048 2048.0 True 0.0 0.0 0.0 False 0.0 2048.0 2048 BF16 generic none CUDA_EVENT False q2k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
79 0.0 0.0 0.0 0.0 0.0 0.026240000501275063 0.03446400165557861 0.028297600522637367 0.027088000439107418 0.0025148441522922374 0.5754240155220032 0.5889919996261597 0.5800191938877105 0.5800191938877105 0.003858869273829596 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 2 40960 1 4096 0.0 True FLASH_ATTN False vllm020_batch_spec [4096] 4096 4096 4096 4096.0 True 0.0 0.0 0.0 False 0.0 4096.0 4096 BF16 generic none CUDA_EVENT False q4k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
80 0.0 0.0 0.0 0.0 0.0 0.04182400181889534 0.047200001776218414 0.043036799877882004 0.0423360001295805 0.0017073405772076728 2.063199996948242 2.0787200927734375 2.067151999473572 2.067151999473572 0.004959651271127835 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 2 40960 1 8192 0.0 True FLASH_ATTN False vllm020_batch_spec [8192] 8192 8192 8192 8192.0 True 0.0 0.0 0.0 False 0.0 8192.0 8192 BF16 generic none CUDA_EVENT False q8k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
81 0.0 0.0 0.0 0.0 0.0 0.014399999752640724 0.05648000165820122 0.02270399993285537 0.017935999669134617 0.012377915150727689 0.049536000937223434 0.07196799665689468 0.056015999615192415 0.056015999615192415 0.0070476637552742884 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 2 40960 1 64 448.0 True FLASH_ATTN False vllm020_batch_spec [64] 64 64 64 64.0 True 0.0 0.0 0.0 True 448.0 512.0 64 BF16 generic none CUDA_EVENT False q64s512 measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
82 0.0 0.0 0.0 0.0 0.0 0.01408000010997057 0.021344000473618507 0.01611520005390048 0.014928000047802925 0.0025499884993961702 0.07072000205516815 0.2642880082130432 0.1588256008923054 0.1588256008923054 0.053220347086102376 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 2 40960 1 128 896.0 True FLASH_ATTN False vllm020_batch_spec [128] 128 128 128 128.0 True 0.0 0.0 0.0 True 896.0 1024.0 128 BF16 generic none CUDA_EVENT False q128s1k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
83 0.0 0.0 0.0 0.0 0.0 0.01360000018030405 0.03014400042593479 0.016975999902933837 0.015056000091135502 0.004715159697364111 0.08393599838018417 0.11036799848079681 0.09160000011324881 0.09160000011324881 0.007911669434472792 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 2 40960 1 256 1792.0 True FLASH_ATTN False vllm020_batch_spec [256] 256 256 256 256.0 True 0.0 0.0 0.0 True 1792.0 2048.0 256 BF16 generic none CUDA_EVENT False q256s2k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
84 0.0 0.0 0.0 0.0 0.0 0.014495999552309513 0.020767999812960625 0.016233599931001663 0.015392000321298838 0.002202615907995427 0.1998399943113327 0.22070400416851044 0.20855360180139543 0.20855360180139543 0.00689307200230667 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 2 40960 1 512 3584.0 True FLASH_ATTN False vllm020_batch_spec [512] 512 512 512 512.0 True 0.0 0.0 0.0 True 3584.0 4096.0 512 BF16 generic none CUDA_EVENT False q512s4k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
85 0.0 0.0 0.0 0.0 0.0 0.01484800036996603 0.033440001308918 0.018684800155460833 0.016048000194132328 0.00553991005639174 0.6043199896812439 0.635807991027832 0.6126143991947173 0.6126143991947173 0.008745933953408096 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 2 40960 1 1024 7168.0 True FLASH_ATTN False vllm020_batch_spec [1024] 1024 1024 1024 1024.0 True 0.0 0.0 0.0 True 7168.0 8192.0 1024 BF16 generic none CUDA_EVENT False q1ks8k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
86 0.0 0.0 0.0 0.0 0.0 0.013824000023305416 0.03359999880194664 0.017811199743300678 0.014944000169634819 0.005926140483565472 0.0 0.0 0.0 0.0 0.0 0.045471999794244766 0.07539200037717819 0.054758400097489356 0.054758400097489356 0.010253548506101549 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 2 40960 1 0 128.0 False FLASH_ATTN False vllm020_batch_spec [1] 1 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False q1s128 measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
87 0.0 0.0 0.0 0.0 0.0 0.014047999866306782 0.02409599907696247 0.01641279999166727 0.01473599998280406 0.003347158638713987 0.0 0.0 0.0 0.0 0.0 0.0461760014295578 0.07529599964618683 0.05460800044238568 0.05460800044238568 0.009748937798340135 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 2 40960 8 0 128.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1] 8 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 8q1s128 measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
88 0.0 0.0 0.0 0.0 0.0 0.014271999709308147 0.028416000306606293 0.018611199874430894 0.01598400017246604 0.005209875093863647 0.0 0.0 0.0 0.0 0.0 0.048128001391887665 0.07897599786520004 0.061353600397706036 0.061353600397706036 0.010488153655157845 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 2 40960 16 0 128.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] 16 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 16q1s128 measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
89 0.0 0.0 0.0 0.0 0.0 0.014112000353634357 0.025728000327944756 0.016092800162732603 0.01512000011280179 0.003300888123052605 0.0 0.0 0.0 0.0 0.0 0.04864000156521797 0.07648000121116638 0.05810560062527656 0.05810560062527656 0.009473544218404408 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 2 40960 32 0 128.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] 32 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 32q1s128 measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
90 0.0 0.0 0.0 0.0 0.0 0.014655999839305878 0.03551999852061272 0.018588799890130757 0.016064000315964222 0.006071057437992447 0.0 0.0 0.0 0.0 0.0 0.04822399839758873 0.07862400263547897 0.05660480037331582 0.05660480037331582 0.009565394213730401 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 2 40960 64 0 128.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] 64 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 64q1s128 measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
91 0.0 0.0 0.0 0.0 0.0 0.014303999952971935 0.028831999748945236 0.01699519995599985 0.015024000313133001 0.004285000744868188 0.0 0.0 0.0 0.0 0.0 0.04854400083422661 0.06719999760389328 0.05778240002691746 0.05778240002691746 0.006852125554679805 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 2 40960 8 0 1024.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1] 8 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 8q1s1k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
92 0.0 0.0 0.0 0.0 0.0 0.0144640002399683 0.024927999824285507 0.0161183999851346 0.01521599991247058 0.0029774808524673907 0.0 0.0 0.0 0.0 0.0 0.05379199981689453 0.08966399729251862 0.06270079985260964 0.06270079985260964 0.009983591277092696 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 2 40960 16 0 1024.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] 16 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 16q1s1k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
93 0.0 0.0 0.0 0.0 0.0 0.014431999996304512 0.03001599945127964 0.017648000083863736 0.01547200046479702 0.004585126309484848 0.0 0.0 0.0 0.0 0.0 0.061664000153541565 0.07692799717187881 0.06704320013523103 0.06704320013523103 0.005500191798490629 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 2 40960 32 0 1024.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] 32 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 32q1s1k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
94 0.0 0.0 0.0 0.0 0.0 0.014175999909639359 0.026367999613285065 0.016947199776768684 0.015039999969303608 0.0037951566103550205 0.0 0.0 0.0 0.0 0.0 0.08799999952316284 0.111455999314785 0.0964031994342804 0.0964031994342804 0.007397541615558088 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 2 40960 64 0 1024.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] 64 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 64q1s1k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
95 0.0 0.0 0.0 0.0 0.0 0.01369599997997284 0.021247999742627144 0.015359999984502793 0.014640000183135271 0.0020942770950814317 0.0 0.0 0.0 0.0 0.0 0.051711998879909515 0.07065600156784058 0.058054400235414506 0.058054400235414506 0.006633034815910025 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 2 40960 8 0 2048.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1] 8 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 8q1s2k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
96 0.0 0.0 0.0 0.0 0.0 0.014208000153303146 0.0307839997112751 0.017148799914866685 0.015039999969303608 0.004882374686312284 0.0 0.0 0.0 0.0 0.0 0.061919998377561576 0.07843200117349625 0.06628479920327664 0.06628479920327664 0.004962852689801192 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 2 40960 16 0 2048.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] 16 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 16q1s2k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
97 0.0 0.0 0.0 0.0 0.0 0.013887999579310417 0.02969600073993206 0.017500799987465142 0.015232000034302473 0.00467273319705314 0.0 0.0 0.0 0.0 0.0 0.08819200098514557 0.11097600311040878 0.09493440166115762 0.09493440166115762 0.007509235042985577 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 2 40960 32 0 2048.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] 32 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 32q1s2k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
98 0.0 0.0 0.0 0.0 0.0 0.014399999752640724 0.022112000733613968 0.01751680001616478 0.016047999262809753 0.0030306621792915785 0.0 0.0 0.0 0.0 0.0 0.13065600395202637 0.15110400319099426 0.13857279866933822 0.13857279866933822 0.00750137841249771 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 2 40960 64 0 2048.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] 64 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 64q1s2k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
99 0.0 0.0 0.0 0.0 0.0 0.013919999822974205 0.04012800008058548 0.01892479993402958 0.01550400024279952 0.007459545267816961 0.0 0.0 0.0 0.0 0.0 0.06278400123119354 0.08259200304746628 0.0704512007534504 0.0704512007534504 0.005979055984382744 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 2 40960 8 0 4096.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1] 8 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 8q1s4k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
100 0.0 0.0 0.0 0.0 0.0 0.014336000196635723 0.02236800082027912 0.016332800220698118 0.014928000047802925 0.002784044363186731 0.0 0.0 0.0 0.0 0.0 0.1003199964761734 0.1279360055923462 0.10921279862523078 0.10921279862523078 0.00862769617046716 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 2 40960 16 0 4096.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] 16 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 16q1s4k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
101 0.0 0.0 0.0 0.0 0.0 0.013919999822974205 0.0225600004196167 0.016128000058233737 0.014800000004470348 0.002958953332547708 0.0 0.0 0.0 0.0 0.0 0.13116799294948578 0.14812800288200378 0.13783999979496003 0.13783999979496003 0.005696384361148053 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 2 40960 32 0 4096.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] 32 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 32q1s4k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
102 0.0 0.0 0.0 0.0 0.0 0.014208000153303146 0.029983999207615852 0.017468800116330386 0.01508800033479929 0.0047379550962483065 0.0 0.0 0.0 0.0 0.0 0.217631995677948 0.2447360008955002 0.22715839892625808 0.22715839892625808 0.008831828221847138 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 2 40960 64 0 4096.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] 64 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 64q1s4k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
103 0.0 0.0 0.0 0.0 0.0 0.014047999866306782 0.02304000034928322 0.016512000095099212 0.014512000139802694 0.0035026774828624254 0.0 0.0 0.0 0.0 0.0 0.11020799726247787 0.12307199835777283 0.11600959971547126 0.11600959971547126 0.004667637266950902 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 2 40960 8 0 8192.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1] 8 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 8q1s8k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
104 0.0 0.0 0.0 0.0 0.0 0.014112000353634357 0.042080000042915344 0.017510399967432023 0.014607999939471483 0.00823117883530167 0.0 0.0 0.0 0.0 0.0 0.15702399611473083 0.17587199807167053 0.16399359852075576 0.16399359852075576 0.006588299074676393 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 2 40960 16 0 8192.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] 16 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 16q1s8k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
105 0.0 0.0 0.0 0.0 0.0 0.013919999822974205 0.0208320003002882 0.015299199987202883 0.01462399959564209 0.001916768086505386 0.0 0.0 0.0 0.0 0.0 0.21779200434684753 0.2415360063314438 0.2260768011212349 0.2260768011212349 0.007251352080685236 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 2 40960 32 0 8192.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] 32 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 32q1s8k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
106 0.0 0.0 0.0 0.0 0.0 0.014015999622642994 0.021088000386953354 0.015619200188666582 0.01473599998280406 0.002013227452302141 0.0 0.0 0.0 0.0 0.0 0.3959999978542328 0.4152640104293823 0.40332479774951924 0.40332479774951924 0.006942401431914052 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 2 40960 64 0 8192.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] 64 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 64q1s8k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
107 0.0 0.0 0.0 0.0 0.0 0.014175999909639359 0.03379200026392937 0.020595200080424547 0.019600000232458115 0.006548754248881344 0.026192623739694512 0.040006882507168426 0.029155180178492036 0.029155180178492036 0.00408828748438241 0.024591375524545756 0.037561119537986146 0.027372820355088746 0.027372820355088746 0.0038383559348575944 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 2 40960 9 0 0 True FLASH_ATTN False true_mixed_fused_projected 72 False 64 BF16 generic none CUDA_EVENT True [64] [0] [512, 512, 512, 512, 512, 512, 512, 512] 1 8 8 9 8 512.0 0.1111111111111111 q64_8q1s512 fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
108 0.0 0.0 0.0 0.0 0.0 0.013856000266969204 0.020896000787615776 0.015318400040268899 0.014431999996304512 0.0019640312960926966 0.029349018208693862 0.06236262941356679 0.03714313592014963 0.03714313592014963 0.009618313791872569 0.028826981462526914 0.06125337308649041 0.036482463672774496 0.036482463672774496 0.009447230957011863 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 2 40960 9 0 0 True FLASH_ATTN False true_mixed_fused_projected 136 False 128 BF16 generic none CUDA_EVENT True [128] [0] [1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024] 1 8 8 9 8 1024.0 0.1111111111111111 q128_8q1s1k fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
109 0.0 0.0 0.0 0.0 0.0 0.013856000266969204 0.032127998769283295 0.017286399938166143 0.014479999896138906 0.005536495446636193 0.03273085874558354 0.04394578491937082 0.03563837490653034 0.03563837490653034 0.003429601730256567 0.03488514202593899 0.04683821345079978 0.037984025407085474 0.037984025407085474 0.0036553316361902684 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 2 40960 17 0 0 True FLASH_ATTN False true_mixed_fused_projected 144 False 128 BF16 generic none CUDA_EVENT True [128] [0] [1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024] 1 16 16 17 16 1024.0 0.058823529411764705 q128_16q1s1k fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
110 0.0 0.0 0.0 0.0 0.0 0.013856000266969204 0.021503999829292297 0.015078400075435639 0.01425600005313754 0.002238435975478849 0.042100813549974185 0.051784144690147756 0.045378692890289625 0.045378692890289625 0.003184109940650555 0.05111518843748309 0.06287185663927425 0.05509490773570219 0.05509490773570219 0.0038658725544299132 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 2 40960 17 0 0 True FLASH_ATTN False true_mixed_fused_projected 272 False 256 BF16 generic none CUDA_EVENT True [256] [0] [2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048] 1 16 16 17 16 2048.0 0.058823529411764705 q256_16q1s2k fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
111 0.0 0.0 0.0 0.0 0.0 0.014399999752640724 0.02687999978661537 0.017667199857532977 0.01566399959847331 0.003864811846387173 0.048475323773821306 0.05599956978723733 0.05123499252968345 0.05123499252968345 0.002427379820423941 0.08429268135885387 0.09737642835214408 0.0890914090616175 0.0890914090616175 0.0042209177332077005 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 2 40960 33 0 0 True FLASH_ATTN False true_mixed_fused_projected 288 False 256 BF16 generic none CUDA_EVENT True [256] [0] [2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048] 1 32 32 33 32 2048.0 0.030303030303030304 q256_32q1s2k fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
112 0.0 0.0 0.0 0.0 0.0 0.01462399959564209 0.03359999880194664 0.019804799742996693 0.016032000072300434 0.006750519908145474 0.07121508474579985 0.07292308367136396 0.07194410845270183 0.07194410845270183 0.0005500065915943844 0.14760091249712767 0.15114092373010238 0.14911189243564577 0.14911189243564577 0.0011399477384397005 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 2 40960 33 0 0 True FLASH_ATTN False true_mixed_fused_projected 544 False 512 BF16 generic none CUDA_EVENT True [512] [0] [4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096] 1 32 32 33 32 4096.0 0.030303030303030304 q512_32q1s4k fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
113 0.0 0.0 0.0 0.0 0.0 0.014399999752640724 0.021856000646948814 0.016835200227797033 0.015263999812304974 0.00283854448975065 0.08146338272142935 0.08560865714384096 0.0834932625520734 0.0834932625520734 0.0012259635905347874 0.2782486219401307 0.29240733788179385 0.2851819365989658 0.2851819365989658 0.004187435731481665 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 2 40960 65 0 0 True FLASH_ATTN False true_mixed_fused_projected 576 False 512 BF16 generic none CUDA_EVENT True [512] [0] [4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096] 1 64 64 65 64 4096.0 0.015384615384615385 q512_64q1s4k fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
114 0.0 0.0 0.0 0.0 0.0 0.014399999752640724 0.04495999962091446 0.02143679987639189 0.015856000129133463 0.009709987872683342 0.1194459208702178 0.12302524755181463 0.12053885718696096 0.12053885718696096 0.001005118119341339 0.5597220650459199 0.5764947443228802 0.5648435507167102 0.5648435507167102 0.004709970715400788 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 2 40960 65 0 0 True FLASH_ATTN False true_mixed_fused_projected 1088 False 1024 BF16 generic none CUDA_EVENT True [1024] [0] [8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] 1 64 64 65 64 8192.0 0.015384615384615385 q1k_64q1s8k fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
115 0.0 0.0 0.0 0.0 0.0 0.018112000077962875 0.026496000587940216 0.019318400137126445 0.018432000651955605 0.0024249605112359905 0.19770253574610402 0.222811797868348 0.2054811520619412 0.2054811520619412 0.008000944254868043 0.13941746080159492 0.157124211777114 0.14490284788179197 0.14490284788179197 0.005642170080515992 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 2 40960 33 0 0 True FLASH_ATTN False true_mixed_fused_projected 2080 False 2048 BF16 generic none CUDA_EVENT True [2048] [0] [4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096] 1 32 32 33 32 4096.0 0.030303030303030304 q2k_32q1s4k fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
116 0.0 0.0 0.0 0.0 0.0 0.02595200017094612 0.03155200183391571 0.02727359998971224 0.026575999334454536 0.0016260948764843166 0.6014684881116659 0.6095472988212 0.6046757700946376 0.6046757700946376 0.0023537435563177303 0.11325152264578045 0.11477269562841545 0.11385542721485654 0.11385542721485654 0.0004431903698023628 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 2 40960 17 0 0 True FLASH_ATTN False true_mixed_fused_projected 4112 False 4096 BF16 generic none CUDA_EVENT True [4096] [0] [4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096] 1 16 16 17 16 4096.0 0.058823529411764705 q4k_16q1s4k fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
117 0.0 0.0 0.0 0.0 0.0 0.014303999952971935 0.03359999880194664 0.01814719969406724 0.015375999733805656 0.005678506740662529 0.06774400174617767 0.07891199737787247 0.07312640026211739 0.07312640026211739 0.003826583051422731 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 2 40960 2 1024 0.0 True FLASH_ATTN False vllm020_batch_spec [512, 512] 1024 512 512 512.0 True 0.0 0.0 0.0 False 0.0 1024.0 1024 BF16 generic none CUDA_EVENT False 2q512 measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
118 0.0 0.0 0.0 0.0 0.0 0.01788800023496151 0.028543999418616295 0.021340799890458582 0.019504000432789326 0.003718627037219618 0.09548799693584442 0.1327359974384308 0.10823359936475753 0.10823359936475753 0.013230635292043864 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 2 40960 4 2048 0.0 True FLASH_ATTN False vllm020_batch_spec [512, 512, 512, 512] 2048 512 512 512.0 True 0.0 0.0 0.0 False 0.0 2048.0 2048 BF16 generic none CUDA_EVENT False 4q512 measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
119 0.0 0.0 0.0 0.0 0.0 0.02627200074493885 0.030368000268936157 0.027184000052511693 0.02643200010061264 0.0013545679205210022 0.14364799857139587 0.1597760021686554 0.15008639842271806 0.15008639842271806 0.005367723415171222 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 2 40960 8 4096 0.0 True FLASH_ATTN False vllm020_batch_spec [512, 512, 512, 512, 512, 512, 512, 512] 4096 512 512 512.0 True 0.0 0.0 0.0 False 0.0 4096.0 4096 BF16 generic none CUDA_EVENT False 8q512 measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
120 0.0 0.0 0.0 0.0 0.0 0.04150399938225746 0.04726399853825569 0.042950399965047834 0.04224000126123428 0.001720147501765378 0.2433920055627823 0.28963199257850647 0.2549152016639709 0.2549152016639709 0.013450991019687407 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 2 40960 16 8192 0.0 True FLASH_ATTN False vllm020_batch_spec [512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512] 8192 512 512 512.0 True 0.0 0.0 0.0 False 0.0 8192.0 8192 BF16 generic none CUDA_EVENT False 16q512 measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
121 0.0 0.0 0.0 0.0 0.0 0.025887999683618546 0.03407999873161316 0.028303999826312064 0.026367999613285065 0.0028877495368841042 0.325439989566803 0.3441599905490875 0.33442879617214205 0.33442879617214205 0.005693597811441922 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 2 40960 2 4096 0.0 True FLASH_ATTN False vllm020_batch_spec [2048, 2048] 4096 2048 2048 2048.0 True 0.0 0.0 0.0 False 0.0 4096.0 4096 BF16 generic none CUDA_EVENT False 2q2k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
122 0.0 0.0 0.0 0.0 0.0 0.04137599840760231 0.05084799975156784 0.044828799366950986 0.043087998405098915 0.003560363865797613 0.6110399961471558 0.6421759724617004 0.6204223990440368 0.6204223990440368 0.011214490370794758 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 2 40960 4 8192 0.0 True FLASH_ATTN False vllm020_batch_spec [2048, 2048, 2048, 2048] 8192 2048 2048 2048.0 True 0.0 0.0 0.0 False 0.0 8192.0 8192 BF16 generic none CUDA_EVENT False 4q2k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
123 0.0 0.0 0.0 0.0 0.0 0.014816000126302242 0.02364799939095974 0.016668799985200166 0.01592000015079975 0.0025515238179941247 0.0 0.0 0.0 0.0 0.0 0.0544000007212162 0.09014400094747543 0.06511679962277411 0.06511679962277411 0.013005726711295521 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 2 40960 1 0 16384.0 False FLASH_ATTN False vllm020_batch_spec [1] 1 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False q1s16k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
124 0.0 0.0 0.0 0.0 0.0 0.014527999795973301 0.0208320003002882 0.016604799963533878 0.01566399959847331 0.0021353594266203244 0.0 0.0 0.0 0.0 0.0 0.17871999740600586 0.1961279958486557 0.18568639904260634 0.18568639904260634 0.004566142885274747 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 2 40960 8 0 16384.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1] 8 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 8q1s16k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
125 0.0 0.0 0.0 0.0 0.0 0.014431999996304512 0.02006400004029274 0.015619200002402068 0.01508800033479929 0.001572984783715635 0.0 0.0 0.0 0.0 0.0 0.2730880081653595 0.29721599817276 0.2815328001976013 0.2815328001976013 0.007016534727616923 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 2 40960 16 0 16384.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] 16 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 16q1s16k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
126 0.0 0.0 0.0 0.0 0.0 0.014879999682307243 0.02755199931561947 0.018678399827331306 0.015919999685138464 0.004323555372382858 0.0 0.0 0.0 0.0 0.0 0.3893119990825653 0.44041600823402405 0.40225600004196166 0.40225600004196166 0.013442998165884852 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 2 40960 32 0 16384.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] 32 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 32q1s16k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
127 0.0 0.0 0.0 0.0 0.0 0.014303999952971935 0.02659199945628643 0.016748800035566093 0.015343999955803156 0.0036412341868394082 0.0 0.0 0.0 0.0 0.0 0.7404800057411194 0.9689919948577881 0.8451807916164398 0.8451807916164398 0.08637455920036795 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 2 40960 64 0 16384.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] 64 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 64q1s16k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
128 0.0 0.0 0.0 0.0 0.0 0.014751999638974667 0.02160000056028366 0.016752000153064727 0.01521599991247058 0.0025167650350367246 0.0 0.0 0.0 0.0 0.0 0.06412799656391144 0.08956799656152725 0.07242240011692047 0.07242240011692047 0.009797120588901621 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 2 40960 1 0 32768.0 False FLASH_ATTN False vllm020_batch_spec [1] 1 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False q1s32k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
129 0.0 0.0 0.0 0.0 0.0 0.014495999552309513 0.022911999374628067 0.016137600038200618 0.015343999955803156 0.002330911555470513 0.0 0.0 0.0 0.0 0.0 0.3128319978713989 0.3282879889011383 0.3203647971153259 0.3203647971153259 0.004598568238907996 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 2 40960 8 0 32768.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1] 8 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 8q1s32k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
130 0.0 0.0 0.0 0.0 0.0 0.014655999839305878 0.03574400022625923 0.01919359974563122 0.01583999954164028 0.006642521913489215 0.0 0.0 0.0 0.0 0.0 0.5050879716873169 0.5311999917030334 0.511932796239853 0.511932796239853 0.007150929647317677 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 2 40960 16 0 32768.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] 16 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 16q1s32k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
131 0.0 0.0 0.0 0.0 0.0 0.014655999839305878 0.02595200017094612 0.016992000117897987 0.015456000342965126 0.003390731937723684 0.0 0.0 0.0 0.0 0.0 0.7385600209236145 0.7681919932365417 0.7483008027076722 0.7483008027076722 0.010214373356208012 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 2 40960 32 0 32768.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] 32 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 32q1s32k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
132 0.0 0.0 0.0 0.0 0.0 0.014527999795973301 0.038975998759269714 0.01952639976516366 0.015327999833971262 0.007695927285621061 0.0 0.0 0.0 0.0 0.0 1.4228800535202026 1.5237760543823242 1.4435008168220522 1.4435008168220522 0.0342955291725742 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 2 40960 64 0 32768.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] 64 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 64q1s32k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
133 0.0 0.0 0.0 0.0 0.0 0.014495999552309513 0.03907199949026108 0.01793599994853139 0.015248000156134367 0.007167103363234667 0.0 0.0 0.0 0.0 0.0 0.06947200000286102 0.08857599645853043 0.07469440028071403 0.07469440028071403 0.00583249952929747 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 2 40960 1 0 40960.0 False FLASH_ATTN False vllm020_batch_spec [1] 1 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False q1s40k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
134 0.0 0.0 0.0 0.0 0.0 0.014495999552309513 0.02175999991595745 0.016710399929434062 0.015327999833971262 0.002705383977079183 0.0 0.0 0.0 0.0 0.0 0.388480007648468 0.4073280096054077 0.3966591984033584 0.3966591984033584 0.004487315516095964 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 2 40960 8 0 40960.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1] 8 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 8q1s40k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
135 0.0 0.0 0.0 0.0 0.0 0.0144640002399683 0.022048000246286392 0.017011200170964004 0.015392000321298838 0.002776899649845722 0.0 0.0 0.0 0.0 0.0 0.622048020362854 0.6347839832305908 0.6262047946453094 0.6262047946453094 0.004593360122958751 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 2 40960 16 0 40960.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] 16 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 16q1s40k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
136 0.0 0.0 0.0 0.0 0.0 0.014592000283300877 0.02937600016593933 0.01847040019929409 0.015344000421464443 0.005043809142122341 0.0 0.0 0.0 0.0 0.0 0.9105280041694641 0.9304640293121338 0.914108806848526 0.914108806848526 0.00563919268816644 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 2 40960 32 0 40960.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] 32 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 32q1s40k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
137 0.0 0.0 0.0 0.0 0.0 0.01462399959564209 0.039903998374938965 0.01791359977796674 0.015440000221133232 0.007367547649297217 0.0 0.0 0.0 0.0 0.0 1.764799952507019 1.7965760231018066 1.770739197731018 1.770739197731018 0.009082122770320404 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 2 40960 64 0 40960.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] 64 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 64q1s40k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
138 0.0 0.0 0.0 0.0 0.0 0.015168000012636185 0.03187200054526329 0.018553599901497363 0.016127999871969223 0.0050628774268006264 0.16963527081512533 0.1737871320906266 0.17088926838108623 0.17088926838108623 0.0011703114258121128 0.47362872483230506 0.48522089403242025 0.47712993814280913 0.47712993814280913 0.0032675581298664976 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 2 40960 9 0 0 True FLASH_ATTN False true_mixed_fused_projected 520 False 512 BF16 generic none CUDA_EVENT True [512] [0] [16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] 1 8 8 9 8 16384.0 0.1111111111111111 q512_8q1s16k fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
139 0.0 0.0 0.0 0.0 0.0 0.014879999682307243 0.03868800029158592 0.01921919994056225 0.016959999687969685 0.006691309533030663 0.1589600576212269 0.16117782913137854 0.15969057520605917 0.15969057520605917 0.0006212984448748182 0.5199519263456005 0.5272061673552852 0.5223414198520004 0.5223414198520004 0.002032242112153396 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 2 40960 17 0 0 True FLASH_ATTN False true_mixed_fused_projected 1040 False 1024 BF16 generic none CUDA_EVENT True [1024] [0] [16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] 1 16 16 17 16 16384.0 0.058823529411764705 q1k_16q1s16k fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
140 0.0 0.0 0.0 0.0 0.0 0.018400000408291817 0.024320000782608986 0.019910399988293647 0.0191040001809597 0.001841532763984352 0.25650752966102763 0.27066608538463055 0.259736894547936 0.259736894547936 0.00462927315947332 0.5278764825612624 0.5570139063374764 0.5345223138928448 0.5345223138928448 0.009526755161797178 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 2 40960 33 0 0 True FLASH_ATTN False true_mixed_fused_projected 2080 False 2048 BF16 generic none CUDA_EVENT True [2048] [0] [16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] 1 32 32 33 32 16384.0 0.030303030303030304 q2k_32q1s16k fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
141 0.0 0.0 0.0 0.0 0.0 0.026048000901937485 0.03728000074625015 0.028636799938976765 0.02705600019544363 0.003313953649011259 0.9270006318443208 0.9712965120641314 0.93480767601568 0.93480767601568 0.012612551450284608 0.8181833128578277 0.8572794566782395 0.8250739157811953 0.8250739157811953 0.01113200873299586 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 2 40960 17 0 0 True FLASH_ATTN False true_mixed_fused_projected 4112 False 4096 BF16 generic none CUDA_EVENT True [4096] [0] [32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] 1 16 16 17 16 32768.0 0.058823529411764705 q4k_16q1s32k fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
142 0.0 0.0 0.0 0.0 0.0 0.018079999834299088 0.037696000188589096 0.022092800214886667 0.019024000503122807 0.005986278786633071 0.2839857165542317 0.2876110048757805 0.2849506939696605 0.2849506939696605 0.0009995178425916847 1.0871823008331585 1.101060989333509 1.0908765231323903 1.0908765231323903 0.0038264533900426207 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 2 40960 33 0 0 True FLASH_ATTN False true_mixed_fused_projected 2080 False 2048 BF16 generic none CUDA_EVENT True [2048] [0] [32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] 1 32 32 33 32 32768.0 0.030303030303030304 q2k_32q1s32k fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
143 0.0 0.0 0.0 0.0 0.0 0.015072000212967396 0.025567999109625816 0.01726400014013052 0.015728000551462173 0.0031324465940990903 0.137270464802061 0.13799613818579368 0.13751499486424038 0.13751499486424038 0.00025221761530472904 2.3021855212208884 2.314355908780759 2.3062865750744197 2.3062865750744197 0.004229983070201486 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 2 40960 65 0 0 True FLASH_ATTN False true_mixed_fused_projected 1088 False 1024 BF16 generic none CUDA_EVENT True [1024] [0] [32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] 1 64 64 65 64 32768.0 0.015384615384615385 q1k_64q1s32k fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
144 0.0 0.0 0.0 0.0 0.0 0.014560000039637089 0.022784000262618065 0.016435200069099664 0.015519999898970127 0.0023688301421469523 0.04879999905824661 0.09139200299978256 0.06054079942405224 0.06054079942405224 0.012152608702448775 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 4 40960 1 64 0.0 True FLASH_ATTN False vllm020_batch_spec [64] 64 64 64 64.0 True 0.0 0.0 0.0 False 0.0 64.0 64 BF16 generic none CUDA_EVENT False q64 measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
145 0.0 0.0 0.0 0.0 0.0 0.014816000126302242 0.02844800055027008 0.017008000146597625 0.015696000307798386 0.003941466294662679 0.047807998955249786 0.07356800138950348 0.05626560002565384 0.05626560002565384 0.00842179125412236 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 4 40960 1 128 0.0 True FLASH_ATTN False vllm020_batch_spec [128] 128 128 128 128.0 True 0.0 0.0 0.0 False 0.0 128.0 128 BF16 generic none CUDA_EVENT False q128 measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
146 0.0 0.0 0.0 0.0 0.0 0.014688000082969666 0.053408000618219376 0.019353600218892097 0.01532800029963255 0.01138302400841626 0.048448000103235245 0.0785600021481514 0.0556256003677845 0.0556256003677845 0.009348274502616908 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 4 40960 1 256 0.0 True FLASH_ATTN False vllm020_batch_spec [256] 256 256 256 256.0 True 0.0 0.0 0.0 False 0.0 256.0 256 BF16 generic none CUDA_EVENT False q256 measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
147 0.0 0.0 0.0 0.0 0.0 0.015039999969303608 0.03139200061559677 0.01823679991066456 0.016159999649971724 0.004860071238302512 0.055424001067876816 0.08505599945783615 0.0640383992344141 0.0640383992344141 0.00921448636178623 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 4 40960 1 512 0.0 True FLASH_ATTN False vllm020_batch_spec [512] 512 512 512 512.0 True 0.0 0.0 0.0 False 0.0 512.0 512 BF16 generic none CUDA_EVENT False q512 measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
148 0.0 0.0 0.0 0.0 0.0 0.014751999638974667 0.026528000831604004 0.017324799951165915 0.01536000007763505 0.0036004822686428305 0.07660800218582153 0.0942080020904541 0.08209280073642732 0.08209280073642732 0.00515895587669752 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 4 40960 1 1024 0.0 True FLASH_ATTN False vllm020_batch_spec [1024] 1024 1024 1024 1024.0 True 0.0 0.0 0.0 False 0.0 1024.0 1024 BF16 generic none CUDA_EVENT False q1k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
149 0.0 0.0 0.0 0.0 0.0 0.014879999682307243 0.021247999742627144 0.016403199825435876 0.01532800029963255 0.001995897022647934 0.11395200341939926 0.15113599598407745 0.12431039959192276 0.12431039959192276 0.011164431123683732 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 4 40960 1 2048 0.0 True FLASH_ATTN False vllm020_batch_spec [2048] 2048 2048 2048 2048.0 True 0.0 0.0 0.0 False 0.0 2048.0 2048 BF16 generic none CUDA_EVENT False q2k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
150 0.0 0.0 0.0 0.0 0.0 0.017184000462293625 0.028672000393271446 0.019865600019693376 0.017823999747633934 0.0035798491315929977 0.3171840012073517 0.3341119885444641 0.3261695951223373 0.3261695951223373 0.005111046452878918 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 4 40960 1 4096 0.0 True FLASH_ATTN False vllm020_batch_spec [4096] 4096 4096 4096 4096.0 True 0.0 0.0 0.0 False 0.0 4096.0 4096 BF16 generic none CUDA_EVENT False q4k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
151 0.0 0.0 0.0 0.0 0.0 0.024639999493956566 0.030400000512599945 0.02656640000641346 0.02556800004094839 0.0020189846603237303 1.0648640394210815 1.0828479528427124 1.0715327858924866 1.0715327858924866 0.005558639263049851 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 4 40960 1 8192 0.0 True FLASH_ATTN False vllm020_batch_spec [8192] 8192 8192 8192 8192.0 True 0.0 0.0 0.0 False 0.0 8192.0 8192 BF16 generic none CUDA_EVENT False q8k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
152 0.0 0.0 0.0 0.0 0.0 0.014976000413298607 0.021695999428629875 0.017305599898099898 0.01593599934130907 0.0025718046517268054 0.04956800118088722 0.07932800054550171 0.06228480041027069 0.06228480041027069 0.01027160349757827 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 4 40960 1 64 448.0 True FLASH_ATTN False vllm020_batch_spec [64] 64 64 64 64.0 True 0.0 0.0 0.0 True 448.0 512.0 64 BF16 generic none CUDA_EVENT False q64s512 measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
153 0.0 0.0 0.0 0.0 0.0 0.01539199985563755 0.03580800071358681 0.019686400331556796 0.017136000096797943 0.005918387811304003 0.05158400163054466 0.10713600367307663 0.06364160068333148 0.06364160068333148 0.015832957809696766 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 4 40960 1 128 896.0 True FLASH_ATTN False vllm020_batch_spec [128] 128 128 128 128.0 True 0.0 0.0 0.0 True 896.0 1024.0 128 BF16 generic none CUDA_EVENT False q128s1k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
154 0.0 0.0 0.0 0.0 0.0 0.015168000012636185 0.02223999984562397 0.016672000009566545 0.015887999907135963 0.0020934945946034563 0.06521599739789963 0.08902399986982346 0.07520959973335266 0.07520959973335266 0.007913840684102929 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 4 40960 1 256 1792.0 True FLASH_ATTN False vllm020_batch_spec [256] 256 256 256 256.0 True 0.0 0.0 0.0 True 1792.0 2048.0 256 BF16 generic none CUDA_EVENT False q256s2k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
155 0.0 0.0 0.0 0.0 0.0 0.015231999568641186 0.04028800129890442 0.019971200078725816 0.01646399963647127 0.007351305886577286 0.14467200636863708 0.16844800114631653 0.15363519936800005 0.15363519936800005 0.008174435899956223 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 4 40960 1 512 3584.0 True FLASH_ATTN False vllm020_batch_spec [512] 512 512 512 512.0 True 0.0 0.0 0.0 True 3584.0 4096.0 512 BF16 generic none CUDA_EVENT False q512s4k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
156 0.0 0.0 0.0 0.0 0.0 0.015519999898970127 0.02304000034928322 0.01775679988786578 0.016784000210464 0.0025077243712082584 0.33740800619125366 0.35343998670578003 0.3445120006799698 0.3445120006799698 0.00445648463590358 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 4 40960 1 1024 7168.0 True FLASH_ATTN False vllm020_batch_spec [1024] 1024 1024 1024 1024.0 True 0.0 0.0 0.0 True 7168.0 8192.0 1024 BF16 generic none CUDA_EVENT False q1ks8k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
157 0.0 0.0 0.0 0.0 0.0 0.015647999942302704 0.02393599972128868 0.01809599995613098 0.01654400024563074 0.002998393102466254 0.0 0.0 0.0 0.0 0.0 0.0504320003092289 0.0843840017914772 0.060083200410008426 0.060083200410008426 0.00986959318572296 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 4 40960 1 0 128.0 False FLASH_ATTN False vllm020_batch_spec [1] 1 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False q1s128 measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
158 0.0 0.0 0.0 0.0 0.0 0.01603199914097786 0.030047999694943428 0.019510399922728537 0.017680000513792038 0.004065185265963332 0.0 0.0 0.0 0.0 0.0 0.05004800111055374 0.07036799937486649 0.059315200522542 0.059315200522542 0.006537768821329647 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 4 40960 8 0 128.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1] 8 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 8q1s128 measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
159 0.0 0.0 0.0 0.0 0.0 0.01484800036996603 0.02393599972128868 0.018035199772566558 0.016512000001966953 0.0030667689116777724 0.0 0.0 0.0 0.0 0.0 0.05100800096988678 0.06735999882221222 0.058387200161814694 0.058387200161814694 0.005832787739241381 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 4 40960 16 0 128.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] 16 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 16q1s128 measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
160 0.0 0.0 0.0 0.0 0.0 0.01500799972563982 0.026208000257611275 0.017500799987465142 0.01646399963647127 0.0030666103306165714 0.0 0.0 0.0 0.0 0.0 0.05023999884724617 0.07254400104284286 0.057254400476813315 0.057254400476813315 0.006890853653068151 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 4 40960 32 0 128.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] 32 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 32q1s128 measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
161 0.0 0.0 0.0 0.0 0.0 0.014688000082969666 0.027327999472618103 0.0179776000790298 0.01648000068962574 0.003783417447531392 0.0 0.0 0.0 0.0 0.0 0.04819199815392494 0.07100799679756165 0.05621119923889638 0.05621119923889638 0.007824391484124725 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 4 40960 64 0 128.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] 64 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 64q1s128 measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
162 0.0 0.0 0.0 0.0 0.0 0.015647999942302704 0.036448001861572266 0.019136000238358975 0.016704000532627106 0.006076530095624803 0.0 0.0 0.0 0.0 0.0 0.047488000243902206 0.08441600203514099 0.0588383998721838 0.0588383998721838 0.011275940726332522 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 4 40960 8 0 1024.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1] 8 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 8q1s1k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
163 0.0 0.0 0.0 0.0 0.0 0.015135999768972397 0.033952001482248306 0.02119360016658902 0.018000000156462193 0.007136646614305304 0.0 0.0 0.0 0.0 0.0 0.04931199923157692 0.06992000341415405 0.05755840018391609 0.05755840018391609 0.007297587694315226 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 4 40960 16 0 1024.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] 16 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 16q1s1k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
164 0.0 0.0 0.0 0.0 0.0 0.014879999682307243 0.034272000193595886 0.0204415999352932 0.017311999574303627 0.00695404701803013 0.0 0.0 0.0 0.0 0.0 0.05215999856591225 0.0735040009021759 0.06117440015077591 0.06117440015077591 0.007381118436894922 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 4 40960 32 0 1024.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] 32 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 32q1s1k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
165 0.0 0.0 0.0 0.0 0.0 0.015359999611973763 0.03743999823927879 0.02012479966506362 0.01775999926030636 0.006181045509079057 0.0 0.0 0.0 0.0 0.0 0.06355199962854385 0.08508799970149994 0.07019200026988984 0.07019200026988984 0.0062246580442117845 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 4 40960 64 0 1024.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] 64 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 64q1s1k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
166 0.0 0.0 0.0 0.0 0.0 0.015552000142633915 0.032607998698949814 0.01959999995306134 0.017487999983131886 0.004882475068460749 0.0 0.0 0.0 0.0 0.0 0.04918399825692177 0.0865280032157898 0.05973760038614274 0.05973760038614274 0.011546847942113974 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 4 40960 8 0 2048.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1] 8 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 8q1s2k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
167 0.0 0.0 0.0 0.0 0.0 0.015168000012636185 0.02675200067460537 0.017430400010198355 0.016560000367462635 0.003243135094239819 0.0 0.0 0.0 0.0 0.0 0.052671998739242554 0.08367999643087387 0.06238719932734965 0.06238719932734965 0.011207824780630104 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 4 40960 16 0 2048.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] 16 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 16q1s2k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
168 0.0 0.0 0.0 0.0 0.0 0.01500799972563982 0.03612799942493439 0.019327999837696553 0.01688000001013279 0.006058251425153085 0.0 0.0 0.0 0.0 0.0 0.06364800035953522 0.07878399640321732 0.06935679838061332 0.06935679838061332 0.004515588328677643 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 4 40960 32 0 2048.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] 32 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 32q1s2k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
169 0.0 0.0 0.0 0.0 0.0 0.015104000456631184 0.03711999952793121 0.019670399930328132 0.017152000218629837 0.006165994646522264 0.0 0.0 0.0 0.0 0.0 0.09071999788284302 0.11507199704647064 0.10252480059862136 0.10252480059862136 0.008782544051535657 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 4 40960 64 0 2048.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] 64 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 64q1s2k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
170 0.0 0.0 0.0 0.0 0.0 0.015039999969303608 0.026528000831604004 0.018396800104528665 0.01601599995046854 0.004279788048986283 0.0 0.0 0.0 0.0 0.0 0.05331199988722801 0.07977599650621414 0.06076480001211167 0.06076480001211167 0.008761579122069606 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 4 40960 8 0 4096.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1] 8 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 8q1s4k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
171 0.0 0.0 0.0 0.0 0.0 0.015359999611973763 0.02643200010061264 0.01726400004699826 0.015855999663472176 0.003210008417242029 0.0 0.0 0.0 0.0 0.0 0.062431998550891876 0.08246400207281113 0.06970879957079888 0.06970879957079888 0.007016341676068233 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 4 40960 16 0 4096.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] 16 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 16q1s4k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
172 0.0 0.0 0.0 0.0 0.0 0.014911999925971031 0.02223999984562397 0.0166015999391675 0.015887999907135963 0.00204913969129354 0.0 0.0 0.0 0.0 0.0 0.10127999633550644 0.1141119971871376 0.10621120035648347 0.10621120035648347 0.004029318311754605 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 4 40960 32 0 4096.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] 32 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 32q1s4k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
173 0.0 0.0 0.0 0.0 0.0 0.015359999611973763 0.03363199904561043 0.019305599946528675 0.016671999357640743 0.0055445582307981234 0.0 0.0 0.0 0.0 0.0 0.1319040060043335 0.15839999914169312 0.14040640145540234 0.14040640145540234 0.007998041073596942 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 4 40960 64 0 4096.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] 64 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 64q1s4k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
174 0.0 0.0 0.0 0.0 0.0 0.014368000440299511 0.04447999969124794 0.018908800091594458 0.0157279996201396 0.0086814937461721 0.0 0.0 0.0 0.0 0.0 0.06428799778223038 0.08982399851083755 0.07349760085344315 0.07349760085344315 0.008605284512197258 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 4 40960 8 0 8192.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1] 8 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 8q1s8k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
175 0.0 0.0 0.0 0.0 0.0 0.015104000456631184 0.028672000393271446 0.01890560006722808 0.016080000437796116 0.004797584627815021 0.0 0.0 0.0 0.0 0.0 0.11123199760913849 0.14115199446678162 0.11942399889230729 0.11942399889230729 0.008513394706791027 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 4 40960 16 0 8192.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] 16 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 16q1s8k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
176 0.0 0.0 0.0 0.0 0.0 0.014271999709308147 0.024927999824285507 0.016915200091898442 0.015216000378131866 0.003374147499442635 0.0 0.0 0.0 0.0 0.0 0.15887999534606934 0.18111999332904816 0.16934399753808976 0.16934399753808976 0.007415181260761123 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 4 40960 32 0 8192.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] 32 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 32q1s8k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
177 0.0 0.0 0.0 0.0 0.0 0.014336000196635723 0.026944000273942947 0.017737600207328796 0.015199999790638685 0.00415598714375255 0.0 0.0 0.0 0.0 0.0 0.2192319929599762 0.23472000658512115 0.22809920012950893 0.22809920012950893 0.004730841376327335 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 4 40960 64 0 8192.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] 64 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 64q1s8k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
178 0.0 0.0 0.0 0.0 0.0 0.014688000082969666 0.05052800104022026 0.020320000313222408 0.015520000364631414 0.010604522177924678 0.024270629882498958 0.03340247625954076 0.028446344104128624 0.028446344104128624 0.0032330369099793834 0.023697370291069768 0.03261352722995356 0.027774456325453972 0.027774456325453972 0.0031566742680923386 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 4 40960 9 0 0 True FLASH_ATTN False true_mixed_fused_projected 72 False 64 BF16 generic none CUDA_EVENT True [64] [0] [512, 512, 512, 512, 512, 512, 512, 512] 1 8 8 9 8 512.0 0.1111111111111111 q64_8q1s512 fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
179 0.0 0.0 0.0 0.0 0.0 0.015359999611973763 0.033663999289274216 0.019987199828028678 0.018240000121295452 0.005522929139253732 0.0259194055660947 0.03719755183990719 0.029916029687899703 0.029916029687899703 0.003966666590554318 0.027104595853449518 0.03889844644729374 0.03128396954022015 0.03128396954022015 0.004148046317967881 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 4 40960 9 0 0 True FLASH_ATTN False true_mixed_fused_projected 136 False 128 BF16 generic none CUDA_EVENT True [128] [0] [1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024] 1 8 8 9 8 1024.0 0.1111111111111111 q128_8q1s1k fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
180 0.0 0.0 0.0 0.0 0.0 0.014592000283300877 0.03299200162291527 0.019840000104159115 0.016207999549806118 0.006546343008726814 0.02587869595769926 0.03763167265431482 0.030174939058162802 0.030174939058162802 0.0037303581782277364 0.026473304070195713 0.03849632587654989 0.03086826083864964 0.03086826083864964 0.003816069654529222 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 4 40960 17 0 0 True FLASH_ATTN False true_mixed_fused_projected 144 False 128 BF16 generic none CUDA_EVENT True [128] [0] [1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024] 1 16 16 17 16 1024.0 0.058823529411764705 q128_16q1s1k fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
181 0.0 0.0 0.0 0.0 0.0 0.013824000023305416 0.021023999899625778 0.016304000187665223 0.015584000386297703 0.0023236165806545476 0.031810621525966996 0.04156949936878106 0.0346749350032807 0.0346749350032807 0.003130209181143985 0.035677378270900374 0.04662250161636451 0.038889864871740294 0.038889864871740294 0.0035107033960828727 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 4 40960 17 0 0 True FLASH_ATTN False true_mixed_fused_projected 272 False 256 BF16 generic none CUDA_EVENT True [256] [0] [2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048] 1 16 16 17 16 2048.0 0.058823529411764705 q256_16q1s2k fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
182 0.0 0.0 0.0 0.0 0.0 0.014527999795973301 0.02175999991595745 0.01569600012153387 0.015008000191301107 0.0020882666168036863 0.038211712107062853 0.07212229256520057 0.04364509673334097 0.04364509673334097 0.009671953074025085 0.0476442859917874 0.08992570455184198 0.05441890342616107 0.05441890342616107 0.01205947791784038 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 4 40960 33 0 0 True FLASH_ATTN False true_mixed_fused_projected 288 False 256 BF16 generic none CUDA_EVENT True [256] [0] [2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048] 1 32 32 33 32 2048.0 0.030303030303030304 q256_32q1s2k fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
183 0.0 0.0 0.0 0.0 0.0 0.014368000440299511 0.033824000507593155 0.017628800217062236 0.014864000026136637 0.005791495403857738 0.05214261250030033 0.06266261508706669 0.05677791295527661 0.05677791295527661 0.0030298223079651514 0.08648138506878382 0.10392938682791132 0.09416928531647478 0.09416928531647478 0.005025126612204489 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 4 40960 33 0 0 True FLASH_ATTN False true_mixed_fused_projected 544 False 512 BF16 generic none CUDA_EVENT True [512] [0] [4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096] 1 32 32 33 32 4096.0 0.030303030303030304 q512_32q1s4k fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
184 0.0 0.0 0.0 0.0 0.0 0.014527999795973301 0.031199999153614044 0.017900799959897996 0.015343999955803156 0.004974475661316249 0.06411958891421111 0.07405276123263956 0.06793047918211377 0.06793047918211377 0.0033918658556700006 0.14058441263169497 0.16236323591492058 0.1489399211274489 0.1489399211274489 0.00743678300375353 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 4 40960 65 0 0 True FLASH_ATTN False true_mixed_fused_projected 576 False 512 BF16 generic none CUDA_EVENT True [512] [0] [4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096] 1 64 64 65 64 4096.0 0.015384615384615385 q512_64q1s4k fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
185 0.0 0.0 0.0 0.0 0.0 0.014720000326633453 0.03855999931693077 0.018495999928563833 0.015647999942302704 0.006918530056761627 0.09872985549401277 0.10683454583043753 0.10185316839884552 0.10185316839884552 0.0020802254037042074 0.2743261390166379 0.2968454508984596 0.2830044295482752 0.2830044295482752 0.005780016596065092 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 4 40960 65 0 0 True FLASH_ATTN False true_mixed_fused_projected 1088 False 1024 BF16 generic none CUDA_EVENT True [1024] [0] [8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] 1 64 64 65 64 8192.0 0.015384615384615385 q1k_64q1s8k fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
186 0.0 0.0 0.0 0.0 0.0 0.014911999925971031 0.030400000512599945 0.01977920001372695 0.01726400014013052 0.005350883464206169 0.13918872472233365 0.16279522855335207 0.1501912864839173 0.1501912864839173 0.008992185529011513 0.11892328861766266 0.13909276049083735 0.1283239123428725 0.1283239123428725 0.007682951884956939 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 4 40960 33 0 0 True FLASH_ATTN False true_mixed_fused_projected 2080 False 2048 BF16 generic none CUDA_EVENT True [2048] [0] [4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096] 1 32 32 33 32 4096.0 0.030303030303030304 q2k_32q1s4k fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
187 0.0 0.0 0.0 0.0 0.0 0.01727999933063984 0.02223999984562397 0.01819519978016615 0.017680000513792038 0.0014397298270620873 0.3728044181625443 0.38295504353701676 0.3761264483787333 0.3761264483787333 0.0033660272332490977 0.07967557017401883 0.08184495665371805 0.08038555277807365 0.08038555277807365 0.0007193856241088455 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 4 40960 17 0 0 True FLASH_ATTN False true_mixed_fused_projected 4112 False 4096 BF16 generic none CUDA_EVENT True [4096] [0] [4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096] 1 16 16 17 16 4096.0 0.058823529411764705 q4k_16q1s4k fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
188 0.0 0.0 0.0 0.0 0.0 0.01500799972563982 0.034591998904943466 0.01941439984366298 0.01756799966096878 0.005541380584373771 0.05951999872922897 0.08268799632787704 0.06715519949793816 0.06715519949793816 0.006657802116288364 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 4 40960 2 1024 0.0 True FLASH_ATTN False vllm020_batch_spec [512, 512] 1024 512 512 512.0 True 0.0 0.0 0.0 False 0.0 1024.0 1024 BF16 generic none CUDA_EVENT False 2q512 measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
189 0.0 0.0 0.0 0.0 0.0 0.015104000456631184 0.03283200040459633 0.019180799927562477 0.016543999314308167 0.0052187911233635975 0.07199999690055847 0.08675199747085571 0.07749439924955369 0.07749439924955369 0.004849874741156772 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 4 40960 4 2048 0.0 True FLASH_ATTN False vllm020_batch_spec [512, 512, 512, 512] 2048 512 512 512.0 True 0.0 0.0 0.0 False 0.0 2048.0 2048 BF16 generic none CUDA_EVENT False 4q512 measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
190 0.0 0.0 0.0 0.0 0.0 0.01724799908697605 0.02284800074994564 0.01928640007972717 0.018400000408291817 0.0020641390157565697 0.09676799923181534 0.12310399860143663 0.10618879944086074 0.10618879944086074 0.008982738207839057 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 4 40960 8 4096 0.0 True FLASH_ATTN False vllm020_batch_spec [512, 512, 512, 512, 512, 512, 512, 512] 4096 512 512 512.0 True 0.0 0.0 0.0 False 0.0 4096.0 4096 BF16 generic none CUDA_EVENT False 8q512 measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
191 0.0 0.0 0.0 0.0 0.0 0.024224000051617622 0.03315199911594391 0.027436799928545953 0.027328000403940678 0.002679154874546328 0.14716799557209015 0.16412800550460815 0.15470399856567385 0.15470399856567385 0.005423454433987419 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 4 40960 16 8192 0.0 True FLASH_ATTN False vllm020_batch_spec [512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512] 8192 512 512 512.0 True 0.0 0.0 0.0 False 0.0 8192.0 8192 BF16 generic none CUDA_EVENT False 16q512 measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
192 0.0 0.0 0.0 0.0 0.0 0.01833599992096424 0.04806400090456009 0.026252799853682517 0.022672000341117382 0.00878438440429033 0.1844799965620041 0.2072959989309311 0.19359359890222552 0.19359359890222552 0.007663539820967659 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 4 40960 2 4096 0.0 True FLASH_ATTN False vllm020_batch_spec [2048, 2048] 4096 2048 2048 2048.0 True 0.0 0.0 0.0 False 0.0 4096.0 4096 BF16 generic none CUDA_EVENT False 2q2k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
193 0.0 0.0 0.0 0.0 0.0 0.024480000138282776 0.04979199916124344 0.029081599973142146 0.025679999962449074 0.007227422044689197 0.34147199988365173 0.37968000769615173 0.3583200007677078 0.3583200007677078 0.011804422100726231 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 4 40960 4 8192 0.0 True FLASH_ATTN False vllm020_batch_spec [2048, 2048, 2048, 2048] 8192 2048 2048 2048.0 True 0.0 0.0 0.0 False 0.0 8192.0 8192 BF16 generic none CUDA_EVENT False 4q2k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
194 0.0 0.0 0.0 0.0 0.0 0.014720000326633453 0.023744000121951103 0.017753600236028434 0.016368000768125057 0.003061040281616705 0.0 0.0 0.0 0.0 0.0 0.05686400085687637 0.1090880036354065 0.06629760004580021 0.06629760004580021 0.014803727026812366 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 4 40960 1 0 16384.0 False FLASH_ATTN False vllm020_batch_spec [1] 1 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False q1s16k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
195 0.0 0.0 0.0 0.0 0.0 0.014879999682307243 0.021088000386953354 0.01647359998896718 0.015840000472962856 0.0017964402433206 0.0 0.0 0.0 0.0 0.0 0.08540800213813782 0.09961599856615067 0.09146559983491898 0.09146559983491898 0.00447423706371901 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 4 40960 8 0 16384.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1] 8 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 8q1s16k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
196 0.0 0.0 0.0 0.0 0.0 0.014560000039637089 0.04156799986958504 0.018512000143527985 0.015584000386297703 0.007849701681877904 0.0 0.0 0.0 0.0 0.0 0.17948800325393677 0.194815993309021 0.1866239994764328 0.1866239994764328 0.004156328241374654 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 4 40960 16 0 16384.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] 16 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 16q1s16k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
197 0.0 0.0 0.0 0.0 0.0 0.014879999682307243 0.029632000252604485 0.019449600111693145 0.016736000776290894 0.005126811425136928 0.0 0.0 0.0 0.0 0.0 0.27529600262641907 0.2898879945278168 0.2825664013624191 0.2825664013624191 0.004542801609674093 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 4 40960 32 0 16384.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] 32 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 32q1s16k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
198 0.0 0.0 0.0 0.0 0.0 0.014399999752640724 0.031328000128269196 0.019759999960660933 0.016367999836802483 0.006266303740589241 0.0 0.0 0.0 0.0 0.0 0.39190399646759033 0.4079039990901947 0.3987520009279252 0.3987520009279252 0.0037367727509362725 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 4 40960 64 0 16384.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] 64 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 64q1s16k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
199 0.0 0.0 0.0 0.0 0.0 0.01539199985563755 0.040031999349594116 0.0212032001465559 0.01649599988013506 0.008068547381446682 0.0 0.0 0.0 0.0 0.0 0.06588800251483917 0.07897599786520004 0.0703904002904892 0.0703904002904892 0.004214826869769361 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 4 40960 1 0 32768.0 False FLASH_ATTN False vllm020_batch_spec [1] 1 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False q1s32k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
200 0.0 0.0 0.0 0.0 0.0 0.014495999552309513 0.022495999932289124 0.016006399970501663 0.01515199989080429 0.002265581884342885 0.0 0.0 0.0 0.0 0.0 0.1231679990887642 0.13526399433612823 0.1288223996758461 0.1288223996758461 0.004328976434897094 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 4 40960 8 0 32768.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1] 8 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 8q1s32k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
201 0.0 0.0 0.0 0.0 0.0 0.014592000283300877 0.023231999948620796 0.01705600004643202 0.016048000194132328 0.002810904312746391 0.0 0.0 0.0 0.0 0.0 0.3158079981803894 0.33129599690437317 0.32348800003528594 0.32348800003528594 0.00417599076136664 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 4 40960 16 0 32768.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] 16 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 16q1s32k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
202 0.0 0.0 0.0 0.0 0.0 0.014527999795973301 0.021888000890612602 0.01708160014823079 0.01609600055962801 0.0025289234621475062 0.0 0.0 0.0 0.0 0.0 0.5103679895401001 0.5200319886207581 0.5132320046424866 0.5132320046424866 0.0035759433157749533 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 4 40960 32 0 32768.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] 32 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 32q1s32k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
203 0.0 0.0 0.0 0.0 0.0 0.014688000082969666 0.032416000962257385 0.019353600032627583 0.016176000237464905 0.0061458798960684425 0.0 0.0 0.0 0.0 0.0 0.7400320172309875 0.7516480088233948 0.7449311971664427 0.7449311971664427 0.004573461776168607 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 4 40960 64 0 32768.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] 64 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 64q1s32k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
204 0.0 0.0 0.0 0.0 0.0 0.0144640002399683 0.02006400004029274 0.01573119992390275 0.014992000069469213 0.001654446341262991 0.0 0.0 0.0 0.0 0.0 0.07097599655389786 0.09932799637317657 0.07828159928321837 0.07828159928321837 0.009464602895232642 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 4 40960 1 0 40960.0 False FLASH_ATTN False vllm020_batch_spec [1] 1 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False q1s40k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
205 0.0 0.0 0.0 0.0 0.0 0.01484800036996603 0.02319999970495701 0.016883199848234654 0.015775999519973993 0.0026693906156048403 0.0 0.0 0.0 0.0 0.0 0.14176000654697418 0.1598079949617386 0.15063679963350293 0.15063679963350293 0.005741662969679433 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 4 40960 8 0 40960.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1] 8 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 8q1s40k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
206 0.0 0.0 0.0 0.0 0.0 0.01484800036996603 0.03203200176358223 0.022224000189453363 0.02247999981045723 0.005996176183124981 0.0 0.0 0.0 0.0 0.0 0.39180800318717957 0.41046398878097534 0.4003200054168701 0.4003200054168701 0.005552433764307601 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 4 40960 16 0 40960.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] 16 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 16q1s40k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
207 0.0 0.0 0.0 0.0 0.0 0.015072000212967396 0.028192000463604927 0.017152000125497578 0.015536000020802021 0.003847486121602065 0.0 0.0 0.0 0.0 0.0 0.6239359974861145 0.6367359757423401 0.6285343945026398 0.6285343945026398 0.004006084286606002 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 4 40960 32 0 40960.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] 32 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 32q1s40k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
208 0.0 0.0 0.0 0.0 0.0 0.014879999682307243 0.026688000187277794 0.01718079997226596 0.015488000120967627 0.0037132536813398722 0.0 0.0 0.0 0.0 0.0 0.9141119718551636 0.9721279740333557 0.927455997467041 0.927455997467041 0.02148767779232235 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 4 40960 64 0 40960.0 False FLASH_ATTN False vllm020_batch_spec [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] 64 1 1 1.0 True 0.0 0.0 0.0 False 0 0 0 BF16 generic none CUDA_EVENT False 64q1s40k measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median
209 0.0 0.0 0.0 0.0 0.0 0.014751999638974667 0.047520000487565994 0.028710400220006704 0.0266720000654459 0.010774688401598903 0.06094816381288764 0.06816969726785131 0.0644060660218149 0.0644060660218149 0.002203106589073183 0.087051838094461 0.09736630405680231 0.09199073574791852 0.09199073574791852 0.0031466818046499644 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 4 40960 9 0 0 True FLASH_ATTN False true_mixed_fused_projected 520 False 512 BF16 generic none CUDA_EVENT True [512] [0] [16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] 1 8 8 9 8 16384.0 0.1111111111111111 q512_8q1s16k fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
210 0.0 0.0 0.0 0.0 0.0 0.014911999925971031 0.02489599958062172 0.017427200078964235 0.01595200039446354 0.003071906489592143 0.19733789497223914 0.2016295616652644 0.19860388994013833 0.19860388994013833 0.001512389485439305 0.44861409134062713 0.45837046456077934 0.45149211526120153 0.45149211526120153 0.0034381598874302305 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 4 40960 17 0 0 True FLASH_ATTN False true_mixed_fused_projected 1040 False 1024 BF16 generic none CUDA_EVENT True [1024] [0] [16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] 1 16 16 17 16 16384.0 0.058823529411764705 q1k_16q1s16k fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
211 0.0 0.0 0.0 0.0 0.0 0.015039999969303608 0.03046399913728237 0.020643199887126686 0.016944000497460365 0.006320148675595729 0.2157924314537054 0.22215709640166995 0.21769400765743155 0.21769400765743155 0.0019679921844645526 0.4905115822753901 0.5049789194903589 0.4948340005651485 0.4948340005651485 0.0044733865493072344 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 4 40960 33 0 0 True FLASH_ATTN False true_mixed_fused_projected 2080 False 2048 BF16 generic none CUDA_EVENT True [2048] [0] [16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] 1 32 32 33 32 16384.0 0.030303030303030304 q2k_32q1s16k fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
212 0.0 0.0 0.0 0.0 0.0 0.017152000218629837 0.02425600029528141 0.01865920014679432 0.018112000077962875 0.0019454553198986453 0.7430866512973927 0.7578834738720781 0.7455351005236347 0.7455351005236347 0.00416692487556653 0.7369773831645824 0.7516525540362471 0.7394057025273603 0.7394057025273603 0.004132666607961175 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 4 40960 17 0 0 True FLASH_ATTN False true_mixed_fused_projected 4112 False 4096 BF16 generic none CUDA_EVENT True [4096] [0] [32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] 1 16 16 17 16 32768.0 0.058823529411764705 q4k_16q1s32k fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
213 0.0 0.0 0.0 0.0 0.0 0.015263999812304974 0.02470399998128414 0.018009600043296815 0.01657600048929453 0.0031544532774534346 0.25251173919752334 0.25566890792461106 0.2536729054481981 0.2536729054481981 0.0010279562245342983 1.0425282722084215 1.0555630628624373 1.0473223013846877 1.0473223013846877 0.004244053880724073 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 4 40960 33 0 0 True FLASH_ATTN False true_mixed_fused_projected 2080 False 2048 BF16 generic none CUDA_EVENT True [2048] [0] [32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] 1 32 32 33 32 32768.0 0.030303030303030304 q2k_32q1s32k fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio
214 0.0 0.0 0.0 0.0 0.0 0.01500799972563982 0.027807999402284622 0.01912960009649396 0.01643200032413006 0.004688164695934997 0.1251379565220268 0.14341821167748953 0.12742497577885914 0.12742497577885914 0.005353812167343766 1.1355340167064276 1.3014137556763312 1.1562870179154463 1.1562870179154463 0.048581869194943325 0.0 0.0 0.0 0.0 0.0 2048 32 4 16 4 40960 65 0 0 True FLASH_ATTN False true_mixed_fused_projected 1088 False 1024 BF16 generic none CUDA_EVENT True [1024] [0] [32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] 1 64 64 65 64 32768.0 0.015384615384615385 q1k_64q1s32k fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio

49
runs/frontier-fidelity-envelope-v1/profiles/profile-v4-trace-final/attention_true_mixed_fused.csv Normal file
View File

@@ -0,0 +1,49 @@
num_tensor_parallel_workers,batch_spec,num_prefill_seqs,num_decode_seqs,total_prefill_tokens,total_decode_tokens,decode_avg_kv_cache_size,attention_core_mean_ms,attention_core_mean_as_median_ms,kv_cache_update_median_ms,pure_prefill_reference_mean_ms,pure_decode_reference_mean_ms,projected_prefill_mean_ms,projected_decode_mean_ms,projection_sum_error_ms,representation
1,q64_8q1s512,1,8,64,8,512.0,0.06604800000786781,0.06604800000786781,0.021359999664127827,0.05810240097343922,0.0600041144660541,0.032492258074592426,0.03355574193327539,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation
1,q128_8q1s1k,1,8,128,8,1024.0,0.07483199909329416,0.07483199909329416,0.024255999363958836,0.05587520003318787,0.06090559959411622,0.03580428402241854,0.03902771507087562,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation
1,q128_16q1s1k,1,16,128,16,1024.0,0.10597119852900506,0.10597119852900506,0.018112000077962875,0.05587520003318787,0.06891520097851753,0.04744885718421094,0.05852234134479412,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation
1,q256_16q1s2k,1,16,256,16,2048.0,0.13697599917650224,0.13697599917650224,0.01673599984496832,0.056396800279617305,0.11392960175871848,0.04535414343408448,0.09162185574241775,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation
1,q256_32q1s2k,1,32,256,32,2048.0,0.2163648009300232,0.2163648009300232,0.01515199989080429,0.056396800279617305,0.13807999789714814,0.06274415549817247,0.15362064543185072,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation
1,q512_32q1s4k,1,32,512,32,4096.0,0.3746495962142944,0.3746495962142944,0.016191999427974224,0.07432000041007995,0.2231455981731415,0.09360396051475776,0.2810456356995366,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation
1,q512_64q1s4k,1,64,512,64,4096.0,0.6718560099601746,0.6718560099601746,0.024848000146448612,0.07432000041007995,0.4058080047369003,0.10399797220840973,0.5678580377517648,-1.1102230246251565e-16,one_fused_FA3_call_projected_for_Frontier_with_total_conservation
1,q1k_64q1s8k,1,64,1024,64,8192.0,1.3472000002861022,1.3472000002861022,0.019600000232458115,0.1307712011039257,0.7464000046253203,0.2008444429250931,1.1463555573610091,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation
1,q2k_32q1s4k,1,32,2048,32,4096.0,0.6351647913455962,0.6351647913455962,0.028256000019609928,0.33396480381488797,0.2231455981731415,0.38075520430942184,0.25440958703617433,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation
1,q4k_16q1s4k,1,16,4096,16,4096.0,1.2515872120857237,1.2515872120857237,0.04395199939608574,1.0999775886535645,0.1389280006289482,1.1112371236754128,0.14035008841031085,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation
1,q512_8q1s16k,1,8,512,8,16384.0,0.6654527902603149,0.6654527902603149,0.016848000697791576,0.07432000041007995,0.28281279802322384,0.13848196486571557,0.5269708253945994,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation
1,q1k_16q1s16k,1,16,1024,16,16384.0,0.7349536061286925,0.7349536061286925,0.01976000051945448,0.1307712011039257,0.39996159672737125,0.18109068484526028,0.5538629212834322,-1.1102230246251565e-16,one_fused_FA3_call_projected_for_Frontier_with_total_conservation
1,q2k_32q1s16k,1,32,2048,32,16384.0,1.5246047973632815,1.5246047973632815,0.02759999968111515,0.33396480381488797,0.7430047929286957,0.4727750380198454,1.051829759343436,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation
1,q4k_16q1s32k,1,16,4096,16,32768.0,2.285759997367859,2.285759997367859,0.04617599956691265,1.0999775886535645,0.7502080142498017,1.3589365122072,0.9268234851606594,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation
1,q2k_32q1s32k,1,32,2048,32,32768.0,2.6958079814910887,2.6958079814910887,0.02817599941045046,0.33396480381488797,1.4325888037681578,0.5096392092471257,2.186168772243963,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation
1,q1k_64q1s32k,1,64,1024,64,32768.0,4.842601585388183,4.842601585388183,0.018943999893963337,0.1307712011039257,2.8293471813201903,0.21393496609766816,4.628666619290515,-8.881784197001252e-16,one_fused_FA3_call_projected_for_Frontier_with_total_conservation
2,q64_8q1s512,1,8,64,8,512.0,0.05652800053358078,0.05652800053358078,0.019600000232458115,0.05961279980838299,0.055968457407185014,0.029155180178492036,0.027372820355088746,6.938893903907228e-18,one_fused_FA3_call_projected_for_Frontier_with_total_conservation
2,q128_8q1s1k,1,8,128,8,1024.0,0.07362559959292413,0.07362559959292413,0.014431999996304512,0.05882879942655563,0.05778240002691746,0.03714313592014963,0.036482463672774496,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation
2,q128_16q1s1k,1,16,128,16,1024.0,0.0736224003136158,0.0736224003136158,0.014479999896138906,0.05882879942655563,0.06270079985260964,0.03563837490653034,0.037984025407085474,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation
2,q256_16q1s2k,1,16,256,16,2048.0,0.10047360062599181,0.10047360062599181,0.01425600005313754,0.05459520071744919,0.06628479920327664,0.045378692890289625,0.05509490773570219,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation
2,q256_32q1s2k,1,32,256,32,2048.0,0.14032640159130094,0.14032640159130094,0.01566399959847331,0.05459520071744919,0.09493440166115762,0.05123499252968345,0.0890914090616175,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation
2,q512_32q1s4k,1,32,512,32,4096.0,0.22105600088834762,0.22105600088834762,0.016032000072300434,0.06650560013949873,0.13783999979496003,0.07194410845270183,0.14911189243564577,-2.7755575615628914e-17,one_fused_FA3_call_projected_for_Frontier_with_total_conservation
2,q512_64q1s4k,1,64,512,64,4096.0,0.36867519915103913,0.36867519915103913,0.015263999812304974,0.06650560013949873,0.22715839892625808,0.0834932625520734,0.2851819365989658,5.551115123125783e-17,one_fused_FA3_call_projected_for_Frontier_with_total_conservation
2,q1k_64q1s8k,1,64,1024,64,8192.0,0.6853824079036711,0.6853824079036711,0.015856000129133463,0.08607039973139763,0.40332479774951924,0.12053885718696096,0.5648435507167102,1.1102230246251565e-16,one_fused_FA3_call_projected_for_Frontier_with_total_conservation
2,q2k_32q1s4k,1,32,2048,32,4096.0,0.3503839999437332,0.3503839999437332,0.018432000651955605,0.19546559900045393,0.13783999979496003,0.2054811520619412,0.14490284788179197,-5.551115123125783e-17,one_fused_FA3_call_projected_for_Frontier_with_total_conservation
2,q4k_16q1s4k,1,16,4096,16,4096.0,0.7185311973094941,0.7185311973094941,0.026575999334454536,0.5800191938877105,0.10921279862523078,0.6046757700946376,0.11385542721485654,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation
2,q512_8q1s16k,1,8,512,8,16384.0,0.6480192065238953,0.6480192065238953,0.016127999871969223,0.06650560013949873,0.18568639904260634,0.17088926838108623,0.47712993814280913,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation
2,q1k_16q1s16k,1,16,1024,16,16384.0,0.6820319950580597,0.6820319950580597,0.016959999687969685,0.08607039973139763,0.2815328001976013,0.15969057520605917,0.5223414198520004,-1.1102230246251565e-16,one_fused_FA3_call_projected_for_Frontier_with_total_conservation
2,q2k_32q1s16k,1,32,2048,32,16384.0,0.7942592084407808,0.7942592084407808,0.0191040001809597,0.19546559900045393,0.40225600004196166,0.259736894547936,0.5345223138928448,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation
2,q4k_16q1s32k,1,16,4096,16,32768.0,1.759881591796875,1.759881591796875,0.02705600019544363,0.5800191938877105,0.511932796239853,0.93480767601568,0.8250739157811953,2.220446049250313e-16,one_fused_FA3_call_projected_for_Frontier_with_total_conservation
2,q2k_32q1s32k,1,32,2048,32,32768.0,1.3758272171020507,1.3758272171020507,0.019024000503122807,0.19546559900045393,0.7483008027076722,0.2849506939696605,1.0908765231323903,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation
2,q1k_64q1s32k,1,64,1024,64,32768.0,2.44380156993866,2.44380156993866,0.015728000551462173,0.08607039973139763,1.4435008168220522,0.13751499486424038,2.3062865750744197,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation
4,q64_8q1s512,1,8,64,8,512.0,0.0562208004295826,0.0562208004295826,0.015520000364631414,0.06054079942405224,0.0591108573866742,0.028446344104128624,0.027774456325453972,-6.938893903907228e-18,one_fused_FA3_call_projected_for_Frontier_with_total_conservation
4,q128_8q1s1k,1,8,128,8,1024.0,0.061199999228119854,0.061199999228119854,0.018240000121295452,0.05626560002565384,0.0588383998721838,0.029916029687899703,0.03128396954022015,-6.938893903907228e-18,one_fused_FA3_call_projected_for_Frontier_with_total_conservation
4,q128_16q1s1k,1,16,128,16,1024.0,0.06104319989681244,0.06104319989681244,0.016207999549806118,0.05626560002565384,0.05755840018391609,0.030174939058162802,0.03086826083864964,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation
4,q256_16q1s2k,1,16,256,16,2048.0,0.07356479987502099,0.07356479987502099,0.015584000386297703,0.0556256003677845,0.06238719932734965,0.0346749350032807,0.038889864871740294,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation
4,q256_32q1s2k,1,32,256,32,2048.0,0.09806400015950203,0.09806400015950203,0.015008000191301107,0.0556256003677845,0.06935679838061332,0.04364509673334097,0.05441890342616107,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation
4,q512_32q1s4k,1,32,512,32,4096.0,0.1509471982717514,0.1509471982717514,0.014864000026136637,0.0640383992344141,0.10621120035648347,0.05677791295527661,0.09416928531647478,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation
4,q512_64q1s4k,1,64,512,64,4096.0,0.21687040030956264,0.21687040030956264,0.015343999955803156,0.0640383992344141,0.14040640145540234,0.06793047918211377,0.1489399211274489,2.7755575615628914e-17,one_fused_FA3_call_projected_for_Frontier_with_total_conservation
4,q1k_64q1s8k,1,64,1024,64,8192.0,0.3848575979471207,0.3848575979471207,0.015647999942302704,0.08209280073642732,0.22809920012950893,0.10185316839884552,0.2830044295482752,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation
4,q2k_32q1s4k,1,32,2048,32,4096.0,0.2785151988267898,0.2785151988267898,0.01726400014013052,0.12431039959192276,0.10621120035648347,0.1501912864839173,0.1283239123428725,-5.551115123125783e-17,one_fused_FA3_call_projected_for_Frontier_with_total_conservation
4,q4k_16q1s4k,1,16,4096,16,4096.0,0.456512001156807,0.456512001156807,0.017680000513792038,0.3261695951223373,0.06970879957079888,0.3761264483787333,0.08038555277807365,-5.551115123125783e-17,one_fused_FA3_call_projected_for_Frontier_with_total_conservation
4,q512_8q1s16k,1,8,512,8,16384.0,0.15639680176973342,0.15639680176973342,0.0266720000654459,0.0640383992344141,0.09146559983491898,0.0644060660218149,0.09199073574791852,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation
4,q1k_16q1s16k,1,16,1024,16,16384.0,0.6500960052013398,0.6500960052013398,0.01595200039446354,0.08209280073642732,0.1866239994764328,0.19860388994013833,0.45149211526120153,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation
4,q2k_32q1s16k,1,32,2048,32,16384.0,0.71252800822258,0.71252800822258,0.016944000497460365,0.12431039959192276,0.2825664013624191,0.21769400765743155,0.4948340005651485,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation
4,q4k_16q1s32k,1,16,4096,16,32768.0,1.484940803050995,1.484940803050995,0.018112000077962875,0.3261695951223373,0.32348800003528594,0.7455351005236347,0.7394057025273603,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation
4,q2k_32q1s32k,1,32,2048,32,32768.0,1.3009952068328858,1.3009952068328858,0.01657600048929453,0.12431039959192276,0.5132320046424866,0.2536729054481981,1.0473223013846877,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation
4,q1k_64q1s32k,1,64,1024,64,32768.0,1.2837119936943056,1.2837119936943056,0.01643200032413006,0.08209280073642732,0.7449311971664427,0.12742497577885914,1.1562870179154463,-2.220446049250313e-16,one_fused_FA3_call_projected_for_Frontier_with_total_conservation
1 num_tensor_parallel_workers batch_spec num_prefill_seqs num_decode_seqs total_prefill_tokens total_decode_tokens decode_avg_kv_cache_size attention_core_mean_ms attention_core_mean_as_median_ms kv_cache_update_median_ms pure_prefill_reference_mean_ms pure_decode_reference_mean_ms projected_prefill_mean_ms projected_decode_mean_ms projection_sum_error_ms representation
2 1 q64_8q1s512 1 8 64 8 512.0 0.06604800000786781 0.06604800000786781 0.021359999664127827 0.05810240097343922 0.0600041144660541 0.032492258074592426 0.03355574193327539 0.0 one_fused_FA3_call_projected_for_Frontier_with_total_conservation
3 1 q128_8q1s1k 1 8 128 8 1024.0 0.07483199909329416 0.07483199909329416 0.024255999363958836 0.05587520003318787 0.06090559959411622 0.03580428402241854 0.03902771507087562 0.0 one_fused_FA3_call_projected_for_Frontier_with_total_conservation
4 1 q128_16q1s1k 1 16 128 16 1024.0 0.10597119852900506 0.10597119852900506 0.018112000077962875 0.05587520003318787 0.06891520097851753 0.04744885718421094 0.05852234134479412 0.0 one_fused_FA3_call_projected_for_Frontier_with_total_conservation
5 1 q256_16q1s2k 1 16 256 16 2048.0 0.13697599917650224 0.13697599917650224 0.01673599984496832 0.056396800279617305 0.11392960175871848 0.04535414343408448 0.09162185574241775 0.0 one_fused_FA3_call_projected_for_Frontier_with_total_conservation
6 1 q256_32q1s2k 1 32 256 32 2048.0 0.2163648009300232 0.2163648009300232 0.01515199989080429 0.056396800279617305 0.13807999789714814 0.06274415549817247 0.15362064543185072 0.0 one_fused_FA3_call_projected_for_Frontier_with_total_conservation
7 1 q512_32q1s4k 1 32 512 32 4096.0 0.3746495962142944 0.3746495962142944 0.016191999427974224 0.07432000041007995 0.2231455981731415 0.09360396051475776 0.2810456356995366 0.0 one_fused_FA3_call_projected_for_Frontier_with_total_conservation
8 1 q512_64q1s4k 1 64 512 64 4096.0 0.6718560099601746 0.6718560099601746 0.024848000146448612 0.07432000041007995 0.4058080047369003 0.10399797220840973 0.5678580377517648 -1.1102230246251565e-16 one_fused_FA3_call_projected_for_Frontier_with_total_conservation
9 1 q1k_64q1s8k 1 64 1024 64 8192.0 1.3472000002861022 1.3472000002861022 0.019600000232458115 0.1307712011039257 0.7464000046253203 0.2008444429250931 1.1463555573610091 0.0 one_fused_FA3_call_projected_for_Frontier_with_total_conservation
10 1 q2k_32q1s4k 1 32 2048 32 4096.0 0.6351647913455962 0.6351647913455962 0.028256000019609928 0.33396480381488797 0.2231455981731415 0.38075520430942184 0.25440958703617433 0.0 one_fused_FA3_call_projected_for_Frontier_with_total_conservation
11 1 q4k_16q1s4k 1 16 4096 16 4096.0 1.2515872120857237 1.2515872120857237 0.04395199939608574 1.0999775886535645 0.1389280006289482 1.1112371236754128 0.14035008841031085 0.0 one_fused_FA3_call_projected_for_Frontier_with_total_conservation
12 1 q512_8q1s16k 1 8 512 8 16384.0 0.6654527902603149 0.6654527902603149 0.016848000697791576 0.07432000041007995 0.28281279802322384 0.13848196486571557 0.5269708253945994 0.0 one_fused_FA3_call_projected_for_Frontier_with_total_conservation
13 1 q1k_16q1s16k 1 16 1024 16 16384.0 0.7349536061286925 0.7349536061286925 0.01976000051945448 0.1307712011039257 0.39996159672737125 0.18109068484526028 0.5538629212834322 -1.1102230246251565e-16 one_fused_FA3_call_projected_for_Frontier_with_total_conservation
14 1 q2k_32q1s16k 1 32 2048 32 16384.0 1.5246047973632815 1.5246047973632815 0.02759999968111515 0.33396480381488797 0.7430047929286957 0.4727750380198454 1.051829759343436 0.0 one_fused_FA3_call_projected_for_Frontier_with_total_conservation
15 1 q4k_16q1s32k 1 16 4096 16 32768.0 2.285759997367859 2.285759997367859 0.04617599956691265 1.0999775886535645 0.7502080142498017 1.3589365122072 0.9268234851606594 0.0 one_fused_FA3_call_projected_for_Frontier_with_total_conservation
16 1 q2k_32q1s32k 1 32 2048 32 32768.0 2.6958079814910887 2.6958079814910887 0.02817599941045046 0.33396480381488797 1.4325888037681578 0.5096392092471257 2.186168772243963 0.0 one_fused_FA3_call_projected_for_Frontier_with_total_conservation
17 1 q1k_64q1s32k 1 64 1024 64 32768.0 4.842601585388183 4.842601585388183 0.018943999893963337 0.1307712011039257 2.8293471813201903 0.21393496609766816 4.628666619290515 -8.881784197001252e-16 one_fused_FA3_call_projected_for_Frontier_with_total_conservation
18 2 q64_8q1s512 1 8 64 8 512.0 0.05652800053358078 0.05652800053358078 0.019600000232458115 0.05961279980838299 0.055968457407185014 0.029155180178492036 0.027372820355088746 6.938893903907228e-18 one_fused_FA3_call_projected_for_Frontier_with_total_conservation
19 2 q128_8q1s1k 1 8 128 8 1024.0 0.07362559959292413 0.07362559959292413 0.014431999996304512 0.05882879942655563 0.05778240002691746 0.03714313592014963 0.036482463672774496 0.0 one_fused_FA3_call_projected_for_Frontier_with_total_conservation
20 2 q128_16q1s1k 1 16 128 16 1024.0 0.0736224003136158 0.0736224003136158 0.014479999896138906 0.05882879942655563 0.06270079985260964 0.03563837490653034 0.037984025407085474 0.0 one_fused_FA3_call_projected_for_Frontier_with_total_conservation
21 2 q256_16q1s2k 1 16 256 16 2048.0 0.10047360062599181 0.10047360062599181 0.01425600005313754 0.05459520071744919 0.06628479920327664 0.045378692890289625 0.05509490773570219 0.0 one_fused_FA3_call_projected_for_Frontier_with_total_conservation
22 2 q256_32q1s2k 1 32 256 32 2048.0 0.14032640159130094 0.14032640159130094 0.01566399959847331 0.05459520071744919 0.09493440166115762 0.05123499252968345 0.0890914090616175 0.0 one_fused_FA3_call_projected_for_Frontier_with_total_conservation
23 2 q512_32q1s4k 1 32 512 32 4096.0 0.22105600088834762 0.22105600088834762 0.016032000072300434 0.06650560013949873 0.13783999979496003 0.07194410845270183 0.14911189243564577 -2.7755575615628914e-17 one_fused_FA3_call_projected_for_Frontier_with_total_conservation
24 2 q512_64q1s4k 1 64 512 64 4096.0 0.36867519915103913 0.36867519915103913 0.015263999812304974 0.06650560013949873 0.22715839892625808 0.0834932625520734 0.2851819365989658 5.551115123125783e-17 one_fused_FA3_call_projected_for_Frontier_with_total_conservation
25 2 q1k_64q1s8k 1 64 1024 64 8192.0 0.6853824079036711 0.6853824079036711 0.015856000129133463 0.08607039973139763 0.40332479774951924 0.12053885718696096 0.5648435507167102 1.1102230246251565e-16 one_fused_FA3_call_projected_for_Frontier_with_total_conservation
26 2 q2k_32q1s4k 1 32 2048 32 4096.0 0.3503839999437332 0.3503839999437332 0.018432000651955605 0.19546559900045393 0.13783999979496003 0.2054811520619412 0.14490284788179197 -5.551115123125783e-17 one_fused_FA3_call_projected_for_Frontier_with_total_conservation
27 2 q4k_16q1s4k 1 16 4096 16 4096.0 0.7185311973094941 0.7185311973094941 0.026575999334454536 0.5800191938877105 0.10921279862523078 0.6046757700946376 0.11385542721485654 0.0 one_fused_FA3_call_projected_for_Frontier_with_total_conservation
28 2 q512_8q1s16k 1 8 512 8 16384.0 0.6480192065238953 0.6480192065238953 0.016127999871969223 0.06650560013949873 0.18568639904260634 0.17088926838108623 0.47712993814280913 0.0 one_fused_FA3_call_projected_for_Frontier_with_total_conservation
29 2 q1k_16q1s16k 1 16 1024 16 16384.0 0.6820319950580597 0.6820319950580597 0.016959999687969685 0.08607039973139763 0.2815328001976013 0.15969057520605917 0.5223414198520004 -1.1102230246251565e-16 one_fused_FA3_call_projected_for_Frontier_with_total_conservation
30 2 q2k_32q1s16k 1 32 2048 32 16384.0 0.7942592084407808 0.7942592084407808 0.0191040001809597 0.19546559900045393 0.40225600004196166 0.259736894547936 0.5345223138928448 0.0 one_fused_FA3_call_projected_for_Frontier_with_total_conservation
31 2 q4k_16q1s32k 1 16 4096 16 32768.0 1.759881591796875 1.759881591796875 0.02705600019544363 0.5800191938877105 0.511932796239853 0.93480767601568 0.8250739157811953 2.220446049250313e-16 one_fused_FA3_call_projected_for_Frontier_with_total_conservation
32 2 q2k_32q1s32k 1 32 2048 32 32768.0 1.3758272171020507 1.3758272171020507 0.019024000503122807 0.19546559900045393 0.7483008027076722 0.2849506939696605 1.0908765231323903 0.0 one_fused_FA3_call_projected_for_Frontier_with_total_conservation
33 2 q1k_64q1s32k 1 64 1024 64 32768.0 2.44380156993866 2.44380156993866 0.015728000551462173 0.08607039973139763 1.4435008168220522 0.13751499486424038 2.3062865750744197 0.0 one_fused_FA3_call_projected_for_Frontier_with_total_conservation
34 4 q64_8q1s512 1 8 64 8 512.0 0.0562208004295826 0.0562208004295826 0.015520000364631414 0.06054079942405224 0.0591108573866742 0.028446344104128624 0.027774456325453972 -6.938893903907228e-18 one_fused_FA3_call_projected_for_Frontier_with_total_conservation
35 4 q128_8q1s1k 1 8 128 8 1024.0 0.061199999228119854 0.061199999228119854 0.018240000121295452 0.05626560002565384 0.0588383998721838 0.029916029687899703 0.03128396954022015 -6.938893903907228e-18 one_fused_FA3_call_projected_for_Frontier_with_total_conservation
36 4 q128_16q1s1k 1 16 128 16 1024.0 0.06104319989681244 0.06104319989681244 0.016207999549806118 0.05626560002565384 0.05755840018391609 0.030174939058162802 0.03086826083864964 0.0 one_fused_FA3_call_projected_for_Frontier_with_total_conservation
37 4 q256_16q1s2k 1 16 256 16 2048.0 0.07356479987502099 0.07356479987502099 0.015584000386297703 0.0556256003677845 0.06238719932734965 0.0346749350032807 0.038889864871740294 0.0 one_fused_FA3_call_projected_for_Frontier_with_total_conservation
38 4 q256_32q1s2k 1 32 256 32 2048.0 0.09806400015950203 0.09806400015950203 0.015008000191301107 0.0556256003677845 0.06935679838061332 0.04364509673334097 0.05441890342616107 0.0 one_fused_FA3_call_projected_for_Frontier_with_total_conservation
39 4 q512_32q1s4k 1 32 512 32 4096.0 0.1509471982717514 0.1509471982717514 0.014864000026136637 0.0640383992344141 0.10621120035648347 0.05677791295527661 0.09416928531647478 0.0 one_fused_FA3_call_projected_for_Frontier_with_total_conservation
40 4 q512_64q1s4k 1 64 512 64 4096.0 0.21687040030956264 0.21687040030956264 0.015343999955803156 0.0640383992344141 0.14040640145540234 0.06793047918211377 0.1489399211274489 2.7755575615628914e-17 one_fused_FA3_call_projected_for_Frontier_with_total_conservation
41 4 q1k_64q1s8k 1 64 1024 64 8192.0 0.3848575979471207 0.3848575979471207 0.015647999942302704 0.08209280073642732 0.22809920012950893 0.10185316839884552 0.2830044295482752 0.0 one_fused_FA3_call_projected_for_Frontier_with_total_conservation
42 4 q2k_32q1s4k 1 32 2048 32 4096.0 0.2785151988267898 0.2785151988267898 0.01726400014013052 0.12431039959192276 0.10621120035648347 0.1501912864839173 0.1283239123428725 -5.551115123125783e-17 one_fused_FA3_call_projected_for_Frontier_with_total_conservation
43 4 q4k_16q1s4k 1 16 4096 16 4096.0 0.456512001156807 0.456512001156807 0.017680000513792038 0.3261695951223373 0.06970879957079888 0.3761264483787333 0.08038555277807365 -5.551115123125783e-17 one_fused_FA3_call_projected_for_Frontier_with_total_conservation
44 4 q512_8q1s16k 1 8 512 8 16384.0 0.15639680176973342 0.15639680176973342 0.0266720000654459 0.0640383992344141 0.09146559983491898 0.0644060660218149 0.09199073574791852 0.0 one_fused_FA3_call_projected_for_Frontier_with_total_conservation
45 4 q1k_16q1s16k 1 16 1024 16 16384.0 0.6500960052013398 0.6500960052013398 0.01595200039446354 0.08209280073642732 0.1866239994764328 0.19860388994013833 0.45149211526120153 0.0 one_fused_FA3_call_projected_for_Frontier_with_total_conservation
46 4 q2k_32q1s16k 1 32 2048 32 16384.0 0.71252800822258 0.71252800822258 0.016944000497460365 0.12431039959192276 0.2825664013624191 0.21769400765743155 0.4948340005651485 0.0 one_fused_FA3_call_projected_for_Frontier_with_total_conservation
47 4 q4k_16q1s32k 1 16 4096 16 32768.0 1.484940803050995 1.484940803050995 0.018112000077962875 0.3261695951223373 0.32348800003528594 0.7455351005236347 0.7394057025273603 0.0 one_fused_FA3_call_projected_for_Frontier_with_total_conservation
48 4 q2k_32q1s32k 1 32 2048 32 32768.0 1.3009952068328858 1.3009952068328858 0.01657600048929453 0.12431039959192276 0.5132320046424866 0.2536729054481981 1.0473223013846877 0.0 one_fused_FA3_call_projected_for_Frontier_with_total_conservation
49 4 q1k_64q1s32k 1 64 1024 64 32768.0 1.2837119936943056 1.2837119936943056 0.01643200032413006 0.08209280073642732 0.7449311971664427 0.12742497577885914 1.1562870179154463 -2.220446049250313e-16 one_fused_FA3_call_projected_for_Frontier_with_total_conservation

37
runs/frontier-fidelity-envelope-v1/profiles/profile-v4-trace-final/linear_op.csv Normal file
View File

@@ -0,0 +1,37 @@
time_stats.emb.min,time_stats.emb.max,time_stats.emb.mean,time_stats.emb.median,time_stats.emb.std,time_stats.input_layernorm.min,time_stats.input_layernorm.max,time_stats.input_layernorm.mean,time_stats.input_layernorm.median,time_stats.input_layernorm.std,time_stats.attn_pre_proj.min,time_stats.attn_pre_proj.max,time_stats.attn_pre_proj.mean,time_stats.attn_pre_proj.median,time_stats.attn_pre_proj.std,time_stats.attn_rope.min,time_stats.attn_rope.max,time_stats.attn_rope.mean,time_stats.attn_rope.median,time_stats.attn_rope.std,time_stats.attn_post_proj.min,time_stats.attn_post_proj.max,time_stats.attn_post_proj.mean,time_stats.attn_post_proj.median,time_stats.attn_post_proj.std,time_stats.post_attention_layernorm.min,time_stats.post_attention_layernorm.max,time_stats.post_attention_layernorm.mean,time_stats.post_attention_layernorm.median,time_stats.post_attention_layernorm.std,n_head,n_kv_head,n_embd,n_expanded_embd,vocab_size,use_gated_mlp,use_qk_norm,attn_output_gate,num_tokens,num_tensor_parallel_workers,padded_n_embd,padded_n_expanded_embd,model_arch,is_step2_mini,share_expert_dim,share_q_dim,measurement_type,profiling_precision,quant_signature
0.029184000566601753,0.06780800223350525,0.03157280012965202,0.030736000277101994,0.005874173435341216,0.033215999603271484,0.04825599864125252,0.03443359974771738,0.0337119996547699,0.0031825678429048183,1.438431978225708,1.505568027496338,1.446228802204132,1.4429279565811157,0.014128607642643025,0.538752019405365,0.5440319776535034,0.5416463971138,0.5420799851417542,0.0016099306164432847,1.0073280334472656,1.0163840055465698,1.0098415970802308,1.0081279873847961,0.0032980457197329728,0.04022400081157684,0.041728001087903976,0.04091359991580248,0.04081599973142147,0.0004728505416767937,32,4,2048,768,151936,True,True,False,8192,1,2048,768,generic,False,,,CUDA_EVENT,BF16,none
0.01360000018030405,0.06784000247716904,0.017755200061947106,0.0179840000346303,0.00837681421401443,0.01836800016462803,0.03651199862360954,0.019606399815529585,0.018719999119639397,0.0038821958848767424,0.7512000203132629,0.8208960294723511,0.7566704005002975,0.75382399559021,0.014793092586577971,0.28995200991630554,0.29337599873542786,0.29135999977588656,0.29150401055812836,0.000998381071258815,0.5149760246276855,0.5169600248336792,0.5160208016633987,0.5158880054950714,0.0005038652783291977,0.02051199972629547,0.021503999829292297,0.021067200042307378,0.021104000508785248,0.00023542855306902367,32,4,2048,768,151936,True,True,False,4096,1,2048,768,generic,False,,,CUDA_EVENT,BF16,none
0.0080960001796484,0.04281599819660187,0.011092799971811474,0.011039999779313803,0.005489135752949302,0.012223999947309494,0.026335999369621277,0.013187199970707298,0.01247999956831336,0.0030236510562153375,0.3928639888763428,0.45372799038887024,0.3976895987987518,0.39528000354766846,0.012905176716136006,0.1547199934720993,0.15884800255298615,0.15712319910526276,0.1573439985513687,0.001165121945135037,0.26633599400520325,0.268095999956131,0.26719200164079665,0.2671840041875839,0.0004242740199415328,0.013024000450968742,0.013887999579310417,0.013489600038155913,0.013520000036805868,0.0002382817554057738,32,4,2048,768,151936,True,True,False,2048,1,2048,768,generic,False,,,CUDA_EVENT,BF16,none
0.00825599953532219,0.04755200073122978,0.02398160002194345,0.01961600035429001,0.00874683840888523,0.018880000337958336,0.03868800029158592,0.021590400114655496,0.0208320003002882,0.004057015751746236,0.24316799640655518,0.2710399925708771,0.2521967992186546,0.25065599381923676,0.007998418528894075,0.09644799679517746,0.19120000302791595,0.10407840013504029,0.09963199868798256,0.020043722414992166,0.13600000739097595,0.18892799317836761,0.15760480016469955,0.15760000050067902,0.0112223677907762,0.009664000011980534,0.010015999898314476,0.009836799977347255,0.009824000298976898,9.016971952948177e-05,32,4,2048,768,151936,True,True,False,1024,1,2048,768,generic,False,,,CUDA_EVENT,BF16,none
0.017376000061631203,0.044704001396894455,0.026318399980664254,0.027312000282108784,0.007565344034680866,0.01836800016462803,0.03014400042593479,0.021276800055056812,0.020655999891459942,0.002632977855097951,0.1438719928264618,0.17132799327373505,0.152497598528862,0.15012799948453903,0.007947150319625347,0.1430719941854477,0.19305600225925446,0.16630879789590836,0.1685439944267273,0.014628024163894684,0.08899199962615967,0.10467199981212616,0.0943599995225668,0.09374399855732918,0.003472669494074113,0.007615999784320593,0.007935999892652035,0.007769599952735007,0.0077760000713169575,7.680004540222077e-05,32,4,2048,768,151936,True,True,False,512,1,2048,768,generic,False,,,CUDA_EVENT,BF16,none
0.016992000862956047,0.0544000007212162,0.02573199989274144,0.026016000658273697,0.00803323401720434,0.018432000651955605,0.02348800003528595,0.020648000109940768,0.02062400057911873,0.0013939985047930988,0.10220800340175629,0.1361600011587143,0.11850560046732425,0.11684799939393997,0.010637754898360304,0.16710400581359863,0.21110400557518005,0.19078560024499894,0.19409599900245667,0.013714352646558832,0.06265600025653839,0.0740479975938797,0.06842879951000214,0.0690080001950264,0.0031292240446560557,0.00979200005531311,0.033440001308918,0.01864320016466081,0.017280000261962414,0.006957998188876383,32,4,2048,768,151936,True,True,False,256,1,2048,768,generic,False,,,CUDA_EVENT,BF16,none
0.017152000218629837,0.04396799951791763,0.025907999789342284,0.02598400041460991,0.007714554737915267,0.018400000408291817,0.03667199984192848,0.024132800102233887,0.02112000063061714,0.006120348733354326,0.10678400099277496,0.1363839954137802,0.1193264003843069,0.11583999916911125,0.009838647443214228,0.17017599940299988,0.22748799622058868,0.18853759989142418,0.18433599919080734,0.015728078443174653,0.04569600149989128,0.06652799993753433,0.05192639995366335,0.05151999928057194,0.004516605694976449,0.021856000646948814,0.026335999369621277,0.02384479995816946,0.023599999956786633,0.0011301153013314744,32,4,2048,768,151936,True,True,False,128,1,2048,768,generic,False,,,CUDA_EVENT,BF16,none
0.017343999817967415,1.0683200359344482,0.05748240072280168,0.029504000209271908,0.16366824814254827,0.018688000738620758,0.3317759931087494,0.03685439983382821,0.021151999942958355,0.06767422466731164,0.10255999863147736,0.9434880018234253,0.16412640027701855,0.11956800147891045,0.17964606281728834,0.1714559942483902,2.1306240558624268,0.3011296011507511,0.1926399990916252,0.42310127734378766,0.03574400022625923,0.6859520077705383,0.08389280084520578,0.04279999993741512,0.14321647071615612,0.020479999482631683,0.1831360012292862,0.033024000097066165,0.023856000043451786,0.03470987082429836,32,4,2048,768,151936,True,True,False,64,1,2048,768,generic,False,,,CUDA_EVENT,BF16,none
0.016095999628305435,0.05142400041222572,0.026363200135529043,0.02676799986511469,0.008637725852473854,0.01849599927663803,0.03577600046992302,0.022193600237369538,0.020848000422120094,0.004615266181181815,0.10540799796581268,0.15014399588108063,0.12211520001292228,0.11896000057458878,0.012772013396624768,0.17315199971199036,0.21478399634361267,0.1881632000207901,0.1873439997434616,0.011761164657572015,0.03481600061058998,0.058079998940229416,0.042200000025331974,0.03969600051641464,0.006461281521769989,0.02143999934196472,0.038816001266241074,0.02466559996828437,0.023856000043451786,0.0035418033748569927,32,4,2048,768,151936,True,True,False,32,1,2048,768,generic,False,,,CUDA_EVENT,BF16,none
0.015904000028967857,0.03753599897027016,0.023127999808639287,0.02527999971061945,0.0054230027890305385,0.018400000408291817,0.03001599945127964,0.020488000102341176,0.020096000283956528,0.002585688394137697,0.10355199873447418,0.1438400000333786,0.11536479964852334,0.11124800145626068,0.011136028294919255,0.16502399742603302,0.2072959989309311,0.18646399974822997,0.19075199961662292,0.013296437761247597,0.03142400085926056,0.05215999856591225,0.03888959977775812,0.03750399872660637,0.0057399302067536314,0.020128000527620316,0.04064000025391579,0.023937600292265417,0.023648000322282314,0.004144197400898248,32,4,2048,768,151936,True,True,False,16,1,2048,768,generic,False,,,CUDA_EVENT,BF16,none
0.015200000256299973,0.037151999771595,0.02144480012357235,0.02195199951529503,0.005576364091933697,0.017952000722289085,0.0226879995316267,0.019934400077909233,0.019952000118792057,0.0011812499470458758,0.10063999891281128,0.13468800485134125,0.11595199964940547,0.1207519993185997,0.011974301621092394,0.16332800686359406,0.20748800039291382,0.18162400051951408,0.1767839938402176,0.01474120743720334,0.03222399950027466,0.043455999344587326,0.03829439990222454,0.03859200142323971,0.0031378131240041122,0.020959999412298203,0.03587200120091438,0.023795200139284135,0.023423999547958374,0.0031339489695198443,32,4,2048,768,151936,True,True,False,8,1,2048,768,generic,False,,,CUDA_EVENT,BF16,none
0.014944000169634819,0.04022400081157684,0.02162720002233982,0.022463999688625336,0.006017219317026815,0.018464000895619392,0.026528000831604004,0.021264000236988066,0.020896000787615776,0.001934095017586082,0.10156799852848053,0.1703999936580658,0.12565439902245998,0.1244799979031086,0.01641776722785409,0.1653759926557541,0.23865599930286407,0.1969360001385212,0.19223999977111816,0.02058903050274278,0.03254399821162224,0.06752000004053116,0.0443536002188921,0.041519999504089355,0.00984829071098989,0.020096000283956528,0.040031999349594116,0.026934400014579297,0.02478400059044361,0.00562071702648593,32,4,2048,768,151936,True,True,False,1,1,2048,768,generic,False,,,CUDA_EVENT,BF16,none
,,,,,,,,,,0.7531200051307678,0.8461440205574036,0.7618160009384155,0.7576479911804199,0.019464233617982506,0.2922559976577759,0.2985599935054779,0.2953856036067009,0.29576000571250916,0.001575901077689139,0.510047972202301,0.5140479803085327,0.5121696025133133,0.5123839974403381,0.0012263137313476844,,,,,,32,4,2048,768,151936,True,True,False,8192,2,2048,768,generic,False,,,CUDA_EVENT,BF16,none
,,,,,,,,,,0.39529600739479065,0.47279998660087585,0.40216960161924364,0.39825600385665894,0.0162749796240819,0.15625600516796112,0.16211199760437012,0.15959519892930984,0.15988799929618835,0.0011493418942396922,0.2635200023651123,0.2642880082130432,0.2639120012521744,0.26392000913619995,0.0001903593401384135,,,,,,32,4,2048,768,151936,True,True,False,4096,2,2048,768,generic,False,,,CUDA_EVENT,BF16,none
,,,,,,,,,,0.2531839907169342,0.32419198751449585,0.26924319565296173,0.26049599051475525,0.01657194262368116,0.10051199793815613,0.2375359982252121,0.12411200068891048,0.10311999917030334,0.039466165428540506,0.15881599485874176,0.19289599359035492,0.16896959990262986,0.1640480011701584,0.009162416086418127,,,,,,32,4,2048,768,151936,True,True,False,2048,2,2048,768,generic,False,,,CUDA_EVENT,BF16,none
,,,,,,,,,,0.15302400290966034,0.1780800074338913,0.16284480094909667,0.16113600134849548,0.00740355661356144,0.14521600306034088,0.20233599841594696,0.17807039842009545,0.1796799972653389,0.013908448560094403,0.0907519981265068,0.09750399738550186,0.09460479989647866,0.09478399902582169,0.0017965487667361475,,,,,,32,4,2048,768,151936,True,True,False,1024,2,2048,768,generic,False,,,CUDA_EVENT,BF16,none
,,,,,,,,,,0.11740799993276596,0.17958399653434753,0.13262080028653145,0.12878400087356567,0.014146060532423056,0.17468799650669098,0.21161599457263947,0.1910431995987892,0.18966399878263474,0.01270903647700971,0.05926400050520897,0.07577600330114365,0.06530559975653887,0.06404799968004227,0.00479458462514625,,,,,,32,4,2048,768,151936,True,True,False,512,2,2048,768,generic,False,,,CUDA_EVENT,BF16,none
,,,,,,,,,,0.11260800063610077,0.17209599912166595,0.13610880002379416,0.13809599727392197,0.017675922458993677,0.18729600310325623,0.26822400093078613,0.20815680101513861,0.2078079953789711,0.01700718593658771,0.04499199986457825,0.06537599861621857,0.0507551996037364,0.04787199944257736,0.005727123232692158,,,,,,32,4,2048,768,151936,True,True,False,256,2,2048,768,generic,False,,,CUDA_EVENT,BF16,none
,,,,,,,,,,0.11257600039243698,0.23452800512313843,0.13866880126297473,0.13964799791574478,0.026590047697062115,0.1828799992799759,0.24751999974250793,0.20735519900918006,0.20670399814844131,0.01500438131586336,0.03654399886727333,0.05593600124120712,0.0408239996060729,0.03892800025641918,0.004864409450710354,,,,,,32,4,2048,768,151936,True,True,False,128,2,2048,768,generic,False,,,CUDA_EVENT,BF16,none
,,,,,,,,,,0.11734399944543839,0.18726399540901184,0.13890240006148816,0.13308800011873245,0.019086167340006257,0.16991999745368958,0.2443840056657791,0.19185120090842248,0.19257599860429764,0.018017536159219673,0.03033600002527237,0.04956800118088722,0.038387199863791466,0.03750400058925152,0.00500897018702887,,,,,,32,4,2048,768,151936,True,True,False,64,2,2048,768,generic,False,,,CUDA_EVENT,BF16,none
,,,,,,,,,,0.11420799791812897,0.18115200102329254,0.13938880078494548,0.1393439993262291,0.018141313962922536,0.1693439930677414,0.221343994140625,0.19187839925289155,0.19438399374485016,0.01430752121272052,0.03017600066959858,0.06019200012087822,0.03866560012102127,0.0364960003644228,0.0074199790031205266,,,,,,32,4,2048,768,151936,True,True,False,32,2,2048,768,generic,False,,,CUDA_EVENT,BF16,none
,,,,,,,,,,0.11276800185441971,0.16223999857902527,0.1331360016018152,0.13305599987506866,0.01432493740801146,0.17187200486660004,0.23625600337982178,0.19287680014967917,0.1913280040025711,0.01754628831589704,0.029823999851942062,0.07574400305747986,0.03900959976017475,0.036927999928593636,0.009277465083962879,,,,,,32,4,2048,768,151936,True,True,False,16,2,2048,768,generic,False,,,CUDA_EVENT,BF16,none
,,,,,,,,,,0.1130559965968132,0.19814400374889374,0.13379519879817964,0.12531199678778648,0.020074427302248836,0.16841599345207214,0.21139200031757355,0.1893615983426571,0.19092799723148346,0.012418720676762711,0.029503999277949333,0.07558400183916092,0.040144000016152856,0.03728000074625015,0.010223239196498205,,,,,,32,4,2048,768,151936,True,True,False,8,2,2048,768,generic,False,,,CUDA_EVENT,BF16,none
,,,,,,,,,,0.115167997777462,0.2640640139579773,0.13805920109152794,0.13118399679660797,0.03132849683515479,0.17103999853134155,0.2977280020713806,0.19899839907884598,0.1976960003376007,0.028928046763067948,0.0297279991209507,0.05990400165319443,0.03834720011800528,0.0363520011305809,0.0070885330713495905,,,,,,32,4,2048,768,151936,True,True,False,1,2,2048,768,generic,False,,,CUDA_EVENT,BF16,none
,,,,,,,,,,0.3940800130367279,0.46483200788497925,0.39923040121793746,0.3957759886980057,0.015108903906233569,0.16022400557994843,0.1634880006313324,0.16203359961509706,0.16228799521923065,0.0010220720912414007,0.26073598861694336,0.2627840042114258,0.26138080209493636,0.2613760083913803,0.00041640363494172775,,,,,,32,4,2048,768,151936,True,True,False,8192,4,2048,768,generic,False,,,CUDA_EVENT,BF16,none
,,,,,,,,,,0.2536959946155548,0.27161601185798645,0.25960480123758317,0.2577280104160309,0.005142017404245015,0.09849599748849869,0.10281600058078766,0.10057279989123344,0.10063999891281128,0.0008836232237268523,0.13913600146770477,0.17606399953365326,0.15875840038061143,0.15988799929618835,0.010058952112389172,,,,,,32,4,2048,768,151936,True,True,False,4096,4,2048,768,generic,False,,,CUDA_EVENT,BF16,none
,,,,,,,,,,0.15142400562763214,0.1780479997396469,0.16139679849147798,0.1602879986166954,0.0070656055731385115,0.14601600170135498,0.23343999683856964,0.17913119941949845,0.179967999458313,0.023065274309176566,0.09388799965381622,0.12108799815177917,0.10317599996924401,0.10073599964380264,0.007364345618470178,,,,,,32,4,2048,768,151936,True,True,False,2048,4,2048,768,generic,False,,,CUDA_EVENT,BF16,none
,,,,,,,,,,0.12198399752378464,0.18406400084495544,0.13770400024950505,0.13232000172138214,0.01623164885687898,0.1773120015859604,0.20688000321388245,0.1870912007987499,0.18535999953746796,0.00825628058448234,0.05766399949789047,0.06739199906587601,0.06187200043350458,0.061216000467538834,0.003004312467037863,,,,,,32,4,2048,768,151936,True,True,False,1024,4,2048,768,generic,False,,,CUDA_EVENT,BF16,none
,,,,,,,,,,0.11737599968910217,0.18614399433135986,0.13109439946711063,0.12771200388669968,0.01372168725934724,0.17587199807167053,0.2699519991874695,0.19926720038056372,0.19075199961662292,0.02327478982490538,0.041728001087903976,0.06224000081419945,0.05000480003654957,0.049375999718904495,0.006185850944273961,,,,,,32,4,2048,768,151936,True,True,False,512,4,2048,768,generic,False,,,CUDA_EVENT,BF16,none
,,,,,,,,,,0.1159679964184761,0.1610880047082901,0.12988320142030715,0.12494400516152382,0.011865375783184065,0.1767680048942566,0.27379199862480164,0.21035519987344742,0.21275199949741364,0.023206964017550125,0.037087999284267426,0.062144000083208084,0.04439679980278015,0.04283200018107891,0.006155226392511492,,,,,,32,4,2048,768,151936,True,True,False,256,4,2048,768,generic,False,,,CUDA_EVENT,BF16,none
,,,,,,,,,,0.12015999853610992,0.158720001578331,0.13573280088603495,0.1343199983239174,0.010666830836014414,0.1737920045852661,0.22127999365329742,0.20261440128087999,0.20321600139141083,0.011153965849067301,0.03587200120091438,0.04944000020623207,0.038265600241720675,0.037328001111745834,0.0030431580113575636,,,,,,32,4,2048,768,151936,True,True,False,128,4,2048,768,generic,False,,,CUDA_EVENT,BF16,none
,,,,,,,,,,0.1212799996137619,0.15139199793338776,0.13183839991688728,0.13014400005340576,0.00851207547022104,0.17257599532604218,0.2250880002975464,0.1872655987739563,0.18193599581718445,0.013638284975248818,0.02969600073993206,0.0525440014898777,0.03840640028938651,0.03444799967110157,0.007598511754254174,,,,,,32,4,2048,768,151936,True,True,False,64,4,2048,768,generic,False,,,CUDA_EVENT,BF16,none
,,,,,,,,,,0.11270400136709213,0.1624639928340912,0.13508000001311302,0.13180799782276154,0.014243564451606775,0.17052799463272095,0.23715199530124664,0.1953311987221241,0.1966560035943985,0.01500670718978398,0.030239999294281006,0.05270399898290634,0.03807039987295866,0.03742399998009205,0.004906165602021684,,,,,,32,4,2048,768,151936,True,True,False,32,4,2048,768,generic,False,,,CUDA_EVENT,BF16,none
,,,,,,,,,,0.12319999933242798,0.17919999361038208,0.14059039913117885,0.14156799763441086,0.016217662982107223,0.1701119989156723,0.21987199783325195,0.18903039917349815,0.1870879977941513,0.014117854507841239,0.030368000268936157,0.06406400352716446,0.04045119984075427,0.03710399940609932,0.00878942205983628,,,,,,32,4,2048,768,151936,True,True,False,16,4,2048,768,generic,False,,,CUDA_EVENT,BF16,none
,,,,,,,,,,0.11296000331640244,0.16761599481105804,0.13843199908733367,0.13814399391412735,0.014925286935582404,0.1711679995059967,0.21561600267887115,0.1906527981162071,0.1876479983329773,0.011803992622137974,0.0306560005992651,0.09216000139713287,0.041129599791020155,0.036847999319434166,0.013739721468154687,,,,,,32,4,2048,768,151936,True,True,False,8,4,2048,768,generic,False,,,CUDA_EVENT,BF16,none
,,,,,,,,,,0.11929599940776825,0.2008959949016571,0.14535359852015972,0.13971199840307236,0.024769473061576282,0.16803200542926788,0.25123199820518494,0.19573760256171227,0.19366399943828583,0.023196011081705884,0.03049599938094616,0.08089599758386612,0.04739360017701984,0.0453919991850853,0.01286389846889463,,,,,,32,4,2048,768,151936,True,True,False,1,4,2048,768,generic,False,,,CUDA_EVENT,BF16,none
1 time_stats.emb.min time_stats.emb.max time_stats.emb.mean time_stats.emb.median time_stats.emb.std time_stats.input_layernorm.min time_stats.input_layernorm.max time_stats.input_layernorm.mean time_stats.input_layernorm.median time_stats.input_layernorm.std time_stats.attn_pre_proj.min time_stats.attn_pre_proj.max time_stats.attn_pre_proj.mean time_stats.attn_pre_proj.median time_stats.attn_pre_proj.std time_stats.attn_rope.min time_stats.attn_rope.max time_stats.attn_rope.mean time_stats.attn_rope.median time_stats.attn_rope.std time_stats.attn_post_proj.min time_stats.attn_post_proj.max time_stats.attn_post_proj.mean time_stats.attn_post_proj.median time_stats.attn_post_proj.std time_stats.post_attention_layernorm.min time_stats.post_attention_layernorm.max time_stats.post_attention_layernorm.mean time_stats.post_attention_layernorm.median time_stats.post_attention_layernorm.std n_head n_kv_head n_embd n_expanded_embd vocab_size use_gated_mlp use_qk_norm attn_output_gate num_tokens num_tensor_parallel_workers padded_n_embd padded_n_expanded_embd model_arch is_step2_mini share_expert_dim share_q_dim measurement_type profiling_precision quant_signature
2 0.029184000566601753 0.06780800223350525 0.03157280012965202 0.030736000277101994 0.005874173435341216 0.033215999603271484 0.04825599864125252 0.03443359974771738 0.0337119996547699 0.0031825678429048183 1.438431978225708 1.505568027496338 1.446228802204132 1.4429279565811157 0.014128607642643025 0.538752019405365 0.5440319776535034 0.5416463971138 0.5420799851417542 0.0016099306164432847 1.0073280334472656 1.0163840055465698 1.0098415970802308 1.0081279873847961 0.0032980457197329728 0.04022400081157684 0.041728001087903976 0.04091359991580248 0.04081599973142147 0.0004728505416767937 32 4 2048 768 151936 True True False 8192 1 2048 768 generic False CUDA_EVENT BF16 none
3 0.01360000018030405 0.06784000247716904 0.017755200061947106 0.0179840000346303 0.00837681421401443 0.01836800016462803 0.03651199862360954 0.019606399815529585 0.018719999119639397 0.0038821958848767424 0.7512000203132629 0.8208960294723511 0.7566704005002975 0.75382399559021 0.014793092586577971 0.28995200991630554 0.29337599873542786 0.29135999977588656 0.29150401055812836 0.000998381071258815 0.5149760246276855 0.5169600248336792 0.5160208016633987 0.5158880054950714 0.0005038652783291977 0.02051199972629547 0.021503999829292297 0.021067200042307378 0.021104000508785248 0.00023542855306902367 32 4 2048 768 151936 True True False 4096 1 2048 768 generic False CUDA_EVENT BF16 none
4 0.0080960001796484 0.04281599819660187 0.011092799971811474 0.011039999779313803 0.005489135752949302 0.012223999947309494 0.026335999369621277 0.013187199970707298 0.01247999956831336 0.0030236510562153375 0.3928639888763428 0.45372799038887024 0.3976895987987518 0.39528000354766846 0.012905176716136006 0.1547199934720993 0.15884800255298615 0.15712319910526276 0.1573439985513687 0.001165121945135037 0.26633599400520325 0.268095999956131 0.26719200164079665 0.2671840041875839 0.0004242740199415328 0.013024000450968742 0.013887999579310417 0.013489600038155913 0.013520000036805868 0.0002382817554057738 32 4 2048 768 151936 True True False 2048 1 2048 768 generic False CUDA_EVENT BF16 none
5 0.00825599953532219 0.04755200073122978 0.02398160002194345 0.01961600035429001 0.00874683840888523 0.018880000337958336 0.03868800029158592 0.021590400114655496 0.0208320003002882 0.004057015751746236 0.24316799640655518 0.2710399925708771 0.2521967992186546 0.25065599381923676 0.007998418528894075 0.09644799679517746 0.19120000302791595 0.10407840013504029 0.09963199868798256 0.020043722414992166 0.13600000739097595 0.18892799317836761 0.15760480016469955 0.15760000050067902 0.0112223677907762 0.009664000011980534 0.010015999898314476 0.009836799977347255 0.009824000298976898 9.016971952948177e-05 32 4 2048 768 151936 True True False 1024 1 2048 768 generic False CUDA_EVENT BF16 none
6 0.017376000061631203 0.044704001396894455 0.026318399980664254 0.027312000282108784 0.007565344034680866 0.01836800016462803 0.03014400042593479 0.021276800055056812 0.020655999891459942 0.002632977855097951 0.1438719928264618 0.17132799327373505 0.152497598528862 0.15012799948453903 0.007947150319625347 0.1430719941854477 0.19305600225925446 0.16630879789590836 0.1685439944267273 0.014628024163894684 0.08899199962615967 0.10467199981212616 0.0943599995225668 0.09374399855732918 0.003472669494074113 0.007615999784320593 0.007935999892652035 0.007769599952735007 0.0077760000713169575 7.680004540222077e-05 32 4 2048 768 151936 True True False 512 1 2048 768 generic False CUDA_EVENT BF16 none
7 0.016992000862956047 0.0544000007212162 0.02573199989274144 0.026016000658273697 0.00803323401720434 0.018432000651955605 0.02348800003528595 0.020648000109940768 0.02062400057911873 0.0013939985047930988 0.10220800340175629 0.1361600011587143 0.11850560046732425 0.11684799939393997 0.010637754898360304 0.16710400581359863 0.21110400557518005 0.19078560024499894 0.19409599900245667 0.013714352646558832 0.06265600025653839 0.0740479975938797 0.06842879951000214 0.0690080001950264 0.0031292240446560557 0.00979200005531311 0.033440001308918 0.01864320016466081 0.017280000261962414 0.006957998188876383 32 4 2048 768 151936 True True False 256 1 2048 768 generic False CUDA_EVENT BF16 none
8 0.017152000218629837 0.04396799951791763 0.025907999789342284 0.02598400041460991 0.007714554737915267 0.018400000408291817 0.03667199984192848 0.024132800102233887 0.02112000063061714 0.006120348733354326 0.10678400099277496 0.1363839954137802 0.1193264003843069 0.11583999916911125 0.009838647443214228 0.17017599940299988 0.22748799622058868 0.18853759989142418 0.18433599919080734 0.015728078443174653 0.04569600149989128 0.06652799993753433 0.05192639995366335 0.05151999928057194 0.004516605694976449 0.021856000646948814 0.026335999369621277 0.02384479995816946 0.023599999956786633 0.0011301153013314744 32 4 2048 768 151936 True True False 128 1 2048 768 generic False CUDA_EVENT BF16 none
9 0.017343999817967415 1.0683200359344482 0.05748240072280168 0.029504000209271908 0.16366824814254827 0.018688000738620758 0.3317759931087494 0.03685439983382821 0.021151999942958355 0.06767422466731164 0.10255999863147736 0.9434880018234253 0.16412640027701855 0.11956800147891045 0.17964606281728834 0.1714559942483902 2.1306240558624268 0.3011296011507511 0.1926399990916252 0.42310127734378766 0.03574400022625923 0.6859520077705383 0.08389280084520578 0.04279999993741512 0.14321647071615612 0.020479999482631683 0.1831360012292862 0.033024000097066165 0.023856000043451786 0.03470987082429836 32 4 2048 768 151936 True True False 64 1 2048 768 generic False CUDA_EVENT BF16 none
10 0.016095999628305435 0.05142400041222572 0.026363200135529043 0.02676799986511469 0.008637725852473854 0.01849599927663803 0.03577600046992302 0.022193600237369538 0.020848000422120094 0.004615266181181815 0.10540799796581268 0.15014399588108063 0.12211520001292228 0.11896000057458878 0.012772013396624768 0.17315199971199036 0.21478399634361267 0.1881632000207901 0.1873439997434616 0.011761164657572015 0.03481600061058998 0.058079998940229416 0.042200000025331974 0.03969600051641464 0.006461281521769989 0.02143999934196472 0.038816001266241074 0.02466559996828437 0.023856000043451786 0.0035418033748569927 32 4 2048 768 151936 True True False 32 1 2048 768 generic False CUDA_EVENT BF16 none
11 0.015904000028967857 0.03753599897027016 0.023127999808639287 0.02527999971061945 0.0054230027890305385 0.018400000408291817 0.03001599945127964 0.020488000102341176 0.020096000283956528 0.002585688394137697 0.10355199873447418 0.1438400000333786 0.11536479964852334 0.11124800145626068 0.011136028294919255 0.16502399742603302 0.2072959989309311 0.18646399974822997 0.19075199961662292 0.013296437761247597 0.03142400085926056 0.05215999856591225 0.03888959977775812 0.03750399872660637 0.0057399302067536314 0.020128000527620316 0.04064000025391579 0.023937600292265417 0.023648000322282314 0.004144197400898248 32 4 2048 768 151936 True True False 16 1 2048 768 generic False CUDA_EVENT BF16 none
12 0.015200000256299973 0.037151999771595 0.02144480012357235 0.02195199951529503 0.005576364091933697 0.017952000722289085 0.0226879995316267 0.019934400077909233 0.019952000118792057 0.0011812499470458758 0.10063999891281128 0.13468800485134125 0.11595199964940547 0.1207519993185997 0.011974301621092394 0.16332800686359406 0.20748800039291382 0.18162400051951408 0.1767839938402176 0.01474120743720334 0.03222399950027466 0.043455999344587326 0.03829439990222454 0.03859200142323971 0.0031378131240041122 0.020959999412298203 0.03587200120091438 0.023795200139284135 0.023423999547958374 0.0031339489695198443 32 4 2048 768 151936 True True False 8 1 2048 768 generic False CUDA_EVENT BF16 none
13 0.014944000169634819 0.04022400081157684 0.02162720002233982 0.022463999688625336 0.006017219317026815 0.018464000895619392 0.026528000831604004 0.021264000236988066 0.020896000787615776 0.001934095017586082 0.10156799852848053 0.1703999936580658 0.12565439902245998 0.1244799979031086 0.01641776722785409 0.1653759926557541 0.23865599930286407 0.1969360001385212 0.19223999977111816 0.02058903050274278 0.03254399821162224 0.06752000004053116 0.0443536002188921 0.041519999504089355 0.00984829071098989 0.020096000283956528 0.040031999349594116 0.026934400014579297 0.02478400059044361 0.00562071702648593 32 4 2048 768 151936 True True False 1 1 2048 768 generic False CUDA_EVENT BF16 none
14 0.7531200051307678 0.8461440205574036 0.7618160009384155 0.7576479911804199 0.019464233617982506 0.2922559976577759 0.2985599935054779 0.2953856036067009 0.29576000571250916 0.001575901077689139 0.510047972202301 0.5140479803085327 0.5121696025133133 0.5123839974403381 0.0012263137313476844 32 4 2048 768 151936 True True False 8192 2 2048 768 generic False CUDA_EVENT BF16 none
15 0.39529600739479065 0.47279998660087585 0.40216960161924364 0.39825600385665894 0.0162749796240819 0.15625600516796112 0.16211199760437012 0.15959519892930984 0.15988799929618835 0.0011493418942396922 0.2635200023651123 0.2642880082130432 0.2639120012521744 0.26392000913619995 0.0001903593401384135 32 4 2048 768 151936 True True False 4096 2 2048 768 generic False CUDA_EVENT BF16 none
16 0.2531839907169342 0.32419198751449585 0.26924319565296173 0.26049599051475525 0.01657194262368116 0.10051199793815613 0.2375359982252121 0.12411200068891048 0.10311999917030334 0.039466165428540506 0.15881599485874176 0.19289599359035492 0.16896959990262986 0.1640480011701584 0.009162416086418127 32 4 2048 768 151936 True True False 2048 2 2048 768 generic False CUDA_EVENT BF16 none
17 0.15302400290966034 0.1780800074338913 0.16284480094909667 0.16113600134849548 0.00740355661356144 0.14521600306034088 0.20233599841594696 0.17807039842009545 0.1796799972653389 0.013908448560094403 0.0907519981265068 0.09750399738550186 0.09460479989647866 0.09478399902582169 0.0017965487667361475 32 4 2048 768 151936 True True False 1024 2 2048 768 generic False CUDA_EVENT BF16 none
18 0.11740799993276596 0.17958399653434753 0.13262080028653145 0.12878400087356567 0.014146060532423056 0.17468799650669098 0.21161599457263947 0.1910431995987892 0.18966399878263474 0.01270903647700971 0.05926400050520897 0.07577600330114365 0.06530559975653887 0.06404799968004227 0.00479458462514625 32 4 2048 768 151936 True True False 512 2 2048 768 generic False CUDA_EVENT BF16 none
19 0.11260800063610077 0.17209599912166595 0.13610880002379416 0.13809599727392197 0.017675922458993677 0.18729600310325623 0.26822400093078613 0.20815680101513861 0.2078079953789711 0.01700718593658771 0.04499199986457825 0.06537599861621857 0.0507551996037364 0.04787199944257736 0.005727123232692158 32 4 2048 768 151936 True True False 256 2 2048 768 generic False CUDA_EVENT BF16 none
20 0.11257600039243698 0.23452800512313843 0.13866880126297473 0.13964799791574478 0.026590047697062115 0.1828799992799759 0.24751999974250793 0.20735519900918006 0.20670399814844131 0.01500438131586336 0.03654399886727333 0.05593600124120712 0.0408239996060729 0.03892800025641918 0.004864409450710354 32 4 2048 768 151936 True True False 128 2 2048 768 generic False CUDA_EVENT BF16 none
21 0.11734399944543839 0.18726399540901184 0.13890240006148816 0.13308800011873245 0.019086167340006257 0.16991999745368958 0.2443840056657791 0.19185120090842248 0.19257599860429764 0.018017536159219673 0.03033600002527237 0.04956800118088722 0.038387199863791466 0.03750400058925152 0.00500897018702887 32 4 2048 768 151936 True True False 64 2 2048 768 generic False CUDA_EVENT BF16 none
22 0.11420799791812897 0.18115200102329254 0.13938880078494548 0.1393439993262291 0.018141313962922536 0.1693439930677414 0.221343994140625 0.19187839925289155 0.19438399374485016 0.01430752121272052 0.03017600066959858 0.06019200012087822 0.03866560012102127 0.0364960003644228 0.0074199790031205266 32 4 2048 768 151936 True True False 32 2 2048 768 generic False CUDA_EVENT BF16 none
23 0.11276800185441971 0.16223999857902527 0.1331360016018152 0.13305599987506866 0.01432493740801146 0.17187200486660004 0.23625600337982178 0.19287680014967917 0.1913280040025711 0.01754628831589704 0.029823999851942062 0.07574400305747986 0.03900959976017475 0.036927999928593636 0.009277465083962879 32 4 2048 768 151936 True True False 16 2 2048 768 generic False CUDA_EVENT BF16 none
24 0.1130559965968132 0.19814400374889374 0.13379519879817964 0.12531199678778648 0.020074427302248836 0.16841599345207214 0.21139200031757355 0.1893615983426571 0.19092799723148346 0.012418720676762711 0.029503999277949333 0.07558400183916092 0.040144000016152856 0.03728000074625015 0.010223239196498205 32 4 2048 768 151936 True True False 8 2 2048 768 generic False CUDA_EVENT BF16 none
25 0.115167997777462 0.2640640139579773 0.13805920109152794 0.13118399679660797 0.03132849683515479 0.17103999853134155 0.2977280020713806 0.19899839907884598 0.1976960003376007 0.028928046763067948 0.0297279991209507 0.05990400165319443 0.03834720011800528 0.0363520011305809 0.0070885330713495905 32 4 2048 768 151936 True True False 1 2 2048 768 generic False CUDA_EVENT BF16 none
26 0.3940800130367279 0.46483200788497925 0.39923040121793746 0.3957759886980057 0.015108903906233569 0.16022400557994843 0.1634880006313324 0.16203359961509706 0.16228799521923065 0.0010220720912414007 0.26073598861694336 0.2627840042114258 0.26138080209493636 0.2613760083913803 0.00041640363494172775 32 4 2048 768 151936 True True False 8192 4 2048 768 generic False CUDA_EVENT BF16 none
27 0.2536959946155548 0.27161601185798645 0.25960480123758317 0.2577280104160309 0.005142017404245015 0.09849599748849869 0.10281600058078766 0.10057279989123344 0.10063999891281128 0.0008836232237268523 0.13913600146770477 0.17606399953365326 0.15875840038061143 0.15988799929618835 0.010058952112389172 32 4 2048 768 151936 True True False 4096 4 2048 768 generic False CUDA_EVENT BF16 none
28 0.15142400562763214 0.1780479997396469 0.16139679849147798 0.1602879986166954 0.0070656055731385115 0.14601600170135498 0.23343999683856964 0.17913119941949845 0.179967999458313 0.023065274309176566 0.09388799965381622 0.12108799815177917 0.10317599996924401 0.10073599964380264 0.007364345618470178 32 4 2048 768 151936 True True False 2048 4 2048 768 generic False CUDA_EVENT BF16 none
29 0.12198399752378464 0.18406400084495544 0.13770400024950505 0.13232000172138214 0.01623164885687898 0.1773120015859604 0.20688000321388245 0.1870912007987499 0.18535999953746796 0.00825628058448234 0.05766399949789047 0.06739199906587601 0.06187200043350458 0.061216000467538834 0.003004312467037863 32 4 2048 768 151936 True True False 1024 4 2048 768 generic False CUDA_EVENT BF16 none
30 0.11737599968910217 0.18614399433135986 0.13109439946711063 0.12771200388669968 0.01372168725934724 0.17587199807167053 0.2699519991874695 0.19926720038056372 0.19075199961662292 0.02327478982490538 0.041728001087903976 0.06224000081419945 0.05000480003654957 0.049375999718904495 0.006185850944273961 32 4 2048 768 151936 True True False 512 4 2048 768 generic False CUDA_EVENT BF16 none
31 0.1159679964184761 0.1610880047082901 0.12988320142030715 0.12494400516152382 0.011865375783184065 0.1767680048942566 0.27379199862480164 0.21035519987344742 0.21275199949741364 0.023206964017550125 0.037087999284267426 0.062144000083208084 0.04439679980278015 0.04283200018107891 0.006155226392511492 32 4 2048 768 151936 True True False 256 4 2048 768 generic False CUDA_EVENT BF16 none
32 0.12015999853610992 0.158720001578331 0.13573280088603495 0.1343199983239174 0.010666830836014414 0.1737920045852661 0.22127999365329742 0.20261440128087999 0.20321600139141083 0.011153965849067301 0.03587200120091438 0.04944000020623207 0.038265600241720675 0.037328001111745834 0.0030431580113575636 32 4 2048 768 151936 True True False 128 4 2048 768 generic False CUDA_EVENT BF16 none
33 0.1212799996137619 0.15139199793338776 0.13183839991688728 0.13014400005340576 0.00851207547022104 0.17257599532604218 0.2250880002975464 0.1872655987739563 0.18193599581718445 0.013638284975248818 0.02969600073993206 0.0525440014898777 0.03840640028938651 0.03444799967110157 0.007598511754254174 32 4 2048 768 151936 True True False 64 4 2048 768 generic False CUDA_EVENT BF16 none
34 0.11270400136709213 0.1624639928340912 0.13508000001311302 0.13180799782276154 0.014243564451606775 0.17052799463272095 0.23715199530124664 0.1953311987221241 0.1966560035943985 0.01500670718978398 0.030239999294281006 0.05270399898290634 0.03807039987295866 0.03742399998009205 0.004906165602021684 32 4 2048 768 151936 True True False 32 4 2048 768 generic False CUDA_EVENT BF16 none
35 0.12319999933242798 0.17919999361038208 0.14059039913117885 0.14156799763441086 0.016217662982107223 0.1701119989156723 0.21987199783325195 0.18903039917349815 0.1870879977941513 0.014117854507841239 0.030368000268936157 0.06406400352716446 0.04045119984075427 0.03710399940609932 0.00878942205983628 32 4 2048 768 151936 True True False 16 4 2048 768 generic False CUDA_EVENT BF16 none
36 0.11296000331640244 0.16761599481105804 0.13843199908733367 0.13814399391412735 0.014925286935582404 0.1711679995059967 0.21561600267887115 0.1906527981162071 0.1876479983329773 0.011803992622137974 0.0306560005992651 0.09216000139713287 0.041129599791020155 0.036847999319434166 0.013739721468154687 32 4 2048 768 151936 True True False 8 4 2048 768 generic False CUDA_EVENT BF16 none
37 0.11929599940776825 0.2008959949016571 0.14535359852015972 0.13971199840307236 0.024769473061576282 0.16803200542926788 0.25123199820518494 0.19573760256171227 0.19366399943828583 0.023196011081705884 0.03049599938094616 0.08089599758386612 0.04739360017701984 0.0453919991850853 0.01286389846889463 32 4 2048 768 151936 True True False 1 4 2048 768 generic False CUDA_EVENT BF16 none

129
runs/frontier-fidelity-envelope-v1/profiles/profile-v4-trace-final/manifest.json Normal file
View File

@@ -0,0 +1,129 @@
{
"attention_long_context_coverage": {
"by_tp": {
"1": {
"decode_kv_lengths": [
128,
1024,
2048,
4096,
8192,
16384,
32768,
40960
],
"true_mixed_decode_avg_kv_lengths": [
512,
1024,
2048,
4096,
8192,
16384,
32768
]
},
"2": {
"decode_kv_lengths": [
128,
1024,
2048,
4096,
8192,
16384,
32768,
40960
],
"true_mixed_decode_avg_kv_lengths": [
512,
1024,
2048,
4096,
8192,
16384,
32768
]
},
"4": {
"decode_kv_lengths": [
128,
1024,
2048,
4096,
8192,
16384,
32768,
40960
],
"true_mixed_decode_avg_kv_lengths": [
512,
1024,
2048,
4096,
8192,
16384,
32768
]
}
},
"included": true
},
"attention_tp_coverage": [
1,
2,
4
],
"environment_contract": {
"dtype": "bfloat16",
"frontier_commit": "d9cfeb6d8791fbf2f295dd9744c56a666171776e",
"hardware": "NVIDIA H20",
"model": "Qwen3-30B-A3B",
"tensor_parallel_sizes": [
1,
2,
4
],
"vllm_source_commit": "88d34c6409e9fb3c7b8ca0c04756f061d2099eb1",
"vllm_version": "0.20.0"
},
"inputs": {
"/home/gahow/phd/aituner/runs/frontier-fidelity-envelope-v1/fleet-artifacts/qwen30-attention-composition-tp1-20260717-v1-20260717T014550833983Z/artifacts/artifacts/attention-composition-tp1-v1/raw/flashattn-composition-tp1.json": "ce461be92a5059e2d168b2cbf21b55d9a55ee71c3b0586a3e63cd8288938168d",
"/home/gahow/phd/aituner/runs/frontier-fidelity-envelope-v1/fleet-artifacts/qwen30-attention-composition-tp2-20260717-v1-20260717T014551831545Z/artifacts/artifacts/attention-composition-tp2-v1/raw/flashattn-composition-tp2.json": "997d67fb3a28b488c3aa4945ae7c6d4ecf8a08726ee3cb4652d16c221ff395ad",
"/home/gahow/phd/aituner/runs/frontier-fidelity-envelope-v1/fleet-artifacts/qwen30-attention-composition-tp4-20260717-v1-20260717T014552897206Z/artifacts/artifacts/attention-composition-tp4-v1/raw/flashattn-composition-tp4.json": "046cffd4c11d8a36513167ec25d3e1ca016324715679ea688a58d4a71dfe62b2",
"/home/gahow/phd/aituner/runs/frontier-fidelity-envelope-v1/fleet-artifacts/qwen30-attention-long-context-tp1-20260717-v1-20260717T022700381447Z/artifacts/artifacts/attention-long-context-tp1-v1/raw/flashattn-long-context-tp1.json": "da3b5adaa0d06f6a0ab7394d3c85d8d0524f0bd571613785f1a6183f70eb457f",
"/home/gahow/phd/aituner/runs/frontier-fidelity-envelope-v1/fleet-artifacts/qwen30-attention-long-context-tp2-20260717-v1-20260717T022701357659Z/artifacts/artifacts/attention-long-context-tp2-v1/raw/flashattn-long-context-tp2.json": "d503acb959f91b1713ff02d3251c781c240c4b057208033f4fd4a637af76e2cb",
"/home/gahow/phd/aituner/runs/frontier-fidelity-envelope-v1/fleet-artifacts/qwen30-attention-long-context-tp4-20260717-v1-20260717T022702416725Z/artifacts/artifacts/attention-long-context-tp4-v1/raw/flashattn-long-context-tp4.json": "a24acd44ab4075a3ac0aad00b91e676b325b2427dbe14282260cae6769481cf7",
"/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-allreduce-full-tp2-20260716-v1-dispatch-aware-20260716T140743025781Z/artifacts/artifacts/allreduce-full-tp2-v1/raw/allreduce-tp2.json": "97c3c76b5a04e95bd9192423c2b891667c668f39cc0dfecbd097d749939f2d0a",
"/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-allreduce-full-tp4-20260716-v1-dispatch-aware-20260716T141106009788Z/artifacts/artifacts/allreduce-full-tp4-v1/raw/allreduce-tp4.json": "809df9baa6f468cf12bf0c99827475acc67894dd9f3f948976590b665fac0e76",
"/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-flashattn-kv-full-tp1-20260716-v2-20260716T135132587012Z/artifacts/artifacts/flashattn-kv-full-v2-tp1/raw/flashattn-tp1.json": "dcb4c1bf7e76b9c765f78ddd2b8a734f2d7ba2adac13ce017689a8a77fe69a27",
"/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-flashattn-kv-full-tp2-20260716-v2-20260716T135134194295Z/artifacts/artifacts/flashattn-kv-full-v2-tp2/raw/flashattn-tp2.json": "43ce042556ba887c8860614b43ccf0f564e5cebc1a0cffbce299d0acb9fa8d07",
"/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-flashattn-kv-full-tp4-20260716-v2-20260716T135135197200Z/artifacts/artifacts/flashattn-kv-full-v2-tp4/raw/flashattn-tp4.json": "84eef31bcad0f556907a093318a420959d14fdc94474823d11f659704bdfec73",
"/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-frontier-linear-full-20260716-v2-max-tokens-20260716T144444676943Z/artifacts/artifacts/frontier-linear-full-v2/profiles/compute/h20/qwen3-a3b-30b-moe/linear_op.csv": "67666cb0a4901b74599d468df2e31bcaa2a11a7842cc0cefba24ffce62508e0c",
"/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-moe-full-20260716-v1-local-shard-20260716T141334565164Z/artifacts/artifacts/moe-full-v1/raw/moe-full.json": "588f6ad0d69c9636d1b852e3df0a12d13cfe731f050ea7ec7aea457cceefbde8",
"/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-router-full-20260716-v3-tp-context-20260716T145446098505Z/artifacts/artifacts/router-full-v3/raw/router.json": "1962972e983bff3e06a721ef4ae4ec65728ff669681497a4a7e7f769b88b4931"
},
"outputs": {
"allreduce.json": "b38d14f990578d668523d25b107aceed433da5020d8ada3b6e44d3562261a3b3",
"attention.csv": "4f0c42d2c47f52880d81b8d2b2d7d52bd62d2c34e0f3a7f94c4c6cbd64d751c8",
"attention_true_mixed_fused.csv": "c84c129d466aec97b42f1255f3b7363fe7f73948dc27ed65b6ce599d6fb9af6d",
"linear_op.csv": "67666cb0a4901b74599d468df2e31bcaa2a11a7842cc0cefba24ffce62508e0c",
"moe.csv": "dd698113975e5ed77f51dfd8a69aebf764ed651c3f6f34b71914d25d6e001415"
},
"profile_id": "qwen3-30b-a3b-bf16-vllm020-h20-tp1-2-4-fused-mixed-total-conserving-pure-prefill-batch-composition-long-context-decode-mixed",
"projection_contract": {
"allreduce": "Frozen exact runtime measurements; base profile-only comparison keeps the historical Frontier CC backend fixed to isolate compute profile fidelity",
"attention": "Pure prefill/extend/decode FA3 core plus separately measured KV update; input/output reshape assumed zero; exported mean is used as median target; true mixed rows use a total-conserving compatibility projection",
"attention_long_context": "Direct FA3 measurements at decode KV lengths 16K/32K/40K, plus true-mixed prefill/decode measurements at 16K/32K, for TP1/2/4",
"attention_pure_prefill_batch_composition": "Direct FA3 measurements for 2/4 requests at query length 2048 and 2/4/8/16 requests at query length 512 for each TP; included only when batch-composition attention inputs are supplied",
"attention_true_mixed": "The directly measured fused total is preserved in diagnostics. Frontier's two targets are projected by the same-TP pure prefill/decode reference ratio, with projected prefill + decode exactly equal to the fused total; the split is a schema compatibility attribution, not an observation",
"linear": "Frontier profiler using vLLM 0.20 CUDA operators",
"moe": "Replicated gate and fused top-k plus TP-local modular expert kernel; expert measurement already includes prepare/finalize so shuffling is zero"
},
"row_counts": {
"allreduce": 24,
"attention_frontier_compatible": 213,
"attention_true_mixed_fused_diagnostic": 48,
"linear": 36,
"moe": 72
},
"schema_version": "frontier_qwen30_vllm020_frozen_profile.v4"
}

73
runs/frontier-fidelity-envelope-v1/profiles/profile-v4-trace-final/moe.csv Normal file
View File

@@ -0,0 +1,73 @@
time_stats.moe_gating_linear.min,time_stats.moe_gating_linear.max,time_stats.moe_gating_linear.mean,time_stats.moe_gating_linear.median,time_stats.moe_gating_linear.std,time_stats.moe_gating_routing_topk.min,time_stats.moe_gating_routing_topk.max,time_stats.moe_gating_routing_topk.mean,time_stats.moe_gating_routing_topk.median,time_stats.moe_gating_routing_topk.std,time_stats.moe_shuffling.min,time_stats.moe_shuffling.max,time_stats.moe_shuffling.mean,time_stats.moe_shuffling.median,time_stats.moe_shuffling.std,time_stats.moe_grouped_gemm.min,time_stats.moe_grouped_gemm.max,time_stats.moe_grouped_gemm.mean,time_stats.moe_grouped_gemm.median,time_stats.moe_grouped_gemm.std,num_tokens,num_experts,num_experts_per_device,expert_parallel_size,routing_runtime_path,routing_assignment_policy,routing_weight_policy,routing_uses_router_logits,gating_runtime_context,gating_runtime_context_impl,router_topk,hidden_dim,expert_hidden_dim,use_gated,num_tensor_parallel_workers,total_routed_tokens,model_expansion_ratio,tokens_per_expert_avg,tokens_to_experts_ratio,expert_utilization,min_load_ratio,load_imbalance_cv,max_load_ratio,load_entropy,load_gini_coefficient,load_distribution,seed,moe_grouped_gemm_backend,measurement_type,profiling_precision,model_arch,quant_signature,router_median_nonadditivity_ratio,projection_policy
0.02502400055527687,0.06092799827456474,0.033839999698102474,0.028672000393271446,0.010315255343709818,0.019360000267624855,0.0352960005402565,0.023424000293016434,0.022064000368118286,0.0038898102289194572,0.0,0.0,0.0,0.0,0.0,0.33926400542259216,0.405023992061615,0.36780479848384856,0.36507199704647064,0.01690507644474779,1,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,8,0.375,0.0625,0.0625,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0233364439829928,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.025280000641942024,0.05132799968123436,0.030641599837690593,0.027343999594449997,0.007562409232763304,0.020160000771284103,0.052960000932216644,0.024145600199699403,0.021424000151455402,0.007450198645599985,0.0,0.0,0.0,0.0,0.0,1.1943039894104004,1.286784052848816,1.228384006023407,1.2273280024528503,0.02832547242381263,8,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,64,0.375,0.5,0.5,0.3984375,0.0,1.346291201783626,4.0,5.59375,0.661865234375,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9806430689981738,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.024831999093294144,0.046560000628232956,0.030371200107038022,0.02763199992477894,0.005878487205028602,0.020320000126957893,0.04560000076889992,0.02601920012384653,0.02270400058478117,0.00751129965906799,0.0,0.0,0.0,0.0,0.0,1.679744005203247,1.766144037246704,1.7095808148384095,1.7015680074691772,0.02438921262998535,16,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,128,0.375,1.0,1.0,0.625,0.0,1.015504800579495,5.0,6.15516433212955,0.529052734375,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9103623678483975,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.024288000538945198,0.049375999718904495,0.03086080001667142,0.0267359996214509,0.0070864531384997225,0.020479999482631683,0.030912000685930252,0.02274719988927245,0.021359999664127827,0.0029966813650672505,0.0,0.0,0.0,0.0,0.0,2.1576640605926514,2.2921600341796875,2.2097824096679686,2.188944101333618,0.045572321842012986,32,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,256,0.375,2.0,2.0,0.875,0.0,0.6343057228182637,2.5,6.64370748444639,0.35369873046875,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9610778571819444,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.02393599972128868,0.04342399910092354,0.029380799923092126,0.026688000187277794,0.0057374782000526574,0.020031999796628952,0.036607999354600906,0.022487999964505435,0.020911999978125095,0.0038135203062103235,0.0,0.0,0.0,0.0,0.0,2.422368049621582,2.5130879878997803,2.4516672134399413,2.434159994125366,0.03278900287381846,64,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,512,0.375,4.0,4.0,0.984375,0.0,0.4921254921257382,2.25,6.817190042344769,0.272369384765625,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9952941013961014,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.025407999753952026,0.05510399863123894,0.031430399790406224,0.02798399981111288,0.007335050273982725,0.020479999482631683,0.03561599925160408,0.02275839988142252,0.021551999263465405,0.003545718365165811,0.0,0.0,0.0,0.0,0.0,2.2217600345611572,2.289599895477295,2.2571327924728393,2.263375997543335,0.021660416089449488,128,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,1024,0.375,8.0,8.0,1.0,0.125,0.3486861500690843,1.875,6.908192310183997,0.197662353515625,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9273256282883522,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.0244159996509552,0.05395200103521347,0.03188959984108806,0.026031999848783016,0.00943995927387483,0.02051199972629547,0.036607999354600906,0.023247999791055917,0.02147199958562851,0.003910623837069648,0.0,0.0,0.0,0.0,0.0,2.18668794631958,2.318079948425293,2.2218016147613526,2.211087942123413,0.035380897213135316,256,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,2048,0.375,16.0,16.0,1.0,0.4375,0.2525504668006971,1.875,6.953347743053017,0.1410369873046875,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0380599882396606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.024512000381946564,0.04620800167322159,0.02884640023112297,0.026320000179111958,0.005516557583355925,0.02054399996995926,0.03846399858593941,0.023401600029319524,0.021263999864459038,0.0044742944198265374,0.0,0.0,0.0,0.0,0.0,2.2291839122772217,2.3929600715637207,2.2908096313476562,2.2804640531539917,0.04479348924786221,512,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,4096,0.375,32.0,32.0,1.0,0.65625,0.15765965680164504,1.5625,6.98229848728205,0.08779525756835938,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9569603278386984,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.02412799932062626,0.06940799951553345,0.030817600432783365,0.026800000108778477,0.010047085187652939,0.020128000527620316,0.044096000492572784,0.024606400076299904,0.022304000332951546,0.00622857166823714,0.0,0.0,0.0,0.0,0.0,2.0678720474243164,2.1297600269317627,2.0837119817733765,2.0779199600219727,0.017880044357986735,1024,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,8192,0.375,64.0,64.0,1.0,0.625,0.12169081635504074,1.3125,6.9892029662356325,0.06879425048828125,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9524274993623046,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.02831999957561493,0.043487999588251114,0.031744000129401685,0.02991999965161085,0.003973415836428909,0.02070399932563305,0.029343999922275543,0.022886400017887353,0.021743999794125557,0.0026873065045088873,0.0,0.0,0.0,0.0,0.0,2.916032075881958,3.0819520950317383,2.9805248022079467,2.9656319618225098,0.05482195799019572,2048,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,16384,0.375,128.0,128.0,1.0,0.796875,0.07935434147688751,1.1796875,6.9954297964750305,0.044734954833984375,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8971818172100244,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.03830400109291077,0.06268800050020218,0.043337599746882914,0.040511999279260635,0.005823016247946116,0.023135999217629433,0.03747199848294258,0.025206399988383053,0.02393599972128868,0.003271381256231161,0.0,0.0,0.0,0.0,0.0,4.421599864959717,4.535359859466553,4.486294317245483,4.497056007385254,0.036990243787549344,4096,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,32768,0.375,256.0,256.0,1.0,0.8203125,0.060849326483103046,1.17578125,6.9973188375859685,0.033740997314453125,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8113207890716184,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.05660799890756607,0.07932800054550171,0.06249920018017292,0.06039999984204769,0.005636461691480845,0.02956799976527691,0.03747199848294258,0.031126399897038935,0.030287999659776688,0.002157571006631541,0.0,0.0,0.0,0.0,0.0,7.302591800689697,7.402751922607422,7.354758310317993,7.3464319705963135,0.032142662400335566,8192,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,65536,0.375,512.0,512.0,1.0,0.890625,0.0412323087266341,1.08984375,6.998772433185578,0.02334284782409668,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.883909666885606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.02502400055527687,0.06092799827456474,0.033839999698102474,0.028672000393271446,0.010315255343709818,0.019360000267624855,0.0352960005402565,0.023424000293016434,0.022064000368118286,0.0038898102289194572,0.0,0.0,0.0,0.0,0.0,0.35280001163482666,0.39692801237106323,0.37662720382213594,0.37196800112724304,0.013401318050665304,1,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,8,0.375,0.0625,0.0625,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0233364439829928,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.025280000641942024,0.05132799968123436,0.030641599837690593,0.027343999594449997,0.007562409232763304,0.020160000771284103,0.052960000932216644,0.024145600199699403,0.021424000151455402,0.007450198645599985,0.0,0.0,0.0,0.0,0.0,0.4692479968070984,0.5523840188980103,0.5134752035140991,0.5100640058517456,0.02291433464135784,8,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,64,0.375,0.5,0.5,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9806430689981738,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.024831999093294144,0.046560000628232956,0.030371200107038022,0.02763199992477894,0.005878487205028602,0.020320000126957893,0.04560000076889992,0.02601920012384653,0.02270400058478117,0.00751129965906799,0.0,0.0,0.0,0.0,0.0,0.34652799367904663,0.4119040071964264,0.3789471983909607,0.38550400733947754,0.02073945105803335,16,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,128,0.375,1.0,1.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9103623678483975,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.024288000538945198,0.049375999718904495,0.03086080001667142,0.0267359996214509,0.0070864531384997225,0.020479999482631683,0.030912000685930252,0.02274719988927245,0.021359999664127827,0.0029966813650672505,0.0,0.0,0.0,0.0,0.0,0.31462401151657104,0.7456960082054138,0.38617280423641204,0.34545600414276123,0.12230201266253077,32,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,256,0.375,2.0,2.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9610778571819444,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.02393599972128868,0.04342399910092354,0.029380799923092126,0.026688000187277794,0.0057374782000526574,0.020031999796628952,0.036607999354600906,0.022487999964505435,0.020911999978125095,0.0038135203062103235,0.0,0.0,0.0,0.0,0.0,0.32521599531173706,0.419871985912323,0.36325119733810424,0.34968000650405884,0.03161798848223672,64,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,512,0.375,4.0,4.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9952941013961014,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.025407999753952026,0.05510399863123894,0.031430399790406224,0.02798399981111288,0.007335050273982725,0.020479999482631683,0.03561599925160408,0.02275839988142252,0.021551999263465405,0.003545718365165811,0.0,0.0,0.0,0.0,0.0,0.289792001247406,0.4663360118865967,0.4091839998960495,0.41655999422073364,0.0446001986506615,128,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,1024,0.375,8.0,8.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9273256282883522,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.0244159996509552,0.05395200103521347,0.03188959984108806,0.026031999848783016,0.00943995927387483,0.02051199972629547,0.036607999354600906,0.023247999791055917,0.02147199958562851,0.003910623837069648,0.0,0.0,0.0,0.0,0.0,0.3761279881000519,0.4416320025920868,0.40686399936676027,0.40540799498558044,0.02257778769899645,256,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,2048,0.375,16.0,16.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0380599882396606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.024512000381946564,0.04620800167322159,0.02884640023112297,0.026320000179111958,0.005516557583355925,0.02054399996995926,0.03846399858593941,0.023401600029319524,0.021263999864459038,0.0044742944198265374,0.0,0.0,0.0,0.0,0.0,0.7172480225563049,0.8663039803504944,0.7723807990550995,0.7591840028762817,0.04164772379451242,512,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,4096,0.375,32.0,32.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9569603278386984,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.02412799932062626,0.06940799951553345,0.030817600432783365,0.026800000108778477,0.010047085187652939,0.020128000527620316,0.044096000492572784,0.024606400076299904,0.022304000332951546,0.00622857166823714,0.0,0.0,0.0,0.0,0.0,1.0195519924163818,1.2216639518737793,1.1253888130187988,1.1453600525856018,0.06548005322243594,1024,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,8192,0.375,64.0,64.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9524274993623046,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.02831999957561493,0.043487999588251114,0.031744000129401685,0.02991999965161085,0.003973415836428909,0.02070399932563305,0.029343999922275543,0.022886400017887353,0.021743999794125557,0.0026873065045088873,0.0,0.0,0.0,0.0,0.0,1.7490559816360474,1.9644800424575806,1.8529024004936219,1.814303994178772,0.08042565617327288,2048,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,16384,0.375,128.0,128.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8971818172100244,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.03830400109291077,0.06268800050020218,0.043337599746882914,0.040511999279260635,0.005823016247946116,0.023135999217629433,0.03747199848294258,0.025206399988383053,0.02393599972128868,0.003271381256231161,0.0,0.0,0.0,0.0,0.0,3.2479360103607178,3.385279893875122,3.296070408821106,3.2800960540771484,0.04525529026442046,4096,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,32768,0.375,256.0,256.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8113207890716184,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.05660799890756607,0.07932800054550171,0.06249920018017292,0.06039999984204769,0.005636461691480845,0.02956799976527691,0.03747199848294258,0.031126399897038935,0.030287999659776688,0.002157571006631541,0.0,0.0,0.0,0.0,0.0,6.344799995422363,6.517856121063232,6.464438438415527,6.478623867034912,0.05116674443145098,8192,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,65536,0.375,512.0,512.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.883909666885606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.02502400055527687,0.06092799827456474,0.033839999698102474,0.028672000393271446,0.010315255343709818,0.019360000267624855,0.0352960005402565,0.023424000293016434,0.022064000368118286,0.0038898102289194572,0.0,0.0,0.0,0.0,0.0,0.2648000121116638,0.325439989566803,0.28852800130844114,0.28390398621559143,0.01933778377635077,1,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,8,0.375,0.0625,0.0625,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0233364439829928,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.025280000641942024,0.05132799968123436,0.030641599837690593,0.027343999594449997,0.007562409232763304,0.020160000771284103,0.052960000932216644,0.024145600199699403,0.021424000151455402,0.007450198645599985,0.0,0.0,0.0,0.0,0.0,0.7347840070724487,0.862496018409729,0.7769344031810761,0.769216001033783,0.03290485328796285,8,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,64,0.375,0.5,0.5,0.421875,0.0,1.346291201783626,6.0,5.652114648336087,0.636962890625,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9806430689981738,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.024831999093294144,0.046560000628232956,0.030371200107038022,0.02763199992477894,0.005878487205028602,0.020320000126957893,0.04560000076889992,0.02601920012384653,0.02270400058478117,0.00751129965906799,0.0,0.0,0.0,0.0,0.0,0.9198399782180786,0.9646080136299133,0.9412063956260681,0.9411839842796326,0.014939365085478117,16,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,128,0.375,1.0,1.0,0.5703125,0.0,1.118033988749895,5.0,6.008641773518898,0.580810546875,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9103623678483975,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.024288000538945198,0.049375999718904495,0.03086080001667142,0.0267359996214509,0.0070864531384997225,0.020479999482631683,0.030912000685930252,0.02274719988927245,0.021359999664127827,0.0029966813650672505,0.0,0.0,0.0,0.0,0.0,1.2796800136566162,1.3484159708023071,1.3006976008415223,1.2929120063781738,0.020807998177176254,32,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,256,0.375,2.0,2.0,0.8828125,0.0,0.6959705453537527,3.0,6.60872850615583,0.38055419921875,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9610778571819444,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.02393599972128868,0.04342399910092354,0.029380799923092126,0.026688000187277794,0.0057374782000526574,0.020031999796628952,0.036607999354600906,0.022487999964505435,0.020911999978125095,0.0038135203062103235,0.0,0.0,0.0,0.0,0.0,1.3630399703979492,1.4430400133132935,1.3909215927124023,1.3892319798469543,0.022335744492366926,64,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,512,0.375,4.0,4.0,0.984375,0.0,0.5201036555341637,3.0,6.798826509158851,0.28302001953125,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9952941013961014,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.025407999753952026,0.05510399863123894,0.031430399790406224,0.02798399981111288,0.007335050273982725,0.020479999482631683,0.03561599925160408,0.02275839988142252,0.021551999263465405,0.003545718365165811,0.0,0.0,0.0,0.0,0.0,1.27948796749115,1.3904000520706177,1.3176063895225525,1.309440016746521,0.038060887827312775,128,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,1024,0.375,8.0,8.0,1.0,0.25,0.3511282039725661,1.875,6.91002266305238,0.1970977783203125,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9273256282883522,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.0244159996509552,0.05395200103521347,0.03188959984108806,0.026031999848783016,0.00943995927387483,0.02051199972629547,0.036607999354600906,0.023247999791055917,0.02147199958562851,0.003910623837069648,0.0,0.0,0.0,0.0,0.0,1.264415979385376,1.3145920038223267,1.2791999936103822,1.2753440141677856,0.014130605249568332,256,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,2048,0.375,16.0,16.0,1.0,0.375,0.24692938483248605,1.6875,6.955481130775285,0.13909912109375,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0380599882396606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.024512000381946564,0.04620800167322159,0.02884640023112297,0.026320000179111958,0.005516557583355925,0.02054399996995926,0.03846399858593941,0.023401600029319524,0.021263999864459038,0.0044742944198265374,0.0,0.0,0.0,0.0,0.0,1.3081920146942139,1.347648024559021,1.3292255997657776,1.329967975616455,0.014558863679016933,512,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,4096,0.375,32.0,32.0,1.0,0.625,0.17143053326165383,1.5625,6.9786675275754035,0.09520339965820312,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9569603278386984,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.02412799932062626,0.06940799951553345,0.030817600432783365,0.026800000108778477,0.010047085187652939,0.020128000527620316,0.044096000492572784,0.024606400076299904,0.022304000332951546,0.00622857166823714,0.0,0.0,0.0,0.0,0.0,1.242751955986023,1.3112000226974487,1.2747935891151427,1.266207993030548,0.021093073517695057,1024,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,8192,0.375,64.0,64.0,1.0,0.78125,0.11000099875256815,1.296875,6.991308871213679,0.062183380126953125,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9524274993623046,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.02831999957561493,0.043487999588251114,0.031744000129401685,0.02991999965161085,0.003973415836428909,0.02070399932563305,0.029343999922275543,0.022886400017887353,0.021743999794125557,0.0026873065045088873,0.0,0.0,0.0,0.0,0.0,1.7388160228729248,1.8077759742736816,1.772764801979065,1.772704005241394,0.021056644077284283,2048,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,16384,0.375,128.0,128.0,1.0,0.78125,0.0864630150197678,1.1796875,6.994552526394139,0.048796653747558594,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8971818172100244,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.03830400109291077,0.06268800050020218,0.043337599746882914,0.040511999279260635,0.005823016247946116,0.023135999217629433,0.03747199848294258,0.025206399988383053,0.02393599972128868,0.003271381256231161,0.0,0.0,0.0,0.0,0.0,2.6563520431518555,2.7063679695129395,2.6785055875778196,2.6791679859161377,0.01639963463052944,4096,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,32768,0.375,256.0,256.0,1.0,0.8671875,0.06127686514721937,1.16015625,6.997291583027146,0.03497934341430664,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8113207890716184,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.05660799890756607,0.07932800054550171,0.06249920018017292,0.06039999984204769,0.005636461691480845,0.02956799976527691,0.03747199848294258,0.031126399897038935,0.030287999659776688,0.002157571006631541,0.0,0.0,0.0,0.0,0.0,4.386879920959473,4.452256202697754,4.4108480453491214,4.406303882598877,0.019768161791937636,8192,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,65536,0.375,512.0,512.0,1.0,0.884765625,0.041723768525324195,1.1171875,6.998746434318934,0.02298593521118164,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.883909666885606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.02502400055527687,0.06092799827456474,0.033839999698102474,0.028672000393271446,0.010315255343709818,0.019360000267624855,0.0352960005402565,0.023424000293016434,0.022064000368118286,0.0038898102289194572,0.0,0.0,0.0,0.0,0.0,0.24208000302314758,0.4028480052947998,0.3041536003351212,0.277103990316391,0.05660721484881584,1,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,8,0.375,0.0625,0.0625,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0233364439829928,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.025280000641942024,0.05132799968123436,0.030641599837690593,0.027343999594449997,0.007562409232763304,0.020160000771284103,0.052960000932216644,0.024145600199699403,0.021424000151455402,0.007450198645599985,0.0,0.0,0.0,0.0,0.0,0.2447039932012558,0.30502399802207947,0.26446720361709597,0.26265600323677063,0.016744548364435372,8,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,64,0.375,0.5,0.5,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9806430689981738,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.024831999093294144,0.046560000628232956,0.030371200107038022,0.02763199992477894,0.005878487205028602,0.020320000126957893,0.04560000076889992,0.02601920012384653,0.02270400058478117,0.00751129965906799,0.0,0.0,0.0,0.0,0.0,0.2337920069694519,0.2881599962711334,0.26074880361557007,0.264384001493454,0.016469850143940968,16,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,128,0.375,1.0,1.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9103623678483975,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.024288000538945198,0.049375999718904495,0.03086080001667142,0.0267359996214509,0.0070864531384997225,0.020479999482631683,0.030912000685930252,0.02274719988927245,0.021359999664127827,0.0029966813650672505,0.0,0.0,0.0,0.0,0.0,0.23369599878787994,0.28591999411582947,0.25465920120477675,0.25385600328445435,0.01593204652619194,32,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,256,0.375,2.0,2.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9610778571819444,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.02393599972128868,0.04342399910092354,0.029380799923092126,0.026688000187277794,0.0057374782000526574,0.020031999796628952,0.036607999354600906,0.022487999964505435,0.020911999978125095,0.0038135203062103235,0.0,0.0,0.0,0.0,0.0,0.2295999974012375,0.26556798815727234,0.24674240052700042,0.2497600018978119,0.010345732066199003,64,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,512,0.375,4.0,4.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9952941013961014,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.025407999753952026,0.05510399863123894,0.031430399790406224,0.02798399981111288,0.007335050273982725,0.020479999482631683,0.03561599925160408,0.02275839988142252,0.021551999263465405,0.003545718365165811,0.0,0.0,0.0,0.0,0.0,0.21721599996089935,0.29020801186561584,0.2394208014011383,0.2346400022506714,0.018747330357768585,128,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,1024,0.375,8.0,8.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9273256282883522,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.0244159996509552,0.05395200103521347,0.03188959984108806,0.026031999848783016,0.00943995927387483,0.02051199972629547,0.036607999354600906,0.023247999791055917,0.02147199958562851,0.003910623837069648,0.0,0.0,0.0,0.0,0.0,0.2717759907245636,0.305184006690979,0.28813759982585907,0.28809599578380585,0.01183422600545854,256,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,2048,0.375,16.0,16.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0380599882396606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.024512000381946564,0.04620800167322159,0.02884640023112297,0.026320000179111958,0.005516557583355925,0.02054399996995926,0.03846399858593941,0.023401600029319524,0.021263999864459038,0.0044742944198265374,0.0,0.0,0.0,0.0,0.0,0.3917759954929352,0.43772798776626587,0.41130879521369934,0.4131519943475723,0.012992473640805227,512,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,4096,0.375,32.0,32.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9569603278386984,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.02412799932062626,0.06940799951553345,0.030817600432783365,0.026800000108778477,0.010047085187652939,0.020128000527620316,0.044096000492572784,0.024606400076299904,0.022304000332951546,0.00622857166823714,0.0,0.0,0.0,0.0,0.0,0.6176639795303345,0.7009919881820679,0.642767995595932,0.6330719888210297,0.024074084919938756,1024,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,8192,0.375,64.0,64.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9524274993623046,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.02831999957561493,0.043487999588251114,0.031744000129401685,0.02991999965161085,0.003973415836428909,0.02070399932563305,0.029343999922275543,0.022886400017887353,0.021743999794125557,0.0026873065045088873,0.0,0.0,0.0,0.0,0.0,1.0820800065994263,1.1674879789352417,1.1034304022789,1.0977439880371094,0.0233067292981566,2048,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,16384,0.375,128.0,128.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8971818172100244,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.03830400109291077,0.06268800050020218,0.043337599746882914,0.040511999279260635,0.005823016247946116,0.023135999217629433,0.03747199848294258,0.025206399988383053,0.02393599972128868,0.003271381256231161,0.0,0.0,0.0,0.0,0.0,1.9809919595718384,2.0415360927581787,2.003715181350708,1.992751955986023,0.022066076434645737,4096,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,32768,0.375,256.0,256.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8113207890716184,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.05660799890756607,0.07932800054550171,0.06249920018017292,0.06039999984204769,0.005636461691480845,0.02956799976527691,0.03747199848294258,0.031126399897038935,0.030287999659776688,0.002157571006631541,0.0,0.0,0.0,0.0,0.0,3.790112018585205,3.8651199340820312,3.829139161109924,3.8230879306793213,0.025177464160110564,8192,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,65536,0.375,512.0,512.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.883909666885606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.02502400055527687,0.06092799827456474,0.033839999698102474,0.028672000393271446,0.010315255343709818,0.019360000267624855,0.0352960005402565,0.023424000293016434,0.022064000368118286,0.0038898102289194572,0.0,0.0,0.0,0.0,0.0,0.212351992726326,0.24383999407291412,0.22760000079870224,0.22723200172185898,0.01050568575837594,1,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,8,0.375,0.0625,0.0625,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0233364439829928,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.025280000641942024,0.05132799968123436,0.030641599837690593,0.027343999594449997,0.007562409232763304,0.020160000771284103,0.052960000932216644,0.024145600199699403,0.021424000151455402,0.007450198645599985,0.0,0.0,0.0,0.0,0.0,0.47494399547576904,0.5184000134468079,0.4920704007148743,0.49169600009918213,0.011991064701471855,8,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,64,0.375,0.5,0.5,0.3984375,0.0,1.3919410907075054,6.0,5.570159765557392,0.667236328125,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9806430689981738,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.024831999093294144,0.046560000628232956,0.030371200107038022,0.02763199992477894,0.005878487205028602,0.020320000126957893,0.04560000076889992,0.02601920012384653,0.02270400058478117,0.00751129965906799,0.0,0.0,0.0,0.0,0.0,0.6360960006713867,0.7004479765892029,0.6608384013175964,0.6572319865226746,0.020416877242438597,16,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,128,0.375,1.0,1.0,0.625,0.0,1.0307764064044151,4.0,6.138251855282827,0.5382080078125,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9103623678483975,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.024288000538945198,0.049375999718904495,0.03086080001667142,0.0267359996214509,0.0070864531384997225,0.020479999482631683,0.030912000685930252,0.02274719988927245,0.021359999664127827,0.0029966813650672505,0.0,0.0,0.0,0.0,0.0,0.780896008014679,0.8301439881324768,0.80346559882164,0.8030399978160858,0.016801230312128875,32,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,256,0.375,2.0,2.0,0.859375,0.0,0.6903350635742038,3.0,6.5943747091218174,0.38067626953125,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9610778571819444,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.02393599972128868,0.04342399910092354,0.029380799923092126,0.026688000187277794,0.0057374782000526574,0.020031999796628952,0.036607999354600906,0.022487999964505435,0.020911999978125095,0.0038135203062103235,0.0,0.0,0.0,0.0,0.0,0.8607040047645569,0.9195200204849243,0.8783008038997651,0.8751039803028107,0.01719059253115595,64,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,512,0.375,4.0,4.0,0.9765625,0.0,0.49410588440130926,2.75,6.814452474347134,0.271270751953125,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9952941013961014,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.025407999753952026,0.05510399863123894,0.031430399790406224,0.02798399981111288,0.007335050273982725,0.020479999482631683,0.03561599925160408,0.02275839988142252,0.021551999263465405,0.003545718365165811,0.0,0.0,0.0,0.0,0.0,0.833952009677887,0.894752025604248,0.8619967997074127,0.863215982913971,0.018716378851797198,128,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,1024,0.375,8.0,8.0,1.0,0.25,0.33693529145074724,2.125,6.9186075263155535,0.1867218017578125,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9273256282883522,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.0244159996509552,0.05395200103521347,0.03188959984108806,0.026031999848783016,0.00943995927387483,0.02051199972629547,0.036607999354600906,0.023247999791055917,0.02147199958562851,0.003910623837069648,0.0,0.0,0.0,0.0,0.0,0.834879994392395,0.8871039748191833,0.8651552021503448,0.8638879954814911,0.015262430894400969,256,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,2048,0.375,16.0,16.0,1.0,0.375,0.25567294018677456,1.8125,6.952441049154937,0.14349365234375,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0380599882396606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.024512000381946564,0.04620800167322159,0.02884640023112297,0.026320000179111958,0.005516557583355925,0.02054399996995926,0.03846399858593941,0.023401600029319524,0.021263999864459038,0.0044742944198265374,0.0,0.0,0.0,0.0,0.0,0.8518080115318298,0.9097599983215332,0.8810272097587586,0.8751040101051331,0.017005819633271906,512,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,4096,0.375,32.0,32.0,1.0,0.625,0.1747801353218523,1.53125,6.978069554482723,0.09820938110351562,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9569603278386984,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.02412799932062626,0.06940799951553345,0.030817600432783365,0.026800000108778477,0.010047085187652939,0.020128000527620316,0.044096000492572784,0.024606400076299904,0.022304000332951546,0.00622857166823714,0.0,0.0,0.0,0.0,0.0,0.8470079898834229,0.9010239839553833,0.8694015920162201,0.8716959953308105,0.016138881171190216,1024,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,8192,0.375,64.0,64.0,1.0,0.65625,0.1158122428154187,1.3125,6.9901908183358845,0.06445503234863281,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9524274993623046,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.02831999957561493,0.043487999588251114,0.031744000129401685,0.02991999965161085,0.003973415836428909,0.02070399932563305,0.029343999922275543,0.022886400017887353,0.021743999794125557,0.0026873065045088873,0.0,0.0,0.0,0.0,0.0,1.1698240041732788,1.2311359643936157,1.1888479948043824,1.1890720129013062,0.017978797794492758,2048,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,16384,0.375,128.0,128.0,1.0,0.78125,0.08347181893108634,1.1796875,6.994921772573154,0.046871185302734375,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8971818172100244,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.03830400109291077,0.06268800050020218,0.043337599746882914,0.040511999279260635,0.005823016247946116,0.023135999217629433,0.03747199848294258,0.025206399988383053,0.02393599972128868,0.003271381256231161,0.0,0.0,0.0,0.0,0.0,1.7702720165252686,1.8097599744796753,1.7919103980064393,1.7956640124320984,0.012641295676021557,4096,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,32768,0.375,256.0,256.0,1.0,0.78125,0.06866734477822484,1.20703125,6.996602562938728,0.03801727294921875,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8113207890716184,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.05660799890756607,0.07932800054550171,0.06249920018017292,0.06039999984204769,0.005636461691480845,0.02956799976527691,0.03747199848294258,0.031126399897038935,0.030287999659776688,0.002157571006631541,0.0,0.0,0.0,0.0,0.0,2.968672037124634,3.0278079509735107,2.9899007797241213,2.9824799299240112,0.01816414122631667,8192,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,65536,0.375,512.0,512.0,1.0,0.8984375,0.04399546833977376,1.126953125,6.998607314922362,0.024699926376342773,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.883909666885606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.02502400055527687,0.06092799827456474,0.033839999698102474,0.028672000393271446,0.010315255343709818,0.019360000267624855,0.0352960005402565,0.023424000293016434,0.022064000368118286,0.0038898102289194572,0.0,0.0,0.0,0.0,0.0,0.19574399292469025,0.2512960135936737,0.21939200013875962,0.2199999988079071,0.017532156418212565,1,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,8,0.375,0.0625,0.0625,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0233364439829928,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.025280000641942024,0.05132799968123436,0.030641599837690593,0.027343999594449997,0.007562409232763304,0.020160000771284103,0.052960000932216644,0.024145600199699403,0.021424000151455402,0.007450198645599985,0.0,0.0,0.0,0.0,0.0,0.20483200252056122,0.24006399512290955,0.22215040028095245,0.22433599829673767,0.00969639786132892,8,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,64,0.375,0.5,0.5,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9806430689981738,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.024831999093294144,0.046560000628232956,0.030371200107038022,0.02763199992477894,0.005878487205028602,0.020320000126957893,0.04560000076889992,0.02601920012384653,0.02270400058478117,0.00751129965906799,0.0,0.0,0.0,0.0,0.0,0.20559999346733093,0.24726399779319763,0.22126719802618028,0.22207999974489212,0.01328093478485499,16,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,128,0.375,1.0,1.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9103623678483975,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.024288000538945198,0.049375999718904495,0.03086080001667142,0.0267359996214509,0.0070864531384997225,0.020479999482631683,0.030912000685930252,0.02274719988927245,0.021359999664127827,0.0029966813650672505,0.0,0.0,0.0,0.0,0.0,0.20003199577331543,0.2301120012998581,0.21453119963407516,0.21598400175571442,0.010402239855151332,32,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,256,0.375,2.0,2.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9610778571819444,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.02393599972128868,0.04342399910092354,0.029380799923092126,0.026688000187277794,0.0057374782000526574,0.020031999796628952,0.036607999354600906,0.022487999964505435,0.020911999978125095,0.0038135203062103235,0.0,0.0,0.0,0.0,0.0,0.19551999866962433,0.22972799837589264,0.21238719969987868,0.21488000452518463,0.010706095835489097,64,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,512,0.375,4.0,4.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9952941013961014,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.025407999753952026,0.05510399863123894,0.031430399790406224,0.02798399981111288,0.007335050273982725,0.020479999482631683,0.03561599925160408,0.02275839988142252,0.021551999263465405,0.003545718365165811,0.0,0.0,0.0,0.0,0.0,0.19420799612998962,0.2903999984264374,0.2211231991648674,0.21476799994707108,0.025955008886196004,128,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,1024,0.375,8.0,8.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9273256282883522,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.0244159996509552,0.05395200103521347,0.03188959984108806,0.026031999848783016,0.00943995927387483,0.02051199972629547,0.036607999354600906,0.023247999791055917,0.02147199958562851,0.003910623837069648,0.0,0.0,0.0,0.0,0.0,0.2290560007095337,0.289247989654541,0.2543327987194061,0.24939200282096863,0.01663236753503115,256,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,2048,0.375,16.0,16.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0380599882396606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.024512000381946564,0.04620800167322159,0.02884640023112297,0.026320000179111958,0.005516557583355925,0.02054399996995926,0.03846399858593941,0.023401600029319524,0.021263999864459038,0.0044742944198265374,0.0,0.0,0.0,0.0,0.0,0.30831998586654663,0.36953601241111755,0.3324000000953674,0.3288639932870865,0.018617999572156707,512,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,4096,0.375,32.0,32.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9569603278386984,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.02412799932062626,0.06940799951553345,0.030817600432783365,0.026800000108778477,0.010047085187652939,0.020128000527620316,0.044096000492572784,0.024606400076299904,0.022304000332951546,0.00622857166823714,0.0,0.0,0.0,0.0,0.0,0.462911993265152,0.5497919917106628,0.4893856018781662,0.4816960096359253,0.023348152887178286,1024,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,8192,0.375,64.0,64.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9524274993623046,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.02831999957561493,0.043487999588251114,0.031744000129401685,0.02991999965161085,0.003973415836428909,0.02070399932563305,0.029343999922275543,0.022886400017887353,0.021743999794125557,0.0026873065045088873,0.0,0.0,0.0,0.0,0.0,0.7662079930305481,0.8717759847640991,0.788454395532608,0.7744799852371216,0.03174746482604812,2048,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,16384,0.375,128.0,128.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8971818172100244,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.03830400109291077,0.06268800050020218,0.043337599746882914,0.040511999279260635,0.005823016247946116,0.023135999217629433,0.03747199848294258,0.025206399988383053,0.02393599972128868,0.003271381256231161,0.0,0.0,0.0,0.0,0.0,1.363935947418213,1.4143040180206299,1.3812703967094422,1.3798720240592957,0.014770450075530007,4096,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,32768,0.375,256.0,256.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8113207890716184,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
0.05660799890756607,0.07932800054550171,0.06249920018017292,0.06039999984204769,0.005636461691480845,0.02956799976527691,0.03747199848294258,0.031126399897038935,0.030287999659776688,0.002157571006631541,0.0,0.0,0.0,0.0,0.0,2.5507519245147705,2.680704116821289,2.579859209060669,2.566223978996277,0.03673283558365955,8192,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,65536,0.375,512.0,512.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.883909666885606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
1 time_stats.moe_gating_linear.min time_stats.moe_gating_linear.max time_stats.moe_gating_linear.mean time_stats.moe_gating_linear.median time_stats.moe_gating_linear.std time_stats.moe_gating_routing_topk.min time_stats.moe_gating_routing_topk.max time_stats.moe_gating_routing_topk.mean time_stats.moe_gating_routing_topk.median time_stats.moe_gating_routing_topk.std time_stats.moe_shuffling.min time_stats.moe_shuffling.max time_stats.moe_shuffling.mean time_stats.moe_shuffling.median time_stats.moe_shuffling.std time_stats.moe_grouped_gemm.min time_stats.moe_grouped_gemm.max time_stats.moe_grouped_gemm.mean time_stats.moe_grouped_gemm.median time_stats.moe_grouped_gemm.std num_tokens num_experts num_experts_per_device expert_parallel_size routing_runtime_path routing_assignment_policy routing_weight_policy routing_uses_router_logits gating_runtime_context gating_runtime_context_impl router_topk hidden_dim expert_hidden_dim use_gated num_tensor_parallel_workers total_routed_tokens model_expansion_ratio tokens_per_expert_avg tokens_to_experts_ratio expert_utilization min_load_ratio load_imbalance_cv max_load_ratio load_entropy load_gini_coefficient load_distribution seed moe_grouped_gemm_backend measurement_type profiling_precision model_arch quant_signature router_median_nonadditivity_ratio projection_policy
2 0.02502400055527687 0.06092799827456474 0.033839999698102474 0.028672000393271446 0.010315255343709818 0.019360000267624855 0.0352960005402565 0.023424000293016434 0.022064000368118286 0.0038898102289194572 0.0 0.0 0.0 0.0 0.0 0.33926400542259216 0.405023992061615 0.36780479848384856 0.36507199704647064 0.01690507644474779 1 128 128 1 standard_fused_topk logit_topk softmax_renorm True standalone_legacy vllm020_replicated_linear 8 2048 768 True 1 8 0.375 0.0625 0.0625 0.0625 0.0 3.872983346207417 16.0 3.0 0.9375 uniform_random_logits 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 1.0233364439829928 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
3 0.025280000641942024 0.05132799968123436 0.030641599837690593 0.027343999594449997 0.007562409232763304 0.020160000771284103 0.052960000932216644 0.024145600199699403 0.021424000151455402 0.007450198645599985 0.0 0.0 0.0 0.0 0.0 1.1943039894104004 1.286784052848816 1.228384006023407 1.2273280024528503 0.02832547242381263 8 128 128 1 standard_fused_topk logit_topk softmax_renorm True standalone_legacy vllm020_replicated_linear 8 2048 768 True 1 64 0.375 0.5 0.5 0.3984375 0.0 1.346291201783626 4.0 5.59375 0.661865234375 uniform_random_logits 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 0.9806430689981738 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
4 0.024831999093294144 0.046560000628232956 0.030371200107038022 0.02763199992477894 0.005878487205028602 0.020320000126957893 0.04560000076889992 0.02601920012384653 0.02270400058478117 0.00751129965906799 0.0 0.0 0.0 0.0 0.0 1.679744005203247 1.766144037246704 1.7095808148384095 1.7015680074691772 0.02438921262998535 16 128 128 1 standard_fused_topk logit_topk softmax_renorm True standalone_legacy vllm020_replicated_linear 8 2048 768 True 1 128 0.375 1.0 1.0 0.625 0.0 1.015504800579495 5.0 6.15516433212955 0.529052734375 uniform_random_logits 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 0.9103623678483975 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
5 0.024288000538945198 0.049375999718904495 0.03086080001667142 0.0267359996214509 0.0070864531384997225 0.020479999482631683 0.030912000685930252 0.02274719988927245 0.021359999664127827 0.0029966813650672505 0.0 0.0 0.0 0.0 0.0 2.1576640605926514 2.2921600341796875 2.2097824096679686 2.188944101333618 0.045572321842012986 32 128 128 1 standard_fused_topk logit_topk softmax_renorm True standalone_legacy vllm020_replicated_linear 8 2048 768 True 1 256 0.375 2.0 2.0 0.875 0.0 0.6343057228182637 2.5 6.64370748444639 0.35369873046875 uniform_random_logits 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 0.9610778571819444 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
6 0.02393599972128868 0.04342399910092354 0.029380799923092126 0.026688000187277794 0.0057374782000526574 0.020031999796628952 0.036607999354600906 0.022487999964505435 0.020911999978125095 0.0038135203062103235 0.0 0.0 0.0 0.0 0.0 2.422368049621582 2.5130879878997803 2.4516672134399413 2.434159994125366 0.03278900287381846 64 128 128 1 standard_fused_topk logit_topk softmax_renorm True standalone_legacy vllm020_replicated_linear 8 2048 768 True 1 512 0.375 4.0 4.0 0.984375 0.0 0.4921254921257382 2.25 6.817190042344769 0.272369384765625 uniform_random_logits 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 0.9952941013961014 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
7 0.025407999753952026 0.05510399863123894 0.031430399790406224 0.02798399981111288 0.007335050273982725 0.020479999482631683 0.03561599925160408 0.02275839988142252 0.021551999263465405 0.003545718365165811 0.0 0.0 0.0 0.0 0.0 2.2217600345611572 2.289599895477295 2.2571327924728393 2.263375997543335 0.021660416089449488 128 128 128 1 standard_fused_topk logit_topk softmax_renorm True standalone_legacy vllm020_replicated_linear 8 2048 768 True 1 1024 0.375 8.0 8.0 1.0 0.125 0.3486861500690843 1.875 6.908192310183997 0.197662353515625 uniform_random_logits 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 0.9273256282883522 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
8 0.0244159996509552 0.05395200103521347 0.03188959984108806 0.026031999848783016 0.00943995927387483 0.02051199972629547 0.036607999354600906 0.023247999791055917 0.02147199958562851 0.003910623837069648 0.0 0.0 0.0 0.0 0.0 2.18668794631958 2.318079948425293 2.2218016147613526 2.211087942123413 0.035380897213135316 256 128 128 1 standard_fused_topk logit_topk softmax_renorm True standalone_legacy vllm020_replicated_linear 8 2048 768 True 1 2048 0.375 16.0 16.0 1.0 0.4375 0.2525504668006971 1.875 6.953347743053017 0.1410369873046875 uniform_random_logits 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 1.0380599882396606 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
9 0.024512000381946564 0.04620800167322159 0.02884640023112297 0.026320000179111958 0.005516557583355925 0.02054399996995926 0.03846399858593941 0.023401600029319524 0.021263999864459038 0.0044742944198265374 0.0 0.0 0.0 0.0 0.0 2.2291839122772217 2.3929600715637207 2.2908096313476562 2.2804640531539917 0.04479348924786221 512 128 128 1 standard_fused_topk logit_topk softmax_renorm True standalone_legacy vllm020_replicated_linear 8 2048 768 True 1 4096 0.375 32.0 32.0 1.0 0.65625 0.15765965680164504 1.5625 6.98229848728205 0.08779525756835938 uniform_random_logits 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 0.9569603278386984 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
10 0.02412799932062626 0.06940799951553345 0.030817600432783365 0.026800000108778477 0.010047085187652939 0.020128000527620316 0.044096000492572784 0.024606400076299904 0.022304000332951546 0.00622857166823714 0.0 0.0 0.0 0.0 0.0 2.0678720474243164 2.1297600269317627 2.0837119817733765 2.0779199600219727 0.017880044357986735 1024 128 128 1 standard_fused_topk logit_topk softmax_renorm True standalone_legacy vllm020_replicated_linear 8 2048 768 True 1 8192 0.375 64.0 64.0 1.0 0.625 0.12169081635504074 1.3125 6.9892029662356325 0.06879425048828125 uniform_random_logits 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 0.9524274993623046 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
11 0.02831999957561493 0.043487999588251114 0.031744000129401685 0.02991999965161085 0.003973415836428909 0.02070399932563305 0.029343999922275543 0.022886400017887353 0.021743999794125557 0.0026873065045088873 0.0 0.0 0.0 0.0 0.0 2.916032075881958 3.0819520950317383 2.9805248022079467 2.9656319618225098 0.05482195799019572 2048 128 128 1 standard_fused_topk logit_topk softmax_renorm True standalone_legacy vllm020_replicated_linear 8 2048 768 True 1 16384 0.375 128.0 128.0 1.0 0.796875 0.07935434147688751 1.1796875 6.9954297964750305 0.044734954833984375 uniform_random_logits 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 0.8971818172100244 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
12 0.03830400109291077 0.06268800050020218 0.043337599746882914 0.040511999279260635 0.005823016247946116 0.023135999217629433 0.03747199848294258 0.025206399988383053 0.02393599972128868 0.003271381256231161 0.0 0.0 0.0 0.0 0.0 4.421599864959717 4.535359859466553 4.486294317245483 4.497056007385254 0.036990243787549344 4096 128 128 1 standard_fused_topk logit_topk softmax_renorm True standalone_legacy vllm020_replicated_linear 8 2048 768 True 1 32768 0.375 256.0 256.0 1.0 0.8203125 0.060849326483103046 1.17578125 6.9973188375859685 0.033740997314453125 uniform_random_logits 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 0.8113207890716184 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
13 0.05660799890756607 0.07932800054550171 0.06249920018017292 0.06039999984204769 0.005636461691480845 0.02956799976527691 0.03747199848294258 0.031126399897038935 0.030287999659776688 0.002157571006631541 0.0 0.0 0.0 0.0 0.0 7.302591800689697 7.402751922607422 7.354758310317993 7.3464319705963135 0.032142662400335566 8192 128 128 1 standard_fused_topk logit_topk softmax_renorm True standalone_legacy vllm020_replicated_linear 8 2048 768 True 1 65536 0.375 512.0 512.0 1.0 0.890625 0.0412323087266341 1.08984375 6.998772433185578 0.02334284782409668 uniform_random_logits 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 0.883909666885606 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
14 0.02502400055527687 0.06092799827456474 0.033839999698102474 0.028672000393271446 0.010315255343709818 0.019360000267624855 0.0352960005402565 0.023424000293016434 0.022064000368118286 0.0038898102289194572 0.0 0.0 0.0 0.0 0.0 0.35280001163482666 0.39692801237106323 0.37662720382213594 0.37196800112724304 0.013401318050665304 1 128 128 1 standard_fused_topk fixed_hotset8 softmax_renorm False standalone_legacy vllm020_replicated_linear 8 2048 768 True 1 8 0.375 0.0625 0.0625 0.0625 0.0 3.872983346207417 16.0 3.0 0.9375 hotset8 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 1.0233364439829928 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
15 0.025280000641942024 0.05132799968123436 0.030641599837690593 0.027343999594449997 0.007562409232763304 0.020160000771284103 0.052960000932216644 0.024145600199699403 0.021424000151455402 0.007450198645599985 0.0 0.0 0.0 0.0 0.0 0.4692479968070984 0.5523840188980103 0.5134752035140991 0.5100640058517456 0.02291433464135784 8 128 128 1 standard_fused_topk fixed_hotset8 softmax_renorm False standalone_legacy vllm020_replicated_linear 8 2048 768 True 1 64 0.375 0.5 0.5 0.0625 0.0 3.872983346207417 16.0 3.0 0.9375 hotset8 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 0.9806430689981738 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
16 0.024831999093294144 0.046560000628232956 0.030371200107038022 0.02763199992477894 0.005878487205028602 0.020320000126957893 0.04560000076889992 0.02601920012384653 0.02270400058478117 0.00751129965906799 0.0 0.0 0.0 0.0 0.0 0.34652799367904663 0.4119040071964264 0.3789471983909607 0.38550400733947754 0.02073945105803335 16 128 128 1 standard_fused_topk fixed_hotset8 softmax_renorm False standalone_legacy vllm020_replicated_linear 8 2048 768 True 1 128 0.375 1.0 1.0 0.0625 0.0 3.872983346207417 16.0 3.0 0.9375 hotset8 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 0.9103623678483975 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
17 0.024288000538945198 0.049375999718904495 0.03086080001667142 0.0267359996214509 0.0070864531384997225 0.020479999482631683 0.030912000685930252 0.02274719988927245 0.021359999664127827 0.0029966813650672505 0.0 0.0 0.0 0.0 0.0 0.31462401151657104 0.7456960082054138 0.38617280423641204 0.34545600414276123 0.12230201266253077 32 128 128 1 standard_fused_topk fixed_hotset8 softmax_renorm False standalone_legacy vllm020_replicated_linear 8 2048 768 True 1 256 0.375 2.0 2.0 0.0625 0.0 3.872983346207417 16.0 3.0 0.9375 hotset8 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 0.9610778571819444 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
18 0.02393599972128868 0.04342399910092354 0.029380799923092126 0.026688000187277794 0.0057374782000526574 0.020031999796628952 0.036607999354600906 0.022487999964505435 0.020911999978125095 0.0038135203062103235 0.0 0.0 0.0 0.0 0.0 0.32521599531173706 0.419871985912323 0.36325119733810424 0.34968000650405884 0.03161798848223672 64 128 128 1 standard_fused_topk fixed_hotset8 softmax_renorm False standalone_legacy vllm020_replicated_linear 8 2048 768 True 1 512 0.375 4.0 4.0 0.0625 0.0 3.872983346207417 16.0 3.0 0.9375 hotset8 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 0.9952941013961014 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
19 0.025407999753952026 0.05510399863123894 0.031430399790406224 0.02798399981111288 0.007335050273982725 0.020479999482631683 0.03561599925160408 0.02275839988142252 0.021551999263465405 0.003545718365165811 0.0 0.0 0.0 0.0 0.0 0.289792001247406 0.4663360118865967 0.4091839998960495 0.41655999422073364 0.0446001986506615 128 128 128 1 standard_fused_topk fixed_hotset8 softmax_renorm False standalone_legacy vllm020_replicated_linear 8 2048 768 True 1 1024 0.375 8.0 8.0 0.0625 0.0 3.872983346207417 16.0 3.0 0.9375 hotset8 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 0.9273256282883522 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
20 0.0244159996509552 0.05395200103521347 0.03188959984108806 0.026031999848783016 0.00943995927387483 0.02051199972629547 0.036607999354600906 0.023247999791055917 0.02147199958562851 0.003910623837069648 0.0 0.0 0.0 0.0 0.0 0.3761279881000519 0.4416320025920868 0.40686399936676027 0.40540799498558044 0.02257778769899645 256 128 128 1 standard_fused_topk fixed_hotset8 softmax_renorm False standalone_legacy vllm020_replicated_linear 8 2048 768 True 1 2048 0.375 16.0 16.0 0.0625 0.0 3.872983346207417 16.0 3.0 0.9375 hotset8 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 1.0380599882396606 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
21 0.024512000381946564 0.04620800167322159 0.02884640023112297 0.026320000179111958 0.005516557583355925 0.02054399996995926 0.03846399858593941 0.023401600029319524 0.021263999864459038 0.0044742944198265374 0.0 0.0 0.0 0.0 0.0 0.7172480225563049 0.8663039803504944 0.7723807990550995 0.7591840028762817 0.04164772379451242 512 128 128 1 standard_fused_topk fixed_hotset8 softmax_renorm False standalone_legacy vllm020_replicated_linear 8 2048 768 True 1 4096 0.375 32.0 32.0 0.0625 0.0 3.872983346207417 16.0 3.0 0.9375 hotset8 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 0.9569603278386984 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
22 0.02412799932062626 0.06940799951553345 0.030817600432783365 0.026800000108778477 0.010047085187652939 0.020128000527620316 0.044096000492572784 0.024606400076299904 0.022304000332951546 0.00622857166823714 0.0 0.0 0.0 0.0 0.0 1.0195519924163818 1.2216639518737793 1.1253888130187988 1.1453600525856018 0.06548005322243594 1024 128 128 1 standard_fused_topk fixed_hotset8 softmax_renorm False standalone_legacy vllm020_replicated_linear 8 2048 768 True 1 8192 0.375 64.0 64.0 0.0625 0.0 3.872983346207417 16.0 3.0 0.9375 hotset8 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 0.9524274993623046 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
23 0.02831999957561493 0.043487999588251114 0.031744000129401685 0.02991999965161085 0.003973415836428909 0.02070399932563305 0.029343999922275543 0.022886400017887353 0.021743999794125557 0.0026873065045088873 0.0 0.0 0.0 0.0 0.0 1.7490559816360474 1.9644800424575806 1.8529024004936219 1.814303994178772 0.08042565617327288 2048 128 128 1 standard_fused_topk fixed_hotset8 softmax_renorm False standalone_legacy vllm020_replicated_linear 8 2048 768 True 1 16384 0.375 128.0 128.0 0.0625 0.0 3.872983346207417 16.0 3.0 0.9375 hotset8 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 0.8971818172100244 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
24 0.03830400109291077 0.06268800050020218 0.043337599746882914 0.040511999279260635 0.005823016247946116 0.023135999217629433 0.03747199848294258 0.025206399988383053 0.02393599972128868 0.003271381256231161 0.0 0.0 0.0 0.0 0.0 3.2479360103607178 3.385279893875122 3.296070408821106 3.2800960540771484 0.04525529026442046 4096 128 128 1 standard_fused_topk fixed_hotset8 softmax_renorm False standalone_legacy vllm020_replicated_linear 8 2048 768 True 1 32768 0.375 256.0 256.0 0.0625 0.0 3.872983346207417 16.0 3.0 0.9375 hotset8 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 0.8113207890716184 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
25 0.05660799890756607 0.07932800054550171 0.06249920018017292 0.06039999984204769 0.005636461691480845 0.02956799976527691 0.03747199848294258 0.031126399897038935 0.030287999659776688 0.002157571006631541 0.0 0.0 0.0 0.0 0.0 6.344799995422363 6.517856121063232 6.464438438415527 6.478623867034912 0.05116674443145098 8192 128 128 1 standard_fused_topk fixed_hotset8 softmax_renorm False standalone_legacy vllm020_replicated_linear 8 2048 768 True 1 65536 0.375 512.0 512.0 0.0625 0.0 3.872983346207417 16.0 3.0 0.9375 hotset8 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 0.883909666885606 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
26 0.02502400055527687 0.06092799827456474 0.033839999698102474 0.028672000393271446 0.010315255343709818 0.019360000267624855 0.0352960005402565 0.023424000293016434 0.022064000368118286 0.0038898102289194572 0.0 0.0 0.0 0.0 0.0 0.2648000121116638 0.325439989566803 0.28852800130844114 0.28390398621559143 0.01933778377635077 1 128 128 1 standard_fused_topk logit_topk softmax_renorm True standalone_legacy vllm020_replicated_linear 8 2048 768 True 2 8 0.375 0.0625 0.0625 0.0625 0.0 3.872983346207417 16.0 3.0 0.9375 uniform_random_logits 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 1.0233364439829928 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
27 0.025280000641942024 0.05132799968123436 0.030641599837690593 0.027343999594449997 0.007562409232763304 0.020160000771284103 0.052960000932216644 0.024145600199699403 0.021424000151455402 0.007450198645599985 0.0 0.0 0.0 0.0 0.0 0.7347840070724487 0.862496018409729 0.7769344031810761 0.769216001033783 0.03290485328796285 8 128 128 1 standard_fused_topk logit_topk softmax_renorm True standalone_legacy vllm020_replicated_linear 8 2048 768 True 2 64 0.375 0.5 0.5 0.421875 0.0 1.346291201783626 6.0 5.652114648336087 0.636962890625 uniform_random_logits 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 0.9806430689981738 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
28 0.024831999093294144 0.046560000628232956 0.030371200107038022 0.02763199992477894 0.005878487205028602 0.020320000126957893 0.04560000076889992 0.02601920012384653 0.02270400058478117 0.00751129965906799 0.0 0.0 0.0 0.0 0.0 0.9198399782180786 0.9646080136299133 0.9412063956260681 0.9411839842796326 0.014939365085478117 16 128 128 1 standard_fused_topk logit_topk softmax_renorm True standalone_legacy vllm020_replicated_linear 8 2048 768 True 2 128 0.375 1.0 1.0 0.5703125 0.0 1.118033988749895 5.0 6.008641773518898 0.580810546875 uniform_random_logits 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 0.9103623678483975 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
29 0.024288000538945198 0.049375999718904495 0.03086080001667142 0.0267359996214509 0.0070864531384997225 0.020479999482631683 0.030912000685930252 0.02274719988927245 0.021359999664127827 0.0029966813650672505 0.0 0.0 0.0 0.0 0.0 1.2796800136566162 1.3484159708023071 1.3006976008415223 1.2929120063781738 0.020807998177176254 32 128 128 1 standard_fused_topk logit_topk softmax_renorm True standalone_legacy vllm020_replicated_linear 8 2048 768 True 2 256 0.375 2.0 2.0 0.8828125 0.0 0.6959705453537527 3.0 6.60872850615583 0.38055419921875 uniform_random_logits 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 0.9610778571819444 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
30 0.02393599972128868 0.04342399910092354 0.029380799923092126 0.026688000187277794 0.0057374782000526574 0.020031999796628952 0.036607999354600906 0.022487999964505435 0.020911999978125095 0.0038135203062103235 0.0 0.0 0.0 0.0 0.0 1.3630399703979492 1.4430400133132935 1.3909215927124023 1.3892319798469543 0.022335744492366926 64 128 128 1 standard_fused_topk logit_topk softmax_renorm True standalone_legacy vllm020_replicated_linear 8 2048 768 True 2 512 0.375 4.0 4.0 0.984375 0.0 0.5201036555341637 3.0 6.798826509158851 0.28302001953125 uniform_random_logits 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 0.9952941013961014 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
31 0.025407999753952026 0.05510399863123894 0.031430399790406224 0.02798399981111288 0.007335050273982725 0.020479999482631683 0.03561599925160408 0.02275839988142252 0.021551999263465405 0.003545718365165811 0.0 0.0 0.0 0.0 0.0 1.27948796749115 1.3904000520706177 1.3176063895225525 1.309440016746521 0.038060887827312775 128 128 128 1 standard_fused_topk logit_topk softmax_renorm True standalone_legacy vllm020_replicated_linear 8 2048 768 True 2 1024 0.375 8.0 8.0 1.0 0.25 0.3511282039725661 1.875 6.91002266305238 0.1970977783203125 uniform_random_logits 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 0.9273256282883522 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
32 0.0244159996509552 0.05395200103521347 0.03188959984108806 0.026031999848783016 0.00943995927387483 0.02051199972629547 0.036607999354600906 0.023247999791055917 0.02147199958562851 0.003910623837069648 0.0 0.0 0.0 0.0 0.0 1.264415979385376 1.3145920038223267 1.2791999936103822 1.2753440141677856 0.014130605249568332 256 128 128 1 standard_fused_topk logit_topk softmax_renorm True standalone_legacy vllm020_replicated_linear 8 2048 768 True 2 2048 0.375 16.0 16.0 1.0 0.375 0.24692938483248605 1.6875 6.955481130775285 0.13909912109375 uniform_random_logits 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 1.0380599882396606 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
33 0.024512000381946564 0.04620800167322159 0.02884640023112297 0.026320000179111958 0.005516557583355925 0.02054399996995926 0.03846399858593941 0.023401600029319524 0.021263999864459038 0.0044742944198265374 0.0 0.0 0.0 0.0 0.0 1.3081920146942139 1.347648024559021 1.3292255997657776 1.329967975616455 0.014558863679016933 512 128 128 1 standard_fused_topk logit_topk softmax_renorm True standalone_legacy vllm020_replicated_linear 8 2048 768 True 2 4096 0.375 32.0 32.0 1.0 0.625 0.17143053326165383 1.5625 6.9786675275754035 0.09520339965820312 uniform_random_logits 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 0.9569603278386984 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
34 0.02412799932062626 0.06940799951553345 0.030817600432783365 0.026800000108778477 0.010047085187652939 0.020128000527620316 0.044096000492572784 0.024606400076299904 0.022304000332951546 0.00622857166823714 0.0 0.0 0.0 0.0 0.0 1.242751955986023 1.3112000226974487 1.2747935891151427 1.266207993030548 0.021093073517695057 1024 128 128 1 standard_fused_topk logit_topk softmax_renorm True standalone_legacy vllm020_replicated_linear 8 2048 768 True 2 8192 0.375 64.0 64.0 1.0 0.78125 0.11000099875256815 1.296875 6.991308871213679 0.062183380126953125 uniform_random_logits 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 0.9524274993623046 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
35 0.02831999957561493 0.043487999588251114 0.031744000129401685 0.02991999965161085 0.003973415836428909 0.02070399932563305 0.029343999922275543 0.022886400017887353 0.021743999794125557 0.0026873065045088873 0.0 0.0 0.0 0.0 0.0 1.7388160228729248 1.8077759742736816 1.772764801979065 1.772704005241394 0.021056644077284283 2048 128 128 1 standard_fused_topk logit_topk softmax_renorm True standalone_legacy vllm020_replicated_linear 8 2048 768 True 2 16384 0.375 128.0 128.0 1.0 0.78125 0.0864630150197678 1.1796875 6.994552526394139 0.048796653747558594 uniform_random_logits 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 0.8971818172100244 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
36 0.03830400109291077 0.06268800050020218 0.043337599746882914 0.040511999279260635 0.005823016247946116 0.023135999217629433 0.03747199848294258 0.025206399988383053 0.02393599972128868 0.003271381256231161 0.0 0.0 0.0 0.0 0.0 2.6563520431518555 2.7063679695129395 2.6785055875778196 2.6791679859161377 0.01639963463052944 4096 128 128 1 standard_fused_topk logit_topk softmax_renorm True standalone_legacy vllm020_replicated_linear 8 2048 768 True 2 32768 0.375 256.0 256.0 1.0 0.8671875 0.06127686514721937 1.16015625 6.997291583027146 0.03497934341430664 uniform_random_logits 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 0.8113207890716184 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
37 0.05660799890756607 0.07932800054550171 0.06249920018017292 0.06039999984204769 0.005636461691480845 0.02956799976527691 0.03747199848294258 0.031126399897038935 0.030287999659776688 0.002157571006631541 0.0 0.0 0.0 0.0 0.0 4.386879920959473 4.452256202697754 4.4108480453491214 4.406303882598877 0.019768161791937636 8192 128 128 1 standard_fused_topk logit_topk softmax_renorm True standalone_legacy vllm020_replicated_linear 8 2048 768 True 2 65536 0.375 512.0 512.0 1.0 0.884765625 0.041723768525324195 1.1171875 6.998746434318934 0.02298593521118164 uniform_random_logits 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 0.883909666885606 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
38 0.02502400055527687 0.06092799827456474 0.033839999698102474 0.028672000393271446 0.010315255343709818 0.019360000267624855 0.0352960005402565 0.023424000293016434 0.022064000368118286 0.0038898102289194572 0.0 0.0 0.0 0.0 0.0 0.24208000302314758 0.4028480052947998 0.3041536003351212 0.277103990316391 0.05660721484881584 1 128 128 1 standard_fused_topk fixed_hotset8 softmax_renorm False standalone_legacy vllm020_replicated_linear 8 2048 768 True 2 8 0.375 0.0625 0.0625 0.0625 0.0 3.872983346207417 16.0 3.0 0.9375 hotset8 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 1.0233364439829928 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
39 0.025280000641942024 0.05132799968123436 0.030641599837690593 0.027343999594449997 0.007562409232763304 0.020160000771284103 0.052960000932216644 0.024145600199699403 0.021424000151455402 0.007450198645599985 0.0 0.0 0.0 0.0 0.0 0.2447039932012558 0.30502399802207947 0.26446720361709597 0.26265600323677063 0.016744548364435372 8 128 128 1 standard_fused_topk fixed_hotset8 softmax_renorm False standalone_legacy vllm020_replicated_linear 8 2048 768 True 2 64 0.375 0.5 0.5 0.0625 0.0 3.872983346207417 16.0 3.0 0.9375 hotset8 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 0.9806430689981738 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
40 0.024831999093294144 0.046560000628232956 0.030371200107038022 0.02763199992477894 0.005878487205028602 0.020320000126957893 0.04560000076889992 0.02601920012384653 0.02270400058478117 0.00751129965906799 0.0 0.0 0.0 0.0 0.0 0.2337920069694519 0.2881599962711334 0.26074880361557007 0.264384001493454 0.016469850143940968 16 128 128 1 standard_fused_topk fixed_hotset8 softmax_renorm False standalone_legacy vllm020_replicated_linear 8 2048 768 True 2 128 0.375 1.0 1.0 0.0625 0.0 3.872983346207417 16.0 3.0 0.9375 hotset8 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 0.9103623678483975 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
41 0.024288000538945198 0.049375999718904495 0.03086080001667142 0.0267359996214509 0.0070864531384997225 0.020479999482631683 0.030912000685930252 0.02274719988927245 0.021359999664127827 0.0029966813650672505 0.0 0.0 0.0 0.0 0.0 0.23369599878787994 0.28591999411582947 0.25465920120477675 0.25385600328445435 0.01593204652619194 32 128 128 1 standard_fused_topk fixed_hotset8 softmax_renorm False standalone_legacy vllm020_replicated_linear 8 2048 768 True 2 256 0.375 2.0 2.0 0.0625 0.0 3.872983346207417 16.0 3.0 0.9375 hotset8 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 0.9610778571819444 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
42 0.02393599972128868 0.04342399910092354 0.029380799923092126 0.026688000187277794 0.0057374782000526574 0.020031999796628952 0.036607999354600906 0.022487999964505435 0.020911999978125095 0.0038135203062103235 0.0 0.0 0.0 0.0 0.0 0.2295999974012375 0.26556798815727234 0.24674240052700042 0.2497600018978119 0.010345732066199003 64 128 128 1 standard_fused_topk fixed_hotset8 softmax_renorm False standalone_legacy vllm020_replicated_linear 8 2048 768 True 2 512 0.375 4.0 4.0 0.0625 0.0 3.872983346207417 16.0 3.0 0.9375 hotset8 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 0.9952941013961014 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
43 0.025407999753952026 0.05510399863123894 0.031430399790406224 0.02798399981111288 0.007335050273982725 0.020479999482631683 0.03561599925160408 0.02275839988142252 0.021551999263465405 0.003545718365165811 0.0 0.0 0.0 0.0 0.0 0.21721599996089935 0.29020801186561584 0.2394208014011383 0.2346400022506714 0.018747330357768585 128 128 128 1 standard_fused_topk fixed_hotset8 softmax_renorm False standalone_legacy vllm020_replicated_linear 8 2048 768 True 2 1024 0.375 8.0 8.0 0.0625 0.0 3.872983346207417 16.0 3.0 0.9375 hotset8 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 0.9273256282883522 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
44 0.0244159996509552 0.05395200103521347 0.03188959984108806 0.026031999848783016 0.00943995927387483 0.02051199972629547 0.036607999354600906 0.023247999791055917 0.02147199958562851 0.003910623837069648 0.0 0.0 0.0 0.0 0.0 0.2717759907245636 0.305184006690979 0.28813759982585907 0.28809599578380585 0.01183422600545854 256 128 128 1 standard_fused_topk fixed_hotset8 softmax_renorm False standalone_legacy vllm020_replicated_linear 8 2048 768 True 2 2048 0.375 16.0 16.0 0.0625 0.0 3.872983346207417 16.0 3.0 0.9375 hotset8 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 1.0380599882396606 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
45 0.024512000381946564 0.04620800167322159 0.02884640023112297 0.026320000179111958 0.005516557583355925 0.02054399996995926 0.03846399858593941 0.023401600029319524 0.021263999864459038 0.0044742944198265374 0.0 0.0 0.0 0.0 0.0 0.3917759954929352 0.43772798776626587 0.41130879521369934 0.4131519943475723 0.012992473640805227 512 128 128 1 standard_fused_topk fixed_hotset8 softmax_renorm False standalone_legacy vllm020_replicated_linear 8 2048 768 True 2 4096 0.375 32.0 32.0 0.0625 0.0 3.872983346207417 16.0 3.0 0.9375 hotset8 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 0.9569603278386984 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
46 0.02412799932062626 0.06940799951553345 0.030817600432783365 0.026800000108778477 0.010047085187652939 0.020128000527620316 0.044096000492572784 0.024606400076299904 0.022304000332951546 0.00622857166823714 0.0 0.0 0.0 0.0 0.0 0.6176639795303345 0.7009919881820679 0.642767995595932 0.6330719888210297 0.024074084919938756 1024 128 128 1 standard_fused_topk fixed_hotset8 softmax_renorm False standalone_legacy vllm020_replicated_linear 8 2048 768 True 2 8192 0.375 64.0 64.0 0.0625 0.0 3.872983346207417 16.0 3.0 0.9375 hotset8 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 0.9524274993623046 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
47 0.02831999957561493 0.043487999588251114 0.031744000129401685 0.02991999965161085 0.003973415836428909 0.02070399932563305 0.029343999922275543 0.022886400017887353 0.021743999794125557 0.0026873065045088873 0.0 0.0 0.0 0.0 0.0 1.0820800065994263 1.1674879789352417 1.1034304022789 1.0977439880371094 0.0233067292981566 2048 128 128 1 standard_fused_topk fixed_hotset8 softmax_renorm False standalone_legacy vllm020_replicated_linear 8 2048 768 True 2 16384 0.375 128.0 128.0 0.0625 0.0 3.872983346207417 16.0 3.0 0.9375 hotset8 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 0.8971818172100244 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
48 0.03830400109291077 0.06268800050020218 0.043337599746882914 0.040511999279260635 0.005823016247946116 0.023135999217629433 0.03747199848294258 0.025206399988383053 0.02393599972128868 0.003271381256231161 0.0 0.0 0.0 0.0 0.0 1.9809919595718384 2.0415360927581787 2.003715181350708 1.992751955986023 0.022066076434645737 4096 128 128 1 standard_fused_topk fixed_hotset8 softmax_renorm False standalone_legacy vllm020_replicated_linear 8 2048 768 True 2 32768 0.375 256.0 256.0 0.0625 0.0 3.872983346207417 16.0 3.0 0.9375 hotset8 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 0.8113207890716184 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
49 0.05660799890756607 0.07932800054550171 0.06249920018017292 0.06039999984204769 0.005636461691480845 0.02956799976527691 0.03747199848294258 0.031126399897038935 0.030287999659776688 0.002157571006631541 0.0 0.0 0.0 0.0 0.0 3.790112018585205 3.8651199340820312 3.829139161109924 3.8230879306793213 0.025177464160110564 8192 128 128 1 standard_fused_topk fixed_hotset8 softmax_renorm False standalone_legacy vllm020_replicated_linear 8 2048 768 True 2 65536 0.375 512.0 512.0 0.0625 0.0 3.872983346207417 16.0 3.0 0.9375 hotset8 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 0.883909666885606 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
50 0.02502400055527687 0.06092799827456474 0.033839999698102474 0.028672000393271446 0.010315255343709818 0.019360000267624855 0.0352960005402565 0.023424000293016434 0.022064000368118286 0.0038898102289194572 0.0 0.0 0.0 0.0 0.0 0.212351992726326 0.24383999407291412 0.22760000079870224 0.22723200172185898 0.01050568575837594 1 128 128 1 standard_fused_topk logit_topk softmax_renorm True standalone_legacy vllm020_replicated_linear 8 2048 768 True 4 8 0.375 0.0625 0.0625 0.0625 0.0 3.872983346207417 16.0 3.0 0.9375 uniform_random_logits 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 1.0233364439829928 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
51 0.025280000641942024 0.05132799968123436 0.030641599837690593 0.027343999594449997 0.007562409232763304 0.020160000771284103 0.052960000932216644 0.024145600199699403 0.021424000151455402 0.007450198645599985 0.0 0.0 0.0 0.0 0.0 0.47494399547576904 0.5184000134468079 0.4920704007148743 0.49169600009918213 0.011991064701471855 8 128 128 1 standard_fused_topk logit_topk softmax_renorm True standalone_legacy vllm020_replicated_linear 8 2048 768 True 4 64 0.375 0.5 0.5 0.3984375 0.0 1.3919410907075054 6.0 5.570159765557392 0.667236328125 uniform_random_logits 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 0.9806430689981738 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
52 0.024831999093294144 0.046560000628232956 0.030371200107038022 0.02763199992477894 0.005878487205028602 0.020320000126957893 0.04560000076889992 0.02601920012384653 0.02270400058478117 0.00751129965906799 0.0 0.0 0.0 0.0 0.0 0.6360960006713867 0.7004479765892029 0.6608384013175964 0.6572319865226746 0.020416877242438597 16 128 128 1 standard_fused_topk logit_topk softmax_renorm True standalone_legacy vllm020_replicated_linear 8 2048 768 True 4 128 0.375 1.0 1.0 0.625 0.0 1.0307764064044151 4.0 6.138251855282827 0.5382080078125 uniform_random_logits 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 0.9103623678483975 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
53 0.024288000538945198 0.049375999718904495 0.03086080001667142 0.0267359996214509 0.0070864531384997225 0.020479999482631683 0.030912000685930252 0.02274719988927245 0.021359999664127827 0.0029966813650672505 0.0 0.0 0.0 0.0 0.0 0.780896008014679 0.8301439881324768 0.80346559882164 0.8030399978160858 0.016801230312128875 32 128 128 1 standard_fused_topk logit_topk softmax_renorm True standalone_legacy vllm020_replicated_linear 8 2048 768 True 4 256 0.375 2.0 2.0 0.859375 0.0 0.6903350635742038 3.0 6.5943747091218174 0.38067626953125 uniform_random_logits 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 0.9610778571819444 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
54 0.02393599972128868 0.04342399910092354 0.029380799923092126 0.026688000187277794 0.0057374782000526574 0.020031999796628952 0.036607999354600906 0.022487999964505435 0.020911999978125095 0.0038135203062103235 0.0 0.0 0.0 0.0 0.0 0.8607040047645569 0.9195200204849243 0.8783008038997651 0.8751039803028107 0.01719059253115595 64 128 128 1 standard_fused_topk logit_topk softmax_renorm True standalone_legacy vllm020_replicated_linear 8 2048 768 True 4 512 0.375 4.0 4.0 0.9765625 0.0 0.49410588440130926 2.75 6.814452474347134 0.271270751953125 uniform_random_logits 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 0.9952941013961014 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
55 0.025407999753952026 0.05510399863123894 0.031430399790406224 0.02798399981111288 0.007335050273982725 0.020479999482631683 0.03561599925160408 0.02275839988142252 0.021551999263465405 0.003545718365165811 0.0 0.0 0.0 0.0 0.0 0.833952009677887 0.894752025604248 0.8619967997074127 0.863215982913971 0.018716378851797198 128 128 128 1 standard_fused_topk logit_topk softmax_renorm True standalone_legacy vllm020_replicated_linear 8 2048 768 True 4 1024 0.375 8.0 8.0 1.0 0.25 0.33693529145074724 2.125 6.9186075263155535 0.1867218017578125 uniform_random_logits 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 0.9273256282883522 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
56 0.0244159996509552 0.05395200103521347 0.03188959984108806 0.026031999848783016 0.00943995927387483 0.02051199972629547 0.036607999354600906 0.023247999791055917 0.02147199958562851 0.003910623837069648 0.0 0.0 0.0 0.0 0.0 0.834879994392395 0.8871039748191833 0.8651552021503448 0.8638879954814911 0.015262430894400969 256 128 128 1 standard_fused_topk logit_topk softmax_renorm True standalone_legacy vllm020_replicated_linear 8 2048 768 True 4 2048 0.375 16.0 16.0 1.0 0.375 0.25567294018677456 1.8125 6.952441049154937 0.14349365234375 uniform_random_logits 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 1.0380599882396606 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
57 0.024512000381946564 0.04620800167322159 0.02884640023112297 0.026320000179111958 0.005516557583355925 0.02054399996995926 0.03846399858593941 0.023401600029319524 0.021263999864459038 0.0044742944198265374 0.0 0.0 0.0 0.0 0.0 0.8518080115318298 0.9097599983215332 0.8810272097587586 0.8751040101051331 0.017005819633271906 512 128 128 1 standard_fused_topk logit_topk softmax_renorm True standalone_legacy vllm020_replicated_linear 8 2048 768 True 4 4096 0.375 32.0 32.0 1.0 0.625 0.1747801353218523 1.53125 6.978069554482723 0.09820938110351562 uniform_random_logits 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 0.9569603278386984 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
58 0.02412799932062626 0.06940799951553345 0.030817600432783365 0.026800000108778477 0.010047085187652939 0.020128000527620316 0.044096000492572784 0.024606400076299904 0.022304000332951546 0.00622857166823714 0.0 0.0 0.0 0.0 0.0 0.8470079898834229 0.9010239839553833 0.8694015920162201 0.8716959953308105 0.016138881171190216 1024 128 128 1 standard_fused_topk logit_topk softmax_renorm True standalone_legacy vllm020_replicated_linear 8 2048 768 True 4 8192 0.375 64.0 64.0 1.0 0.65625 0.1158122428154187 1.3125 6.9901908183358845 0.06445503234863281 uniform_random_logits 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 0.9524274993623046 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
59 0.02831999957561493 0.043487999588251114 0.031744000129401685 0.02991999965161085 0.003973415836428909 0.02070399932563305 0.029343999922275543 0.022886400017887353 0.021743999794125557 0.0026873065045088873 0.0 0.0 0.0 0.0 0.0 1.1698240041732788 1.2311359643936157 1.1888479948043824 1.1890720129013062 0.017978797794492758 2048 128 128 1 standard_fused_topk logit_topk softmax_renorm True standalone_legacy vllm020_replicated_linear 8 2048 768 True 4 16384 0.375 128.0 128.0 1.0 0.78125 0.08347181893108634 1.1796875 6.994921772573154 0.046871185302734375 uniform_random_logits 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 0.8971818172100244 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
60 0.03830400109291077 0.06268800050020218 0.043337599746882914 0.040511999279260635 0.005823016247946116 0.023135999217629433 0.03747199848294258 0.025206399988383053 0.02393599972128868 0.003271381256231161 0.0 0.0 0.0 0.0 0.0 1.7702720165252686 1.8097599744796753 1.7919103980064393 1.7956640124320984 0.012641295676021557 4096 128 128 1 standard_fused_topk logit_topk softmax_renorm True standalone_legacy vllm020_replicated_linear 8 2048 768 True 4 32768 0.375 256.0 256.0 1.0 0.78125 0.06866734477822484 1.20703125 6.996602562938728 0.03801727294921875 uniform_random_logits 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 0.8113207890716184 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
61 0.05660799890756607 0.07932800054550171 0.06249920018017292 0.06039999984204769 0.005636461691480845 0.02956799976527691 0.03747199848294258 0.031126399897038935 0.030287999659776688 0.002157571006631541 0.0 0.0 0.0 0.0 0.0 2.968672037124634 3.0278079509735107 2.9899007797241213 2.9824799299240112 0.01816414122631667 8192 128 128 1 standard_fused_topk logit_topk softmax_renorm True standalone_legacy vllm020_replicated_linear 8 2048 768 True 4 65536 0.375 512.0 512.0 1.0 0.8984375 0.04399546833977376 1.126953125 6.998607314922362 0.024699926376342773 uniform_random_logits 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 0.883909666885606 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
62 0.02502400055527687 0.06092799827456474 0.033839999698102474 0.028672000393271446 0.010315255343709818 0.019360000267624855 0.0352960005402565 0.023424000293016434 0.022064000368118286 0.0038898102289194572 0.0 0.0 0.0 0.0 0.0 0.19574399292469025 0.2512960135936737 0.21939200013875962 0.2199999988079071 0.017532156418212565 1 128 128 1 standard_fused_topk fixed_hotset8 softmax_renorm False standalone_legacy vllm020_replicated_linear 8 2048 768 True 4 8 0.375 0.0625 0.0625 0.0625 0.0 3.872983346207417 16.0 3.0 0.9375 hotset8 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 1.0233364439829928 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
63 0.025280000641942024 0.05132799968123436 0.030641599837690593 0.027343999594449997 0.007562409232763304 0.020160000771284103 0.052960000932216644 0.024145600199699403 0.021424000151455402 0.007450198645599985 0.0 0.0 0.0 0.0 0.0 0.20483200252056122 0.24006399512290955 0.22215040028095245 0.22433599829673767 0.00969639786132892 8 128 128 1 standard_fused_topk fixed_hotset8 softmax_renorm False standalone_legacy vllm020_replicated_linear 8 2048 768 True 4 64 0.375 0.5 0.5 0.0625 0.0 3.872983346207417 16.0 3.0 0.9375 hotset8 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 0.9806430689981738 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
64 0.024831999093294144 0.046560000628232956 0.030371200107038022 0.02763199992477894 0.005878487205028602 0.020320000126957893 0.04560000076889992 0.02601920012384653 0.02270400058478117 0.00751129965906799 0.0 0.0 0.0 0.0 0.0 0.20559999346733093 0.24726399779319763 0.22126719802618028 0.22207999974489212 0.01328093478485499 16 128 128 1 standard_fused_topk fixed_hotset8 softmax_renorm False standalone_legacy vllm020_replicated_linear 8 2048 768 True 4 128 0.375 1.0 1.0 0.0625 0.0 3.872983346207417 16.0 3.0 0.9375 hotset8 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 0.9103623678483975 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
65 0.024288000538945198 0.049375999718904495 0.03086080001667142 0.0267359996214509 0.0070864531384997225 0.020479999482631683 0.030912000685930252 0.02274719988927245 0.021359999664127827 0.0029966813650672505 0.0 0.0 0.0 0.0 0.0 0.20003199577331543 0.2301120012998581 0.21453119963407516 0.21598400175571442 0.010402239855151332 32 128 128 1 standard_fused_topk fixed_hotset8 softmax_renorm False standalone_legacy vllm020_replicated_linear 8 2048 768 True 4 256 0.375 2.0 2.0 0.0625 0.0 3.872983346207417 16.0 3.0 0.9375 hotset8 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 0.9610778571819444 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
66 0.02393599972128868 0.04342399910092354 0.029380799923092126 0.026688000187277794 0.0057374782000526574 0.020031999796628952 0.036607999354600906 0.022487999964505435 0.020911999978125095 0.0038135203062103235 0.0 0.0 0.0 0.0 0.0 0.19551999866962433 0.22972799837589264 0.21238719969987868 0.21488000452518463 0.010706095835489097 64 128 128 1 standard_fused_topk fixed_hotset8 softmax_renorm False standalone_legacy vllm020_replicated_linear 8 2048 768 True 4 512 0.375 4.0 4.0 0.0625 0.0 3.872983346207417 16.0 3.0 0.9375 hotset8 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 0.9952941013961014 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
67 0.025407999753952026 0.05510399863123894 0.031430399790406224 0.02798399981111288 0.007335050273982725 0.020479999482631683 0.03561599925160408 0.02275839988142252 0.021551999263465405 0.003545718365165811 0.0 0.0 0.0 0.0 0.0 0.19420799612998962 0.2903999984264374 0.2211231991648674 0.21476799994707108 0.025955008886196004 128 128 128 1 standard_fused_topk fixed_hotset8 softmax_renorm False standalone_legacy vllm020_replicated_linear 8 2048 768 True 4 1024 0.375 8.0 8.0 0.0625 0.0 3.872983346207417 16.0 3.0 0.9375 hotset8 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 0.9273256282883522 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
68 0.0244159996509552 0.05395200103521347 0.03188959984108806 0.026031999848783016 0.00943995927387483 0.02051199972629547 0.036607999354600906 0.023247999791055917 0.02147199958562851 0.003910623837069648 0.0 0.0 0.0 0.0 0.0 0.2290560007095337 0.289247989654541 0.2543327987194061 0.24939200282096863 0.01663236753503115 256 128 128 1 standard_fused_topk fixed_hotset8 softmax_renorm False standalone_legacy vllm020_replicated_linear 8 2048 768 True 4 2048 0.375 16.0 16.0 0.0625 0.0 3.872983346207417 16.0 3.0 0.9375 hotset8 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 1.0380599882396606 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
69 0.024512000381946564 0.04620800167322159 0.02884640023112297 0.026320000179111958 0.005516557583355925 0.02054399996995926 0.03846399858593941 0.023401600029319524 0.021263999864459038 0.0044742944198265374 0.0 0.0 0.0 0.0 0.0 0.30831998586654663 0.36953601241111755 0.3324000000953674 0.3288639932870865 0.018617999572156707 512 128 128 1 standard_fused_topk fixed_hotset8 softmax_renorm False standalone_legacy vllm020_replicated_linear 8 2048 768 True 4 4096 0.375 32.0 32.0 0.0625 0.0 3.872983346207417 16.0 3.0 0.9375 hotset8 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 0.9569603278386984 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
70 0.02412799932062626 0.06940799951553345 0.030817600432783365 0.026800000108778477 0.010047085187652939 0.020128000527620316 0.044096000492572784 0.024606400076299904 0.022304000332951546 0.00622857166823714 0.0 0.0 0.0 0.0 0.0 0.462911993265152 0.5497919917106628 0.4893856018781662 0.4816960096359253 0.023348152887178286 1024 128 128 1 standard_fused_topk fixed_hotset8 softmax_renorm False standalone_legacy vllm020_replicated_linear 8 2048 768 True 4 8192 0.375 64.0 64.0 0.0625 0.0 3.872983346207417 16.0 3.0 0.9375 hotset8 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 0.9524274993623046 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
71 0.02831999957561493 0.043487999588251114 0.031744000129401685 0.02991999965161085 0.003973415836428909 0.02070399932563305 0.029343999922275543 0.022886400017887353 0.021743999794125557 0.0026873065045088873 0.0 0.0 0.0 0.0 0.0 0.7662079930305481 0.8717759847640991 0.788454395532608 0.7744799852371216 0.03174746482604812 2048 128 128 1 standard_fused_topk fixed_hotset8 softmax_renorm False standalone_legacy vllm020_replicated_linear 8 2048 768 True 4 16384 0.375 128.0 128.0 0.0625 0.0 3.872983346207417 16.0 3.0 0.9375 hotset8 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 0.8971818172100244 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
72 0.03830400109291077 0.06268800050020218 0.043337599746882914 0.040511999279260635 0.005823016247946116 0.023135999217629433 0.03747199848294258 0.025206399988383053 0.02393599972128868 0.003271381256231161 0.0 0.0 0.0 0.0 0.0 1.363935947418213 1.4143040180206299 1.3812703967094422 1.3798720240592957 0.014770450075530007 4096 128 128 1 standard_fused_topk fixed_hotset8 softmax_renorm False standalone_legacy vllm020_replicated_linear 8 2048 768 True 4 32768 0.375 256.0 256.0 0.0625 0.0 3.872983346207417 16.0 3.0 0.9375 hotset8 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 0.8113207890716184 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize
73 0.05660799890756607 0.07932800054550171 0.06249920018017292 0.06039999984204769 0.005636461691480845 0.02956799976527691 0.03747199848294258 0.031126399897038935 0.030287999659776688 0.002157571006631541 0.0 0.0 0.0 0.0 0.0 2.5507519245147705 2.680704116821289 2.579859209060669 2.566223978996277 0.03673283558365955 8192 128 128 1 standard_fused_topk fixed_hotset8 softmax_renorm False standalone_legacy vllm020_replicated_linear 8 2048 768 True 4 65536 0.375 512.0 512.0 0.0625 0.0 3.872983346207417 16.0 3.0 0.9375 hotset8 20260716 FlashInfer CUTLASS CUDA_EVENT BF16 generic none 0.883909666885606 measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize

49
runs/frontier-qwen30-vllm020-profile-v1/freeze_frontier_profiles.py
View File

@@ -644,16 +644,51 @@ def main() -> None:
args.output / "allreduce.json",
]
batch_composition_augmented = len(args.attention) > 3
long_context_augmented = any(
"long-context" in path.name for path in args.attention
)
long_context_coverage: dict[str, Any] = {"included": long_context_augmented}
if long_context_augmented:
with (args.output / "attention.csv").open(newline="") as handle:
frozen_attention = list(csv.DictReader(handle))
by_tp = {}
for tp in (1, 2, 4):
decode_kv = {
int(float(row["kv_cache_size"]))
for row in frozen_attention
if int(row["num_tensor_parallel_workers"]) == tp
and row["is_prefill"].lower() == "false"
}
mixed_kv = {
int(float(row["decode_avg_kv_cache_size"]))
for row in frozen_attention
if int(row["num_tensor_parallel_workers"]) == tp
and row.get("is_true_mixed_batch", "").lower() == "true"
}
if not {16384, 32768, 40960}.issubset(decode_kv):
raise ValueError(f"long-context decode coverage mismatch for TP{tp}")
if not {16384, 32768}.issubset(mixed_kv):
raise ValueError(f"long-context mixed coverage mismatch for TP{tp}")
by_tp[str(tp)] = {
"decode_kv_lengths": sorted(decode_kv),
"true_mixed_decode_avg_kv_lengths": sorted(mixed_kv),
}
long_context_coverage["by_tp"] = by_tp
manifest = {
"schema_version": (
"frontier_qwen30_vllm020_frozen_profile.v3"
if batch_composition_augmented
else "frontier_qwen30_vllm020_frozen_profile.v2"
"frontier_qwen30_vllm020_frozen_profile.v4"
if long_context_augmented
else (
"frontier_qwen30_vllm020_frozen_profile.v3"
if batch_composition_augmented
else "frontier_qwen30_vllm020_frozen_profile.v2"
)
),
"profile_id": (
"qwen3-30b-a3b-bf16-vllm020-h20-tp1-2-4-"
"fused-mixed-total-conserving"
+ ("-pure-prefill-batch-composition" if batch_composition_augmented else "")
+ ("-long-context-decode-mixed" if long_context_augmented else "")
),
"environment_contract": {
"hardware": "NVIDIA H20",
@@ -672,6 +707,7 @@ def main() -> None:
"allreduce": allreduce_rows,
},
"attention_tp_coverage": attention_tps,
"attention_long_context_coverage": long_context_coverage,
"projection_contract": {
"linear": "Frontier profiler using vLLM 0.20 CUDA operators",
"attention": (
@@ -692,6 +728,13 @@ def main() -> None:
if batch_composition_augmented
else "not included"
),
"attention_long_context": (
"Direct FA3 measurements at decode KV lengths 16K/32K/40K, "
"plus true-mixed prefill/decode measurements at 16K/32K, "
"for TP1/2/4"
if long_context_augmented
else "not included"
),
"moe": (
"Replicated gate and fused top-k plus TP-local modular expert kernel; "
"expert measurement already includes prepare/finalize so shuffling is zero"