gahow/aituner
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Record TP2 prefill serving-path verdict

Browse Source
This commit is contained in:
Gahow Wang
2026-07-23 18:08:32 +08:00
parent cf610003ed
commit 4f22688bfd
9 changed files with 2013 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

463
runs/frontier-tp2-prefill-serving-v0/results/serving-smoke.json Normal file
View File

@@ -0,0 +1,463 @@
{
"contract": {
"component_time": "sum of CUDA kernel duration within selected window",
"critical_path": "rank with largest selected execute wall",
"selection": "longest execute annotation per TP rank"
},
"critical_rank": {
"all_execute_windows": [
{
"duration_ms": 408.19065,
"name": "execute_context_1(8192)_generation_0(0)"
},
{
"duration_ms": 5.17557,
"name": "execute_context_0(0)_generation_1(1)"
}
],
"components_ms": {
"attention": 99.67212900000001,
"collective": 27.829565000000006,
"linear_norm_rope": 57.39125199999998,
"moe": 214.52378299999995,
"other": 1.9318720000000007,
"router": 0.7856679999999999
},
"execute_annotation_histogram": {
"execute_context_0(0)_generation_1(1)": 1,
"execute_context_1(8192)_generation_0(0)": 1
},
"execute_wall_ms": 408.19065,
"gpu_kernel_busy_ms": 402.13426899999996,
"kernel_rows": [
{
"duration_ms": 122.02009999999999,
"name": "void fused_moe::run_global<fused_moe::Fused_Moe_Kernel_sm80<cutlass::bfloat16_t, cutlass::bfloat16_t, cutlass::bfloat16_t, 32, 128, 64, 3, (fused_moe::Activation_Type)3> >(fused_moe::Fused_Moe_Kernel_sm80<cutlass::bfloat16_t, cutlass::bfloat16_t, cutlass::bfloat16_t, 32, 128, 64, 3, (fused_moe::Activation_Type)3>::Params)"
},
{
"duration_ms": 97.15557600000001,
"name": "void cutlass::device_kernel<flash::enable_sm90_or_later<flash::FlashAttnFwdSm90<flash::CollectiveMainloopFwdSm90<2, cute::tuple<cute::C<1>, cute::C<1>, cute::C<1> >, cute::tuple<cute::C<128>, cute::C<128>, cute::C<128> >, 128, cutlass::bfloat16_t, float, cutlass::arch::Sm90, true, false, false, true, true, false, false, true, true, true, false, false, cutlass::bfloat16_t, 8>, flash::CollectiveEpilogueFwd<cute::tuple<cute::C<128>, cute::C<128>, cute::C<128> >, cute::tuple<cute::C<1>, cute::C<1>, cute::C<1> >, cutlass::bfloat16_t, cutlass::arch::Sm90, 256, true, true, false, false, 8>, flash::VarlenDynamicPersistentTileScheduler<128, 128, 256, 128, false, true, true, true, false, true> > > >(flash::enable_sm90_or_later<flash::FlashAttnFwdSm90<flash::CollectiveMainloopFwdSm90<2, cute::tuple<cute::C<1>, cute::C<1>, cute::C<1> >, cute::tuple<cute::C<128>, cute::C<128>, cute::C<128> >, 128, cutlass::bfloat16_t, float, cutlass::arch::Sm90, true, false, false, true, true, false, false, true, true, true, false, false, cutlass::bfloat16_t, 8>, flash::CollectiveEpilogueFwd<cute::tuple<cute::C<128>, cute::C<128>, cute::C<128> >, cute::tuple<cute::C<1>, cute::C<1>, cute::C<1> >, cutlass::bfloat16_t, cutlass::arch::Sm90, 256, true, true, false, false, 8>, flash::VarlenDynamicPersistentTileScheduler<128, 128, 256, 128, false, true, true, true, false, true> > >::Params)"
},
{
"duration_ms": 80.135489,
"name": "_ZN7cutlass13device_kernelINS_4gemm6kernel13GemmUniversalINS1_17GroupProblemShapeIN4cute5tupleIJlllEEEEENS1_10collective13CollectiveMmaINS1_39MainloopSm90ArrayTmaGmmaWarpSpecializedILi12ENS6_IJNS5_1CILi1EEESD_SD_EEENS1_43KernelPtrArrayTmaWarpSpecializedCooperativeEEENS6_IJNSC_ILi128EEENSC_ILi16EEENSC_ILi64EEEEEENS_10bfloat16_tEPNS6_IJlSD_NSC_ILi0EEEEEESL_SO_NS5_8TiledMMAINS5_8MMA_AtomIJNS5_4SM904GMMA27MMA_64x16x16_F32BF16BF16_SSILNSS_5MajorE0ELSU_0ELNSS_7ScaleInE1ELSV_1EEEEEENS5_6LayoutINS6_IJNSC_ILi2EEESD_SD_EEENS6_IJSD_SM_SM_EEEEENS6_IJNS5_10UnderscoreES13_S13_EEEEENS5_13SM90_TMA_LOADENS5_14ComposedLayoutINS5_7SwizzleILi3ELi4ELi3EEENS5_18smem_ptr_flag_bitsILi16EEENSY_INS6_IJNSC_ILi8EEESJ_EEENS6_IJSJ_SD_EEEEEEEvNS5_8identityES16_S1G_vS1H_EENS_8epilogue10collective18CollectiveEpilogueINS1J_30Sm90PtrArrayTmaWarpSpecializedILi1ELi1ELi8ELb0ELb0ELi2EEEJSK_NS6_IJSH_SI_EEEvPNS6_IJSD_lSM_EEEvS1Q_NS1J_6fusion15FusionCallbacksIS1N_NS1R_37ScaledAccPerRowBiasPerColScaleScatterINS_6layout11ColumnMajorESL_fSL_ffLi8ELi8ELNS_15FloatRoundStyleE2EEESK_S1O_JNS17_IS19_S1B_NSY_INS6_IJSJ_S1C_EEENS6_IJSD_SJ_EEEEEEENS5_17SM90_U16x8_STSM_TEEEES16_S21_NS5_17SM75_U16x8_LDSM_TENS5_14SM90_TMA_STOREES21_S22_NS5_9Copy_AtomIJNS5_17SM90_U32x4_STSM_NENS_6half_tEEEEvEEEvvEEEEvNT_6ParamsE"
},
{
"duration_ms": 31.169920999999995,
"name": "nvjet_tst_320x128_64x3_1x2_h_bz_coopB_TNT"
},
{
"duration_ms": 27.829565000000006,
"name": "void flashinfer::trtllm_allreduce_fusion::allreduce_fusion_kernel_oneshot_lamport<(flashinfer::trtllm_allreduce_fusion::AllReduceFusionPattern)1, __nv_bfloat16, 2, true, true>(flashinfer::trtllm_allreduce_fusion::AllReduceFusionParams<__nv_bfloat16>)"
},
{
"duration_ms": 24.150314000000005,
"name": "nvjet_tst_128x192_64x5_2x1_v_bz_coopB_TNN"
},
{
"duration_ms": 9.48525,
"name": "void tensorrt_llm::kernels::cutlass_kernels::expandInputRowsKernel<__nv_bfloat16, __nv_bfloat16, (tensorrt_llm::kernels::cutlass_kernels::TmaWarpSpecializedGroupedGemmInput::FpXBlockScalingType)2, false>(__nv_bfloat16 const*, __nv_bfloat16*, float const*, float*, int const*, long, long, long, float const*, bool, long const*, unsigned char*, unsigned char const*, bool, long, __nv_bfloat16 const*)"
},
{
"duration_ms": 2.5150810000000003,
"name": "void vllm::reshape_and_cache_flash_kernel<__nv_bfloat16, __nv_bfloat16, (vllm::Fp8KVCacheDataType)0>(__nv_bfloat16 const*, __nv_bfloat16 const*, __nv_bfloat16*, __nv_bfloat16*, long const*, long, long, long, long, long, int, int, int, float const*, float const*, int)"
},
{
"duration_ms": 1.9769679999999998,
"name": "nvjet_tst_128x128_64x6_1x2_h_bz_TNT"
},
{
"duration_ms": 1.356286,
"name": "void tensorrt_llm::kernels::cutlass_kernels::blockExpertPrefixSumKernel<1024>(int const*, int*, int*, long, long, int)"
},
{
"duration_ms": 1.1989450000000001,
"name": "triton_poi_fused_1"
},
{
"duration_ms": 1.1752029999999996,
"name": "void tensorrt_llm::kernels::cutlass_kernels::computeStridesTmaWarpSpecializedKernel<__nv_bfloat16, __nv_bfloat16, __nv_bfloat16, __nv_bfloat16>(long const*, tensorrt_llm::kernels::cutlass_kernels::TmaWarpSpecializedGroupedGemmInput, tensorrt_llm::kernels::cutlass_kernels::TmaWarpSpecializedGroupedGemmInput, long, long, long, long, long, long, long, __nv_bfloat16 const*, __nv_bfloat16 const*, __nv_bfloat16 const*, __nv_bfloat16 const*, float const*, float const*, unsigned char const*, unsigned char const*, tensorrt_llm::kernels::cutlass_kernels::QuantParams, __nv_bfloat16 const*, __nv_bfloat16 const*, __nv_bfloat16*, __nv_bfloat16*, float const*, int const*)"
},
{
"duration_ms": 0.7856679999999999,
"name": "void vllm::moe::topkGating<8, 128, 4, 16, 32, int, __nv_bfloat16, (vllm::moe::ScoringFunc)0>(__nv_bfloat16 const*, bool const*, float*, int, int*, int*, int, int, int, bool, float const*)"
},
{
"duration_ms": 0.6054410000000001,
"name": "triton_red_fused_0"
},
{
"duration_ms": 0.264768,
"name": "tensorrt_llm::kernels::cutlass_kernels::mergeExpertPrefixSumKernel(int const*, int const*, int const*, int*, int*, int*, int)"
},
{
"duration_ms": 0.09404900000000001,
"name": "nvjet_tst_512x8_64x3_2x1_v_bz_TNT"
},
{
"duration_ms": 0.086687,
"name": "void tensorrt_llm::kernels::cutlass_kernels::globalExpertPrefixSumKernel<1024>(int const*, int*, long*, long, long)"
},
{
"duration_ms": 0.034815,
"name": "void at::native::vectorized_elementwise_kernel<4, at::native::FillFunctor<int>, std::array<char*, 1ul> >(int, at::native::FillFunctor<int>, std::array<char*, 1ul>)"
},
{
"duration_ms": 0.025472,
"name": "triton_poi_fused_2"
},
{
"duration_ms": 0.022335999999999998,
"name": "triton_poi_fused_0"
},
{
"duration_ms": 0.015392,
"name": "_compute_slot_mapping_kernel"
},
{
"duration_ms": 0.013439999999999999,
"name": "triton_red_fused_1"
},
{
"duration_ms": 0.004256,
"name": "void at::native::unrolled_elementwise_kernel<at::native::direct_copy_kernel_cuda(at::TensorIteratorBase&)::{lambda()#3}::operator()() const::{lambda()#4}::operator()() const::{lambda(long)#1}, std::array<char*, 2ul>, 4, TrivialOffsetCalculator<1, unsigned int>, TrivialOffsetCalculator<1, unsigned int>, at::native::memory::LoadWithCast<1>, at::native::memory::StoreWithCast<1> >(int, at::native::direct_copy_kernel_cuda(at::TensorIteratorBase&)::{lambda()#3}::operator()() const::{lambda()#4}::operator()() const::{lambda(long)#1}, std::array<char*, 2ul>, TrivialOffsetCalculator<1, unsigned int>, TrivialOffsetCalculator<1, unsigned int>, at::native::memory::LoadWithCast<1>, at::native::memory::StoreWithCast<1>)"
},
{
"duration_ms": 0.0041600000000000005,
"name": "void at::native::index_elementwise_kernel<128, 4, at::native::gpu_index_kernel<at::native::index_kernel_impl<at::native::OpaqueType<4> >(at::TensorIteratorBase&, c10::ArrayRef<long>, c10::ArrayRef<long>)::{lambda(char*, char const*, long)#1}>(at::TensorIteratorBase&, c10::ArrayRef<long>, c10::ArrayRef<long>, at::native::index_kernel_impl<at::native::OpaqueType<4> >(at::TensorIteratorBase&, c10::ArrayRef<long>, c10::ArrayRef<long>)::{lambda(char*, char const*, long)#1} const&, bool)::{lambda(int)#1}>(long, at::native::gpu_index_kernel<at::native::index_kernel_impl<at::native::OpaqueType<4> >(at::TensorIteratorBase&, c10::ArrayRef<long>, c10::ArrayRef<long>)::{lambda(char*, char const*, long)#1}>(at::TensorIteratorBase&, c10::ArrayRef<long>, c10::ArrayRef<long>, at::native::index_kernel_impl<at::native::OpaqueType<4> >(at::TensorIteratorBase&, c10::ArrayRef<long>, c10::ArrayRef<long>)::{lambda(char*, char const*, long)#1} const&, bool)::{lambda(int)#1})"
},
{
"duration_ms": 0.003072,
"name": "void at::native::index_elementwise_kernel<128, 4, at::native::gpu_index_kernel<at::native::index_kernel_impl<at::native::OpaqueType<2> >(at::TensorIteratorBase&, c10::ArrayRef<long>, c10::ArrayRef<long>)::{lambda(char*, char const*, long)#1}>(at::TensorIteratorBase&, c10::ArrayRef<long>, c10::ArrayRef<long>, at::native::index_kernel_impl<at::native::OpaqueType<2> >(at::TensorIteratorBase&, c10::ArrayRef<long>, c10::ArrayRef<long>)::{lambda(char*, char const*, long)#1} const&, bool)::{lambda(int)#1}>(long, at::native::gpu_index_kernel<at::native::index_kernel_impl<at::native::OpaqueType<2> >(at::TensorIteratorBase&, c10::ArrayRef<long>, c10::ArrayRef<long>)::{lambda(char*, char const*, long)#1}>(at::TensorIteratorBase&, c10::ArrayRef<long>, c10::ArrayRef<long>, at::native::index_kernel_impl<at::native::OpaqueType<2> >(at::TensorIteratorBase&, c10::ArrayRef<long>, c10::ArrayRef<long>)::{lambda(char*, char const*, long)#1} const&, bool)::{lambda(int)#1})"
},
{
"duration_ms": 0.001472,
"name": "void at::native::vectorized_elementwise_kernel<2, at::native::CUDAFunctor_add<long>, std::array<char*, 3ul> >(int, at::native::CUDAFunctor_add<long>, std::array<char*, 3ul>)"
},
{
"duration_ms": 0.001472,
"name": "void flash::prepare_varlen_num_blocks_kernel<1, false>(int, int, int, int const*, int const*, int const*, int const*, int const*, int const*, int, int, int, int, int, cutlass::FastDivmod, cutlass::FastDivmod, int*, int*, int*, int*, int*, bool, bool, bool, int)"
},
{
"duration_ms": 0.001184,
"name": "void at::native::unrolled_elementwise_kernel<at::native::FillFunctor<int>, std::array<char*, 1ul>, 4, TrivialOffsetCalculator<0, unsigned int>, TrivialOffsetCalculator<1, unsigned int>, at::native::memory::LoadWithoutCast, at::native::memory::StoreWithoutCast>(int, at::native::FillFunctor<int>, std::array<char*, 1ul>, TrivialOffsetCalculator<0, unsigned int>, TrivialOffsetCalculator<1, unsigned int>, at::native::memory::LoadWithoutCast, at::native::memory::StoreWithoutCast)"
},
{
"duration_ms": 0.0011200000000000001,
"name": "void at::native::vectorized_elementwise_kernel<4, at::native::CUDAFunctor_add<int>, std::array<char*, 3ul> >(int, at::native::CUDAFunctor_add<int>, std::array<char*, 3ul>)"
},
{
"duration_ms": 0.000767,
"name": "void at::native::unrolled_elementwise_kernel<at::native::CUDAFunctorOnSelf_add<int>, std::array<char*, 2ul>, 4, TrivialOffsetCalculator<1, unsigned int>, TrivialOffsetCalculator<1, unsigned int>, at::native::memory::LoadWithoutCast, at::native::memory::StoreWithoutCast>(int, at::native::CUDAFunctorOnSelf_add<int>, std::array<char*, 2ul>, TrivialOffsetCalculator<1, unsigned int>, TrivialOffsetCalculator<1, unsigned int>, at::native::memory::LoadWithoutCast, at::native::memory::StoreWithoutCast)"
}
],
"non_kernel_gap_ms": 6.056381000000044,
"selected_execute_annotation": "execute_context_1(8192)_generation_0(0)",
"trace": "runs/frontier-tp2-prefill-serving-v0/fleet-artifacts/tp2-prefill-serving-smoke-20260723-v1-20260723T082454699036Z/artifacts/runs/frontier-tp2-prefill-serving-v0/remote-outputs/tp2-smoke-r1/traces/profile/dp0_pp0_tp0_dcp0_ep0_rank0.1784795312743204872.pt.trace.json.gz"
},
"ranks": [
{
"all_execute_windows": [
{
"duration_ms": 408.19065,
"name": "execute_context_1(8192)_generation_0(0)"
},
{
"duration_ms": 5.17557,
"name": "execute_context_0(0)_generation_1(1)"
}
],
"components_ms": {
"attention": 99.67212900000001,
"collective": 27.829565000000006,
"linear_norm_rope": 57.39125199999998,
"moe": 214.52378299999995,
"other": 1.9318720000000007,
"router": 0.7856679999999999
},
"execute_annotation_histogram": {
"execute_context_0(0)_generation_1(1)": 1,
"execute_context_1(8192)_generation_0(0)": 1
},
"execute_wall_ms": 408.19065,
"gpu_kernel_busy_ms": 402.13426899999996,
"kernel_rows": [
{
"duration_ms": 122.02009999999999,
"name": "void fused_moe::run_global<fused_moe::Fused_Moe_Kernel_sm80<cutlass::bfloat16_t, cutlass::bfloat16_t, cutlass::bfloat16_t, 32, 128, 64, 3, (fused_moe::Activation_Type)3> >(fused_moe::Fused_Moe_Kernel_sm80<cutlass::bfloat16_t, cutlass::bfloat16_t, cutlass::bfloat16_t, 32, 128, 64, 3, (fused_moe::Activation_Type)3>::Params)"
},
{
"duration_ms": 97.15557600000001,
"name": "void cutlass::device_kernel<flash::enable_sm90_or_later<flash::FlashAttnFwdSm90<flash::CollectiveMainloopFwdSm90<2, cute::tuple<cute::C<1>, cute::C<1>, cute::C<1> >, cute::tuple<cute::C<128>, cute::C<128>, cute::C<128> >, 128, cutlass::bfloat16_t, float, cutlass::arch::Sm90, true, false, false, true, true, false, false, true, true, true, false, false, cutlass::bfloat16_t, 8>, flash::CollectiveEpilogueFwd<cute::tuple<cute::C<128>, cute::C<128>, cute::C<128> >, cute::tuple<cute::C<1>, cute::C<1>, cute::C<1> >, cutlass::bfloat16_t, cutlass::arch::Sm90, 256, true, true, false, false, 8>, flash::VarlenDynamicPersistentTileScheduler<128, 128, 256, 128, false, true, true, true, false, true> > > >(flash::enable_sm90_or_later<flash::FlashAttnFwdSm90<flash::CollectiveMainloopFwdSm90<2, cute::tuple<cute::C<1>, cute::C<1>, cute::C<1> >, cute::tuple<cute::C<128>, cute::C<128>, cute::C<128> >, 128, cutlass::bfloat16_t, float, cutlass::arch::Sm90, true, false, false, true, true, false, false, true, true, true, false, false, cutlass::bfloat16_t, 8>, flash::CollectiveEpilogueFwd<cute::tuple<cute::C<128>, cute::C<128>, cute::C<128> >, cute::tuple<cute::C<1>, cute::C<1>, cute::C<1> >, cutlass::bfloat16_t, cutlass::arch::Sm90, 256, true, true, false, false, 8>, flash::VarlenDynamicPersistentTileScheduler<128, 128, 256, 128, false, true, true, true, false, true> > >::Params)"
},
{
"duration_ms": 80.135489,
"name": "_ZN7cutlass13device_kernelINS_4gemm6kernel13GemmUniversalINS1_17GroupProblemShapeIN4cute5tupleIJlllEEEEENS1_10collective13CollectiveMmaINS1_39MainloopSm90ArrayTmaGmmaWarpSpecializedILi12ENS6_IJNS5_1CILi1EEESD_SD_EEENS1_43KernelPtrArrayTmaWarpSpecializedCooperativeEEENS6_IJNSC_ILi128EEENSC_ILi16EEENSC_ILi64EEEEEENS_10bfloat16_tEPNS6_IJlSD_NSC_ILi0EEEEEESL_SO_NS5_8TiledMMAINS5_8MMA_AtomIJNS5_4SM904GMMA27MMA_64x16x16_F32BF16BF16_SSILNSS_5MajorE0ELSU_0ELNSS_7ScaleInE1ELSV_1EEEEEENS5_6LayoutINS6_IJNSC_ILi2EEESD_SD_EEENS6_IJSD_SM_SM_EEEEENS6_IJNS5_10UnderscoreES13_S13_EEEEENS5_13SM90_TMA_LOADENS5_14ComposedLayoutINS5_7SwizzleILi3ELi4ELi3EEENS5_18smem_ptr_flag_bitsILi16EEENSY_INS6_IJNSC_ILi8EEESJ_EEENS6_IJSJ_SD_EEEEEEEvNS5_8identityES16_S1G_vS1H_EENS_8epilogue10collective18CollectiveEpilogueINS1J_30Sm90PtrArrayTmaWarpSpecializedILi1ELi1ELi8ELb0ELb0ELi2EEEJSK_NS6_IJSH_SI_EEEvPNS6_IJSD_lSM_EEEvS1Q_NS1J_6fusion15FusionCallbacksIS1N_NS1R_37ScaledAccPerRowBiasPerColScaleScatterINS_6layout11ColumnMajorESL_fSL_ffLi8ELi8ELNS_15FloatRoundStyleE2EEESK_S1O_JNS17_IS19_S1B_NSY_INS6_IJSJ_S1C_EEENS6_IJSD_SJ_EEEEEEENS5_17SM90_U16x8_STSM_TEEEES16_S21_NS5_17SM75_U16x8_LDSM_TENS5_14SM90_TMA_STOREES21_S22_NS5_9Copy_AtomIJNS5_17SM90_U32x4_STSM_NENS_6half_tEEEEvEEEvvEEEEvNT_6ParamsE"
},
{
"duration_ms": 31.169920999999995,
"name": "nvjet_tst_320x128_64x3_1x2_h_bz_coopB_TNT"
},
{
"duration_ms": 27.829565000000006,
"name": "void flashinfer::trtllm_allreduce_fusion::allreduce_fusion_kernel_oneshot_lamport<(flashinfer::trtllm_allreduce_fusion::AllReduceFusionPattern)1, __nv_bfloat16, 2, true, true>(flashinfer::trtllm_allreduce_fusion::AllReduceFusionParams<__nv_bfloat16>)"
},
{
"duration_ms": 24.150314000000005,
"name": "nvjet_tst_128x192_64x5_2x1_v_bz_coopB_TNN"
},
{
"duration_ms": 9.48525,
"name": "void tensorrt_llm::kernels::cutlass_kernels::expandInputRowsKernel<__nv_bfloat16, __nv_bfloat16, (tensorrt_llm::kernels::cutlass_kernels::TmaWarpSpecializedGroupedGemmInput::FpXBlockScalingType)2, false>(__nv_bfloat16 const*, __nv_bfloat16*, float const*, float*, int const*, long, long, long, float const*, bool, long const*, unsigned char*, unsigned char const*, bool, long, __nv_bfloat16 const*)"
},
{
"duration_ms": 2.5150810000000003,
"name": "void vllm::reshape_and_cache_flash_kernel<__nv_bfloat16, __nv_bfloat16, (vllm::Fp8KVCacheDataType)0>(__nv_bfloat16 const*, __nv_bfloat16 const*, __nv_bfloat16*, __nv_bfloat16*, long const*, long, long, long, long, long, int, int, int, float const*, float const*, int)"
},
{
"duration_ms": 1.9769679999999998,
"name": "nvjet_tst_128x128_64x6_1x2_h_bz_TNT"
},
{
"duration_ms": 1.356286,
"name": "void tensorrt_llm::kernels::cutlass_kernels::blockExpertPrefixSumKernel<1024>(int const*, int*, int*, long, long, int)"
},
{
"duration_ms": 1.1989450000000001,
"name": "triton_poi_fused_1"
},
{
"duration_ms": 1.1752029999999996,
"name": "void tensorrt_llm::kernels::cutlass_kernels::computeStridesTmaWarpSpecializedKernel<__nv_bfloat16, __nv_bfloat16, __nv_bfloat16, __nv_bfloat16>(long const*, tensorrt_llm::kernels::cutlass_kernels::TmaWarpSpecializedGroupedGemmInput, tensorrt_llm::kernels::cutlass_kernels::TmaWarpSpecializedGroupedGemmInput, long, long, long, long, long, long, long, __nv_bfloat16 const*, __nv_bfloat16 const*, __nv_bfloat16 const*, __nv_bfloat16 const*, float const*, float const*, unsigned char const*, unsigned char const*, tensorrt_llm::kernels::cutlass_kernels::QuantParams, __nv_bfloat16 const*, __nv_bfloat16 const*, __nv_bfloat16*, __nv_bfloat16*, float const*, int const*)"
},
{
"duration_ms": 0.7856679999999999,
"name": "void vllm::moe::topkGating<8, 128, 4, 16, 32, int, __nv_bfloat16, (vllm::moe::ScoringFunc)0>(__nv_bfloat16 const*, bool const*, float*, int, int*, int*, int, int, int, bool, float const*)"
},
{
"duration_ms": 0.6054410000000001,
"name": "triton_red_fused_0"
},
{
"duration_ms": 0.264768,
"name": "tensorrt_llm::kernels::cutlass_kernels::mergeExpertPrefixSumKernel(int const*, int const*, int const*, int*, int*, int*, int)"
},
{
"duration_ms": 0.09404900000000001,
"name": "nvjet_tst_512x8_64x3_2x1_v_bz_TNT"
},
{
"duration_ms": 0.086687,
"name": "void tensorrt_llm::kernels::cutlass_kernels::globalExpertPrefixSumKernel<1024>(int const*, int*, long*, long, long)"
},
{
"duration_ms": 0.034815,
"name": "void at::native::vectorized_elementwise_kernel<4, at::native::FillFunctor<int>, std::array<char*, 1ul> >(int, at::native::FillFunctor<int>, std::array<char*, 1ul>)"
},
{
"duration_ms": 0.025472,
"name": "triton_poi_fused_2"
},
{
"duration_ms": 0.022335999999999998,
"name": "triton_poi_fused_0"
},
{
"duration_ms": 0.015392,
"name": "_compute_slot_mapping_kernel"
},
{
"duration_ms": 0.013439999999999999,
"name": "triton_red_fused_1"
},
{
"duration_ms": 0.004256,
"name": "void at::native::unrolled_elementwise_kernel<at::native::direct_copy_kernel_cuda(at::TensorIteratorBase&)::{lambda()#3}::operator()() const::{lambda()#4}::operator()() const::{lambda(long)#1}, std::array<char*, 2ul>, 4, TrivialOffsetCalculator<1, unsigned int>, TrivialOffsetCalculator<1, unsigned int>, at::native::memory::LoadWithCast<1>, at::native::memory::StoreWithCast<1> >(int, at::native::direct_copy_kernel_cuda(at::TensorIteratorBase&)::{lambda()#3}::operator()() const::{lambda()#4}::operator()() const::{lambda(long)#1}, std::array<char*, 2ul>, TrivialOffsetCalculator<1, unsigned int>, TrivialOffsetCalculator<1, unsigned int>, at::native::memory::LoadWithCast<1>, at::native::memory::StoreWithCast<1>)"
},
{
"duration_ms": 0.0041600000000000005,
"name": "void at::native::index_elementwise_kernel<128, 4, at::native::gpu_index_kernel<at::native::index_kernel_impl<at::native::OpaqueType<4> >(at::TensorIteratorBase&, c10::ArrayRef<long>, c10::ArrayRef<long>)::{lambda(char*, char const*, long)#1}>(at::TensorIteratorBase&, c10::ArrayRef<long>, c10::ArrayRef<long>, at::native::index_kernel_impl<at::native::OpaqueType<4> >(at::TensorIteratorBase&, c10::ArrayRef<long>, c10::ArrayRef<long>)::{lambda(char*, char const*, long)#1} const&, bool)::{lambda(int)#1}>(long, at::native::gpu_index_kernel<at::native::index_kernel_impl<at::native::OpaqueType<4> >(at::TensorIteratorBase&, c10::ArrayRef<long>, c10::ArrayRef<long>)::{lambda(char*, char const*, long)#1}>(at::TensorIteratorBase&, c10::ArrayRef<long>, c10::ArrayRef<long>, at::native::index_kernel_impl<at::native::OpaqueType<4> >(at::TensorIteratorBase&, c10::ArrayRef<long>, c10::ArrayRef<long>)::{lambda(char*, char const*, long)#1} const&, bool)::{lambda(int)#1})"
},
{
"duration_ms": 0.003072,
"name": "void at::native::index_elementwise_kernel<128, 4, at::native::gpu_index_kernel<at::native::index_kernel_impl<at::native::OpaqueType<2> >(at::TensorIteratorBase&, c10::ArrayRef<long>, c10::ArrayRef<long>)::{lambda(char*, char const*, long)#1}>(at::TensorIteratorBase&, c10::ArrayRef<long>, c10::ArrayRef<long>, at::native::index_kernel_impl<at::native::OpaqueType<2> >(at::TensorIteratorBase&, c10::ArrayRef<long>, c10::ArrayRef<long>)::{lambda(char*, char const*, long)#1} const&, bool)::{lambda(int)#1}>(long, at::native::gpu_index_kernel<at::native::index_kernel_impl<at::native::OpaqueType<2> >(at::TensorIteratorBase&, c10::ArrayRef<long>, c10::ArrayRef<long>)::{lambda(char*, char const*, long)#1}>(at::TensorIteratorBase&, c10::ArrayRef<long>, c10::ArrayRef<long>, at::native::index_kernel_impl<at::native::OpaqueType<2> >(at::TensorIteratorBase&, c10::ArrayRef<long>, c10::ArrayRef<long>)::{lambda(char*, char const*, long)#1} const&, bool)::{lambda(int)#1})"
},
{
"duration_ms": 0.001472,
"name": "void at::native::vectorized_elementwise_kernel<2, at::native::CUDAFunctor_add<long>, std::array<char*, 3ul> >(int, at::native::CUDAFunctor_add<long>, std::array<char*, 3ul>)"
},
{
"duration_ms": 0.001472,
"name": "void flash::prepare_varlen_num_blocks_kernel<1, false>(int, int, int, int const*, int const*, int const*, int const*, int const*, int const*, int, int, int, int, int, cutlass::FastDivmod, cutlass::FastDivmod, int*, int*, int*, int*, int*, bool, bool, bool, int)"
},
{
"duration_ms": 0.001184,
"name": "void at::native::unrolled_elementwise_kernel<at::native::FillFunctor<int>, std::array<char*, 1ul>, 4, TrivialOffsetCalculator<0, unsigned int>, TrivialOffsetCalculator<1, unsigned int>, at::native::memory::LoadWithoutCast, at::native::memory::StoreWithoutCast>(int, at::native::FillFunctor<int>, std::array<char*, 1ul>, TrivialOffsetCalculator<0, unsigned int>, TrivialOffsetCalculator<1, unsigned int>, at::native::memory::LoadWithoutCast, at::native::memory::StoreWithoutCast)"
},
{
"duration_ms": 0.0011200000000000001,
"name": "void at::native::vectorized_elementwise_kernel<4, at::native::CUDAFunctor_add<int>, std::array<char*, 3ul> >(int, at::native::CUDAFunctor_add<int>, std::array<char*, 3ul>)"
},
{
"duration_ms": 0.000767,
"name": "void at::native::unrolled_elementwise_kernel<at::native::CUDAFunctorOnSelf_add<int>, std::array<char*, 2ul>, 4, TrivialOffsetCalculator<1, unsigned int>, TrivialOffsetCalculator<1, unsigned int>, at::native::memory::LoadWithoutCast, at::native::memory::StoreWithoutCast>(int, at::native::CUDAFunctorOnSelf_add<int>, std::array<char*, 2ul>, TrivialOffsetCalculator<1, unsigned int>, TrivialOffsetCalculator<1, unsigned int>, at::native::memory::LoadWithoutCast, at::native::memory::StoreWithoutCast)"
}
],
"non_kernel_gap_ms": 6.056381000000044,
"selected_execute_annotation": "execute_context_1(8192)_generation_0(0)",
"trace": "runs/frontier-tp2-prefill-serving-v0/fleet-artifacts/tp2-prefill-serving-smoke-20260723-v1-20260723T082454699036Z/artifacts/runs/frontier-tp2-prefill-serving-v0/remote-outputs/tp2-smoke-r1/traces/profile/dp0_pp0_tp0_dcp0_ep0_rank0.1784795312743204872.pt.trace.json.gz"
},
{
"all_execute_windows": [
{
"duration_ms": 407.337292,
"name": "execute_context_1(8192)_generation_0(0)"
},
{
"duration_ms": 5.175196,
"name": "execute_context_0(0)_generation_1(1)"
}
],
"components_ms": {
"attention": 99.75531199999998,
"collective": 25.666238000000003,
"linear_norm_rope": 57.43388800000001,
"moe": 214.55563300000009,
"other": 1.9427889999999999,
"router": 0.7898920000000001
},
"execute_annotation_histogram": {
"execute_context_0(0)_generation_1(1)": 1,
"execute_context_1(8192)_generation_0(0)": 1
},
"execute_wall_ms": 407.337292,
"gpu_kernel_busy_ms": 400.14375200000006,
"kernel_rows": [
{
"duration_ms": 122.088157,
"name": "void fused_moe::run_global<fused_moe::Fused_Moe_Kernel_sm80<cutlass::bfloat16_t, cutlass::bfloat16_t, cutlass::bfloat16_t, 32, 128, 64, 3, (fused_moe::Activation_Type)3> >(fused_moe::Fused_Moe_Kernel_sm80<cutlass::bfloat16_t, cutlass::bfloat16_t, cutlass::bfloat16_t, 32, 128, 64, 3, (fused_moe::Activation_Type)3>::Params)"
},
{
"duration_ms": 97.24454899999998,
"name": "void cutlass::device_kernel<flash::enable_sm90_or_later<flash::FlashAttnFwdSm90<flash::CollectiveMainloopFwdSm90<2, cute::tuple<cute::C<1>, cute::C<1>, cute::C<1> >, cute::tuple<cute::C<128>, cute::C<128>, cute::C<128> >, 128, cutlass::bfloat16_t, float, cutlass::arch::Sm90, true, false, false, true, true, false, false, true, true, true, false, false, cutlass::bfloat16_t, 8>, flash::CollectiveEpilogueFwd<cute::tuple<cute::C<128>, cute::C<128>, cute::C<128> >, cute::tuple<cute::C<1>, cute::C<1>, cute::C<1> >, cutlass::bfloat16_t, cutlass::arch::Sm90, 256, true, true, false, false, 8>, flash::VarlenDynamicPersistentTileScheduler<128, 128, 256, 128, false, true, true, true, false, true> > > >(flash::enable_sm90_or_later<flash::FlashAttnFwdSm90<flash::CollectiveMainloopFwdSm90<2, cute::tuple<cute::C<1>, cute::C<1>, cute::C<1> >, cute::tuple<cute::C<128>, cute::C<128>, cute::C<128> >, 128, cutlass::bfloat16_t, float, cutlass::arch::Sm90, true, false, false, true, true, false, false, true, true, true, false, false, cutlass::bfloat16_t, 8>, flash::CollectiveEpilogueFwd<cute::tuple<cute::C<128>, cute::C<128>, cute::C<128> >, cute::tuple<cute::C<1>, cute::C<1>, cute::C<1> >, cutlass::bfloat16_t, cutlass::arch::Sm90, 256, true, true, false, false, 8>, flash::VarlenDynamicPersistentTileScheduler<128, 128, 256, 128, false, true, true, true, false, true> > >::Params)"
},
{
"duration_ms": 80.12219199999998,
"name": "_ZN7cutlass13device_kernelINS_4gemm6kernel13GemmUniversalINS1_17GroupProblemShapeIN4cute5tupleIJlllEEEEENS1_10collective13CollectiveMmaINS1_39MainloopSm90ArrayTmaGmmaWarpSpecializedILi12ENS6_IJNS5_1CILi1EEESD_SD_EEENS1_43KernelPtrArrayTmaWarpSpecializedCooperativeEEENS6_IJNSC_ILi128EEENSC_ILi16EEENSC_ILi64EEEEEENS_10bfloat16_tEPNS6_IJlSD_NSC_ILi0EEEEEESL_SO_NS5_8TiledMMAINS5_8MMA_AtomIJNS5_4SM904GMMA27MMA_64x16x16_F32BF16BF16_SSILNSS_5MajorE0ELSU_0ELNSS_7ScaleInE1ELSV_1EEEEEENS5_6LayoutINS6_IJNSC_ILi2EEESD_SD_EEENS6_IJSD_SM_SM_EEEEENS6_IJNS5_10UnderscoreES13_S13_EEEEENS5_13SM90_TMA_LOADENS5_14ComposedLayoutINS5_7SwizzleILi3ELi4ELi3EEENS5_18smem_ptr_flag_bitsILi16EEENSY_INS6_IJNSC_ILi8EEESJ_EEENS6_IJSJ_SD_EEEEEEEvNS5_8identityES16_S1G_vS1H_EENS_8epilogue10collective18CollectiveEpilogueINS1J_30Sm90PtrArrayTmaWarpSpecializedILi1ELi1ELi8ELb0ELb0ELi2EEEJSK_NS6_IJSH_SI_EEEvPNS6_IJSD_lSM_EEEvS1Q_NS1J_6fusion15FusionCallbacksIS1N_NS1R_37ScaledAccPerRowBiasPerColScaleScatterINS_6layout11ColumnMajorESL_fSL_ffLi8ELi8ELNS_15FloatRoundStyleE2EEESK_S1O_JNS17_IS19_S1B_NSY_INS6_IJSJ_S1C_EEENS6_IJSD_SJ_EEEEEEENS5_17SM90_U16x8_STSM_TEEEES16_S21_NS5_17SM75_U16x8_LDSM_TENS5_14SM90_TMA_STOREES21_S22_NS5_9Copy_AtomIJNS5_17SM90_U32x4_STSM_NENS_6half_tEEEEvEEEvvEEEEvNT_6ParamsE"
},
{
"duration_ms": 31.199706,
"name": "nvjet_tst_320x128_64x3_1x2_h_bz_coopB_TNT"
},
{
"duration_ms": 25.666238000000003,
"name": "void flashinfer::trtllm_allreduce_fusion::allreduce_fusion_kernel_oneshot_lamport<(flashinfer::trtllm_allreduce_fusion::AllReduceFusionPattern)1, __nv_bfloat16, 2, true, true>(flashinfer::trtllm_allreduce_fusion::AllReduceFusionParams<__nv_bfloat16>)"
},
{
"duration_ms": 24.162201000000007,
"name": "nvjet_tst_128x192_64x5_2x1_v_bz_coopB_TNN"
},
{
"duration_ms": 9.475503000000002,
"name": "void tensorrt_llm::kernels::cutlass_kernels::expandInputRowsKernel<__nv_bfloat16, __nv_bfloat16, (tensorrt_llm::kernels::cutlass_kernels::TmaWarpSpecializedGroupedGemmInput::FpXBlockScalingType)2, false>(__nv_bfloat16 const*, __nv_bfloat16*, float const*, float*, int const*, long, long, long, float const*, bool, long const*, unsigned char*, unsigned char const*, bool, long, __nv_bfloat16 const*)"
},
{
"duration_ms": 2.5092910000000006,
"name": "void vllm::reshape_and_cache_flash_kernel<__nv_bfloat16, __nv_bfloat16, (vllm::Fp8KVCacheDataType)0>(__nv_bfloat16 const*, __nv_bfloat16 const*, __nv_bfloat16*, __nv_bfloat16*, long const*, long, long, long, long, long, int, int, int, float const*, float const*, int)"
},
{
"duration_ms": 1.9775490000000004,
"name": "nvjet_tst_128x128_64x6_1x2_h_bz_TNT"
},
{
"duration_ms": 1.3556549999999998,
"name": "void tensorrt_llm::kernels::cutlass_kernels::blockExpertPrefixSumKernel<1024>(int const*, int*, int*, long, long, int)"
},
{
"duration_ms": 1.2008649999999996,
"name": "triton_poi_fused_1"
},
{
"duration_ms": 1.164903,
"name": "void tensorrt_llm::kernels::cutlass_kernels::computeStridesTmaWarpSpecializedKernel<__nv_bfloat16, __nv_bfloat16, __nv_bfloat16, __nv_bfloat16>(long const*, tensorrt_llm::kernels::cutlass_kernels::TmaWarpSpecializedGroupedGemmInput, tensorrt_llm::kernels::cutlass_kernels::TmaWarpSpecializedGroupedGemmInput, long, long, long, long, long, long, long, __nv_bfloat16 const*, __nv_bfloat16 const*, __nv_bfloat16 const*, __nv_bfloat16 const*, float const*, float const*, unsigned char const*, unsigned char const*, tensorrt_llm::kernels::cutlass_kernels::QuantParams, __nv_bfloat16 const*, __nv_bfloat16 const*, __nv_bfloat16*, __nv_bfloat16*, float const*, int const*)"
},
{
"duration_ms": 0.7898920000000001,
"name": "void vllm::moe::topkGating<8, 128, 4, 16, 32, int, __nv_bfloat16, (vllm::moe::ScoringFunc)0>(__nv_bfloat16 const*, bool const*, float*, int, int*, int*, int, int, int, bool, float const*)"
},
{
"duration_ms": 0.6094719999999998,
"name": "triton_red_fused_0"
},
{
"duration_ms": 0.2641649999999999,
"name": "tensorrt_llm::kernels::cutlass_kernels::mergeExpertPrefixSumKernel(int const*, int const*, int const*, int*, int*, int*, int)"
},
{
"duration_ms": 0.094432,
"name": "nvjet_tst_512x8_64x3_2x1_v_bz_TNT"
},
{
"duration_ms": 0.08505799999999997,
"name": "void tensorrt_llm::kernels::cutlass_kernels::globalExpertPrefixSumKernel<1024>(int const*, int*, long*, long, long)"
},
{
"duration_ms": 0.034820000000000004,
"name": "void at::native::vectorized_elementwise_kernel<4, at::native::FillFunctor<int>, std::array<char*, 1ul> >(int, at::native::FillFunctor<int>, std::array<char*, 1ul>)"
},
{
"duration_ms": 0.027615999999999998,
"name": "triton_poi_fused_0"
},
{
"duration_ms": 0.025792000000000002,
"name": "triton_poi_fused_2"
},
{
"duration_ms": 0.015392,
"name": "_compute_slot_mapping_kernel"
},
{
"duration_ms": 0.012928,
"name": "triton_red_fused_1"
},
{
"duration_ms": 0.004096,
"name": "void at::native::unrolled_elementwise_kernel<at::native::direct_copy_kernel_cuda(at::TensorIteratorBase&)::{lambda()#3}::operator()() const::{lambda()#4}::operator()() const::{lambda(long)#1}, std::array<char*, 2ul>, 4, TrivialOffsetCalculator<1, unsigned int>, TrivialOffsetCalculator<1, unsigned int>, at::native::memory::LoadWithCast<1>, at::native::memory::StoreWithCast<1> >(int, at::native::direct_copy_kernel_cuda(at::TensorIteratorBase&)::{lambda()#3}::operator()() const::{lambda()#4}::operator()() const::{lambda(long)#1}, std::array<char*, 2ul>, TrivialOffsetCalculator<1, unsigned int>, TrivialOffsetCalculator<1, unsigned int>, at::native::memory::LoadWithCast<1>, at::native::memory::StoreWithCast<1>)"
},
{
"duration_ms": 0.004,
"name": "void at::native::index_elementwise_kernel<128, 4, at::native::gpu_index_kernel<at::native::index_kernel_impl<at::native::OpaqueType<4> >(at::TensorIteratorBase&, c10::ArrayRef<long>, c10::ArrayRef<long>)::{lambda(char*, char const*, long)#1}>(at::TensorIteratorBase&, c10::ArrayRef<long>, c10::ArrayRef<long>, at::native::index_kernel_impl<at::native::OpaqueType<4> >(at::TensorIteratorBase&, c10::ArrayRef<long>, c10::ArrayRef<long>)::{lambda(char*, char const*, long)#1} const&, bool)::{lambda(int)#1}>(long, at::native::gpu_index_kernel<at::native::index_kernel_impl<at::native::OpaqueType<4> >(at::TensorIteratorBase&, c10::ArrayRef<long>, c10::ArrayRef<long>)::{lambda(char*, char const*, long)#1}>(at::TensorIteratorBase&, c10::ArrayRef<long>, c10::ArrayRef<long>, at::native::index_kernel_impl<at::native::OpaqueType<4> >(at::TensorIteratorBase&, c10::ArrayRef<long>, c10::ArrayRef<long>)::{lambda(char*, char const*, long)#1} const&, bool)::{lambda(int)#1})"
},
{
"duration_ms": 0.003328,
"name": "void at::native::index_elementwise_kernel<128, 4, at::native::gpu_index_kernel<at::native::index_kernel_impl<at::native::OpaqueType<2> >(at::TensorIteratorBase&, c10::ArrayRef<long>, c10::ArrayRef<long>)::{lambda(char*, char const*, long)#1}>(at::TensorIteratorBase&, c10::ArrayRef<long>, c10::ArrayRef<long>, at::native::index_kernel_impl<at::native::OpaqueType<2> >(at::TensorIteratorBase&, c10::ArrayRef<long>, c10::ArrayRef<long>)::{lambda(char*, char const*, long)#1} const&, bool)::{lambda(int)#1}>(long, at::native::gpu_index_kernel<at::native::index_kernel_impl<at::native::OpaqueType<2> >(at::TensorIteratorBase&, c10::ArrayRef<long>, c10::ArrayRef<long>)::{lambda(char*, char const*, long)#1}>(at::TensorIteratorBase&, c10::ArrayRef<long>, c10::ArrayRef<long>, at::native::index_kernel_impl<at::native::OpaqueType<2> >(at::TensorIteratorBase&, c10::ArrayRef<long>, c10::ArrayRef<long>)::{lambda(char*, char const*, long)#1} const&, bool)::{lambda(int)#1})"
},
{
"duration_ms": 0.001472,
"name": "void flash::prepare_varlen_num_blocks_kernel<1, false>(int, int, int, int const*, int const*, int const*, int const*, int const*, int const*, int, int, int, int, int, cutlass::FastDivmod, cutlass::FastDivmod, int*, int*, int*, int*, int*, bool, bool, bool, int)"
},
{
"duration_ms": 0.001312,
"name": "void at::native::vectorized_elementwise_kernel<2, at::native::CUDAFunctor_add<long>, std::array<char*, 3ul> >(int, at::native::CUDAFunctor_add<long>, std::array<char*, 3ul>)"
},
{
"duration_ms": 0.001184,
"name": "void at::native::vectorized_elementwise_kernel<4, at::native::CUDAFunctor_add<int>, std::array<char*, 3ul> >(int, at::native::CUDAFunctor_add<int>, std::array<char*, 3ul>)"
},
{
"duration_ms": 0.001184,
"name": "void at::native::unrolled_elementwise_kernel<at::native::FillFunctor<int>, std::array<char*, 1ul>, 4, TrivialOffsetCalculator<0, unsigned int>, TrivialOffsetCalculator<1, unsigned int>, at::native::memory::LoadWithoutCast, at::native::memory::StoreWithoutCast>(int, at::native::FillFunctor<int>, std::array<char*, 1ul>, TrivialOffsetCalculator<0, unsigned int>, TrivialOffsetCalculator<1, unsigned int>, at::native::memory::LoadWithoutCast, at::native::memory::StoreWithoutCast)"
},
{
"duration_ms": 0.0008,
"name": "void at::native::unrolled_elementwise_kernel<at::native::CUDAFunctorOnSelf_add<int>, std::array<char*, 2ul>, 4, TrivialOffsetCalculator<1, unsigned int>, TrivialOffsetCalculator<1, unsigned int>, at::native::memory::LoadWithoutCast, at::native::memory::StoreWithoutCast>(int, at::native::CUDAFunctorOnSelf_add<int>, std::array<char*, 2ul>, TrivialOffsetCalculator<1, unsigned int>, TrivialOffsetCalculator<1, unsigned int>, at::native::memory::LoadWithoutCast, at::native::memory::StoreWithoutCast)"
}
],
"non_kernel_gap_ms": 7.193539999999928,
"selected_execute_annotation": "execute_context_1(8192)_generation_0(0)",
"trace": "runs/frontier-tp2-prefill-serving-v0/fleet-artifacts/tp2-prefill-serving-smoke-20260723-v1-20260723T082454699036Z/artifacts/runs/frontier-tp2-prefill-serving-v0/remote-outputs/tp2-smoke-r1/traces/profile/dp0_pp0_tp1_dcp0_ep1_rank1.1784795312743314215.pt.trace.json.gz"
}
],
"schema": "frontier-tp2-prefill-serving-smoke.v1"
}