chore: vendor sglang v0.5.10 snapshot
This commit is contained in:
91
third_party/sglang/sgl-kernel/csrc/moe/cutlass_moe/w4a8/scaled_mm_entry.cu
vendored
Normal file
91
third_party/sglang/sgl-kernel/csrc/moe/cutlass_moe/w4a8/scaled_mm_entry.cu
vendored
Normal file
@@ -0,0 +1,91 @@
|
||||
#include <c10/cuda/CUDAGuard.h>
|
||||
#include <cudaTypedefs.h>
|
||||
#include <torch/all.h>
|
||||
|
||||
int32_t get_sm_version_num() {
|
||||
int32_t major_capability, minor_capability;
|
||||
cudaDeviceGetAttribute(&major_capability, cudaDevAttrComputeCapabilityMajor, 0);
|
||||
cudaDeviceGetAttribute(&minor_capability, cudaDevAttrComputeCapabilityMinor, 0);
|
||||
int32_t version_num = major_capability * 10 + minor_capability;
|
||||
return version_num;
|
||||
}
|
||||
|
||||
void cutlass_w4a8_moe_mm_sm90(
|
||||
torch::Tensor& d_tensors,
|
||||
torch::Tensor const& a_tensors,
|
||||
torch::Tensor const& b_tensors,
|
||||
torch::Tensor const& a_scales,
|
||||
torch::Tensor const& b_scales,
|
||||
torch::Tensor const& expert_offsets,
|
||||
torch::Tensor const& problem_sizes,
|
||||
torch::Tensor const& a_strides,
|
||||
torch::Tensor const& b_strides,
|
||||
torch::Tensor const& d_strides,
|
||||
torch::Tensor const& s_strides,
|
||||
int64_t chunk_size,
|
||||
int64_t topk);
|
||||
|
||||
void get_cutlass_w4a8_moe_mm_data_caller(
|
||||
const torch::Tensor& topk_ids,
|
||||
torch::Tensor& expert_offsets,
|
||||
torch::Tensor& problem_sizes1,
|
||||
torch::Tensor& problem_sizes2,
|
||||
torch::Tensor& input_permutation,
|
||||
torch::Tensor& output_permutation,
|
||||
const int64_t num_experts,
|
||||
const int64_t n,
|
||||
const int64_t k);
|
||||
|
||||
void cutlass_w4a8_moe_mm(
|
||||
torch::Tensor& d_tensors,
|
||||
torch::Tensor const& a_tensors,
|
||||
torch::Tensor const& b_tensors,
|
||||
torch::Tensor const& a_scales,
|
||||
torch::Tensor const& b_scales,
|
||||
torch::Tensor const& expert_offsets,
|
||||
torch::Tensor const& problem_sizes,
|
||||
torch::Tensor const& a_strides,
|
||||
torch::Tensor const& b_strides,
|
||||
torch::Tensor const& d_strides,
|
||||
torch::Tensor const& s_strides,
|
||||
int64_t chunk_size,
|
||||
int64_t topk) {
|
||||
cutlass_w4a8_moe_mm_sm90(
|
||||
d_tensors,
|
||||
a_tensors,
|
||||
b_tensors,
|
||||
a_scales,
|
||||
b_scales,
|
||||
expert_offsets,
|
||||
problem_sizes,
|
||||
a_strides,
|
||||
b_strides,
|
||||
d_strides,
|
||||
s_strides,
|
||||
chunk_size,
|
||||
topk);
|
||||
return;
|
||||
}
|
||||
|
||||
void get_cutlass_w4a8_moe_mm_data(
|
||||
const torch::Tensor& topk_ids,
|
||||
torch::Tensor& expert_offsets,
|
||||
torch::Tensor& problem_sizes1,
|
||||
torch::Tensor& problem_sizes2,
|
||||
torch::Tensor& input_permutation,
|
||||
torch::Tensor& output_permutation,
|
||||
const int64_t num_experts,
|
||||
const int64_t n,
|
||||
const int64_t k) {
|
||||
get_cutlass_w4a8_moe_mm_data_caller(
|
||||
topk_ids,
|
||||
expert_offsets,
|
||||
problem_sizes1,
|
||||
problem_sizes2,
|
||||
input_permutation,
|
||||
output_permutation,
|
||||
num_experts,
|
||||
n,
|
||||
k);
|
||||
return;
|
||||
}
|
||||
158
third_party/sglang/sgl-kernel/csrc/moe/cutlass_moe/w4a8/w4a8_get_group_starts.cuh
vendored
Normal file
158
third_party/sglang/sgl-kernel/csrc/moe/cutlass_moe/w4a8/w4a8_get_group_starts.cuh
vendored
Normal file
@@ -0,0 +1,158 @@
|
||||
#pragma once
|
||||
|
||||
#include <c10/cuda/CUDAStream.h>
|
||||
#include <cuda.h>
|
||||
#include <torch/all.h>
|
||||
|
||||
#include "cutlass/bfloat16.h"
|
||||
#include "cutlass/float8.h"
|
||||
|
||||
template <typename ElementA, typename ElementB, typename ElementC, typename ElementAccumulator>
|
||||
__global__ void int4_fp8_get_group_gemm_starts(
|
||||
int32_t* expert_offsets,
|
||||
ElementA** a_offsets,
|
||||
ElementB** b_offsets,
|
||||
ElementC** out_offsets,
|
||||
ElementAccumulator** a_scales_offsets,
|
||||
cutlass::bfloat16_t** b_scales_offsets,
|
||||
ElementA* a_base_as_int,
|
||||
ElementB* b_base_as_int,
|
||||
ElementC* out_base_as_int,
|
||||
ElementAccumulator* a_scales_base_as_int,
|
||||
cutlass::bfloat16_t* b_scales_base_as_int,
|
||||
int64_t n,
|
||||
int64_t k,
|
||||
bool per_act_token,
|
||||
bool per_out_ch) {
|
||||
int expert_id = threadIdx.x;
|
||||
int32_t expert_offset = expert_offsets[expert_id];
|
||||
|
||||
a_offsets[expert_id] = a_base_as_int + expert_offset * k;
|
||||
b_offsets[expert_id] = b_base_as_int + expert_id * k * n / 2;
|
||||
out_offsets[expert_id] = out_base_as_int + expert_offset * n;
|
||||
a_scales_offsets[expert_id] = a_scales_base_as_int + (per_act_token ? expert_offset : 0);
|
||||
b_scales_offsets[expert_id] = b_scales_base_as_int + (per_out_ch ? expert_id * n * k / 128 : expert_id);
|
||||
}
|
||||
|
||||
template <typename ElementA, typename ElementB, typename ElementC, typename ElementAccumulator>
|
||||
__global__ void int4_fp8_get_group_gemm_starts_3d(
|
||||
ElementA** a_offsets,
|
||||
ElementB** b_offsets,
|
||||
ElementC** out_offsets,
|
||||
ElementAccumulator** a_scales_offsets,
|
||||
cutlass::bfloat16_t** b_scales_offsets,
|
||||
ElementA* a_base_as_int,
|
||||
ElementB* b_base_as_int,
|
||||
ElementC* out_base_as_int,
|
||||
ElementAccumulator* a_scales_base_as_int,
|
||||
cutlass::bfloat16_t* b_scales_base_as_int,
|
||||
int64_t n,
|
||||
int64_t m,
|
||||
int64_t k,
|
||||
bool per_act_token,
|
||||
bool per_out_ch,
|
||||
int num_experts) {
|
||||
int expert_id = blockIdx.x * blockDim.x + threadIdx.x;
|
||||
if (expert_id >= num_experts) return;
|
||||
|
||||
int64_t a_offset = expert_id * m * k;
|
||||
int64_t b_offset = expert_id * k * n / 2;
|
||||
int64_t out_offset = expert_id * m * n;
|
||||
int64_t a_scales_offset = 0;
|
||||
int64_t b_scales_offset = per_out_ch ? expert_id * n * 4 * k / 512 : expert_id;
|
||||
|
||||
a_offsets[expert_id] = a_base_as_int + a_offset;
|
||||
b_offsets[expert_id] = b_base_as_int + b_offset;
|
||||
out_offsets[expert_id] = out_base_as_int + out_offset;
|
||||
a_scales_offsets[expert_id] = a_scales_base_as_int + a_scales_offset;
|
||||
b_scales_offsets[expert_id] = b_scales_base_as_int + b_scales_offset;
|
||||
}
|
||||
|
||||
#define __CALL_W4A8_GET_STARTS_KERNEL(TENSOR_C_TYPE, C_TYPE) \
|
||||
else if (out_tensors.dtype() == TENSOR_C_TYPE) { \
|
||||
int4_fp8_get_group_gemm_starts<cutlass::float_e4m3_t, cutlass::int8_t, C_TYPE, float> \
|
||||
<<<1, num_experts, 0, stream>>>( \
|
||||
static_cast<int32_t*>(expert_offsets.data_ptr()), \
|
||||
static_cast<cutlass::float_e4m3_t**>(a_ptrs.data_ptr()), \
|
||||
static_cast<cutlass::int8_t**>(b_ptrs.data_ptr()), \
|
||||
static_cast<C_TYPE**>(out_ptrs.data_ptr()), \
|
||||
static_cast<float**>(a_scales_ptrs.data_ptr()), \
|
||||
static_cast<cutlass::bfloat16_t**>(b_scales_ptrs.data_ptr()), \
|
||||
static_cast<cutlass::float_e4m3_t*>(a_tensors.data_ptr()), \
|
||||
static_cast<cutlass::int8_t*>(b_tensors.data_ptr()), \
|
||||
static_cast<C_TYPE*>(out_tensors.data_ptr()), \
|
||||
static_cast<float*>(a_scales.data_ptr()), \
|
||||
static_cast<cutlass::bfloat16_t*>(b_scales.data_ptr()), \
|
||||
out_tensors.size(1), \
|
||||
a_tensors.size(1), \
|
||||
per_act_token, \
|
||||
per_out_ch); \
|
||||
}
|
||||
|
||||
#define __CALL_W4A8_GET_STARTS_KERNEL_3D(TENSOR_C_TYPE, C_TYPE) \
|
||||
else if (out_tensors.dtype() == TENSOR_C_TYPE) { \
|
||||
int4_fp8_get_group_gemm_starts_3d<cutlass::float_e4m3_t, cutlass::int8_t, C_TYPE, float> \
|
||||
<<<1, num_experts, 0, stream>>>( \
|
||||
static_cast<cutlass::float_e4m3_t**>(a_ptrs.data_ptr()), \
|
||||
static_cast<cutlass::int8_t**>(b_ptrs.data_ptr()), \
|
||||
static_cast<C_TYPE**>(out_ptrs.data_ptr()), \
|
||||
static_cast<float**>(a_scales_ptrs.data_ptr()), \
|
||||
static_cast<cutlass::bfloat16_t**>(b_scales_ptrs.data_ptr()), \
|
||||
static_cast<cutlass::float_e4m3_t*>(a_tensors.data_ptr()), \
|
||||
static_cast<cutlass::int8_t*>(b_tensors.data_ptr()), \
|
||||
static_cast<C_TYPE*>(out_tensors.data_ptr()), \
|
||||
static_cast<float*>(a_scales.data_ptr()), \
|
||||
static_cast<cutlass::bfloat16_t*>(b_scales.data_ptr()), \
|
||||
out_tensors.size(2), \
|
||||
a_tensors.size(1), \
|
||||
a_tensors.size(2), \
|
||||
per_act_token, \
|
||||
per_out_ch, \
|
||||
num_experts); \
|
||||
}
|
||||
|
||||
namespace {
|
||||
|
||||
void run_int4_fp8_get_group_gemm_starts(
|
||||
torch::Tensor const& expert_offsets,
|
||||
torch::Tensor& a_ptrs,
|
||||
torch::Tensor& b_ptrs,
|
||||
torch::Tensor& out_ptrs,
|
||||
torch::Tensor& a_scales_ptrs,
|
||||
torch::Tensor& b_scales_ptrs,
|
||||
torch::Tensor const& a_tensors,
|
||||
torch::Tensor const& b_tensors,
|
||||
torch::Tensor& out_tensors,
|
||||
torch::Tensor const& a_scales,
|
||||
torch::Tensor const& b_scales) {
|
||||
TORCH_CHECK(a_tensors.dtype() == torch::kFloat8_e4m3fn);
|
||||
TORCH_CHECK(b_tensors.dtype() == torch::kInt8);
|
||||
TORCH_CHECK(a_scales.dtype() == torch::kFloat32);
|
||||
TORCH_CHECK(b_scales.dtype() == torch::kBFloat16);
|
||||
|
||||
int num_experts = static_cast<int>(expert_offsets.size(0));
|
||||
bool per_act_token = a_scales.numel() != 1;
|
||||
bool per_out_ch = b_scales.numel() != num_experts;
|
||||
|
||||
auto stream = at::cuda::getCurrentCUDAStream(expert_offsets.device().index());
|
||||
|
||||
if (a_tensors.dim() == 3) {
|
||||
if (false) {
|
||||
}
|
||||
__CALL_W4A8_GET_STARTS_KERNEL_3D(torch::kBFloat16, cutlass::bfloat16_t)
|
||||
__CALL_W4A8_GET_STARTS_KERNEL_3D(torch::kFloat16, half)
|
||||
else {
|
||||
TORCH_CHECK(false, "Invalid output type (must be float16 or bfloat16)");
|
||||
}
|
||||
} else {
|
||||
if (false) {
|
||||
}
|
||||
__CALL_W4A8_GET_STARTS_KERNEL(torch::kBFloat16, cutlass::bfloat16_t)
|
||||
__CALL_W4A8_GET_STARTS_KERNEL(torch::kFloat16, half)
|
||||
else {
|
||||
TORCH_CHECK(false, "Invalid output type (must be float16 or bfloat16)");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace
|
||||
214
third_party/sglang/sgl-kernel/csrc/moe/cutlass_moe/w4a8/w4a8_grouped_mm_c3x.cu
vendored
Normal file
214
third_party/sglang/sgl-kernel/csrc/moe/cutlass_moe/w4a8/w4a8_grouped_mm_c3x.cu
vendored
Normal file
@@ -0,0 +1,214 @@
|
||||
#include <c10/cuda/CUDAGuard.h>
|
||||
#include <cudaTypedefs.h>
|
||||
#include <torch/all.h>
|
||||
|
||||
#include <type_traits>
|
||||
|
||||
#include "cutlass/cutlass.h"
|
||||
#include "w4a8_grouped_mm_c3x.cuh"
|
||||
|
||||
using namespace cute;
|
||||
|
||||
namespace {
|
||||
|
||||
enum class Sched { PP, CO };
|
||||
|
||||
template <int M, int N, int K, int A, int B, int C, Sched S>
|
||||
struct SM90W4A8Config {
|
||||
using KernelSchedule = std::conditional_t<
|
||||
S == Sched::PP,
|
||||
cutlass::gemm::KernelPtrArrayTmaWarpSpecializedPingpong,
|
||||
cutlass::gemm::KernelPtrArrayTmaWarpSpecializedCooperative>;
|
||||
|
||||
using EpilogueSchedule = std::conditional_t<
|
||||
S == Sched::PP,
|
||||
cutlass::epilogue::PtrArrayTmaWarpSpecializedPingpong,
|
||||
cutlass::epilogue::PtrArrayTmaWarpSpecializedCooperative>;
|
||||
|
||||
using TileShape = cute::Shape<cute::Int<M>, cute::Int<N>, cute::Int<K>>;
|
||||
using ClusterShape = cute::Shape<cute::Int<A>, cute::Int<B>, cute::Int<C>>;
|
||||
using Cutlass3xW4A8Gemm = cutlass_3x_w4a8_group_gemm<TileShape, ClusterShape, KernelSchedule, EpilogueSchedule>;
|
||||
};
|
||||
|
||||
template <int M, int N, int K, int A, int B, int C>
|
||||
using SM90_PP = SM90W4A8Config<M, N, K, A, B, C, Sched::PP>;
|
||||
|
||||
template <int M, int N, int K, int A, int B, int C>
|
||||
using SM90_CO = SM90W4A8Config<M, N, K, A, B, C, Sched::CO>;
|
||||
|
||||
template <typename Config>
|
||||
inline void invoke_gemm(
|
||||
torch::Tensor& d_tensors,
|
||||
torch::Tensor const& a_tensors,
|
||||
torch::Tensor const& b_tensors,
|
||||
torch::Tensor const& a_scales,
|
||||
torch::Tensor const& b_scales,
|
||||
torch::Tensor const& expert_offsets,
|
||||
torch::Tensor const& problem_sizes,
|
||||
torch::Tensor const& a_strides,
|
||||
torch::Tensor const& b_strides,
|
||||
torch::Tensor const& d_strides,
|
||||
torch::Tensor const& s_strides,
|
||||
int64_t chunk_size) {
|
||||
using GemmT = typename Config::Cutlass3xW4A8Gemm;
|
||||
cutlass_w4a8_group_gemm_caller<GemmT>(
|
||||
d_tensors,
|
||||
a_tensors,
|
||||
b_tensors,
|
||||
a_scales,
|
||||
b_scales,
|
||||
expert_offsets,
|
||||
problem_sizes,
|
||||
a_strides,
|
||||
b_strides,
|
||||
d_strides,
|
||||
s_strides,
|
||||
chunk_size);
|
||||
}
|
||||
|
||||
// Helper macro to reduce code duplication
|
||||
// Note: Config must be wrapped in parentheses when it contains commas (e.g., template parameters)
|
||||
// This uses a helper macro to strip the parentheses from the template parameter
|
||||
#define INVOKE_GEMM_WITH_CONFIG_HELPER(...) \
|
||||
invoke_gemm<__VA_ARGS__>( \
|
||||
d_tensors, \
|
||||
a_tensors, \
|
||||
b_tensors, \
|
||||
a_scales, \
|
||||
b_scales, \
|
||||
expert_offsets, \
|
||||
problem_sizes, \
|
||||
a_strides, \
|
||||
b_strides, \
|
||||
d_strides, \
|
||||
s_strides, \
|
||||
chunk_size)
|
||||
#define INVOKE_GEMM_WITH_CONFIG(Config) INVOKE_GEMM_WITH_CONFIG_HELPER Config
|
||||
|
||||
void dispatch_w4a8_moe_mm_sm90(
|
||||
torch::Tensor& d_tensors,
|
||||
torch::Tensor const& a_tensors,
|
||||
torch::Tensor const& b_tensors,
|
||||
torch::Tensor const& a_scales,
|
||||
torch::Tensor const& b_scales,
|
||||
torch::Tensor const& expert_offsets,
|
||||
torch::Tensor const& problem_sizes,
|
||||
torch::Tensor const& a_strides,
|
||||
torch::Tensor const& b_strides,
|
||||
torch::Tensor const& d_strides,
|
||||
torch::Tensor const& s_strides,
|
||||
int64_t chunk_size,
|
||||
int64_t topk) {
|
||||
uint32_t const m = a_tensors.size(0) / topk;
|
||||
uint32_t const n = d_tensors.size(1);
|
||||
uint32_t const k = a_tensors.size(1);
|
||||
|
||||
if (n == 4096 && k == 7168) {
|
||||
// group gemm 1
|
||||
if (m <= 4) {
|
||||
INVOKE_GEMM_WITH_CONFIG((SM90_PP<64, 32, 512, 2, 1, 1>));
|
||||
} else if (m <= 32) {
|
||||
INVOKE_GEMM_WITH_CONFIG((SM90_CO<128, 16, 512, 2, 1, 1>));
|
||||
} else if (m <= 256) {
|
||||
INVOKE_GEMM_WITH_CONFIG((SM90_CO<128, 16, 512, 1, 1, 1>));
|
||||
} else if (m <= 1024) {
|
||||
INVOKE_GEMM_WITH_CONFIG((SM90_CO<128, 32, 512, 2, 1, 1>));
|
||||
} else if (m <= 4096) {
|
||||
// Optimized for prefill: seq_len up to 4096 (m=4096 with topk=1)
|
||||
INVOKE_GEMM_WITH_CONFIG((SM90_CO<128, 64, 512, 2, 1, 1>));
|
||||
} else {
|
||||
// Optimized for prefill: seq_len up to 8192 (m=8192 with topk=1)
|
||||
INVOKE_GEMM_WITH_CONFIG((SM90_CO<128, 64, 512, 1, 1, 1>));
|
||||
}
|
||||
} else if (n == 7168 && k == 2048) {
|
||||
// group gemm 2
|
||||
if (m <= 8) {
|
||||
INVOKE_GEMM_WITH_CONFIG((SM90_PP<64, 16, 512, 1, 1, 1>));
|
||||
} else if (m <= 512) {
|
||||
INVOKE_GEMM_WITH_CONFIG((SM90_CO<128, 32, 512, 1, 1, 1>));
|
||||
} else if (m <= 4096) {
|
||||
// Optimized for prefill: larger cluster for better throughput
|
||||
INVOKE_GEMM_WITH_CONFIG((SM90_CO<128, 64, 512, 2, 1, 1>));
|
||||
} else {
|
||||
INVOKE_GEMM_WITH_CONFIG((SM90_CO<128, 64, 512, 1, 1, 1>));
|
||||
}
|
||||
} else if (n == 512 && k == 7168) {
|
||||
// group gemm 1 for tp
|
||||
if (m <= 4) {
|
||||
INVOKE_GEMM_WITH_CONFIG((SM90_PP<64, 32, 512, 2, 1, 1>));
|
||||
} else if (m <= 32) {
|
||||
INVOKE_GEMM_WITH_CONFIG((SM90_CO<128, 16, 512, 2, 1, 1>));
|
||||
} else if (m <= 256) {
|
||||
INVOKE_GEMM_WITH_CONFIG((SM90_CO<128, 16, 512, 1, 1, 1>));
|
||||
} else if (m <= 1024) {
|
||||
INVOKE_GEMM_WITH_CONFIG((SM90_CO<128, 32, 512, 2, 1, 1>));
|
||||
} else {
|
||||
INVOKE_GEMM_WITH_CONFIG((SM90_CO<128, 64, 512, 1, 1, 1>));
|
||||
}
|
||||
} else if (n == 7168 && k == 256) {
|
||||
// group gemm 2 for tp
|
||||
if (m <= 8) {
|
||||
INVOKE_GEMM_WITH_CONFIG((SM90_PP<64, 16, 128, 1, 1, 1>));
|
||||
} else if (m <= 32) {
|
||||
INVOKE_GEMM_WITH_CONFIG((SM90_PP<128, 32, 128, 1, 1, 1>));
|
||||
} else if (m <= 512) {
|
||||
INVOKE_GEMM_WITH_CONFIG((SM90_PP<128, 32, 128, 2, 1, 1>));
|
||||
} else {
|
||||
INVOKE_GEMM_WITH_CONFIG((SM90_PP<128, 64, 128, 1, 1, 1>));
|
||||
}
|
||||
} else {
|
||||
if (k % 512 == 0) {
|
||||
// For large m (prefill), prefer larger cluster
|
||||
if (m <= 32) {
|
||||
// Decode: target batch size (16-32) - use cluster size 1 for better latency
|
||||
INVOKE_GEMM_WITH_CONFIG((SM90_CO<128, 16, 512, 1, 1, 1>));
|
||||
} else if (m <= 1024) {
|
||||
// Decode: large batch or small prefill
|
||||
INVOKE_GEMM_WITH_CONFIG((SM90_CO<128, 32, 512, 1, 1, 1>));
|
||||
} else {
|
||||
// Prefill: large sequence length - prefer larger cluster
|
||||
INVOKE_GEMM_WITH_CONFIG((SM90_CO<128, 64, 512, 1, 1, 1>));
|
||||
}
|
||||
} else {
|
||||
if (m <= 32) {
|
||||
// Decode: target batch size (16-32) - use larger tile for better throughput
|
||||
INVOKE_GEMM_WITH_CONFIG((SM90_PP<128, 32, 128, 1, 1, 1>));
|
||||
} else {
|
||||
// Prefill: larger sequence length
|
||||
INVOKE_GEMM_WITH_CONFIG((SM90_PP<128, 64, 128, 1, 1, 1>));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
void cutlass_w4a8_moe_mm_sm90(
|
||||
torch::Tensor& d_tensors,
|
||||
torch::Tensor const& a_tensors,
|
||||
torch::Tensor const& b_tensors,
|
||||
torch::Tensor const& a_scales,
|
||||
torch::Tensor const& b_scales,
|
||||
torch::Tensor const& expert_offsets,
|
||||
torch::Tensor const& problem_sizes,
|
||||
torch::Tensor const& a_strides,
|
||||
torch::Tensor const& b_strides,
|
||||
torch::Tensor const& d_strides,
|
||||
torch::Tensor const& s_strides,
|
||||
int64_t chunk_size,
|
||||
int64_t topk) {
|
||||
dispatch_w4a8_moe_mm_sm90(
|
||||
d_tensors,
|
||||
a_tensors,
|
||||
b_tensors,
|
||||
a_scales,
|
||||
b_scales,
|
||||
expert_offsets,
|
||||
problem_sizes,
|
||||
a_strides,
|
||||
b_strides,
|
||||
d_strides,
|
||||
s_strides,
|
||||
chunk_size,
|
||||
topk);
|
||||
}
|
||||
279
third_party/sglang/sgl-kernel/csrc/moe/cutlass_moe/w4a8/w4a8_grouped_mm_c3x.cuh
vendored
Normal file
279
third_party/sglang/sgl-kernel/csrc/moe/cutlass_moe/w4a8/w4a8_grouped_mm_c3x.cuh
vendored
Normal file
@@ -0,0 +1,279 @@
|
||||
#pragma once
|
||||
|
||||
/**
|
||||
* @file w4a8_grouped_mm_c3x.cuh
|
||||
* @brief Implementation of grouped GEMM operation with int4 and fp8 mixed
|
||||
* precision
|
||||
*
|
||||
* This file implements a grouped GEMM operation that multiplies FP8 matrices
|
||||
* (A) with quantized INT4 matrices (B), applying per-block scaling factors.
|
||||
* The implementation is optimized for NVIDIA Hopper GPUs, leveraging Tensor
|
||||
* Cores for mixed precision arithmetic.
|
||||
*
|
||||
* Key features:
|
||||
* - Supports grouped GEMM operations with multiple experts
|
||||
* - Uses FP8 (e4m3) for matrix A
|
||||
* - Uses INT4 quantization for matrix B with per-block scaling
|
||||
* - Implements preprocessing for INT4 encoding and scale packing
|
||||
* - Optimized for Hopper architecture with Tensor Core operations
|
||||
*/
|
||||
|
||||
#include <ATen/cuda/CUDAContext.h>
|
||||
#include <cuda_fp8.h>
|
||||
#include <cuda_runtime.h>
|
||||
#include <torch/all.h>
|
||||
|
||||
#include "cutlass/cutlass.h"
|
||||
#include "cutlass/epilogue/collective/collective_builder.hpp"
|
||||
#include "cutlass/gemm/device/gemm_universal_adapter.h"
|
||||
#include "cutlass/gemm/dispatch_policy.hpp"
|
||||
#include "cutlass/gemm/group_array_problem_shape.hpp"
|
||||
#include "cutlass/gemm/kernel/gemm_universal.hpp"
|
||||
#include "cutlass_extensions/gemm/collective/collective_builder_mixed_input.hpp"
|
||||
#include "w4a8_get_group_starts.cuh"
|
||||
|
||||
using namespace cute;
|
||||
|
||||
namespace {
|
||||
|
||||
// Type definitions
|
||||
using MmaType = cutlass::float_e4m3_t; // FP8 e4m3 type
|
||||
using QuantType = cutlass::int4b_t; // 4-bit integer type
|
||||
using ElementAccumulator = float; // Accumulator type
|
||||
using ElementScale = cutlass::bfloat16_t; // Scale type
|
||||
using ElementC = cutlass::bfloat16_t; // Output type
|
||||
using ElementD = ElementC; // Output type
|
||||
using ProblemShape = cutlass::gemm::GroupProblemShape<Shape<int, int, int>>;
|
||||
|
||||
// Architecture-specific configurations
|
||||
using ArchTag = cutlass::arch::Sm90;
|
||||
using OperatorClass = cutlass::arch::OpClassTensorOp;
|
||||
// constexpr int TileShapeK = 512;
|
||||
// using TileShape = Shape<_128, _32, cute::Int<TileShapeK>>;
|
||||
// using ClusterShape = Shape<_1, _1, _1>;
|
||||
|
||||
// Layout configurations
|
||||
using LayoutA = cutlass::layout::RowMajor;
|
||||
using LayoutB = cutlass::layout::ColumnMajor;
|
||||
using LayoutC = cutlass::layout::RowMajor;
|
||||
using LayoutD = LayoutC;
|
||||
|
||||
// Transposed layouts
|
||||
using LayoutA_Transpose = typename cutlass::layout::LayoutTranspose<LayoutA>::type;
|
||||
using LayoutB_Transpose = typename cutlass::layout::LayoutTranspose<LayoutB>::type;
|
||||
using LayoutC_Transpose = typename cutlass::layout::LayoutTranspose<LayoutC>::type;
|
||||
using LayoutD_Transpose = typename cutlass::layout::LayoutTranspose<LayoutD>::type;
|
||||
|
||||
// Alignments
|
||||
static constexpr int AlignmentA = 128 / cutlass::sizeof_bits<MmaType>::value;
|
||||
static constexpr int AlignmentB = 128 / cutlass::sizeof_bits<QuantType>::value;
|
||||
static constexpr int AlignmentC = 128 / cutlass::sizeof_bits<ElementC>::value;
|
||||
static constexpr int AlignmentD = 128 / cutlass::sizeof_bits<ElementD>::value;
|
||||
|
||||
template <typename TileShape, typename ClusterShape, typename KernelSchedule, typename EpilogueSchedule>
|
||||
struct cutlass_3x_w4a8_group_gemm {
|
||||
static constexpr int GroupSize = 128;
|
||||
static constexpr int PackedScalesNum = get<2>(TileShape{}) / GroupSize;
|
||||
using ElementScalePacked = cutlass::Array<ElementScale, PackedScalesNum>;
|
||||
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
ArchTag,
|
||||
OperatorClass,
|
||||
TileShape,
|
||||
ClusterShape,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
ElementAccumulator,
|
||||
ElementAccumulator,
|
||||
ElementC,
|
||||
LayoutC_Transpose*,
|
||||
AlignmentC,
|
||||
ElementD,
|
||||
LayoutD_Transpose*,
|
||||
AlignmentD,
|
||||
EpilogueSchedule>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloopScaleOnly = typename cutlass::gemm::collective::CollectiveBuilderMixedInput<
|
||||
ArchTag,
|
||||
OperatorClass,
|
||||
cute::tuple<QuantType, ElementScalePacked>,
|
||||
LayoutB_Transpose*,
|
||||
AlignmentB,
|
||||
MmaType,
|
||||
LayoutA_Transpose*,
|
||||
AlignmentA,
|
||||
ElementAccumulator,
|
||||
TileShape,
|
||||
ClusterShape,
|
||||
cutlass::gemm::collective::StageCountAutoCarveout<static_cast<int>(
|
||||
sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
KernelSchedule>::CollectiveOp;
|
||||
|
||||
// Define the final kernel and GEMM operation types
|
||||
using GemmKernelScaleOnly =
|
||||
cutlass::gemm::kernel::GemmUniversal<ProblemShape, CollectiveMainloopScaleOnly, CollectiveEpilogue>;
|
||||
|
||||
using GemmScaleOnly = cutlass::gemm::device::GemmUniversalAdapter<GemmKernelScaleOnly>;
|
||||
|
||||
using StrideA = cute::remove_pointer_t<cutlass::detail::TagToStrideA_t<LayoutA*>>;
|
||||
using StrideB = cute::remove_pointer_t<cutlass::detail::TagToStrideB_t<LayoutB*>>;
|
||||
using StrideC = typename GemmKernelScaleOnly::InternalStrideC;
|
||||
using StrideD = typename GemmKernelScaleOnly::InternalStrideD;
|
||||
using StrideS = typename CollectiveMainloopScaleOnly::StrideScale;
|
||||
};
|
||||
|
||||
/**
|
||||
* @brief Main function to run int4 * fp8 grouped GEMM from PyTorch
|
||||
*
|
||||
* This function performs multiple GEMM operations in parallel where each
|
||||
* operation multiplies an FP8 matrix (A) with a quantized INT4 matrix (B),
|
||||
* applying per-channel scaling factors. It's designed for efficient execution
|
||||
* on NVIDIA Hopper GPUs, leveraging Tensor Cores for optimal performance with
|
||||
* mixed precision arithmetic.
|
||||
*
|
||||
* The function includes preprocessing steps for both INT4 tensors and scale
|
||||
* factors to ensure optimal performance and correct operation.
|
||||
*
|
||||
* @param d_tensors Output tensor D with shape [total_m, total_n]
|
||||
* @param a_tensors Tensor containing all A matrices (fp8_e4m3) with shape
|
||||
* [total_m, K]
|
||||
* @param b_tensors Tensor containing all B matrices (int4 packed as int8) with
|
||||
* shape [E, N, K/2]
|
||||
* @param a_scales Tensor containing A matrix scale factors
|
||||
* @param b_scales Tensor containing B matrix scale factors with shape [E,
|
||||
* K//512, N*4]
|
||||
* @param expert_offsets Tensor containing expert offsets for determining group
|
||||
* boundaries (int32)
|
||||
* @param problem_sizes Tensor containing problem sizes with shape [num_experts,
|
||||
* 3] (M, N, K for each group) (int32)
|
||||
* @param a_strides Stride information for A tensors
|
||||
* @param b_strides Stride information for B tensors
|
||||
* @param d_strides Stride information for D tensors
|
||||
* @param s_strides Stride information for scale tensors
|
||||
* @param chunk_size Size of each chunk for scales (K / number of scale chunks)
|
||||
*/
|
||||
// template <typename TileShape, typename ClusterShape, typename KernelSchedule, typename EpilogueSchedule>
|
||||
template <typename Gemm>
|
||||
void cutlass_w4a8_group_gemm_caller(
|
||||
torch::Tensor& d_tensors,
|
||||
torch::Tensor const& a_tensors,
|
||||
torch::Tensor const& b_tensors,
|
||||
torch::Tensor const& a_scales,
|
||||
torch::Tensor const& b_scales,
|
||||
torch::Tensor const& expert_offsets,
|
||||
torch::Tensor const& problem_sizes,
|
||||
torch::Tensor const& a_strides,
|
||||
torch::Tensor const& b_strides,
|
||||
torch::Tensor const& d_strides,
|
||||
torch::Tensor const& s_strides,
|
||||
int64_t chunk_size) {
|
||||
// using Gemm = cutlass_3x_w4a8_group_gemm<TileShape, ClusterShape, KernelSchedule, EpilogueSchedule>;
|
||||
using Args = typename Gemm::GemmScaleOnly::Arguments;
|
||||
|
||||
int num_experts = static_cast<int>(expert_offsets.size(0));
|
||||
bool per_act_token = a_scales.numel() != 1;
|
||||
bool per_out_ch = b_scales.numel() != num_experts;
|
||||
|
||||
// Check inputs
|
||||
TORCH_CHECK(a_tensors.dim() == 2 or a_tensors.dim() == 3, "A tensor must be 2D/3D");
|
||||
TORCH_CHECK(b_tensors.dim() == 3, "B tensor must be 3D [E, N, K/2]");
|
||||
TORCH_CHECK(b_scales.dim() == 3, "Scale tensor must be 3D [E, K//512, N*4]");
|
||||
TORCH_CHECK(a_scales.dim() == 1, "A Scale tensor must be 1D [1]");
|
||||
TORCH_CHECK(expert_offsets.dim() == 1, "expert_offsets must be a 1D tensor");
|
||||
TORCH_CHECK(problem_sizes.dim() == 2, "problem_sizes must be 2D tensor");
|
||||
|
||||
// Check tensor shapes
|
||||
TORCH_CHECK(problem_sizes.size(0) == num_experts, "problem_sizes must have num_experts rows");
|
||||
TORCH_CHECK(problem_sizes.size(1) == 3, "problem_sizes must have 3 columns (N, M, K)");
|
||||
TORCH_CHECK(b_tensors.size(0) == num_experts, "B tensor first dimension must match number of groups");
|
||||
TORCH_CHECK(b_scales.size(0) == num_experts, "Scale tensor first dimension must match number of groups");
|
||||
TORCH_CHECK(
|
||||
b_tensors.size(2) * 2 == a_tensors.size(1) or b_tensors.size(2) * 2 == a_tensors.size(2),
|
||||
"B tensor K/2 dimension must match A tensor K dimension");
|
||||
|
||||
// Check tensor types
|
||||
TORCH_CHECK(a_tensors.scalar_type() == torch::kFloat8_e4m3fn, "A tensor must be fp8 (float_e4m3_t) type");
|
||||
TORCH_CHECK(b_tensors.scalar_type() == torch::kInt8, "B tensor must contain packed int4 values (stored as int8)");
|
||||
TORCH_CHECK(expert_offsets.scalar_type() == torch::kInt32, "Expert offsets must be int32 type");
|
||||
TORCH_CHECK(problem_sizes.scalar_type() == torch::kInt32, "Problem sizes must be int32 type");
|
||||
|
||||
auto stream = at::cuda::getCurrentCUDAStream(a_tensors.device().index());
|
||||
auto options_int = torch::TensorOptions().dtype(torch::kInt64).device(a_tensors.device());
|
||||
|
||||
torch::Tensor a_ptrs = torch::empty(num_experts, options_int);
|
||||
torch::Tensor b_ptrs = torch::empty(num_experts, options_int);
|
||||
torch::Tensor out_ptrs = torch::empty(num_experts, options_int);
|
||||
torch::Tensor a_scales_ptrs = torch::empty(num_experts, options_int);
|
||||
torch::Tensor b_scales_ptrs = torch::empty(num_experts, options_int);
|
||||
|
||||
cutlass::KernelHardwareInfo hw_info;
|
||||
hw_info.device_id = a_tensors.device().index();
|
||||
hw_info.sm_count = cutlass::KernelHardwareInfo::query_device_multiprocessor_count(hw_info.device_id);
|
||||
|
||||
Args arguments;
|
||||
decltype(arguments.epilogue.thread) fusion_args;
|
||||
fusion_args.alpha = 0;
|
||||
fusion_args.beta = 0;
|
||||
fusion_args.alpha_ptr = a_scales.data_ptr<float>();
|
||||
;
|
||||
fusion_args.beta_ptr = nullptr;
|
||||
fusion_args.alpha_ptr_array = nullptr;
|
||||
fusion_args.beta_ptr_array = nullptr;
|
||||
fusion_args.dAlpha = {cute::_0{}, cute::_0{}, 0};
|
||||
fusion_args.dBeta = {cute::_0{}, cute::_0{}, 0};
|
||||
|
||||
ProblemShape::UnderlyingProblemShape* problem_sizes_as_shapes =
|
||||
static_cast<ProblemShape::UnderlyingProblemShape*>(problem_sizes.data_ptr());
|
||||
|
||||
run_int4_fp8_get_group_gemm_starts(
|
||||
expert_offsets,
|
||||
a_ptrs,
|
||||
b_ptrs,
|
||||
out_ptrs,
|
||||
a_scales_ptrs,
|
||||
b_scales_ptrs,
|
||||
a_tensors,
|
||||
b_tensors,
|
||||
d_tensors,
|
||||
a_scales,
|
||||
b_scales);
|
||||
|
||||
arguments = Args{
|
||||
cutlass::gemm::GemmUniversalMode::kGrouped,
|
||||
{num_experts, problem_sizes_as_shapes, nullptr},
|
||||
{static_cast<const QuantType**>(b_ptrs.data_ptr()),
|
||||
static_cast<typename Gemm::StrideB*>(b_strides.data_ptr()),
|
||||
static_cast<const MmaType**>(a_ptrs.data_ptr()),
|
||||
static_cast<typename Gemm::StrideA*>(a_strides.data_ptr()),
|
||||
static_cast<const typename Gemm::ElementScalePacked**>(b_scales_ptrs.data_ptr()),
|
||||
static_cast<typename Gemm::StrideS*>(s_strides.data_ptr()),
|
||||
static_cast<int>(chunk_size)},
|
||||
{fusion_args,
|
||||
nullptr,
|
||||
nullptr,
|
||||
static_cast<ElementD**>(out_ptrs.data_ptr()),
|
||||
static_cast<typename Gemm::StrideD*>(d_strides.data_ptr())},
|
||||
hw_info};
|
||||
|
||||
// Instantiate and run GEMM
|
||||
typename Gemm::GemmScaleOnly gemm;
|
||||
size_t workspace_size = Gemm::GemmScaleOnly::get_workspace_size(arguments);
|
||||
auto const workspace_options = torch::TensorOptions().dtype(torch::kUInt8).device(a_tensors.device());
|
||||
auto workspace = torch::empty(workspace_size, workspace_options);
|
||||
|
||||
cutlass::Status status = gemm.can_implement(arguments);
|
||||
if (status != cutlass::Status::kSuccess) {
|
||||
TORCH_CHECK(false, "GEMM implementation not supported");
|
||||
}
|
||||
|
||||
status = gemm.initialize(arguments, workspace.data_ptr(), stream);
|
||||
if (status != cutlass::Status::kSuccess) {
|
||||
TORCH_CHECK(false, "GEMM initialization failed");
|
||||
}
|
||||
|
||||
status = gemm.run(stream);
|
||||
if (status != cutlass::Status::kSuccess) {
|
||||
TORCH_CHECK(false, "GEMM execution failed");
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace
|
||||
148
third_party/sglang/sgl-kernel/csrc/moe/cutlass_moe/w4a8/w4a8_moe_data.cu
vendored
Normal file
148
third_party/sglang/sgl-kernel/csrc/moe/cutlass_moe/w4a8/w4a8_moe_data.cu
vendored
Normal file
@@ -0,0 +1,148 @@
|
||||
#include <c10/cuda/CUDAGuard.h>
|
||||
#include <cudaTypedefs.h>
|
||||
#include <torch/all.h>
|
||||
|
||||
#include <cub/block/block_reduce.cuh>
|
||||
#include <cub/block/block_scan.cuh>
|
||||
|
||||
template <int BLOCK_SIZE>
|
||||
__global__ void compute_problem_sizes_w4a8(
|
||||
const int32_t* __restrict__ topk_ids,
|
||||
int32_t* problem_sizes1,
|
||||
int32_t* problem_sizes2,
|
||||
const int topk_length,
|
||||
const int n,
|
||||
const int k) {
|
||||
int expert_id = blockIdx.x;
|
||||
|
||||
int occurrences = 0;
|
||||
// Optimized: vectorized memory access using int4 for better memory bandwidth
|
||||
// Process vectorized chunks first
|
||||
bool aligned = (reinterpret_cast<uintptr_t>(topk_ids) % 16 == 0);
|
||||
|
||||
if (aligned) {
|
||||
const int4* vec_ptr = reinterpret_cast<const int4*>(topk_ids);
|
||||
int vec_length = topk_length / 4;
|
||||
|
||||
for (int i = threadIdx.x; i < vec_length; i += BLOCK_SIZE) {
|
||||
int4 vec_data = vec_ptr[i];
|
||||
occurrences +=
|
||||
(vec_data.x == expert_id) + (vec_data.y == expert_id) + (vec_data.z == expert_id) + (vec_data.w == expert_id);
|
||||
}
|
||||
|
||||
for (int i = vec_length * 4 + threadIdx.x; i < topk_length; i += BLOCK_SIZE) {
|
||||
occurrences += (topk_ids[i] == expert_id);
|
||||
}
|
||||
} else {
|
||||
for (int i = threadIdx.x; i < topk_length; i += BLOCK_SIZE) {
|
||||
occurrences += (topk_ids[i] == expert_id);
|
||||
}
|
||||
}
|
||||
|
||||
using BlockReduce = cub::BlockReduce<int, BLOCK_SIZE>;
|
||||
__shared__ typename BlockReduce::TempStorage temp_storage;
|
||||
int final_occurrences = BlockReduce(temp_storage).Sum(occurrences);
|
||||
|
||||
if (threadIdx.x == 0) {
|
||||
problem_sizes1[expert_id * 3] = 2 * n;
|
||||
problem_sizes1[expert_id * 3 + 1] = final_occurrences;
|
||||
problem_sizes1[expert_id * 3 + 2] = k;
|
||||
problem_sizes2[expert_id * 3] = k;
|
||||
problem_sizes2[expert_id * 3 + 1] = final_occurrences;
|
||||
problem_sizes2[expert_id * 3 + 2] = n;
|
||||
}
|
||||
}
|
||||
|
||||
template <int BLOCK_SIZE>
|
||||
__device__ void
|
||||
cumsum_block_scan(const int32_t* __restrict__ input, int32_t* __restrict__ output, int n, int input_stride) {
|
||||
using BlockScan = cub::BlockScan<int32_t, BLOCK_SIZE>;
|
||||
__shared__ typename BlockScan::TempStorage temp_scan_storage;
|
||||
__shared__ int32_t s_broadcast_val;
|
||||
|
||||
int tid = threadIdx.x;
|
||||
int32_t base_prefix_sum = 0;
|
||||
const int num_chunks = (n + BLOCK_SIZE - 1) / BLOCK_SIZE;
|
||||
|
||||
for (int chunk = 0; chunk < num_chunks; chunk++) {
|
||||
const int base_idx = chunk * BLOCK_SIZE;
|
||||
const int index = base_idx + tid;
|
||||
|
||||
const int32_t val = (index < n) ? input[index * input_stride] : 0;
|
||||
int32_t local_prefix_sum;
|
||||
BlockScan(temp_scan_storage).InclusiveSum(val, local_prefix_sum);
|
||||
const int32_t prefix_sum = local_prefix_sum + base_prefix_sum;
|
||||
if (index < n) {
|
||||
output[index] = prefix_sum;
|
||||
}
|
||||
if (chunk < num_chunks - 1) {
|
||||
if (tid == BLOCK_SIZE - 1) {
|
||||
s_broadcast_val = prefix_sum;
|
||||
}
|
||||
__syncthreads();
|
||||
base_prefix_sum = s_broadcast_val;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
template <int BLOCK_SIZE>
|
||||
__global__ void compute_expert_offsets_w4a8_kernel(
|
||||
const int32_t* __restrict__ problem_sizes1, int32_t* __restrict__ expert_offsets, int n, int stride) {
|
||||
if (threadIdx.x == 0) {
|
||||
expert_offsets[0] = 0;
|
||||
}
|
||||
cumsum_block_scan<BLOCK_SIZE>(problem_sizes1, expert_offsets + 1, n, stride);
|
||||
}
|
||||
|
||||
void compute_expert_offsets_w4a8(
|
||||
cudaStream_t stream, const int32_t* problem_sizes1, int32_t* expert_offsets, int n, int stride = 1, int off = 0) {
|
||||
#define compute_expert_offsets_w4a8_call(BLOCK_SIZE) \
|
||||
compute_expert_offsets_w4a8_kernel<BLOCK_SIZE> \
|
||||
<<<1, BLOCK_SIZE, 0, stream>>>(problem_sizes1 + off, expert_offsets, n, stride);
|
||||
|
||||
if (n <= 32) {
|
||||
compute_expert_offsets_w4a8_call(32);
|
||||
} else if (n <= 64) {
|
||||
compute_expert_offsets_w4a8_call(64);
|
||||
} else if (n <= 128) {
|
||||
compute_expert_offsets_w4a8_call(128);
|
||||
} else if (n <= 256) {
|
||||
compute_expert_offsets_w4a8_call(256);
|
||||
} else if (n <= 512) {
|
||||
compute_expert_offsets_w4a8_call(512);
|
||||
} else {
|
||||
compute_expert_offsets_w4a8_call(1024);
|
||||
}
|
||||
#undef compute_expert_offsets_w4a8_call
|
||||
}
|
||||
|
||||
void get_cutlass_w4a8_moe_mm_data_caller(
|
||||
const torch::Tensor& topk_ids,
|
||||
torch::Tensor& expert_offsets,
|
||||
torch::Tensor& problem_sizes1,
|
||||
torch::Tensor& problem_sizes2,
|
||||
torch::Tensor& input_permutation,
|
||||
torch::Tensor& output_permutation,
|
||||
const int64_t num_experts,
|
||||
const int64_t n,
|
||||
const int64_t k) {
|
||||
auto stream = at::cuda::getCurrentCUDAStream(topk_ids.device().index());
|
||||
auto options_int32 = torch::TensorOptions().dtype(torch::kInt32).device(topk_ids.device());
|
||||
|
||||
constexpr uint64_t BLOCK_SIZE = 512;
|
||||
compute_problem_sizes_w4a8<BLOCK_SIZE><<<num_experts, BLOCK_SIZE, 0, stream>>>(
|
||||
static_cast<const int32_t*>(topk_ids.data_ptr()),
|
||||
static_cast<int32_t*>(problem_sizes1.data_ptr()),
|
||||
static_cast<int32_t*>(problem_sizes2.data_ptr()),
|
||||
topk_ids.numel(),
|
||||
n,
|
||||
k);
|
||||
|
||||
compute_expert_offsets_w4a8(
|
||||
stream,
|
||||
static_cast<const int32_t*>(problem_sizes1.data_ptr()),
|
||||
static_cast<int32_t*>(expert_offsets.data_ptr()),
|
||||
num_experts,
|
||||
3,
|
||||
1);
|
||||
}
|
||||
142
third_party/sglang/sgl-kernel/csrc/moe/cutlass_moe_helper.cu
vendored
Normal file
142
third_party/sglang/sgl-kernel/csrc/moe/cutlass_moe_helper.cu
vendored
Normal file
@@ -0,0 +1,142 @@
|
||||
#pragma once
|
||||
|
||||
#include <c10/cuda/CUDAStream.h>
|
||||
#include <cuda.h>
|
||||
#include <torch/all.h>
|
||||
|
||||
#include "cutlass/bfloat16.h"
|
||||
#include "cutlass/float8.h"
|
||||
|
||||
template <
|
||||
typename ElementAB,
|
||||
typename ElementC,
|
||||
typename ElementAccumulator,
|
||||
typename LayoutSFA,
|
||||
typename LayoutSFB,
|
||||
typename ScaleConfig>
|
||||
__global__ void get_group_gemm_starts(
|
||||
int32_t* expert_offsets,
|
||||
ElementAB** a_offsets,
|
||||
ElementAB** b_offsets,
|
||||
ElementC** out_offsets,
|
||||
ElementAccumulator** a_scales_offsets,
|
||||
ElementAccumulator** b_scales_offsets,
|
||||
ElementAB* a_base_as_int,
|
||||
ElementAB* b_base_as_int,
|
||||
ElementC* out_base_as_int,
|
||||
ElementAccumulator* a_scales_base_as_int,
|
||||
ElementAccumulator* b_scales_base_as_int,
|
||||
LayoutSFA* layout_sfa_base_as_int,
|
||||
LayoutSFB* layout_sfb_base_as_int,
|
||||
int* problem_sizes,
|
||||
int* problem_sizes_transpose,
|
||||
bool transpose = false) {
|
||||
int64_t expert_id = static_cast<int64_t>(threadIdx.x);
|
||||
|
||||
if (expert_id >= gridDim.x * blockDim.x) {
|
||||
return;
|
||||
}
|
||||
|
||||
int m = problem_sizes[expert_id * 3];
|
||||
int n = problem_sizes[expert_id * 3 + 1];
|
||||
int k = problem_sizes[expert_id * 3 + 2];
|
||||
if (transpose) {
|
||||
problem_sizes_transpose[expert_id * 3] = n;
|
||||
problem_sizes_transpose[expert_id * 3 + 1] = m;
|
||||
problem_sizes_transpose[expert_id * 3 + 2] = k;
|
||||
}
|
||||
|
||||
int64_t expert_offset = static_cast<int64_t>(expert_offsets[expert_id]);
|
||||
int64_t a_stride = 0;
|
||||
int64_t b_stride = 0;
|
||||
int64_t a_scale_stride = 0;
|
||||
int64_t b_scale_stride = 0;
|
||||
if (!transpose) {
|
||||
a_stride = expert_offset * k;
|
||||
b_stride = expert_id * k * n;
|
||||
a_scale_stride = expert_offset * k / 128;
|
||||
b_scale_stride = expert_id * k * n / 128 / 128;
|
||||
} else {
|
||||
a_stride = expert_id * k * n;
|
||||
b_stride = expert_offset * k;
|
||||
a_scale_stride = expert_id * k * n / 128 / 128;
|
||||
b_scale_stride = expert_offset * k / 128;
|
||||
}
|
||||
a_offsets[expert_id] = a_base_as_int + a_stride;
|
||||
b_offsets[expert_id] = b_base_as_int + b_stride;
|
||||
out_offsets[expert_id] = out_base_as_int + expert_offset * n;
|
||||
a_scales_offsets[expert_id] = a_scales_base_as_int + a_scale_stride;
|
||||
b_scales_offsets[expert_id] = b_scales_base_as_int + b_scale_stride;
|
||||
|
||||
LayoutSFA* layout_sfa_ptr = layout_sfa_base_as_int + expert_id;
|
||||
LayoutSFB* layout_sfb_ptr = layout_sfb_base_as_int + expert_id;
|
||||
|
||||
if (!transpose) {
|
||||
*layout_sfa_ptr = ScaleConfig::tile_atom_to_shape_SFA(cute::make_shape(m, n, k, 1));
|
||||
*layout_sfb_ptr = ScaleConfig::tile_atom_to_shape_SFB(cute::make_shape(m, n, k, 1));
|
||||
} else {
|
||||
*layout_sfa_ptr = ScaleConfig::tile_atom_to_shape_SFA(cute::make_shape(n, m, k, 1));
|
||||
*layout_sfb_ptr = ScaleConfig::tile_atom_to_shape_SFB(cute::make_shape(n, m, k, 1));
|
||||
}
|
||||
}
|
||||
|
||||
#define __CALL_GET_STARTS_KERNEL(TENSOR_C_TYPE, C_TYPE, LayoutSFA, LayoutSFB, ScaleConfig) \
|
||||
else if (out_tensors.dtype() == TENSOR_C_TYPE) { \
|
||||
get_group_gemm_starts<cutlass::float_e4m3_t, C_TYPE, float, LayoutSFA, LayoutSFB, ScaleConfig> \
|
||||
<<<1, num_experts, 0, stream>>>( \
|
||||
static_cast<int32_t*>(expert_offsets.data_ptr()), \
|
||||
static_cast<cutlass::float_e4m3_t**>(a_ptrs.data_ptr()), \
|
||||
static_cast<cutlass::float_e4m3_t**>(b_ptrs.data_ptr()), \
|
||||
static_cast<C_TYPE**>(out_ptrs.data_ptr()), \
|
||||
static_cast<float**>(a_scales_ptrs.data_ptr()), \
|
||||
static_cast<float**>(b_scales_ptrs.data_ptr()), \
|
||||
static_cast<cutlass::float_e4m3_t*>(a_tensors.data_ptr()), \
|
||||
static_cast<cutlass::float_e4m3_t*>(b_tensors.data_ptr()), \
|
||||
static_cast<C_TYPE*>(out_tensors.data_ptr()), \
|
||||
static_cast<float*>(a_scales.data_ptr()), \
|
||||
static_cast<float*>(b_scales.data_ptr()), \
|
||||
reinterpret_cast<LayoutSFA*>(layout_sfa.data_ptr()), \
|
||||
reinterpret_cast<LayoutSFB*>(layout_sfb.data_ptr()), \
|
||||
static_cast<int*>(problem_sizes.data_ptr()), \
|
||||
static_cast<int*>(problem_sizes_transpose.data_ptr()), \
|
||||
transpose); \
|
||||
}
|
||||
|
||||
namespace {
|
||||
template <typename LayoutSFA, typename LayoutSFB, typename ScaleConfig>
|
||||
void run_get_group_gemm_starts(
|
||||
torch::Tensor const& expert_offsets,
|
||||
torch::Tensor& a_ptrs,
|
||||
torch::Tensor& b_ptrs,
|
||||
torch::Tensor& out_ptrs,
|
||||
torch::Tensor& a_scales_ptrs,
|
||||
torch::Tensor& b_scales_ptrs,
|
||||
torch::Tensor const& a_tensors,
|
||||
torch::Tensor const& b_tensors,
|
||||
torch::Tensor& out_tensors,
|
||||
torch::Tensor const& a_scales,
|
||||
torch::Tensor const& b_scales,
|
||||
torch::Tensor const& layout_sfa,
|
||||
torch::Tensor const& layout_sfb,
|
||||
torch::Tensor const& problem_sizes,
|
||||
torch::Tensor& problem_sizes_transpose,
|
||||
bool transpose = false) {
|
||||
TORCH_CHECK(a_tensors.dtype() == torch::kFloat8_e4m3fn);
|
||||
TORCH_CHECK(b_tensors.dtype() == torch::kFloat8_e4m3fn);
|
||||
TORCH_CHECK(a_scales.dtype() == torch::kFloat32);
|
||||
TORCH_CHECK(b_scales.dtype() == torch::kFloat32);
|
||||
TORCH_CHECK(out_tensors.size(1) % 128 == 0 or out_tensors.size(0) % 128 == 0);
|
||||
TORCH_CHECK(a_tensors.size(1) % 128 == 0 or a_tensors.size(0) % 128 == 0);
|
||||
|
||||
int num_experts = (int)expert_offsets.size(0);
|
||||
auto stream = at::cuda::getCurrentCUDAStream(a_tensors.device().index());
|
||||
|
||||
if (false) {
|
||||
}
|
||||
__CALL_GET_STARTS_KERNEL(torch::kBFloat16, cutlass::bfloat16_t, LayoutSFA, LayoutSFB, ScaleConfig)
|
||||
__CALL_GET_STARTS_KERNEL(torch::kFloat16, half, LayoutSFA, LayoutSFB, ScaleConfig)
|
||||
else {
|
||||
TORCH_CHECK(false, "Invalid output type (must be float16 or bfloat16)");
|
||||
}
|
||||
}
|
||||
} // namespace
|
||||
810
third_party/sglang/sgl-kernel/csrc/moe/fp8_blockwise_moe_kernel.cu
vendored
Normal file
810
third_party/sglang/sgl-kernel/csrc/moe/fp8_blockwise_moe_kernel.cu
vendored
Normal file
@@ -0,0 +1,810 @@
|
||||
#include <ATen/cuda/CUDAContext.h>
|
||||
#include <c10/cuda/CUDAGuard.h>
|
||||
#include <cutlass/arch/arch.h>
|
||||
#include <torch/all.h>
|
||||
|
||||
#include "cute/tensor.hpp"
|
||||
#include "cutlass/cutlass.h"
|
||||
#include "cutlass/epilogue/collective/collective_builder.hpp"
|
||||
#include "cutlass/epilogue/collective/default_epilogue.hpp"
|
||||
#include "cutlass/epilogue/dispatch_policy.hpp"
|
||||
#include "cutlass/epilogue/thread/activation.h"
|
||||
#include "cutlass/epilogue/thread/linear_combination.h"
|
||||
#include "cutlass/gemm/collective/collective_builder.hpp"
|
||||
#include "cutlass/gemm/device/gemm_universal_adapter.h"
|
||||
#include "cutlass/gemm/dispatch_policy.hpp"
|
||||
#include "cutlass/gemm/group_array_problem_shape.hpp"
|
||||
#include "cutlass/gemm/kernel/gemm_universal.hpp"
|
||||
#include "cutlass/gemm/kernel/tile_scheduler_params.h"
|
||||
#include "cutlass/tensor_ref.h"
|
||||
#include "cutlass/util/command_line.h"
|
||||
#include "cutlass/util/distribution.h"
|
||||
#include "cutlass/util/host_tensor.h"
|
||||
#include "cutlass/util/packed_stride.hpp"
|
||||
#include "cutlass/util/reference/device/gemm.h"
|
||||
#include "cutlass/util/reference/device/tensor_compare.h"
|
||||
#include "cutlass/util/tensor_view_io.h"
|
||||
#include "cutlass_moe_helper.cu"
|
||||
#include "utils.h"
|
||||
|
||||
using namespace cute;
|
||||
|
||||
using ProblemShape = cutlass::gemm::GroupProblemShape<Shape<int, int, int>>;
|
||||
|
||||
template <typename OutType, typename ScheduleConfig, typename LayoutD>
|
||||
void launch_sm90_fp8_blockwise_scaled_group_mm(
|
||||
torch::Tensor& out_ptrs,
|
||||
const torch::Tensor& a_ptrs,
|
||||
const torch::Tensor& b_ptrs,
|
||||
const torch::Tensor& a_scales_ptrs,
|
||||
const torch::Tensor& b_scales_ptrs,
|
||||
const torch::Tensor& stride_a,
|
||||
const torch::Tensor& stride_b,
|
||||
const torch::Tensor& stride_c,
|
||||
const torch::Tensor& layout_sfa,
|
||||
const torch::Tensor& layout_sfb,
|
||||
const torch::Tensor& problem_sizes,
|
||||
const torch::Tensor& expert_offsets,
|
||||
const torch::Tensor& workspace) {
|
||||
using ElementA = cutlass::float_e4m3_t;
|
||||
using ElementB = cutlass::float_e4m3_t;
|
||||
using ElementC = void;
|
||||
using ElementD = OutType;
|
||||
using ElementAccumulator = float;
|
||||
using LayoutA = cutlass::layout::RowMajor;
|
||||
using LayoutB = cutlass::layout::ColumnMajor;
|
||||
using LayoutC = LayoutD;
|
||||
|
||||
static constexpr int AlignmentA = 128 / cutlass::sizeof_bits<ElementA>::value;
|
||||
static constexpr int AlignmentB = 128 / cutlass::sizeof_bits<ElementB>::value;
|
||||
static constexpr int AlignmentC = 128 / cutlass::sizeof_bits<ElementD>::value;
|
||||
|
||||
using ArchTag = cutlass::arch::Sm90;
|
||||
using OperatorClass = cutlass::arch::OpClassTensorOp;
|
||||
static constexpr auto RoundStyle = cutlass::FloatRoundStyle::round_to_nearest;
|
||||
using CustomEVTIdentity = // acc
|
||||
cutlass::epilogue::fusion::Sm90EVT<
|
||||
cutlass::epilogue::fusion::
|
||||
Sm90Compute<cutlass::epilogue::thread::Identity, ElementD, ElementAccumulator, RoundStyle>,
|
||||
cutlass::epilogue::fusion::Sm90AccFetch>;
|
||||
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
ArchTag,
|
||||
OperatorClass,
|
||||
typename ScheduleConfig::MmaTileShape,
|
||||
typename ScheduleConfig::ClusterShape,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
ElementAccumulator,
|
||||
ElementAccumulator,
|
||||
ElementC, // Use void to avoid load Matrix C
|
||||
LayoutC*,
|
||||
AlignmentC,
|
||||
ElementD,
|
||||
LayoutC*,
|
||||
AlignmentC,
|
||||
typename ScheduleConfig::EpilogueSchedule,
|
||||
CustomEVTIdentity>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
ArchTag,
|
||||
OperatorClass,
|
||||
ElementA,
|
||||
cute::tuple<LayoutA*, typename ScheduleConfig::LayoutSFA*>,
|
||||
AlignmentA,
|
||||
ElementB,
|
||||
cute::tuple<LayoutB*, typename ScheduleConfig::LayoutSFB*>,
|
||||
AlignmentB,
|
||||
ElementAccumulator,
|
||||
typename ScheduleConfig::MmaTileShape,
|
||||
typename ScheduleConfig::ClusterShape,
|
||||
cutlass::gemm::collective::StageCountAutoCarveout<static_cast<int>(
|
||||
sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
typename ScheduleConfig::KernelSchedule>::CollectiveOp;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<ProblemShape, CollectiveMainloop, CollectiveEpilogue, void>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
using UnderlyingProblemShape = ProblemShape::UnderlyingProblemShape;
|
||||
using StrideA = typename Gemm::GemmKernel::InternalStrideA;
|
||||
using StrideB = typename Gemm::GemmKernel::InternalStrideB;
|
||||
using StrideC = typename Gemm::GemmKernel::InternalStrideC;
|
||||
using StrideD = typename Gemm::GemmKernel::InternalStrideD;
|
||||
|
||||
int num_experts = (int)expert_offsets.size(0);
|
||||
Gemm gemm_op;
|
||||
|
||||
typename GemmKernel::MainloopArguments mainloop_args{
|
||||
static_cast<const ElementA**>(a_ptrs.data_ptr()),
|
||||
static_cast<StrideA*>(stride_a.data_ptr()),
|
||||
static_cast<const ElementB**>(b_ptrs.data_ptr()),
|
||||
static_cast<StrideB*>(stride_b.data_ptr()),
|
||||
static_cast<const ElementAccumulator**>(a_scales_ptrs.data_ptr()),
|
||||
reinterpret_cast<typename ScheduleConfig::LayoutSFA*>(layout_sfa.data_ptr()),
|
||||
static_cast<const ElementAccumulator**>(b_scales_ptrs.data_ptr()),
|
||||
reinterpret_cast<typename ScheduleConfig::LayoutSFB*>(layout_sfb.data_ptr())};
|
||||
|
||||
cutlass::KernelHardwareInfo hw_info;
|
||||
hw_info.device_id = c10::cuda::current_device();
|
||||
hw_info.sm_count = at::cuda::getCurrentDeviceProperties()->multiProcessorCount;
|
||||
|
||||
typename GemmKernel::EpilogueArguments epilogue_args{
|
||||
{},
|
||||
nullptr,
|
||||
static_cast<StrideC*>(stride_c.data_ptr()),
|
||||
static_cast<ElementD**>(out_ptrs.data_ptr()),
|
||||
static_cast<StrideC*>(stride_c.data_ptr())};
|
||||
|
||||
UnderlyingProblemShape* problem_sizes_as_shapes = static_cast<UnderlyingProblemShape*>(problem_sizes.data_ptr());
|
||||
typename GemmKernel::Arguments args{
|
||||
cutlass::gemm::GemmUniversalMode::kGrouped,
|
||||
{num_experts, problem_sizes_as_shapes, nullptr},
|
||||
mainloop_args,
|
||||
epilogue_args,
|
||||
hw_info};
|
||||
|
||||
at::cuda::CUDAGuard device_guard{(char)a_ptrs.get_device()};
|
||||
const cudaStream_t stream = at::cuda::getCurrentCUDAStream(a_ptrs.get_device());
|
||||
|
||||
auto can_implement_status = gemm_op.can_implement(args);
|
||||
TORCH_CHECK(can_implement_status == cutlass::Status::kSuccess, "Failed to implement GEMM");
|
||||
|
||||
auto status = gemm_op.initialize(args, workspace.data_ptr(), stream);
|
||||
TORCH_CHECK(status == cutlass::Status::kSuccess, "Failed to initialize GEMM");
|
||||
|
||||
status = gemm_op.run(stream);
|
||||
TORCH_CHECK(status == cutlass::Status::kSuccess, "Failed to run GEMM");
|
||||
}
|
||||
|
||||
template <typename OutType, typename ScheduleConfig, typename LayoutD>
|
||||
void launch_sm100_fp8_blockwise_scaled_group_mm(
|
||||
torch::Tensor& out_ptrs,
|
||||
const torch::Tensor& a_ptrs,
|
||||
const torch::Tensor& b_ptrs,
|
||||
const torch::Tensor& a_scales_ptrs,
|
||||
const torch::Tensor& b_scales_ptrs,
|
||||
const torch::Tensor& stride_a,
|
||||
const torch::Tensor& stride_b,
|
||||
const torch::Tensor& stride_c,
|
||||
const torch::Tensor& layout_sfa,
|
||||
const torch::Tensor& layout_sfb,
|
||||
const torch::Tensor& problem_sizes,
|
||||
const torch::Tensor& expert_offsets,
|
||||
const torch::Tensor& workspace) {
|
||||
using ProblemShape = cutlass::gemm::GroupProblemShape<Shape<int, int, int>>;
|
||||
using ElementA = cutlass::float_e4m3_t;
|
||||
using ElementB = cutlass::float_e4m3_t;
|
||||
using ElementC = OutType;
|
||||
using ElementD = ElementC;
|
||||
using ElementAccumulator = float;
|
||||
using LayoutA = cutlass::layout::RowMajor;
|
||||
using LayoutB = cutlass::layout::ColumnMajor;
|
||||
using LayoutC = LayoutD;
|
||||
|
||||
static constexpr int AlignmentA = 128 / cutlass::sizeof_bits<ElementA>::value;
|
||||
static constexpr int AlignmentB = 128 / cutlass::sizeof_bits<ElementB>::value;
|
||||
static constexpr int AlignmentC = 128 / cutlass::sizeof_bits<ElementC>::value;
|
||||
|
||||
using ArchTag = cutlass::arch::Sm100;
|
||||
using OperatorClass = cutlass::arch::OpClassTensorOp;
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
ArchTag,
|
||||
OperatorClass,
|
||||
typename ScheduleConfig::MmaTileShape,
|
||||
typename ScheduleConfig::ClusterShape,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
ElementAccumulator,
|
||||
ElementAccumulator,
|
||||
void,
|
||||
LayoutC*,
|
||||
AlignmentC,
|
||||
ElementD,
|
||||
LayoutC*,
|
||||
AlignmentC,
|
||||
typename ScheduleConfig::EpilogueSchedule>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
ArchTag,
|
||||
OperatorClass,
|
||||
ElementA,
|
||||
cute::tuple<LayoutA*, typename ScheduleConfig::LayoutSFA*>,
|
||||
AlignmentA,
|
||||
ElementB,
|
||||
cute::tuple<LayoutB*, typename ScheduleConfig::LayoutSFB*>,
|
||||
AlignmentB,
|
||||
ElementAccumulator,
|
||||
typename ScheduleConfig::MmaTileShape,
|
||||
typename ScheduleConfig::ClusterShape,
|
||||
cutlass::gemm::collective::StageCountAutoCarveout<static_cast<int>(
|
||||
sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
typename ScheduleConfig::KernelSchedule>::CollectiveOp;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<ProblemShape, CollectiveMainloop, CollectiveEpilogue, void>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
using UnderlyingProblemShape = ProblemShape::UnderlyingProblemShape;
|
||||
using StrideA = typename Gemm::GemmKernel::InternalStrideA;
|
||||
using StrideB = typename Gemm::GemmKernel::InternalStrideB;
|
||||
using StrideC = typename Gemm::GemmKernel::InternalStrideC;
|
||||
using StrideD = typename Gemm::GemmKernel::InternalStrideD;
|
||||
|
||||
int num_experts = (int)expert_offsets.size(0);
|
||||
// Create an instance of the GEMM
|
||||
Gemm gemm_op;
|
||||
|
||||
typename GemmKernel::MainloopArguments mainloop_args{
|
||||
static_cast<const ElementA**>(a_ptrs.data_ptr()),
|
||||
static_cast<StrideA*>(stride_a.data_ptr()),
|
||||
static_cast<const ElementB**>(b_ptrs.data_ptr()),
|
||||
static_cast<StrideB*>(stride_b.data_ptr()),
|
||||
static_cast<const ElementAccumulator**>(a_scales_ptrs.data_ptr()),
|
||||
reinterpret_cast<typename ScheduleConfig::LayoutSFA*>(layout_sfa.data_ptr()),
|
||||
static_cast<const ElementAccumulator**>(b_scales_ptrs.data_ptr()),
|
||||
reinterpret_cast<typename ScheduleConfig::LayoutSFB*>(layout_sfb.data_ptr())};
|
||||
|
||||
cutlass::KernelHardwareInfo hw_info;
|
||||
|
||||
hw_info.device_id = 0;
|
||||
// sm_count is the number of SMs on the current device, since we only support SM100 blackwell, so we set it to 148
|
||||
hw_info.sm_count = 148;
|
||||
typename GemmKernel::EpilogueArguments epilogue_args{
|
||||
{},
|
||||
nullptr,
|
||||
static_cast<StrideC*>(stride_c.data_ptr()),
|
||||
static_cast<ElementD**>(out_ptrs.data_ptr()),
|
||||
static_cast<StrideC*>(stride_c.data_ptr())};
|
||||
|
||||
UnderlyingProblemShape* problem_sizes_as_shapes = static_cast<UnderlyingProblemShape*>(problem_sizes.data_ptr());
|
||||
typename GemmKernel::Arguments args{
|
||||
cutlass::gemm::GemmUniversalMode::kGrouped,
|
||||
{num_experts, problem_sizes_as_shapes, nullptr},
|
||||
mainloop_args,
|
||||
epilogue_args,
|
||||
hw_info};
|
||||
|
||||
at::cuda::CUDAGuard device_guard{(char)a_ptrs.get_device()};
|
||||
const cudaStream_t stream = at::cuda::getCurrentCUDAStream(a_ptrs.get_device());
|
||||
|
||||
auto can_implement_status = gemm_op.can_implement(args);
|
||||
TORCH_CHECK(can_implement_status == cutlass::Status::kSuccess, "Failed to implement GEMM");
|
||||
|
||||
auto status = gemm_op.initialize(args, workspace.data_ptr(), stream);
|
||||
TORCH_CHECK(status == cutlass::Status::kSuccess, "Failed to initialize GEMM");
|
||||
|
||||
status = gemm_op.run(stream);
|
||||
TORCH_CHECK(status == cutlass::Status::kSuccess, "Failed to run GEMM");
|
||||
}
|
||||
|
||||
template <typename OutType>
|
||||
void sm100_fp8_blockwise_group_mm_dispatch_shape(
|
||||
torch::Tensor& output,
|
||||
torch::Tensor& a_ptrs,
|
||||
torch::Tensor& b_ptrs,
|
||||
torch::Tensor& out_ptrs,
|
||||
torch::Tensor& a_scales_ptrs,
|
||||
torch::Tensor& b_scales_ptrs,
|
||||
const torch::Tensor& a,
|
||||
const torch::Tensor& b,
|
||||
const torch::Tensor& scales_a,
|
||||
const torch::Tensor& scales_b,
|
||||
const torch::Tensor& stride_a,
|
||||
const torch::Tensor& stride_b,
|
||||
const torch::Tensor& stride_c,
|
||||
const torch::Tensor& layout_sfa,
|
||||
const torch::Tensor& layout_sfb,
|
||||
const torch::Tensor& problem_sizes,
|
||||
const torch::Tensor& expert_offsets,
|
||||
const torch::Tensor& workspace) {
|
||||
// Check the first matrix size to decide on the configuration
|
||||
// Assuming all matrices in the group have similar size characteristics
|
||||
// bool use_small_config = a[0].size(0) <= 128;
|
||||
struct MmaConfig1 {
|
||||
using ElementA = cutlass::float_e4m3_t;
|
||||
using MmaTileShape = Shape<_256, _32, _128>;
|
||||
using ClusterShape = Shape<_2, _1, _1>; // Layout type for SFB matrix operand
|
||||
using KernelSchedule = cutlass::gemm::KernelPtrArrayTmaWarpSpecializedBlockwise2SmSm100;
|
||||
using EpilogueSchedule = cutlass::epilogue::PtrArrayTmaWarpSpecialized2Sm;
|
||||
using ScaleConfig =
|
||||
cutlass::detail::Sm100BlockwiseScaleConfig<128, 1, 128, cute::UMMA::Major::K, cute::UMMA::Major::K>;
|
||||
using LayoutSFA = decltype(ScaleConfig::deduce_layoutSFA());
|
||||
using LayoutSFB = decltype(ScaleConfig::deduce_layoutSFB());
|
||||
};
|
||||
struct MmaConfig2 {
|
||||
using ElementA = cutlass::float_e4m3_t;
|
||||
using MmaTileShape = Shape<_128, _128, _128>;
|
||||
using ClusterShape = Shape<_1, _1, _1>; // Layout type for SFB matrix operand
|
||||
using KernelSchedule = cutlass::gemm::KernelPtrArrayTmaWarpSpecializedBlockwise1SmSm100;
|
||||
using EpilogueSchedule = cutlass::epilogue::PtrArrayTmaWarpSpecialized1Sm;
|
||||
using ScaleConfig =
|
||||
cutlass::detail::Sm100BlockwiseScaleConfig<1, 128, 128, cute::UMMA::Major::K, cute::UMMA::Major::K>;
|
||||
using LayoutSFA = decltype(ScaleConfig::deduce_layoutSFA());
|
||||
using LayoutSFB = decltype(ScaleConfig::deduce_layoutSFB());
|
||||
};
|
||||
struct MmaConfig3 {
|
||||
using ElementA = cutlass::float_e4m3_t;
|
||||
using MmaTileShape = Shape<_64, _128, _128>;
|
||||
using ClusterShape = Shape<_1, _1, _1>; // Layout type for SFB matrix operand
|
||||
using KernelSchedule = cutlass::gemm::KernelPtrArrayTmaWarpSpecializedBlockwise1SmSm100;
|
||||
using EpilogueSchedule = cutlass::epilogue::PtrArrayTmaWarpSpecialized1Sm;
|
||||
using ScaleConfig =
|
||||
cutlass::detail::Sm100BlockwiseScaleConfig<1, 128, 128, cute::UMMA::Major::K, cute::UMMA::Major::K>;
|
||||
using LayoutSFA = decltype(ScaleConfig::deduce_layoutSFA());
|
||||
using LayoutSFB = decltype(ScaleConfig::deduce_layoutSFB());
|
||||
};
|
||||
int num_experts = (int)expert_offsets.size(0);
|
||||
torch::TensorOptions options_int = torch::TensorOptions().dtype(torch::kInt64).device(a.device());
|
||||
torch::Tensor problem_sizes_transpose = torch::empty(num_experts * 3, options_int);
|
||||
torch::Tensor output_t = output.t();
|
||||
torch::Tensor a_t = a.t();
|
||||
torch::Tensor b_t = b.transpose(1, 2);
|
||||
torch::Tensor scales_a_t = scales_a.t();
|
||||
torch::Tensor scales_b_t = scales_b.transpose(1, 2);
|
||||
|
||||
if (a.size(0) <= 2048 && a.size(1) >= 2048) {
|
||||
run_get_group_gemm_starts<MmaConfig1::LayoutSFA, MmaConfig1::LayoutSFB, MmaConfig1::ScaleConfig>(
|
||||
expert_offsets,
|
||||
a_ptrs,
|
||||
b_ptrs,
|
||||
out_ptrs,
|
||||
a_scales_ptrs,
|
||||
b_scales_ptrs,
|
||||
b_t,
|
||||
a_t,
|
||||
output_t,
|
||||
scales_b_t,
|
||||
scales_a_t,
|
||||
layout_sfa,
|
||||
layout_sfb,
|
||||
problem_sizes,
|
||||
problem_sizes_transpose,
|
||||
true);
|
||||
launch_sm100_fp8_blockwise_scaled_group_mm<OutType, MmaConfig1, cutlass::layout::ColumnMajor>(
|
||||
out_ptrs,
|
||||
a_ptrs,
|
||||
b_ptrs,
|
||||
a_scales_ptrs,
|
||||
b_scales_ptrs,
|
||||
stride_a,
|
||||
stride_b,
|
||||
stride_c,
|
||||
layout_sfa,
|
||||
layout_sfb,
|
||||
problem_sizes_transpose,
|
||||
expert_offsets,
|
||||
workspace);
|
||||
output = output_t.t();
|
||||
} else if (a.size(0) > 2048 && a.size(1) >= 2048) {
|
||||
run_get_group_gemm_starts<MmaConfig2::LayoutSFA, MmaConfig2::LayoutSFB, MmaConfig2::ScaleConfig>(
|
||||
expert_offsets,
|
||||
a_ptrs,
|
||||
b_ptrs,
|
||||
out_ptrs,
|
||||
a_scales_ptrs,
|
||||
b_scales_ptrs,
|
||||
a,
|
||||
b,
|
||||
output,
|
||||
scales_a,
|
||||
scales_b,
|
||||
layout_sfa,
|
||||
layout_sfb,
|
||||
problem_sizes,
|
||||
problem_sizes_transpose);
|
||||
launch_sm100_fp8_blockwise_scaled_group_mm<OutType, MmaConfig2, cutlass::layout::RowMajor>(
|
||||
out_ptrs,
|
||||
a_ptrs,
|
||||
b_ptrs,
|
||||
a_scales_ptrs,
|
||||
b_scales_ptrs,
|
||||
stride_a,
|
||||
stride_b,
|
||||
stride_c,
|
||||
layout_sfa,
|
||||
layout_sfb,
|
||||
problem_sizes,
|
||||
expert_offsets,
|
||||
workspace);
|
||||
} else {
|
||||
run_get_group_gemm_starts<MmaConfig3::LayoutSFA, MmaConfig3::LayoutSFB, MmaConfig3::ScaleConfig>(
|
||||
expert_offsets,
|
||||
a_ptrs,
|
||||
b_ptrs,
|
||||
out_ptrs,
|
||||
a_scales_ptrs,
|
||||
b_scales_ptrs,
|
||||
a,
|
||||
b,
|
||||
output,
|
||||
scales_a,
|
||||
scales_b,
|
||||
layout_sfa,
|
||||
layout_sfb,
|
||||
problem_sizes,
|
||||
problem_sizes_transpose);
|
||||
launch_sm100_fp8_blockwise_scaled_group_mm<OutType, MmaConfig3, cutlass::layout::RowMajor>(
|
||||
out_ptrs,
|
||||
a_ptrs,
|
||||
b_ptrs,
|
||||
a_scales_ptrs,
|
||||
b_scales_ptrs,
|
||||
stride_a,
|
||||
stride_b,
|
||||
stride_c,
|
||||
layout_sfa,
|
||||
layout_sfb,
|
||||
problem_sizes,
|
||||
expert_offsets,
|
||||
workspace);
|
||||
}
|
||||
}
|
||||
|
||||
template <typename OutType>
|
||||
void sm90_fp8_blockwise_group_mm_dispatch_shape(
|
||||
torch::Tensor& output,
|
||||
torch::Tensor& a_ptrs,
|
||||
torch::Tensor& b_ptrs,
|
||||
torch::Tensor& out_ptrs,
|
||||
torch::Tensor& a_scales_ptrs,
|
||||
torch::Tensor& b_scales_ptrs,
|
||||
const torch::Tensor& a,
|
||||
const torch::Tensor& b,
|
||||
const torch::Tensor& scales_a,
|
||||
const torch::Tensor& scales_b,
|
||||
const torch::Tensor& stride_a,
|
||||
const torch::Tensor& stride_b,
|
||||
const torch::Tensor& stride_c,
|
||||
const torch::Tensor& layout_sfa,
|
||||
const torch::Tensor& layout_sfb,
|
||||
const torch::Tensor& problem_sizes,
|
||||
const torch::Tensor& expert_offsets,
|
||||
const torch::Tensor& workspace) {
|
||||
struct MmaConfigSmallM {
|
||||
// Swap A/B
|
||||
using ElementA = cutlass::float_e4m3_t;
|
||||
using MmaTileShape = Shape<_128, _32, _128>;
|
||||
using ClusterShape = Shape<_2, _1, _1>;
|
||||
// TODO: Check Pingpong or Cooperative
|
||||
using KernelSchedule = cutlass::gemm::KernelPtrArrayTmaWarpSpecializedPingpongFP8Blockwise;
|
||||
using EpilogueSchedule = cutlass::epilogue::PtrArrayTmaWarpSpecializedPingpong;
|
||||
using ScaleConfig =
|
||||
cutlass::detail::Sm90BlockwiseScaleConfig<128, 1, 128, cute::GMMA::Major::K, cute::GMMA::Major::K>;
|
||||
using LayoutSFA = decltype(ScaleConfig::deduce_layoutSFA());
|
||||
using LayoutSFB = decltype(ScaleConfig::deduce_layoutSFB());
|
||||
};
|
||||
|
||||
struct MmaConfigH20LargeK {
|
||||
using ElementA = cutlass::float_e4m3_t;
|
||||
using MmaTileShape = Shape<_64, _128, _128>;
|
||||
using ClusterShape = Shape<_2, _1, _1>;
|
||||
using KernelSchedule = cutlass::gemm::KernelPtrArrayTmaWarpSpecializedPingpongFP8Blockwise;
|
||||
using EpilogueSchedule = cutlass::epilogue::PtrArrayTmaWarpSpecializedPingpong;
|
||||
using ScaleConfig =
|
||||
cutlass::detail::Sm90BlockwiseScaleConfig<1, 128, 128, cute::GMMA::Major::K, cute::GMMA::Major::K>;
|
||||
using LayoutSFA = decltype(ScaleConfig::deduce_layoutSFA());
|
||||
using LayoutSFB = decltype(ScaleConfig::deduce_layoutSFB());
|
||||
};
|
||||
|
||||
struct MmaConfigHx00AndH20SmallK {
|
||||
using ElementA = cutlass::float_e4m3_t;
|
||||
using MmaTileShape = Shape<_128, _128, _128>;
|
||||
using ClusterShape = Shape<_1, _2, _1>;
|
||||
using KernelSchedule = cutlass::gemm::KernelPtrArrayTmaWarpSpecializedCooperativeFP8Blockwise;
|
||||
using EpilogueSchedule = cutlass::epilogue::PtrArrayTmaWarpSpecializedCooperative;
|
||||
using ScaleConfig =
|
||||
cutlass::detail::Sm90BlockwiseScaleConfig<1, 128, 128, cute::GMMA::Major::K, cute::GMMA::Major::K>;
|
||||
using LayoutSFA = decltype(ScaleConfig::deduce_layoutSFA());
|
||||
using LayoutSFB = decltype(ScaleConfig::deduce_layoutSFB());
|
||||
};
|
||||
|
||||
int num_experts = (int)expert_offsets.size(0);
|
||||
torch::TensorOptions options_int = torch::TensorOptions().dtype(torch::kInt64).device(a.device());
|
||||
torch::Tensor problem_sizes_transpose = torch::empty(num_experts * 3, options_int);
|
||||
torch::Tensor output_t = output.t();
|
||||
torch::Tensor a_t = a.t();
|
||||
torch::Tensor b_t = b.transpose(1, 2);
|
||||
torch::Tensor scales_a_t = scales_a.t();
|
||||
torch::Tensor scales_b_t = scales_b.transpose(1, 2);
|
||||
|
||||
const std::string H20_device_type_str("NVIDIA H20");
|
||||
bool is_h20_device = std::string(at::cuda::getCurrentDeviceProperties()->name) == H20_device_type_str;
|
||||
|
||||
if (a.size(0) <= 2048) {
|
||||
run_get_group_gemm_starts<MmaConfigSmallM::LayoutSFA, MmaConfigSmallM::LayoutSFB, MmaConfigSmallM::ScaleConfig>(
|
||||
expert_offsets,
|
||||
a_ptrs,
|
||||
b_ptrs,
|
||||
out_ptrs,
|
||||
a_scales_ptrs,
|
||||
b_scales_ptrs,
|
||||
b_t,
|
||||
a_t,
|
||||
output_t,
|
||||
scales_b_t,
|
||||
scales_a_t,
|
||||
layout_sfa,
|
||||
layout_sfb,
|
||||
problem_sizes,
|
||||
problem_sizes_transpose,
|
||||
true);
|
||||
launch_sm90_fp8_blockwise_scaled_group_mm<OutType, MmaConfigSmallM, cutlass::layout::ColumnMajor>(
|
||||
out_ptrs,
|
||||
a_ptrs,
|
||||
b_ptrs,
|
||||
a_scales_ptrs,
|
||||
b_scales_ptrs,
|
||||
stride_a,
|
||||
stride_b,
|
||||
stride_c,
|
||||
layout_sfa,
|
||||
layout_sfb,
|
||||
problem_sizes_transpose,
|
||||
expert_offsets,
|
||||
workspace);
|
||||
output = output_t.t();
|
||||
} else {
|
||||
if (is_h20_device && a.size(1) > 128) {
|
||||
// For H20 with K > 128, use Pingpong Schedule
|
||||
run_get_group_gemm_starts<
|
||||
MmaConfigH20LargeK::LayoutSFA,
|
||||
MmaConfigH20LargeK::LayoutSFB,
|
||||
MmaConfigH20LargeK::ScaleConfig>(
|
||||
expert_offsets,
|
||||
a_ptrs,
|
||||
b_ptrs,
|
||||
out_ptrs,
|
||||
a_scales_ptrs,
|
||||
b_scales_ptrs,
|
||||
a,
|
||||
b,
|
||||
output,
|
||||
scales_a,
|
||||
scales_b,
|
||||
layout_sfa,
|
||||
layout_sfb,
|
||||
problem_sizes,
|
||||
problem_sizes_transpose);
|
||||
launch_sm90_fp8_blockwise_scaled_group_mm<OutType, MmaConfigH20LargeK, cutlass::layout::RowMajor>(
|
||||
out_ptrs,
|
||||
a_ptrs,
|
||||
b_ptrs,
|
||||
a_scales_ptrs,
|
||||
b_scales_ptrs,
|
||||
stride_a,
|
||||
stride_b,
|
||||
stride_c,
|
||||
layout_sfa,
|
||||
layout_sfb,
|
||||
problem_sizes,
|
||||
expert_offsets,
|
||||
workspace);
|
||||
} else {
|
||||
// For H20 with K <= 128, and H100 & H200 & H800, use Cooperative Schedule
|
||||
run_get_group_gemm_starts<
|
||||
MmaConfigHx00AndH20SmallK::LayoutSFA,
|
||||
MmaConfigHx00AndH20SmallK::LayoutSFB,
|
||||
MmaConfigHx00AndH20SmallK::ScaleConfig>(
|
||||
expert_offsets,
|
||||
a_ptrs,
|
||||
b_ptrs,
|
||||
out_ptrs,
|
||||
a_scales_ptrs,
|
||||
b_scales_ptrs,
|
||||
a,
|
||||
b,
|
||||
output,
|
||||
scales_a,
|
||||
scales_b,
|
||||
layout_sfa,
|
||||
layout_sfb,
|
||||
problem_sizes,
|
||||
problem_sizes_transpose);
|
||||
launch_sm90_fp8_blockwise_scaled_group_mm<OutType, MmaConfigHx00AndH20SmallK, cutlass::layout::RowMajor>(
|
||||
out_ptrs,
|
||||
a_ptrs,
|
||||
b_ptrs,
|
||||
a_scales_ptrs,
|
||||
b_scales_ptrs,
|
||||
stride_a,
|
||||
stride_b,
|
||||
stride_c,
|
||||
layout_sfa,
|
||||
layout_sfb,
|
||||
problem_sizes,
|
||||
expert_offsets,
|
||||
workspace);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Performs blockwise grouped matrix multiplication on FP8 quantized inputs,
|
||||
* with per-block scaling.
|
||||
*
|
||||
* This function dispatches to hardware-specific implementations (e.g., SM100 FP8)
|
||||
* to compute:
|
||||
* C_i = scale_a[i] * A_i * scale_b[i] * B_i
|
||||
* for each expert group `i`, using input `problem_sizes` and `expert_offsets`
|
||||
* to describe the individual matrix dimensions and their offsets.
|
||||
*
|
||||
* Input tensors A and B must be quantized to 8-bit formats and dequantized before multiplication.
|
||||
* The output tensor is written with bfloat16 or half precision.
|
||||
*
|
||||
* @param output Output tensor (must be of type bfloat16 or half).
|
||||
* @param a Input tensor A (must be kFloat8_e4m3fn).
|
||||
* @param b Input tensor B (must be kFloat8_e4m3fn).
|
||||
* @param scales_a Scaling factors for tensor A, float32 per expert group.
|
||||
* @param scales_b Scaling factors for tensor B, float32 per expert group.
|
||||
* @param stride_a Stride information for tensor A (int32).
|
||||
* @param stride_b Stride information for tensor B (int32).
|
||||
* @param stride_c Stride information for output tensor C (int32).
|
||||
* @param layout_sfa Layout descriptor for A (int32), e.g., row-major/column-major.
|
||||
* @param layout_sfb Layout descriptor for B (int32).
|
||||
* @param problem_sizes 2D int32 tensor of shape (num_experts, 3), specifying (M, N, K)
|
||||
* for each grouped matrix multiplication problem.
|
||||
* @param expert_offsets 1D int32 tensor of size (num_experts), used to index into
|
||||
* the grouped input tensors for dispatch.
|
||||
* @note Performance Optimization:
|
||||
* If the batch size (a.size(0)) is smaller than 512, the implementation
|
||||
* will internally transpose input matrices to align with the optimal memory access
|
||||
* pattern for better GPU efficiency. This transformation is done within the kernel.
|
||||
*/
|
||||
void fp8_blockwise_scaled_grouped_mm(
|
||||
torch::Tensor& output,
|
||||
torch::Tensor& a_ptrs,
|
||||
torch::Tensor& b_ptrs,
|
||||
torch::Tensor& out_ptrs,
|
||||
torch::Tensor& a_scales_ptrs,
|
||||
torch::Tensor& b_scales_ptrs,
|
||||
const torch::Tensor& a,
|
||||
const torch::Tensor& b,
|
||||
const torch::Tensor& scales_a,
|
||||
const torch::Tensor& scales_b,
|
||||
const torch::Tensor& stride_a,
|
||||
const torch::Tensor& stride_b,
|
||||
const torch::Tensor& stride_c,
|
||||
const torch::Tensor& layout_sfa,
|
||||
const torch::Tensor& layout_sfb,
|
||||
const torch::Tensor& problem_sizes,
|
||||
const torch::Tensor& expert_offsets,
|
||||
const torch::Tensor& workspace) {
|
||||
TORCH_CHECK(problem_sizes.dim() == 2, "problem_sizes must be 2D tensor");
|
||||
TORCH_CHECK(problem_sizes.size(1) == 3, "problem_sizes must have shape (num_experts, 3)");
|
||||
TORCH_CHECK(
|
||||
problem_sizes.size(0) == expert_offsets.size(0), "Number of experts in problem_sizes must match expert_offsets");
|
||||
TORCH_CHECK(problem_sizes.dtype() == torch::kInt32, "problem_sizes must be int32");
|
||||
TORCH_CHECK(a.scalar_type() == torch::kFloat8_e4m3fn, "a must be kFloat8_e4m3fn");
|
||||
TORCH_CHECK(b.scalar_type() == torch::kFloat8_e4m3fn, "b must be kFloat8_e4m3fn");
|
||||
TORCH_CHECK(
|
||||
output.scalar_type() == torch::kBFloat16 || output.scalar_type() == torch::kHalf,
|
||||
"output must be bfloat16 or half");
|
||||
TORCH_CHECK(scales_a.scalar_type() == torch::kFloat32, "scales_a must be float32");
|
||||
TORCH_CHECK(scales_b.scalar_type() == torch::kFloat32, "scales_b must be float32");
|
||||
TORCH_CHECK(stride_a.scalar_type() == torch::kInt64, "stride_a must be int64");
|
||||
TORCH_CHECK(stride_b.scalar_type() == torch::kInt64, "stride_b must be int64");
|
||||
TORCH_CHECK(stride_c.scalar_type() == torch::kInt64, "stride_c must be int64");
|
||||
TORCH_CHECK(layout_sfa.scalar_type() == torch::kInt32, "layout_sfa must be int32");
|
||||
TORCH_CHECK(layout_sfb.scalar_type() == torch::kInt32, "layout_sfb must be int32");
|
||||
TORCH_CHECK(expert_offsets.scalar_type() == torch::kInt32, "expert_offsets must be int32");
|
||||
|
||||
TORCH_CHECK(output.dim() == 2, "output must be 2D tensor");
|
||||
TORCH_CHECK(a.dim() == 2, "a must be 2D tensor");
|
||||
TORCH_CHECK(b.dim() == 3, "b must be 3D tensor");
|
||||
TORCH_CHECK(scales_a.dim() == 2, "scales_a must be 2D tensor");
|
||||
TORCH_CHECK(scales_b.dim() == 3, "scales_b must be 3D tensor");
|
||||
TORCH_CHECK(stride_a.dim() == 1, "stride_a must be 1D tensor");
|
||||
TORCH_CHECK(stride_b.dim() == 1, "stride_b must be 1D tensor");
|
||||
TORCH_CHECK(stride_c.dim() == 1, "stride_c must be 1D tensor");
|
||||
TORCH_CHECK(layout_sfa.dim() == 2, "layout_sfa must be 1D tensor");
|
||||
TORCH_CHECK(layout_sfb.dim() == 2, "layout_sfb must be 1D tensor");
|
||||
TORCH_CHECK(a_ptrs.dim() == 1, "a_ptrs must be 1D tensor");
|
||||
TORCH_CHECK(b_ptrs.dim() == 1, "b_ptrs must be 1D tensor");
|
||||
TORCH_CHECK(out_ptrs.dim() == 1, "out_ptrs must be 1D tensor");
|
||||
TORCH_CHECK(a_scales_ptrs.dim() == 1, "a_scales_ptrs must be 1D tensor");
|
||||
TORCH_CHECK(b_scales_ptrs.dim() == 1, "b_scales_ptrs must be 1D tensor");
|
||||
TORCH_CHECK(expert_offsets.dim() == 1, "expert_offsets must be 1D tensor");
|
||||
TORCH_CHECK(workspace.dim() == 1, "workspace must be 1D tensor");
|
||||
|
||||
bool can_implement = false;
|
||||
auto sm_version = getSMVersion();
|
||||
|
||||
#if defined(CUTLASS_ARCH_MMA_SM100A_SUPPORTED) || defined(CUTLASS_ARCH_MMA_SM100_SUPPORTED)
|
||||
#if defined CUDA_VERSION && CUDA_VERSION >= 12080
|
||||
if (sm_version == 100
|
||||
#if CUDA_VERSION >= 12090
|
||||
|| sm_version == 103
|
||||
#endif
|
||||
) {
|
||||
if (output.scalar_type() == torch::kBFloat16) {
|
||||
sm100_fp8_blockwise_group_mm_dispatch_shape<cutlass::bfloat16_t>(
|
||||
output,
|
||||
a_ptrs,
|
||||
b_ptrs,
|
||||
out_ptrs,
|
||||
a_scales_ptrs,
|
||||
b_scales_ptrs,
|
||||
a,
|
||||
b,
|
||||
scales_a,
|
||||
scales_b,
|
||||
stride_a,
|
||||
stride_b,
|
||||
stride_c,
|
||||
layout_sfa,
|
||||
layout_sfb,
|
||||
problem_sizes,
|
||||
expert_offsets,
|
||||
workspace);
|
||||
} else {
|
||||
sm100_fp8_blockwise_group_mm_dispatch_shape<cutlass::half_t>(
|
||||
output,
|
||||
a_ptrs,
|
||||
b_ptrs,
|
||||
out_ptrs,
|
||||
a_scales_ptrs,
|
||||
b_scales_ptrs,
|
||||
a,
|
||||
b,
|
||||
scales_a,
|
||||
scales_b,
|
||||
stride_a,
|
||||
stride_b,
|
||||
stride_c,
|
||||
layout_sfa,
|
||||
layout_sfb,
|
||||
problem_sizes,
|
||||
expert_offsets,
|
||||
workspace);
|
||||
}
|
||||
can_implement = true;
|
||||
}
|
||||
#endif
|
||||
#endif
|
||||
|
||||
#if defined(CUTLASS_ARCH_MMA_SM90_SUPPORTED) && defined(CUTLASS_ARCH_MMA_MODIFIABLE_TMA_SM90_SUPPORTED)
|
||||
if (sm_version == 90) {
|
||||
if (output.scalar_type() == torch::kBFloat16) {
|
||||
sm90_fp8_blockwise_group_mm_dispatch_shape<cutlass::bfloat16_t>(
|
||||
output,
|
||||
a_ptrs,
|
||||
b_ptrs,
|
||||
out_ptrs,
|
||||
a_scales_ptrs,
|
||||
b_scales_ptrs,
|
||||
a,
|
||||
b,
|
||||
scales_a,
|
||||
scales_b,
|
||||
stride_a,
|
||||
stride_b,
|
||||
stride_c,
|
||||
layout_sfa,
|
||||
layout_sfb,
|
||||
problem_sizes,
|
||||
expert_offsets,
|
||||
workspace);
|
||||
} else {
|
||||
sm90_fp8_blockwise_group_mm_dispatch_shape<cutlass::half_t>(
|
||||
output,
|
||||
a_ptrs,
|
||||
b_ptrs,
|
||||
out_ptrs,
|
||||
a_scales_ptrs,
|
||||
b_scales_ptrs,
|
||||
a,
|
||||
b,
|
||||
scales_a,
|
||||
scales_b,
|
||||
stride_a,
|
||||
stride_b,
|
||||
stride_c,
|
||||
layout_sfa,
|
||||
layout_sfb,
|
||||
problem_sizes,
|
||||
expert_offsets,
|
||||
workspace);
|
||||
}
|
||||
can_implement = true;
|
||||
}
|
||||
#endif
|
||||
TORCH_CHECK_NOT_IMPLEMENTED(
|
||||
can_implement, "No implemented fp8_blockwise_scaled_grouped_mm for current compute capability: ", sm_version);
|
||||
}
|
||||
428
third_party/sglang/sgl-kernel/csrc/moe/fused_qknorm_rope_kernel.cu
vendored
Normal file
428
third_party/sglang/sgl-kernel/csrc/moe/fused_qknorm_rope_kernel.cu
vendored
Normal file
@@ -0,0 +1,428 @@
|
||||
/*
|
||||
* Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
// Adapted from
|
||||
// https://github.com/NVIDIA/TensorRT-LLM/blob/main/cpp/tensorrt_llm/kernels/fusedQKNormRopeKernel.cu
|
||||
|
||||
#include <ATen/cuda/Exceptions.h>
|
||||
#include <c10/cuda/CUDAGuard.h>
|
||||
#include <c10/cuda/CUDAStream.h>
|
||||
#include <cuda_bf16.h>
|
||||
#include <cuda_fp16.h>
|
||||
#include <cuda_fp8.h>
|
||||
#include <cuda_runtime.h>
|
||||
#include <torch/all.h>
|
||||
#include <torch/cuda.h>
|
||||
|
||||
#include <cmath>
|
||||
|
||||
#define CHECK_TYPE(x, st) \
|
||||
TORCH_CHECK(x.scalar_type() == st, #x " dtype is ", x.scalar_type(), ", while ", st, " is expected")
|
||||
#define CHECK_TH_CUDA(x) TORCH_CHECK(x.is_cuda(), #x " must be a CUDA tensor")
|
||||
#define CHECK_CONTIGUOUS(x) TORCH_CHECK(x.is_contiguous(), #x " must be contiguous")
|
||||
#define CHECK_INPUT(x, st) \
|
||||
CHECK_TH_CUDA(x); \
|
||||
CHECK_CONTIGUOUS(x); \
|
||||
CHECK_TYPE(x, st)
|
||||
|
||||
#define FINAL_MASK 0xffffffff
|
||||
|
||||
namespace tensorrt_llm::common {
|
||||
template <typename T, int num>
|
||||
struct packed_as;
|
||||
|
||||
// Specialization for packed_as used in this kernel.
|
||||
template <>
|
||||
struct packed_as<uint, 1> {
|
||||
using type = uint;
|
||||
};
|
||||
|
||||
template <>
|
||||
struct packed_as<uint, 2> {
|
||||
using type = uint2;
|
||||
};
|
||||
|
||||
template <>
|
||||
struct packed_as<uint, 4> {
|
||||
using type = uint4;
|
||||
};
|
||||
|
||||
template <typename T>
|
||||
__inline__ __device__ T warpReduceSum(T val) {
|
||||
#pragma unroll
|
||||
for (int mask = 16; mask > 0; mask >>= 1)
|
||||
val += __shfl_xor_sync(FINAL_MASK, val, mask,
|
||||
32); //__shfl_sync bf16 return float when sm < 80
|
||||
return val;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
inline __device__ __host__ T divUp(T m, T n) {
|
||||
return (m + n - 1) / n;
|
||||
}
|
||||
|
||||
} // namespace tensorrt_llm::common
|
||||
namespace tensorrt_llm::kernels {
|
||||
|
||||
__device__ inline float compute_freq_yarn(float base, int head_dim, int half_dim, float factor, float low, float high) {
|
||||
float freq = powf(base, -2.0f * half_dim / static_cast<float>(head_dim));
|
||||
|
||||
if (factor != 1.0f) {
|
||||
float inv_freq_extrapolation = freq;
|
||||
float inv_freq_interpolation = freq / factor;
|
||||
|
||||
float high_adj = high;
|
||||
if (fabsf(low - high_adj) <= 1e-6f) {
|
||||
high_adj += 0.001f;
|
||||
}
|
||||
|
||||
float linear_func = (static_cast<float>(half_dim) - low) / (high_adj - low);
|
||||
float ramp_func = fminf(fmaxf(linear_func, 0.0f), 1.0f);
|
||||
float inv_freq_extrapolation_factor = 1.0f - ramp_func;
|
||||
|
||||
freq = inv_freq_interpolation * (1.0f - inv_freq_extrapolation_factor) +
|
||||
inv_freq_extrapolation * inv_freq_extrapolation_factor;
|
||||
}
|
||||
|
||||
return freq;
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
// Perform per-head QK Norm and RoPE in a single kernel.
|
||||
// head_dim: the dimension of each head
|
||||
// interleave: interleave=!is_neox.
|
||||
template <int head_dim, bool interleave>
|
||||
__global__ void fusedQKNormRopeKernel(
|
||||
__nv_bfloat16* qkv, // Combined QKV tensor [num_tokens, (num_heads_q+num_heads_k+num_heads_v)*head_dim]
|
||||
int const num_heads_q, // Number of query heads
|
||||
int const num_heads_k, // Number of key heads
|
||||
int const num_heads_v, // Number of value heads
|
||||
float const eps, // Epsilon for RMS normalization
|
||||
__nv_bfloat16 const* q_weight, // RMSNorm weights for query
|
||||
__nv_bfloat16 const* k_weight, // RMSNorm weights for key
|
||||
float const base, // Base for RoPE computation
|
||||
int const* position_ids, // Position IDs for RoPE
|
||||
int const num_tokens, // Number of tokens
|
||||
// parameters for yarn
|
||||
float factor, // factor in rope_scaling in config.json. When it is not 1.0, it means the model is using yarn.
|
||||
float low, // threshold for high frequency
|
||||
float high, // threshold for low frequency
|
||||
float attention_factor, // attention_factor applied on cos and sin
|
||||
int const rotary_dim) {
|
||||
int const warpsPerBlock = blockDim.x / 32;
|
||||
int const warpId = threadIdx.x / 32;
|
||||
int const laneId = threadIdx.x % 32;
|
||||
|
||||
// Calculate global warp index to determine which head/token this warp processes
|
||||
int const globalWarpIdx = blockIdx.x * warpsPerBlock + warpId;
|
||||
|
||||
// Total number of attention heads (Q and K)
|
||||
int const total_qk_heads = num_heads_q + num_heads_k;
|
||||
|
||||
// Determine which token and head type (Q or K) this warp processes
|
||||
int const tokenIdx = globalWarpIdx / total_qk_heads;
|
||||
int const localHeadIdx = globalWarpIdx % total_qk_heads;
|
||||
|
||||
// Skip if this warp is assigned beyond the number of tokens
|
||||
if (tokenIdx >= num_tokens) return;
|
||||
|
||||
bool const isQ = localHeadIdx < num_heads_q;
|
||||
int const headIdx = isQ ? localHeadIdx : localHeadIdx - num_heads_q;
|
||||
int const num_heads = num_heads_q + num_heads_k + num_heads_v;
|
||||
static_assert(
|
||||
head_dim % (32 * 2) == 0,
|
||||
"head_dim must be divisible by 64 (each warp processes one head, and each thread gets even number of "
|
||||
"elements)");
|
||||
constexpr int numElemsPerThread = head_dim / 32;
|
||||
float elements[numElemsPerThread];
|
||||
constexpr int elemSizeBytes = numElemsPerThread * sizeof(__nv_bfloat16);
|
||||
static_assert(elemSizeBytes % 4 == 0, "numSizeBytes must be a multiple of 4");
|
||||
constexpr int vecSize = elemSizeBytes / 4; // Use packed_as<uint, vecSize> to perform loading/saving.
|
||||
using vec_T = typename tensorrt_llm::common::packed_as<uint, vecSize>::type;
|
||||
|
||||
int offsetWarp; // Offset for the warp
|
||||
if (isQ) {
|
||||
// Q segment: token offset + head offset within Q segment
|
||||
offsetWarp = tokenIdx * num_heads * head_dim + headIdx * head_dim;
|
||||
} else {
|
||||
// K segment: token offset + entire Q segment + head offset within K segment
|
||||
offsetWarp = tokenIdx * num_heads * head_dim + num_heads_q * head_dim + headIdx * head_dim;
|
||||
}
|
||||
int offsetThread = offsetWarp + laneId * numElemsPerThread;
|
||||
|
||||
// Sum of squares for RMSNorm
|
||||
float sumOfSquares = 0.0f;
|
||||
|
||||
// Load.
|
||||
{
|
||||
vec_T vec = *reinterpret_cast<vec_T const*>(&qkv[offsetThread]);
|
||||
for (int i = 0; i < vecSize; i++) {
|
||||
float2 vals = __bfloat1622float2(*reinterpret_cast<__nv_bfloat162*>(reinterpret_cast<uint*>(&vec) + i));
|
||||
sumOfSquares += vals.x * vals.x;
|
||||
sumOfSquares += vals.y * vals.y;
|
||||
elements[2 * i] = vals.x;
|
||||
elements[2 * i + 1] = vals.y;
|
||||
}
|
||||
}
|
||||
|
||||
// Reduce sum across warp using the utility function
|
||||
sumOfSquares = tensorrt_llm::common::warpReduceSum(sumOfSquares);
|
||||
|
||||
// Compute RMS normalization factor
|
||||
float rms_rcp = rsqrtf(sumOfSquares / static_cast<float>(head_dim) + eps);
|
||||
|
||||
// Normalize elements
|
||||
for (int i = 0; i < numElemsPerThread; i++) {
|
||||
int dim = laneId * numElemsPerThread + i;
|
||||
float weight = isQ ? __bfloat162float(q_weight[dim]) : __bfloat162float(k_weight[dim]);
|
||||
elements[i] *= rms_rcp * weight;
|
||||
}
|
||||
// Apply RoPE to normalized elements
|
||||
float elements2[numElemsPerThread]; // Additional buffer required for RoPE.
|
||||
float cos_vals[numElemsPerThread];
|
||||
float sin_vals[numElemsPerThread];
|
||||
float pos_id = static_cast<float>(position_ids[tokenIdx]);
|
||||
int const rotary_lanes = rotary_dim / numElemsPerThread; // rotary range
|
||||
bool const applyRotary = (laneId < rotary_lanes);
|
||||
if (applyRotary) {
|
||||
if constexpr (interleave) {
|
||||
// Perform interleaving. Fill cos_vals and sin_vals.
|
||||
for (int i = 0; i < numElemsPerThread; i++) {
|
||||
elements2[i] = (i % 2 == 0) ? -elements[i + 1] : elements[i - 1];
|
||||
|
||||
int dim_idx = laneId * numElemsPerThread + i;
|
||||
int half_dim = dim_idx / 2;
|
||||
float freq = compute_freq_yarn(base, rotary_dim, half_dim, factor, low, high);
|
||||
float theta = pos_id * freq;
|
||||
__sincosf(theta, &sin_vals[i], &cos_vals[i]);
|
||||
}
|
||||
|
||||
} else {
|
||||
// Neox style
|
||||
// Before data exchange with in warp, we need to sync.
|
||||
__syncwarp();
|
||||
int const half_rotary_lanes = rotary_lanes / 2;
|
||||
unsigned int active_mask = (1u << rotary_lanes) - 1;
|
||||
// Limitation: The operation below requires half_rotary_lanes to be a power of 2.
|
||||
// because it relies on __shfl_xor_sync to exchange data within a warp.
|
||||
for (int i = 0; i < numElemsPerThread; i++) {
|
||||
elements2[i] = __shfl_xor_sync(active_mask, elements[i], half_rotary_lanes);
|
||||
if (laneId < half_rotary_lanes) {
|
||||
elements2[i] = -elements2[i];
|
||||
}
|
||||
|
||||
int dim_idx = laneId * numElemsPerThread + i;
|
||||
dim_idx = (dim_idx * 2) % rotary_dim;
|
||||
int half_dim = dim_idx / 2;
|
||||
float freq = compute_freq_yarn(base, rotary_dim, half_dim, factor, low, high);
|
||||
float theta = pos_id * freq;
|
||||
__sincosf(theta, &sin_vals[i], &cos_vals[i]);
|
||||
}
|
||||
// __shfl_xor_sync does not provide memfence. Need to sync again.
|
||||
__syncwarp();
|
||||
}
|
||||
|
||||
for (int i = 0; i < numElemsPerThread; i++) {
|
||||
elements[i] = (elements[i] * cos_vals[i] + elements2[i] * sin_vals[i]) * attention_factor;
|
||||
}
|
||||
}
|
||||
|
||||
// Store.
|
||||
{
|
||||
vec_T vec;
|
||||
for (int i = 0; i < vecSize; i++) {
|
||||
__nv_bfloat162 vals = __float22bfloat162_rn(make_float2(elements[2 * i], elements[2 * i + 1]));
|
||||
reinterpret_cast<__nv_bfloat162&>(*(reinterpret_cast<uint*>(&vec) + i)) = vals;
|
||||
}
|
||||
vec_T* outputPtr = reinterpret_cast<vec_T*>(&qkv[offsetThread]);
|
||||
*outputPtr = vec;
|
||||
}
|
||||
}
|
||||
// Borrowed from
|
||||
// https://github.com/flashinfer-ai/flashinfer/blob/8125d079a43e9a0ba463a4ed1b639cefd084cec9/include/flashinfer/pos_enc.cuh#L568
|
||||
#define DISPATCH_INTERLEAVE(interleave, INTERLEAVE, ...) \
|
||||
if (interleave) { \
|
||||
const bool INTERLEAVE = true; \
|
||||
__VA_ARGS__ \
|
||||
} else { \
|
||||
const bool INTERLEAVE = false; \
|
||||
__VA_ARGS__ \
|
||||
}
|
||||
void launchFusedQKNormRope(
|
||||
void* qkv,
|
||||
int const num_tokens,
|
||||
int const num_heads_q,
|
||||
int const num_heads_k,
|
||||
int const num_heads_v,
|
||||
int const head_dim,
|
||||
float const eps,
|
||||
void const* q_weight,
|
||||
void const* k_weight,
|
||||
float const base,
|
||||
bool const interleave,
|
||||
int const* position_ids,
|
||||
float factor,
|
||||
float low,
|
||||
float high,
|
||||
float attention_factor,
|
||||
int const rotary_dim,
|
||||
cudaStream_t stream) {
|
||||
constexpr int blockSize = 256;
|
||||
int const warpsPerBlock = blockSize / 32;
|
||||
int const totalQKHeads = num_heads_q + num_heads_k;
|
||||
int const totalWarps = num_tokens * totalQKHeads;
|
||||
int const gridSize = common::divUp(totalWarps, warpsPerBlock);
|
||||
dim3 gridDim(gridSize);
|
||||
dim3 blockDim(blockSize);
|
||||
// Head dimensions should be a multiple of 64
|
||||
// Add more cases as needed
|
||||
switch (head_dim) {
|
||||
case 64:
|
||||
DISPATCH_INTERLEAVE(interleave, INTERLEAVE, {
|
||||
fusedQKNormRopeKernel<64, INTERLEAVE><<<gridDim, blockDim, 0, stream>>>(
|
||||
reinterpret_cast<__nv_bfloat16*>(qkv),
|
||||
num_heads_q,
|
||||
num_heads_k,
|
||||
num_heads_v,
|
||||
eps,
|
||||
reinterpret_cast<__nv_bfloat16 const*>(q_weight),
|
||||
reinterpret_cast<__nv_bfloat16 const*>(k_weight),
|
||||
base,
|
||||
position_ids,
|
||||
num_tokens,
|
||||
factor,
|
||||
low,
|
||||
high,
|
||||
attention_factor,
|
||||
rotary_dim);
|
||||
});
|
||||
break;
|
||||
case 128:
|
||||
DISPATCH_INTERLEAVE(interleave, INTERLEAVE, {
|
||||
fusedQKNormRopeKernel<128, INTERLEAVE><<<gridDim, blockDim, 0, stream>>>(
|
||||
reinterpret_cast<__nv_bfloat16*>(qkv),
|
||||
num_heads_q,
|
||||
num_heads_k,
|
||||
num_heads_v,
|
||||
eps,
|
||||
reinterpret_cast<__nv_bfloat16 const*>(q_weight),
|
||||
reinterpret_cast<__nv_bfloat16 const*>(k_weight),
|
||||
base,
|
||||
position_ids,
|
||||
num_tokens,
|
||||
factor,
|
||||
low,
|
||||
high,
|
||||
attention_factor,
|
||||
rotary_dim);
|
||||
});
|
||||
break;
|
||||
case 256:
|
||||
DISPATCH_INTERLEAVE(interleave, INTERLEAVE, {
|
||||
fusedQKNormRopeKernel<256, INTERLEAVE><<<gridDim, blockDim, 0, stream>>>(
|
||||
reinterpret_cast<__nv_bfloat16*>(qkv),
|
||||
num_heads_q,
|
||||
num_heads_k,
|
||||
num_heads_v,
|
||||
eps,
|
||||
reinterpret_cast<__nv_bfloat16 const*>(q_weight),
|
||||
reinterpret_cast<__nv_bfloat16 const*>(k_weight),
|
||||
base,
|
||||
position_ids,
|
||||
num_tokens,
|
||||
factor,
|
||||
low,
|
||||
high,
|
||||
attention_factor,
|
||||
rotary_dim);
|
||||
});
|
||||
break;
|
||||
default:
|
||||
TORCH_CHECK(false, "Unsupported head dimension for fusedQKNormRope: ", head_dim);
|
||||
}
|
||||
}
|
||||
} // namespace tensorrt_llm::kernels
|
||||
|
||||
// Function for fused QK Norm and RoPE
|
||||
// This operator applies RMS normalization and RoPE to Q and K tensors in a single CUDA kernel.
|
||||
// The OP performs operations in-place on the input qkv tensor.
|
||||
void fused_qk_norm_rope(
|
||||
torch::Tensor& qkv, // Combined QKV tensor [num_tokens, (num_heads_q+num_heads_k+num_heads_v)*head_dim]
|
||||
int64_t num_heads_q, // Number of query heads
|
||||
int64_t num_heads_k, // Number of key heads
|
||||
int64_t num_heads_v, // Number of value heads
|
||||
int64_t head_dim, // Dimension per head
|
||||
double eps, // Epsilon for RMS normalization
|
||||
torch::Tensor& q_weight, // RMSNorm weights for query [head_dim]
|
||||
torch::Tensor& k_weight, // RMSNorm weights for key [head_dim]
|
||||
double base, // Base for RoPE computation
|
||||
bool is_neox, // Whether RoPE is applied in Neox style
|
||||
torch::Tensor& position_ids, // Position IDs for RoPE [num_tokens]
|
||||
// parameters for yarn
|
||||
double factor, // factor in rope_scaling in config.json. When it is not 1.0, it means the model is using yarn.
|
||||
double low, // threshold for high frequency
|
||||
double high, // threshold for low frequency
|
||||
double attention_factor, // attention_factor applied on cos and sin
|
||||
int64_t rotary_dim) {
|
||||
// Input validation
|
||||
TORCH_CHECK(qkv.dim() == 2, "QKV tensor must be 2D: [num_tokens, (num_heads_q+num_heads_k+num_heads_v)*head_dim]");
|
||||
TORCH_CHECK(position_ids.dim() == 1, "Position IDs must be 1D: [num_tokens]");
|
||||
TORCH_CHECK(q_weight.dim() == 1, "Query weights must be 1D: [head_dim]");
|
||||
TORCH_CHECK(k_weight.dim() == 1, "Key weights must be 1D: [head_dim]");
|
||||
TORCH_CHECK(q_weight.size(0) == head_dim, "Query weights size must match head dimension");
|
||||
TORCH_CHECK(k_weight.size(0) == head_dim, "Key weights size must match head dimension");
|
||||
TORCH_CHECK(rotary_dim % (head_dim / 32) == 0, "rotary_dim must be divisible by numElemsPerThread");
|
||||
if (is_neox) {
|
||||
int64_t half_rotary_lanes = rotary_dim / (head_dim / 32) / 2;
|
||||
TORCH_CHECK(
|
||||
half_rotary_lanes >= 1 && (half_rotary_lanes & (half_rotary_lanes - 1)) == 0,
|
||||
"half_rotary_lanes must be a power of 2 for neox style, got ",
|
||||
half_rotary_lanes);
|
||||
}
|
||||
CHECK_INPUT(qkv, torch::kBFloat16);
|
||||
CHECK_INPUT(position_ids, torch::kInt32);
|
||||
CHECK_INPUT(q_weight, torch::kBFloat16);
|
||||
CHECK_INPUT(k_weight, torch::kBFloat16);
|
||||
|
||||
int64_t num_tokens = qkv.size(0);
|
||||
TORCH_CHECK(position_ids.size(0) == num_tokens, "Number of tokens in position_ids must match QKV");
|
||||
|
||||
int64_t total_heads = num_heads_q + num_heads_k + num_heads_v;
|
||||
TORCH_CHECK(
|
||||
qkv.size(1) == total_heads * head_dim, "QKV tensor size must match total number of heads and head dimension");
|
||||
|
||||
auto stream = at::cuda::getCurrentCUDAStream(qkv.get_device());
|
||||
|
||||
tensorrt_llm::kernels::launchFusedQKNormRope(
|
||||
reinterpret_cast<__nv_bfloat16*>(qkv.data_ptr()),
|
||||
static_cast<int>(num_tokens),
|
||||
static_cast<int>(num_heads_q),
|
||||
static_cast<int>(num_heads_k),
|
||||
static_cast<int>(num_heads_v),
|
||||
static_cast<int>(head_dim),
|
||||
static_cast<float>(eps),
|
||||
reinterpret_cast<__nv_bfloat16*>(q_weight.data_ptr()),
|
||||
reinterpret_cast<__nv_bfloat16*>(k_weight.data_ptr()),
|
||||
static_cast<float>(base),
|
||||
!is_neox, // interleave
|
||||
reinterpret_cast<int const*>(position_ids.data_ptr()),
|
||||
static_cast<float>(factor),
|
||||
static_cast<float>(low),
|
||||
static_cast<float>(high),
|
||||
static_cast<float>(attention_factor),
|
||||
static_cast<int>(rotary_dim),
|
||||
stream);
|
||||
}
|
||||
319
third_party/sglang/sgl-kernel/csrc/moe/kimi_k2_moe_fused_gate.cu
vendored
Normal file
319
third_party/sglang/sgl-kernel/csrc/moe/kimi_k2_moe_fused_gate.cu
vendored
Normal file
@@ -0,0 +1,319 @@
|
||||
#include <ATen/cuda/CUDAContext.h>
|
||||
#include <cuda_runtime.h>
|
||||
#include <torch/all.h>
|
||||
|
||||
#include <cfloat>
|
||||
|
||||
// Kimi K2 specific constants
|
||||
static constexpr int WARP_SIZE = 32;
|
||||
static constexpr int WARPS_PER_CTA = 6;
|
||||
static constexpr int NUM_EXPERTS = 384;
|
||||
static constexpr int VPT = 12; // 384 / 32 = 12
|
||||
|
||||
// Small token optimization constants
|
||||
static constexpr int SMALL_TOKEN_THRESHOLD = 512;
|
||||
static constexpr int WARPS_PER_TOKEN_SMALL = 12; // Use 12 warps per token for small batches
|
||||
static constexpr int THREADS_PER_BLOCK_SMALL = WARPS_PER_TOKEN_SMALL * WARP_SIZE; // 384 threads
|
||||
|
||||
// Vectorization constants (used by large token kernel)
|
||||
static constexpr int VEC_SIZE = 4; // Use float4 for vectorized loads
|
||||
|
||||
// Small token optimized kernel: Each warp independently finds top-k, then merge, using warp-level topk
|
||||
__global__ void kimi_k2_moe_fused_gate_kernel_small_token(
|
||||
float* input,
|
||||
float* bias,
|
||||
float* output_ptr,
|
||||
int32_t* indices_ptr,
|
||||
int64_t num_rows,
|
||||
int64_t topk,
|
||||
bool renormalize,
|
||||
double routed_scaling_factor,
|
||||
bool apply_routed_scaling_factor_on_output) {
|
||||
int64_t row_idx = blockIdx.x;
|
||||
if (row_idx >= num_rows) return;
|
||||
|
||||
int tid = threadIdx.x;
|
||||
int warp_id = tid / WARP_SIZE;
|
||||
int lane_id = tid % WARP_SIZE;
|
||||
|
||||
// Shared memory: biased scores and original scores
|
||||
__shared__ float shared_scores[NUM_EXPERTS];
|
||||
__shared__ float shared_original_scores[NUM_EXPERTS];
|
||||
// For storing selected top-k indices and values
|
||||
__shared__ int selected_experts[8]; // Up to topk=6, I use 8 for alignment
|
||||
__shared__ float selected_vals[8];
|
||||
// For warp-level reduction
|
||||
__shared__ float warp_maxs[WARPS_PER_TOKEN_SMALL];
|
||||
__shared__ int warp_experts[WARPS_PER_TOKEN_SMALL];
|
||||
|
||||
// Load data: all 384 threads load one expert each
|
||||
if (tid < NUM_EXPERTS) {
|
||||
float input_val = input[row_idx * NUM_EXPERTS + tid];
|
||||
float bias_val = bias[tid];
|
||||
float sigmoid_val = 1.0f / (1.0f + expf(-input_val));
|
||||
float biased_val = sigmoid_val + bias_val;
|
||||
shared_scores[tid] = biased_val;
|
||||
shared_original_scores[tid] = sigmoid_val;
|
||||
}
|
||||
|
||||
__syncthreads();
|
||||
|
||||
// Find top-k using iterative selection, each iteration finds the next maximum
|
||||
for (int k = 0; k < topk; k++) {
|
||||
// Each thread holds one expert's value
|
||||
float my_val = (tid < NUM_EXPERTS) ? shared_scores[tid] : -FLT_MAX;
|
||||
int my_expert = tid;
|
||||
|
||||
// Use warp-level reduction first
|
||||
float warp_max_val = my_val;
|
||||
int warp_max_expert = my_expert;
|
||||
|
||||
#pragma unroll
|
||||
for (int offset = 16; offset > 0; offset /= 2) {
|
||||
float other_val = __shfl_down_sync(0xFFFFFFFF, warp_max_val, offset);
|
||||
int other_expert = __shfl_down_sync(0xFFFFFFFF, warp_max_expert, offset);
|
||||
if (other_val > warp_max_val) {
|
||||
warp_max_val = other_val;
|
||||
warp_max_expert = other_expert;
|
||||
}
|
||||
}
|
||||
|
||||
// Warp leaders write to shared memory
|
||||
if (lane_id == 0) {
|
||||
warp_maxs[warp_id] = warp_max_val;
|
||||
warp_experts[warp_id] = warp_max_expert;
|
||||
}
|
||||
|
||||
__syncthreads();
|
||||
|
||||
// Final reduction among warps (done by first warp)
|
||||
if (warp_id == 0) {
|
||||
float final_max = (lane_id < WARPS_PER_TOKEN_SMALL) ? warp_maxs[lane_id] : -FLT_MAX;
|
||||
int final_expert = (lane_id < WARPS_PER_TOKEN_SMALL) ? warp_experts[lane_id] : -1;
|
||||
|
||||
#pragma unroll
|
||||
for (int offset = 16; offset > 0; offset /= 2) {
|
||||
float other_val = __shfl_down_sync(0xFFFFFFFF, final_max, offset);
|
||||
int other_expert = __shfl_down_sync(0xFFFFFFFF, final_expert, offset);
|
||||
if (other_val > final_max) {
|
||||
final_max = other_val;
|
||||
final_expert = other_expert;
|
||||
}
|
||||
}
|
||||
|
||||
if (lane_id == 0) {
|
||||
selected_experts[k] = final_expert;
|
||||
selected_vals[k] = final_max;
|
||||
}
|
||||
}
|
||||
|
||||
__syncthreads();
|
||||
|
||||
// Mark the selected expert as used for next iteration
|
||||
// All threads can read from selected_experts[k]
|
||||
int selected = selected_experts[k];
|
||||
if (tid == selected) {
|
||||
shared_scores[tid] = -FLT_MAX;
|
||||
}
|
||||
|
||||
__syncthreads();
|
||||
}
|
||||
|
||||
// Write output (done by thread 0)
|
||||
if (tid == 0) {
|
||||
for (int k = 0; k < topk; k++) {
|
||||
int expert_id = selected_experts[k];
|
||||
if (expert_id >= 0 && expert_id < NUM_EXPERTS) {
|
||||
output_ptr[row_idx * topk + k] = shared_original_scores[expert_id];
|
||||
indices_ptr[row_idx * topk + k] = expert_id;
|
||||
} else {
|
||||
output_ptr[row_idx * topk + k] = 0.0f;
|
||||
indices_ptr[row_idx * topk + k] = 0;
|
||||
}
|
||||
}
|
||||
|
||||
// Renormalization
|
||||
if (renormalize) {
|
||||
float sum = 0.0f;
|
||||
for (int k = 0; k < topk; k++) {
|
||||
sum += output_ptr[row_idx * topk + k];
|
||||
}
|
||||
|
||||
if (sum > 0.0f) {
|
||||
for (int k = 0; k < topk; k++) {
|
||||
int64_t idx = row_idx * topk + k;
|
||||
output_ptr[idx] /= sum;
|
||||
if (apply_routed_scaling_factor_on_output) {
|
||||
output_ptr[idx] *= static_cast<float>(routed_scaling_factor);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Large token kernel: Original implementation with vectorized loads
|
||||
__global__ void kimi_k2_moe_fused_gate_kernel(
|
||||
float* input,
|
||||
float* bias,
|
||||
float* output_ptr,
|
||||
int32_t* indices_ptr,
|
||||
int64_t num_rows,
|
||||
int64_t topk,
|
||||
bool renormalize,
|
||||
double routed_scaling_factor,
|
||||
bool apply_routed_scaling_factor_on_output) {
|
||||
int64_t row_idx = blockIdx.x * WARPS_PER_CTA + threadIdx.y;
|
||||
if (row_idx >= num_rows) return;
|
||||
|
||||
int lane_id = threadIdx.x;
|
||||
int warp_id = threadIdx.y;
|
||||
|
||||
__shared__ float shared_scores[NUM_EXPERTS * WARPS_PER_CTA];
|
||||
__shared__ float shared_original_scores[NUM_EXPERTS * WARPS_PER_CTA];
|
||||
|
||||
float* warp_scores = shared_scores + warp_id * NUM_EXPERTS;
|
||||
float* warp_original_scores = shared_original_scores + warp_id * NUM_EXPERTS;
|
||||
|
||||
// Vectorized loading: each lane loads multiple float4 chunks
|
||||
// VPT = 12, so we load 12/4 = 3 float4 per lane
|
||||
const int VEC_PER_LANE = VPT / VEC_SIZE; // 3
|
||||
float4* input_vec = reinterpret_cast<float4*>(input + row_idx * NUM_EXPERTS);
|
||||
float4* bias_vec = reinterpret_cast<float4*>(bias);
|
||||
|
||||
#pragma unroll
|
||||
for (int i = 0; i < VEC_PER_LANE; i++) {
|
||||
int vec_idx = lane_id * VEC_PER_LANE + i;
|
||||
float4 input_val = input_vec[vec_idx];
|
||||
float4 bias_val = bias_vec[vec_idx];
|
||||
|
||||
#pragma unroll
|
||||
for (int j = 0; j < VEC_SIZE; j++) {
|
||||
int expert = vec_idx * VEC_SIZE + j;
|
||||
float inp = ((float*)&input_val)[j];
|
||||
float b = ((float*)&bias_val)[j];
|
||||
float sigmoid_val = 1.0f / (1.0f + expf(-inp));
|
||||
float biased_val = sigmoid_val + b;
|
||||
warp_scores[expert] = biased_val;
|
||||
warp_original_scores[expert] = sigmoid_val;
|
||||
}
|
||||
}
|
||||
|
||||
__syncthreads();
|
||||
|
||||
for (int k = 0; k < topk; k++) {
|
||||
float max_val = -FLT_MAX;
|
||||
int max_expert = -1;
|
||||
|
||||
for (int expert = lane_id; expert < NUM_EXPERTS; expert += WARP_SIZE) {
|
||||
if (warp_scores[expert] > max_val) {
|
||||
max_val = warp_scores[expert];
|
||||
max_expert = expert;
|
||||
}
|
||||
}
|
||||
|
||||
for (int offset = WARP_SIZE / 2; offset > 0; offset /= 2) {
|
||||
float other_val = __shfl_down_sync(0xFFFFFFFF, max_val, offset);
|
||||
int other_expert = __shfl_down_sync(0xFFFFFFFF, max_expert, offset);
|
||||
|
||||
if (other_val > max_val || (other_val == max_val && other_expert < max_expert)) {
|
||||
max_val = other_val;
|
||||
max_expert = other_expert;
|
||||
}
|
||||
}
|
||||
|
||||
if (lane_id == 0) {
|
||||
int64_t output_idx = row_idx * topk + k;
|
||||
if (max_expert != -1) {
|
||||
output_ptr[output_idx] = warp_original_scores[max_expert];
|
||||
indices_ptr[output_idx] = max_expert;
|
||||
warp_scores[max_expert] = -FLT_MAX;
|
||||
} else {
|
||||
output_ptr[output_idx] = 0.0f;
|
||||
indices_ptr[output_idx] = 0;
|
||||
}
|
||||
}
|
||||
|
||||
__syncwarp();
|
||||
}
|
||||
|
||||
__syncthreads();
|
||||
|
||||
if (renormalize && lane_id == 0) {
|
||||
float sum = 0.0f;
|
||||
for (int k = 0; k < topk; k++) {
|
||||
sum += output_ptr[row_idx * topk + k];
|
||||
}
|
||||
|
||||
if (sum > 0.0f) {
|
||||
for (int k = 0; k < topk; k++) {
|
||||
int64_t idx = row_idx * topk + k;
|
||||
output_ptr[idx] /= sum;
|
||||
if (apply_routed_scaling_factor_on_output) {
|
||||
output_ptr[idx] *= static_cast<float>(routed_scaling_factor);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
std::vector<at::Tensor> kimi_k2_moe_fused_gate(
|
||||
at::Tensor& input,
|
||||
at::Tensor& bias,
|
||||
int64_t topk,
|
||||
bool renormalize,
|
||||
double routed_scaling_factor,
|
||||
bool apply_routed_scaling_factor_on_output) {
|
||||
int64_t num_rows = input.size(0);
|
||||
int32_t num_experts = input.size(1);
|
||||
|
||||
// Assert: Only support 384 experts
|
||||
TORCH_CHECK(num_experts == 384, "kimi_k2_moe_fused_gate only supports 384 experts, but got ", num_experts);
|
||||
TORCH_CHECK(input.dtype() == bias.dtype(), "input and bias should have the same dtype");
|
||||
|
||||
auto options = torch::TensorOptions().dtype(torch::kFloat32).device(torch::kCUDA);
|
||||
auto output = torch::empty({num_rows, topk}, options);
|
||||
auto indices = torch::empty({num_rows, topk}, options.dtype(torch::kInt32));
|
||||
|
||||
const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
|
||||
|
||||
// Only support float32
|
||||
TORCH_CHECK(input.scalar_type() == at::kFloat, "kimi_k2_moe_fused_gate only supports float32 input");
|
||||
TORCH_CHECK(bias.scalar_type() == at::kFloat, "kimi_k2_moe_fused_gate only supports float32 bias");
|
||||
|
||||
bool use_small_token_kernel = num_rows <= SMALL_TOKEN_THRESHOLD;
|
||||
|
||||
if (use_small_token_kernel) {
|
||||
// Small token kernel: Each block handles 1 token with multiple warps collaborating
|
||||
int64_t num_blocks = num_rows;
|
||||
dim3 block_dim(THREADS_PER_BLOCK_SMALL);
|
||||
|
||||
kimi_k2_moe_fused_gate_kernel_small_token<<<num_blocks, block_dim, 0, stream>>>(
|
||||
input.data_ptr<float>(),
|
||||
bias.data_ptr<float>(),
|
||||
output.data_ptr<float>(),
|
||||
indices.data_ptr<int32_t>(),
|
||||
num_rows,
|
||||
topk,
|
||||
renormalize,
|
||||
routed_scaling_factor,
|
||||
apply_routed_scaling_factor_on_output);
|
||||
} else {
|
||||
// Large token kernel: Original implementation
|
||||
int64_t num_blocks = (num_rows + WARPS_PER_CTA - 1) / WARPS_PER_CTA;
|
||||
dim3 block_dim(WARP_SIZE, WARPS_PER_CTA);
|
||||
|
||||
kimi_k2_moe_fused_gate_kernel<<<num_blocks, block_dim, 0, stream>>>(
|
||||
input.data_ptr<float>(),
|
||||
bias.data_ptr<float>(),
|
||||
output.data_ptr<float>(),
|
||||
indices.data_ptr<int32_t>(),
|
||||
num_rows,
|
||||
topk,
|
||||
renormalize,
|
||||
routed_scaling_factor,
|
||||
apply_routed_scaling_factor_on_output);
|
||||
}
|
||||
|
||||
return {output, indices};
|
||||
}
|
||||
383
third_party/sglang/sgl-kernel/csrc/moe/moe_align_kernel.cu
vendored
Normal file
383
third_party/sglang/sgl-kernel/csrc/moe/moe_align_kernel.cu
vendored
Normal file
@@ -0,0 +1,383 @@
|
||||
/* Copyright 2025 SGLang Team. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
#include <ATen/ATen.h>
|
||||
#include <ATen/cuda/CUDAContext.h>
|
||||
#include <c10/cuda/CUDAGuard.h>
|
||||
|
||||
#include <THC/THCAtomics.cuh>
|
||||
|
||||
#include "utils.h"
|
||||
|
||||
#define VEC_SIZE 4
|
||||
using Vec = int4;
|
||||
|
||||
template <typename scalar_t>
|
||||
__global__ void count_and_sort_expert_tokens_kernel(
|
||||
const scalar_t* __restrict__ topk_ids,
|
||||
int32_t* __restrict__ sorted_token_ids,
|
||||
int32_t* __restrict__ cumsum_buffer,
|
||||
size_t numel) {
|
||||
const size_t tid = blockIdx.x * blockDim.x + threadIdx.x;
|
||||
const size_t stride = blockDim.x * gridDim.x;
|
||||
|
||||
for (size_t i = tid; i < numel; i += stride) {
|
||||
int32_t expert_id = topk_ids[i] + 1;
|
||||
int32_t rank_post_pad = atomicAdd(&cumsum_buffer[expert_id], 1);
|
||||
sorted_token_ids[rank_post_pad] = i;
|
||||
}
|
||||
}
|
||||
|
||||
#ifdef __CUDA_ARCH__
|
||||
__device__ __forceinline__ int warp_exclusive_scan(int v, unsigned mask = 0xffffffffu) {
|
||||
int original = v;
|
||||
#pragma unroll
|
||||
for (int offset = 1; offset < WARP_SIZE; offset <<= 1) {
|
||||
int n = __shfl_up_sync(mask, v, offset);
|
||||
if ((threadIdx.x & (WARP_SIZE - 1)) >= offset) v += n;
|
||||
}
|
||||
return v - original;
|
||||
}
|
||||
#endif
|
||||
|
||||
template <typename scalar_t>
|
||||
__global__ void moe_align_block_size_kernel(
|
||||
const scalar_t* __restrict__ topk_ids,
|
||||
int32_t* __restrict__ sorted_token_ids,
|
||||
int32_t* __restrict__ expert_ids,
|
||||
int32_t* __restrict__ total_tokens_post_pad,
|
||||
int32_t num_experts,
|
||||
int32_t block_size,
|
||||
size_t numel,
|
||||
int32_t* __restrict__ cumsum,
|
||||
bool pad_sorted_token_ids,
|
||||
const int32_t scan_size,
|
||||
int32_t max_num_tokens_padded) {
|
||||
// Use a separate thread block to populate sorted_token_ids
|
||||
if (blockIdx.x == 1) {
|
||||
if (pad_sorted_token_ids) {
|
||||
Vec fill_vec;
|
||||
fill_vec.x = fill_vec.y = fill_vec.z = fill_vec.w = numel;
|
||||
int32_t total_vecs = (max_num_tokens_padded + VEC_SIZE - 1) / VEC_SIZE;
|
||||
Vec* out_ptr = reinterpret_cast<Vec*>(sorted_token_ids);
|
||||
for (int32_t i = threadIdx.x; i < total_vecs; i += blockDim.x) {
|
||||
out_ptr[i] = fill_vec;
|
||||
}
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
extern __shared__ int32_t smem[];
|
||||
int32_t* shared_counts = smem; // [num_experts]
|
||||
int32_t* prefix = shared_counts + num_experts; // [num_experts + 1]
|
||||
int32_t* scan_buf = prefix + num_experts + 1; // [scan_size]
|
||||
__shared__ int32_t s_total_tokens_post_pad;
|
||||
|
||||
const size_t tid = threadIdx.x;
|
||||
const size_t stride = blockDim.x;
|
||||
|
||||
if (tid < num_experts) {
|
||||
shared_counts[tid] = 0;
|
||||
}
|
||||
|
||||
__syncthreads();
|
||||
|
||||
for (size_t i = tid; i < numel; i += stride) {
|
||||
int expert_id = topk_ids[i] + 1;
|
||||
atomicAdd(&shared_counts[expert_id], 1);
|
||||
}
|
||||
|
||||
__syncthreads();
|
||||
|
||||
int32_t padded_count = 0;
|
||||
if (tid < num_experts) {
|
||||
int32_t count = shared_counts[tid];
|
||||
padded_count = (count + block_size - 1) / block_size * block_size;
|
||||
scan_buf[tid] = padded_count;
|
||||
}
|
||||
|
||||
#ifndef __CUDA_ARCH__ // HIP
|
||||
|
||||
if (tid >= num_experts && tid < scan_size) {
|
||||
scan_buf[tid] = 0;
|
||||
}
|
||||
|
||||
__syncthreads();
|
||||
|
||||
// Blelloch scan
|
||||
int offset = 1;
|
||||
#pragma unroll
|
||||
for (int d = scan_size >> 1; d > 0; d >>= 1) {
|
||||
if (tid < d) {
|
||||
int ai = offset * (2 * tid + 1) - 1;
|
||||
int bi = offset * (2 * tid + 2) - 1;
|
||||
scan_buf[bi] += scan_buf[ai];
|
||||
}
|
||||
offset <<= 1;
|
||||
__syncthreads();
|
||||
}
|
||||
|
||||
// down-sweep
|
||||
if (tid == 0) {
|
||||
prefix[num_experts] = scan_buf[scan_size - 1];
|
||||
scan_buf[scan_size - 1] = 0;
|
||||
}
|
||||
__syncthreads();
|
||||
|
||||
#pragma unroll
|
||||
for (int d = 1; d < scan_size; d <<= 1) {
|
||||
offset >>= 1;
|
||||
if (tid < d) {
|
||||
int ai = offset * (2 * tid + 1) - 1;
|
||||
int bi = offset * (2 * tid + 2) - 1;
|
||||
if (bi < scan_size) {
|
||||
int temp = scan_buf[ai];
|
||||
scan_buf[ai] = scan_buf[bi];
|
||||
scan_buf[bi] += temp;
|
||||
}
|
||||
}
|
||||
__syncthreads();
|
||||
}
|
||||
|
||||
if (tid < num_experts) {
|
||||
prefix[tid] = scan_buf[tid];
|
||||
}
|
||||
|
||||
if (tid == 0) {
|
||||
s_total_tokens_post_pad = prefix[num_experts];
|
||||
*total_tokens_post_pad = s_total_tokens_post_pad;
|
||||
}
|
||||
__syncthreads();
|
||||
|
||||
#else // CUDA
|
||||
|
||||
// Intra warp prefix sum
|
||||
int32_t* warp_sums = scan_buf + scan_size; // [<= 32]
|
||||
const int warp_id = tid / WARP_SIZE;
|
||||
const int lane_id = tid & (WARP_SIZE - 1);
|
||||
const int num_warps_for_scan = (scan_size + WARP_SIZE - 1) / WARP_SIZE;
|
||||
const int warp_sum = warp_exclusive_scan(padded_count) + padded_count;
|
||||
if (lane_id == WARP_SIZE - 1) warp_sums[warp_id] = warp_sum;
|
||||
__syncthreads();
|
||||
|
||||
// warp0 accumulate all the block's prefix sum
|
||||
if (tid < WARP_SIZE) {
|
||||
int val = (tid < num_warps_for_scan) ? warp_sums[tid] : 0;
|
||||
int incl = warp_exclusive_scan(val) + val;
|
||||
warp_sums[tid] = incl;
|
||||
}
|
||||
__syncthreads();
|
||||
|
||||
// Every thread obtains the whole block's sum
|
||||
if (tid == 0) {
|
||||
prefix[num_experts] = warp_sums[num_warps_for_scan - 1];
|
||||
s_total_tokens_post_pad = prefix[num_experts];
|
||||
*total_tokens_post_pad = s_total_tokens_post_pad;
|
||||
}
|
||||
__syncthreads();
|
||||
|
||||
// Fill 0 to scan_buf extended area (tid >= num_expert)
|
||||
if (tid >= num_experts && tid < scan_size) scan_buf[tid] = 0;
|
||||
__syncthreads();
|
||||
|
||||
// Perform 2 level exclusive-prefix-sum to scan_buf
|
||||
int v = (tid < scan_size) ? scan_buf[tid] : 0;
|
||||
int pre = warp_exclusive_scan(v);
|
||||
if (lane_id == WARP_SIZE - 1) warp_sums[warp_id] = pre + v;
|
||||
__syncthreads();
|
||||
|
||||
if (warp_id == 0) {
|
||||
int val = (lane_id < num_warps_for_scan) ? warp_sums[lane_id] : 0;
|
||||
warp_sums[lane_id] = warp_exclusive_scan(val);
|
||||
}
|
||||
__syncthreads();
|
||||
|
||||
int offset = warp_sums[warp_id];
|
||||
if (tid < scan_size) scan_buf[tid] = pre + offset;
|
||||
__syncthreads();
|
||||
|
||||
// Write prefix[0..num_experts - 1] and cumsum
|
||||
if (tid < num_experts) prefix[tid] = scan_buf[tid];
|
||||
#endif
|
||||
|
||||
if (tid <= num_experts) {
|
||||
cumsum[tid] = prefix[tid];
|
||||
}
|
||||
// fill expert_ids
|
||||
const int32_t num_blocks = s_total_tokens_post_pad / block_size;
|
||||
for (int32_t i = tid; i < num_blocks; i += stride) {
|
||||
int32_t block_start = i * block_size;
|
||||
int left = 0, right = num_experts;
|
||||
while (left < right) {
|
||||
int mid = (left + right) >> 1;
|
||||
if (prefix[mid] <= block_start) {
|
||||
left = mid + 1;
|
||||
} else {
|
||||
right = mid;
|
||||
}
|
||||
}
|
||||
expert_ids[i] = left - 2;
|
||||
}
|
||||
}
|
||||
|
||||
template <typename scalar_t, int32_t fill_threads>
|
||||
__global__ void moe_align_block_size_small_batch_expert_kernel(
|
||||
const scalar_t* __restrict__ topk_ids,
|
||||
int32_t* __restrict__ sorted_token_ids,
|
||||
int32_t* __restrict__ expert_ids,
|
||||
int32_t* __restrict__ total_tokens_post_pad,
|
||||
int32_t num_experts,
|
||||
int32_t block_size,
|
||||
size_t numel,
|
||||
bool pad_sorted_token_ids,
|
||||
int32_t max_num_tokens_padded) {
|
||||
// Adapted from
|
||||
// https://github.com/vllm-project/vllm/pull/29642/files#diff-5647b1413f4ae9aacba904eca8f8a8aee9079321eadff4c10101a2c6962dcc53R226
|
||||
// Use an additional group of threads to fill sorted_token_ids.
|
||||
// Since the kernel will use sorted_token_ids afterward,
|
||||
// we fill sorted_token_ids within the same threadblock to make
|
||||
// synchronization easier.
|
||||
if (threadIdx.x < fill_threads) {
|
||||
// Initialize sorted_token_ids with numel
|
||||
if (pad_sorted_token_ids) {
|
||||
for (int32_t it = threadIdx.x; it < max_num_tokens_padded; it += fill_threads) {
|
||||
sorted_token_ids[it] = numel;
|
||||
}
|
||||
}
|
||||
// Three __syncthreads() corresponding to the other threads
|
||||
__syncthreads();
|
||||
__syncthreads();
|
||||
__syncthreads();
|
||||
return;
|
||||
}
|
||||
|
||||
const size_t tid = threadIdx.x - fill_threads;
|
||||
const size_t stride = blockDim.x - fill_threads;
|
||||
|
||||
extern __shared__ int32_t shared_mem[];
|
||||
int32_t* cumsum = shared_mem;
|
||||
int32_t* tokens_cnts = (int32_t*)(shared_mem + num_experts + 1);
|
||||
|
||||
for (int i = 0; i < num_experts; ++i) {
|
||||
tokens_cnts[(tid + 1) * num_experts + i] = 0;
|
||||
}
|
||||
|
||||
for (size_t i = tid; i < numel; i += stride) {
|
||||
int32_t expert_id = topk_ids[i] + 1;
|
||||
++tokens_cnts[(tid + 1) * num_experts + expert_id];
|
||||
}
|
||||
|
||||
__syncthreads();
|
||||
|
||||
if (tid < num_experts) {
|
||||
tokens_cnts[tid] = 0;
|
||||
for (int i = 1; i <= stride; ++i) {
|
||||
tokens_cnts[i * num_experts + tid] += tokens_cnts[(i - 1) * num_experts + tid];
|
||||
}
|
||||
}
|
||||
|
||||
__syncthreads();
|
||||
|
||||
if (tid == 0) {
|
||||
cumsum[0] = 0;
|
||||
for (int i = 1; i <= num_experts; ++i) {
|
||||
cumsum[i] = cumsum[i - 1] + CEILDIV(tokens_cnts[stride * num_experts + i - 1], block_size) * block_size;
|
||||
}
|
||||
*total_tokens_post_pad = static_cast<int32_t>(cumsum[num_experts]);
|
||||
}
|
||||
|
||||
__syncthreads();
|
||||
|
||||
if (tid < num_experts) {
|
||||
for (int i = cumsum[tid]; i < cumsum[tid + 1]; i += block_size) {
|
||||
expert_ids[i / block_size] = tid - 1;
|
||||
}
|
||||
}
|
||||
|
||||
for (size_t i = tid; i < numel; i += stride) {
|
||||
int32_t expert_id = topk_ids[i] + 1;
|
||||
int32_t rank_post_pad = tokens_cnts[tid * num_experts + expert_id] + cumsum[expert_id];
|
||||
sorted_token_ids[rank_post_pad] = i;
|
||||
++tokens_cnts[tid * num_experts + expert_id];
|
||||
}
|
||||
}
|
||||
|
||||
void moe_align_block_size(
|
||||
torch::Tensor topk_ids,
|
||||
int64_t num_experts,
|
||||
int64_t block_size,
|
||||
torch::Tensor sorted_token_ids,
|
||||
torch::Tensor experts_ids,
|
||||
torch::Tensor num_tokens_post_pad,
|
||||
torch::Tensor cumsum_buffer,
|
||||
bool pad_sorted_token_ids) {
|
||||
const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
|
||||
|
||||
int threads = 1024;
|
||||
threads = ((threads + WARP_SIZE - 1) / WARP_SIZE) * WARP_SIZE;
|
||||
|
||||
int64_t max_num_tokens_padded = sorted_token_ids.size(0);
|
||||
|
||||
DISPATCH_INTEGRAL_TYPES(topk_ids.scalar_type(), "moe_align_block_size_kernel", [&] {
|
||||
bool small_batch_expert_mode = (topk_ids.numel() < 1024) && (num_experts <= 64);
|
||||
|
||||
if (small_batch_expert_mode) {
|
||||
const int32_t threads = max((int32_t)num_experts, WARP_SIZE);
|
||||
constexpr int32_t fill_threads = 256;
|
||||
const int32_t shared_mem_size = ((threads + 1) * num_experts + (num_experts + 1)) * sizeof(int32_t);
|
||||
|
||||
auto small_batch_expert_kernel = moe_align_block_size_small_batch_expert_kernel<scalar_t, fill_threads>;
|
||||
small_batch_expert_kernel<<<1, fill_threads + threads, shared_mem_size, stream>>>(
|
||||
topk_ids.data_ptr<scalar_t>(),
|
||||
sorted_token_ids.data_ptr<int32_t>(),
|
||||
experts_ids.data_ptr<int32_t>(),
|
||||
num_tokens_post_pad.data_ptr<int32_t>(),
|
||||
num_experts,
|
||||
block_size,
|
||||
topk_ids.numel(),
|
||||
pad_sorted_token_ids,
|
||||
max_num_tokens_padded);
|
||||
} else {
|
||||
auto align_kernel = moe_align_block_size_kernel<scalar_t>;
|
||||
|
||||
const size_t scan_size = next_pow2(num_experts);
|
||||
const size_t shared_mem_size = (num_experts + (num_experts + 1) + scan_size + WARP_SIZE) * sizeof(int32_t);
|
||||
align_kernel<<<2, threads, shared_mem_size, stream>>>(
|
||||
topk_ids.data_ptr<scalar_t>(),
|
||||
sorted_token_ids.data_ptr<int32_t>(),
|
||||
experts_ids.data_ptr<int32_t>(),
|
||||
num_tokens_post_pad.data_ptr<int32_t>(),
|
||||
num_experts,
|
||||
block_size,
|
||||
topk_ids.numel(),
|
||||
cumsum_buffer.data_ptr<int32_t>(),
|
||||
pad_sorted_token_ids,
|
||||
scan_size,
|
||||
max_num_tokens_padded);
|
||||
|
||||
const int block_threads = std::min(256, (int)threads);
|
||||
const int num_blocks = (topk_ids.numel() + block_threads - 1) / block_threads;
|
||||
const int max_blocks = 65535;
|
||||
const int actual_blocks = std::min(num_blocks, max_blocks);
|
||||
|
||||
auto sort_kernel = count_and_sort_expert_tokens_kernel<scalar_t>;
|
||||
sort_kernel<<<actual_blocks, block_threads, 0, stream>>>(
|
||||
topk_ids.data_ptr<scalar_t>(),
|
||||
sorted_token_ids.data_ptr<int32_t>(),
|
||||
cumsum_buffer.data_ptr<int32_t>(),
|
||||
topk_ids.numel());
|
||||
}
|
||||
});
|
||||
}
|
||||
523
third_party/sglang/sgl-kernel/csrc/moe/moe_fused_gate.cu
vendored
Normal file
523
third_party/sglang/sgl-kernel/csrc/moe/moe_fused_gate.cu
vendored
Normal file
@@ -0,0 +1,523 @@
|
||||
#include <ATen/cuda/CUDAContext.h>
|
||||
#include <cuda_runtime.h>
|
||||
#include <cutlass/array.h>
|
||||
#include <cutlass/cutlass.h>
|
||||
#include <cutlass/numeric_types.h>
|
||||
#include <stdio.h>
|
||||
#include <torch/all.h>
|
||||
|
||||
#include <cfloat>
|
||||
#include <type_traits>
|
||||
template <typename T, int N>
|
||||
using AlignedArray = cutlass::AlignedArray<T, N>;
|
||||
using bfloat16_t = cutlass::bfloat16_t;
|
||||
using float16_t = cutlass::half_t;
|
||||
using float32_t = float;
|
||||
|
||||
// QQ NOTE: to handle the case for at::Half, error: more than one operator ">" matches these operands: built-in operator
|
||||
// "arithmetic > arithmetic" function "operator>(const __half &, const __half &)"
|
||||
template <typename T>
|
||||
__device__ inline bool cmp_gt(const T& a, const T& b) {
|
||||
if constexpr (std::is_same<T, at::Half>::value) {
|
||||
// at::Half (or float16_t in our native case) causes ambiguity, so we cast to float.
|
||||
return static_cast<float>(a) > static_cast<float>(b);
|
||||
} else {
|
||||
// For types like float, at::BFloat16, or cutlass::half_t / cutlass::bfloat16_t, assume operator> works as expected.
|
||||
return a > b;
|
||||
}
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
__device__ inline bool cmp_eq(const T& a, const T& b) {
|
||||
if constexpr (std::is_same<T, at::Half>::value) {
|
||||
return static_cast<float>(a) == static_cast<float>(b);
|
||||
} else {
|
||||
return a == b;
|
||||
}
|
||||
}
|
||||
|
||||
// Fixed constants common to both dynamic and static template versions:
|
||||
static constexpr int WARP_SIZE = 32;
|
||||
static constexpr int WARPS_PER_CTA = 6;
|
||||
static constexpr int MAX_VPT = 32; // maximum VPT we support, > params.VPT = num_expert / num_expert_group
|
||||
|
||||
// Create an alias for Array using AlignedArray
|
||||
template <typename T, int N>
|
||||
using Array = AlignedArray<T, N>;
|
||||
// QQ: NOTE expression must have a constant value, this has to be > params.VPT
|
||||
template <typename T>
|
||||
using AccessType = AlignedArray<T, MAX_VPT>;
|
||||
|
||||
template <typename T, typename Params>
|
||||
__device__ void moe_fused_gate_impl(
|
||||
void* input,
|
||||
void* bias,
|
||||
float* output_ptr,
|
||||
int32_t* indices_ptr,
|
||||
int64_t num_rows,
|
||||
int64_t topk_group,
|
||||
int64_t topk,
|
||||
int64_t num_fused_shared_experts,
|
||||
double routed_scaling_factor,
|
||||
bool apply_routed_scaling_factor_on_output,
|
||||
Params params) {
|
||||
int tidx = threadIdx.x;
|
||||
int64_t thread_row =
|
||||
blockIdx.x * params.ROWS_PER_CTA + threadIdx.y * params.ROWS_PER_WARP + tidx / params.THREADS_PER_ROW;
|
||||
if (thread_row >= num_rows) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Calculate topk_excluding_share_expert_fusion from topk
|
||||
int64_t topk_excluding_share_expert_fusion = topk - num_fused_shared_experts;
|
||||
|
||||
// Cast pointers to type T:
|
||||
auto* input_ptr = reinterpret_cast<T*>(input);
|
||||
auto* bias_ptr = reinterpret_cast<T*>(bias);
|
||||
auto* thread_row_ptr = input_ptr + thread_row * params.NUM_EXPERTS;
|
||||
|
||||
int thread_group_idx = tidx % params.THREADS_PER_ROW;
|
||||
int first_elt_read_by_thread = thread_group_idx * params.VPT;
|
||||
|
||||
// Create local arrays for the row chunk and bias chunk and then reinterpret the address of row_chunk as a pointer to
|
||||
// AccessType.
|
||||
T* thread_read_ptr = thread_row_ptr + first_elt_read_by_thread;
|
||||
Array<T, MAX_VPT> row_chunk;
|
||||
AccessType<T> const* vec_thread_read_ptr = reinterpret_cast<AccessType<T> const*>(thread_read_ptr);
|
||||
|
||||
T* bias_thread_read_ptr = bias_ptr + first_elt_read_by_thread;
|
||||
Array<T, MAX_VPT> bias_chunk;
|
||||
AccessType<T> const* vec_bias_thread_read_ptr = reinterpret_cast<AccessType<T> const*>(bias_thread_read_ptr);
|
||||
|
||||
// QQ NOTE: doing the follow will be slower than loop assign and more importantly
|
||||
// have misaligned address issue when params.VPT < 8 and mismatch with MAX_VPT
|
||||
// AccessType<T>* row_chunk_vec_ptr = reinterpret_cast<AccessType<T>*>(&row_chunk);
|
||||
// row_chunk_vec_ptr[0] = vec_thread_read_ptr[0];
|
||||
#pragma unroll
|
||||
for (int ii = 0; ii < params.VPT; ++ii) {
|
||||
row_chunk[ii] = vec_thread_read_ptr[0][ii];
|
||||
bias_chunk[ii] = vec_bias_thread_read_ptr[0][ii];
|
||||
}
|
||||
|
||||
__syncthreads();
|
||||
|
||||
////////////////////// Sigmoid //////////////////////
|
||||
#pragma unroll
|
||||
for (int ii = 0; ii < params.VPT; ++ii) {
|
||||
row_chunk[ii] = static_cast<T>(1.0f / (1.0f + expf(-float(row_chunk[ii]))));
|
||||
}
|
||||
__syncthreads();
|
||||
|
||||
////////////////////// Add Bias //////////////////////
|
||||
#pragma unroll
|
||||
for (int ii = 0; ii < params.VPT; ++ii) {
|
||||
bias_chunk[ii] = row_chunk[ii] + bias_chunk[ii];
|
||||
}
|
||||
|
||||
////////////////////// Exclude Groups //////////////////////
|
||||
#pragma unroll
|
||||
for (int k_idx = 0; k_idx < params.THREADS_PER_ROW - topk_group;
|
||||
++k_idx) { // QQ NOTE Here params.THREADS_PER_ROW = num_expert_group
|
||||
int expert = first_elt_read_by_thread;
|
||||
// local argmax
|
||||
T max_val = static_cast<T>(-FLT_MAX);
|
||||
T max_val_second = static_cast<T>(-FLT_MAX);
|
||||
#pragma unroll
|
||||
for (int ii = 0; ii < params.VPT; ++ii) {
|
||||
T val = bias_chunk[ii];
|
||||
|
||||
if (cmp_gt(val, max_val)) {
|
||||
max_val_second = max_val;
|
||||
max_val = val;
|
||||
} else if (cmp_gt(val, max_val_second)) {
|
||||
max_val_second = val;
|
||||
}
|
||||
}
|
||||
|
||||
// QQ NOTE: currently fixed to pick top2 sigmoid weight value in each expert group and sum them as the group weight
|
||||
// to select expert groups
|
||||
T max_sum = max_val + max_val_second;
|
||||
|
||||
// argmin reduce
|
||||
#pragma unroll
|
||||
for (int mask = params.THREADS_PER_ROW / 2; mask > 0; mask /= 2) {
|
||||
T other_max_sum =
|
||||
static_cast<T>(__shfl_xor_sync(0xFFFFFFFF, static_cast<float>(max_sum), mask, params.THREADS_PER_ROW));
|
||||
int other_expert = __shfl_xor_sync(0xFFFFFFFF, expert, mask, params.THREADS_PER_ROW);
|
||||
|
||||
// higher indices win
|
||||
if (cmp_gt(max_sum, other_max_sum) || (cmp_eq(other_max_sum, max_sum) && other_expert > expert)) {
|
||||
max_sum = other_max_sum;
|
||||
expert = other_expert;
|
||||
}
|
||||
}
|
||||
|
||||
// clear the max value in the thread
|
||||
if (k_idx < params.THREADS_PER_ROW - topk_group) {
|
||||
int const thread_to_clear_in_group = expert / params.VPT;
|
||||
|
||||
if (thread_group_idx == thread_to_clear_in_group) {
|
||||
#pragma unroll
|
||||
for (int ii = 0; ii < params.VPT; ++ii) {
|
||||
bias_chunk[ii] = static_cast<T>(FLT_MAX);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
__syncthreads();
|
||||
|
||||
////////////////////// Topk //////////////////////
|
||||
float output_sum = 0.0f;
|
||||
for (int k_idx = 0; k_idx < topk_excluding_share_expert_fusion; ++k_idx) {
|
||||
// local argmax
|
||||
T max_val = bias_chunk[0];
|
||||
int expert = first_elt_read_by_thread;
|
||||
|
||||
if (!cmp_eq(max_val, static_cast<T>(FLT_MAX))) {
|
||||
#pragma unroll
|
||||
for (int ii = 1; ii < params.VPT; ++ii) {
|
||||
T val = bias_chunk[ii];
|
||||
if (cmp_gt(val, max_val)) {
|
||||
max_val = val;
|
||||
expert = first_elt_read_by_thread + ii;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
max_val = static_cast<T>(-FLT_MAX);
|
||||
}
|
||||
|
||||
// argmax reduce
|
||||
#pragma unroll
|
||||
for (int mask = params.THREADS_PER_ROW / 2; mask > 0; mask /= 2) {
|
||||
T other_max =
|
||||
static_cast<T>(__shfl_xor_sync(0xFFFFFFFF, static_cast<float>(max_val), mask, params.THREADS_PER_ROW));
|
||||
int other_expert = __shfl_xor_sync(0xFFFFFFFF, expert, mask, params.THREADS_PER_ROW);
|
||||
|
||||
// lower indices to win
|
||||
if (cmp_gt(other_max, max_val) || (cmp_eq(other_max, max_val) && other_expert < expert)) {
|
||||
max_val = other_max;
|
||||
expert = other_expert;
|
||||
}
|
||||
}
|
||||
|
||||
int thread_to_clear_in_group = expert / params.VPT;
|
||||
int64_t idx = topk * thread_row + k_idx;
|
||||
|
||||
if (thread_group_idx == thread_to_clear_in_group) {
|
||||
int expert_to_clear_in_thread = expert % params.VPT;
|
||||
|
||||
// clear the max value in the thread
|
||||
bias_chunk[expert_to_clear_in_thread] = static_cast<T>(-FLT_MAX);
|
||||
|
||||
// store output
|
||||
output_ptr[idx] = static_cast<float>(row_chunk[expert_to_clear_in_thread]);
|
||||
indices_ptr[idx] = static_cast<int32_t>(expert);
|
||||
}
|
||||
|
||||
// accumulate sum for all elements
|
||||
if (thread_group_idx == 0) {
|
||||
output_sum += output_ptr[idx];
|
||||
}
|
||||
|
||||
__syncthreads();
|
||||
}
|
||||
|
||||
if (thread_group_idx == 0 && num_fused_shared_experts > 0) {
|
||||
int64_t last_idx = topk * thread_row + topk_excluding_share_expert_fusion;
|
||||
int64_t expert_offset = 0;
|
||||
indices_ptr[last_idx] = static_cast<int32_t>(params.NUM_EXPERTS + expert_offset);
|
||||
|
||||
// Set the weight to the sum of all weights divided by routed_scaling_factor
|
||||
output_ptr[last_idx] = output_sum / routed_scaling_factor;
|
||||
|
||||
if (num_fused_shared_experts > 1) {
|
||||
for (int i = 1; i < num_fused_shared_experts; ++i) {
|
||||
++last_idx;
|
||||
++expert_offset;
|
||||
indices_ptr[last_idx] = static_cast<int32_t>(params.NUM_EXPERTS + expert_offset);
|
||||
// Set the weight to the sum of all weights divided by routed_scaling_factor
|
||||
output_ptr[last_idx] = output_sum / routed_scaling_factor;
|
||||
}
|
||||
}
|
||||
}
|
||||
__syncthreads();
|
||||
|
||||
////////////////////// Rescale Output //////////////////////
|
||||
if (thread_group_idx == 0) {
|
||||
#pragma unroll
|
||||
for (int ii = 0; ii < topk; ++ii) {
|
||||
int64_t const idx = topk * thread_row + ii;
|
||||
output_ptr[idx] = output_ptr[idx] / output_sum;
|
||||
if (apply_routed_scaling_factor_on_output) {
|
||||
output_ptr[idx] *= routed_scaling_factor;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
//------------------------------------------------------------------------------
|
||||
// Templated Kernel Version (using compile-time constants)
|
||||
//------------------------------------------------------------------------------
|
||||
template <int VPT_, int NUM_EXPERTS_, int THREADS_PER_ROW_, int ROWS_PER_WARP_, int ROWS_PER_CTA_, int WARPS_PER_CTA_>
|
||||
struct KernelParams {
|
||||
static constexpr int VPT = VPT_;
|
||||
static constexpr int NUM_EXPERTS = NUM_EXPERTS_;
|
||||
static constexpr int THREADS_PER_ROW = THREADS_PER_ROW_;
|
||||
static constexpr int ROWS_PER_WARP = ROWS_PER_WARP_;
|
||||
static constexpr int ROWS_PER_CTA = ROWS_PER_CTA_;
|
||||
static constexpr int WARPS_PER_CTA = WARPS_PER_CTA_;
|
||||
};
|
||||
|
||||
template <
|
||||
typename T,
|
||||
int VPT,
|
||||
int NUM_EXPERTS,
|
||||
int THREADS_PER_ROW,
|
||||
int ROWS_PER_WARP,
|
||||
int ROWS_PER_CTA,
|
||||
int WARPS_PER_CTA>
|
||||
__global__ void moe_fused_gate_kernel(
|
||||
void* input,
|
||||
void* bias,
|
||||
float* output_ptr,
|
||||
int32_t* indices_ptr,
|
||||
int64_t num_rows,
|
||||
int64_t topk_group,
|
||||
int64_t topk,
|
||||
int64_t num_fused_shared_experts,
|
||||
double routed_scaling_factor,
|
||||
bool apply_routed_scaling_factor_on_output) {
|
||||
KernelParams<VPT, NUM_EXPERTS, THREADS_PER_ROW, ROWS_PER_WARP, ROWS_PER_CTA, WARPS_PER_CTA> params;
|
||||
moe_fused_gate_impl<T>(
|
||||
input,
|
||||
bias,
|
||||
output_ptr,
|
||||
indices_ptr,
|
||||
num_rows,
|
||||
topk_group,
|
||||
topk,
|
||||
num_fused_shared_experts,
|
||||
routed_scaling_factor,
|
||||
apply_routed_scaling_factor_on_output,
|
||||
params);
|
||||
}
|
||||
|
||||
// Macro to compute compile-time constants and launch the kernel.
|
||||
#define LAUNCH_MOE_GATE_CONFIG(T, EXPERTS, EXPERT_GROUP) \
|
||||
do { \
|
||||
constexpr int VPT = (EXPERTS) / (EXPERT_GROUP); \
|
||||
/* If EXPERT_GROUP > WARP_SIZE, fall back to 1 row per warp */ \
|
||||
constexpr int ROWS_PER_WARP = ((EXPERT_GROUP) <= WARP_SIZE) ? (WARP_SIZE / (EXPERT_GROUP)) : 1; \
|
||||
constexpr int ROWS_PER_CTA = WARPS_PER_CTA * ROWS_PER_WARP; \
|
||||
moe_fused_gate_kernel<T, VPT, (EXPERTS), (EXPERT_GROUP), ROWS_PER_WARP, ROWS_PER_CTA, WARPS_PER_CTA> \
|
||||
<<<num_blocks, block_dim, 0, stream>>>( \
|
||||
input.data_ptr(), \
|
||||
bias.data_ptr(), \
|
||||
output.data_ptr<float>(), \
|
||||
indices.data_ptr<int32_t>(), \
|
||||
num_rows, \
|
||||
topk_group, \
|
||||
topk, \
|
||||
num_fused_shared_experts, \
|
||||
routed_scaling_factor, \
|
||||
apply_routed_scaling_factor_on_output); \
|
||||
dispatched = true; \
|
||||
} while (0)
|
||||
|
||||
//------------------------------------------------------------------------------
|
||||
// Dynamic Kernel Version (parameters computed at runtime)
|
||||
//------------------------------------------------------------------------------
|
||||
struct KernelParamsDynamic {
|
||||
int VPT;
|
||||
int NUM_EXPERTS;
|
||||
int THREADS_PER_ROW;
|
||||
int ROWS_PER_WARP;
|
||||
int ROWS_PER_CTA;
|
||||
int WARPS_PER_CTA;
|
||||
};
|
||||
|
||||
template <typename T>
|
||||
__global__ void moe_fused_gate_kernel_dynamic(
|
||||
void* input,
|
||||
void* bias,
|
||||
float* output_ptr,
|
||||
int32_t* indices_ptr,
|
||||
int64_t num_rows,
|
||||
int64_t num_experts,
|
||||
int64_t num_expert_group,
|
||||
int64_t topk_group,
|
||||
int64_t topk,
|
||||
int64_t num_fused_shared_experts,
|
||||
double routed_scaling_factor,
|
||||
bool apply_routed_scaling_factor_on_output) {
|
||||
KernelParamsDynamic params;
|
||||
params.NUM_EXPERTS = num_experts; // e.g, for deepseek v3, this is 256
|
||||
params.VPT = num_experts / num_expert_group; // e.g., for deepseek v3, this is 256 / 8 = 32
|
||||
params.THREADS_PER_ROW = num_expert_group; // fixed as num_expert_group, e.g., for deepseek v3, this is 8
|
||||
params.WARPS_PER_CTA = WARPS_PER_CTA; // fixed as 6
|
||||
params.ROWS_PER_WARP = std::max<int64_t>(1, WARP_SIZE / num_expert_group); // WARP_SIZE is fixed as 32
|
||||
params.ROWS_PER_CTA = params.WARPS_PER_CTA * params.ROWS_PER_WARP;
|
||||
|
||||
moe_fused_gate_impl<T>(
|
||||
input,
|
||||
bias,
|
||||
output_ptr,
|
||||
indices_ptr,
|
||||
num_rows,
|
||||
topk_group,
|
||||
topk,
|
||||
num_fused_shared_experts,
|
||||
routed_scaling_factor,
|
||||
apply_routed_scaling_factor_on_output,
|
||||
params);
|
||||
}
|
||||
|
||||
//------------------------------------------------------------------------------
|
||||
// Host Launcher Function
|
||||
//------------------------------------------------------------------------------
|
||||
std::vector<at::Tensor> moe_fused_gate(
|
||||
at::Tensor& input,
|
||||
at::Tensor& bias,
|
||||
int64_t num_expert_group,
|
||||
int64_t topk_group,
|
||||
int64_t topk,
|
||||
int64_t num_fused_shared_experts,
|
||||
double routed_scaling_factor,
|
||||
bool apply_routed_scaling_factor_on_output) {
|
||||
TORCH_CHECK(input.dtype() == bias.dtype(), "input and bias should have the same dtype");
|
||||
|
||||
int64_t num_rows = input.size(0);
|
||||
int32_t num_experts = input.size(1);
|
||||
auto options = torch::TensorOptions().dtype(torch::kFloat32).device(torch::kCUDA);
|
||||
auto output = torch::empty({num_rows, topk}, options);
|
||||
auto indices = torch::empty({num_rows, topk}, options.dtype(torch::kInt32));
|
||||
|
||||
// Compute grid dimensions based on runtime value for num_expert_group.
|
||||
int64_t rows_per_warp = std::max<int64_t>(1, WARP_SIZE / num_expert_group);
|
||||
int64_t num_warps = (num_rows + rows_per_warp - 1) / rows_per_warp;
|
||||
int64_t num_blocks = (num_warps + WARPS_PER_CTA - 1) / WARPS_PER_CTA;
|
||||
const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
|
||||
dim3 block_dim(WARP_SIZE, WARPS_PER_CTA);
|
||||
|
||||
// Check 1: Ensure that num_experts is a power of 2.
|
||||
TORCH_CHECK((num_experts & (num_experts - 1)) == 0, "num_experts must be a power of 2, but got ", num_experts);
|
||||
|
||||
// Check 2: Ensure that num_experts is divisible by num_expert_group. (this also means num_expert_group is power of 2)
|
||||
TORCH_CHECK(
|
||||
num_experts % num_expert_group == 0,
|
||||
"num_experts must be divisible by num_expert_group, but got ",
|
||||
num_experts,
|
||||
" / ",
|
||||
num_expert_group);
|
||||
|
||||
int computed_vpt = num_experts / num_expert_group;
|
||||
// Check 3: Ensure that num_experts/num_expert_group does not exceed MAX_VPT=32. Maximum VPT indicate max value per
|
||||
// threads we can process.
|
||||
TORCH_CHECK(
|
||||
computed_vpt <= MAX_VPT,
|
||||
"Per group experts: num_experts / num_expert_group = (",
|
||||
computed_vpt,
|
||||
") exceeds the maximum supported (",
|
||||
MAX_VPT,
|
||||
")");
|
||||
|
||||
// Dispatch to templated kernel for known compile-time configurations.
|
||||
// We currently only support for:
|
||||
// Case 1: 256 experts, with 8 or 16 groups.
|
||||
// Case 2: 128 experts, with 4 or 8 groups.
|
||||
// Case 3: other cases, require 8 <= num_experts / num_expert_group <= 32
|
||||
bool dispatched = false;
|
||||
switch (num_experts) {
|
||||
case 256:
|
||||
if (num_expert_group == 8)
|
||||
// This is deepseek v3 case. Here VPT = 256/8 = 32, ROWS_PER_WARP = 32/8 = 4, ROWS_PER_CTA = 6 * 4 = 24.
|
||||
if (input.scalar_type() == at::kBFloat16) {
|
||||
LAUNCH_MOE_GATE_CONFIG(bfloat16_t, 256, 8);
|
||||
} else if (input.scalar_type() == at::kHalf) {
|
||||
LAUNCH_MOE_GATE_CONFIG(float16_t, 256, 8);
|
||||
} else if (input.scalar_type() == at::kFloat) {
|
||||
LAUNCH_MOE_GATE_CONFIG(float32_t, 256, 8);
|
||||
} else if (num_expert_group == 16)
|
||||
// Here VPT = 256/16 = 16, ROWS_PER_WARP = 32/16 = 2, ROWS_PER_CTA = 6 * 2 = 12.
|
||||
if (input.scalar_type() == at::kBFloat16) {
|
||||
LAUNCH_MOE_GATE_CONFIG(bfloat16_t, 256, 16);
|
||||
} else if (input.scalar_type() == at::kHalf) {
|
||||
LAUNCH_MOE_GATE_CONFIG(float16_t, 256, 16);
|
||||
} else if (input.scalar_type() == at::kFloat) {
|
||||
LAUNCH_MOE_GATE_CONFIG(float32_t, 256, 16);
|
||||
}
|
||||
break;
|
||||
case 128:
|
||||
if (num_expert_group == 4)
|
||||
// VPT = 128/4 = 32, ROWS_PER_WARP = 32/16 = 2, ROWS_PER_CTA = 6 * 2 = 12.
|
||||
if (input.scalar_type() == at::kBFloat16) {
|
||||
LAUNCH_MOE_GATE_CONFIG(bfloat16_t, 128, 4);
|
||||
} else if (input.scalar_type() == at::kHalf) {
|
||||
LAUNCH_MOE_GATE_CONFIG(float16_t, 128, 4);
|
||||
} else if (input.scalar_type() == at::kFloat) {
|
||||
LAUNCH_MOE_GATE_CONFIG(float32_t, 128, 4);
|
||||
} else if (num_expert_group == 8)
|
||||
// VPT = 128/8 = 16, ROWS_PER_WARP = 32/8 = 4, ROWS_PER_CTA = 6 * 4 = 24.
|
||||
if (input.scalar_type() == at::kBFloat16) {
|
||||
LAUNCH_MOE_GATE_CONFIG(bfloat16_t, 128, 8);
|
||||
} else if (input.scalar_type() == at::kHalf) {
|
||||
LAUNCH_MOE_GATE_CONFIG(float16_t, 128, 8);
|
||||
} else if (input.scalar_type() == at::kFloat) {
|
||||
LAUNCH_MOE_GATE_CONFIG(float32_t, 128, 8);
|
||||
}
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
if (!dispatched) {
|
||||
// Fallback to the dynamic kernel if none of the supported combinations match.
|
||||
// currently only support num_experts / num_expert_group <= 32 for dynamic kernels
|
||||
if (input.scalar_type() == at::kBFloat16) {
|
||||
moe_fused_gate_kernel_dynamic<bfloat16_t><<<num_blocks, block_dim, 0, stream>>>(
|
||||
input.data_ptr(),
|
||||
bias.data_ptr(),
|
||||
output.data_ptr<float>(),
|
||||
indices.data_ptr<int32_t>(),
|
||||
num_rows,
|
||||
num_experts,
|
||||
num_expert_group,
|
||||
topk_group,
|
||||
topk,
|
||||
num_fused_shared_experts,
|
||||
routed_scaling_factor,
|
||||
apply_routed_scaling_factor_on_output);
|
||||
} else if (input.scalar_type() == at::kHalf) {
|
||||
moe_fused_gate_kernel_dynamic<float16_t><<<num_blocks, block_dim, 0, stream>>>(
|
||||
input.data_ptr(),
|
||||
bias.data_ptr(),
|
||||
output.data_ptr<float>(),
|
||||
indices.data_ptr<int32_t>(),
|
||||
num_rows,
|
||||
num_experts,
|
||||
num_expert_group,
|
||||
topk_group,
|
||||
topk,
|
||||
num_fused_shared_experts,
|
||||
routed_scaling_factor,
|
||||
apply_routed_scaling_factor_on_output);
|
||||
} else if (input.scalar_type() == at::kFloat) {
|
||||
moe_fused_gate_kernel_dynamic<float32_t><<<num_blocks, block_dim, 0, stream>>>(
|
||||
input.data_ptr(),
|
||||
bias.data_ptr(),
|
||||
output.data_ptr<float>(),
|
||||
indices.data_ptr<int32_t>(),
|
||||
num_rows,
|
||||
num_experts,
|
||||
num_expert_group,
|
||||
topk_group,
|
||||
topk,
|
||||
num_fused_shared_experts,
|
||||
routed_scaling_factor,
|
||||
apply_routed_scaling_factor_on_output);
|
||||
} else {
|
||||
TORCH_CHECK(false, "Unsupported data type for moe_fused_gate");
|
||||
}
|
||||
}
|
||||
return {output, indices};
|
||||
}
|
||||
66
third_party/sglang/sgl-kernel/csrc/moe/moe_sum.cu
vendored
Normal file
66
third_party/sglang/sgl-kernel/csrc/moe/moe_sum.cu
vendored
Normal file
@@ -0,0 +1,66 @@
|
||||
#include <ATen/ATen.h>
|
||||
#include <ATen/cuda/CUDAContext.h>
|
||||
#include <c10/cuda/CUDAGuard.h>
|
||||
#include <torch/all.h>
|
||||
|
||||
#include <ATen/cuda/Atomic.cuh>
|
||||
#include <cub/cub.cuh>
|
||||
|
||||
#include "utils.h"
|
||||
|
||||
template <typename scalar_t, int TOPK>
|
||||
__global__ void moe_sum_kernel(
|
||||
scalar_t* __restrict__ out, // [..., d]
|
||||
const scalar_t* __restrict__ input, // [..., topk, d]
|
||||
const int d) {
|
||||
const int64_t token_idx = blockIdx.x;
|
||||
for (int64_t idx = threadIdx.x; idx < d; idx += blockDim.x) {
|
||||
scalar_t x = 0.0;
|
||||
#pragma unroll
|
||||
for (int k = 0; k < TOPK; ++k) {
|
||||
x += SGLANG_LDG(&input[token_idx * TOPK * d + k * d + idx]);
|
||||
}
|
||||
out[token_idx * d + idx] = x;
|
||||
}
|
||||
}
|
||||
|
||||
void moe_sum(
|
||||
torch::Tensor& input, // [num_tokens, topk, hidden_size]
|
||||
torch::Tensor& output) // [num_tokens, hidden_size]
|
||||
{
|
||||
const int hidden_size = input.size(-1);
|
||||
const auto num_tokens = output.numel() / hidden_size;
|
||||
const int topk = input.size(1);
|
||||
|
||||
dim3 grid(num_tokens);
|
||||
dim3 block(std::min(hidden_size, 1024));
|
||||
const at::cuda::OptionalCUDAGuard device_guard(device_of(output));
|
||||
const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
|
||||
|
||||
switch (topk) {
|
||||
case 2:
|
||||
DISPATCH_FLOAT_TYPES(input.scalar_type(), "moe_sum_kernel", [&] {
|
||||
moe_sum_kernel<scalar_t, 2>
|
||||
<<<grid, block, 0, stream>>>(output.data_ptr<scalar_t>(), input.data_ptr<scalar_t>(), hidden_size);
|
||||
});
|
||||
break;
|
||||
|
||||
case 3:
|
||||
DISPATCH_FLOAT_TYPES(input.scalar_type(), "moe_sum_kernel", [&] {
|
||||
moe_sum_kernel<scalar_t, 3>
|
||||
<<<grid, block, 0, stream>>>(output.data_ptr<scalar_t>(), input.data_ptr<scalar_t>(), hidden_size);
|
||||
});
|
||||
break;
|
||||
|
||||
case 4:
|
||||
DISPATCH_FLOAT_TYPES(input.scalar_type(), "moe_sum_kernel", [&] {
|
||||
moe_sum_kernel<scalar_t, 4>
|
||||
<<<grid, block, 0, stream>>>(output.data_ptr<scalar_t>(), input.data_ptr<scalar_t>(), hidden_size);
|
||||
});
|
||||
break;
|
||||
|
||||
default:
|
||||
at::sum_out(output, input, 1);
|
||||
break;
|
||||
}
|
||||
}
|
||||
403
third_party/sglang/sgl-kernel/csrc/moe/moe_sum_reduce.cu
vendored
Normal file
403
third_party/sglang/sgl-kernel/csrc/moe/moe_sum_reduce.cu
vendored
Normal file
@@ -0,0 +1,403 @@
|
||||
#include <ATen/OpMathType.h>
|
||||
#include <ATen/cuda/CUDAContext.h>
|
||||
#include <c10/cuda/CUDAGuard.h>
|
||||
#include <cuda.h>
|
||||
#include <cudaTypedefs.h>
|
||||
#include <cuda_runtime.h>
|
||||
#include <torch/all.h>
|
||||
|
||||
#include <iostream>
|
||||
#include <type_traits>
|
||||
|
||||
#include "cutlass/array.h"
|
||||
#include "utils.h"
|
||||
|
||||
template <typename T>
|
||||
using opmath_t = at::opmath_type<T>;
|
||||
|
||||
template <typename T>
|
||||
__device__ __forceinline__ opmath_t<T> to_acc(T x) {
|
||||
return static_cast<opmath_t<T>>(x);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
__device__ __forceinline__ T from_acc(opmath_t<T> x) {
|
||||
return static_cast<T>(x);
|
||||
}
|
||||
|
||||
template <>
|
||||
__device__ __forceinline__ opmath_t<at::Half> to_acc<at::Half>(at::Half x) {
|
||||
return __half2float(__nv_half(x));
|
||||
}
|
||||
template <>
|
||||
__device__ __forceinline__ at::Half from_acc<at::Half>(opmath_t<at::Half> x) {
|
||||
return __float2half_rn(x);
|
||||
}
|
||||
|
||||
template <>
|
||||
__device__ __forceinline__ opmath_t<at::BFloat16> to_acc<at::BFloat16>(at::BFloat16 x) {
|
||||
return __bfloat162float(__nv_bfloat16(x));
|
||||
}
|
||||
template <>
|
||||
__device__ __forceinline__ at::BFloat16 from_acc<at::BFloat16>(opmath_t<at::BFloat16> x) {
|
||||
return __float2bfloat16_rn(x);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
__device__ __forceinline__ T ldg_cg(const T* p) {
|
||||
return __ldg(p);
|
||||
}
|
||||
|
||||
union Pack16B {
|
||||
uint4 v;
|
||||
__nv_bfloat16 u16[8];
|
||||
};
|
||||
|
||||
template <int WARPS_PER_BLOCK>
|
||||
__global__ void moe_sum_reduce_warp_per_token_vec_kernel(
|
||||
const at::BFloat16* __restrict__ x,
|
||||
at::BFloat16* __restrict__ y,
|
||||
const int64_t token_num,
|
||||
const int64_t hidden_dim,
|
||||
const int64_t topk_num,
|
||||
const int64_t stride_token, // in elements
|
||||
const int64_t stride_topk, // in elements
|
||||
const int64_t out_stride_token, // in elements
|
||||
const float scale) {
|
||||
constexpr int VEC = 16;
|
||||
constexpr int PACKS = VEC / 8;
|
||||
|
||||
const int warp_id = threadIdx.x / 32;
|
||||
const int lane = threadIdx.x % 32;
|
||||
const int64_t t = (int64_t)blockIdx.y * WARPS_PER_BLOCK + warp_id;
|
||||
if (t >= token_num) return;
|
||||
|
||||
const int64_t n_chunks = hidden_dim / VEC;
|
||||
|
||||
for (int64_t chunk = (int64_t)blockIdx.x * 32 + lane; chunk < n_chunks; chunk += (int64_t)gridDim.x * 32) {
|
||||
const int64_t d = chunk * VEC;
|
||||
const int64_t base = t * stride_token + d;
|
||||
|
||||
float acc[VEC];
|
||||
#pragma unroll
|
||||
for (int i = 0; i < VEC; ++i)
|
||||
acc[i] = 0.f;
|
||||
|
||||
#pragma unroll
|
||||
for (int k = 0; k < topk_num; ++k) {
|
||||
#pragma unroll
|
||||
for (int p = 0; p < PACKS; ++p) {
|
||||
const int64_t offset = base + (int64_t)k * stride_topk + p * 8;
|
||||
Pack16B pack = {ldg_cg(reinterpret_cast<const uint4*>(x + offset))};
|
||||
|
||||
#pragma unroll
|
||||
for (int i = 0; i < 8; ++i) {
|
||||
acc[p * 8 + i] += __bfloat162float(pack.u16[i]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#pragma unroll
|
||||
for (int i = 0; i < VEC; ++i)
|
||||
acc[i] *= scale;
|
||||
|
||||
#pragma unroll
|
||||
for (int p = 0; p < PACKS; ++p) {
|
||||
Pack16B outp;
|
||||
#pragma unroll
|
||||
for (int i = 0; i < 8; ++i) {
|
||||
outp.u16[i] = __float2bfloat16_rn(acc[p * 8 + i]);
|
||||
}
|
||||
const int64_t dst = t * out_stride_token + d + p * 8;
|
||||
*reinterpret_cast<uint4*>(y + dst) = outp.v;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
template <typename scalar_t, int TOPK, int WARPS_PER_BLOCK>
|
||||
__global__ void moe_sum_reduce_kernel_warp_token_topk(
|
||||
const scalar_t* __restrict__ x,
|
||||
scalar_t* __restrict__ y,
|
||||
const int64_t token_num,
|
||||
const int64_t hidden_dim,
|
||||
const int64_t stride_token,
|
||||
const int64_t stride_topk,
|
||||
const int64_t out_stride_token,
|
||||
const opmath_t<scalar_t> scale) {
|
||||
const int warp_id = threadIdx.x / 32;
|
||||
const int lane = threadIdx.x % 32;
|
||||
const int64_t t = (int64_t)blockIdx.y * WARPS_PER_BLOCK + warp_id;
|
||||
if (t >= token_num) return;
|
||||
|
||||
for (int64_t d = (int64_t)blockIdx.x * 32 + lane; d < hidden_dim; d += (int64_t)gridDim.x * 32) {
|
||||
opmath_t<scalar_t> acc = opmath_t<scalar_t>(0);
|
||||
const int64_t base = t * stride_token + d;
|
||||
|
||||
#pragma unroll
|
||||
for (int k = 0; k < TOPK; ++k) {
|
||||
acc += to_acc<scalar_t>(x[base + (int64_t)k * stride_topk]);
|
||||
}
|
||||
acc *= scale;
|
||||
y[t * out_stride_token + d] = from_acc<scalar_t>(acc);
|
||||
}
|
||||
}
|
||||
|
||||
template <typename scalar_t, int TOPK>
|
||||
__global__ void moe_sum_reduce_kernel(
|
||||
const scalar_t* __restrict__ x,
|
||||
scalar_t* __restrict__ y,
|
||||
const int64_t token_num,
|
||||
const int64_t hidden_dim,
|
||||
const int64_t stride_token,
|
||||
const int64_t stride_topk,
|
||||
const int64_t out_stride_token,
|
||||
const opmath_t<scalar_t> scale) {
|
||||
for (int t = blockIdx.y; t < token_num; t += gridDim.y) {
|
||||
for (int d = blockIdx.x * blockDim.x + threadIdx.x; d < hidden_dim; d += blockDim.x * gridDim.x) {
|
||||
const int64_t base = t * stride_token + d;
|
||||
opmath_t<scalar_t> acc = opmath_t<scalar_t>(0);
|
||||
|
||||
#pragma unroll
|
||||
for (int k = 0; k < TOPK; ++k) {
|
||||
acc += to_acc<scalar_t>(x[base + (int64_t)k * stride_topk]);
|
||||
}
|
||||
|
||||
acc *= scale;
|
||||
y[t * out_stride_token + d] = from_acc<scalar_t>(acc);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// -------------------- general-topk fallback kernels --------------------
|
||||
// small-token
|
||||
template <typename scalar_t>
|
||||
__global__ void moe_sum_reduce_kernel_general(
|
||||
const scalar_t* __restrict__ x,
|
||||
scalar_t* __restrict__ y,
|
||||
const int64_t token_num,
|
||||
const int64_t hidden_dim,
|
||||
const int64_t stride_token,
|
||||
const int64_t stride_topk,
|
||||
const int64_t out_stride_token,
|
||||
const int topk_num,
|
||||
const opmath_t<scalar_t> scale) {
|
||||
for (int t = blockIdx.y; t < token_num; t += gridDim.y) {
|
||||
for (int d = blockIdx.x * blockDim.x + threadIdx.x; d < hidden_dim; d += blockDim.x * gridDim.x) {
|
||||
const int64_t base = t * stride_token + d;
|
||||
opmath_t<scalar_t> acc = opmath_t<scalar_t>(0);
|
||||
#pragma unroll 1
|
||||
for (int k = 0; k < topk_num; ++k) {
|
||||
acc += to_acc<scalar_t>(x[base + (int64_t)k * stride_topk]);
|
||||
}
|
||||
acc *= scale;
|
||||
y[t * out_stride_token + d] = from_acc<scalar_t>(acc);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// warp-per-token
|
||||
template <typename scalar_t, int WARPS_PER_BLOCK>
|
||||
__global__ void moe_sum_reduce_kernel_warp_token_general(
|
||||
const scalar_t* __restrict__ x,
|
||||
scalar_t* __restrict__ y,
|
||||
const int64_t token_num,
|
||||
const int64_t hidden_dim,
|
||||
const int64_t stride_token,
|
||||
const int64_t stride_topk,
|
||||
const int64_t out_stride_token,
|
||||
const int topk_num,
|
||||
const opmath_t<scalar_t> scale) {
|
||||
const int warp_id = threadIdx.x / 32;
|
||||
const int lane = threadIdx.x % 32;
|
||||
const int64_t t = (int64_t)blockIdx.y * WARPS_PER_BLOCK + warp_id;
|
||||
if (t >= token_num) return;
|
||||
|
||||
for (int64_t d = (int64_t)blockIdx.x * 32 + lane; d < hidden_dim; d += (int64_t)gridDim.x * 32) {
|
||||
opmath_t<scalar_t> acc = opmath_t<scalar_t>(0);
|
||||
const int64_t base = t * stride_token + d;
|
||||
#pragma unroll 1
|
||||
for (int k = 0; k < topk_num; ++k) {
|
||||
acc += to_acc<scalar_t>(x[base + (int64_t)k * stride_topk]);
|
||||
}
|
||||
acc *= scale;
|
||||
y[t * out_stride_token + d] = from_acc<scalar_t>(acc);
|
||||
}
|
||||
}
|
||||
|
||||
void moe_sum_reduce(at::Tensor& input, at::Tensor& output, double routed_scaling_factor) {
|
||||
TORCH_CHECK(input.is_cuda(), "input must be CUDA tensor");
|
||||
TORCH_CHECK(output.is_cuda(), "output must be CUDA tensor");
|
||||
TORCH_CHECK(input.dim() == 3, "input must be a 3D tensor like [token_num, topk_num, hidden_dim]");
|
||||
TORCH_CHECK(output.dim() == 2, "output must be [token_num, hidden_dim]");
|
||||
TORCH_CHECK(input.size(0) == output.size(0), "token dim mismatch");
|
||||
TORCH_CHECK(input.size(2) == output.size(1), "hidden_dim mismatch");
|
||||
|
||||
TORCH_CHECK(input.is_contiguous(), "expect input to be contiguous");
|
||||
TORCH_CHECK(output.is_contiguous(), "expect output to be contiguous");
|
||||
|
||||
const int64_t token_num = input.size(0);
|
||||
const int64_t topk_num = input.size(1);
|
||||
const int64_t hidden_dim = input.size(2);
|
||||
|
||||
const int64_t in_stride_token = input.stride(0);
|
||||
const int64_t in_stride_topk = input.stride(1);
|
||||
const int64_t out_stride_token = output.stride(0);
|
||||
|
||||
auto stream = at::cuda::getCurrentCUDAStream();
|
||||
|
||||
const bool fast_bf16_vec_ok = (input.scalar_type() == at::kBFloat16) && (token_num > 256) && (hidden_dim % 8 == 0);
|
||||
|
||||
// Fast path for bf16 vectorize
|
||||
if (fast_bf16_vec_ok) {
|
||||
constexpr int WARPS_PER_BLOCK = 8;
|
||||
constexpr int THREADS = WARPS_PER_BLOCK * 32;
|
||||
|
||||
const int64_t n_chunks = hidden_dim / 8;
|
||||
int64_t grid_x = (n_chunks + 32 - 1) / 32;
|
||||
if (grid_x > 65535) grid_x = 65535;
|
||||
|
||||
int64_t grid_y = (token_num + WARPS_PER_BLOCK - 1) / WARPS_PER_BLOCK;
|
||||
if (grid_y > 65535) grid_y = 65535;
|
||||
|
||||
dim3 block(THREADS);
|
||||
dim3 grid(static_cast<unsigned>(grid_x), static_cast<unsigned>(grid_y));
|
||||
|
||||
auto stream = at::cuda::getCurrentCUDAStream();
|
||||
|
||||
const float scale = static_cast<float>(routed_scaling_factor);
|
||||
moe_sum_reduce_warp_per_token_vec_kernel<WARPS_PER_BLOCK><<<grid, block, 0, stream>>>(
|
||||
reinterpret_cast<const at::BFloat16*>(input.data_ptr<at::BFloat16>()),
|
||||
reinterpret_cast<at::BFloat16*>(output.data_ptr<at::BFloat16>()),
|
||||
token_num,
|
||||
hidden_dim,
|
||||
topk_num,
|
||||
in_stride_token,
|
||||
in_stride_topk,
|
||||
out_stride_token,
|
||||
scale);
|
||||
|
||||
TORCH_CHECK(cudaGetLastError() == cudaSuccess, "moe_sum_reduce CUDA kernel (bf16 vec) launch failed");
|
||||
return;
|
||||
}
|
||||
|
||||
const bool per_token_use_one_warp = (token_num > 128);
|
||||
|
||||
if (!per_token_use_one_warp) {
|
||||
// ---------- small-token ----------
|
||||
const int block_size = 256;
|
||||
int64_t grid_x = (hidden_dim + block_size - 1) / block_size;
|
||||
grid_x = grid_x > 65535 ? 65535 : grid_x;
|
||||
int64_t grid_y = token_num < 65535 ? token_num : 65535;
|
||||
|
||||
dim3 block(block_size);
|
||||
dim3 grid(static_cast<unsigned>(grid_x), static_cast<unsigned>(grid_y));
|
||||
|
||||
#define LAUNCH_SMALL_TOKEN_KERNEL(TOPK) \
|
||||
moe_sum_reduce_kernel<scalar_t_, TOPK><<<grid, block, 0, stream>>>( \
|
||||
input.data_ptr<scalar_t_>(), \
|
||||
output.data_ptr<scalar_t_>(), \
|
||||
token_num, \
|
||||
hidden_dim, \
|
||||
in_stride_token, \
|
||||
in_stride_topk, \
|
||||
out_stride_token, \
|
||||
scale);
|
||||
|
||||
AT_DISPATCH_FLOATING_TYPES_AND2(
|
||||
at::kHalf, at::kBFloat16, input.scalar_type(), "moe_sum_reduce_cuda_small_token", [&] {
|
||||
using scalar_t_ = scalar_t;
|
||||
using acc_t_ = opmath_t<scalar_t_>;
|
||||
const acc_t_ scale = static_cast<acc_t_>(routed_scaling_factor);
|
||||
|
||||
switch (topk_num) {
|
||||
case 2:
|
||||
LAUNCH_SMALL_TOKEN_KERNEL(2);
|
||||
break;
|
||||
case 4:
|
||||
LAUNCH_SMALL_TOKEN_KERNEL(4);
|
||||
break;
|
||||
case 8:
|
||||
LAUNCH_SMALL_TOKEN_KERNEL(8);
|
||||
break;
|
||||
case 9:
|
||||
LAUNCH_SMALL_TOKEN_KERNEL(9);
|
||||
break;
|
||||
default: // launch general kernel
|
||||
moe_sum_reduce_kernel_general<scalar_t_><<<grid, block, 0, stream>>>(
|
||||
input.data_ptr<scalar_t_>(),
|
||||
output.data_ptr<scalar_t_>(),
|
||||
token_num,
|
||||
hidden_dim,
|
||||
in_stride_token,
|
||||
in_stride_topk,
|
||||
out_stride_token,
|
||||
static_cast<int>(topk_num),
|
||||
scale);
|
||||
}
|
||||
});
|
||||
#undef LAUNCH_SMALL_TOKEN_KERNEL
|
||||
|
||||
TORCH_CHECK(cudaGetLastError() == cudaSuccess, "moe_sum_reduce CUDA kernel (small-token) launch failed");
|
||||
|
||||
} else {
|
||||
// ---------- warp-per-token ----------
|
||||
constexpr int WARPS_PER_BLOCK = 4;
|
||||
constexpr int THREADS = WARPS_PER_BLOCK * 32;
|
||||
|
||||
int64_t gx = (hidden_dim + 32 - 1) / 32;
|
||||
gx = gx > 65535 ? 65535 : gx;
|
||||
|
||||
int64_t gy = (token_num + WARPS_PER_BLOCK - 1) / WARPS_PER_BLOCK;
|
||||
gy = gy > 65535 ? 65535 : gy;
|
||||
|
||||
dim3 block(THREADS);
|
||||
dim3 grid(static_cast<unsigned>(gx), static_cast<unsigned>(gy));
|
||||
|
||||
#define LAUNCH_WARP_PER_TOKEN_KERNEL(TOPK) \
|
||||
moe_sum_reduce_kernel_warp_token_topk<scalar_t_, TOPK, WARPS_PER_BLOCK><<<grid, block, 0, stream>>>( \
|
||||
input.data_ptr<scalar_t_>(), \
|
||||
output.data_ptr<scalar_t_>(), \
|
||||
token_num, \
|
||||
hidden_dim, \
|
||||
in_stride_token, \
|
||||
in_stride_topk, \
|
||||
out_stride_token, \
|
||||
scale);
|
||||
|
||||
AT_DISPATCH_FLOATING_TYPES_AND2(
|
||||
at::kHalf, at::kBFloat16, input.scalar_type(), "moe_sum_reduce_cuda_large_token", [&] {
|
||||
using scalar_t_ = scalar_t;
|
||||
using acc_t_ = opmath_t<scalar_t_>;
|
||||
const acc_t_ scale = static_cast<acc_t_>(routed_scaling_factor);
|
||||
|
||||
switch (topk_num) {
|
||||
case 2:
|
||||
LAUNCH_WARP_PER_TOKEN_KERNEL(2);
|
||||
break;
|
||||
case 4:
|
||||
LAUNCH_WARP_PER_TOKEN_KERNEL(4);
|
||||
break;
|
||||
case 8:
|
||||
LAUNCH_WARP_PER_TOKEN_KERNEL(8);
|
||||
break;
|
||||
case 9:
|
||||
LAUNCH_WARP_PER_TOKEN_KERNEL(9);
|
||||
break;
|
||||
default: // launch general kernel
|
||||
moe_sum_reduce_kernel_warp_token_general<scalar_t_, WARPS_PER_BLOCK><<<grid, block, 0, stream>>>(
|
||||
input.data_ptr<scalar_t_>(),
|
||||
output.data_ptr<scalar_t_>(),
|
||||
token_num,
|
||||
hidden_dim,
|
||||
in_stride_token,
|
||||
in_stride_topk,
|
||||
out_stride_token,
|
||||
static_cast<int>(topk_num),
|
||||
scale);
|
||||
}
|
||||
});
|
||||
#undef LAUNCH_WARP_PER_TOKEN_KERNEL
|
||||
|
||||
TORCH_CHECK(cudaGetLastError() == cudaSuccess, "moe_sum_reduce CUDA kernel (warp-token) launch failed");
|
||||
}
|
||||
}
|
||||
592
third_party/sglang/sgl-kernel/csrc/moe/moe_topk_sigmoid_kernels.cu
vendored
Normal file
592
third_party/sglang/sgl-kernel/csrc/moe/moe_topk_sigmoid_kernels.cu
vendored
Normal file
@@ -0,0 +1,592 @@
|
||||
// Adapt from https://github.com/vllm-project/vllm/blob/v0.7.3/csrc/moe/topk_softmax_kernels.cu
|
||||
// which is originally adapted from
|
||||
// https://github.com/NVIDIA/TensorRT-LLM/blob/v0.7.1/cpp/tensorrt_llm/kernels/mixtureOfExperts/moe_kernels.cu
|
||||
/* Copyright 2025 SGLang Team. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
#include <ATen/cuda/CUDAContext.h>
|
||||
#include <c10/cuda/CUDAGuard.h>
|
||||
#include <torch/all.h>
|
||||
|
||||
#ifndef USE_ROCM
|
||||
#include <cub/cub.cuh>
|
||||
#include <cub/util_type.cuh>
|
||||
#include <cuda/functional>
|
||||
#else
|
||||
#include <hipcub/hipcub.hpp>
|
||||
#include <hipcub/util_type.hpp>
|
||||
#endif
|
||||
|
||||
#include "utils.h"
|
||||
|
||||
#define MAX(a, b) ((a) > (b) ? (a) : (b))
|
||||
#define MIN(a, b) ((a) < (b) ? (a) : (b))
|
||||
|
||||
// Define reduction operators based on CUDA version
|
||||
// CUDA 13 (12.9+) deprecated cub::Max/Min in favor of cuda::maximum/minimum
|
||||
#if CUDA_VERSION >= 12090
|
||||
using MaxReduceOp = cuda::maximum<>;
|
||||
using MinReduceOp = cuda::minimum<>;
|
||||
#else
|
||||
using MaxReduceOp = cub::Max;
|
||||
using MinReduceOp = cub::Min;
|
||||
#endif
|
||||
|
||||
/// Aligned array type
|
||||
template <
|
||||
typename T,
|
||||
/// Number of elements in the array
|
||||
int N,
|
||||
/// Alignment requirement in bytes
|
||||
int Alignment = sizeof(T) * N>
|
||||
class alignas(Alignment) AlignedArray {
|
||||
T data[N];
|
||||
};
|
||||
|
||||
// ========================== Util functions to convert types ==========================
|
||||
template <typename T>
|
||||
__device__ float convert_to_float(T x) {
|
||||
if constexpr (std::is_same_v<T, __half>) {
|
||||
return __half2float(x);
|
||||
} else if constexpr (std::is_same_v<T, __nv_bfloat16>) {
|
||||
return __bfloat162float(x);
|
||||
} else if constexpr (std::is_same_v<T, float>) {
|
||||
return x;
|
||||
} else {
|
||||
return static_cast<float>(x);
|
||||
}
|
||||
}
|
||||
|
||||
// ====================== Sigmoid things ===============================
|
||||
// We have our own implementation of sigmoid here so we can support transposing the output
|
||||
// in the sigmoid kernel when we extend this module to support expert-choice routing.
|
||||
template <typename T, int TPB>
|
||||
__launch_bounds__(TPB) __global__ void moeSigmoid(
|
||||
const T* input, const bool* finished, float* output, const int num_cols, const float* correction_bias) {
|
||||
const int thread_row_offset = blockIdx.x * num_cols;
|
||||
|
||||
// Don't touch finished rows.
|
||||
if ((finished != nullptr) && finished[blockIdx.x]) {
|
||||
return;
|
||||
}
|
||||
|
||||
// First pass: Apply transformation, find max, and write transformed values to output
|
||||
for (int ii = threadIdx.x; ii < num_cols; ii += TPB) {
|
||||
const int idx = thread_row_offset + ii;
|
||||
float val = convert_to_float<T>(input[idx]);
|
||||
|
||||
val = 1.0f / (1.0f + expf(-val));
|
||||
|
||||
// Apply correction bias if provided
|
||||
if (correction_bias != nullptr) {
|
||||
val = val + correction_bias[ii];
|
||||
}
|
||||
|
||||
output[idx] = val; // Store transformed value
|
||||
}
|
||||
}
|
||||
|
||||
template <int TPB>
|
||||
__launch_bounds__(TPB) __global__ void moeTopK(
|
||||
const float* inputs_after_sigmoid,
|
||||
const bool* finished,
|
||||
float* output,
|
||||
int* indices,
|
||||
const int num_experts,
|
||||
const int k,
|
||||
const int start_expert,
|
||||
const int end_expert,
|
||||
const bool renormalize,
|
||||
const float* correction_bias) {
|
||||
using cub_kvp = cub::KeyValuePair<int, float>;
|
||||
using BlockReduce = cub::BlockReduce<cub_kvp, TPB>;
|
||||
__shared__ typename BlockReduce::TempStorage tmpStorage;
|
||||
|
||||
cub_kvp thread_kvp;
|
||||
cub::ArgMax arg_max;
|
||||
|
||||
const int block_row = blockIdx.x;
|
||||
|
||||
const bool row_is_active = finished ? !finished[block_row] : true;
|
||||
const int thread_read_offset = blockIdx.x * num_experts;
|
||||
float row_sum_for_renormalize = 0;
|
||||
for (int k_idx = 0; k_idx < k; ++k_idx) {
|
||||
thread_kvp.key = 0;
|
||||
thread_kvp.value = -1.f; // This is OK because inputs are probabilities
|
||||
|
||||
cub_kvp inp_kvp;
|
||||
for (int expert = threadIdx.x; expert < num_experts; expert += TPB) {
|
||||
const int idx = thread_read_offset + expert;
|
||||
inp_kvp.key = expert;
|
||||
inp_kvp.value = inputs_after_sigmoid[idx];
|
||||
|
||||
for (int prior_k = 0; prior_k < k_idx; ++prior_k) {
|
||||
const int prior_winning_expert = indices[k * block_row + prior_k];
|
||||
|
||||
if (prior_winning_expert == expert) {
|
||||
inp_kvp = thread_kvp;
|
||||
}
|
||||
}
|
||||
|
||||
thread_kvp = arg_max(inp_kvp, thread_kvp);
|
||||
}
|
||||
|
||||
const cub_kvp result_kvp = BlockReduce(tmpStorage).Reduce(thread_kvp, arg_max);
|
||||
if (threadIdx.x == 0) {
|
||||
// Ignore experts the node isn't responsible for with expert parallelism
|
||||
const int expert = result_kvp.key;
|
||||
const bool node_uses_expert = expert >= start_expert && expert < end_expert;
|
||||
const bool should_process_row = row_is_active && node_uses_expert;
|
||||
|
||||
const int idx = k * block_row + k_idx;
|
||||
float val = result_kvp.value;
|
||||
if (correction_bias != nullptr) {
|
||||
val -= correction_bias[expert];
|
||||
}
|
||||
output[idx] = val;
|
||||
indices[idx] = should_process_row ? (expert - start_expert) : num_experts;
|
||||
assert(indices[idx] >= 0);
|
||||
row_sum_for_renormalize += val;
|
||||
}
|
||||
__syncthreads();
|
||||
}
|
||||
|
||||
if (renormalize && threadIdx.x == 0) {
|
||||
float row_sum_for_renormalize_inv = 1.f / row_sum_for_renormalize;
|
||||
for (int k_idx = 0; k_idx < k; ++k_idx) {
|
||||
const int idx = k * block_row + k_idx;
|
||||
output[idx] = output[idx] * row_sum_for_renormalize_inv;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ====================== TopK sigmoid things ===============================
|
||||
|
||||
/*
|
||||
A Top-K gating sigmoid written to exploit when the number of experts in the MoE layers
|
||||
are a small power of 2. This allows us to cleanly share the rows among the threads in
|
||||
a single warp and eliminate communication between warps (so no need to use shared mem).
|
||||
|
||||
It fuses the sigmoid, max and argmax into a single kernel.
|
||||
|
||||
Limitations:
|
||||
1) This implementation is intended for when the number of experts is a small power of 2.
|
||||
2) This implementation assumes k is small, but will work for any k.
|
||||
*/
|
||||
|
||||
template <typename T, int VPT, int NUM_EXPERTS, int WARPS_PER_CTA, int BYTES_PER_LDG>
|
||||
__launch_bounds__(WARPS_PER_CTA* WARP_SIZE) __global__ void topkGatingSigmoid(
|
||||
const T* input,
|
||||
const bool* finished,
|
||||
float* output,
|
||||
const int num_rows,
|
||||
int* indices,
|
||||
const int k,
|
||||
const int start_expert,
|
||||
const int end_expert,
|
||||
const bool renormalize,
|
||||
const float* correction_bias) {
|
||||
// We begin by enforcing compile time assertions and setting up compile time constants.
|
||||
static_assert(VPT == (VPT & -VPT), "VPT must be power of 2");
|
||||
static_assert(NUM_EXPERTS == (NUM_EXPERTS & -NUM_EXPERTS), "NUM_EXPERTS must be power of 2");
|
||||
static_assert(BYTES_PER_LDG == (BYTES_PER_LDG & -BYTES_PER_LDG), "BYTES_PER_LDG must be power of 2");
|
||||
static_assert(BYTES_PER_LDG <= 16, "BYTES_PER_LDG must be leq 16");
|
||||
|
||||
// Number of bytes each thread pulls in per load
|
||||
static constexpr int ELTS_PER_LDG = BYTES_PER_LDG / sizeof(T);
|
||||
static constexpr int ELTS_PER_ROW = NUM_EXPERTS;
|
||||
static constexpr int THREADS_PER_ROW = ELTS_PER_ROW / VPT;
|
||||
static constexpr int LDG_PER_THREAD = VPT / ELTS_PER_LDG;
|
||||
|
||||
// Restrictions based on previous section.
|
||||
static_assert(VPT % ELTS_PER_LDG == 0, "The elements per thread must be a multiple of the elements per ldg");
|
||||
static_assert(WARP_SIZE % THREADS_PER_ROW == 0, "The threads per row must cleanly divide the threads per warp");
|
||||
static_assert(THREADS_PER_ROW == (THREADS_PER_ROW & -THREADS_PER_ROW), "THREADS_PER_ROW must be power of 2");
|
||||
static_assert(THREADS_PER_ROW <= WARP_SIZE, "THREADS_PER_ROW can be at most warp size");
|
||||
|
||||
// We have NUM_EXPERTS elements per row. We specialize for small #experts
|
||||
static constexpr int ELTS_PER_WARP = WARP_SIZE * VPT;
|
||||
static constexpr int ROWS_PER_WARP = ELTS_PER_WARP / ELTS_PER_ROW;
|
||||
static constexpr int ROWS_PER_CTA = WARPS_PER_CTA * ROWS_PER_WARP;
|
||||
|
||||
// Restrictions for previous section.
|
||||
static_assert(ELTS_PER_WARP % ELTS_PER_ROW == 0, "The elts per row must cleanly divide the total elt per warp");
|
||||
|
||||
// ===================== From this point, we finally start computing run-time variables. ========================
|
||||
|
||||
// Compute CTA and warp rows. We pack multiple rows into a single warp, and a block contains WARPS_PER_CTA warps.
|
||||
// This, each block processes a chunk of rows. We start by computing the start row for each block.
|
||||
const int cta_base_row = blockIdx.x * ROWS_PER_CTA;
|
||||
|
||||
// Now, using the base row per thread block, we compute the base row per warp.
|
||||
const int warp_base_row = cta_base_row + threadIdx.y * ROWS_PER_WARP;
|
||||
|
||||
// The threads in a warp are split into sub-groups that will work on a row.
|
||||
// We compute row offset for each thread sub-group
|
||||
const int thread_row_in_warp = threadIdx.x / THREADS_PER_ROW;
|
||||
const int thread_row = warp_base_row + thread_row_in_warp;
|
||||
|
||||
// Threads with indices out of bounds should early exit here.
|
||||
if (thread_row >= num_rows) {
|
||||
return;
|
||||
}
|
||||
const bool row_is_active = finished ? !finished[thread_row] : true;
|
||||
|
||||
// We finally start setting up the read pointers for each thread. First, each thread jumps to the start of the
|
||||
// row it will read.
|
||||
const T* thread_row_ptr = input + thread_row * ELTS_PER_ROW;
|
||||
|
||||
// Now, we compute the group each thread belong to in order to determine the first column to start loads.
|
||||
const int thread_group_idx = threadIdx.x % THREADS_PER_ROW;
|
||||
const int first_elt_read_by_thread = thread_group_idx * ELTS_PER_LDG;
|
||||
const T* thread_read_ptr = thread_row_ptr + first_elt_read_by_thread;
|
||||
|
||||
// Determine the pointer type to use to read in the data depending on the BYTES_PER_LDG template param. In theory,
|
||||
// this can support all powers of 2 up to 16.
|
||||
// NOTE(woosuk): The original implementation uses CUTLASS aligned array here.
|
||||
// We defined our own aligned array and use it here to avoid the dependency on CUTLASS.
|
||||
using AccessType = AlignedArray<T, ELTS_PER_LDG>;
|
||||
|
||||
// Finally, we pull in the data from global mem
|
||||
T row_chunk_temp[VPT];
|
||||
AccessType* row_chunk_vec_ptr = reinterpret_cast<AccessType*>(&row_chunk_temp);
|
||||
const AccessType* vec_thread_read_ptr = reinterpret_cast<const AccessType*>(thread_read_ptr);
|
||||
#pragma unroll
|
||||
// Note(Byron): interleaved loads to achieve better memory coalescing
|
||||
// | thread[0] | thread[1] | thread[2] | thread[3] | thread[0] | thread[1] | thread[2] | thread[3] | ...
|
||||
for (int ii = 0; ii < LDG_PER_THREAD; ++ii) {
|
||||
row_chunk_vec_ptr[ii] = vec_thread_read_ptr[ii * THREADS_PER_ROW];
|
||||
}
|
||||
|
||||
float row_chunk[VPT];
|
||||
#pragma unroll
|
||||
// Note(Byron): upcast logits to float32
|
||||
for (int ii = 0; ii < VPT; ++ii) {
|
||||
float val = convert_to_float<T>(row_chunk_temp[ii]);
|
||||
val = 1.0f / (1.0f + expf(-val));
|
||||
// Apply correction bias if provided
|
||||
if (correction_bias != nullptr) {
|
||||
/*
|
||||
LDG is interleaved
|
||||
|thread0 LDG| |thread1 LDG| |thread0 LDG| |thread1 LDG|
|
||||
|--------- group0 --------| |----------group1 --------|
|
||||
^ local2
|
||||
*/
|
||||
const int group_id = ii / ELTS_PER_LDG;
|
||||
const int local_id = ii % ELTS_PER_LDG;
|
||||
const int expert_idx = first_elt_read_by_thread + group_id * THREADS_PER_ROW * ELTS_PER_LDG + local_id;
|
||||
val = val + correction_bias[expert_idx];
|
||||
}
|
||||
|
||||
row_chunk[ii] = val;
|
||||
}
|
||||
|
||||
// Now, row_chunk contains the sigmoid of the row chunk. Now, I want to find the topk elements in each row, along
|
||||
// with the max index.
|
||||
int start_col = first_elt_read_by_thread;
|
||||
static constexpr int COLS_PER_GROUP_LDG = ELTS_PER_LDG * THREADS_PER_ROW;
|
||||
|
||||
float row_sum_for_renormalize = 0;
|
||||
|
||||
for (int k_idx = 0; k_idx < k; ++k_idx) {
|
||||
// First, each thread does the local argmax
|
||||
float max_val = row_chunk[0];
|
||||
int expert = start_col;
|
||||
#pragma unroll
|
||||
for (int ldg = 0, col = start_col; ldg < LDG_PER_THREAD; ++ldg, col += COLS_PER_GROUP_LDG) {
|
||||
#pragma unroll
|
||||
for (int ii = 0; ii < ELTS_PER_LDG; ++ii) {
|
||||
float val = row_chunk[ldg * ELTS_PER_LDG + ii];
|
||||
|
||||
// No check on the experts here since columns with the smallest index are processed first and only
|
||||
// updated if > (not >=)
|
||||
if (val > max_val) {
|
||||
max_val = val;
|
||||
expert = col + ii;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Now, we perform the argmax reduce. We use the butterfly pattern so threads reach consensus about the max.
|
||||
// This will be useful for K > 1 so that the threads can agree on "who" had the max value. That thread can
|
||||
// then blank out their max with -inf and the warp can run more iterations...
|
||||
#pragma unroll
|
||||
for (int mask = THREADS_PER_ROW / 2; mask > 0; mask /= 2) {
|
||||
float other_max = SGLANG_SHFL_XOR_SYNC_WIDTH(0xffffffff, max_val, mask, THREADS_PER_ROW);
|
||||
int other_expert = SGLANG_SHFL_XOR_SYNC_WIDTH(0xffffffff, expert, mask, THREADS_PER_ROW);
|
||||
|
||||
// We want lower indices to "win" in every thread so we break ties this way
|
||||
if (other_max > max_val || (other_max == max_val && other_expert < expert)) {
|
||||
max_val = other_max;
|
||||
expert = other_expert;
|
||||
}
|
||||
}
|
||||
|
||||
// Write the max for this k iteration to global memory.
|
||||
if (thread_group_idx == 0) {
|
||||
// Add a guard to ignore experts not included by this node
|
||||
const bool node_uses_expert = expert >= start_expert && expert < end_expert;
|
||||
const bool should_process_row = row_is_active && node_uses_expert;
|
||||
|
||||
// The lead thread from each sub-group will write out the final results to global memory. (This will be a
|
||||
// single) thread per row of the input/output matrices.
|
||||
const int idx = k * thread_row + k_idx;
|
||||
if (correction_bias != nullptr) {
|
||||
max_val -= correction_bias[expert];
|
||||
}
|
||||
output[idx] = max_val;
|
||||
indices[idx] = should_process_row ? (expert - start_expert) : NUM_EXPERTS;
|
||||
row_sum_for_renormalize += max_val;
|
||||
}
|
||||
|
||||
// Finally, we clear the value in the thread with the current max if there is another iteration to run.
|
||||
if (k_idx + 1 < k) {
|
||||
const int ldg_group_for_expert = expert / COLS_PER_GROUP_LDG;
|
||||
const int thread_to_clear_in_group = (expert / ELTS_PER_LDG) % THREADS_PER_ROW;
|
||||
|
||||
// Only the thread in the group which produced the max will reset the "winning" value to -inf.
|
||||
if (thread_group_idx == thread_to_clear_in_group) {
|
||||
const int offset_for_expert = expert % ELTS_PER_LDG;
|
||||
// Safe to set to any negative value since row_chunk values must be between 0 and 1.
|
||||
row_chunk[ldg_group_for_expert * ELTS_PER_LDG + offset_for_expert] = -10000.f;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Fuse renormalization of topk_weights into this kernel
|
||||
if (renormalize && thread_group_idx == 0) {
|
||||
float row_sum_for_renormalize_inv = 1.f / row_sum_for_renormalize;
|
||||
#pragma unroll
|
||||
for (int k_idx = 0; k_idx < k; ++k_idx) {
|
||||
const int idx = k * thread_row + k_idx;
|
||||
output[idx] = output[idx] * row_sum_for_renormalize_inv;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
namespace detail {
|
||||
// Constructs some constants needed to partition the work across threads at compile time.
|
||||
template <typename T, int EXPERTS, int BYTES_PER_LDG>
|
||||
struct TopkConstants {
|
||||
static constexpr int ELTS_PER_LDG = BYTES_PER_LDG / sizeof(T);
|
||||
static_assert(EXPERTS / (ELTS_PER_LDG * WARP_SIZE) == 0 || EXPERTS % (ELTS_PER_LDG * WARP_SIZE) == 0, "");
|
||||
static constexpr int VECs_PER_THREAD = MAX(1, EXPERTS / (ELTS_PER_LDG * WARP_SIZE));
|
||||
static constexpr int VPT = VECs_PER_THREAD * ELTS_PER_LDG;
|
||||
static constexpr int THREADS_PER_ROW = EXPERTS / VPT;
|
||||
static constexpr int ROWS_PER_WARP = WARP_SIZE / THREADS_PER_ROW;
|
||||
};
|
||||
} // namespace detail
|
||||
|
||||
template <typename T, int EXPERTS, int WARPS_PER_TB>
|
||||
void topkGatingSigmoidLauncherHelper(
|
||||
const T* input,
|
||||
const bool* finished,
|
||||
float* output,
|
||||
int* indices,
|
||||
const int num_rows,
|
||||
const int k,
|
||||
const int start_expert,
|
||||
const int end_expert,
|
||||
const bool renormalize,
|
||||
const float* correction_bias,
|
||||
cudaStream_t stream) {
|
||||
static constexpr std::size_t MAX_BYTES_PER_LDG = 16;
|
||||
|
||||
static constexpr int BYTES_PER_LDG = MIN(MAX_BYTES_PER_LDG, sizeof(T) * EXPERTS);
|
||||
using Constants = detail::TopkConstants<T, EXPERTS, BYTES_PER_LDG>;
|
||||
static constexpr int VPT = Constants::VPT;
|
||||
static constexpr int ROWS_PER_WARP = Constants::ROWS_PER_WARP;
|
||||
const int num_warps = (num_rows + ROWS_PER_WARP - 1) / ROWS_PER_WARP;
|
||||
const int num_blocks = (num_warps + WARPS_PER_TB - 1) / WARPS_PER_TB;
|
||||
|
||||
dim3 block_dim(WARP_SIZE, WARPS_PER_TB);
|
||||
topkGatingSigmoid<T, VPT, EXPERTS, WARPS_PER_TB, BYTES_PER_LDG><<<num_blocks, block_dim, 0, stream>>>(
|
||||
input, finished, output, num_rows, indices, k, start_expert, end_expert, renormalize, correction_bias);
|
||||
}
|
||||
|
||||
#define LAUNCH_SIGMOID(TYPE, NUM_EXPERTS, WARPS_PER_TB) \
|
||||
topkGatingSigmoidLauncherHelper<TYPE, NUM_EXPERTS, WARPS_PER_TB>( \
|
||||
gating_output, \
|
||||
nullptr, \
|
||||
topk_weights, \
|
||||
topk_indices, \
|
||||
num_tokens, \
|
||||
topk, \
|
||||
0, \
|
||||
num_experts, \
|
||||
renormalize, \
|
||||
correction_bias, \
|
||||
stream);
|
||||
|
||||
template <typename T>
|
||||
void topkGatingSigmoidKernelLauncher(
|
||||
const T* gating_output,
|
||||
float* topk_weights,
|
||||
int* topk_indices,
|
||||
float* sigmoid_workspace,
|
||||
const int num_tokens,
|
||||
const int num_experts,
|
||||
const int topk,
|
||||
const bool renormalize,
|
||||
const float* correction_bias,
|
||||
cudaStream_t stream) {
|
||||
static constexpr int WARPS_PER_TB = 4;
|
||||
switch (num_experts) {
|
||||
case 1:
|
||||
LAUNCH_SIGMOID(T, 1, WARPS_PER_TB);
|
||||
break;
|
||||
case 2:
|
||||
LAUNCH_SIGMOID(T, 2, WARPS_PER_TB);
|
||||
break;
|
||||
case 4:
|
||||
LAUNCH_SIGMOID(T, 4, WARPS_PER_TB);
|
||||
break;
|
||||
case 8:
|
||||
LAUNCH_SIGMOID(T, 8, WARPS_PER_TB);
|
||||
break;
|
||||
case 16:
|
||||
LAUNCH_SIGMOID(T, 16, WARPS_PER_TB);
|
||||
break;
|
||||
case 32:
|
||||
LAUNCH_SIGMOID(T, 32, WARPS_PER_TB);
|
||||
break;
|
||||
case 64:
|
||||
LAUNCH_SIGMOID(T, 64, WARPS_PER_TB);
|
||||
break;
|
||||
case 128:
|
||||
LAUNCH_SIGMOID(T, 128, WARPS_PER_TB);
|
||||
break;
|
||||
case 256:
|
||||
LAUNCH_SIGMOID(T, 256, WARPS_PER_TB);
|
||||
break;
|
||||
default: {
|
||||
TORCH_CHECK(
|
||||
sigmoid_workspace != nullptr,
|
||||
"sigmoid_workspace must be provided for num_experts that are not a power of 2.");
|
||||
static constexpr int TPB = 256;
|
||||
moeSigmoid<T, TPB>
|
||||
<<<num_tokens, TPB, 0, stream>>>(gating_output, nullptr, sigmoid_workspace, num_experts, correction_bias);
|
||||
moeTopK<TPB><<<num_tokens, TPB, 0, stream>>>(
|
||||
sigmoid_workspace,
|
||||
nullptr,
|
||||
topk_weights,
|
||||
topk_indices,
|
||||
num_experts,
|
||||
topk,
|
||||
0,
|
||||
num_experts,
|
||||
renormalize,
|
||||
correction_bias);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void topk_sigmoid(
|
||||
torch::Tensor& topk_weights, // [num_tokens, topk]
|
||||
torch::Tensor& topk_indices, // [num_tokens, topk]
|
||||
torch::Tensor& gating_output, // [num_tokens, num_experts]
|
||||
const bool renormalize,
|
||||
const c10::optional<torch::Tensor>& correction_bias) {
|
||||
// Check data type
|
||||
TORCH_CHECK(
|
||||
gating_output.scalar_type() == at::ScalarType::Float || gating_output.scalar_type() == at::ScalarType::Half ||
|
||||
gating_output.scalar_type() == at::ScalarType::BFloat16,
|
||||
"gating_output must be float32, float16, or bfloat16");
|
||||
|
||||
// Check dimensions
|
||||
TORCH_CHECK(gating_output.dim() == 2, "gating_output must be 2D tensor [num_tokens, num_experts]");
|
||||
TORCH_CHECK(topk_weights.dim() == 2, "topk_weights must be 2D tensor [num_tokens, topk]");
|
||||
TORCH_CHECK(topk_indices.dim() == 2, "topk_indices must be 2D tensor [num_tokens, topk]");
|
||||
|
||||
// Check shapes
|
||||
TORCH_CHECK(
|
||||
gating_output.size(0) == topk_weights.size(0),
|
||||
"First dimension of topk_weights must match num_tokens in gating_output");
|
||||
TORCH_CHECK(
|
||||
gating_output.size(0) == topk_indices.size(0),
|
||||
"First dimension of topk_indices must match num_tokens in gating_output");
|
||||
TORCH_CHECK(
|
||||
topk_weights.size(-1) == topk_indices.size(-1),
|
||||
"Second dimension of topk_indices must match topk in topk_weights");
|
||||
TORCH_CHECK(topk_weights.size(-1) <= gating_output.size(-1), "topk must be less than or equal to num_experts");
|
||||
|
||||
const int num_experts = static_cast<int>(gating_output.size(-1));
|
||||
const int num_tokens = static_cast<int>(gating_output.size(0));
|
||||
const int topk = static_cast<int>(topk_weights.size(-1));
|
||||
|
||||
const bool is_pow_2 = (num_experts != 0) && ((num_experts & (num_experts - 1)) == 0);
|
||||
const bool needs_workspace = !is_pow_2 || num_experts > 256;
|
||||
const int64_t workspace_size = needs_workspace ? num_tokens * num_experts : 0;
|
||||
|
||||
const at::cuda::OptionalCUDAGuard device_guard(device_of(gating_output));
|
||||
const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
|
||||
torch::Tensor sigmoid_workspace =
|
||||
torch::empty({workspace_size}, gating_output.options().dtype(at::ScalarType::Float));
|
||||
|
||||
const at::ScalarType dtype = gating_output.scalar_type();
|
||||
|
||||
// Validate correction_bias if provided - must always be float32
|
||||
const float* bias_ptr = nullptr;
|
||||
if (correction_bias.has_value()) {
|
||||
const torch::Tensor& bias_tensor = correction_bias.value();
|
||||
TORCH_CHECK(bias_tensor.dim() == 1, "correction_bias must be 1D tensor [num_experts]");
|
||||
TORCH_CHECK(bias_tensor.size(0) == num_experts, "correction_bias size must match num_experts");
|
||||
TORCH_CHECK(
|
||||
bias_tensor.scalar_type() == at::ScalarType::Float,
|
||||
"correction_bias must be float32, got ",
|
||||
bias_tensor.scalar_type());
|
||||
bias_ptr = bias_tensor.data_ptr<float>();
|
||||
}
|
||||
|
||||
if (dtype == at::ScalarType::Float) {
|
||||
topkGatingSigmoidKernelLauncher<float>(
|
||||
gating_output.data_ptr<float>(),
|
||||
topk_weights.data_ptr<float>(),
|
||||
topk_indices.data_ptr<int>(),
|
||||
sigmoid_workspace.data_ptr<float>(),
|
||||
num_tokens,
|
||||
num_experts,
|
||||
topk,
|
||||
renormalize,
|
||||
bias_ptr,
|
||||
stream);
|
||||
} else if (dtype == at::ScalarType::Half) {
|
||||
topkGatingSigmoidKernelLauncher<__half>(
|
||||
reinterpret_cast<const __half*>(gating_output.data_ptr<at::Half>()),
|
||||
topk_weights.data_ptr<float>(),
|
||||
topk_indices.data_ptr<int>(),
|
||||
sigmoid_workspace.data_ptr<float>(),
|
||||
num_tokens,
|
||||
num_experts,
|
||||
topk,
|
||||
renormalize,
|
||||
bias_ptr,
|
||||
stream);
|
||||
} else if (dtype == at::ScalarType::BFloat16) {
|
||||
topkGatingSigmoidKernelLauncher<__nv_bfloat16>(
|
||||
reinterpret_cast<const __nv_bfloat16*>(gating_output.data_ptr<at::BFloat16>()),
|
||||
topk_weights.data_ptr<float>(),
|
||||
topk_indices.data_ptr<int>(),
|
||||
sigmoid_workspace.data_ptr<float>(),
|
||||
num_tokens,
|
||||
num_experts,
|
||||
topk,
|
||||
renormalize,
|
||||
bias_ptr,
|
||||
stream);
|
||||
} else {
|
||||
TORCH_CHECK(false, "Unsupported gating_output dtype: ", dtype);
|
||||
}
|
||||
}
|
||||
820
third_party/sglang/sgl-kernel/csrc/moe/moe_topk_softmax_kernels.cu
vendored
Normal file
820
third_party/sglang/sgl-kernel/csrc/moe/moe_topk_softmax_kernels.cu
vendored
Normal file
@@ -0,0 +1,820 @@
|
||||
// Adapt from https://github.com/vllm-project/vllm/blob/v0.7.3/csrc/moe/topk_softmax_kernels.cu
|
||||
// which is originally adapted from
|
||||
// https://github.com/NVIDIA/TensorRT-LLM/blob/v0.7.1/cpp/tensorrt_llm/kernels/mixtureOfExperts/moe_kernels.cu
|
||||
/* Copyright 2025 SGLang Team. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
#include <ATen/cuda/CUDAContext.h>
|
||||
#include <c10/cuda/CUDAGuard.h>
|
||||
#include <torch/all.h>
|
||||
|
||||
#ifndef USE_ROCM
|
||||
#include <cub/cub.cuh>
|
||||
#include <cub/util_type.cuh>
|
||||
#include <cuda/functional>
|
||||
#else
|
||||
#include <hipcub/hipcub.hpp>
|
||||
#include <hipcub/util_type.hpp>
|
||||
#endif
|
||||
|
||||
#include "utils.h"
|
||||
|
||||
#define MAX(a, b) ((a) > (b) ? (a) : (b))
|
||||
#define MIN(a, b) ((a) < (b) ? (a) : (b))
|
||||
|
||||
// Define reduction operators based on CUDA version
|
||||
// CUDA 13 (12.9+) deprecated cub::Max/Min in favor of cuda::maximum/minimum
|
||||
#if CUDA_VERSION >= 12090
|
||||
using MaxReduceOp = cuda::maximum<>;
|
||||
using MinReduceOp = cuda::minimum<>;
|
||||
#else
|
||||
using MaxReduceOp = cub::Max;
|
||||
using MinReduceOp = cub::Min;
|
||||
#endif
|
||||
|
||||
using cub_kvp = cub::KeyValuePair<int, float>;
|
||||
|
||||
/// Aligned array type
|
||||
template <
|
||||
typename T,
|
||||
/// Number of elements in the array
|
||||
int N,
|
||||
/// Alignment requirement in bytes
|
||||
int Alignment = sizeof(T) * N>
|
||||
class alignas(Alignment) AlignedArray {
|
||||
T data[N];
|
||||
};
|
||||
|
||||
// ========================== Util functions to convert types ==========================
|
||||
template <typename T>
|
||||
__device__ float convert_to_float(T x) {
|
||||
if constexpr (std::is_same_v<T, __half>) {
|
||||
return __half2float(x);
|
||||
} else if constexpr (std::is_same_v<T, __nv_bfloat16>) {
|
||||
return __bfloat162float(x);
|
||||
} else if constexpr (std::is_same_v<T, float>) {
|
||||
return x;
|
||||
} else {
|
||||
return static_cast<float>(x);
|
||||
}
|
||||
}
|
||||
|
||||
// ====================== Softmax things ===============================
|
||||
// We have our own implementation of softmax here so we can support transposing the output
|
||||
// in the softmax kernel when we extend this module to support expert-choice routing.
|
||||
template <typename T, int TPB>
|
||||
__launch_bounds__(TPB) __global__ void moeSoftmax(
|
||||
const T* input,
|
||||
const bool* finished,
|
||||
float* output,
|
||||
const int num_cols,
|
||||
const float moe_softcapping,
|
||||
const float* correction_bias) {
|
||||
using BlockReduce = cub::BlockReduce<float, TPB>;
|
||||
__shared__ typename BlockReduce::TempStorage tmpStorage;
|
||||
|
||||
__shared__ float normalizing_factor;
|
||||
__shared__ float float_max;
|
||||
|
||||
const int thread_row_offset = blockIdx.x * num_cols;
|
||||
|
||||
float threadData(-FLT_MAX);
|
||||
|
||||
// Don't touch finished rows.
|
||||
if ((finished != nullptr) && finished[blockIdx.x]) {
|
||||
return;
|
||||
}
|
||||
|
||||
// First pass: Apply transformation, find max, and write transformed values to output
|
||||
for (int ii = threadIdx.x; ii < num_cols; ii += TPB) {
|
||||
const int idx = thread_row_offset + ii;
|
||||
float val = convert_to_float<T>(input[idx]);
|
||||
|
||||
// Apply tanh softcapping if enabled
|
||||
if (moe_softcapping != 0.0f) {
|
||||
val = tanhf(val / moe_softcapping) * moe_softcapping;
|
||||
}
|
||||
|
||||
// Apply correction bias if provided
|
||||
if (correction_bias != nullptr) {
|
||||
val = val + correction_bias[ii];
|
||||
}
|
||||
|
||||
output[idx] = val; // Store transformed value
|
||||
threadData = max(val, threadData);
|
||||
}
|
||||
|
||||
const float maxElem = BlockReduce(tmpStorage).Reduce(threadData, MaxReduceOp());
|
||||
|
||||
if (threadIdx.x == 0) {
|
||||
float_max = maxElem;
|
||||
}
|
||||
__syncthreads();
|
||||
|
||||
// Second pass: Compute sum using transformed values from output
|
||||
threadData = 0;
|
||||
for (int ii = threadIdx.x; ii < num_cols; ii += TPB) {
|
||||
const int idx = thread_row_offset + ii;
|
||||
threadData += exp((output[idx] - float_max));
|
||||
}
|
||||
|
||||
const auto Z = BlockReduce(tmpStorage).Sum(threadData);
|
||||
|
||||
if (threadIdx.x == 0) {
|
||||
normalizing_factor = 1.f / Z;
|
||||
}
|
||||
__syncthreads();
|
||||
|
||||
// Third pass: Compute final softmax using transformed values from output
|
||||
for (int ii = threadIdx.x; ii < num_cols; ii += TPB) {
|
||||
const int idx = thread_row_offset + ii;
|
||||
const float softmax_val = exp((output[idx] - float_max)) * normalizing_factor;
|
||||
output[idx] = softmax_val;
|
||||
}
|
||||
}
|
||||
|
||||
namespace moe {
|
||||
struct TopKPair {
|
||||
static const int PAIR = 2;
|
||||
static const int MAX_INDEX = 0;
|
||||
cub_kvp max;
|
||||
cub_kvp secondMax;
|
||||
|
||||
__device__ TopKPair() {}
|
||||
__device__ TopKPair(cub_kvp max, cub_kvp secondMax) : max(max), secondMax(secondMax) {}
|
||||
};
|
||||
|
||||
struct TopKPairArgMax {
|
||||
__device__ TopKPairArgMax() {}
|
||||
__device__ __forceinline__ TopKPair operator()(const TopKPair& candidate1, const TopKPair& candidate2) const {
|
||||
cub_kvp globalMax, globalSecondMax;
|
||||
|
||||
// Determine the global maximum
|
||||
if (candidate1.max.value > candidate2.max.value) {
|
||||
globalMax = candidate1.max;
|
||||
} else {
|
||||
globalMax = candidate2.max;
|
||||
}
|
||||
|
||||
// Determine the global second maximum
|
||||
if (globalMax.key == candidate1.max.key) {
|
||||
// If candidate1 contributed the max, compare its secondMax with candidate2's max
|
||||
globalSecondMax = (candidate1.secondMax.value > candidate2.max.value) ? candidate1.secondMax : candidate2.max;
|
||||
} else {
|
||||
// If candidate2 contributed the max, compare its secondMax with candidate1's max
|
||||
globalSecondMax = (candidate2.secondMax.value > candidate1.max.value) ? candidate2.secondMax : candidate1.max;
|
||||
}
|
||||
return TopKPair(globalMax, globalSecondMax);
|
||||
}
|
||||
};
|
||||
} // namespace moe
|
||||
|
||||
template <int TPB>
|
||||
__launch_bounds__(TPB) __global__ void moeTopKFast(
|
||||
float* inputs_after_softmax,
|
||||
const bool* finished,
|
||||
float* output,
|
||||
int* indices,
|
||||
const int num_experts,
|
||||
const int k,
|
||||
const int start_expert,
|
||||
const int end_expert,
|
||||
const bool renormalize) {
|
||||
using namespace moe;
|
||||
using BlockReduce = cub::BlockReduce<TopKPair, TPB>;
|
||||
__shared__ typename BlockReduce::TempStorage tmpStorage;
|
||||
TopKPair thread_pair;
|
||||
|
||||
const int block_row = blockIdx.x;
|
||||
|
||||
const bool row_is_active = finished ? !finished[block_row] : true;
|
||||
const int thread_read_offset = blockIdx.x * num_experts;
|
||||
float row_sum_for_renormalize = 0;
|
||||
// Each loop finds the top 2 elements,
|
||||
// thus requiring only ⌈k/2⌉ loops (calculated as (k + 1) / 2).
|
||||
for (int k_idx = 0; k_idx < (k + TopKPair::PAIR - 1) / TopKPair::PAIR; ++k_idx) {
|
||||
// Initializing the top 2 elements by the minimum value.
|
||||
thread_pair.max.key = 0;
|
||||
thread_pair.max.value = -1.f;
|
||||
thread_pair.secondMax.key = 0;
|
||||
thread_pair.secondMax.value = -1.f;
|
||||
|
||||
cub_kvp inp_kvp;
|
||||
for (int expert = threadIdx.x; expert < num_experts; expert += TPB) {
|
||||
const int idx = thread_read_offset + expert;
|
||||
inp_kvp.key = expert;
|
||||
inp_kvp.value = inputs_after_softmax[idx];
|
||||
// updating the thread_pair according to inp_kvp's value
|
||||
if (inp_kvp.value > thread_pair.max.value) {
|
||||
thread_pair.secondMax = thread_pair.max;
|
||||
thread_pair.max = inp_kvp;
|
||||
} else if (inp_kvp.value > thread_pair.secondMax.value) {
|
||||
thread_pair.secondMax = inp_kvp;
|
||||
}
|
||||
}
|
||||
|
||||
TopKPairArgMax reducer;
|
||||
const TopKPair result_pair = BlockReduce(tmpStorage).Reduce(thread_pair, reducer);
|
||||
if (threadIdx.x == 0) {
|
||||
#pragma unroll
|
||||
// updating 2 elements to the result.
|
||||
for (int i = 0; i < TopKPair::PAIR; i++) {
|
||||
if (k_idx * 2 + i >= k) break;
|
||||
cub_kvp result = (i == TopKPair::MAX_INDEX) ? result_pair.max : result_pair.secondMax;
|
||||
int expert = result.key;
|
||||
bool node_uses_expert = expert >= start_expert && expert < end_expert;
|
||||
bool should_process_row = row_is_active && node_uses_expert;
|
||||
// The inputs_after_softmax is modified in-place to avoid unnecessary loops for finding the top k-1 value.
|
||||
// 1.f represents the minimum value.
|
||||
inputs_after_softmax[thread_read_offset + expert] = -1.f;
|
||||
int idx = k * block_row + k_idx * 2 + i;
|
||||
output[idx] = result.value;
|
||||
indices[idx] = should_process_row ? (expert - start_expert) : num_experts;
|
||||
assert(indices[idx] >= 0);
|
||||
row_sum_for_renormalize += result.value;
|
||||
}
|
||||
}
|
||||
__syncthreads();
|
||||
}
|
||||
|
||||
if (renormalize && threadIdx.x == 0) {
|
||||
float row_sum_for_renormalize_inv = 1.f / row_sum_for_renormalize;
|
||||
for (int k_idx = 0; k_idx < k; ++k_idx) {
|
||||
const int idx = k * block_row + k_idx;
|
||||
output[idx] = output[idx] * row_sum_for_renormalize_inv;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
template <int TPB>
|
||||
__launch_bounds__(TPB) __global__ void moeTopK(
|
||||
float* inputs_after_softmax,
|
||||
const bool* finished,
|
||||
float* output,
|
||||
int* indices,
|
||||
const int num_experts,
|
||||
const int k,
|
||||
const int start_expert,
|
||||
const int end_expert,
|
||||
const bool renormalize) {
|
||||
using cub_kvp = cub::KeyValuePair<int, float>;
|
||||
using BlockReduce = cub::BlockReduce<cub_kvp, TPB>;
|
||||
__shared__ typename BlockReduce::TempStorage tmpStorage;
|
||||
|
||||
cub_kvp thread_kvp;
|
||||
cub::ArgMax arg_max;
|
||||
|
||||
const int block_row = blockIdx.x;
|
||||
|
||||
const bool row_is_active = finished ? !finished[block_row] : true;
|
||||
const int thread_read_offset = blockIdx.x * num_experts;
|
||||
float row_sum_for_renormalize = 0;
|
||||
for (int k_idx = 0; k_idx < k; ++k_idx) {
|
||||
thread_kvp.key = 0;
|
||||
thread_kvp.value = -1.f; // This is OK because inputs are probabilities
|
||||
|
||||
cub_kvp inp_kvp;
|
||||
for (int expert = threadIdx.x; expert < num_experts; expert += TPB) {
|
||||
const int idx = thread_read_offset + expert;
|
||||
inp_kvp.key = expert;
|
||||
inp_kvp.value = inputs_after_softmax[idx];
|
||||
thread_kvp = arg_max(inp_kvp, thread_kvp);
|
||||
}
|
||||
|
||||
const cub_kvp result_kvp = BlockReduce(tmpStorage).Reduce(thread_kvp, arg_max);
|
||||
if (threadIdx.x == 0) {
|
||||
// Ignore experts the node isn't responsible for with expert parallelism
|
||||
const int expert = result_kvp.key;
|
||||
const bool node_uses_expert = expert >= start_expert && expert < end_expert;
|
||||
const bool should_process_row = row_is_active && node_uses_expert;
|
||||
|
||||
const int idx = k * block_row + k_idx;
|
||||
output[idx] = result_kvp.value;
|
||||
indices[idx] = should_process_row ? (expert - start_expert) : num_experts;
|
||||
assert(indices[idx] >= 0);
|
||||
row_sum_for_renormalize += result_kvp.value;
|
||||
// The inputs_after_softmax is modified in-place to avoid unnecessary loops for finding the top k-1 value.
|
||||
// 1.f represents the minimum value.
|
||||
inputs_after_softmax[thread_read_offset + expert] = -1.f;
|
||||
}
|
||||
__syncthreads();
|
||||
}
|
||||
|
||||
if (renormalize && threadIdx.x == 0) {
|
||||
float row_sum_for_renormalize_inv = 1.f / row_sum_for_renormalize;
|
||||
for (int k_idx = 0; k_idx < k; ++k_idx) {
|
||||
const int idx = k * block_row + k_idx;
|
||||
output[idx] = output[idx] * row_sum_for_renormalize_inv;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ====================== TopK softmax things ===============================
|
||||
|
||||
/*
|
||||
A Top-K gating softmax written to exploit when the number of experts in the MoE layers
|
||||
are a small power of 2. This allows us to cleanly share the rows among the threads in
|
||||
a single warp and eliminate communication between warps (so no need to use shared mem).
|
||||
|
||||
It fuses the softmax, max and argmax into a single kernel.
|
||||
|
||||
Limitations:
|
||||
1) This implementation is intended for when the number of experts is a small power of 2.
|
||||
2) This implementation assumes k is small, but will work for any k.
|
||||
*/
|
||||
|
||||
template <typename T, int VPT, int NUM_EXPERTS, int WARPS_PER_CTA, int BYTES_PER_LDG>
|
||||
__launch_bounds__(WARPS_PER_CTA* WARP_SIZE) __global__ void topkGatingSoftmax(
|
||||
const T* input,
|
||||
const bool* finished,
|
||||
float* output,
|
||||
const int num_rows,
|
||||
int* indices,
|
||||
const int k,
|
||||
const int start_expert,
|
||||
const int end_expert,
|
||||
const bool renormalize,
|
||||
const float moe_softcapping,
|
||||
const float* correction_bias) {
|
||||
// We begin by enforcing compile time assertions and setting up compile time constants.
|
||||
static_assert(VPT == (VPT & -VPT), "VPT must be power of 2");
|
||||
static_assert(NUM_EXPERTS == (NUM_EXPERTS & -NUM_EXPERTS), "NUM_EXPERTS must be power of 2");
|
||||
static_assert(BYTES_PER_LDG == (BYTES_PER_LDG & -BYTES_PER_LDG), "BYTES_PER_LDG must be power of 2");
|
||||
static_assert(BYTES_PER_LDG <= 16, "BYTES_PER_LDG must be leq 16");
|
||||
|
||||
// Number of bytes each thread pulls in per load
|
||||
static constexpr int ELTS_PER_LDG = BYTES_PER_LDG / sizeof(T);
|
||||
static constexpr int ELTS_PER_ROW = NUM_EXPERTS;
|
||||
static constexpr int THREADS_PER_ROW = ELTS_PER_ROW / VPT;
|
||||
static constexpr int LDG_PER_THREAD = VPT / ELTS_PER_LDG;
|
||||
|
||||
// Restrictions based on previous section.
|
||||
static_assert(VPT % ELTS_PER_LDG == 0, "The elements per thread must be a multiple of the elements per ldg");
|
||||
static_assert(WARP_SIZE % THREADS_PER_ROW == 0, "The threads per row must cleanly divide the threads per warp");
|
||||
static_assert(THREADS_PER_ROW == (THREADS_PER_ROW & -THREADS_PER_ROW), "THREADS_PER_ROW must be power of 2");
|
||||
static_assert(THREADS_PER_ROW <= WARP_SIZE, "THREADS_PER_ROW can be at most warp size");
|
||||
|
||||
// We have NUM_EXPERTS elements per row. We specialize for small #experts
|
||||
static constexpr int ELTS_PER_WARP = WARP_SIZE * VPT;
|
||||
static constexpr int ROWS_PER_WARP = ELTS_PER_WARP / ELTS_PER_ROW;
|
||||
static constexpr int ROWS_PER_CTA = WARPS_PER_CTA * ROWS_PER_WARP;
|
||||
|
||||
// Restrictions for previous section.
|
||||
static_assert(ELTS_PER_WARP % ELTS_PER_ROW == 0, "The elts per row must cleanly divide the total elt per warp");
|
||||
|
||||
// ===================== From this point, we finally start computing run-time variables. ========================
|
||||
|
||||
// Compute CTA and warp rows. We pack multiple rows into a single warp, and a block contains WARPS_PER_CTA warps.
|
||||
// This, each block processes a chunk of rows. We start by computing the start row for each block.
|
||||
const int cta_base_row = blockIdx.x * ROWS_PER_CTA;
|
||||
|
||||
// Now, using the base row per thread block, we compute the base row per warp.
|
||||
const int warp_base_row = cta_base_row + threadIdx.y * ROWS_PER_WARP;
|
||||
|
||||
// The threads in a warp are split into sub-groups that will work on a row.
|
||||
// We compute row offset for each thread sub-group
|
||||
const int thread_row_in_warp = threadIdx.x / THREADS_PER_ROW;
|
||||
const int thread_row = warp_base_row + thread_row_in_warp;
|
||||
|
||||
// Threads with indices out of bounds should early exit here.
|
||||
if (thread_row >= num_rows) {
|
||||
return;
|
||||
}
|
||||
const bool row_is_active = finished ? !finished[thread_row] : true;
|
||||
|
||||
// We finally start setting up the read pointers for each thread. First, each thread jumps to the start of the
|
||||
// row it will read.
|
||||
const T* thread_row_ptr = input + thread_row * ELTS_PER_ROW;
|
||||
|
||||
// Now, we compute the group each thread belong to in order to determine the first column to start loads.
|
||||
const int thread_group_idx = threadIdx.x % THREADS_PER_ROW;
|
||||
const int first_elt_read_by_thread = thread_group_idx * ELTS_PER_LDG;
|
||||
const T* thread_read_ptr = thread_row_ptr + first_elt_read_by_thread;
|
||||
|
||||
// Determine the pointer type to use to read in the data depending on the BYTES_PER_LDG template param. In theory,
|
||||
// this can support all powers of 2 up to 16.
|
||||
// NOTE(woosuk): The original implementation uses CUTLASS aligned array here.
|
||||
// We defined our own aligned array and use it here to avoid the dependency on CUTLASS.
|
||||
using AccessType = AlignedArray<T, ELTS_PER_LDG>;
|
||||
|
||||
// Finally, we pull in the data from global mem
|
||||
T row_chunk_temp[VPT];
|
||||
AccessType* row_chunk_vec_ptr = reinterpret_cast<AccessType*>(&row_chunk_temp);
|
||||
const AccessType* vec_thread_read_ptr = reinterpret_cast<const AccessType*>(thread_read_ptr);
|
||||
#pragma unroll
|
||||
// Note(Byron): interleaved loads to achieve better memory coalescing
|
||||
// | thread[0] | thread[1] | thread[2] | thread[3] | thread[0] | thread[1] | thread[2] | thread[3] | ...
|
||||
for (int ii = 0; ii < LDG_PER_THREAD; ++ii) {
|
||||
row_chunk_vec_ptr[ii] = vec_thread_read_ptr[ii * THREADS_PER_ROW];
|
||||
}
|
||||
|
||||
float row_chunk[VPT];
|
||||
#pragma unroll
|
||||
// Note(Byron): upcast logits to float32
|
||||
for (int ii = 0; ii < VPT; ++ii) {
|
||||
row_chunk[ii] = convert_to_float<T>(row_chunk_temp[ii]);
|
||||
}
|
||||
|
||||
// Apply tanh softcapping and correction bias
|
||||
if (moe_softcapping != 0.0f || correction_bias != nullptr) {
|
||||
#pragma unroll
|
||||
for (int ii = 0; ii < VPT; ++ii) {
|
||||
float val = row_chunk[ii];
|
||||
|
||||
// Apply tanh softcapping if enabled
|
||||
if (moe_softcapping != 0.0f) {
|
||||
val = tanhf(val / moe_softcapping) * moe_softcapping;
|
||||
}
|
||||
|
||||
// Apply correction bias if provided
|
||||
if (correction_bias != nullptr) {
|
||||
/*
|
||||
LDG is interleaved
|
||||
|thread0 LDG| |thread1 LDG| |thread0 LDG| |thread1 LDG|
|
||||
|--------- group0 --------| |----------group1 --------|
|
||||
^ local2
|
||||
*/
|
||||
const int group_id = ii / ELTS_PER_LDG;
|
||||
const int local_id = ii % ELTS_PER_LDG;
|
||||
const int expert_idx = first_elt_read_by_thread + group_id * THREADS_PER_ROW * ELTS_PER_LDG + local_id;
|
||||
val = val + correction_bias[expert_idx];
|
||||
}
|
||||
|
||||
row_chunk[ii] = val;
|
||||
}
|
||||
}
|
||||
|
||||
// First, we perform a max reduce within the thread. We can do the max in fp16 safely (I think) and just
|
||||
// convert to float afterwards for the exp + sum reduction.
|
||||
float thread_max = row_chunk[0];
|
||||
#pragma unroll
|
||||
for (int ii = 1; ii < VPT; ++ii) {
|
||||
thread_max = max(thread_max, row_chunk[ii]);
|
||||
}
|
||||
|
||||
/*********************************/
|
||||
/********* Softmax Begin *********/
|
||||
/*********************************/
|
||||
|
||||
// Now, we find the max within the thread group and distribute among the threads. We use a butterfly reduce.
|
||||
// lane id: 0-31 within a warp
|
||||
#pragma unroll
|
||||
for (int mask = THREADS_PER_ROW / 2; mask > 0; mask /= 2) {
|
||||
// butterfly reduce with (lane id ^ mask)
|
||||
thread_max = max(thread_max, SGLANG_SHFL_XOR_SYNC_WIDTH(0xffffffff, thread_max, mask, THREADS_PER_ROW));
|
||||
}
|
||||
|
||||
// From this point, thread max in all the threads have the max within the row.
|
||||
// Now, we subtract the max from each element in the thread and take the exp. We also compute the thread local sum.
|
||||
float row_sum = 0;
|
||||
#pragma unroll
|
||||
for (int ii = 0; ii < VPT; ++ii) {
|
||||
row_chunk[ii] = expf(row_chunk[ii] - thread_max);
|
||||
row_sum += row_chunk[ii];
|
||||
}
|
||||
|
||||
// Now, we perform the sum reduce within each thread group. Similar to the max reduce, we use a bufferfly pattern.
|
||||
#pragma unroll
|
||||
for (int mask = THREADS_PER_ROW / 2; mask > 0; mask /= 2) {
|
||||
row_sum += SGLANG_SHFL_XOR_SYNC_WIDTH(0xffffffff, row_sum, mask, THREADS_PER_ROW);
|
||||
}
|
||||
|
||||
// From this point, all threads have the max and the sum for their rows in the thread_max and thread_sum variables
|
||||
// respectively. Finally, we can scale the rows for the softmax. Technically, for top-k gating we don't need to
|
||||
// compute the entire softmax row. We can likely look at the maxes and only compute for the top-k values in the row.
|
||||
// However, this kernel will likely not be a bottle neck and it seems better to closer match torch and find the
|
||||
// argmax after computing the softmax.
|
||||
const float reciprocal_row_sum = 1.f / row_sum;
|
||||
|
||||
#pragma unroll
|
||||
for (int ii = 0; ii < VPT; ++ii) {
|
||||
row_chunk[ii] = row_chunk[ii] * reciprocal_row_sum;
|
||||
}
|
||||
/*******************************/
|
||||
/********* Softmax End *********/
|
||||
/*******************************/
|
||||
|
||||
// Now, softmax_res contains the softmax of the row chunk. Now, I want to find the topk elements in each row, along
|
||||
// with the max index.
|
||||
int start_col = first_elt_read_by_thread;
|
||||
static constexpr int COLS_PER_GROUP_LDG = ELTS_PER_LDG * THREADS_PER_ROW;
|
||||
|
||||
float row_sum_for_renormalize = 0;
|
||||
|
||||
for (int k_idx = 0; k_idx < k; ++k_idx) {
|
||||
// First, each thread does the local argmax
|
||||
float max_val = row_chunk[0];
|
||||
int expert = start_col;
|
||||
#pragma unroll
|
||||
for (int ldg = 0, col = start_col; ldg < LDG_PER_THREAD; ++ldg, col += COLS_PER_GROUP_LDG) {
|
||||
#pragma unroll
|
||||
for (int ii = 0; ii < ELTS_PER_LDG; ++ii) {
|
||||
float val = row_chunk[ldg * ELTS_PER_LDG + ii];
|
||||
|
||||
// No check on the experts here since columns with the smallest index are processed first and only
|
||||
// updated if > (not >=)
|
||||
if (val > max_val) {
|
||||
max_val = val;
|
||||
expert = col + ii;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Now, we perform the argmax reduce. We use the butterfly pattern so threads reach consensus about the max.
|
||||
// This will be useful for K > 1 so that the threads can agree on "who" had the max value. That thread can
|
||||
// then blank out their max with -inf and the warp can run more iterations...
|
||||
#pragma unroll
|
||||
for (int mask = THREADS_PER_ROW / 2; mask > 0; mask /= 2) {
|
||||
float other_max = SGLANG_SHFL_XOR_SYNC_WIDTH(0xffffffff, max_val, mask, THREADS_PER_ROW);
|
||||
int other_expert = SGLANG_SHFL_XOR_SYNC_WIDTH(0xffffffff, expert, mask, THREADS_PER_ROW);
|
||||
|
||||
// We want lower indices to "win" in every thread so we break ties this way
|
||||
if (other_max > max_val || (other_max == max_val && other_expert < expert)) {
|
||||
max_val = other_max;
|
||||
expert = other_expert;
|
||||
}
|
||||
}
|
||||
|
||||
// Write the max for this k iteration to global memory.
|
||||
if (thread_group_idx == 0) {
|
||||
// Add a guard to ignore experts not included by this node
|
||||
const bool node_uses_expert = expert >= start_expert && expert < end_expert;
|
||||
const bool should_process_row = row_is_active && node_uses_expert;
|
||||
|
||||
// The lead thread from each sub-group will write out the final results to global memory. (This will be a
|
||||
// single) thread per row of the input/output matrices.
|
||||
const int idx = k * thread_row + k_idx;
|
||||
output[idx] = max_val;
|
||||
indices[idx] = should_process_row ? (expert - start_expert) : NUM_EXPERTS;
|
||||
row_sum_for_renormalize += max_val;
|
||||
}
|
||||
|
||||
// Finally, we clear the value in the thread with the current max if there is another iteration to run.
|
||||
if (k_idx + 1 < k) {
|
||||
const int ldg_group_for_expert = expert / COLS_PER_GROUP_LDG;
|
||||
const int thread_to_clear_in_group = (expert / ELTS_PER_LDG) % THREADS_PER_ROW;
|
||||
|
||||
// Only the thread in the group which produced the max will reset the "winning" value to -inf.
|
||||
if (thread_group_idx == thread_to_clear_in_group) {
|
||||
const int offset_for_expert = expert % ELTS_PER_LDG;
|
||||
// Safe to set to any negative value since row_chunk values must be between 0 and 1.
|
||||
row_chunk[ldg_group_for_expert * ELTS_PER_LDG + offset_for_expert] = -10000.f;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Fuse renormalization of topk_weights into this kernel
|
||||
if (renormalize && thread_group_idx == 0) {
|
||||
float row_sum_for_renormalize_inv = 1.f / row_sum_for_renormalize;
|
||||
#pragma unroll
|
||||
for (int k_idx = 0; k_idx < k; ++k_idx) {
|
||||
const int idx = k * thread_row + k_idx;
|
||||
output[idx] = output[idx] * row_sum_for_renormalize_inv;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
namespace detail {
|
||||
// Constructs some constants needed to partition the work across threads at compile time.
|
||||
template <typename T, int EXPERTS, int BYTES_PER_LDG>
|
||||
struct TopkConstants {
|
||||
static constexpr int ELTS_PER_LDG = BYTES_PER_LDG / sizeof(T);
|
||||
static_assert(EXPERTS / (ELTS_PER_LDG * WARP_SIZE) == 0 || EXPERTS % (ELTS_PER_LDG * WARP_SIZE) == 0, "");
|
||||
static constexpr int VECs_PER_THREAD = MAX(1, EXPERTS / (ELTS_PER_LDG * WARP_SIZE));
|
||||
static constexpr int VPT = VECs_PER_THREAD * ELTS_PER_LDG;
|
||||
static constexpr int THREADS_PER_ROW = EXPERTS / VPT;
|
||||
static constexpr int ROWS_PER_WARP = WARP_SIZE / THREADS_PER_ROW;
|
||||
};
|
||||
} // namespace detail
|
||||
|
||||
template <typename T, int EXPERTS, int WARPS_PER_TB>
|
||||
void topkGatingSoftmaxLauncherHelper(
|
||||
const T* input,
|
||||
const bool* finished,
|
||||
float* output,
|
||||
int* indices,
|
||||
const int num_rows,
|
||||
const int k,
|
||||
const int start_expert,
|
||||
const int end_expert,
|
||||
const bool renormalize,
|
||||
const float moe_softcapping,
|
||||
const float* correction_bias,
|
||||
cudaStream_t stream) {
|
||||
static constexpr std::size_t MAX_BYTES_PER_LDG = 16;
|
||||
|
||||
static constexpr int BYTES_PER_LDG = MIN(MAX_BYTES_PER_LDG, sizeof(T) * EXPERTS);
|
||||
using Constants = detail::TopkConstants<T, EXPERTS, BYTES_PER_LDG>;
|
||||
static constexpr int VPT = Constants::VPT;
|
||||
static constexpr int ROWS_PER_WARP = Constants::ROWS_PER_WARP;
|
||||
const int num_warps = (num_rows + ROWS_PER_WARP - 1) / ROWS_PER_WARP;
|
||||
const int num_blocks = (num_warps + WARPS_PER_TB - 1) / WARPS_PER_TB;
|
||||
|
||||
dim3 block_dim(WARP_SIZE, WARPS_PER_TB);
|
||||
topkGatingSoftmax<T, VPT, EXPERTS, WARPS_PER_TB, BYTES_PER_LDG><<<num_blocks, block_dim, 0, stream>>>(
|
||||
input,
|
||||
finished,
|
||||
output,
|
||||
num_rows,
|
||||
indices,
|
||||
k,
|
||||
start_expert,
|
||||
end_expert,
|
||||
renormalize,
|
||||
moe_softcapping,
|
||||
correction_bias);
|
||||
}
|
||||
|
||||
#define LAUNCH_SOFTMAX(TYPE, NUM_EXPERTS, WARPS_PER_TB) \
|
||||
topkGatingSoftmaxLauncherHelper<TYPE, NUM_EXPERTS, WARPS_PER_TB>( \
|
||||
gating_output, \
|
||||
nullptr, \
|
||||
topk_weights, \
|
||||
topk_indices, \
|
||||
num_tokens, \
|
||||
topk, \
|
||||
0, \
|
||||
num_experts, \
|
||||
renormalize, \
|
||||
moe_softcapping, \
|
||||
correction_bias, \
|
||||
stream);
|
||||
|
||||
template <typename T>
|
||||
void topkGatingSoftmaxKernelLauncher(
|
||||
const T* gating_output,
|
||||
float* topk_weights,
|
||||
int* topk_indices,
|
||||
float* softmax_workspace,
|
||||
const int num_tokens,
|
||||
const int num_experts,
|
||||
const int topk,
|
||||
const bool renormalize,
|
||||
const float moe_softcapping,
|
||||
const float* correction_bias,
|
||||
cudaStream_t stream) {
|
||||
static constexpr int WARPS_PER_TB = 4;
|
||||
switch (num_experts) {
|
||||
case 1:
|
||||
LAUNCH_SOFTMAX(T, 1, WARPS_PER_TB);
|
||||
break;
|
||||
case 2:
|
||||
LAUNCH_SOFTMAX(T, 2, WARPS_PER_TB);
|
||||
break;
|
||||
case 4:
|
||||
LAUNCH_SOFTMAX(T, 4, WARPS_PER_TB);
|
||||
break;
|
||||
case 8:
|
||||
LAUNCH_SOFTMAX(T, 8, WARPS_PER_TB);
|
||||
break;
|
||||
case 16:
|
||||
LAUNCH_SOFTMAX(T, 16, WARPS_PER_TB);
|
||||
break;
|
||||
case 32:
|
||||
LAUNCH_SOFTMAX(T, 32, WARPS_PER_TB);
|
||||
break;
|
||||
case 64:
|
||||
LAUNCH_SOFTMAX(T, 64, WARPS_PER_TB);
|
||||
break;
|
||||
case 128:
|
||||
LAUNCH_SOFTMAX(T, 128, WARPS_PER_TB);
|
||||
break;
|
||||
case 256:
|
||||
LAUNCH_SOFTMAX(T, 256, WARPS_PER_TB);
|
||||
break;
|
||||
default: {
|
||||
TORCH_CHECK(
|
||||
softmax_workspace != nullptr,
|
||||
"softmax_workspace must be provided for num_experts that are not a power of 2.");
|
||||
static constexpr int TPB = 256;
|
||||
moeSoftmax<T, TPB><<<num_tokens, TPB, 0, stream>>>(
|
||||
gating_output, nullptr, softmax_workspace, num_experts, moe_softcapping, correction_bias);
|
||||
if (topk == 1) {
|
||||
// Note: As an optimization for better performance,
|
||||
// the softmax_workspace is overwritten in-place by both moeTopK and moeTopKFast.
|
||||
moeTopK<TPB><<<num_tokens, TPB, 0, stream>>>(
|
||||
softmax_workspace, nullptr, topk_weights, topk_indices, num_experts, topk, 0, num_experts, renormalize);
|
||||
} else {
|
||||
moeTopKFast<TPB><<<num_tokens, TPB, 0, stream>>>(
|
||||
softmax_workspace, nullptr, topk_weights, topk_indices, num_experts, topk, 0, num_experts, renormalize);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void topk_softmax(
|
||||
torch::Tensor& topk_weights, // [num_tokens, topk]
|
||||
torch::Tensor& topk_indices, // [num_tokens, topk]
|
||||
torch::Tensor& gating_output, // [num_tokens, num_experts]
|
||||
const bool renormalize,
|
||||
const double moe_softcapping,
|
||||
const c10::optional<torch::Tensor>& correction_bias) {
|
||||
// Check data type
|
||||
TORCH_CHECK(
|
||||
gating_output.scalar_type() == at::ScalarType::Float || gating_output.scalar_type() == at::ScalarType::Half ||
|
||||
gating_output.scalar_type() == at::ScalarType::BFloat16,
|
||||
"gating_output must be float32, float16, or bfloat16");
|
||||
|
||||
// Check dimensions
|
||||
TORCH_CHECK(gating_output.dim() == 2, "gating_output must be 2D tensor [num_tokens, num_experts]");
|
||||
TORCH_CHECK(topk_weights.dim() == 2, "topk_weights must be 2D tensor [num_tokens, topk]");
|
||||
TORCH_CHECK(topk_indices.dim() == 2, "topk_indices must be 2D tensor [num_tokens, topk]");
|
||||
|
||||
// Check shapes
|
||||
TORCH_CHECK(
|
||||
gating_output.size(0) == topk_weights.size(0),
|
||||
"First dimension of topk_weights must match num_tokens in gating_output");
|
||||
TORCH_CHECK(
|
||||
gating_output.size(0) == topk_indices.size(0),
|
||||
"First dimension of topk_indices must match num_tokens in gating_output");
|
||||
TORCH_CHECK(
|
||||
topk_weights.size(-1) == topk_indices.size(-1),
|
||||
"Second dimension of topk_indices must match topk in topk_weights");
|
||||
TORCH_CHECK(topk_weights.size(-1) <= gating_output.size(-1), "topk must be less than or equal to num_experts");
|
||||
|
||||
const int num_experts = static_cast<int>(gating_output.size(-1));
|
||||
const int num_tokens = static_cast<int>(gating_output.size(0));
|
||||
const int topk = static_cast<int>(topk_weights.size(-1));
|
||||
|
||||
const bool is_pow_2 = (num_experts != 0) && ((num_experts & (num_experts - 1)) == 0);
|
||||
const bool needs_workspace = !is_pow_2 || num_experts > 256;
|
||||
const int64_t workspace_size = needs_workspace ? num_tokens * num_experts : 0;
|
||||
|
||||
const at::cuda::OptionalCUDAGuard device_guard(device_of(gating_output));
|
||||
const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
|
||||
torch::Tensor softmax_workspace =
|
||||
torch::empty({workspace_size}, gating_output.options().dtype(at::ScalarType::Float));
|
||||
|
||||
const at::ScalarType dtype = gating_output.scalar_type();
|
||||
|
||||
// Validate correction_bias if provided - must always be float32
|
||||
const float* bias_ptr = nullptr;
|
||||
if (correction_bias.has_value()) {
|
||||
const torch::Tensor& bias_tensor = correction_bias.value();
|
||||
TORCH_CHECK(bias_tensor.dim() == 1, "correction_bias must be 1D tensor [num_experts]");
|
||||
TORCH_CHECK(bias_tensor.size(0) == num_experts, "correction_bias size must match num_experts");
|
||||
TORCH_CHECK(
|
||||
bias_tensor.scalar_type() == at::ScalarType::Float,
|
||||
"correction_bias must be float32, got ",
|
||||
bias_tensor.scalar_type());
|
||||
bias_ptr = bias_tensor.data_ptr<float>();
|
||||
}
|
||||
|
||||
// Cast moe_softcapping from double to float for CUDA kernels
|
||||
const float moe_softcapping_f = static_cast<float>(moe_softcapping);
|
||||
|
||||
if (dtype == at::ScalarType::Float) {
|
||||
topkGatingSoftmaxKernelLauncher<float>(
|
||||
gating_output.data_ptr<float>(),
|
||||
topk_weights.data_ptr<float>(),
|
||||
topk_indices.data_ptr<int>(),
|
||||
softmax_workspace.data_ptr<float>(),
|
||||
num_tokens,
|
||||
num_experts,
|
||||
topk,
|
||||
renormalize,
|
||||
moe_softcapping_f,
|
||||
bias_ptr,
|
||||
stream);
|
||||
} else if (dtype == at::ScalarType::Half) {
|
||||
topkGatingSoftmaxKernelLauncher<__half>(
|
||||
reinterpret_cast<const __half*>(gating_output.data_ptr<at::Half>()),
|
||||
topk_weights.data_ptr<float>(),
|
||||
topk_indices.data_ptr<int>(),
|
||||
softmax_workspace.data_ptr<float>(),
|
||||
num_tokens,
|
||||
num_experts,
|
||||
topk,
|
||||
renormalize,
|
||||
moe_softcapping_f,
|
||||
bias_ptr,
|
||||
stream);
|
||||
} else if (dtype == at::ScalarType::BFloat16) {
|
||||
topkGatingSoftmaxKernelLauncher<__nv_bfloat16>(
|
||||
reinterpret_cast<const __nv_bfloat16*>(gating_output.data_ptr<at::BFloat16>()),
|
||||
topk_weights.data_ptr<float>(),
|
||||
topk_indices.data_ptr<int>(),
|
||||
softmax_workspace.data_ptr<float>(),
|
||||
num_tokens,
|
||||
num_experts,
|
||||
topk,
|
||||
renormalize,
|
||||
moe_softcapping_f,
|
||||
bias_ptr,
|
||||
stream);
|
||||
} else {
|
||||
TORCH_CHECK(false, "Unsupported gating_output dtype: ", dtype);
|
||||
}
|
||||
}
|
||||
392
third_party/sglang/sgl-kernel/csrc/moe/prepare_moe_input.cu
vendored
Normal file
392
third_party/sglang/sgl-kernel/csrc/moe/prepare_moe_input.cu
vendored
Normal file
@@ -0,0 +1,392 @@
|
||||
#include <c10/cuda/CUDAGuard.h>
|
||||
#include <cudaTypedefs.h>
|
||||
#include <torch/all.h>
|
||||
|
||||
#include <flashinfer/vec_dtypes.cuh>
|
||||
#include <iostream>
|
||||
|
||||
#include "cutlass/array.h"
|
||||
#include "utils.h"
|
||||
|
||||
constexpr uint64_t THREADS_PER_EXPERT = 512;
|
||||
|
||||
__global__ void compute_problem_sizes(
|
||||
const int* __restrict__ topk_ids,
|
||||
int32_t* problem_sizes1,
|
||||
int32_t* problem_sizes2,
|
||||
int32_t* atomic_buffer,
|
||||
const int64_t topk_length,
|
||||
const int64_t n,
|
||||
const int64_t k) {
|
||||
int expert_id = blockIdx.x;
|
||||
|
||||
int occurrences = 0;
|
||||
for (int i = threadIdx.x; i < topk_length; i += THREADS_PER_EXPERT) {
|
||||
occurrences += (topk_ids[i] == expert_id);
|
||||
}
|
||||
atomicAdd(&atomic_buffer[expert_id], occurrences);
|
||||
__syncthreads();
|
||||
|
||||
if (threadIdx.x == 0) {
|
||||
int final_occurrences = atomic_buffer[expert_id];
|
||||
problem_sizes1[expert_id * 3] = final_occurrences;
|
||||
problem_sizes1[expert_id * 3 + 1] = static_cast<int32_t>(2 * n);
|
||||
problem_sizes1[expert_id * 3 + 2] = static_cast<int32_t>(k);
|
||||
problem_sizes2[expert_id * 3] = final_occurrences;
|
||||
problem_sizes2[expert_id * 3 + 1] = static_cast<int32_t>(k);
|
||||
problem_sizes2[expert_id * 3 + 2] = static_cast<int32_t>(n);
|
||||
}
|
||||
}
|
||||
|
||||
__global__ void compute_expert_offsets(
|
||||
const int32_t* __restrict__ problem_sizes1,
|
||||
int32_t* expert_offsets,
|
||||
int32_t* atomic_buffer,
|
||||
const int64_t num_experts) {
|
||||
int32_t tot_offset = 0;
|
||||
expert_offsets[0] = 0;
|
||||
for (int i = 0; i < num_experts; ++i) {
|
||||
atomic_buffer[i] = tot_offset;
|
||||
tot_offset += problem_sizes1[i * 3];
|
||||
expert_offsets[i + 1] = tot_offset;
|
||||
}
|
||||
}
|
||||
|
||||
__global__ void compute_expert_blockscale_offsets(
|
||||
const int32_t* __restrict__ problem_sizes1,
|
||||
int32_t* expert_offsets,
|
||||
int32_t* blockscale_offsets,
|
||||
int32_t* atomic_buffer,
|
||||
const int64_t num_experts) {
|
||||
int32_t tot_offset = 0;
|
||||
int32_t tot_rounded_offset = 0;
|
||||
expert_offsets[0] = 0;
|
||||
blockscale_offsets[0] = 0;
|
||||
for (int i = 0; i < num_experts; ++i) {
|
||||
atomic_buffer[i] = tot_offset;
|
||||
int num_tokens = problem_sizes1[i * 3];
|
||||
int rounded_num_tokens = (num_tokens + (128 - 1)) / 128 * 128;
|
||||
tot_offset += num_tokens;
|
||||
tot_rounded_offset += rounded_num_tokens;
|
||||
expert_offsets[i + 1] = tot_offset;
|
||||
blockscale_offsets[i + 1] = tot_rounded_offset;
|
||||
}
|
||||
}
|
||||
|
||||
__global__ void compute_arg_sorts(
|
||||
const int32_t* __restrict__ topk_ids,
|
||||
int32_t* input_permutation,
|
||||
int32_t* output_permutation,
|
||||
int32_t* atomic_buffer,
|
||||
const int64_t topk_length,
|
||||
const int64_t topk) {
|
||||
int expert_id = blockIdx.x;
|
||||
|
||||
for (int i = threadIdx.x; i < topk_length; i += THREADS_PER_EXPERT) {
|
||||
if (topk_ids[i] == expert_id) {
|
||||
int start = atomicAdd(&atomic_buffer[expert_id], 1);
|
||||
input_permutation[start] = i / topk;
|
||||
output_permutation[i] = start;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void get_moe_prepare_input_caller(
|
||||
const torch::Tensor& topk_ids,
|
||||
torch::Tensor& expert_offsets,
|
||||
const std::optional<torch::Tensor>& blockscale_offsets,
|
||||
torch::Tensor& problem_sizes1,
|
||||
torch::Tensor& problem_sizes2,
|
||||
torch::Tensor& input_permutation,
|
||||
torch::Tensor& output_permutation,
|
||||
const int64_t num_experts,
|
||||
const int64_t n,
|
||||
const int64_t k) {
|
||||
auto stream = at::cuda::getCurrentCUDAStream(topk_ids.device().index());
|
||||
auto options_int32 = torch::TensorOptions().dtype(torch::kInt32).device(topk_ids.device());
|
||||
torch::Tensor atomic_buffer = torch::zeros(num_experts, options_int32);
|
||||
|
||||
uint32_t num_threads = static_cast<uint32_t>(min(THREADS_PER_EXPERT, topk_ids.numel()));
|
||||
uint32_t num_blocks = static_cast<uint32_t>(num_experts);
|
||||
|
||||
compute_problem_sizes<<<num_blocks, num_threads, 0, stream>>>(
|
||||
static_cast<const int32_t*>(topk_ids.data_ptr()),
|
||||
static_cast<int32_t*>(problem_sizes1.data_ptr()),
|
||||
static_cast<int32_t*>(problem_sizes2.data_ptr()),
|
||||
static_cast<int32_t*>(atomic_buffer.data_ptr()),
|
||||
topk_ids.numel(),
|
||||
n,
|
||||
k);
|
||||
if (blockscale_offsets.has_value()) {
|
||||
compute_expert_blockscale_offsets<<<1, 1, 0, stream>>>(
|
||||
static_cast<const int32_t*>(problem_sizes1.data_ptr()),
|
||||
static_cast<int32_t*>(expert_offsets.data_ptr()),
|
||||
static_cast<int32_t*>(blockscale_offsets.value().data_ptr()),
|
||||
static_cast<int32_t*>(atomic_buffer.data_ptr()),
|
||||
num_experts);
|
||||
} else {
|
||||
compute_expert_offsets<<<1, 1, 0, stream>>>(
|
||||
static_cast<const int32_t*>(problem_sizes1.data_ptr()),
|
||||
static_cast<int32_t*>(expert_offsets.data_ptr()),
|
||||
static_cast<int32_t*>(atomic_buffer.data_ptr()),
|
||||
num_experts);
|
||||
}
|
||||
compute_arg_sorts<<<num_blocks, num_threads, 0, stream>>>(
|
||||
static_cast<const int32_t*>(topk_ids.data_ptr()),
|
||||
static_cast<int32_t*>(input_permutation.data_ptr()),
|
||||
static_cast<int32_t*>(output_permutation.data_ptr()),
|
||||
static_cast<int32_t*>(atomic_buffer.data_ptr()),
|
||||
topk_ids.numel(),
|
||||
topk_ids.size(1));
|
||||
}
|
||||
|
||||
void prepare_moe_input(
|
||||
const torch::Tensor& topk_ids,
|
||||
torch::Tensor& expert_offsets,
|
||||
const std::optional<torch::Tensor>& blockscale_offsets,
|
||||
torch::Tensor& problem_sizes1,
|
||||
torch::Tensor& problem_sizes2,
|
||||
torch::Tensor& input_permutation,
|
||||
torch::Tensor& output_permutation,
|
||||
const int64_t num_experts,
|
||||
const int64_t n,
|
||||
const int64_t k) {
|
||||
TORCH_CHECK(topk_ids.dtype() == torch::kInt32);
|
||||
get_moe_prepare_input_caller(
|
||||
topk_ids,
|
||||
expert_offsets,
|
||||
blockscale_offsets,
|
||||
problem_sizes1,
|
||||
problem_sizes2,
|
||||
input_permutation,
|
||||
output_permutation,
|
||||
num_experts,
|
||||
n,
|
||||
k);
|
||||
return;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
__global__ void shuffleRowsKernel(
|
||||
const T* input,
|
||||
const int32_t* dst2src_map,
|
||||
T* output,
|
||||
int64_t num_src_rows,
|
||||
int64_t num_dst_rows,
|
||||
int64_t num_cols) {
|
||||
int64_t dest_row_idx = blockIdx.x;
|
||||
int64_t const source_row_idx = dst2src_map[dest_row_idx];
|
||||
|
||||
if (blockIdx.x < num_dst_rows) {
|
||||
// Load 128-bits per thread
|
||||
constexpr uint64_t ELEM_PER_THREAD = 128 / sizeof(T) / 8;
|
||||
using DataElem = cutlass::Array<T, ELEM_PER_THREAD>;
|
||||
|
||||
// Duplicate and permute rows
|
||||
auto const* source_row_ptr = reinterpret_cast<DataElem const*>(input + source_row_idx * num_cols);
|
||||
auto* dest_row_ptr = reinterpret_cast<DataElem*>(output + dest_row_idx * num_cols);
|
||||
|
||||
auto const start_offset = threadIdx.x;
|
||||
auto const stride = blockDim.x;
|
||||
auto const num_elems_in_col = num_cols / ELEM_PER_THREAD;
|
||||
|
||||
for (auto elem_index = start_offset; elem_index < num_elems_in_col; elem_index += stride) {
|
||||
dest_row_ptr[elem_index] = source_row_ptr[elem_index];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#define DECLARE_SHUFFLE_ROWS(T) \
|
||||
__global__ void shuffleRowsKernel( \
|
||||
const T* input, \
|
||||
const int32_t* dst2src_map, \
|
||||
T* output, \
|
||||
int64_t num_src_rows, \
|
||||
int64_t num_dest_rows, \
|
||||
int64_t num_cols);
|
||||
|
||||
DECLARE_SHUFFLE_ROWS(float);
|
||||
DECLARE_SHUFFLE_ROWS(half);
|
||||
DECLARE_SHUFFLE_ROWS(__nv_bfloat16);
|
||||
DECLARE_SHUFFLE_ROWS(__nv_fp8_e4m3);
|
||||
DECLARE_SHUFFLE_ROWS(uint8_t);
|
||||
|
||||
#define SHUFFLE_ROWS(T) \
|
||||
shuffleRowsKernel<T><<<blocks, threads, 0, stream>>>( \
|
||||
reinterpret_cast<const T*>(input), \
|
||||
static_cast<const int32_t*>(dst2src_map.data_ptr()), \
|
||||
reinterpret_cast<T*>(output), \
|
||||
num_src_rows, \
|
||||
num_dst_rows, \
|
||||
num_cols)
|
||||
|
||||
#define DTYPE_DISPATCH_CASE(T, CUDA_T) \
|
||||
case T: \
|
||||
SHUFFLE_ROWS(CUDA_T); \
|
||||
break;
|
||||
|
||||
void shuffle_rows_caller(
|
||||
const torch::Tensor& input_tensor, const torch::Tensor& dst2src_map, torch::Tensor& output_tensor) {
|
||||
TORCH_CHECK(
|
||||
input_tensor.scalar_type() == output_tensor.scalar_type(),
|
||||
"Input and output tensors must have the same data type");
|
||||
auto stream = at::cuda::getCurrentCUDAStream().stream();
|
||||
uint32_t blocks = static_cast<uint32_t>(output_tensor.size(0));
|
||||
uint32_t threads = 256;
|
||||
int64_t num_dst_rows = output_tensor.size(0);
|
||||
int64_t num_src_rows = input_tensor.size(0);
|
||||
int64_t num_cols = input_tensor.size(1);
|
||||
const void* input = input_tensor.data_ptr();
|
||||
void* output = output_tensor.data_ptr();
|
||||
switch (input_tensor.scalar_type()) {
|
||||
DTYPE_DISPATCH_CASE(torch::kFloat16, half);
|
||||
DTYPE_DISPATCH_CASE(torch::kBFloat16, __nv_bfloat16);
|
||||
DTYPE_DISPATCH_CASE(torch::kFloat32, float);
|
||||
DTYPE_DISPATCH_CASE(torch::kFloat8_e4m3fn, __nv_fp8_e4m3);
|
||||
DTYPE_DISPATCH_CASE(torch::kUInt8, uint8_t);
|
||||
default:
|
||||
TORCH_CHECK(false, "[moe replicate input] data type dispatch fail!");
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
void shuffle_rows(const torch::Tensor& input_tensor, const torch::Tensor& dst2src_map, torch::Tensor& output_tensor) {
|
||||
shuffle_rows_caller(input_tensor, dst2src_map, output_tensor);
|
||||
return;
|
||||
}
|
||||
|
||||
template <typename scalar_t>
|
||||
__global__ void apply_shuffle_mul_sum_kernel(
|
||||
const scalar_t* __restrict__ input_tensor, // [m * topk, k] (expert-major layout)
|
||||
scalar_t* __restrict__ output_tensor, // [m, k] (token-major layout)
|
||||
const int32_t* __restrict__ permutation, // [m * topk] (c_map: token-major-idx -> expert-major-idx)
|
||||
int m,
|
||||
int topk,
|
||||
int row_stride,
|
||||
const scalar_t* __restrict__ factors) // [m * topk] (topk_weights, token-major layout)
|
||||
{
|
||||
int i = blockIdx.x;
|
||||
if (i >= m) {
|
||||
return;
|
||||
}
|
||||
|
||||
constexpr uint32_t vec_size = 16 / sizeof(scalar_t);
|
||||
using t = float;
|
||||
using vec_t = flashinfer::vec_t<t, vec_size>;
|
||||
int thread_idx = threadIdx.x;
|
||||
int stride = blockDim.x;
|
||||
|
||||
for (int d_vec_idx = thread_idx; d_vec_idx < row_stride / vec_size; d_vec_idx += stride) {
|
||||
int d = d_vec_idx * vec_size;
|
||||
vec_t sum_vec;
|
||||
sum_vec.fill(0.0f);
|
||||
|
||||
for (int j = 0; j < topk; ++j) {
|
||||
int token_major_idx = i * topk + j;
|
||||
int src_row = permutation[token_major_idx];
|
||||
|
||||
vec_t val_vec;
|
||||
val_vec.cast_load(input_tensor + src_row * row_stride + d);
|
||||
|
||||
t factor = 1.0;
|
||||
if (factors != nullptr) {
|
||||
factor = factors[token_major_idx];
|
||||
}
|
||||
|
||||
#pragma unroll
|
||||
for (int k = 0; k < vec_size; ++k) {
|
||||
sum_vec[k] += factor * val_vec[k];
|
||||
}
|
||||
}
|
||||
sum_vec.cast_store(output_tensor + i * row_stride + d);
|
||||
}
|
||||
|
||||
// remainder part
|
||||
int remainder_start = (row_stride / vec_size) * vec_size;
|
||||
for (int d = remainder_start + thread_idx; d < row_stride; d += stride) {
|
||||
t sum_val = 0.0;
|
||||
for (int j = 0; j < topk; ++j) {
|
||||
int token_major_idx = i * topk + j;
|
||||
int src_row = permutation[token_major_idx];
|
||||
t val = input_tensor[src_row * row_stride + d];
|
||||
|
||||
t factor = 1.0;
|
||||
if (factors != nullptr) {
|
||||
factor = factors[token_major_idx];
|
||||
}
|
||||
sum_val += factor * val;
|
||||
}
|
||||
output_tensor[i * row_stride + d] = sum_val;
|
||||
}
|
||||
}
|
||||
|
||||
void get_apply_shuffle_mul_sum_caller(
|
||||
const torch::Tensor& input_tensor, // [m * topk, row_stride], bf16/f16
|
||||
torch::Tensor& output_tensor, // [m, row_stride], bf16/f16
|
||||
const torch::Tensor& permutation, // [m * topk], int32
|
||||
const std::optional<torch::Tensor>& factors_opt) // optional [m * topk], bf16/f16
|
||||
{
|
||||
TORCH_CHECK(input_tensor.dim() == 2, "input_tensor must be 2D [m * topk, row_stride]");
|
||||
TORCH_CHECK(output_tensor.dim() == 2, "output_tensor must be 2D [m, row_stride]");
|
||||
TORCH_CHECK(permutation.dim() == 1, "permutation must be 1D [m * topk]");
|
||||
|
||||
int m = output_tensor.size(0);
|
||||
int topk = int(permutation.size(0) / m);
|
||||
int row_stride = output_tensor.size(1);
|
||||
|
||||
TORCH_CHECK(permutation.size(0) == m * topk, "permutation size must match m * topk");
|
||||
|
||||
auto scalar_type = output_tensor.scalar_type();
|
||||
uint32_t vec_size = 16 / sizeof(scalar_type);
|
||||
auto blockDim = std::min(row_stride / vec_size, 1024U);
|
||||
dim3 block(blockDim);
|
||||
|
||||
dim3 grid(m); // blockIdx.x = j, blockIdx.y = i
|
||||
auto stream = at::cuda::getCurrentCUDAStream(input_tensor.device().index());
|
||||
|
||||
const int32_t* perm_ptr = permutation.data_ptr<int32_t>();
|
||||
|
||||
void* factors_ptr = nullptr;
|
||||
if (factors_opt.has_value()) {
|
||||
TORCH_CHECK(factors_opt->dtype() == output_tensor.dtype(), "Factors must match output dtype");
|
||||
TORCH_CHECK(factors_opt->numel() == m * topk, "Factors must have shape [m * topk]");
|
||||
factors_ptr = factors_opt->data_ptr();
|
||||
}
|
||||
|
||||
DISPATCH_PYTORCH_DTYPE_TO_CTYPE_FLOAT_FP16(output_tensor.scalar_type(), scalar_t, [&] {
|
||||
apply_shuffle_mul_sum_kernel<scalar_t><<<grid, block, 0, stream>>>(
|
||||
static_cast<const scalar_t*>(input_tensor.data_ptr()),
|
||||
static_cast<scalar_t*>(output_tensor.data_ptr()),
|
||||
perm_ptr,
|
||||
m,
|
||||
topk,
|
||||
row_stride,
|
||||
static_cast<const scalar_t*>(factors_ptr));
|
||||
return true;
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Applies a permutation-based shuffle, element-wise multiplication, and reduction over the second dimension.
|
||||
*
|
||||
* This function performs the equivalent of the following PyTorch expression:
|
||||
*
|
||||
* (c2[c_map].view(m, topk, k) * topk_weights.view(m, topk, 1).to(out_dtype)).sum(dim=1)
|
||||
*
|
||||
* Specifically:
|
||||
* - `input` is shuffled using the `permutation` tensor.
|
||||
* - The shuffled tensor is reshaped and multiplied element-wise with `factors` (e.g., top-k weights).
|
||||
* - The result is summed along dimension 1 (the top-k dimension), and stored in `output`.
|
||||
*
|
||||
* @param input Input tensor of shape (m * topk, k), representing c2.
|
||||
* @param output Output tensor of shape (m, k), where the final reduced results are stored.
|
||||
* @param permutation Index tensor (e.g., c_map) that maps positions in `input` to shuffled layout.
|
||||
* @param factors Optional scaling factors (e.g., top-k weights), shape (m * topk) or (m, topk).
|
||||
*/
|
||||
void apply_shuffle_mul_sum(
|
||||
const torch::Tensor& input,
|
||||
torch::Tensor& output,
|
||||
const torch::Tensor& permutation,
|
||||
const std::optional<torch::Tensor>& factors) {
|
||||
get_apply_shuffle_mul_sum_caller(input, output, permutation, factors);
|
||||
}
|
||||
Reference in New Issue
Block a user