Gahow Wang
c8e8153702
CUDA kernels (csrc/): - common.cuh: shared warp_reduce_sum/max, block_reduce_sum/max - normalization/rmsnorm.cu: RMSNorm (F32 + BF16) - normalization/layernorm.cu: LayerNorm with Welford (F32 + BF16) - activation/activations.cu: GELU tanh-approx + SiLU (F32 + BF16) - reduce/softmax.cu: safe softmax, 3-pass (F32 + BF16) - embedding/embedding.cu: gather lookup (F32 + BF16) - embedding/rope.cu: RoPE in-place + precomputed cos/sin cache (F32 + BF16) Rust wrappers (xserv-kernels/src/): - rmsnorm.rs, layernorm.rs, activation.rs, softmax.rs, embedding.rs, rope.rs - RopeCache struct with GPU-side precomputation Tests: 12 new tests (ops_test.rs), all passing with good precision: - F32: max_err 1e-6 ~ 1e-9 - BF16: max_err 2e-3 ~ 7e-3 Total: 29 kernel tests + 27 prior = 56 tests passing Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
67 lines
2.2 KiB
Plaintext
67 lines
2.2 KiB
Plaintext
#include <cuda_bf16.h>
|
|
#include <math.h>
|
|
|
|
// GELU (tanh approximation):
|
|
// gelu(x) = 0.5 * x * (1 + tanh(sqrt(2/pi) * (x + 0.044715 * x^3)))
|
|
__device__ __forceinline__ float gelu_f(float x) {
|
|
const float SQRT_2_OVER_PI = 0.7978845608f;
|
|
float cube = x * x * x;
|
|
float inner = SQRT_2_OVER_PI * (x + 0.044715f * cube);
|
|
return 0.5f * x * (1.0f + tanhf(inner));
|
|
}
|
|
|
|
// SiLU (Swish): silu(x) = x * sigmoid(x) = x / (1 + exp(-x))
|
|
__device__ __forceinline__ float silu_f(float x) {
|
|
return x / (1.0f + expf(-x));
|
|
}
|
|
|
|
__global__ void gelu_f32(const float* x, float* out, int n) {
|
|
int idx = blockIdx.x * blockDim.x + threadIdx.x;
|
|
if (idx < n) out[idx] = gelu_f(x[idx]);
|
|
}
|
|
|
|
__global__ void gelu_bf16(const __nv_bfloat16* x, __nv_bfloat16* out, int n) {
|
|
int idx = blockIdx.x * blockDim.x + threadIdx.x;
|
|
if (idx < n) out[idx] = __float2bfloat16(gelu_f(__bfloat162float(x[idx])));
|
|
}
|
|
|
|
__global__ void silu_f32(const float* x, float* out, int n) {
|
|
int idx = blockIdx.x * blockDim.x + threadIdx.x;
|
|
if (idx < n) out[idx] = silu_f(x[idx]);
|
|
}
|
|
|
|
__global__ void silu_bf16(const __nv_bfloat16* x, __nv_bfloat16* out, int n) {
|
|
int idx = blockIdx.x * blockDim.x + threadIdx.x;
|
|
if (idx < n) out[idx] = __float2bfloat16(silu_f(__bfloat162float(x[idx])));
|
|
}
|
|
|
|
extern "C" {
|
|
|
|
void launch_gelu_f32(const void* x, void* out, int n, void* stream) {
|
|
int block = 256;
|
|
int grid = (n + block - 1) / block;
|
|
gelu_f32<<<grid, block, 0, (cudaStream_t)stream>>>((const float*)x, (float*)out, n);
|
|
}
|
|
|
|
void launch_gelu_bf16(const void* x, void* out, int n, void* stream) {
|
|
int block = 256;
|
|
int grid = (n + block - 1) / block;
|
|
gelu_bf16<<<grid, block, 0, (cudaStream_t)stream>>>(
|
|
(const __nv_bfloat16*)x, (__nv_bfloat16*)out, n);
|
|
}
|
|
|
|
void launch_silu_f32(const void* x, void* out, int n, void* stream) {
|
|
int block = 256;
|
|
int grid = (n + block - 1) / block;
|
|
silu_f32<<<grid, block, 0, (cudaStream_t)stream>>>((const float*)x, (float*)out, n);
|
|
}
|
|
|
|
void launch_silu_bf16(const void* x, void* out, int n, void* stream) {
|
|
int block = 256;
|
|
int grid = (n + block - 1) / block;
|
|
silu_bf16<<<grid, block, 0, (cudaStream_t)stream>>>(
|
|
(const __nv_bfloat16*)x, (__nv_bfloat16*)out, n);
|
|
}
|
|
|
|
}
|