Gahow Wang
c8e8153702
CUDA kernels (csrc/): - common.cuh: shared warp_reduce_sum/max, block_reduce_sum/max - normalization/rmsnorm.cu: RMSNorm (F32 + BF16) - normalization/layernorm.cu: LayerNorm with Welford (F32 + BF16) - activation/activations.cu: GELU tanh-approx + SiLU (F32 + BF16) - reduce/softmax.cu: safe softmax, 3-pass (F32 + BF16) - embedding/embedding.cu: gather lookup (F32 + BF16) - embedding/rope.cu: RoPE in-place + precomputed cos/sin cache (F32 + BF16) Rust wrappers (xserv-kernels/src/): - rmsnorm.rs, layernorm.rs, activation.rs, softmax.rs, embedding.rs, rope.rs - RopeCache struct with GPU-side precomputation Tests: 12 new tests (ops_test.rs), all passing with good precision: - F32: max_err 1e-6 ~ 1e-9 - BF16: max_err 2e-3 ~ 7e-3 Total: 29 kernel tests + 27 prior = 56 tests passing Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
40 lines
1.6 KiB
Rust
40 lines
1.6 KiB
Rust
use std::ffi::c_void;
|
|
use xserv_tensor::{DType, Device, Tensor};
|
|
|
|
unsafe extern "C" {
|
|
fn launch_layernorm_f32(x: *const c_void, gamma: *const c_void, beta: *const c_void,
|
|
out: *mut c_void, rows: i32, hidden_size: i32, eps: f32, stream: *mut c_void);
|
|
fn launch_layernorm_bf16(x: *const c_void, gamma: *const c_void, beta: *const c_void,
|
|
out: *mut c_void, rows: i32, hidden_size: i32, eps: f32, stream: *mut c_void);
|
|
}
|
|
|
|
pub fn layernorm(x: &Tensor, gamma: &Tensor, beta: &Tensor, eps: f32) -> Tensor {
|
|
assert!(x.ndim() >= 1);
|
|
assert!(x.is_contiguous() && gamma.is_contiguous() && beta.is_contiguous());
|
|
assert!(matches!(x.device(), Device::Cuda(_)));
|
|
let hidden_size = *x.shape().last().unwrap();
|
|
assert_eq!(gamma.shape(), &[hidden_size]);
|
|
assert_eq!(beta.shape(), &[hidden_size]);
|
|
|
|
let rows = x.numel() / hidden_size;
|
|
let out = Tensor::zeros(x.shape(), x.dtype(), x.device());
|
|
|
|
unsafe {
|
|
match x.dtype() {
|
|
DType::F32 => launch_layernorm_f32(
|
|
x.data_ptr() as _, gamma.data_ptr() as _, beta.data_ptr() as _,
|
|
out.data_ptr() as *mut c_void,
|
|
rows as i32, hidden_size as i32, eps, std::ptr::null_mut(),
|
|
),
|
|
DType::BF16 => launch_layernorm_bf16(
|
|
x.data_ptr() as _, gamma.data_ptr() as _, beta.data_ptr() as _,
|
|
out.data_ptr() as *mut c_void,
|
|
rows as i32, hidden_size as i32, eps, std::ptr::null_mut(),
|
|
),
|
|
_ => panic!("unsupported dtype for layernorm"),
|
|
}
|
|
}
|
|
xserv_cuda::device::synchronize().unwrap();
|
|
out
|
|
}
|