124 lines
3.7 KiB
Rust
124 lines
3.7 KiB
Rust
use std::ffi::c_void;
|
|
use xserv_tensor::{DType, Device, Tensor};
|
|
|
|
unsafe extern "C" {
|
|
fn launch_rmsnorm_f32(
|
|
x: *const c_void,
|
|
gamma: *const c_void,
|
|
out: *mut c_void,
|
|
rows: i32,
|
|
hidden_size: i32,
|
|
eps: f32,
|
|
stream: *mut c_void,
|
|
);
|
|
fn launch_rmsnorm_bf16(
|
|
x: *const c_void,
|
|
gamma: *const c_void,
|
|
out: *mut c_void,
|
|
rows: i32,
|
|
hidden_size: i32,
|
|
eps: f32,
|
|
stream: *mut c_void,
|
|
);
|
|
fn launch_add_rmsnorm_bf16(
|
|
x: *const c_void,
|
|
residual: *const c_void,
|
|
gamma: *const c_void,
|
|
normed_out: *mut c_void,
|
|
sum_out: *mut c_void,
|
|
rows: i32,
|
|
hidden_size: i32,
|
|
eps: f32,
|
|
stream: *mut c_void,
|
|
);
|
|
}
|
|
|
|
pub fn rmsnorm(x: &Tensor, gamma: &Tensor, eps: f32) -> Tensor {
|
|
assert!(x.ndim() >= 1);
|
|
assert!(x.is_contiguous() && gamma.is_contiguous());
|
|
assert!(matches!(x.device(), Device::Cuda(_)));
|
|
let hidden_size = *x.shape().last().unwrap();
|
|
assert_eq!(gamma.shape(), &[hidden_size]);
|
|
assert_eq!(x.dtype(), gamma.dtype());
|
|
|
|
let rows = x.numel() / hidden_size;
|
|
assert!(
|
|
rows <= i32::MAX as usize,
|
|
"too many rows for i32 kernel param"
|
|
);
|
|
assert!(
|
|
hidden_size <= i32::MAX as usize,
|
|
"hidden_size too large for i32 kernel param"
|
|
);
|
|
let out = Tensor::empty(x.shape(), x.dtype(), x.device());
|
|
|
|
unsafe {
|
|
match x.dtype() {
|
|
DType::F32 => launch_rmsnorm_f32(
|
|
x.data_ptr() as _,
|
|
gamma.data_ptr() as _,
|
|
out.data_ptr() as *mut c_void,
|
|
rows as i32,
|
|
hidden_size as i32,
|
|
eps,
|
|
xserv_cuda::current_stream_raw(),
|
|
),
|
|
DType::BF16 => launch_rmsnorm_bf16(
|
|
x.data_ptr() as _,
|
|
gamma.data_ptr() as _,
|
|
out.data_ptr() as *mut c_void,
|
|
rows as i32,
|
|
hidden_size as i32,
|
|
eps,
|
|
xserv_cuda::current_stream_raw(),
|
|
),
|
|
_ => panic!("unsupported dtype for rmsnorm"),
|
|
}
|
|
}
|
|
out
|
|
}
|
|
|
|
/// Fused Add + RMSNorm: computes sum = x + residual, then normed = rmsnorm(sum, gamma, eps).
|
|
/// Returns (normed, sum). BF16 only.
|
|
/// Saves one kernel launch and one full HBM round-trip per layer.
|
|
pub fn add_rmsnorm(x: &Tensor, residual: &Tensor, gamma: &Tensor, eps: f32) -> (Tensor, Tensor) {
|
|
assert!(x.ndim() >= 1);
|
|
assert_eq!(x.shape(), residual.shape());
|
|
assert!(x.is_contiguous() && residual.is_contiguous() && gamma.is_contiguous());
|
|
assert!(matches!(x.device(), Device::Cuda(_)));
|
|
assert_eq!(x.dtype(), DType::BF16, "add_rmsnorm requires BF16");
|
|
assert_eq!(residual.dtype(), DType::BF16);
|
|
assert_eq!(gamma.dtype(), DType::BF16);
|
|
|
|
let hidden_size = *x.shape().last().unwrap();
|
|
assert_eq!(gamma.shape(), &[hidden_size]);
|
|
|
|
let rows = x.numel() / hidden_size;
|
|
assert!(
|
|
rows <= i32::MAX as usize,
|
|
"too many rows for i32 kernel param"
|
|
);
|
|
assert!(
|
|
hidden_size <= i32::MAX as usize,
|
|
"hidden_size too large for i32 kernel param"
|
|
);
|
|
let normed_out = Tensor::empty(x.shape(), DType::BF16, x.device());
|
|
let sum_out = Tensor::empty(x.shape(), DType::BF16, x.device());
|
|
|
|
unsafe {
|
|
launch_add_rmsnorm_bf16(
|
|
x.data_ptr() as *const c_void,
|
|
residual.data_ptr() as *const c_void,
|
|
gamma.data_ptr() as *const c_void,
|
|
normed_out.data_ptr() as *mut c_void,
|
|
sum_out.data_ptr() as *mut c_void,
|
|
rows as i32,
|
|
hidden_size as i32,
|
|
eps,
|
|
xserv_cuda::current_stream_raw(),
|
|
);
|
|
}
|
|
|
|
(normed_out, sum_out)
|
|
}
|