use std::ffi::c_void; use xserv_tensor::{DType, Device, Tensor}; unsafe extern "C" { fn launch_rmsnorm_f32(x: *const c_void, gamma: *const c_void, out: *mut c_void, rows: i32, hidden_size: i32, eps: f32, stream: *mut c_void); fn launch_rmsnorm_bf16(x: *const c_void, gamma: *const c_void, out: *mut c_void, rows: i32, hidden_size: i32, eps: f32, stream: *mut c_void); fn launch_add_rmsnorm_bf16(x: *const c_void, residual: *const c_void, gamma: *const c_void, normed_out: *mut c_void, sum_out: *mut c_void, rows: i32, hidden_size: i32, eps: f32, stream: *mut c_void); } pub fn rmsnorm(x: &Tensor, gamma: &Tensor, eps: f32) -> Tensor { assert!(x.ndim() >= 1); assert!(x.is_contiguous() && gamma.is_contiguous()); assert!(matches!(x.device(), Device::Cuda(_))); let hidden_size = *x.shape().last().unwrap(); assert_eq!(gamma.shape(), &[hidden_size]); assert_eq!(x.dtype(), gamma.dtype()); let rows = x.numel() / hidden_size; assert!(rows <= i32::MAX as usize, "too many rows for i32 kernel param"); assert!(hidden_size <= i32::MAX as usize, "hidden_size too large for i32 kernel param"); let out = Tensor::empty(x.shape(), x.dtype(), x.device()); unsafe { match x.dtype() { DType::F32 => launch_rmsnorm_f32( x.data_ptr() as _, gamma.data_ptr() as _, out.data_ptr() as *mut c_void, rows as i32, hidden_size as i32, eps, std::ptr::null_mut(), ), DType::BF16 => launch_rmsnorm_bf16( x.data_ptr() as _, gamma.data_ptr() as _, out.data_ptr() as *mut c_void, rows as i32, hidden_size as i32, eps, std::ptr::null_mut(), ), _ => panic!("unsupported dtype for rmsnorm"), } } out } /// Fused Add + RMSNorm: computes sum = x + residual, then normed = rmsnorm(sum, gamma, eps). /// Returns (normed, sum). BF16 only. /// Saves one kernel launch and one full HBM round-trip per layer. pub fn add_rmsnorm(x: &Tensor, residual: &Tensor, gamma: &Tensor, eps: f32) -> (Tensor, Tensor) { assert!(x.ndim() >= 1); assert_eq!(x.shape(), residual.shape()); assert!(x.is_contiguous() && residual.is_contiguous() && gamma.is_contiguous()); assert!(matches!(x.device(), Device::Cuda(_))); assert_eq!(x.dtype(), DType::BF16, "add_rmsnorm requires BF16"); assert_eq!(residual.dtype(), DType::BF16); assert_eq!(gamma.dtype(), DType::BF16); let hidden_size = *x.shape().last().unwrap(); assert_eq!(gamma.shape(), &[hidden_size]); let rows = x.numel() / hidden_size; assert!(rows <= i32::MAX as usize, "too many rows for i32 kernel param"); assert!(hidden_size <= i32::MAX as usize, "hidden_size too large for i32 kernel param"); let normed_out = Tensor::empty(x.shape(), DType::BF16, x.device()); let sum_out = Tensor::empty(x.shape(), DType::BF16, x.device()); unsafe { launch_add_rmsnorm_bf16( x.data_ptr() as *const c_void, residual.data_ptr() as *const c_void, gamma.data_ptr() as *const c_void, normed_out.data_ptr() as *mut c_void, sum_out.data_ptr() as *mut c_void, rows as i32, hidden_size as i32, eps, std::ptr::null_mut(), ); } (normed_out, sum_out) }