Gahow Wang
4c3f914459
CUDA layer for the paged-KV + swap work: - csrc: new paged_attention.cu plus updates across attention/gemm/norm/ activation/embedding/reduce kernels and common.cuh. - xserv-kernels: new dispatch module and kernel-binding updates. - xserv-cuda: cudaMallocHost/FreeHost bindings + PinnedBuffer (host swap pool backing) and offset-aware D2H/H2D copies used to move KV blocks between the GPU pool and pinned host memory. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
36 lines
1.3 KiB
Rust
36 lines
1.3 KiB
Rust
use std::ffi::c_void;
|
|
use xserv_tensor::{DType, Device, Tensor};
|
|
|
|
unsafe extern "C" {
|
|
fn launch_softmax_f32(x: *const c_void, out: *mut c_void, rows: i32, cols: i32, stream: *mut c_void);
|
|
fn launch_softmax_bf16(x: *const c_void, out: *mut c_void, rows: i32, cols: i32, stream: *mut c_void);
|
|
}
|
|
|
|
/// Softmax along the last dimension.
|
|
pub fn softmax(x: &Tensor) -> Tensor {
|
|
assert!(x.ndim() >= 1);
|
|
assert!(x.is_contiguous());
|
|
assert!(matches!(x.device(), Device::Cuda(_)));
|
|
|
|
let cols = *x.shape().last().unwrap();
|
|
let rows = x.numel() / cols;
|
|
assert!(rows <= i32::MAX as usize, "too many rows for i32 kernel param");
|
|
assert!(cols <= i32::MAX as usize, "cols too large for i32 kernel param");
|
|
let out = Tensor::empty(x.shape(), x.dtype(), x.device());
|
|
|
|
unsafe {
|
|
match x.dtype() {
|
|
DType::F32 => launch_softmax_f32(
|
|
x.data_ptr() as _, out.data_ptr() as *mut c_void,
|
|
rows as i32, cols as i32, std::ptr::null_mut(),
|
|
),
|
|
DType::BF16 => launch_softmax_bf16(
|
|
x.data_ptr() as _, out.data_ptr() as *mut c_void,
|
|
rows as i32, cols as i32, std::ptr::null_mut(),
|
|
),
|
|
_ => panic!("unsupported dtype for softmax"),
|
|
}
|
|
}
|
|
out
|
|
}
|