Gahow Wang
4088f49b7d
- Thread-local launch stream (xserv_cuda::stream): every kernel wrapper, cublasSetStream, and NCCL collective now launches on current_stream_raw() — the legacy null stream by default (behavior unchanged), or the capture stream installed via push_stream during graph capture. Capture is impossible on the legacy stream. - Allocator retain mode: blocks freed inside a retain window are quarantined (RetainedBlocks) instead of pooled, so an instantiated graph keeps exclusive ownership of every intermediate buffer it references across replays. - Capture mode GLOBAL -> THREAD_LOCAL: concurrent TP rank threads must not poison each other's captures with their own cudaMallocs. - embedding_device_ids / rope_inplace_device_pos: variants reading token ids / positions from persistent device buffers, replacing the per-call host upload that a captured region cannot contain. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
70 lines
2.9 KiB
Rust
70 lines
2.9 KiB
Rust
use std::ffi::c_void;
|
|
use xserv_tensor::{DType, Device, Tensor};
|
|
|
|
unsafe extern "C" {
|
|
fn launch_embedding_f32(table: *const c_void, token_ids: *const c_void, out: *mut c_void,
|
|
num_tokens: i32, hidden_size: i32, vocab_size: i32, stream: *mut c_void);
|
|
fn launch_embedding_bf16(table: *const c_void, token_ids: *const c_void, out: *mut c_void,
|
|
num_tokens: i32, hidden_size: i32, vocab_size: i32, stream: *mut c_void);
|
|
}
|
|
|
|
/// Embedding lookup: table[token_ids[i]] for each i.
|
|
/// table: [vocab_size, hidden_size], token_ids: [num_tokens] (i32 on CPU)
|
|
pub fn embedding(table: &Tensor, token_ids: &[u32]) -> Tensor {
|
|
assert_eq!(table.ndim(), 2);
|
|
assert!(table.is_contiguous());
|
|
assert!(matches!(table.device(), Device::Cuda(_)));
|
|
|
|
let hidden_size = table.shape()[1];
|
|
let num_tokens = token_ids.len();
|
|
let vocab_size = table.shape()[0];
|
|
assert!(num_tokens <= i32::MAX as usize, "too many tokens for i32 kernel param");
|
|
assert!(hidden_size <= i32::MAX as usize, "hidden_size too large for i32 kernel param");
|
|
|
|
// Upload token_ids to GPU
|
|
let ids_bytes = unsafe {
|
|
std::slice::from_raw_parts(
|
|
token_ids.as_ptr() as *const u8,
|
|
num_tokens * std::mem::size_of::<u32>(),
|
|
)
|
|
};
|
|
let mut ids_gpu = xserv_cuda::allocator::cached_alloc(ids_bytes.len()).expect("alloc token_ids");
|
|
ids_gpu.copy_from_host(ids_bytes).unwrap();
|
|
|
|
for &tid in token_ids {
|
|
assert!((tid as usize) < vocab_size, "token_id {tid} out of bounds (vocab_size={vocab_size})");
|
|
}
|
|
|
|
embedding_device_ids(table, ids_gpu.as_ptr() as *const c_void, num_tokens)
|
|
}
|
|
|
|
/// Embedding lookup with token ids already on the GPU (u32, [num_tokens]).
|
|
/// Used by the CUDA-graph decode path, where ids live in a persistent device
|
|
/// buffer updated outside the captured region (no bounds check possible here).
|
|
pub fn embedding_device_ids(table: &Tensor, ids_gpu: *const c_void, num_tokens: usize) -> Tensor {
|
|
assert_eq!(table.ndim(), 2);
|
|
assert!(table.is_contiguous());
|
|
assert!(matches!(table.device(), Device::Cuda(_)));
|
|
let hidden_size = table.shape()[1];
|
|
let vocab_size = table.shape()[0];
|
|
|
|
let out = Tensor::empty(&[num_tokens, hidden_size], table.dtype(), table.device());
|
|
|
|
unsafe {
|
|
match table.dtype() {
|
|
DType::F32 => launch_embedding_f32(
|
|
table.data_ptr() as _, ids_gpu,
|
|
out.data_ptr() as *mut c_void,
|
|
num_tokens as i32, hidden_size as i32, vocab_size as i32, xserv_cuda::current_stream_raw(),
|
|
),
|
|
DType::BF16 => launch_embedding_bf16(
|
|
table.data_ptr() as _, ids_gpu,
|
|
out.data_ptr() as *mut c_void,
|
|
num_tokens as i32, hidden_size as i32, vocab_size as i32, xserv_cuda::current_stream_raw(),
|
|
),
|
|
_ => panic!("unsupported dtype for embedding"),
|
|
}
|
|
}
|
|
out
|
|
}
|