use std::ffi::c_void; use xserv_tensor::{DType, Device, Tensor}; unsafe extern "C" { fn launch_embedding_f32(table: *const c_void, token_ids: *const c_void, out: *mut c_void, num_tokens: i32, hidden_size: i32, vocab_size: i32, stream: *mut c_void); fn launch_embedding_bf16(table: *const c_void, token_ids: *const c_void, out: *mut c_void, num_tokens: i32, hidden_size: i32, vocab_size: i32, stream: *mut c_void); } /// Embedding lookup: table[token_ids[i]] for each i. /// table: [vocab_size, hidden_size], token_ids: [num_tokens] (i32 on CPU) pub fn embedding(table: &Tensor, token_ids: &[u32]) -> Tensor { assert_eq!(table.ndim(), 2); assert!(table.is_contiguous()); assert!(matches!(table.device(), Device::Cuda(_))); let hidden_size = table.shape()[1]; let num_tokens = token_ids.len(); let vocab_size = table.shape()[0]; assert!(num_tokens <= i32::MAX as usize, "too many tokens for i32 kernel param"); assert!(hidden_size <= i32::MAX as usize, "hidden_size too large for i32 kernel param"); // Upload token_ids to GPU let ids_bytes = unsafe { std::slice::from_raw_parts( token_ids.as_ptr() as *const u8, num_tokens * std::mem::size_of::(), ) }; let mut ids_gpu = xserv_cuda::allocator::cached_alloc(ids_bytes.len()).expect("alloc token_ids"); ids_gpu.copy_from_host(ids_bytes).unwrap(); for &tid in token_ids { assert!((tid as usize) < vocab_size, "token_id {tid} out of bounds (vocab_size={vocab_size})"); } embedding_device_ids(table, ids_gpu.as_ptr() as *const c_void, num_tokens) } /// Embedding lookup with token ids already on the GPU (u32, [num_tokens]). /// Used by the CUDA-graph decode path, where ids live in a persistent device /// buffer updated outside the captured region (no bounds check possible here). pub fn embedding_device_ids(table: &Tensor, ids_gpu: *const c_void, num_tokens: usize) -> Tensor { assert_eq!(table.ndim(), 2); assert!(table.is_contiguous()); assert!(matches!(table.device(), Device::Cuda(_))); let hidden_size = table.shape()[1]; let vocab_size = table.shape()[0]; let out = Tensor::empty(&[num_tokens, hidden_size], table.dtype(), table.device()); unsafe { match table.dtype() { DType::F32 => launch_embedding_f32( table.data_ptr() as _, ids_gpu, out.data_ptr() as *mut c_void, num_tokens as i32, hidden_size as i32, vocab_size as i32, xserv_cuda::current_stream_raw(), ), DType::BF16 => launch_embedding_bf16( table.data_ptr() as _, ids_gpu, out.data_ptr() as *mut c_void, num_tokens as i32, hidden_size as i32, vocab_size as i32, xserv_cuda::current_stream_raw(), ), _ => panic!("unsupported dtype for embedding"), } } out }