diff --git a/crates/xserv-model/src/sampling.rs b/crates/xserv-model/src/sampling.rs index 762b62e..efa2c4c 100644 --- a/crates/xserv-model/src/sampling.rs +++ b/crates/xserv-model/src/sampling.rs @@ -19,6 +19,18 @@ impl Default for SamplingParams { /// Uses the last position's logits. Handles both F32 and BF16 dtypes. pub fn sample(logits: &Tensor, params: &SamplingParams) -> u32 { assert_eq!(logits.ndim(), 2); + // Greedy fast path: GPU argmax + 4-byte D2H instead of copying the whole + // [seq, vocab] logits to the host and scanning it (~201k bf16/token). + // NaN logits lose every `>` comparison in the kernel, matching the + // NaN-safe host argmax below. + if params.temperature == 0.0 + && logits.dtype() == DType::BF16 + && matches!(logits.device(), Device::Cuda(_)) + && logits.is_contiguous() + { + let ids = xserv_kernels::argmax_bf16_to_host(logits); + return *ids.last().unwrap(); + } let vocab_size = logits.shape()[1]; let seq_len = logits.shape()[0]; let logits_cpu = logits.to_device(Device::Cpu);