Gahow Wang
be5c64ea8a
Kernel additions: - add_f32/bf16, mul_f32/bf16 CUDA kernels (element-wise, on GPU) - Refactored activation.rs with dispatch_unary/dispatch_binary helpers - Qwen3 and GPT-2 now use GPU add/mul instead of CPU round-trips GPT-2 add_bias also moved to GPU (broadcast via tile + GPU add) BF16 precision analysis (docs/benchmarks/phase10-qwen3.md): - Root cause: separate attention kernels materialize BF16 intermediates (QK^T→BF16→scale→BF16→mask→BF16→softmax→BF16 vs HF's fused FP32 path) - HF itself SDPA vs Eager also differs by ~0.125 logit - xserv vs HF: ~1-2 logit systematic offset, but same top-1 in 84% cases - Industry standard for BF16: top-5 overlap (we achieve 100%) - Fix path: Flash Attention (Phase 14) to fuse attention in FP32 Performance: TTFT 138→119ms, TBT 144→137ms (GPU ops faster than CPU) Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
45 lines
1.7 KiB
Rust
45 lines
1.7 KiB
Rust
use std::path::PathBuf;
|
|
use xserv_model::{loader, KVCache, ModelConfig, Qwen3};
|
|
use xserv_tensor::{DType, Device};
|
|
use xserv_tokenizer::Tokenizer;
|
|
use half::bf16;
|
|
|
|
fn main() {
|
|
let args: Vec<String> = std::env::args().collect();
|
|
let model_dir = PathBuf::from(&args[1]);
|
|
let prompt = &args[2];
|
|
|
|
xserv_cuda::device::set_device(0).unwrap();
|
|
let config = ModelConfig::from_file(&model_dir.join("config.json"));
|
|
let weights = loader::load_model_dir(&model_dir, Device::Cuda(0));
|
|
let model = Qwen3::from_weights(config.clone(), weights);
|
|
let tokenizer = Tokenizer::from_file(&model_dir.join("tokenizer.json"));
|
|
|
|
let token_ids = tokenizer.encode(prompt);
|
|
eprintln!("Prompt: {prompt}");
|
|
eprintln!("Token IDs: {token_ids:?}");
|
|
|
|
let mut cache = KVCache::new(
|
|
config.num_layers(), config.num_kv_heads(), config.head_dim(),
|
|
DType::BF16, Device::Cuda(0),
|
|
);
|
|
let logits = model.forward_with_cache(&token_ids, &mut cache);
|
|
let logits_cpu = logits.to_device(Device::Cpu);
|
|
let data = logits_cpu.as_slice::<bf16>();
|
|
let vocab_size = logits.shape()[1];
|
|
let seq_len = logits.shape()[0];
|
|
|
|
// Print top-20 logits for the last position
|
|
let last_row = &data[(seq_len - 1) * vocab_size..seq_len * vocab_size];
|
|
let mut indexed: Vec<(usize, f32)> = last_row.iter().enumerate()
|
|
.map(|(i, v)| (i, v.to_f32()))
|
|
.collect();
|
|
indexed.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap());
|
|
|
|
println!("Top-20 logits (last position):");
|
|
for (rank, (id, val)) in indexed.iter().take(20).enumerate() {
|
|
let tok = tokenizer.decode(&[*id as u32]);
|
|
println!(" [{rank:>2}] id={id:>6} logit={val:>10.4} token={tok:?}");
|
|
}
|
|
}
|