server: support GptOss in TP engine + benchmark script
- tp_engine.rs: TpModel enum dispatches between Qwen3 and GptOss based on config.is_moe(). Server auto-detects model type on startup. - tools/run_gpt_oss_bench.sh: one-click benchmark comparing xserv (TP=2) vs llama.cpp (BF16 GGUF) on GSM8K quality + speed Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -19,8 +19,8 @@ use std::thread;
|
||||
|
||||
use xserv_distributed::{TpContext, UniqueId};
|
||||
use xserv_model::loader;
|
||||
use xserv_model::{sample, ModelConfig, PagedKVCache, Qwen3, BLOCK_SIZE};
|
||||
use xserv_tensor::{DType, Device};
|
||||
use xserv_model::{sample, GptOss, ModelConfig, PagedKVCache, Qwen3, BLOCK_SIZE};
|
||||
use xserv_tensor::{DType, Device, Tensor};
|
||||
use xserv_tokenizer::Tokenizer;
|
||||
|
||||
use crate::engine::{GenerateEvent, GenerateRequest};
|
||||
@@ -34,8 +34,29 @@ enum TpCommand {
|
||||
Shutdown,
|
||||
}
|
||||
|
||||
enum TpModel {
|
||||
Qwen3(Qwen3),
|
||||
GptOss(GptOss),
|
||||
}
|
||||
|
||||
impl TpModel {
|
||||
fn forward_prefill_paged(&self, tokens: &[u32], slot: usize, cache: &mut PagedKVCache) -> Tensor {
|
||||
match self {
|
||||
TpModel::Qwen3(m) => m.forward_prefill_paged(tokens, slot, cache),
|
||||
TpModel::GptOss(m) => m.forward_prefill_paged(tokens, slot, cache),
|
||||
}
|
||||
}
|
||||
|
||||
fn forward_decode_paged(&self, tokens: &[u32], positions: &[usize], slots: &[usize], cache: &mut PagedKVCache) -> Tensor {
|
||||
match self {
|
||||
TpModel::Qwen3(m) => m.forward_decode_paged(tokens, positions, slots, cache),
|
||||
TpModel::GptOss(m) => m.forward_decode_paged(tokens, positions, slots, cache),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct RankCtx {
|
||||
model: Qwen3,
|
||||
model: TpModel,
|
||||
cache: PagedKVCache,
|
||||
}
|
||||
|
||||
@@ -49,9 +70,13 @@ fn build_rank(
|
||||
tp: Option<Arc<TpContext>>,
|
||||
) -> RankCtx {
|
||||
let weights = loader::load_model_dir(model_dir, Device::Cpu);
|
||||
let model = Qwen3::from_weights_tp(config.clone(), weights, rank, world, device, tp);
|
||||
let model = if config.is_moe() {
|
||||
TpModel::GptOss(GptOss::from_weights_tp(config.clone(), weights, rank, world, device, tp))
|
||||
} else {
|
||||
TpModel::Qwen3(Qwen3::from_weights_tp(config.clone(), weights, rank, world, device, tp))
|
||||
};
|
||||
let local_kv = config.num_kv_heads() / world;
|
||||
let max_blocks_per_seq = max_seq_len.div_ceil(BLOCK_SIZE);
|
||||
let max_blocks_per_seq = (max_seq_len + BLOCK_SIZE - 1) / BLOCK_SIZE;
|
||||
let total_blocks = max_blocks_per_seq + 8;
|
||||
let cache = PagedKVCache::new_tp(
|
||||
config, local_kv, total_blocks, 0, 4, max_blocks_per_seq, DType::BF16, device,
|
||||
|
||||
Reference in New Issue
Block a user