server: support GptOss in TP engine + benchmark script

- tp_engine.rs: TpModel enum dispatches between Qwen3 and GptOss based on
  config.is_moe(). Server auto-detects model type on startup.
- tools/run_gpt_oss_bench.sh: one-click benchmark comparing xserv (TP=2)
  vs llama.cpp (BF16 GGUF) on GSM8K quality + speed

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Gahow Wang
2026-05-30 15:39:44 +08:00
parent d29c39d74e
commit 15c51f143e
3 changed files with 198 additions and 5 deletions

View File

@@ -19,8 +19,8 @@ use std::thread;
use xserv_distributed::{TpContext, UniqueId};
use xserv_model::loader;
use xserv_model::{sample, ModelConfig, PagedKVCache, Qwen3, BLOCK_SIZE};
use xserv_tensor::{DType, Device};
use xserv_model::{sample, GptOss, ModelConfig, PagedKVCache, Qwen3, BLOCK_SIZE};
use xserv_tensor::{DType, Device, Tensor};
use xserv_tokenizer::Tokenizer;
use crate::engine::{GenerateEvent, GenerateRequest};
@@ -34,8 +34,29 @@ enum TpCommand {
Shutdown,
}
enum TpModel {
Qwen3(Qwen3),
GptOss(GptOss),
}
impl TpModel {
fn forward_prefill_paged(&self, tokens: &[u32], slot: usize, cache: &mut PagedKVCache) -> Tensor {
match self {
TpModel::Qwen3(m) => m.forward_prefill_paged(tokens, slot, cache),
TpModel::GptOss(m) => m.forward_prefill_paged(tokens, slot, cache),
}
}
fn forward_decode_paged(&self, tokens: &[u32], positions: &[usize], slots: &[usize], cache: &mut PagedKVCache) -> Tensor {
match self {
TpModel::Qwen3(m) => m.forward_decode_paged(tokens, positions, slots, cache),
TpModel::GptOss(m) => m.forward_decode_paged(tokens, positions, slots, cache),
}
}
}
struct RankCtx {
model: Qwen3,
model: TpModel,
cache: PagedKVCache,
}
@@ -49,9 +70,13 @@ fn build_rank(
tp: Option<Arc<TpContext>>,
) -> RankCtx {
let weights = loader::load_model_dir(model_dir, Device::Cpu);
let model = Qwen3::from_weights_tp(config.clone(), weights, rank, world, device, tp);
let model = if config.is_moe() {
TpModel::GptOss(GptOss::from_weights_tp(config.clone(), weights, rank, world, device, tp))
} else {
TpModel::Qwen3(Qwen3::from_weights_tp(config.clone(), weights, rank, world, device, tp))
};
let local_kv = config.num_kv_heads() / world;
let max_blocks_per_seq = max_seq_len.div_ceil(BLOCK_SIZE);
let max_blocks_per_seq = (max_seq_len + BLOCK_SIZE - 1) / BLOCK_SIZE;
let total_blocks = max_blocks_per_seq + 8;
let cache = PagedKVCache::new_tp(
config, local_kv, total_blocks, 0, 4, max_blocks_per_seq, DType::BF16, device,