style: format Rust workspace
This commit is contained in:
@@ -3,10 +3,13 @@ mod engine;
|
||||
mod pp_engine;
|
||||
mod tp_engine;
|
||||
|
||||
use axum::{routing::{get, post}, Extension, Router};
|
||||
use std::path::PathBuf;
|
||||
use std::sync::{mpsc, Arc, Mutex};
|
||||
use axum::{
|
||||
Extension, Router,
|
||||
routing::{get, post},
|
||||
};
|
||||
use engine::GenerateRequest;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::{Arc, Mutex, mpsc};
|
||||
use xserv_model::ModelConfig;
|
||||
|
||||
pub struct AppState {
|
||||
@@ -21,40 +24,48 @@ pub struct AppState {
|
||||
async fn main() {
|
||||
let args: Vec<String> = std::env::args().collect();
|
||||
if args.len() < 2 {
|
||||
eprintln!("Usage: xserv-server <model-dir> [--port PORT] [--max-batch N] [--max-seq-len N] [--swap-space-gb N] [--tp N] [--pp N]");
|
||||
eprintln!(
|
||||
"Usage: xserv-server <model-dir> [--port PORT] [--max-batch N] [--max-seq-len N] [--swap-space-gb N] [--tp N] [--pp N]"
|
||||
);
|
||||
std::process::exit(1);
|
||||
}
|
||||
|
||||
let model_dir = PathBuf::from(&args[1]);
|
||||
let port: u16 = args.iter()
|
||||
let port: u16 = args
|
||||
.iter()
|
||||
.position(|a| a == "--port")
|
||||
.and_then(|i| args.get(i + 1))
|
||||
.and_then(|s| s.parse().ok())
|
||||
.unwrap_or(8080);
|
||||
let max_batch: usize = args.iter()
|
||||
let max_batch: usize = args
|
||||
.iter()
|
||||
.position(|a| a == "--max-batch")
|
||||
.and_then(|i| args.get(i + 1))
|
||||
.and_then(|s| s.parse().ok())
|
||||
.unwrap_or(4)
|
||||
.max(1);
|
||||
let requested_max_seq_len: usize = args.iter()
|
||||
let requested_max_seq_len: usize = args
|
||||
.iter()
|
||||
.position(|a| a == "--max-seq-len")
|
||||
.and_then(|i| args.get(i + 1))
|
||||
.and_then(|s| s.parse().ok())
|
||||
.unwrap_or(2048)
|
||||
.max(1);
|
||||
let swap_space_gb: usize = args.iter()
|
||||
let swap_space_gb: usize = args
|
||||
.iter()
|
||||
.position(|a| a == "--swap-space-gb")
|
||||
.and_then(|i| args.get(i + 1))
|
||||
.and_then(|s| s.parse().ok())
|
||||
.unwrap_or(8);
|
||||
let tp: usize = args.iter()
|
||||
let tp: usize = args
|
||||
.iter()
|
||||
.position(|a| a == "--tp")
|
||||
.and_then(|i| args.get(i + 1))
|
||||
.and_then(|s| s.parse().ok())
|
||||
.unwrap_or(1)
|
||||
.max(1);
|
||||
let pp: usize = args.iter()
|
||||
let pp: usize = args
|
||||
.iter()
|
||||
.position(|a| a == "--pp")
|
||||
.and_then(|i| args.get(i + 1))
|
||||
.and_then(|s| s.parse().ok())
|
||||
@@ -69,7 +80,9 @@ async fn main() {
|
||||
// tp=1 (single-rank world) so quantized models can serve on one GPU.
|
||||
let is_gpt_oss = model_config.model_type.as_deref() == Some("gpt_oss");
|
||||
if pp > 1 && is_gpt_oss {
|
||||
eprintln!("gpt-oss is not supported by the pipeline-parallel engine (Qwen3 only); use --tp instead");
|
||||
eprintln!(
|
||||
"gpt-oss is not supported by the pipeline-parallel engine (Qwen3 only); use --tp instead"
|
||||
);
|
||||
std::process::exit(1);
|
||||
}
|
||||
let model_max_seq_len = model_config.max_seq_len();
|
||||
@@ -84,7 +97,8 @@ async fn main() {
|
||||
);
|
||||
}
|
||||
|
||||
let model_name = model_dir.file_name()
|
||||
let model_name = model_dir
|
||||
.file_name()
|
||||
.map(|n| n.to_string_lossy().to_string())
|
||||
.unwrap_or_else(|| "unknown".to_string());
|
||||
|
||||
@@ -99,7 +113,12 @@ async fn main() {
|
||||
// Pipeline-parallel path: stage-0 coordinator + worker stage threads.
|
||||
pp_engine::run_pp(&model_dir_clone, pp, max_seq_len, rx);
|
||||
} else if tp <= 1 && !is_gpt_oss {
|
||||
let mut engine = engine::Engine::load_with_swap(&model_dir_clone, max_batch, max_seq_len, swap_space_gb);
|
||||
let mut engine = engine::Engine::load_with_swap(
|
||||
&model_dir_clone,
|
||||
max_batch,
|
||||
max_seq_len,
|
||||
swap_space_gb,
|
||||
);
|
||||
engine.run(rx);
|
||||
} else {
|
||||
// Tensor-parallel path: rank-0 coordinator + worker rank threads.
|
||||
|
||||
Reference in New Issue
Block a user