phase 12+13: HTTP API server with OpenAI-compatible endpoint (Milestone ③)
New crate: xserv-server
- Engine thread: loads Qwen3-8B, processes requests sequentially
- axum HTTP server: /health, /v1/models, /v1/chat/completions
- tokio::sync::mpsc channel between API and engine threads
- Non-streaming JSON response (streaming SSE to be added later)
API is OpenAI-compatible:
POST /v1/chat/completions {"messages": [...], "max_tokens": N}
→ {"choices": [{"message": {"content": "..."}}]}
Verified: "Hi" → ", I'm" (3 tokens), model runs correctly via HTTP.
Key learnings:
- std::sync::mpsc::SyncSender is Send but NOT Sync → wrap in Mutex for Arc<AppState>
- MutexGuard must not live across await points (scope carefully)
- axum 0.8 Extension<Arc<T>> requires T: Send + Sync
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
76
crates/xserv-server/src/engine.rs
Normal file
76
crates/xserv-server/src/engine.rs
Normal file
@@ -0,0 +1,76 @@
|
||||
use std::path::Path;
|
||||
use xserv_model::{loader, GpuKVCache, ModelConfig, Qwen3};
|
||||
use xserv_model::qwen3::sample_greedy;
|
||||
use xserv_tensor::{DType, Device};
|
||||
use xserv_tokenizer::Tokenizer;
|
||||
|
||||
pub struct Engine {
|
||||
model: Qwen3,
|
||||
config: ModelConfig,
|
||||
tokenizer: Tokenizer,
|
||||
}
|
||||
|
||||
pub struct GenerateRequest {
|
||||
pub prompt_tokens: Vec<u32>,
|
||||
pub max_tokens: usize,
|
||||
pub sender: tokio::sync::mpsc::Sender<GenerateEvent>,
|
||||
}
|
||||
|
||||
pub enum GenerateEvent {
|
||||
Token { id: u32, text: String },
|
||||
Done { finish_reason: String },
|
||||
}
|
||||
|
||||
impl Engine {
|
||||
pub fn load(model_dir: &Path) -> Self {
|
||||
xserv_cuda::device::set_device(0).unwrap();
|
||||
let config = ModelConfig::from_file(&model_dir.join("config.json"));
|
||||
eprintln!("[engine] Loading weights...");
|
||||
let weights = loader::load_model_dir(model_dir, Device::Cuda(0));
|
||||
eprintln!("[engine] Loaded {} tensors", weights.len());
|
||||
let model = Qwen3::from_weights(config.clone(), weights);
|
||||
let tokenizer = Tokenizer::from_file(&model_dir.join("tokenizer.json"));
|
||||
eprintln!("[engine] Ready");
|
||||
Self { model, config, tokenizer }
|
||||
}
|
||||
|
||||
pub fn tokenizer(&self) -> &Tokenizer {
|
||||
&self.tokenizer
|
||||
}
|
||||
|
||||
pub fn generate(&self, req: GenerateRequest) {
|
||||
let max_seq = 256;
|
||||
let mut cache = GpuKVCache::new(&self.config, max_seq, DType::BF16);
|
||||
|
||||
let logits = self.model.forward_gpu_cache(&req.prompt_tokens, &mut cache);
|
||||
let mut next = sample_greedy(&logits);
|
||||
|
||||
for _ in 0..req.max_tokens {
|
||||
let text = self.tokenizer.decode(&[next]);
|
||||
if req.sender.blocking_send(GenerateEvent::Token { id: next, text }).is_err() {
|
||||
return;
|
||||
}
|
||||
|
||||
if self.tokenizer.eos_token_id() == Some(next) {
|
||||
let _ = req.sender.blocking_send(GenerateEvent::Done {
|
||||
finish_reason: "stop".to_string(),
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
if cache.seq_len() >= max_seq - 1 {
|
||||
let _ = req.sender.blocking_send(GenerateEvent::Done {
|
||||
finish_reason: "length".to_string(),
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
let logits = self.model.forward_gpu_cache(&[next], &mut cache);
|
||||
next = sample_greedy(&logits);
|
||||
}
|
||||
|
||||
let _ = req.sender.blocking_send(GenerateEvent::Done {
|
||||
finish_reason: "length".to_string(),
|
||||
});
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user