phase 12+13: HTTP API server with OpenAI-compatible endpoint (Milestone ③)
New crate: xserv-server
- Engine thread: loads Qwen3-8B, processes requests sequentially
- axum HTTP server: /health, /v1/models, /v1/chat/completions
- tokio::sync::mpsc channel between API and engine threads
- Non-streaming JSON response (streaming SSE to be added later)
API is OpenAI-compatible:
POST /v1/chat/completions {"messages": [...], "max_tokens": N}
→ {"choices": [{"message": {"content": "..."}}]}
Verified: "Hi" → ", I'm" (3 tokens), model runs correctly via HTTP.
Key learnings:
- std::sync::mpsc::SyncSender is Send but NOT Sync → wrap in Mutex for Arc<AppState>
- MutexGuard must not live across await points (scope carefully)
- axum 0.8 Extension<Arc<T>> requires T: Send + Sync
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
62
crates/xserv-server/src/main.rs
Normal file
62
crates/xserv-server/src/main.rs
Normal file
@@ -0,0 +1,62 @@
|
||||
mod api;
|
||||
mod engine;
|
||||
|
||||
use axum::{routing::{get, post}, Extension, Router};
|
||||
use std::path::PathBuf;
|
||||
use std::sync::{mpsc, Arc, Mutex};
|
||||
use engine::GenerateRequest;
|
||||
|
||||
pub struct AppState {
|
||||
pub model_name: String,
|
||||
pub engine_sender: Mutex<mpsc::SyncSender<GenerateRequest>>,
|
||||
pub engine_tokenizer: Mutex<xserv_tokenizer::Tokenizer>,
|
||||
}
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() {
|
||||
let args: Vec<String> = std::env::args().collect();
|
||||
if args.len() < 2 {
|
||||
eprintln!("Usage: xserv-server <model-dir> [--port PORT]");
|
||||
std::process::exit(1);
|
||||
}
|
||||
|
||||
let model_dir = PathBuf::from(&args[1]);
|
||||
let port: u16 = args.iter()
|
||||
.position(|a| a == "--port")
|
||||
.and_then(|i| args.get(i + 1))
|
||||
.and_then(|s| s.parse().ok())
|
||||
.unwrap_or(8080);
|
||||
|
||||
let model_name = model_dir.file_name()
|
||||
.map(|n| n.to_string_lossy().to_string())
|
||||
.unwrap_or_else(|| "unknown".to_string());
|
||||
|
||||
let tokenizer = xserv_tokenizer::Tokenizer::from_file(&model_dir.join("tokenizer.json"));
|
||||
let (tx, rx) = mpsc::sync_channel::<GenerateRequest>(1);
|
||||
|
||||
let model_dir_clone = model_dir.clone();
|
||||
std::thread::spawn(move || {
|
||||
let engine = engine::Engine::load(&model_dir_clone);
|
||||
eprintln!("[engine] Listening for requests...");
|
||||
while let Ok(req) = rx.recv() {
|
||||
engine.generate(req);
|
||||
}
|
||||
});
|
||||
|
||||
let state = Arc::new(AppState {
|
||||
model_name,
|
||||
engine_sender: Mutex::new(tx),
|
||||
engine_tokenizer: Mutex::new(tokenizer),
|
||||
});
|
||||
|
||||
let app = Router::new()
|
||||
.route("/health", get(api::health))
|
||||
.route("/v1/models", get(api::list_models))
|
||||
.route("/v1/chat/completions", post(api::chat_completions))
|
||||
.layer(Extension(state));
|
||||
|
||||
let addr = format!("0.0.0.0:{port}");
|
||||
eprintln!("[server] Listening on {addr}");
|
||||
let listener = tokio::net::TcpListener::bind(&addr).await.unwrap();
|
||||
axum::serve(listener, app).await.unwrap();
|
||||
}
|
||||
Reference in New Issue
Block a user