server: VRAM-sized KV pool + vLLM-style swap scheduler

Fixes the paged-KV OOM at large --max-seq-len and adds elastic memory:

- Size the GPU block pool to available VRAM (cudaMemGetInfo) instead of the
  worst-case blocks_per_seq * max_batch * 2 reservation, which OOM'd at 8192.
- Scheduler tracks waiting/running/swapped sets: block-aware admission,
  swap-in of resumable sequences when blocks free, and preemption of the
  newest running sequence to host when the pool can't cover a decode step.
- --swap-space-gb (default 8) sizes the pinned host swap pool;
  XSERV_MAX_KV_BLOCKS forces a small pool to exercise swapping.
- api: poison-tolerant lock + clean 503 when the engine thread is gone,
  instead of cascading mutex-poison panics.

Verified on RTX 5090: serves at --max-seq-len 8192 (previously OOM), and a
forced 40-block pool drives 48 lossless swap-out/swap-in cycles under
concurrency with coherent output.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>

crates/xserv-server/src/api.rs

@@ -85,18 +85,20 @@ async fn chat_non_stream(state: Arc<AppState>, req: ChatRequest) -> Response {
     let model_name = state.model_name.clone();
     let created = unix_timestamp();
 
+    if let Some(response) = validate_request(&req, &model_name) {
+        return response;
+    }
+
     let prompt = build_prompt(&req.messages);
     let prompt_tokens = state.engine_tokenizer.lock().unwrap().encode(&prompt);
     let prompt_token_count = prompt_tokens.len();
 
     let max_seq_len = state.max_seq_len;
     if prompt_token_count >= max_seq_len {
-        return (StatusCode::BAD_REQUEST, Json(serde_json::json!({
-            "error": {
-                "message": format!("prompt is {} tokens, exceeds max_seq_len {}", prompt_token_count, max_seq_len),
-                "type": "invalid_request_error"
-            }
-        }))).into_response();
+        return bad_request(format!(
+            "prompt is {} tokens, exceeds max_seq_len {}",
+            prompt_token_count, max_seq_len
+        ));
     }
     let max_tokens = req.max_tokens.min(max_seq_len - prompt_token_count);
 
@@ -107,12 +109,9 @@ async fn chat_non_stream(state: Arc<AppState>, req: ChatRequest) -> Response {
         sampling: sampling_params(&req),
         sender: tx,
     };
-    state
-        .engine_sender
-        .lock()
-        .unwrap()
-        .send(gen_req)
-        .expect("engine channel closed");
+    if let Err(resp) = submit_to_engine(&state, gen_req) {
+        return resp;
+    }
 
     let mut content = String::new();
     let mut completion_token_count: usize = 0;
@@ -156,17 +155,19 @@ fn chat_stream(
     let model_name = state.model_name.clone();
     let created = unix_timestamp();
 
+    if let Some(response) = validate_request(&req, &model_name) {
+        return response;
+    }
+
     let prompt = build_prompt(&req.messages);
     let prompt_tokens = state.engine_tokenizer.lock().unwrap().encode(&prompt);
 
     let max_seq_len = state.max_seq_len;
     if prompt_tokens.len() >= max_seq_len {
-        return (StatusCode::BAD_REQUEST, Json(serde_json::json!({
-            "error": {
-                "message": format!("prompt is {} tokens, exceeds max_seq_len {}", prompt_tokens.len(), max_seq_len),
-                "type": "invalid_request_error"
-            }
-        }))).into_response();
+        return bad_request(format!(
+            "prompt is {} tokens, exceeds max_seq_len {}",
+            prompt_tokens.len(), max_seq_len
+        ));
     }
     let max_tokens = req.max_tokens.min(max_seq_len - prompt_tokens.len());
 
@@ -177,12 +178,9 @@ fn chat_stream(
         sampling: sampling_params(&req),
         sender: engine_tx,
     };
-    state
-        .engine_sender
-        .lock()
-        .unwrap()
-        .send(gen_req)
-        .expect("engine channel closed");
+    if let Err(resp) = submit_to_engine(&state, gen_req) {
+        return resp;
+    }
 
     // SSE event channel: engine events -> SSE events
     let (sse_tx, sse_rx) = tokio::sync::mpsc::channel::<Result<Event, Infallible>>(64);
@@ -228,6 +226,53 @@ fn chat_stream(
     Sse::new(ReceiverStream::new(sse_rx)).keep_alive(KeepAlive::default()).into_response()
 }
 
+fn validate_request(req: &ChatRequest, model_name: &str) -> Option<Response> {
+    if let Some(model) = &req.model {
+        if model != model_name {
+            return Some(bad_request(format!(
+                "model '{model}' is not loaded; available model is '{model_name}'"
+            )));
+        }
+    }
+
+    if req.max_tokens == 0 {
+        return Some(bad_request("max_tokens must be greater than 0"));
+    }
+
+    None
+}
+
+/// Hand a request to the engine thread. Poison-tolerant (recovers the lock if a
+/// prior handler panicked) and returns a clean 503 instead of panicking when the
+/// engine thread is gone, so one dead engine doesn't cascade into every request.
+fn submit_to_engine(state: &AppState, req: GenerateRequest) -> Result<(), Response> {
+    let sender = state.engine_sender.lock().unwrap_or_else(|e| e.into_inner());
+    sender.send(req).map_err(|_| service_unavailable("inference engine is not available"))
+}
+
+fn service_unavailable(message: impl Into<String>) -> Response {
+    (
+        StatusCode::SERVICE_UNAVAILABLE,
+        Json(serde_json::json!({
+            "error": { "message": message.into(), "type": "server_error" }
+        })),
+    )
+        .into_response()
+}
+
+fn bad_request(message: impl Into<String>) -> Response {
+    (
+        StatusCode::BAD_REQUEST,
+        Json(serde_json::json!({
+            "error": {
+                "message": message.into(),
+                "type": "invalid_request_error"
+            }
+        })),
+    )
+        .into_response()
+}
+
 fn make_chunk(
     id: &str,
     model: &str,
@@ -295,5 +340,6 @@ fn build_prompt(messages: &[Message]) -> String {
         }
     }
     prompt.push_str("<|im_start|>assistant\n");
+    prompt.push_str("<think>\n\n</think>\n\n");
     prompt
 }