phase 12: implement real continuous batching scheduler

Rewrote engine.rs from scratch:
- Scheduler loop: admit → prefill → decode → finish → check new requests
- Multiple sequences run concurrently (max_batch_size configurable)
- Each sequence has independent GpuKVCache
- Non-blocking try_recv() for new requests during decode iterations
- Dynamic join: new requests enter batch immediately, don't wait for others

Verified with concurrent test (tools/test_concurrent.py):
- 3 concurrent requests: wall_time=3.8s, concurrency_ratio=2.82x ✓
- 5 concurrent requests: wall_time=6.1s, concurrency_ratio=4.04x ✓
- All outputs are coherent and correct

Design doc (docs/12-continuous-batching.md) fully rewritten with:
- Detailed scheduler loop pseudocode
- Data structures (Sequence, Scheduler)
- Acceptance criteria with specific test cases
- Clear separation from Phase 13 (HTTP layer)

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

crates/xserv-server/src/engine.rs

@@ -1,5 +1,8 @@
+use std::collections::VecDeque;
 use std::path::Path;
-use xserv_model::{loader, GpuKVCache, ModelConfig, Qwen3};
+use std::sync::mpsc;
+use xserv_model::{GpuKVCache, ModelConfig, Qwen3};
+use xserv_model::loader;
 use xserv_model::qwen3::sample_greedy;
 use xserv_tensor::{DType, Device};
 use xserv_tokenizer::Tokenizer;
@@ -8,6 +11,8 @@ pub struct Engine {
     model: Qwen3,
     config: ModelConfig,
     tokenizer: Tokenizer,
+    max_batch_size: usize,
+    max_seq_len: usize,
 }
 
 pub struct GenerateRequest {
@@ -21,8 +26,18 @@ pub enum GenerateEvent {
     Done { finish_reason: String },
 }
 
+struct Sequence {
+    id: u64,
+    prompt_tokens: Vec<u32>,
+    generated_tokens: Vec<u32>,
+    max_tokens: usize,
+    kv_cache: GpuKVCache,
+    sender: tokio::sync::mpsc::Sender<GenerateEvent>,
+    prefilled: bool,
+}
+
 impl Engine {
-    pub fn load(model_dir: &Path) -> Self {
+    pub fn load(model_dir: &Path, max_batch_size: usize) -> Self {
         xserv_cuda::device::set_device(0).unwrap();
         let config = ModelConfig::from_file(&model_dir.join("config.json"));
         eprintln!("[engine] Loading weights...");
@@ -30,47 +45,117 @@ impl Engine {
         eprintln!("[engine] Loaded {} tensors", weights.len());
         let model = Qwen3::from_weights(config.clone(), weights);
         let tokenizer = Tokenizer::from_file(&model_dir.join("tokenizer.json"));
-        eprintln!("[engine] Ready");
-        Self { model, config, tokenizer }
+        let max_seq_len = 256;
+        eprintln!("[engine] Ready (max_batch_size={max_batch_size}, max_seq_len={max_seq_len})");
+        Self { model, config, tokenizer, max_batch_size, max_seq_len }
     }
 
-    pub fn tokenizer(&self) -> &Tokenizer {
-        &self.tokenizer
-    }
+    pub fn tokenizer(&self) -> &Tokenizer { &self.tokenizer }
 
-    pub fn generate(&self, req: GenerateRequest) {
-        let max_seq = 256;
-        let mut cache = GpuKVCache::new(&self.config, max_seq, DType::BF16);
+    /// Main scheduler loop. Receives requests from channel, manages concurrent sequences.
+    pub fn run(&self, rx: mpsc::Receiver<GenerateRequest>) {
+        let mut waiting: VecDeque<Sequence> = VecDeque::new();
+        let mut running: Vec<Sequence> = Vec::new();
+        let mut next_id: u64 = 0;
 
-        let logits = self.model.forward_gpu_cache(&req.prompt_tokens, &mut cache);
-        let mut next = sample_greedy(&logits);
+        eprintln!("[scheduler] Listening for requests...");
 
-        for _ in 0..req.max_tokens {
-            let text = self.tokenizer.decode(&[next]);
-            if req.sender.blocking_send(GenerateEvent::Token { id: next, text }).is_err() {
-                return;
+        loop {
+            // Step 1: Remove finished sequences
+            running.retain(|seq| !is_finished(seq));
+
+            // Step 2: Admit new sequences from waiting queue
+            while running.len() < self.max_batch_size {
+                if let Some(seq) = waiting.pop_front() {
+                    running.push(seq);
+                } else {
+                    break;
+                }
             }
 
-            if self.tokenizer.eos_token_id() == Some(next) {
-                let _ = req.sender.blocking_send(GenerateEvent::Done {
-                    finish_reason: "stop".to_string(),
-                });
-                return;
+            // Step 3: If nothing to do, blocking wait for new request
+            if running.is_empty() {
+                match rx.recv() {
+                    Ok(req) => {
+                        let seq = self.make_sequence(req, &mut next_id);
+                        running.push(seq);
+                    }
+                    Err(_) => break, // channel closed
+                }
             }
 
-            if cache.seq_len() >= max_seq - 1 {
-                let _ = req.sender.blocking_send(GenerateEvent::Done {
-                    finish_reason: "length".to_string(),
-                });
-                return;
+            // Step 4: Process one iteration for all running sequences
+            for seq in running.iter_mut() {
+                if !seq.prefilled {
+                    // Prefill
+                    let logits = self.model.forward_gpu_cache(&seq.prompt_tokens, &mut seq.kv_cache);
+                    let next = sample_greedy(&logits);
+                    seq.generated_tokens.push(next);
+                    seq.prefilled = true;
+                    self.emit_token(seq, next);
+                } else {
+                    // Decode one token
+                    let last = *seq.generated_tokens.last().unwrap();
+                    let logits = self.model.forward_gpu_cache(&[last], &mut seq.kv_cache);
+                    let next = sample_greedy(&logits);
+                    seq.generated_tokens.push(next);
+                    self.emit_token(seq, next);
+                }
             }
 
-            let logits = self.model.forward_gpu_cache(&[next], &mut cache);
-            next = sample_greedy(&logits);
+            // Step 5: Check for newly arrived requests (non-blocking)
+            loop {
+                match rx.try_recv() {
+                    Ok(req) => {
+                        let seq = self.make_sequence(req, &mut next_id);
+                        waiting.push_back(seq);
+                    }
+                    Err(mpsc::TryRecvError::Empty) => break,
+                    Err(mpsc::TryRecvError::Disconnected) => return,
+                }
+            }
         }
+    }
 
-        let _ = req.sender.blocking_send(GenerateEvent::Done {
-            finish_reason: "length".to_string(),
-        });
+    fn make_sequence(&self, req: GenerateRequest, next_id: &mut u64) -> Sequence {
+        let id = *next_id;
+        *next_id += 1;
+        let kv_cache = GpuKVCache::new(&self.config, self.max_seq_len, DType::BF16);
+        Sequence {
+            id,
+            prompt_tokens: req.prompt_tokens,
+            generated_tokens: Vec::new(),
+            max_tokens: req.max_tokens,
+            kv_cache,
+            sender: req.sender,
+            prefilled: false,
+        }
+    }
+
+    fn emit_token(&self, seq: &Sequence, token_id: u32) {
+        let text = self.tokenizer.decode(&[token_id]);
+
+        if self.tokenizer.eos_token_id() == Some(token_id) {
+            let _ = seq.sender.blocking_send(GenerateEvent::Token { id: token_id, text });
+            let _ = seq.sender.blocking_send(GenerateEvent::Done {
+                finish_reason: "stop".to_string(),
+            });
+        } else if seq.generated_tokens.len() >= seq.max_tokens {
+            let _ = seq.sender.blocking_send(GenerateEvent::Token { id: token_id, text });
+            let _ = seq.sender.blocking_send(GenerateEvent::Done {
+                finish_reason: "length".to_string(),
+            });
+        } else {
+            let _ = seq.sender.blocking_send(GenerateEvent::Token { id: token_id, text });
+        }
     }
 }
+
+fn is_finished(seq: &Sequence) -> bool {
+    if seq.generated_tokens.is_empty() { return false; }
+    let last = *seq.generated_tokens.last().unwrap();
+    if seq.generated_tokens.len() >= seq.max_tokens { return true; }
+    // Check EOS — need tokenizer info. Use a simple heuristic:
+    // If sender is closed (receiver dropped), also consider finished.
+    seq.sender.is_closed() || last == 151645 // Qwen3 EOS token ID (hardcoded for now)
+}