chore: vendor sglang v0.5.10 snapshot

third_party/sglang/sgl-model-gateway/src/core/circuit_breaker.rs

@@ -0,0 +1,626 @@
+use std::{
+    sync::atomic::{AtomicU32, AtomicU64, AtomicU8, Ordering},
+    time::{Duration, Instant},
+};
+
+use tracing::info;
+
+use crate::observability::metrics::Metrics;
+
+/// Circuit breaker configuration
+#[derive(Debug, Clone)]
+pub struct CircuitBreakerConfig {
+    /// Number of consecutive failures to open the circuit
+    pub failure_threshold: u32,
+    /// Success threshold to close circuit from half-open
+    pub success_threshold: u32,
+    /// Duration to wait before attempting half-open
+    pub timeout_duration: Duration,
+    /// Time window for failure counting
+    pub window_duration: Duration,
+}
+
+impl Default for CircuitBreakerConfig {
+    fn default() -> Self {
+        Self {
+            failure_threshold: 5,
+            success_threshold: 2,
+            timeout_duration: Duration::from_secs(30),
+            window_duration: Duration::from_secs(60),
+        }
+    }
+}
+
+/// Circuit breaker state constants for atomic storage
+const STATE_CLOSED: u8 = 0;
+const STATE_OPEN: u8 = 1;
+const STATE_HALF_OPEN: u8 = 2;
+
+/// Circuit breaker state
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum CircuitState {
+    /// Normal operation - requests are allowed
+    Closed,
+    /// Circuit is open - requests are rejected
+    Open,
+    /// Testing if service has recovered - limited requests allowed
+    HalfOpen,
+}
+
+impl std::fmt::Display for CircuitState {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            CircuitState::Closed => write!(f, "Closed"),
+            CircuitState::Open => write!(f, "Open"),
+            CircuitState::HalfOpen => write!(f, "HalfOpen"),
+        }
+    }
+}
+
+impl CircuitState {
+    pub fn as_str(&self) -> &'static str {
+        match self {
+            CircuitState::Closed => "closed",
+            CircuitState::Open => "open",
+            CircuitState::HalfOpen => "half_open",
+        }
+    }
+
+    pub fn to_int(&self) -> u8 {
+        match self {
+            CircuitState::Closed => STATE_CLOSED,
+            CircuitState::Open => STATE_OPEN,
+            CircuitState::HalfOpen => STATE_HALF_OPEN,
+        }
+    }
+
+    fn from_int(v: u8) -> Self {
+        match v {
+            STATE_CLOSED => CircuitState::Closed,
+            STATE_OPEN => CircuitState::Open,
+            STATE_HALF_OPEN => CircuitState::HalfOpen,
+            _ => CircuitState::Closed, // Default to closed for safety
+        }
+    }
+}
+
+/// Get current time as milliseconds since an arbitrary epoch.
+/// Uses Instant for monotonic time, converting to ms for atomic storage.
+#[inline]
+fn now_ms() -> u64 {
+    // Use a static reference point for consistent timing
+    static START: std::sync::OnceLock<Instant> = std::sync::OnceLock::new();
+    let start = START.get_or_init(Instant::now);
+    start.elapsed().as_millis() as u64
+}
+
+/// Circuit breaker implementation using lock-free atomics for hot paths.
+///
+/// This implementation avoids RwLock contention by using atomic operations
+/// for state checks (the most common operation). Only state transitions
+/// use compare-and-swap which is still lock-free.
+#[derive(Debug)]
+pub struct CircuitBreaker {
+    /// Circuit state stored as atomic u8 (0=Closed, 1=Open, 2=HalfOpen)
+    state: AtomicU8,
+    consecutive_failures: AtomicU32,
+    consecutive_successes: AtomicU32,
+    total_failures: AtomicU64,
+    total_successes: AtomicU64,
+    /// Last failure time in milliseconds (from now_ms())
+    last_failure_time_ms: AtomicU64,
+    /// Last state change time in milliseconds (from now_ms())
+    last_state_change_ms: AtomicU64,
+    config: CircuitBreakerConfig,
+    metric_label: String,
+}
+
+impl CircuitBreaker {
+    /// Create a new circuit breaker with default configuration
+    pub fn new() -> Self {
+        Self::with_config_and_label(CircuitBreakerConfig::default(), String::new())
+    }
+
+    /// Create a new circuit breaker with custom configuration and metric label
+    pub fn with_config_and_label(config: CircuitBreakerConfig, metric_label: String) -> Self {
+        let init_state = CircuitState::Closed;
+        Metrics::set_worker_cb_state(&metric_label, init_state.to_int());
+        Self {
+            state: AtomicU8::new(STATE_CLOSED),
+            consecutive_failures: AtomicU32::new(0),
+            consecutive_successes: AtomicU32::new(0),
+            total_failures: AtomicU64::new(0),
+            total_successes: AtomicU64::new(0),
+            last_failure_time_ms: AtomicU64::new(0),
+            last_state_change_ms: AtomicU64::new(now_ms()),
+            config,
+            metric_label,
+        }
+    }
+
+    /// Get the metric label
+    pub fn metric_label(&self) -> &str {
+        &self.metric_label
+    }
+
+    /// Check if a request can be executed (lock-free hot path)
+    #[inline]
+    pub fn can_execute(&self) -> bool {
+        let state = self.state();
+        match state {
+            CircuitState::Closed => true,
+            CircuitState::Open => false,
+            CircuitState::HalfOpen => true,
+        }
+    }
+
+    /// Get the current state (lock-free)
+    #[inline]
+    pub fn state(&self) -> CircuitState {
+        self.check_and_update_state_returning()
+    }
+
+    /// Check and update state, returning the current state (lock-free)
+    #[inline]
+    fn check_and_update_state_returning(&self) -> CircuitState {
+        let current_state_int = self.state.load(Ordering::Acquire);
+        let current_state = CircuitState::from_int(current_state_int);
+
+        if current_state == CircuitState::Open {
+            let last_change_ms = self.last_state_change_ms.load(Ordering::Acquire);
+            let elapsed_ms = now_ms().saturating_sub(last_change_ms);
+            let timeout_ms = self.config.timeout_duration.as_millis() as u64;
+
+            if elapsed_ms >= timeout_ms {
+                // Try to transition to HalfOpen using CAS
+                if self
+                    .state
+                    .compare_exchange(
+                        STATE_OPEN,
+                        STATE_HALF_OPEN,
+                        Ordering::AcqRel,
+                        Ordering::Acquire,
+                    )
+                    .is_ok()
+                {
+                    self.last_state_change_ms.store(now_ms(), Ordering::Release);
+                    self.consecutive_failures.store(0, Ordering::Release);
+                    self.consecutive_successes.store(0, Ordering::Release);
+
+                    info!("Circuit breaker state transition: open -> half_open");
+                    Metrics::record_worker_cb_transition(&self.metric_label, "open", "half_open");
+                    Metrics::set_worker_cb_state(&self.metric_label, STATE_HALF_OPEN);
+                    self.publish_gauge_metrics();
+                    return CircuitState::HalfOpen;
+                }
+                // Another thread already transitioned, re-read the state
+                return CircuitState::from_int(self.state.load(Ordering::Acquire));
+            }
+        }
+        current_state
+    }
+
+    /// Record the outcome of a request
+    pub fn record_outcome(&self, success: bool) {
+        if success {
+            self.record_success();
+        } else {
+            self.record_failure();
+        }
+
+        let outcome_str = if success { "success" } else { "failure" };
+        Metrics::record_worker_cb_outcome(&self.metric_label, outcome_str);
+        self.publish_gauge_metrics();
+    }
+
+    /// Record a successful request
+    pub fn record_success(&self) {
+        self.total_successes.fetch_add(1, Ordering::Relaxed);
+        self.consecutive_failures.store(0, Ordering::Release);
+        let successes = self.consecutive_successes.fetch_add(1, Ordering::AcqRel) + 1;
+
+        let current_state = CircuitState::from_int(self.state.load(Ordering::Acquire));
+
+        match current_state {
+            CircuitState::HalfOpen => {
+                if successes >= self.config.success_threshold {
+                    self.transition_to(CircuitState::Closed);
+                }
+            }
+            CircuitState::Closed => {}
+            CircuitState::Open => {
+                tracing::warn!("Success recorded while circuit is open");
+            }
+        }
+    }
+
+    /// Record a failed request
+    pub fn record_failure(&self) {
+        self.total_failures.fetch_add(1, Ordering::Relaxed);
+        self.consecutive_successes.store(0, Ordering::Release);
+        let failures = self.consecutive_failures.fetch_add(1, Ordering::AcqRel) + 1;
+
+        // Update last failure time atomically
+        self.last_failure_time_ms.store(now_ms(), Ordering::Release);
+
+        let current_state = CircuitState::from_int(self.state.load(Ordering::Acquire));
+
+        match current_state {
+            CircuitState::Closed => {
+                if failures >= self.config.failure_threshold {
+                    self.transition_to(CircuitState::Open);
+                }
+            }
+            CircuitState::HalfOpen => {
+                self.transition_to(CircuitState::Open);
+            }
+            CircuitState::Open => {}
+        }
+    }
+
+    /// Transition to a new state (uses CAS for lock-free operation)
+    fn transition_to(&self, new_state: CircuitState) {
+        let new_state_int = new_state.to_int();
+        let old_state_int = self.state.swap(new_state_int, Ordering::AcqRel);
+        let old_state = CircuitState::from_int(old_state_int);
+
+        if old_state != new_state {
+            self.last_state_change_ms.store(now_ms(), Ordering::Release);
+
+            match new_state {
+                CircuitState::Closed => {
+                    self.consecutive_failures.store(0, Ordering::Release);
+                    self.consecutive_successes.store(0, Ordering::Release);
+                }
+                CircuitState::Open => {
+                    self.consecutive_successes.store(0, Ordering::Release);
+                }
+                CircuitState::HalfOpen => {
+                    self.consecutive_failures.store(0, Ordering::Release);
+                    self.consecutive_successes.store(0, Ordering::Release);
+                }
+            }
+
+            let from = old_state.as_str();
+            let to = new_state.as_str();
+            info!("Circuit breaker state transition: {} -> {}", from, to);
+            Metrics::record_worker_cb_transition(&self.metric_label, from, to);
+            Metrics::set_worker_cb_state(&self.metric_label, new_state.to_int());
+            self.publish_gauge_metrics();
+        }
+    }
+
+    /// Get the number of consecutive failures
+    pub fn consecutive_failures(&self) -> u32 {
+        self.consecutive_failures.load(Ordering::Acquire)
+    }
+
+    /// Get the number of consecutive successes
+    pub fn consecutive_successes(&self) -> u32 {
+        self.consecutive_successes.load(Ordering::Acquire)
+    }
+
+    /// Get total failures
+    pub fn total_failures(&self) -> u64 {
+        self.total_failures.load(Ordering::Relaxed)
+    }
+
+    /// Get total successes
+    pub fn total_successes(&self) -> u64 {
+        self.total_successes.load(Ordering::Relaxed)
+    }
+
+    /// Get time since last failure
+    pub fn time_since_last_failure(&self) -> Option<Duration> {
+        let last_ms = self.last_failure_time_ms.load(Ordering::Acquire);
+        if last_ms == 0 {
+            None
+        } else {
+            let elapsed_ms = now_ms().saturating_sub(last_ms);
+            Some(Duration::from_millis(elapsed_ms))
+        }
+    }
+
+    /// Get time since last state change
+    pub fn time_since_last_state_change(&self) -> Duration {
+        let last_ms = self.last_state_change_ms.load(Ordering::Acquire);
+        let elapsed_ms = now_ms().saturating_sub(last_ms);
+        Duration::from_millis(elapsed_ms)
+    }
+
+    /// Check if the circuit is in a half-open state
+    pub fn is_half_open(&self) -> bool {
+        self.state() == CircuitState::HalfOpen
+    }
+
+    /// Record a test success (for health check probing)
+    pub fn record_test_success(&self) {
+        if self.is_half_open() {
+            self.record_success();
+        }
+    }
+
+    /// Record a test failure (for health check probing)
+    pub fn record_test_failure(&self) {
+        if self.is_half_open() {
+            self.record_failure();
+        }
+    }
+
+    /// Reset the circuit breaker to closed state
+    pub fn reset(&self) {
+        self.transition_to(CircuitState::Closed);
+        self.consecutive_failures.store(0, Ordering::Release);
+        self.consecutive_successes.store(0, Ordering::Release);
+        self.publish_gauge_metrics();
+    }
+
+    /// Force the circuit to open (for manual intervention)
+    pub fn force_open(&self) {
+        self.transition_to(CircuitState::Open);
+    }
+
+    /// Get circuit breaker statistics
+    pub fn stats(&self) -> CircuitBreakerStats {
+        CircuitBreakerStats {
+            state: self.state(),
+            consecutive_failures: self.consecutive_failures(),
+            consecutive_successes: self.consecutive_successes(),
+            total_failures: self.total_failures(),
+            total_successes: self.total_successes(),
+            time_since_last_failure: self.time_since_last_failure(),
+            time_since_last_state_change: self.time_since_last_state_change(),
+        }
+    }
+
+    fn publish_gauge_metrics(&self) {
+        Metrics::set_worker_cb_consecutive_failures(
+            &self.metric_label,
+            self.consecutive_failures(),
+        );
+        Metrics::set_worker_cb_consecutive_successes(
+            &self.metric_label,
+            self.consecutive_successes(),
+        );
+    }
+}
+
+impl Clone for CircuitBreaker {
+    fn clone(&self) -> Self {
+        Self {
+            state: AtomicU8::new(self.state.load(Ordering::Acquire)),
+            consecutive_failures: AtomicU32::new(self.consecutive_failures.load(Ordering::Acquire)),
+            consecutive_successes: AtomicU32::new(
+                self.consecutive_successes.load(Ordering::Acquire),
+            ),
+            total_failures: AtomicU64::new(self.total_failures.load(Ordering::Relaxed)),
+            total_successes: AtomicU64::new(self.total_successes.load(Ordering::Relaxed)),
+            last_failure_time_ms: AtomicU64::new(self.last_failure_time_ms.load(Ordering::Acquire)),
+            last_state_change_ms: AtomicU64::new(self.last_state_change_ms.load(Ordering::Acquire)),
+            config: self.config.clone(),
+            metric_label: self.metric_label.clone(),
+        }
+    }
+}
+
+impl Default for CircuitBreaker {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+/// Circuit breaker statistics
+#[derive(Debug, Clone)]
+pub struct CircuitBreakerStats {
+    pub state: CircuitState,
+    pub consecutive_failures: u32,
+    pub consecutive_successes: u32,
+    pub total_failures: u64,
+    pub total_successes: u64,
+    pub time_since_last_failure: Option<Duration>,
+    pub time_since_last_state_change: Duration,
+}
+
+#[cfg(test)]
+mod tests {
+    use std::thread;
+
+    use super::*;
+
+    #[test]
+    fn test_circuit_breaker_initial_state() {
+        let cb = CircuitBreaker::new();
+        assert_eq!(cb.state(), CircuitState::Closed);
+        assert!(cb.can_execute());
+        assert_eq!(cb.consecutive_failures(), 0);
+        assert_eq!(cb.consecutive_successes(), 0);
+    }
+
+    #[test]
+    fn test_circuit_opens_on_threshold() {
+        let config = CircuitBreakerConfig {
+            failure_threshold: 3,
+            ..Default::default()
+        };
+        let cb = CircuitBreaker::with_config_and_label(config, String::new());
+
+        assert_eq!(cb.state(), CircuitState::Closed);
+        cb.record_failure();
+        assert_eq!(cb.state(), CircuitState::Closed);
+        cb.record_failure();
+        assert_eq!(cb.state(), CircuitState::Closed);
+        cb.record_failure();
+
+        assert_eq!(cb.state(), CircuitState::Open);
+        assert!(!cb.can_execute());
+        assert_eq!(cb.consecutive_failures(), 3);
+    }
+
+    #[test]
+    fn test_circuit_half_open_after_timeout() {
+        let config = CircuitBreakerConfig {
+            failure_threshold: 1,
+            timeout_duration: Duration::from_millis(100),
+            ..Default::default()
+        };
+        let cb = CircuitBreaker::with_config_and_label(config, String::new());
+
+        cb.record_failure();
+        assert_eq!(cb.state(), CircuitState::Open);
+
+        thread::sleep(Duration::from_millis(150));
+
+        assert_eq!(cb.state(), CircuitState::HalfOpen);
+        assert!(cb.can_execute());
+    }
+
+    #[test]
+    fn test_circuit_closes_on_success_threshold() {
+        let config = CircuitBreakerConfig {
+            failure_threshold: 1,
+            success_threshold: 2,
+            timeout_duration: Duration::from_millis(50),
+            ..Default::default()
+        };
+        let cb = CircuitBreaker::with_config_and_label(config, String::new());
+
+        cb.record_failure();
+        assert_eq!(cb.state(), CircuitState::Open);
+
+        thread::sleep(Duration::from_millis(100));
+        assert_eq!(cb.state(), CircuitState::HalfOpen);
+
+        cb.record_success();
+        assert_eq!(cb.state(), CircuitState::HalfOpen);
+        cb.record_success();
+
+        assert_eq!(cb.state(), CircuitState::Closed);
+        assert!(cb.can_execute());
+    }
+
+    #[test]
+    fn test_circuit_reopens_on_half_open_failure() {
+        let config = CircuitBreakerConfig {
+            failure_threshold: 1,
+            timeout_duration: Duration::from_millis(50),
+            ..Default::default()
+        };
+        let cb = CircuitBreaker::with_config_and_label(config, String::new());
+
+        cb.record_failure();
+        assert_eq!(cb.state(), CircuitState::Open);
+
+        thread::sleep(Duration::from_millis(100));
+        assert_eq!(cb.state(), CircuitState::HalfOpen);
+
+        cb.record_failure();
+
+        assert_eq!(cb.state(), CircuitState::Open);
+        assert!(!cb.can_execute());
+    }
+
+    #[test]
+    fn test_success_resets_failure_count() {
+        let config = CircuitBreakerConfig {
+            failure_threshold: 3,
+            ..Default::default()
+        };
+        let cb = CircuitBreaker::with_config_and_label(config, String::new());
+
+        cb.record_failure();
+        cb.record_failure();
+        assert_eq!(cb.consecutive_failures(), 2);
+
+        cb.record_success();
+        assert_eq!(cb.consecutive_failures(), 0);
+        assert_eq!(cb.consecutive_successes(), 1);
+
+        cb.record_failure();
+        cb.record_failure();
+        assert_eq!(cb.state(), CircuitState::Closed);
+    }
+
+    #[test]
+    fn test_manual_reset() {
+        let config = CircuitBreakerConfig {
+            failure_threshold: 1,
+            ..Default::default()
+        };
+        let cb = CircuitBreaker::with_config_and_label(config, String::new());
+
+        cb.record_failure();
+        assert_eq!(cb.state(), CircuitState::Open);
+
+        cb.reset();
+        assert_eq!(cb.state(), CircuitState::Closed);
+        assert_eq!(cb.consecutive_failures(), 0);
+        assert_eq!(cb.consecutive_successes(), 0);
+    }
+
+    #[test]
+    fn test_force_open() {
+        let cb = CircuitBreaker::new();
+        assert_eq!(cb.state(), CircuitState::Closed);
+
+        cb.force_open();
+        assert_eq!(cb.state(), CircuitState::Open);
+        assert!(!cb.can_execute());
+    }
+
+    #[test]
+    fn test_stats() {
+        let config = CircuitBreakerConfig {
+            failure_threshold: 2,
+            ..Default::default()
+        };
+        let cb = CircuitBreaker::with_config_and_label(config, String::new());
+
+        cb.record_success();
+        cb.record_failure();
+        cb.record_failure();
+
+        let stats = cb.stats();
+        assert_eq!(stats.state, CircuitState::Open);
+        assert_eq!(stats.consecutive_failures, 2);
+        assert_eq!(stats.consecutive_successes, 0);
+        assert_eq!(stats.total_failures, 2);
+        assert_eq!(stats.total_successes, 1);
+    }
+
+    #[test]
+    fn test_clone() {
+        let cb1 = CircuitBreaker::new();
+        cb1.record_failure();
+
+        let cb2 = cb1.clone();
+        assert_eq!(cb2.consecutive_failures(), 1);
+
+        cb1.record_failure();
+        assert_eq!(cb1.consecutive_failures(), 2);
+        assert_eq!(cb2.consecutive_failures(), 1); // cb2 is unchanged
+    }
+
+    #[test]
+    fn test_thread_safety() {
+        use std::sync::Arc;
+
+        let cb = Arc::new(CircuitBreaker::new());
+        let mut handles = vec![];
+
+        for _ in 0..10 {
+            let cb_clone = Arc::clone(&cb);
+            let handle = thread::spawn(move || {
+                for _ in 0..100 {
+                    cb_clone.record_failure();
+                }
+            });
+            handles.push(handle);
+        }
+
+        for handle in handles {
+            handle.join().unwrap();
+        }
+
+        assert_eq!(cb.total_failures(), 1000);
+    }
+}

third_party/sglang/sgl-model-gateway/src/core/error.rs

@@ -0,0 +1,210 @@
+//! Error types for the SGLang router core
+//!
+//! This module defines error types used throughout the router for worker operations.
+
+/// Worker-related errors
+#[derive(Debug, thiserror::Error)]
+pub enum WorkerError {
+    #[error("Health check failed for worker {url}: {reason}")]
+    HealthCheckFailed { url: String, reason: String },
+
+    #[error("Worker not found: {url}")]
+    WorkerNotFound { url: String },
+
+    #[error("Invalid worker configuration: {message}")]
+    InvalidConfiguration { message: String },
+
+    #[error("Network error for worker {url}: {error}")]
+    NetworkError { url: String, error: String },
+
+    #[error("Worker at capacity: {url}")]
+    WorkerAtCapacity { url: String },
+
+    #[error("Invalid URL format: {url}")]
+    InvalidUrl { url: String },
+
+    #[error("Connection failed for worker {url}: {reason}")]
+    ConnectionFailed { url: String, reason: String },
+}
+
+/// Result type for worker operations
+pub type WorkerResult<T> = Result<T, WorkerError>;
+
+/// Convert from reqwest errors to worker errors
+impl From<reqwest::Error> for WorkerError {
+    fn from(err: reqwest::Error) -> Self {
+        WorkerError::NetworkError {
+            url: err.url().map(|u| u.to_string()).unwrap_or_default(),
+            error: err.to_string(),
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::error::Error;
+
+    use super::*;
+
+    #[test]
+    fn test_health_check_failed_display() {
+        let error = WorkerError::HealthCheckFailed {
+            url: "http://worker1:8080".to_string(),
+            reason: "Connection refused".to_string(),
+        };
+        assert_eq!(
+            error.to_string(),
+            "Health check failed for worker http://worker1:8080: Connection refused"
+        );
+    }
+
+    #[test]
+    fn test_worker_not_found_display() {
+        let error = WorkerError::WorkerNotFound {
+            url: "http://worker2:8080".to_string(),
+        };
+        assert_eq!(error.to_string(), "Worker not found: http://worker2:8080");
+    }
+
+    #[test]
+    fn test_invalid_configuration_display() {
+        let error = WorkerError::InvalidConfiguration {
+            message: "Missing port number".to_string(),
+        };
+        assert_eq!(
+            error.to_string(),
+            "Invalid worker configuration: Missing port number"
+        );
+    }
+
+    #[test]
+    fn test_network_error_display() {
+        let error = WorkerError::NetworkError {
+            url: "http://worker3:8080".to_string(),
+            error: "Timeout after 30s".to_string(),
+        };
+        assert_eq!(
+            error.to_string(),
+            "Network error for worker http://worker3:8080: Timeout after 30s"
+        );
+    }
+
+    #[test]
+    fn test_worker_at_capacity_display() {
+        let error = WorkerError::WorkerAtCapacity {
+            url: "http://worker4:8080".to_string(),
+        };
+        assert_eq!(error.to_string(), "Worker at capacity: http://worker4:8080");
+    }
+
+    #[test]
+    fn test_worker_error_implements_std_error() {
+        let error = WorkerError::WorkerNotFound {
+            url: "http://test".to_string(),
+        };
+        let _: &dyn Error = &error;
+        assert!(error.source().is_none());
+    }
+
+    #[test]
+    fn test_error_send_sync() {
+        fn assert_send_sync<T: Send + Sync>() {}
+        assert_send_sync::<WorkerError>();
+    }
+
+    #[test]
+    fn test_worker_result_type_alias() {
+        let result: WorkerResult<i32> = Ok(42);
+        assert!(matches!(result, Ok(42)));
+
+        let error = WorkerError::WorkerNotFound {
+            url: "test".to_string(),
+        };
+        let result: WorkerResult<i32> = Err(error);
+        assert!(result.is_err());
+    }
+
+    #[test]
+    fn test_empty_url_handling() {
+        let error1 = WorkerError::HealthCheckFailed {
+            url: "".to_string(),
+            reason: "No connection".to_string(),
+        };
+        assert_eq!(
+            error1.to_string(),
+            "Health check failed for worker : No connection"
+        );
+
+        let error2 = WorkerError::NetworkError {
+            url: "".to_string(),
+            error: "DNS failure".to_string(),
+        };
+        assert_eq!(error2.to_string(), "Network error for worker : DNS failure");
+
+        let error3 = WorkerError::WorkerNotFound {
+            url: "".to_string(),
+        };
+        assert_eq!(error3.to_string(), "Worker not found: ");
+    }
+
+    #[test]
+    fn test_special_characters_in_messages() {
+        let error = WorkerError::InvalidConfiguration {
+            message: "Invalid JSON: {\"error\": \"test\"}".to_string(),
+        };
+        assert_eq!(
+            error.to_string(),
+            "Invalid worker configuration: Invalid JSON: {\"error\": \"test\"}"
+        );
+
+        let error2 = WorkerError::HealthCheckFailed {
+            url: "http://测试:8080".to_string(),
+            reason: "连接被拒绝".to_string(),
+        };
+        assert_eq!(
+            error2.to_string(),
+            "Health check failed for worker http://测试:8080: 连接被拒绝"
+        );
+    }
+
+    #[test]
+    fn test_very_long_error_messages() {
+        let long_message = "A".repeat(10000);
+        let error = WorkerError::InvalidConfiguration {
+            message: long_message.clone(),
+        };
+        let display = error.to_string();
+        assert!(display.contains(&long_message));
+        assert_eq!(
+            display.len(),
+            "Invalid worker configuration: ".len() + long_message.len()
+        );
+    }
+
+    #[test]
+    fn test_reqwest_error_conversion() {
+        let network_error = WorkerError::NetworkError {
+            url: "http://example.com".to_string(),
+            error: "connection timeout".to_string(),
+        };
+
+        match network_error {
+            WorkerError::NetworkError { url, error } => {
+                assert_eq!(url, "http://example.com");
+                assert_eq!(error, "connection timeout");
+            }
+            _ => panic!("Expected NetworkError variant"),
+        }
+    }
+
+    #[test]
+    fn test_error_equality() {
+        let error1 = WorkerError::WorkerNotFound {
+            url: "http://test".to_string(),
+        };
+        let error2 = WorkerError::WorkerNotFound {
+            url: "http://test".to_string(),
+        };
+        assert_eq!(error1.to_string(), error2.to_string());
+    }
+}

third_party/sglang/sgl-model-gateway/src/core/job_queue.rs

@@ -0,0 +1,798 @@
+//! Async job queue for control plane operations
+//!
+//! Provides non-blocking worker management by queuing operations and processing
+//! them asynchronously in background worker tasks.
+
+use std::{
+    collections::HashMap,
+    sync::{Arc, Weak},
+    time::{Duration, SystemTime},
+};
+
+use dashmap::DashMap;
+use smg_mcp::McpConfig;
+use tokio::sync::{mpsc, Semaphore};
+use tracing::{debug, error, info, warn};
+use wfaas::WorkflowId;
+
+use crate::{
+    app_context::AppContext,
+    config::{RouterConfig, RoutingMode},
+    core::steps::{
+        create_external_worker_workflow_data, create_local_worker_workflow_data,
+        create_mcp_workflow_data, create_tokenizer_workflow_data,
+        create_wasm_registration_workflow_data, create_wasm_removal_workflow_data,
+        create_worker_removal_workflow_data, create_worker_update_workflow_data,
+        McpServerConfigRequest, TokenizerConfigRequest, TokenizerRemovalRequest,
+        WasmModuleConfigRequest, WasmModuleRemovalRequest,
+    },
+    protocols::worker_spec::{JobStatus, WorkerConfigRequest, WorkerUpdateRequest},
+};
+
+/// Job types for control plane operations
+#[derive(Debug, Clone)]
+pub enum Job {
+    AddWorker {
+        config: Box<WorkerConfigRequest>,
+    },
+    UpdateWorker {
+        url: String,
+        update: Box<WorkerUpdateRequest>,
+    },
+    RemoveWorker {
+        url: String,
+    },
+    InitializeWorkersFromConfig {
+        router_config: Box<RouterConfig>,
+    },
+    InitializeMcpServers {
+        mcp_config: Box<McpConfig>,
+    },
+    RegisterMcpServer {
+        config: Box<McpServerConfigRequest>,
+    },
+    AddWasmModule {
+        config: Box<WasmModuleConfigRequest>,
+    },
+    RemoveWasmModule {
+        request: Box<WasmModuleRemovalRequest>,
+    },
+    AddTokenizer {
+        config: Box<TokenizerConfigRequest>,
+    },
+    RemoveTokenizer {
+        request: Box<TokenizerRemovalRequest>,
+    },
+}
+
+impl Job {
+    /// Get job type as string for logging
+    pub fn job_type(&self) -> &'static str {
+        match self {
+            Job::AddWorker { .. } => "AddWorker",
+            Job::UpdateWorker { .. } => "UpdateWorker",
+            Job::RemoveWorker { .. } => "RemoveWorker",
+            Job::InitializeWorkersFromConfig { .. } => "InitializeWorkersFromConfig",
+            Job::InitializeMcpServers { .. } => "InitializeMcpServers",
+            Job::RegisterMcpServer { .. } => "RegisterMcpServer",
+            Job::AddWasmModule { .. } => "AddWasmModule",
+            Job::RemoveWasmModule { .. } => "RemoveWasmModule",
+            Job::AddTokenizer { .. } => "AddTokenizer",
+            Job::RemoveTokenizer { .. } => "RemoveTokenizer",
+        }
+    }
+
+    /// Get worker URL, MCP server name, WASM module, or tokenizer identifier for logging and status tracking
+    pub fn worker_url(&self) -> &str {
+        match self {
+            Job::AddWorker { config } => &config.url,
+            Job::UpdateWorker { url, .. } => url,
+            Job::RemoveWorker { url } => url,
+            Job::InitializeWorkersFromConfig { .. } => "startup",
+            Job::InitializeMcpServers { .. } => "startup",
+            Job::RegisterMcpServer { config } => &config.name,
+            Job::AddWasmModule { config } => &config.descriptor.name,
+            Job::RemoveWasmModule { request } => &request.uuid_string,
+            Job::AddTokenizer { config } => &config.id,
+            Job::RemoveTokenizer { request } => &request.id,
+        }
+    }
+}
+
+/// Job queue configuration
+#[derive(Clone, Debug)]
+pub struct JobQueueConfig {
+    /// Maximum pending jobs in queue
+    pub queue_capacity: usize,
+    /// Maximum number of jobs executing concurrently
+    pub max_concurrent_jobs: usize,
+}
+
+impl Default for JobQueueConfig {
+    fn default() -> Self {
+        Self {
+            queue_capacity: 1000,
+            max_concurrent_jobs: 200,
+        }
+    }
+}
+
+/// Job queue manager for worker validation and removal operations
+pub struct JobQueue {
+    /// Channel for submitting jobs
+    tx: mpsc::Sender<Job>,
+    /// Weak reference to AppContext to avoid circular dependencies
+    context: Weak<AppContext>,
+    /// Job status tracking by worker URL
+    status_map: Arc<DashMap<String, JobStatus>>,
+    /// Semaphore to limit concurrent job execution
+    concurrency_limit: Arc<Semaphore>,
+}
+
+impl std::fmt::Debug for JobQueue {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("JobQueue")
+            .field("status_count", &self.status_map.len())
+            .finish()
+    }
+}
+
+impl JobQueue {
+    /// Create a new job queue with semaphore-based concurrency control
+    ///
+    /// Takes a Weak reference to AppContext to avoid circular strong references.
+    /// Spawns a single dispatcher task that spawns individual job tasks with semaphore control.
+    pub fn new(config: JobQueueConfig, context: Weak<AppContext>) -> Arc<Self> {
+        let (tx, mut rx) = mpsc::channel(config.queue_capacity);
+
+        debug!(
+            "Initializing job queue: capacity={}, max_concurrent={}",
+            config.queue_capacity, config.max_concurrent_jobs
+        );
+
+        let status_map = Arc::new(DashMap::new());
+        let concurrency_limit = Arc::new(Semaphore::new(config.max_concurrent_jobs));
+
+        let queue = Arc::new(Self {
+            tx,
+            context: context.clone(),
+            status_map: status_map.clone(),
+            concurrency_limit: concurrency_limit.clone(),
+        });
+
+        // Single dispatcher task
+        let ctx = context.clone();
+        let status = status_map.clone();
+        let sem = concurrency_limit.clone();
+
+        tokio::spawn(async move {
+            while let Some(job) = rx.recv().await {
+                // Acquire permit (blocks if at concurrency limit)
+                let Ok(permit) = sem.clone().acquire_owned().await else {
+                    error!("Semaphore closed, stopping dispatcher");
+                    break;
+                };
+
+                let ctx_clone = ctx.clone();
+                let status_clone = status.clone();
+
+                tokio::spawn(async move {
+                    Self::process_job(job, ctx_clone, status_clone, permit).await;
+                });
+            }
+
+            debug!("Job dispatcher stopped");
+        });
+
+        // Spawn cleanup task for old job statuses (TTL 5 minutes)
+        let cleanup_status_map = status_map.clone();
+        tokio::spawn(async move {
+            Self::cleanup_old_statuses(cleanup_status_map).await;
+        });
+
+        queue
+    }
+
+    /// Get current queue and concurrency status
+    pub fn get_load_info(&self) -> (usize, usize) {
+        let queue_depth = self.tx.max_capacity() - self.tx.capacity();
+        let available_permits = self.concurrency_limit.available_permits();
+        (queue_depth, available_permits)
+    }
+
+    /// Submit a job with detailed queue status
+    pub async fn submit(&self, job: Job) -> Result<(), String> {
+        // Check if context is still alive before accepting jobs
+        if self.context.upgrade().is_none() {
+            return Err("Job queue shutting down: AppContext dropped".to_string());
+        }
+
+        // Extract values before moving job
+        let job_type = job.job_type();
+        let worker_url = job.worker_url().to_string();
+
+        // Record pending status
+        self.status_map.insert(
+            worker_url.clone(),
+            JobStatus::pending(job_type, &worker_url),
+        );
+
+        match self.tx.send(job).await {
+            Ok(_) => {
+                let (queue_depth, available_permits) = self.get_load_info();
+                debug!(
+                    "Job submitted: type={}, worker={}, queue_depth={}, available_slots={}",
+                    job_type, worker_url, queue_depth, available_permits
+                );
+                Ok(())
+            }
+            Err(_) => {
+                self.status_map.remove(&worker_url);
+                let (queue_depth, _) = self.get_load_info();
+                Err(format!(
+                    "Job queue full: {} jobs pending (capacity: {})",
+                    queue_depth,
+                    self.tx.max_capacity()
+                ))
+            }
+        }
+    }
+
+    /// Get job status by worker URL
+    pub fn get_status(&self, worker_url: &str) -> Option<JobStatus> {
+        self.status_map.get(worker_url).map(|entry| entry.clone())
+    }
+
+    /// Remove job status (called when worker is deleted)
+    pub fn remove_status(&self, worker_url: &str) {
+        self.status_map.remove(worker_url);
+    }
+
+    /// Process a single job with status tracking and error handling
+    async fn process_job(
+        job: Job,
+        context: Weak<AppContext>,
+        status_map: Arc<DashMap<String, JobStatus>>,
+        _permit: tokio::sync::OwnedSemaphorePermit,
+    ) {
+        let job_type = job.job_type();
+        let worker_url = job.worker_url().to_string();
+        let start = std::time::Instant::now();
+
+        // Update to processing
+        status_map.insert(
+            worker_url.clone(),
+            JobStatus::processing(job_type, &worker_url),
+        );
+
+        debug!("Processing job: type={}, worker={}", job_type, worker_url);
+
+        // Execute job
+        match context.upgrade() {
+            Some(ctx) => {
+                let result = Self::execute_job(&job, &ctx).await;
+                let duration = start.elapsed();
+                Self::record_job_completion(job_type, &worker_url, duration, &result, &status_map);
+            }
+            None => {
+                let error_msg = "AppContext dropped".to_string();
+                status_map.insert(
+                    worker_url.clone(),
+                    JobStatus::failed(job_type, &worker_url, error_msg),
+                );
+                error!(
+                    "AppContext dropped, cannot process job: type={}, worker={}",
+                    job_type, worker_url
+                );
+            }
+        }
+
+        // Permit automatically released when dropped
+    }
+
+    /// Execute a specific job
+    async fn execute_job(job: &Job, context: &Arc<AppContext>) -> Result<String, String> {
+        match job {
+            Job::AddWorker { config } => {
+                let engines = context
+                    .workflow_engines
+                    .get()
+                    .ok_or_else(|| "Workflow engines not initialized".to_string())?;
+
+                let timeout_duration =
+                    Duration::from_secs(context.router_config.worker_startup_timeout_secs + 30);
+
+                // Select workflow based on runtime field
+                match config.runtime.as_deref() {
+                    Some("external") => {
+                        let workflow_data = create_external_worker_workflow_data(
+                            (**config).clone(),
+                            Arc::clone(context),
+                        );
+                        let instance_id = engines
+                            .external_worker
+                            .start_workflow(
+                                WorkflowId::new("external_worker_registration"),
+                                workflow_data,
+                            )
+                            .await
+                            .map_err(|e| {
+                                format!(
+                                    "Failed to start external worker registration workflow: {:?}",
+                                    e
+                                )
+                            })?;
+
+                        debug!(
+                            "Started external worker registration workflow for {} (instance: {})",
+                            config.url, instance_id
+                        );
+
+                        engines
+                            .external_worker
+                            .wait_for_completion(instance_id, &config.url, timeout_duration)
+                            .await
+                    }
+                    _ => {
+                        let workflow_data = create_local_worker_workflow_data(
+                            (**config).clone(),
+                            Arc::clone(context),
+                        );
+                        let instance_id = engines
+                            .local_worker
+                            .start_workflow(
+                                WorkflowId::new("local_worker_registration"),
+                                workflow_data,
+                            )
+                            .await
+                            .map_err(|e| {
+                                format!(
+                                    "Failed to start local worker registration workflow: {:?}",
+                                    e
+                                )
+                            })?;
+
+                        debug!(
+                            "Started local worker registration workflow for {} (instance: {})",
+                            config.url, instance_id
+                        );
+
+                        engines
+                            .local_worker
+                            .wait_for_completion(instance_id, &config.url, timeout_duration)
+                            .await
+                    }
+                }
+            }
+            Job::UpdateWorker { url, update } => {
+                let engines = context
+                    .workflow_engines
+                    .get()
+                    .ok_or_else(|| "Workflow engines not initialized".to_string())?;
+
+                let workflow_data = create_worker_update_workflow_data(
+                    url.to_string(),
+                    (**update).clone(),
+                    Arc::clone(context),
+                );
+
+                let instance_id = engines
+                    .worker_update
+                    .start_workflow(WorkflowId::new("worker_update"), workflow_data)
+                    .await
+                    .map_err(|e| format!("Failed to start worker update workflow: {:?}", e))?;
+
+                debug!(
+                    "Started worker update workflow for {} (instance: {})",
+                    url, instance_id
+                );
+
+                let timeout_duration = Duration::from_secs(30);
+
+                engines
+                    .worker_update
+                    .wait_for_completion(instance_id, url, timeout_duration)
+                    .await
+            }
+            Job::RemoveWorker { url } => {
+                let engines = context
+                    .workflow_engines
+                    .get()
+                    .ok_or_else(|| "Workflow engines not initialized".to_string())?;
+
+                let workflow_data = create_worker_removal_workflow_data(
+                    url.to_string(),
+                    context.router_config.dp_aware,
+                    Arc::clone(context),
+                );
+
+                let instance_id = engines
+                    .worker_removal
+                    .start_workflow(WorkflowId::new("worker_removal"), workflow_data)
+                    .await
+                    .map_err(|e| format!("Failed to start worker removal workflow: {:?}", e))?;
+
+                debug!(
+                    "Started worker removal workflow for {} (instance: {})",
+                    url, instance_id
+                );
+
+                let timeout_duration = Duration::from_secs(30);
+
+                let result = engines
+                    .worker_removal
+                    .wait_for_completion(instance_id, url, timeout_duration)
+                    .await;
+
+                // Clean up job status when removing worker
+                if let Some(queue) = context.worker_job_queue.get() {
+                    queue.remove_status(url);
+                }
+
+                result
+            }
+            Job::AddWasmModule { config } => {
+                let engines = context
+                    .workflow_engines
+                    .get()
+                    .ok_or_else(|| "Workflow engines not initialized".to_string())?;
+
+                let workflow_data =
+                    create_wasm_registration_workflow_data(*config.clone(), Arc::clone(context));
+
+                let instance_id = engines
+                    .wasm_registration
+                    .start_workflow(WorkflowId::new("wasm_module_registration"), workflow_data)
+                    .await
+                    .map_err(|e| {
+                        format!("Failed to start WASM module registration workflow: {:?}", e)
+                    })?;
+
+                debug!(
+                    "Started WASM module registration workflow for {} (instance: {})",
+                    config.descriptor.name, instance_id
+                );
+
+                let timeout_duration = Duration::from_secs(300); // 5 minutes
+
+                engines
+                    .wasm_registration
+                    .wait_for_completion(instance_id, &config.descriptor.name, timeout_duration)
+                    .await
+            }
+            Job::RemoveWasmModule { request } => {
+                let engines = context
+                    .workflow_engines
+                    .get()
+                    .ok_or_else(|| "Workflow engines not initialized".to_string())?;
+
+                let workflow_data =
+                    create_wasm_removal_workflow_data(*request.clone(), Arc::clone(context));
+
+                let instance_id = engines
+                    .wasm_removal
+                    .start_workflow(WorkflowId::new("wasm_module_removal"), workflow_data)
+                    .await
+                    .map_err(|e| {
+                        format!("Failed to start WASM module removal workflow: {:?}", e)
+                    })?;
+
+                debug!(
+                    "Started WASM module removal workflow for {} (instance: {})",
+                    request.module_uuid, instance_id
+                );
+
+                let timeout_duration = Duration::from_secs(60); // 1 minute
+
+                engines
+                    .wasm_removal
+                    .wait_for_completion(
+                        instance_id,
+                        &request.module_uuid.to_string(),
+                        timeout_duration,
+                    )
+                    .await
+            }
+            Job::InitializeWorkersFromConfig { router_config } => {
+                let api_key = router_config.api_key.clone();
+                let mut worker_count = 0;
+
+                // Create iterator of (url, worker_type, bootstrap_port) tuples based on mode
+                let workers: Vec<(String, &str, Option<u16>)> = match &router_config.mode {
+                    RoutingMode::Regular { worker_urls } => worker_urls
+                        .iter()
+                        .map(|url| (url.clone(), "regular", None))
+                        .collect(),
+                    RoutingMode::PrefillDecode {
+                        prefill_urls,
+                        decode_urls,
+                        ..
+                    } => {
+                        let prefill_workers = prefill_urls
+                            .iter()
+                            .map(|(url, port)| (url.clone(), "prefill", *port));
+
+                        let decode_workers =
+                            decode_urls.iter().map(|url| (url.clone(), "decode", None));
+
+                        prefill_workers.chain(decode_workers).collect()
+                    }
+                    RoutingMode::OpenAI { worker_urls } => {
+                        // OpenAI mode: submit AddWorker jobs with runtime: "external"
+                        // The external_worker_registration workflow handles model discovery
+                        let api_key = router_config.api_key.clone();
+                        let mut submitted_count = 0;
+
+                        for url in worker_urls {
+                            let url_for_error = url.clone();
+                            let config = WorkerConfigRequest {
+                                url: url.clone(),
+                                api_key: api_key.clone(),
+                                worker_type: Some("regular".to_string()),
+                                labels: HashMap::new(),
+                                model_id: None,
+                                priority: None,
+                                cost: None,
+                                runtime: Some("external".to_string()),
+                                tokenizer_path: None,
+                                reasoning_parser: None,
+                                tool_parser: None,
+                                chat_template: router_config.chat_template.clone(),
+                                bootstrap_port: None,
+                                health_check_timeout_secs: router_config.health_check.timeout_secs,
+                                health_check_interval_secs: router_config
+                                    .health_check
+                                    .check_interval_secs,
+                                health_success_threshold: router_config
+                                    .health_check
+                                    .success_threshold,
+                                health_failure_threshold: router_config
+                                    .health_check
+                                    .failure_threshold,
+                                disable_health_check: router_config
+                                    .health_check
+                                    .disable_health_check,
+                                max_connection_attempts: router_config
+                                    .health_check
+                                    .success_threshold
+                                    * 10,
+                                dp_aware: false,
+                            };
+
+                            let job = Job::AddWorker {
+                                config: Box::new(config),
+                            };
+
+                            if let Some(queue) = context.worker_job_queue.get() {
+                                queue.submit(job).await.map_err(|e| {
+                                    format!(
+                                        "Failed to submit AddWorker job for external endpoint {}: {}",
+                                        url_for_error, e
+                                    )
+                                })?;
+                                submitted_count += 1;
+                            } else {
+                                return Err("JobQueue not available".to_string());
+                            }
+                        }
+
+                        if submitted_count == 0 {
+                            info!("OpenAI mode: no worker URLs provided");
+                            return Ok("OpenAI mode: no worker URLs to initialize".to_string());
+                        }
+
+                        return Ok(format!(
+                            "Submitted {} AddWorker jobs for external endpoints",
+                            submitted_count
+                        ));
+                    }
+                };
+
+                debug!(
+                    "Creating AddWorker jobs for {} workers from config",
+                    workers.len()
+                );
+
+                // Process all workers with unified loop
+                for (url, worker_type, bootstrap_port) in workers {
+                    let url_for_error = url.clone(); // Clone for error message
+                    let config = WorkerConfigRequest {
+                        url,
+                        api_key: api_key.clone(),
+                        worker_type: Some(worker_type.to_string()),
+                        labels: HashMap::new(),
+                        model_id: None,
+                        priority: None,
+                        cost: None,
+                        runtime: None,
+                        tokenizer_path: None,
+                        reasoning_parser: None,
+                        tool_parser: None,
+                        chat_template: router_config.chat_template.clone(),
+                        bootstrap_port,
+                        health_check_timeout_secs: router_config.health_check.timeout_secs,
+                        health_check_interval_secs: router_config.health_check.check_interval_secs,
+                        health_success_threshold: router_config.health_check.success_threshold,
+                        health_failure_threshold: router_config.health_check.failure_threshold,
+                        disable_health_check: router_config.health_check.disable_health_check,
+                        max_connection_attempts: router_config.health_check.success_threshold * 10,
+                        dp_aware: router_config.dp_aware,
+                    };
+
+                    let job = Job::AddWorker {
+                        config: Box::new(config),
+                    };
+
+                    if let Some(queue) = context.worker_job_queue.get() {
+                        queue.submit(job).await.map_err(|e| {
+                            format!(
+                                "Failed to submit AddWorker job for {} worker {}: {}",
+                                worker_type, url_for_error, e
+                            )
+                        })?;
+                        worker_count += 1;
+                    } else {
+                        return Err("JobQueue not available".to_string());
+                    }
+                }
+
+                Ok(format!("Submitted {} AddWorker jobs", worker_count))
+            }
+            Job::InitializeMcpServers { mcp_config } => {
+                let mut server_count = 0;
+
+                debug!(
+                    "Creating RegisterMcpServer jobs for {} MCP servers from config",
+                    mcp_config.servers.len()
+                );
+
+                // Submit RegisterMcpServer jobs for each server in the config
+                for server_config in &mcp_config.servers {
+                    let mcp_server_request = McpServerConfigRequest {
+                        name: server_config.name.clone(),
+                        config: server_config.clone(),
+                    };
+
+                    let job = Job::RegisterMcpServer {
+                        config: Box::new(mcp_server_request),
+                    };
+
+                    if let Some(queue) = context.worker_job_queue.get() {
+                        queue.submit(job).await.map_err(|e| {
+                            format!(
+                                "Failed to submit RegisterMcpServer job for '{}': {}",
+                                server_config.name, e
+                            )
+                        })?;
+                        server_count += 1;
+                    } else {
+                        return Err("JobQueue not available".to_string());
+                    }
+                }
+
+                Ok(format!("Submitted {} RegisterMcpServer jobs", server_count))
+            }
+            Job::RegisterMcpServer { config } => {
+                let engines = context
+                    .workflow_engines
+                    .get()
+                    .ok_or_else(|| "Workflow engines not initialized".to_string())?;
+
+                let workflow_data =
+                    create_mcp_workflow_data((**config).clone(), Arc::clone(context));
+
+                let instance_id = engines
+                    .mcp
+                    .start_workflow(WorkflowId::new("mcp_registration"), workflow_data)
+                    .await
+                    .map_err(|e| format!("Failed to start MCP registration workflow: {:?}", e))?;
+
+                debug!(
+                    "Started MCP registration workflow for {} (instance: {})",
+                    config.name, instance_id
+                );
+
+                let timeout_duration = Duration::from_secs(7200 + 30); // 2hr + margin
+
+                engines
+                    .mcp
+                    .wait_for_completion(instance_id, &config.name, timeout_duration)
+                    .await
+            }
+            Job::AddTokenizer { config } => {
+                let engines = context
+                    .workflow_engines
+                    .get()
+                    .ok_or_else(|| "Workflow engines not initialized".to_string())?;
+
+                let workflow_data =
+                    create_tokenizer_workflow_data(*config.clone(), Arc::clone(context));
+
+                let instance_id = engines
+                    .tokenizer
+                    .start_workflow(WorkflowId::new("tokenizer_registration"), workflow_data)
+                    .await
+                    .map_err(|e| {
+                        format!("Failed to start tokenizer registration workflow: {:?}", e)
+                    })?;
+
+                debug!(
+                    "Started tokenizer registration workflow for '{}' id={} (instance: {})",
+                    config.name, config.id, instance_id
+                );
+
+                // Allow up to 10 minutes for HuggingFace downloads
+                let timeout_duration = Duration::from_secs(600);
+
+                engines
+                    .tokenizer
+                    .wait_for_completion(instance_id, &config.id, timeout_duration)
+                    .await
+            }
+            Job::RemoveTokenizer { request } => {
+                // Tokenizer removal is synchronous and fast
+                if let Some(entry) = context.tokenizer_registry.remove_by_id(&request.id) {
+                    info!(
+                        "Successfully removed tokenizer '{}' (id: {})",
+                        entry.name, entry.id
+                    );
+                    Ok(format!("Tokenizer '{}' removed successfully", entry.name))
+                } else {
+                    Err(format!("Tokenizer with id '{}' not found", request.id))
+                }
+            }
+        }
+    }
+
+    /// Update job status on completion
+    fn record_job_completion(
+        job_type: &'static str,
+        worker_url: &str,
+        _duration: Duration,
+        result: &Result<String, String>,
+        status_map: &Arc<DashMap<String, JobStatus>>,
+    ) {
+        match result {
+            Ok(message) => {
+                status_map.remove(worker_url);
+                debug!(
+                    "Completed job: type={}, worker={}, result={}",
+                    job_type, worker_url, message
+                );
+            }
+            Err(error) => {
+                status_map.insert(
+                    worker_url.to_string(),
+                    JobStatus::failed(job_type, worker_url, error.clone()),
+                );
+                warn!(
+                    "Failed job: type={}, worker={}, error={}",
+                    job_type, worker_url, error
+                );
+            }
+        }
+    }
+
+    /// Cleanup old job statuses (TTL 5 minutes)
+    async fn cleanup_old_statuses(status_map: Arc<DashMap<String, JobStatus>>) {
+        const CLEANUP_INTERVAL: Duration = Duration::from_secs(60); // Run every minute
+        const STATUS_TTL: u64 = 300; // 5 minutes in seconds
+
+        loop {
+            tokio::time::sleep(CLEANUP_INTERVAL).await;
+
+            let now = SystemTime::now()
+                .duration_since(SystemTime::UNIX_EPOCH)
+                .unwrap()
+                .as_secs();
+
+            // Remove statuses older than TTL
+            status_map.retain(|_key, value| now - value.timestamp < STATUS_TTL);
+
+            debug!(
+                "Cleaned up old job statuses, remaining: {}",
+                status_map.len()
+            );
+        }
+    }
+}

third_party/sglang/sgl-model-gateway/src/core/metrics_aggregator.rs

@@ -0,0 +1,91 @@
+use anyhow::ensure;
+use openmetrics_parser::{MetricFamily, MetricsExposition, PrometheusType, PrometheusValue};
+use tracing::warn;
+
+#[derive(Debug)]
+pub struct MetricPack {
+    pub labels: Vec<(String, String)>,
+    pub metrics_text: String,
+}
+
+type PrometheusExposition = MetricsExposition<PrometheusType, PrometheusValue>;
+type PrometheusFamily = MetricFamily<PrometheusType, PrometheusValue>;
+
+/// Aggregate Prometheus metrics scraped from multiple sources into a unified one
+pub fn aggregate_metrics(metric_packs: Vec<MetricPack>) -> anyhow::Result<String> {
+    let mut expositions = vec![];
+    for metric_pack in metric_packs {
+        let metrics_text = &metric_pack.metrics_text;
+        // openmetrics_parser doesn't handle colons in metric names; replace with underscores
+        let metrics_text = metrics_text.replace(":", "_");
+
+        let exposition = match openmetrics_parser::prometheus::parse_prometheus(&metrics_text) {
+            Ok(x) => x,
+            Err(err) => {
+                warn!(
+                    "aggregate_metrics error when parsing text: pack={:?} err={:?}",
+                    metric_pack, err
+                );
+                continue;
+            }
+        };
+        let exposition = transform_metrics(exposition, &metric_pack.labels);
+        expositions.push(exposition);
+    }
+
+    let text = try_reduce(expositions.into_iter(), merge_exposition)?
+        .map(|x| format!("{x}"))
+        .unwrap_or_default();
+    Ok(text)
+}
+
+fn transform_metrics(
+    mut exposition: PrometheusExposition,
+    extra_labels: &[(String, String)],
+) -> PrometheusExposition {
+    for family in exposition.families.values_mut() {
+        *family = family.with_labels(extra_labels.iter().map(|(k, v)| (k.as_str(), v.as_str())));
+    }
+    exposition
+}
+
+fn merge_exposition(
+    a: PrometheusExposition,
+    b: PrometheusExposition,
+) -> anyhow::Result<PrometheusExposition> {
+    let mut ans = a;
+    for (name, family_b) in b.families.into_iter() {
+        let family_merged = if let Some(family_a) = ans.families.remove(&name) {
+            merge_family(family_a, family_b)?
+        } else {
+            family_b
+        };
+        ans.families.insert(name, family_merged);
+    }
+    Ok(ans)
+}
+
+fn merge_family(a: PrometheusFamily, b: PrometheusFamily) -> anyhow::Result<PrometheusFamily> {
+    ensure!(
+        a.get_label_names() == b.get_label_names(),
+        "Label names should agree a={:?} b={:?}",
+        a.get_label_names(),
+        b.get_label_names()
+    );
+    a.with_samples(b.into_iter_samples())
+        .map_err(|e| anyhow::anyhow!("failed to merge samples: {e:?}"))
+}
+
+fn try_reduce<I, T, E, F>(iterable: I, f: F) -> Result<Option<T>, E>
+where
+    I: IntoIterator<Item = T>,
+    F: FnMut(T, T) -> Result<T, E>,
+{
+    let mut it = iterable.into_iter();
+    let first = match it.next() {
+        None => return Ok(None),
+        Some(x) => x,
+    };
+
+    Ok(Some(it.try_fold(first, f)?))
+}

third_party/sglang/sgl-model-gateway/src/core/mod.rs

@@ -0,0 +1,43 @@
+//! Core abstractions for the SGLang router
+//!
+//! This module contains the fundamental types and traits used throughout the router:
+//! - Worker trait and implementations
+//! - Model types and endpoint definitions
+//! - Error types
+//! - Circuit breaker for reliability
+//! - Token buckets for rate limiting
+//! - Workflow steps for multi-step operations
+//! - Common utilities
+
+// Re-export UNKNOWN_MODEL_ID from protocols for use throughout core
+pub use crate::protocols::UNKNOWN_MODEL_ID;
+
+pub mod circuit_breaker;
+pub mod error;
+pub mod job_queue;
+pub mod metrics_aggregator;
+pub mod model_card;
+pub mod model_type;
+pub mod retry;
+pub mod steps;
+pub mod token_bucket;
+pub mod worker;
+pub mod worker_builder;
+pub mod worker_manager;
+pub mod worker_registry;
+pub mod worker_service;
+
+// Re-export commonly used types for convenience
+pub use circuit_breaker::{CircuitBreaker, CircuitBreakerConfig};
+pub use error::{WorkerError, WorkerResult};
+pub use job_queue::{Job, JobQueue, JobQueueConfig};
+pub use model_card::{ModelCard, ProviderType};
+pub use retry::{is_retryable_status, RetryExecutor};
+pub use worker::{
+    AttachedBody, BasicWorker, ConnectionMode, HealthConfig, RuntimeType, Worker, WorkerLoadGuard,
+    WorkerType,
+};
+pub use worker_builder::{BasicWorkerBuilder, DPAwareWorkerBuilder};
+pub use worker_manager::{LoadMonitor, WorkerManager};
+pub use worker_registry::{HashRing, WorkerRegistry};
+pub use worker_service::WorkerService;

third_party/sglang/sgl-model-gateway/src/core/model_card.rs

@@ -0,0 +1,395 @@
+//! Model card definitions for worker model configuration.
+//!
+//! This module defines [`ModelCard`] which consolidates model-related configuration
+//! that was previously scattered in `WorkerMetadata.labels` HashMap.
+//!
+//! Also defines [`ProviderType`] for vendor-specific API transformations.
+//!
+//! Inspired by Dynamo's ModelDeploymentCard but simplified for router needs.
+
+use std::collections::HashMap;
+
+use serde::{Deserialize, Serialize};
+
+use super::{
+    model_type::{Endpoint, ModelType},
+    UNKNOWN_MODEL_ID,
+};
+
+/// Provider type for external API transformations.
+///
+/// Different providers have different API formats and requirements.
+/// This enum identifies which vendor's API format to use for transformations.
+///
+/// Note: `None` (when used as `Option<ProviderType>`) means native/passthrough -
+/// no transformation needed. This is the case for local SGLang backends.
+#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
+#[serde(rename_all = "lowercase")]
+pub enum ProviderType {
+    /// OpenAI API - strip SGLang-specific fields
+    #[serde(alias = "openai")]
+    OpenAI,
+    /// xAI/Grok - special handling for input items
+    #[serde(alias = "xai", alias = "grok")]
+    XAI,
+    /// Anthropic Claude - different API format
+    #[serde(alias = "anthropic", alias = "claude")]
+    Anthropic,
+    /// Google Gemini - special logprobs handling
+    #[serde(alias = "gemini", alias = "google")]
+    Gemini,
+    /// Custom provider with string identifier
+    #[serde(untagged)]
+    Custom(String),
+}
+
+impl ProviderType {
+    /// Get provider name as string
+    pub fn as_str(&self) -> &str {
+        match self {
+            Self::OpenAI => "openai",
+            Self::XAI => "xai",
+            Self::Anthropic => "anthropic",
+            Self::Gemini => "gemini",
+            Self::Custom(s) => s.as_str(),
+        }
+    }
+
+    /// Detect provider from model name (heuristic fallback).
+    /// Returns `None` for models that don't match known external providers
+    /// (i.e., models served by local/native backends).
+    pub fn from_model_name(model: &str) -> Option<Self> {
+        let model_lower = model.to_lowercase();
+        if model_lower.starts_with("grok") {
+            Some(Self::XAI)
+        } else if model_lower.starts_with("gemini") {
+            Some(Self::Gemini)
+        } else if model_lower.starts_with("claude") {
+            Some(Self::Anthropic)
+        } else if model_lower.starts_with("gpt")
+            || model_lower.starts_with("o1")
+            || model_lower.starts_with("o3")
+        {
+            Some(Self::OpenAI)
+        } else {
+            None // Native/local model, no provider transformation needed
+        }
+    }
+}
+
+impl std::fmt::Display for ProviderType {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "{}", self.as_str())
+    }
+}
+
+/// Model card containing model configuration and capabilities.
+///
+/// Consolidates fields previously scattered in `WorkerMetadata.labels`:
+/// - `model_id` -> `id`
+/// - `tokenizer_path` -> `tokenizer_path`
+/// - `chat_template` -> `chat_template`
+/// - `reasoning_parser` -> `reasoning_parser`
+/// - `tool_parser` -> `tool_parser`
+///
+/// # Example
+///
+/// ```
+/// use smg::core::{model_type::ModelType, ModelCard, ProviderType};
+///
+/// let card = ModelCard::new("meta-llama/Llama-3.1-8B-Instruct")
+///     .with_display_name("Llama 3.1 8B Instruct")
+///     .with_alias("llama-3.1-8b")
+///     .with_model_type(ModelType::VISION_LLM)
+///     .with_context_length(128_000)
+///     .with_tokenizer_path("meta-llama/Llama-3.1-8B-Instruct");
+///
+/// assert!(card.matches("llama-3.1-8b"));
+/// assert!(card.model_type.supports_vision());
+/// assert!(card.provider.is_none()); // Local model, no external provider
+/// ```
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct ModelCard {
+    // === Identity ===
+    /// Primary model ID (e.g., "meta-llama/Llama-3.1-8B-Instruct")
+    /// Previously: labels.get("model_id")
+    pub id: String,
+
+    /// Optional display name (e.g., "Llama 3.1 8B Instruct")
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub display_name: Option<String>,
+
+    /// Alternative names/aliases for this model
+    #[serde(default, skip_serializing_if = "Vec::is_empty")]
+    pub aliases: Vec<String>,
+
+    // === Capabilities ===
+    /// Supported endpoint types (bitflags)
+    #[serde(default = "default_model_type")]
+    pub model_type: ModelType,
+
+    /// HuggingFace model type string (e.g., "llama", "qwen2", "gpt-oss")
+    /// This is different from `model_type` which is capability bitflags.
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub hf_model_type: Option<String>,
+
+    /// Model architectures from HuggingFace config (e.g., ["LlamaForCausalLM"])
+    #[serde(default, skip_serializing_if = "Vec::is_empty")]
+    pub architectures: Vec<String>,
+
+    /// Provider hint for API transformations.
+    /// `None` means native/passthrough (no transformation needed).
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub provider: Option<ProviderType>,
+
+    /// Maximum context length in tokens
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub context_length: Option<u32>,
+
+    // === Tokenization & Parsing (previously in labels) ===
+    /// Path to tokenizer (e.g., HuggingFace model ID or local path)
+    /// Previously: labels.get("tokenizer_path")
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub tokenizer_path: Option<String>,
+
+    /// Chat template (Jinja2 template string or path)
+    /// Previously: labels.get("chat_template")
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub chat_template: Option<String>,
+
+    /// Reasoning parser type (e.g., "deepseek", "qwen")
+    /// Previously: labels.get("reasoning_parser")
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub reasoning_parser: Option<String>,
+
+    /// Tool/function calling parser type (e.g., "llama", "mistral")
+    /// Previously: labels.get("tool_parser")
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub tool_parser: Option<String>,
+
+    /// User-defined metadata (for fields not covered above)
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub metadata: Option<serde_json::Value>,
+
+    // === Classification Support ===
+    /// Classification label mapping (class index -> label name).
+    /// Empty if not a classification model.
+    /// Example: {0: "negative", 1: "positive"}
+    #[serde(default, skip_serializing_if = "HashMap::is_empty")]
+    pub id2label: HashMap<u32, String>,
+
+    /// Number of classification labels (0 if not a classifier).
+    #[serde(default, skip_serializing_if = "is_zero")]
+    pub num_labels: u32,
+}
+
+fn is_zero(n: &u32) -> bool {
+    *n == 0
+}
+
+fn default_model_type() -> ModelType {
+    ModelType::LLM
+}
+
+impl ModelCard {
+    /// Create a new model card with minimal configuration.
+    ///
+    /// Defaults to `ModelType::LLM` and no provider (native/passthrough).
+    pub fn new(id: impl Into<String>) -> Self {
+        Self {
+            id: id.into(),
+            display_name: None,
+            aliases: Vec::new(),
+            model_type: ModelType::LLM,
+            hf_model_type: None,
+            architectures: Vec::new(),
+            provider: None,
+            context_length: None,
+            tokenizer_path: None,
+            chat_template: None,
+            reasoning_parser: None,
+            tool_parser: None,
+            metadata: None,
+            id2label: HashMap::new(),
+            num_labels: 0,
+        }
+    }
+
+    // === Builder-style methods ===
+
+    /// Set the display name
+    pub fn with_display_name(mut self, name: impl Into<String>) -> Self {
+        self.display_name = Some(name.into());
+        self
+    }
+
+    /// Add a single alias
+    pub fn with_alias(mut self, alias: impl Into<String>) -> Self {
+        self.aliases.push(alias.into());
+        self
+    }
+
+    /// Add multiple aliases
+    pub fn with_aliases(mut self, aliases: impl IntoIterator<Item = impl Into<String>>) -> Self {
+        self.aliases.extend(aliases.into_iter().map(|a| a.into()));
+        self
+    }
+
+    /// Set the model type (capabilities)
+    pub fn with_model_type(mut self, model_type: ModelType) -> Self {
+        self.model_type = model_type;
+        self
+    }
+
+    /// Set the HuggingFace model type string
+    pub fn with_hf_model_type(mut self, hf_model_type: impl Into<String>) -> Self {
+        self.hf_model_type = Some(hf_model_type.into());
+        self
+    }
+
+    /// Set the model architectures
+    pub fn with_architectures(mut self, architectures: Vec<String>) -> Self {
+        self.architectures = architectures;
+        self
+    }
+
+    /// Set the provider type (for external API transformations)
+    pub fn with_provider(mut self, provider: ProviderType) -> Self {
+        self.provider = Some(provider);
+        self
+    }
+
+    /// Set the context length
+    pub fn with_context_length(mut self, length: u32) -> Self {
+        self.context_length = Some(length);
+        self
+    }
+
+    /// Set the tokenizer path
+    pub fn with_tokenizer_path(mut self, path: impl Into<String>) -> Self {
+        self.tokenizer_path = Some(path.into());
+        self
+    }
+
+    /// Set the chat template
+    pub fn with_chat_template(mut self, template: impl Into<String>) -> Self {
+        self.chat_template = Some(template.into());
+        self
+    }
+
+    /// Set the reasoning parser type
+    pub fn with_reasoning_parser(mut self, parser: impl Into<String>) -> Self {
+        self.reasoning_parser = Some(parser.into());
+        self
+    }
+
+    /// Set the tool parser type
+    pub fn with_tool_parser(mut self, parser: impl Into<String>) -> Self {
+        self.tool_parser = Some(parser.into());
+        self
+    }
+
+    /// Set custom metadata
+    pub fn with_metadata(mut self, metadata: serde_json::Value) -> Self {
+        self.metadata = Some(metadata);
+        self
+    }
+
+    /// Set the id2label mapping for classification models
+    pub fn with_id2label(mut self, id2label: HashMap<u32, String>) -> Self {
+        self.num_labels = id2label.len() as u32;
+        self.id2label = id2label;
+        self
+    }
+
+    /// Set num_labels directly (alternative to with_id2label)
+    pub fn with_num_labels(mut self, num_labels: u32) -> Self {
+        self.num_labels = num_labels;
+        self
+    }
+
+    // === Query methods ===
+
+    /// Check if this model matches the given ID (including aliases)
+    pub fn matches(&self, model_id: &str) -> bool {
+        self.id == model_id || self.aliases.iter().any(|a| a == model_id)
+    }
+
+    /// Check if this model supports a given endpoint
+    pub fn supports_endpoint(&self, endpoint: Endpoint) -> bool {
+        self.model_type.supports_endpoint(endpoint)
+    }
+
+    /// Get the display name or fall back to ID
+    pub fn name(&self) -> &str {
+        self.display_name.as_deref().unwrap_or(&self.id)
+    }
+
+    /// Check if this is a native/local model (no external provider)
+    #[inline]
+    pub fn is_native(&self) -> bool {
+        self.provider.is_none()
+    }
+
+    /// Check if this model uses an external provider
+    #[inline]
+    pub fn has_external_provider(&self) -> bool {
+        self.provider.is_some()
+    }
+
+    /// Check if this is an LLM (supports chat)
+    #[inline]
+    pub fn is_llm(&self) -> bool {
+        self.model_type.is_llm()
+    }
+
+    /// Check if this is an embedding model
+    #[inline]
+    pub fn is_embedding_model(&self) -> bool {
+        self.model_type.is_embedding_model()
+    }
+
+    /// Check if this model supports vision/multimodal
+    #[inline]
+    pub fn supports_vision(&self) -> bool {
+        self.model_type.supports_vision()
+    }
+
+    /// Check if this model supports tools/function calling
+    #[inline]
+    pub fn supports_tools(&self) -> bool {
+        self.model_type.supports_tools()
+    }
+
+    /// Check if this model supports reasoning
+    #[inline]
+    pub fn supports_reasoning(&self) -> bool {
+        self.model_type.supports_reasoning()
+    }
+
+    /// Check if this is a classification model
+    #[inline]
+    pub fn is_classifier(&self) -> bool {
+        self.num_labels > 0
+    }
+
+    /// Get label for a class index, with fallback to generic label (LABEL_N)
+    pub fn get_label(&self, class_idx: u32) -> String {
+        self.id2label
+            .get(&class_idx)
+            .cloned()
+            .unwrap_or_else(|| format!("LABEL_{}", class_idx))
+    }
+}
+
+impl Default for ModelCard {
+    fn default() -> Self {
+        Self::new(UNKNOWN_MODEL_ID)
+    }
+}
+
+impl std::fmt::Display for ModelCard {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "{}", self.name())
+    }
+}

third_party/sglang/sgl-model-gateway/src/core/model_type.rs

@@ -0,0 +1,332 @@
+//! Model type definitions using bitflags for endpoint support.
+//!
+//! This module defines [`ModelType`] using bitflags to represent which endpoints
+//! a model can support. This allows combining capabilities like
+//! `ModelType::CHAT | ModelType::COMPLETIONS`.
+//!
+//! Inspired by Dynamo's model_type.rs implementation.
+
+use bitflags::bitflags;
+use serde::{Deserialize, Serialize};
+
+bitflags! {
+    #[derive(Copy, Debug, Default, Clone, Eq, PartialEq, Hash)]
+    pub struct ModelType: u16 {
+        /// OpenAI Chat Completions API (/v1/chat/completions)
+        const CHAT        = 1 << 0;
+        /// OpenAI Completions API - legacy (/v1/completions)
+        const COMPLETIONS = 1 << 1;
+        /// OpenAI Responses API (/v1/responses)
+        const RESPONSES   = 1 << 2;
+        /// Embeddings API (/v1/embeddings)
+        const EMBEDDINGS  = 1 << 3;
+        /// Rerank API (/v1/rerank)
+        const RERANK      = 1 << 4;
+        /// SGLang Generate API (/generate)
+        const GENERATE    = 1 << 5;
+        /// Vision/multimodal support (images in input)
+        const VISION      = 1 << 6;
+        /// Tool/function calling support
+        const TOOLS       = 1 << 7;
+        /// Reasoning/thinking support (e.g., o1, DeepSeek-R1)
+        const REASONING   = 1 << 8;
+        /// Image generation (DALL-E, Sora, gpt-image)
+        const IMAGE_GEN   = 1 << 9;
+        /// Audio models (TTS, Whisper, realtime, transcribe)
+        const AUDIO       = 1 << 10;
+        /// Content moderation models
+        const MODERATION  = 1 << 11;
+
+        /// Standard LLM: chat + completions + responses + tools
+        const LLM = Self::CHAT.bits() | Self::COMPLETIONS.bits()
+                  | Self::RESPONSES.bits() | Self::TOOLS.bits();
+
+        /// Vision-capable LLM: LLM + vision
+        const VISION_LLM = Self::LLM.bits() | Self::VISION.bits();
+
+        /// Reasoning LLM: LLM + reasoning (e.g., o1, o3, DeepSeek-R1)
+        const REASONING_LLM = Self::LLM.bits() | Self::REASONING.bits();
+
+        /// Full-featured LLM: all text generation capabilities
+        const FULL_LLM = Self::VISION_LLM.bits() | Self::REASONING.bits();
+
+        /// Embedding model only
+        const EMBED_MODEL = Self::EMBEDDINGS.bits();
+
+        /// Reranker model only
+        const RERANK_MODEL = Self::RERANK.bits();
+
+        /// Image generation model only (DALL-E, Sora, gpt-image)
+        const IMAGE_MODEL = Self::IMAGE_GEN.bits();
+
+        /// Audio model only (TTS, Whisper, realtime)
+        const AUDIO_MODEL = Self::AUDIO.bits();
+
+        /// Content moderation model only
+        const MODERATION_MODEL = Self::MODERATION.bits();
+    }
+}
+
+/// Mapping of individual capability flags to their names.
+/// Used by `as_capability_names()` for a data-driven approach.
+const CAPABILITY_NAMES: &[(ModelType, &str)] = &[
+    (ModelType::CHAT, "chat"),
+    (ModelType::COMPLETIONS, "completions"),
+    (ModelType::RESPONSES, "responses"),
+    (ModelType::EMBEDDINGS, "embeddings"),
+    (ModelType::RERANK, "rerank"),
+    (ModelType::GENERATE, "generate"),
+    (ModelType::VISION, "vision"),
+    (ModelType::TOOLS, "tools"),
+    (ModelType::REASONING, "reasoning"),
+    (ModelType::IMAGE_GEN, "image_gen"),
+    (ModelType::AUDIO, "audio"),
+    (ModelType::MODERATION, "moderation"),
+];
+
+impl ModelType {
+    /// Check if this model type supports the chat completions endpoint
+    #[inline]
+    pub fn supports_chat(&self) -> bool {
+        self.contains(Self::CHAT)
+    }
+
+    /// Check if this model type supports the legacy completions endpoint
+    #[inline]
+    pub fn supports_completions(&self) -> bool {
+        self.contains(Self::COMPLETIONS)
+    }
+
+    /// Check if this model type supports the responses endpoint
+    #[inline]
+    pub fn supports_responses(&self) -> bool {
+        self.contains(Self::RESPONSES)
+    }
+
+    /// Check if this model type supports the embeddings endpoint
+    #[inline]
+    pub fn supports_embeddings(&self) -> bool {
+        self.contains(Self::EMBEDDINGS)
+    }
+
+    /// Check if this model type supports the rerank endpoint
+    #[inline]
+    pub fn supports_rerank(&self) -> bool {
+        self.contains(Self::RERANK)
+    }
+
+    /// Check if this model type supports the generate endpoint
+    #[inline]
+    pub fn supports_generate(&self) -> bool {
+        self.contains(Self::GENERATE)
+    }
+
+    /// Check if this model type supports vision/multimodal input
+    #[inline]
+    pub fn supports_vision(&self) -> bool {
+        self.contains(Self::VISION)
+    }
+
+    /// Check if this model type supports tool/function calling
+    #[inline]
+    pub fn supports_tools(&self) -> bool {
+        self.contains(Self::TOOLS)
+    }
+
+    /// Check if this model type supports reasoning/thinking
+    #[inline]
+    pub fn supports_reasoning(&self) -> bool {
+        self.contains(Self::REASONING)
+    }
+
+    /// Check if this model type supports image generation
+    #[inline]
+    pub fn supports_image_gen(&self) -> bool {
+        self.contains(Self::IMAGE_GEN)
+    }
+
+    /// Check if this model type supports audio (TTS, Whisper, etc.)
+    #[inline]
+    pub fn supports_audio(&self) -> bool {
+        self.contains(Self::AUDIO)
+    }
+
+    /// Check if this model type supports content moderation
+    #[inline]
+    pub fn supports_moderation(&self) -> bool {
+        self.contains(Self::MODERATION)
+    }
+
+    /// Check if this model type supports a given endpoint
+    pub fn supports_endpoint(&self, endpoint: Endpoint) -> bool {
+        match endpoint {
+            Endpoint::Chat => self.supports_chat(),
+            Endpoint::Completions => self.supports_completions(),
+            Endpoint::Responses => self.supports_responses(),
+            Endpoint::Embeddings => self.supports_embeddings(),
+            Endpoint::Rerank => self.supports_rerank(),
+            Endpoint::Generate => self.supports_generate(),
+            Endpoint::Models => true, // Models endpoint is always supported
+        }
+    }
+
+    /// Convert to a list of supported capability names
+    pub fn as_capability_names(&self) -> Vec<&'static str> {
+        let mut result = Vec::with_capacity(CAPABILITY_NAMES.len());
+        for &(flag, name) in CAPABILITY_NAMES {
+            if self.contains(flag) {
+                result.push(name);
+            }
+        }
+        result
+    }
+
+    /// Check if this is an LLM (supports at least chat)
+    #[inline]
+    pub fn is_llm(&self) -> bool {
+        self.supports_chat()
+    }
+
+    /// Check if this is an embedding model
+    #[inline]
+    pub fn is_embedding_model(&self) -> bool {
+        self.supports_embeddings() && !self.supports_chat()
+    }
+
+    /// Check if this is a reranker model
+    #[inline]
+    pub fn is_reranker(&self) -> bool {
+        self.supports_rerank() && !self.supports_chat()
+    }
+
+    /// Check if this is an image generation model
+    #[inline]
+    pub fn is_image_model(&self) -> bool {
+        self.supports_image_gen() && !self.supports_chat()
+    }
+
+    /// Check if this is an audio model
+    #[inline]
+    pub fn is_audio_model(&self) -> bool {
+        self.supports_audio() && !self.supports_chat()
+    }
+
+    /// Check if this is a moderation model
+    #[inline]
+    pub fn is_moderation_model(&self) -> bool {
+        self.supports_moderation() && !self.supports_chat()
+    }
+}
+
+impl std::fmt::Display for ModelType {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        let names = self.as_capability_names();
+        if names.is_empty() {
+            write!(f, "none")
+        } else {
+            write!(f, "{}", names.join(","))
+        }
+    }
+}
+
+// Custom Serialize/Deserialize for ModelType to handle bitflags properly
+impl Serialize for ModelType {
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: serde::Serializer,
+    {
+        // Serialize as the underlying u16 bits
+        serializer.serialize_u16(self.bits())
+    }
+}
+
+impl<'de> Deserialize<'de> for ModelType {
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where
+        D: serde::Deserializer<'de>,
+    {
+        let bits = u16::deserialize(deserializer)?;
+        ModelType::from_bits(bits)
+            .ok_or_else(|| serde::de::Error::custom(format!("invalid ModelType bits: {}", bits)))
+    }
+}
+
+/// Endpoint types for routing decisions.
+///
+/// This enum represents the different API endpoints that can be routed to workers.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
+#[serde(rename_all = "lowercase")]
+pub enum Endpoint {
+    /// Chat completions endpoint (/v1/chat/completions)
+    Chat,
+    /// Legacy completions endpoint (/v1/completions)
+    Completions,
+    /// Responses endpoint (/v1/responses)
+    Responses,
+    /// Embeddings endpoint (/v1/embeddings)
+    Embeddings,
+    /// Rerank endpoint (/v1/rerank)
+    Rerank,
+    /// SGLang generate endpoint (/generate)
+    Generate,
+    /// Models listing endpoint (/v1/models)
+    Models,
+}
+
+impl Endpoint {
+    /// Get the URL path for this endpoint
+    pub fn path(&self) -> &'static str {
+        match self {
+            Endpoint::Chat => "/v1/chat/completions",
+            Endpoint::Completions => "/v1/completions",
+            Endpoint::Responses => "/v1/responses",
+            Endpoint::Embeddings => "/v1/embeddings",
+            Endpoint::Rerank => "/v1/rerank",
+            Endpoint::Generate => "/generate",
+            Endpoint::Models => "/v1/models",
+        }
+    }
+
+    /// Parse an endpoint from a URL path
+    pub fn from_path(path: &str) -> Option<Self> {
+        // Normalize: strip trailing slash and match
+        let path = path.trim_end_matches('/');
+        match path {
+            "/v1/chat/completions" => Some(Endpoint::Chat),
+            "/v1/completions" => Some(Endpoint::Completions),
+            "/v1/responses" => Some(Endpoint::Responses),
+            "/v1/embeddings" => Some(Endpoint::Embeddings),
+            "/v1/rerank" => Some(Endpoint::Rerank),
+            "/generate" => Some(Endpoint::Generate),
+            "/v1/models" => Some(Endpoint::Models),
+            _ => None,
+        }
+    }
+
+    /// Get the required ModelType flag for this endpoint
+    pub fn required_capability(&self) -> Option<ModelType> {
+        match self {
+            Endpoint::Chat => Some(ModelType::CHAT),
+            Endpoint::Completions => Some(ModelType::COMPLETIONS),
+            Endpoint::Responses => Some(ModelType::RESPONSES),
+            Endpoint::Embeddings => Some(ModelType::EMBEDDINGS),
+            Endpoint::Rerank => Some(ModelType::RERANK),
+            Endpoint::Generate => Some(ModelType::GENERATE),
+            Endpoint::Models => None, // No specific capability required
+        }
+    }
+}
+
+impl std::fmt::Display for Endpoint {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            Endpoint::Chat => write!(f, "chat"),
+            Endpoint::Completions => write!(f, "completions"),
+            Endpoint::Responses => write!(f, "responses"),
+            Endpoint::Embeddings => write!(f, "embeddings"),
+            Endpoint::Rerank => write!(f, "rerank"),
+            Endpoint::Generate => write!(f, "generate"),
+            Endpoint::Models => write!(f, "models"),
+        }
+    }
+}

third_party/sglang/sgl-model-gateway/src/core/retry.rs

@@ -0,0 +1,320 @@
+use std::time::Duration;
+
+use axum::{http::StatusCode, response::Response};
+use rand::Rng;
+use tracing::debug;
+
+use crate::config::types::RetryConfig;
+
+/// Check if an HTTP status code indicates a retryable error
+pub fn is_retryable_status(status: StatusCode) -> bool {
+    matches!(
+        status,
+        StatusCode::REQUEST_TIMEOUT
+            | StatusCode::TOO_MANY_REQUESTS
+            | StatusCode::INTERNAL_SERVER_ERROR
+            | StatusCode::BAD_GATEWAY
+            | StatusCode::SERVICE_UNAVAILABLE
+            | StatusCode::GATEWAY_TIMEOUT
+    )
+}
+
+/// Computes exponential backoff with optional jitter.
+#[derive(Debug, Clone)]
+pub struct BackoffCalculator;
+
+impl BackoffCalculator {
+    /// Calculate backoff delay for a given attempt index (0-based).
+    pub fn calculate_delay(config: &RetryConfig, attempt: u32) -> Duration {
+        let pow = config.backoff_multiplier.powi(attempt as i32);
+        let delay_ms = ((config.initial_backoff_ms as f32 * pow) as u64).min(config.max_backoff_ms);
+
+        let jitter = config.jitter_factor.clamp(0.0, 1.0);
+        if jitter > 0.0 {
+            let mut rng = rand::rng();
+            let jitter_scale: f32 = rng.random_range(-jitter..=jitter);
+            let jitter_ms = (delay_ms as f32 * jitter_scale)
+                .round()
+                .max(-(delay_ms as f32));
+            let adjusted = (delay_ms as i64 + jitter_ms as i64).max(0) as u64;
+            return Duration::from_millis(adjusted);
+        }
+
+        Duration::from_millis(delay_ms)
+    }
+}
+
+#[derive(Debug, thiserror::Error)]
+pub enum RetryError {
+    #[error("maximum retry attempts exceeded")]
+    MaxRetriesExceeded,
+}
+
+/// A thin async retry executor for generic operations.
+#[derive(Debug, Clone, Default)]
+pub struct RetryExecutor;
+
+impl RetryExecutor {
+    /// Execute an operation that returns an HTTP Response with retries and backoff.
+    ///
+    /// Usage pattern:
+    /// - `operation(attempt)`: perform one attempt (0-based). Construct and send the request,
+    ///   then return the `Response`. Do any per-attempt bookkeeping (e.g., load tracking,
+    ///   circuit-breaker outcome recording) inside this closure.
+    /// - `should_retry(&response, attempt)`: decide if the given response should be retried
+    ///   (e.g., based on HTTP status). Returning false short-circuits and returns the response.
+    /// - `on_backoff(delay, next_attempt)`: called before sleeping between attempts.
+    ///   Use this to record metrics.
+    /// - `on_exhausted()`: called when the executor has exhausted all retry attempts.
+    ///
+    /// Example:
+    /// ```ignore
+    /// let resp = RetryExecutor::execute_response_with_retry(
+    ///     &retry_cfg,
+    ///     |attempt| async move {
+    ///         let worker = select_cb_aware_worker()?;
+    ///         let resp = send_request(worker).await;
+    ///         worker.record_outcome(resp.status().is_success());
+    ///         resp
+    ///     },
+    ///     |res, _| matches!(res.status(), StatusCode::REQUEST_TIMEOUT | StatusCode::TOO_MANY_REQUESTS | StatusCode::INTERNAL_SERVER_ERROR | StatusCode::BAD_GATEWAY | StatusCode::SERVICE_UNAVAILABLE | StatusCode::GATEWAY_TIMEOUT),
+    ///     |delay, _attempt| { /* record backoff metrics */ },
+    ///     || { /* record retries exhausted */ },
+    /// ).await;
+    /// ```
+    pub async fn execute_response_with_retry<Op, Fut, ShouldRetry, OnBackoff, OnExhausted>(
+        config: &RetryConfig,
+        mut operation: Op,
+        should_retry: ShouldRetry,
+        on_backoff: OnBackoff,
+        mut on_exhausted: OnExhausted,
+    ) -> Response
+    where
+        Op: FnMut(u32) -> Fut,
+        Fut: std::future::Future<Output = Response>,
+        ShouldRetry: Fn(&Response, u32) -> bool,
+        OnBackoff: Fn(Duration, u32),
+        OnExhausted: FnMut(),
+    {
+        let max = config.max_retries.max(1);
+
+        let mut attempt: u32 = 0;
+        loop {
+            let response = operation(attempt).await;
+            let is_last = attempt + 1 >= max;
+
+            if !should_retry(&response, attempt) {
+                return response;
+            }
+
+            if is_last {
+                on_exhausted();
+                return response;
+            }
+
+            let next_attempt = attempt + 1;
+            let delay = BackoffCalculator::calculate_delay(config, attempt);
+            debug!(
+                attempt = attempt,
+                next_attempt = next_attempt,
+                delay_ms = delay.as_millis() as u64,
+                "Retry backoff"
+            );
+            on_backoff(delay, next_attempt);
+            tokio::time::sleep(delay).await;
+
+            attempt = next_attempt;
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::sync::{
+        atomic::{AtomicU32, Ordering},
+        Arc,
+    };
+
+    use axum::{http::StatusCode, response::IntoResponse};
+
+    use super::*;
+
+    fn base_retry_config() -> RetryConfig {
+        RetryConfig {
+            max_retries: 3,
+            initial_backoff_ms: 1,
+            max_backoff_ms: 4,
+            backoff_multiplier: 2.0,
+            jitter_factor: 0.0,
+        }
+    }
+
+    #[test]
+    fn test_backoff_no_jitter_progression_and_cap() {
+        let cfg = RetryConfig {
+            max_retries: 10,
+            initial_backoff_ms: 100,
+            max_backoff_ms: 250,
+            backoff_multiplier: 2.0,
+            jitter_factor: 0.0,
+        };
+        assert_eq!(
+            BackoffCalculator::calculate_delay(&cfg, 0),
+            Duration::from_millis(100)
+        );
+        assert_eq!(
+            BackoffCalculator::calculate_delay(&cfg, 1),
+            Duration::from_millis(200)
+        );
+        assert_eq!(
+            BackoffCalculator::calculate_delay(&cfg, 2),
+            Duration::from_millis(250)
+        );
+        assert_eq!(
+            BackoffCalculator::calculate_delay(&cfg, 10),
+            Duration::from_millis(250)
+        );
+    }
+
+    #[test]
+    fn test_backoff_with_jitter_within_bounds() {
+        let cfg = RetryConfig {
+            max_retries: 5,
+            initial_backoff_ms: 100,
+            max_backoff_ms: 10_000,
+            backoff_multiplier: 2.0,
+            jitter_factor: 0.5,
+        };
+        let base = 400.0;
+        for _ in 0..50 {
+            let d = BackoffCalculator::calculate_delay(&cfg, 2).as_millis() as f32;
+            assert!(d >= base * 0.5 - 1.0 && d <= base * 1.5 + 1.0);
+        }
+    }
+
+    #[tokio::test]
+    async fn test_execute_response_with_retry_success_path_and_hooks() {
+        let cfg = base_retry_config();
+        let remaining = Arc::new(AtomicU32::new(2));
+        let calls = Arc::new(AtomicU32::new(0));
+        let backoffs = Arc::new(AtomicU32::new(0));
+        let exhausted = Arc::new(AtomicU32::new(0));
+
+        let response = RetryExecutor::execute_response_with_retry(
+            &cfg,
+            {
+                let remaining = remaining.clone();
+                let calls = calls.clone();
+                move |_attempt| {
+                    calls.fetch_add(1, Ordering::Relaxed);
+                    let remaining = remaining.clone();
+                    async move {
+                        if remaining
+                            .fetch_update(Ordering::AcqRel, Ordering::Acquire, |v| v.checked_sub(1))
+                            .is_ok()
+                        {
+                            (StatusCode::SERVICE_UNAVAILABLE, "fail").into_response()
+                        } else {
+                            (StatusCode::OK, "ok").into_response()
+                        }
+                    }
+                }
+            },
+            |res, _attempt| !res.status().is_success(),
+            {
+                let backoffs = backoffs.clone();
+                move |_delay, _next_attempt| {
+                    backoffs.fetch_add(1, Ordering::Relaxed);
+                }
+            },
+            {
+                let exhausted = exhausted.clone();
+                move || {
+                    exhausted.fetch_add(1, Ordering::Relaxed);
+                }
+            },
+        )
+        .await;
+
+        assert_eq!(response.status(), StatusCode::OK);
+        assert_eq!(calls.load(Ordering::Relaxed), 3);
+        assert_eq!(backoffs.load(Ordering::Relaxed), 2);
+        assert_eq!(exhausted.load(Ordering::Relaxed), 0);
+    }
+
+    #[tokio::test]
+    async fn test_execute_response_with_retry_non_retryable_short_circuit() {
+        let cfg = base_retry_config();
+        let calls = Arc::new(AtomicU32::new(0));
+        let backoffs = Arc::new(AtomicU32::new(0));
+        let exhausted = Arc::new(AtomicU32::new(0));
+
+        let response = RetryExecutor::execute_response_with_retry(
+            &cfg,
+            {
+                let calls = calls.clone();
+                move |_attempt| {
+                    calls.fetch_add(1, Ordering::Relaxed);
+                    async move { (StatusCode::BAD_REQUEST, "bad").into_response() }
+                }
+            },
+            |_res, _attempt| false,
+            {
+                let backoffs = backoffs.clone();
+                move |_delay, _next_attempt| {
+                    backoffs.fetch_add(1, Ordering::Relaxed);
+                }
+            },
+            {
+                let exhausted = exhausted.clone();
+                move || {
+                    exhausted.fetch_add(1, Ordering::Relaxed);
+                }
+            },
+        )
+        .await;
+
+        assert_eq!(response.status(), StatusCode::BAD_REQUEST);
+        assert_eq!(calls.load(Ordering::Relaxed), 1);
+        assert_eq!(backoffs.load(Ordering::Relaxed), 0);
+        assert_eq!(exhausted.load(Ordering::Relaxed), 0);
+    }
+
+    #[tokio::test]
+    async fn test_execute_response_with_retry_exhausted_hooks() {
+        let cfg = base_retry_config();
+        let calls = Arc::new(AtomicU32::new(0));
+        let backoffs = Arc::new(AtomicU32::new(0));
+        let exhausted = Arc::new(AtomicU32::new(0));
+
+        let response = RetryExecutor::execute_response_with_retry(
+            &cfg,
+            {
+                let calls = calls.clone();
+                move |_attempt| {
+                    calls.fetch_add(1, Ordering::Relaxed);
+                    async move { (StatusCode::SERVICE_UNAVAILABLE, "fail").into_response() }
+                }
+            },
+            |_res, _attempt| true,
+            {
+                let backoffs = backoffs.clone();
+                move |_delay, _next_attempt| {
+                    backoffs.fetch_add(1, Ordering::Relaxed);
+                }
+            },
+            {
+                let exhausted = exhausted.clone();
+                move || {
+                    exhausted.fetch_add(1, Ordering::Relaxed);
+                }
+            },
+        )
+        .await;
+
+        assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE);
+        assert_eq!(calls.load(Ordering::Relaxed), cfg.max_retries);
+        assert_eq!(backoffs.load(Ordering::Relaxed), cfg.max_retries - 1);
+        assert_eq!(exhausted.load(Ordering::Relaxed), 1);
+    }
+}

third_party/sglang/sgl-model-gateway/src/core/steps/mcp_registration.rs

@@ -0,0 +1,326 @@
+use std::{sync::Arc, time::Duration};
+
+use async_trait::async_trait;
+use smg_mcp::{config::McpServerConfig, manager::McpManager};
+use tracing::{debug, error, info, warn};
+use wfaas::{
+    BackoffStrategy, FailureAction, RetryPolicy, StepDefinition, StepExecutor, StepId, StepResult,
+    WorkflowContext, WorkflowDefinition, WorkflowError, WorkflowResult,
+};
+
+use super::workflow_data::McpWorkflowData;
+use crate::{app_context::AppContext, observability::metrics::Metrics};
+
+/// MCP server connection configuration
+#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
+pub struct McpServerConfigRequest {
+    /// Server name (unique identifier)
+    pub name: String,
+    /// Server configuration (transport, proxy, etc.)
+    pub config: McpServerConfig,
+}
+
+impl McpServerConfigRequest {
+    /// Check if this server is required for router startup
+    pub fn is_required(&self) -> bool {
+        self.config.required
+    }
+}
+
+/// Step 1: Connect to MCP server
+///
+/// This step establishes a connection to the MCP server using the flat manager architecture.
+/// The connection is retried aggressively (100 attempts) with a long timeout (2 hours)
+/// to handle slow-starting servers or network issues.
+pub struct ConnectMcpServerStep;
+
+#[async_trait]
+impl StepExecutor<McpWorkflowData> for ConnectMcpServerStep {
+    async fn execute(
+        &self,
+        context: &mut WorkflowContext<McpWorkflowData>,
+    ) -> WorkflowResult<StepResult> {
+        let config_request = &context.data.config;
+        let app_context = context
+            .data
+            .app_context
+            .as_ref()
+            .ok_or_else(|| WorkflowError::ContextValueNotFound("app_context".to_string()))?;
+
+        debug!("Connecting to MCP server: {}", config_request.name);
+
+        // Get proxy config from router_config if available, otherwise fall back to env
+        let proxy_config = app_context
+            .router_config
+            .mcp_config
+            .as_ref()
+            .and_then(|cfg| cfg.proxy.as_ref());
+
+        // Connect to MCP server
+        let client = McpManager::connect_server(&config_request.config, proxy_config)
+            .await
+            .map_err(|e| WorkflowError::StepFailed {
+                step_id: StepId::new("connect_mcp_server"),
+                message: format!(
+                    "Failed to connect to MCP server {}: {}",
+                    config_request.name, e
+                ),
+            })?;
+
+        info!(
+            "Successfully connected to MCP server: {}",
+            config_request.name
+        );
+
+        // Store client in typed data
+        context.data.mcp_client = Some(Arc::new(client));
+
+        Ok(StepResult::Success)
+    }
+
+    fn is_retryable(&self, _error: &WorkflowError) -> bool {
+        true // Connection failures are retryable
+    }
+}
+
+/// Step 2: Discover MCP inventory (tools, prompts, resources)
+///
+/// This step queries the MCP server for its capabilities using McpManager::load_server_inventory().
+/// - Tools: Available function calls
+/// - Prompts: Reusable prompt templates
+/// - Resources: Accessible files/data
+pub struct DiscoverMcpInventoryStep;
+
+#[async_trait]
+impl StepExecutor<McpWorkflowData> for DiscoverMcpInventoryStep {
+    async fn execute(
+        &self,
+        context: &mut WorkflowContext<McpWorkflowData>,
+    ) -> WorkflowResult<StepResult> {
+        let config_request = &context.data.config;
+        let app_context = context
+            .data
+            .app_context
+            .as_ref()
+            .ok_or_else(|| WorkflowError::ContextValueNotFound("app_context".to_string()))?;
+        let mcp_client = context
+            .data
+            .mcp_client
+            .as_ref()
+            .ok_or_else(|| WorkflowError::ContextValueNotFound("mcp_client".to_string()))?;
+
+        debug!(
+            "Discovering inventory for MCP server: {}",
+            config_request.name
+        );
+
+        // Get shared ToolInventory from McpManager
+        let mcp_manager =
+            app_context
+                .mcp_manager
+                .get()
+                .ok_or_else(|| WorkflowError::StepFailed {
+                    step_id: StepId::new("discover_mcp_inventory"),
+                    message: "MCP manager not initialized".to_string(),
+                })?;
+
+        let inventory = mcp_manager.inventory();
+
+        // Use the public load_server_inventory method
+        McpManager::load_server_inventory(&inventory, &config_request.name, mcp_client).await;
+
+        info!("Completed inventory discovery for {}", config_request.name);
+
+        Ok(StepResult::Success)
+    }
+
+    fn is_retryable(&self, _error: &WorkflowError) -> bool {
+        true // Discovery failures are retryable
+    }
+}
+
+/// Step 3: Register MCP server in manager
+///
+/// This step adds the MCP client to the McpManager's client map so it can be
+/// used for tool calls and inventory management.
+pub struct RegisterMcpServerStep;
+
+#[async_trait]
+impl StepExecutor<McpWorkflowData> for RegisterMcpServerStep {
+    async fn execute(
+        &self,
+        context: &mut WorkflowContext<McpWorkflowData>,
+    ) -> WorkflowResult<StepResult> {
+        let config_request = &context.data.config;
+        let app_context = context
+            .data
+            .app_context
+            .as_ref()
+            .ok_or_else(|| WorkflowError::ContextValueNotFound("app_context".to_string()))?;
+        let mcp_client = context
+            .data
+            .mcp_client
+            .as_ref()
+            .ok_or_else(|| WorkflowError::ContextValueNotFound("mcp_client".to_string()))?
+            .clone();
+
+        debug!("Registering MCP server: {}", config_request.name);
+
+        // Get MCP manager from app context
+        let mcp_manager =
+            app_context
+                .mcp_manager
+                .get()
+                .ok_or_else(|| WorkflowError::StepFailed {
+                    step_id: StepId::new("register_mcp_server"),
+                    message: "MCP manager not initialized".to_string(),
+                })?;
+
+        // Register the client in the manager's client map
+        mcp_manager.register_static_server(config_request.name.clone(), mcp_client);
+
+        // Update active MCP servers metric
+        Metrics::set_mcp_servers_active(mcp_manager.list_servers().len());
+
+        info!("Registered MCP server: {}", config_request.name);
+
+        Ok(StepResult::Success)
+    }
+
+    fn is_retryable(&self, _error: &WorkflowError) -> bool {
+        false // Registration is a simple operation, not retryable
+    }
+}
+
+/// Step 4: Validate registration based on required flag
+///
+/// This step checks if the server is marked as required. If the server is required
+/// but wasn't successfully registered (client not in context), this step fails the workflow.
+/// For optional servers, this step always succeeds, allowing the workflow to complete
+/// even if earlier steps failed.
+pub struct ValidateRegistrationStep;
+
+#[async_trait]
+impl StepExecutor<McpWorkflowData> for ValidateRegistrationStep {
+    async fn execute(
+        &self,
+        context: &mut WorkflowContext<McpWorkflowData>,
+    ) -> WorkflowResult<StepResult> {
+        let config_request = &context.data.config;
+        let client_registered = context.data.mcp_client.is_some();
+
+        if client_registered {
+            info!(
+                "MCP server '{}' registered successfully",
+                config_request.name
+            );
+
+            // Mark as validated
+            context.data.validated = true;
+
+            return Ok(StepResult::Success);
+        }
+
+        if config_request.is_required() {
+            error!(
+                "Required MCP server '{}' failed to register",
+                config_request.name
+            );
+            Err(WorkflowError::StepFailed {
+                step_id: StepId::new("validate_registration"),
+                message: format!(
+                    "Required MCP server '{}' failed to register",
+                    config_request.name
+                ),
+            })
+        } else {
+            warn!(
+                "Optional MCP server '{}' failed to register, continuing workflow",
+                config_request.name
+            );
+            Ok(StepResult::Success)
+        }
+    }
+
+    fn is_retryable(&self, _error: &WorkflowError) -> bool {
+        false
+    }
+}
+
+/// Create MCP server registration workflow
+///
+/// This workflow adapts its failure behavior based on the `required` field in the server config:
+/// - If `required == true`: Uses FailWorkflow - router startup fails if server cannot be reached
+/// - If `required == false` (default): Uses ContinueNextStep - logs warning but continues
+///
+/// Workflow configuration:
+/// - ConnectMcpServer: 100 retries, 2hr timeout (aggressive retry for slow servers)
+/// - DiscoverMcpInventory: 3 retries, 10s timeout (discovery + caching)
+/// - RegisterMcpServer: No retry, 5s timeout (fast registration)
+/// - ValidateRegistration: Final validation step
+pub fn create_mcp_registration_workflow() -> WorkflowDefinition<McpWorkflowData> {
+    WorkflowDefinition::new("mcp_registration", "MCP Server Registration")
+        .add_step(
+            StepDefinition::new(
+                "connect_mcp_server",
+                "Connect to MCP Server",
+                Arc::new(ConnectMcpServerStep),
+            )
+            .with_retry(RetryPolicy {
+                max_attempts: 100,
+                backoff: BackoffStrategy::Linear {
+                    increment: Duration::from_secs(1),
+                    max: Duration::from_secs(5),
+                },
+            })
+            .with_timeout(Duration::from_secs(7200)) // 2 hours
+            .with_failure_action(FailureAction::ContinueNextStep),
+        )
+        .add_step(
+            StepDefinition::new(
+                "discover_mcp_inventory",
+                "Discover and Cache MCP Inventory",
+                Arc::new(DiscoverMcpInventoryStep),
+            )
+            .with_retry(RetryPolicy {
+                max_attempts: 3,
+                backoff: BackoffStrategy::Fixed(Duration::from_secs(1)),
+            })
+            .with_timeout(Duration::from_secs(10))
+            .with_failure_action(FailureAction::ContinueNextStep)
+            .depends_on(&["connect_mcp_server"]),
+        )
+        .add_step(
+            StepDefinition::new(
+                "register_mcp_server",
+                "Register MCP Server",
+                Arc::new(RegisterMcpServerStep),
+            )
+            .with_timeout(Duration::from_secs(5))
+            .with_failure_action(FailureAction::ContinueNextStep)
+            .depends_on(&["discover_mcp_inventory"]),
+        )
+        .add_step(
+            StepDefinition::new(
+                "validate_registration",
+                "Validate MCP Registration",
+                Arc::new(ValidateRegistrationStep),
+            )
+            .with_timeout(Duration::from_secs(1))
+            .with_failure_action(FailureAction::FailWorkflow)
+            .depends_on(&["register_mcp_server"]),
+        )
+}
+
+/// Helper to create initial workflow data for MCP registration
+pub fn create_mcp_workflow_data(
+    config: McpServerConfigRequest,
+    app_context: Arc<AppContext>,
+) -> McpWorkflowData {
+    McpWorkflowData {
+        config,
+        validated: false,
+        app_context: Some(app_context),
+        mcp_client: None,
+    }
+}

third_party/sglang/sgl-model-gateway/src/core/steps/mod.rs

@@ -0,0 +1,87 @@
+//! Workflow step implementations
+//!
+//! This module contains concrete step implementations for various workflows:
+//! - Worker management (registration, removal, updates)
+//! - MCP server registration
+//! - WASM module registration and removal
+//! - Tokenizer registration
+
+pub mod mcp_registration;
+pub mod tokenizer_registration;
+pub mod wasm_module_registration;
+pub mod wasm_module_removal;
+pub mod worker;
+pub mod workflow_data;
+pub mod workflow_engines;
+
+// Worker management (registration, removal)
+pub use mcp_registration::{
+    create_mcp_registration_workflow, create_mcp_workflow_data, ConnectMcpServerStep,
+    DiscoverMcpInventoryStep, McpServerConfigRequest, RegisterMcpServerStep,
+    ValidateRegistrationStep,
+};
+pub use tokenizer_registration::{
+    create_tokenizer_registration_workflow, create_tokenizer_workflow_data, LoadTokenizerStep,
+    TokenizerConfigRequest, TokenizerRemovalRequest,
+};
+pub use wasm_module_registration::{
+    create_wasm_module_registration_workflow, create_wasm_registration_workflow_data,
+    CalculateHashStep, CheckDuplicateStep, LoadWasmBytesStep, RegisterModuleStep,
+    ValidateDescriptorStep, ValidateWasmComponentStep, WasmModuleConfigRequest,
+};
+pub use wasm_module_removal::{
+    create_wasm_module_removal_workflow, create_wasm_removal_workflow_data, FindModuleToRemoveStep,
+    RemoveModuleStep, WasmModuleRemovalRequest,
+};
+pub use worker::{
+    // Workflow builders
+    create_external_worker_workflow,
+    // Workflow data helpers
+    create_external_worker_workflow_data,
+    create_local_worker_workflow,
+    create_local_worker_workflow_data,
+    create_worker_removal_workflow,
+    create_worker_removal_workflow_data,
+    create_worker_update_workflow,
+    create_worker_update_workflow_data,
+    // Utility functions
+    group_models_into_cards,
+    infer_model_type_from_id,
+    // Shared steps
+    ActivateWorkersStep,
+    // External registration steps
+    CreateExternalWorkersStep,
+    // Local registration steps
+    CreateLocalWorkerStep,
+    DetectConnectionModeStep,
+    DiscoverDPInfoStep,
+    DiscoverMetadataStep,
+    DiscoverModelsStep,
+    DpInfo,
+    // Update steps
+    FindWorkerToUpdateStep,
+    // Removal steps
+    FindWorkersToRemoveStep,
+    ModelInfo,
+    ModelsResponse,
+    RegisterWorkersStep,
+    RemoveFromPolicyRegistryStep,
+    RemoveFromWorkerRegistryStep,
+    UpdatePoliciesForWorkerStep,
+    UpdatePoliciesStep,
+    UpdateRemainingPoliciesStep,
+    UpdateWorkerPropertiesStep,
+    WorkerList,
+    WorkerRemovalRequest,
+};
+// Typed workflow data structures
+pub use workflow_data::{
+    ExternalWorkerWorkflowData, LocalWorkerWorkflowData, McpWorkflowData, ProtocolUpdateRequest,
+    TokenizerWorkflowData, WasmRegistrationWorkflowData, WasmRemovalWorkflowData,
+    WorkerConfigRequest, WorkerList as WorkflowWorkerList, WorkerRegistrationData,
+    WorkerRemovalWorkflowData, WorkerUpdateWorkflowData,
+};
+// Typed workflow engines
+pub use workflow_engines::WorkflowEngines;
+
+pub use crate::config::TokenizerCacheConfig;

third_party/sglang/sgl-model-gateway/src/core/steps/tokenizer_registration.rs

@@ -0,0 +1,330 @@
+//! Tokenizer registration workflow
+//!
+//! This module provides a workflow for registering tokenizers asynchronously.
+//! Tokenizers can be loaded from local paths or downloaded from HuggingFace.
+//!
+//! This is the **single source of truth** for tokenizer registration. All paths
+//! (startup, worker connection, API) should use this workflow to ensure consistent
+//! behavior (validation, caching, deduplication).
+
+use std::{sync::Arc, time::Duration};
+
+use async_trait::async_trait;
+use serde::{Deserialize, Serialize};
+use tracing::{error, info};
+use wfaas::{
+    BackoffStrategy, FailureAction, RetryPolicy, StepDefinition, StepExecutor, StepId, StepResult,
+    WorkflowContext, WorkflowDefinition, WorkflowError, WorkflowResult,
+};
+
+use super::workflow_data::TokenizerWorkflowData;
+use crate::{
+    app_context::AppContext,
+    config::TokenizerCacheConfig,
+    tokenizer::{
+        cache::{CacheConfig, CachedTokenizer},
+        factory,
+        registry::LoadOutcome,
+        traits::Tokenizer,
+    },
+};
+
+/// Configuration for adding a tokenizer
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct TokenizerConfigRequest {
+    /// Pre-generated UUID for this tokenizer
+    pub id: String,
+    /// User-provided name (what to register under in the registry)
+    pub name: String,
+    /// Source: either a local path or HuggingFace model ID
+    pub source: String,
+    /// Optional path to chat template file
+    pub chat_template_path: Option<String>,
+    /// Optional cache configuration. If provided, wraps tokenizer with CachedTokenizer.
+    #[serde(default)]
+    pub cache_config: Option<TokenizerCacheConfig>,
+    /// If true, the workflow fails when a tokenizer with the same name already exists.
+    /// If false (default), the workflow succeeds and returns the existing tokenizer's ID.
+    /// API callers should set this to true.
+    #[serde(default)]
+    pub fail_on_duplicate: bool,
+}
+
+/// Configuration for removing a tokenizer
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct TokenizerRemovalRequest {
+    /// UUID of the tokenizer to remove
+    pub id: String,
+}
+
+// ============================================================================
+// Workflow Steps
+// ============================================================================
+
+/// Load the tokenizer from source (local path or HuggingFace)
+///
+/// This step handles:
+/// - Input validation (via registry.load())
+/// - Deduplication (returns success if already exists)
+/// - Loading from local path or HuggingFace
+/// - Optional caching layer wrapping
+pub struct LoadTokenizerStep;
+
+#[async_trait]
+impl StepExecutor<TokenizerWorkflowData> for LoadTokenizerStep {
+    async fn execute(
+        &self,
+        context: &mut WorkflowContext<TokenizerWorkflowData>,
+    ) -> WorkflowResult<StepResult> {
+        let config = &context.data.config;
+        let app_context = context
+            .data
+            .app_context
+            .as_ref()
+            .ok_or_else(|| WorkflowError::ContextValueNotFound("app_context".to_string()))?
+            .clone();
+
+        info!(
+            "Loading tokenizer '{}' (id: {}) from source: {}{}",
+            config.name,
+            config.id,
+            config.source,
+            if config.cache_config.is_some() {
+                " with caching"
+            } else {
+                ""
+            }
+        );
+
+        // Clone needed values before async move
+        let id = config.id.clone();
+        let name = config.name.clone();
+        let source = config.source.clone();
+        let chat_template = config.chat_template_path.clone();
+        let cache_config = config.cache_config.clone();
+
+        // Load the tokenizer using the registry's load method
+        // This handles: validation, deduplication, and loading
+        let result = app_context
+            .tokenizer_registry
+            .load(&id, &name, &source, || {
+                let source = source.clone();
+                let chat_template = chat_template.clone();
+                let cache_cfg = cache_config.clone();
+                async move {
+                    // Load base tokenizer
+                    let base_tokenizer = factory::create_tokenizer_async_with_chat_template(
+                        &source,
+                        chat_template.as_deref(),
+                    )
+                    .await
+                    .map_err(|e| format!("Failed to load tokenizer: {}", e))?;
+
+                    // Wrap with caching layer if configured
+                    let tokenizer: Arc<dyn Tokenizer> = match cache_cfg {
+                        Some(cfg) if cfg.enable_l0 || cfg.enable_l1 => {
+                            let cache_config = CacheConfig {
+                                enable_l0: cfg.enable_l0,
+                                l0_max_entries: cfg.l0_max_entries,
+                                enable_l1: cfg.enable_l1,
+                                l1_max_memory: cfg.l1_max_memory,
+                            };
+                            Arc::new(CachedTokenizer::new(base_tokenizer, cache_config))
+                        }
+                        _ => base_tokenizer,
+                    };
+
+                    Ok(tokenizer)
+                }
+            })
+            .await;
+
+        match result {
+            Ok(outcome) => {
+                let loaded_id = outcome.id();
+
+                // Get vocab size for logging
+                let vocab_size = app_context
+                    .tokenizer_registry
+                    .get_by_id(loaded_id)
+                    .map(|e| e.tokenizer.vocab_size());
+
+                match &outcome {
+                    LoadOutcome::Loaded { id } => {
+                        info!(
+                            "Successfully loaded tokenizer '{}' (id: {}) with vocab_size: {:?}",
+                            name, id, vocab_size
+                        );
+                    }
+                    LoadOutcome::AlreadyExists { id } => {
+                        if config.fail_on_duplicate {
+                            return Err(WorkflowError::StepFailed {
+                                step_id: StepId::new("load_tokenizer"),
+                                message: format!(
+                                    "Tokenizer '{}' already exists (id: {})",
+                                    name, id
+                                ),
+                            });
+                        }
+                        info!(
+                            "Tokenizer '{}' already exists (id: {}), skipping load",
+                            name, id
+                        );
+                    }
+                }
+
+                // Store vocab size in typed data
+                if let Some(size) = vocab_size {
+                    context.data.vocab_size = Some(size);
+                }
+
+                Ok(StepResult::Success)
+            }
+            Err(e) => {
+                error!("Failed to load tokenizer '{}': {}", name, e);
+                Err(WorkflowError::StepFailed {
+                    step_id: StepId::new("load_tokenizer"),
+                    message: e.to_string(),
+                })
+            }
+        }
+    }
+
+    fn is_retryable(&self, _error: &WorkflowError) -> bool {
+        true // Network/IO errors are retryable
+    }
+}
+
+// ============================================================================
+// Workflow Builder
+// ============================================================================
+
+/// Create the tokenizer registration workflow
+///
+/// This workflow loads and registers a tokenizer. The single LoadTokenizerStep handles:
+/// - Input validation (empty name/source)
+/// - Deduplication (returns success if already exists)
+/// - Loading from local path or HuggingFace
+/// - Optional caching layer wrapping
+///
+/// Configuration:
+/// - 3 retries with 2s backoff (for network issues)
+/// - 5 minute timeout (HuggingFace downloads can be slow)
+pub fn create_tokenizer_registration_workflow() -> WorkflowDefinition<TokenizerWorkflowData> {
+    WorkflowDefinition::new("tokenizer_registration", "Tokenizer Registration").add_step(
+        StepDefinition::new(
+            "load_tokenizer",
+            "Load Tokenizer",
+            Arc::new(LoadTokenizerStep),
+        )
+        .with_retry(RetryPolicy {
+            max_attempts: 3,
+            backoff: BackoffStrategy::Fixed(Duration::from_secs(2)),
+        })
+        .with_timeout(Duration::from_secs(300)) // 5 min for HuggingFace downloads
+        .with_failure_action(FailureAction::FailWorkflow),
+    )
+}
+
+/// Helper to create initial workflow data for tokenizer registration
+pub fn create_tokenizer_workflow_data(
+    config: TokenizerConfigRequest,
+    app_context: Arc<AppContext>,
+) -> TokenizerWorkflowData {
+    TokenizerWorkflowData {
+        config,
+        vocab_size: None,
+        app_context: Some(app_context),
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_tokenizer_config_request_serialization() {
+        let config = TokenizerConfigRequest {
+            id: "test-uuid-1234".to_string(),
+            name: "test-model".to_string(),
+            source: "meta-llama/Llama-2-7b-hf".to_string(),
+            chat_template_path: None,
+            cache_config: None,
+            fail_on_duplicate: false,
+        };
+
+        let json = serde_json::to_string(&config).unwrap();
+        let parsed: TokenizerConfigRequest = serde_json::from_str(&json).unwrap();
+
+        assert_eq!(parsed.id, "test-uuid-1234");
+        assert_eq!(parsed.name, "test-model");
+        assert_eq!(parsed.source, "meta-llama/Llama-2-7b-hf");
+        assert!(parsed.chat_template_path.is_none());
+        assert!(parsed.cache_config.is_none());
+        assert!(!parsed.fail_on_duplicate);
+    }
+
+    #[test]
+    fn test_tokenizer_config_request_fail_on_duplicate_defaults_to_false() {
+        // Test that fail_on_duplicate defaults to false when not specified in JSON
+        let json = r#"{
+            "id": "test-uuid",
+            "name": "test-model",
+            "source": "/path/to/tokenizer"
+        }"#;
+        let parsed: TokenizerConfigRequest = serde_json::from_str(json).unwrap();
+        assert!(!parsed.fail_on_duplicate);
+    }
+
+    #[test]
+    fn test_tokenizer_config_request_fail_on_duplicate_true() {
+        let config = TokenizerConfigRequest {
+            id: "test-uuid-1234".to_string(),
+            name: "test-model".to_string(),
+            source: "meta-llama/Llama-2-7b-hf".to_string(),
+            chat_template_path: None,
+            cache_config: None,
+            fail_on_duplicate: true,
+        };
+
+        let json = serde_json::to_string(&config).unwrap();
+        let parsed: TokenizerConfigRequest = serde_json::from_str(&json).unwrap();
+        assert!(parsed.fail_on_duplicate);
+    }
+
+    #[test]
+    fn test_tokenizer_config_request_with_cache() {
+        let config = TokenizerConfigRequest {
+            id: "test-uuid-1234".to_string(),
+            name: "test-model".to_string(),
+            source: "meta-llama/Llama-2-7b-hf".to_string(),
+            chat_template_path: None,
+            cache_config: Some(TokenizerCacheConfig {
+                enable_l0: true,
+                l0_max_entries: 1000,
+                enable_l1: false,
+                l1_max_memory: 0,
+            }),
+            fail_on_duplicate: false,
+        };
+
+        let json = serde_json::to_string(&config).unwrap();
+        let parsed: TokenizerConfigRequest = serde_json::from_str(&json).unwrap();
+
+        assert!(parsed.cache_config.is_some());
+        let cache = parsed.cache_config.unwrap();
+        assert!(cache.enable_l0);
+        assert_eq!(cache.l0_max_entries, 1000);
+        assert!(!cache.enable_l1);
+    }
+
+    #[test]
+    fn test_workflow_creation() {
+        let mut workflow = create_tokenizer_registration_workflow();
+        assert_eq!(workflow.id.to_string(), "tokenizer_registration");
+        // Validate the workflow DAG
+        workflow
+            .validate()
+            .expect("Workflow validation should pass");
+    }
+}

third_party/sglang/sgl-model-gateway/src/core/steps/wasm_module_registration.rs

@@ -0,0 +1,637 @@
+use std::{
+    path::{Component as PathComponent, Path},
+    sync::Arc,
+    time::Duration,
+};
+
+use async_trait::async_trait;
+use sha2::{Digest, Sha256};
+use tracing::{debug, info, warn};
+use uuid::Uuid;
+use wasmtime::{component::Component, Config, Engine};
+use wfaas::{
+    BackoffStrategy, FailureAction, RetryPolicy, StepDefinition, StepExecutor, StepId, StepResult,
+    WorkflowContext, WorkflowDefinition, WorkflowError, WorkflowResult,
+};
+
+use super::workflow_data::WasmRegistrationWorkflowData;
+use crate::{
+    app_context::AppContext,
+    wasm::module::{WasmModule, WasmModuleDescriptor, WasmModuleMeta},
+};
+
+/// WASM module registration request
+#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
+pub struct WasmModuleConfigRequest {
+    /// Module descriptor containing name, file_path, attach_points, etc.
+    pub descriptor: WasmModuleDescriptor,
+}
+
+/// Sensitive system directories that WASM modules cannot be loaded from.
+/// These are blocked to prevent information disclosure attacks.
+const BLOCKED_PATH_PREFIXES: &[&str] = &[
+    "/etc/",
+    "/proc/",
+    "/sys/",
+    "/dev/",
+    "/boot/",
+    "/root/",
+    "/var/log/",
+    "/var/run/",
+];
+
+/// Check if a path starts with any blocked prefix.
+/// Returns the matched prefix if found, None otherwise.
+fn find_blocked_prefix(path: &str) -> Option<&'static str> {
+    BLOCKED_PATH_PREFIXES
+        .iter()
+        .find(|&&prefix| path.starts_with(prefix))
+        .copied()
+}
+
+/// Check if a path has a .wasm extension (case-insensitive).
+fn has_wasm_extension(path: &Path) -> bool {
+    path.extension()
+        .is_some_and(|ext| ext.eq_ignore_ascii_case("wasm"))
+}
+
+/// Step 1: Validate module descriptor
+///
+/// Validates that the module descriptor has all required fields:
+/// - Module name is not empty
+/// - File path is not empty
+/// - File exists and is readable
+/// - File size is not zero
+pub struct ValidateDescriptorStep;
+
+#[async_trait]
+impl StepExecutor<WasmRegistrationWorkflowData> for ValidateDescriptorStep {
+    async fn execute(
+        &self,
+        context: &mut WorkflowContext<WasmRegistrationWorkflowData>,
+    ) -> WorkflowResult<StepResult> {
+        let descriptor = &context.data.config.descriptor;
+
+        debug!("Validating WASM module descriptor: {}", descriptor.name);
+
+        // Validate name
+        if descriptor.name.is_empty() {
+            return Err(WorkflowError::StepFailed {
+                step_id: StepId::new("validate_descriptor"),
+                message: "Module name cannot be empty".to_string(),
+            });
+        }
+
+        // Validate file path
+        if descriptor.file_path.is_empty() {
+            return Err(WorkflowError::StepFailed {
+                step_id: StepId::new("validate_descriptor"),
+                message: "Module file path cannot be empty".to_string(),
+            });
+        }
+
+        // Security: Validate path to prevent path traversal attacks
+        let path = Path::new(&descriptor.file_path);
+
+        // Must be an absolute path
+        if !path.is_absolute() {
+            return Err(WorkflowError::StepFailed {
+                step_id: StepId::new("validate_descriptor"),
+                message: format!(
+                    "Module file path must be absolute, got: {}",
+                    descriptor.file_path
+                ),
+            });
+        }
+
+        // Check for path traversal components (.. or symbolic links that could escape)
+        for component in path.components() {
+            match component {
+                PathComponent::ParentDir => {
+                    warn!(
+                        "Path traversal attempt detected in WASM module path: {}",
+                        descriptor.file_path
+                    );
+                    return Err(WorkflowError::StepFailed {
+                        step_id: StepId::new("validate_descriptor"),
+                        message: "Path traversal (..) not allowed in module file path".to_string(),
+                    });
+                }
+                PathComponent::CurDir => {
+                    return Err(WorkflowError::StepFailed {
+                        step_id: StepId::new("validate_descriptor"),
+                        message: "Current directory (.) not allowed in module file path"
+                            .to_string(),
+                    });
+                }
+                _ => {}
+            }
+        }
+
+        // Require .wasm extension to prevent loading arbitrary files
+        if !has_wasm_extension(path) {
+            return Err(WorkflowError::StepFailed {
+                step_id: StepId::new("validate_descriptor"),
+                message: "Module file must have .wasm extension".to_string(),
+            });
+        }
+
+        // Block access to sensitive system directories
+        if let Some(prefix) = find_blocked_prefix(&descriptor.file_path) {
+            warn!(
+                "Attempt to access blocked directory in WASM module path: {}",
+                descriptor.file_path
+            );
+            return Err(WorkflowError::StepFailed {
+                step_id: StepId::new("validate_descriptor"),
+                message: format!("Access to {} directory is not allowed", prefix),
+            });
+        }
+
+        // Check if file exists and get size
+        let metadata = tokio::fs::metadata(&descriptor.file_path)
+            .await
+            .map_err(|e| WorkflowError::StepFailed {
+                step_id: StepId::new("validate_descriptor"),
+                message: format!("Failed to access file {}: {}", descriptor.file_path, e),
+            })?;
+
+        if metadata.len() == 0 {
+            return Err(WorkflowError::StepFailed {
+                step_id: StepId::new("validate_descriptor"),
+                message: "Module file size cannot be 0".to_string(),
+            });
+        }
+
+        // Canonicalize the path to resolve symlinks and verify final location is safe
+        let canonical_path = tokio::fs::canonicalize(&descriptor.file_path)
+            .await
+            .map_err(|e| WorkflowError::StepFailed {
+                step_id: StepId::new("validate_descriptor"),
+                message: format!(
+                    "Failed to canonicalize path {}: {}",
+                    descriptor.file_path, e
+                ),
+            })?;
+
+        // Re-check blocked directories after symlink resolution
+        let canonical_str = canonical_path.to_string_lossy();
+        if let Some(prefix) = find_blocked_prefix(&canonical_str) {
+            warn!(
+                "Symlink resolved to blocked directory: {} -> {}",
+                descriptor.file_path, canonical_str
+            );
+            return Err(WorkflowError::StepFailed {
+                step_id: StepId::new("validate_descriptor"),
+                message: format!(
+                    "Path resolves to blocked directory {} (via symlink)",
+                    prefix
+                ),
+            });
+        }
+
+        // Ensure canonicalized path still has .wasm extension (symlink target check)
+        if !has_wasm_extension(&canonical_path) {
+            warn!(
+                "Symlink target is not a .wasm file: {} -> {}",
+                descriptor.file_path, canonical_str
+            );
+            return Err(WorkflowError::StepFailed {
+                step_id: StepId::new("validate_descriptor"),
+                message: "Symlink target must be a .wasm file".to_string(),
+            });
+        }
+
+        // Clone name for logging before mutable borrow
+        let module_name = descriptor.name.clone();
+
+        // Store file size in typed data
+        context.data.file_size_bytes = Some(metadata.len());
+
+        info!(
+            "Descriptor validated successfully for module: {}",
+            module_name
+        );
+        Ok(StepResult::Success)
+    }
+
+    fn is_retryable(&self, _error: &WorkflowError) -> bool {
+        false // Validation errors are not retryable (invalid input)
+    }
+}
+
+/// Step 2: Calculate SHA256 hash of the module file
+///
+/// Reads the file and calculates its SHA256 hash for deduplication.
+/// This step is I/O intensive and may take time for large files.
+pub struct CalculateHashStep;
+
+#[async_trait]
+impl StepExecutor<WasmRegistrationWorkflowData> for CalculateHashStep {
+    async fn execute(
+        &self,
+        context: &mut WorkflowContext<WasmRegistrationWorkflowData>,
+    ) -> WorkflowResult<StepResult> {
+        let file_path = &context.data.config.descriptor.file_path;
+
+        debug!("Calculating SHA256 hash for: {}", file_path);
+
+        // Read file in chunks to handle large files efficiently
+        let mut file =
+            tokio::fs::File::open(file_path)
+                .await
+                .map_err(|e| WorkflowError::StepFailed {
+                    step_id: StepId::new("calculate_hash"),
+                    message: format!("Failed to open file {}: {}", file_path, e),
+                })?;
+
+        let mut hasher = Sha256::new();
+        let mut buffer = vec![0u8; 1024 * 1024]; // 1MB buffer
+
+        loop {
+            use tokio::io::AsyncReadExt;
+            let bytes_read =
+                file.read(&mut buffer)
+                    .await
+                    .map_err(|e| WorkflowError::StepFailed {
+                        step_id: StepId::new("calculate_hash"),
+                        message: format!("Failed to read file {}: {}", file_path, e),
+                    })?;
+
+            if bytes_read == 0 {
+                break;
+            }
+
+            hasher.update(&buffer[..bytes_read]);
+        }
+
+        let hash: [u8; 32] = hasher.finalize().into();
+
+        // Clone path for logging before mutable borrow
+        let path_for_log = file_path.clone();
+
+        // Store hash in typed data
+        context.data.sha256_hash = Some(hash);
+
+        info!("SHA256 hash calculated for: {}", path_for_log);
+        Ok(StepResult::Success)
+    }
+
+    fn is_retryable(&self, _error: &WorkflowError) -> bool {
+        true // File I/O errors are retryable (network filesystem, etc.)
+    }
+}
+
+/// Step 3: Check for duplicate SHA256 hash
+///
+/// Checks if a module with the same SHA256 hash already exists in the manager.
+/// This prevents duplicate modules from being registered.
+pub struct CheckDuplicateStep;
+
+#[async_trait]
+impl StepExecutor<WasmRegistrationWorkflowData> for CheckDuplicateStep {
+    async fn execute(
+        &self,
+        context: &mut WorkflowContext<WasmRegistrationWorkflowData>,
+    ) -> WorkflowResult<StepResult> {
+        let app_context = context
+            .data
+            .app_context
+            .as_ref()
+            .ok_or_else(|| WorkflowError::ContextValueNotFound("app_context".to_string()))?;
+        let sha256_hash = context
+            .data
+            .sha256_hash
+            .as_ref()
+            .ok_or_else(|| WorkflowError::ContextValueNotFound("sha256_hash".to_string()))?;
+
+        debug!(
+            "Checking for duplicate SHA256 hash for module: {}",
+            context.data.config.descriptor.name
+        );
+
+        // Get WASM module manager from app context
+        let wasm_manager =
+            app_context
+                .wasm_manager
+                .as_ref()
+                .ok_or_else(|| WorkflowError::StepFailed {
+                    step_id: StepId::new("check_duplicate"),
+                    message: "WASM module manager not initialized".to_string(),
+                })?;
+
+        // Check for duplicate hash using manager's internal method
+        wasm_manager
+            .check_duplicate_sha256_hash(sha256_hash)
+            .map_err(|e| WorkflowError::StepFailed {
+                step_id: StepId::new("check_duplicate"),
+                message: format!("Duplicate SHA256 hash detected: {}", e),
+            })?;
+
+        info!(
+            "No duplicate found for module: {}",
+            context.data.config.descriptor.name
+        );
+        Ok(StepResult::Success)
+    }
+
+    fn is_retryable(&self, _error: &WorkflowError) -> bool {
+        false // Duplicate check failures are not retryable
+    }
+}
+
+/// Step 4: Load WASM bytes into memory
+///
+/// Reads the entire WASM file into memory for faster execution.
+/// This is an I/O operation that may take time for large files.
+pub struct LoadWasmBytesStep;
+
+#[async_trait]
+impl StepExecutor<WasmRegistrationWorkflowData> for LoadWasmBytesStep {
+    async fn execute(
+        &self,
+        context: &mut WorkflowContext<WasmRegistrationWorkflowData>,
+    ) -> WorkflowResult<StepResult> {
+        let file_path = &context.data.config.descriptor.file_path;
+
+        debug!("Loading WASM bytes from: {}", file_path);
+
+        // Clone path for logging before mutable borrow
+        let path_for_log = file_path.clone();
+
+        let wasm_bytes =
+            tokio::fs::read(file_path)
+                .await
+                .map_err(|e| WorkflowError::StepFailed {
+                    step_id: StepId::new("load_wasm_bytes"),
+                    message: format!("Failed to read WASM file {}: {}", file_path, e),
+                })?;
+
+        // Store WASM bytes in typed data
+        context.data.wasm_bytes = Some(wasm_bytes);
+
+        info!("WASM bytes loaded from: {}", path_for_log);
+        Ok(StepResult::Success)
+    }
+
+    fn is_retryable(&self, _error: &WorkflowError) -> bool {
+        true // File read errors are retryable
+    }
+}
+
+/// Step 5: Validate WASM component format
+///
+/// Validates that the loaded WASM bytes represent a valid component.
+/// This catches format errors early during registration rather than during execution.
+pub struct ValidateWasmComponentStep;
+
+#[async_trait]
+impl StepExecutor<WasmRegistrationWorkflowData> for ValidateWasmComponentStep {
+    async fn execute(
+        &self,
+        context: &mut WorkflowContext<WasmRegistrationWorkflowData>,
+    ) -> WorkflowResult<StepResult> {
+        let wasm_bytes = context
+            .data
+            .wasm_bytes
+            .as_ref()
+            .ok_or_else(|| WorkflowError::ContextValueNotFound("wasm_bytes".to_string()))?;
+
+        debug!(
+            "Validating WASM component format for module: {}",
+            context.data.config.descriptor.name
+        );
+
+        // Create a temporary engine to validate the component
+        let mut config = Config::new();
+        config.async_support(true);
+        config.wasm_component_model(true);
+
+        let engine = Engine::new(&config).map_err(|e| WorkflowError::StepFailed {
+            step_id: StepId::new("validate_wasm_component"),
+            message: format!("Failed to create WASM engine: {}", e),
+        })?;
+
+        // Attempt to compile the component to validate it
+        Component::new(&engine, wasm_bytes)
+            .map_err(|e| WorkflowError::StepFailed {
+                step_id: StepId::new("validate_wasm_component"),
+                message: format!(
+                    "Invalid WASM component: {}. \
+                     Hint: The WASM file must be in component format. \
+                     If you're using wit-bindgen, use 'wasm-tools component new' to wrap the WASM module into a component.",
+                    e
+                ),
+            })?;
+
+        info!(
+            "WASM component validated successfully for module: {}",
+            context.data.config.descriptor.name
+        );
+        Ok(StepResult::Success)
+    }
+
+    fn is_retryable(&self, _error: &WorkflowError) -> bool {
+        false // Validation errors are not retryable (invalid format)
+    }
+}
+
+/// Step 6: Register module in WasmModuleManager
+///
+/// Creates the WasmModule object and registers it in the manager's module map.
+/// This is the final step that makes the module available for execution.
+pub struct RegisterModuleStep;
+
+#[async_trait]
+impl StepExecutor<WasmRegistrationWorkflowData> for RegisterModuleStep {
+    async fn execute(
+        &self,
+        context: &mut WorkflowContext<WasmRegistrationWorkflowData>,
+    ) -> WorkflowResult<StepResult> {
+        let app_context = context
+            .data
+            .app_context
+            .as_ref()
+            .ok_or_else(|| WorkflowError::ContextValueNotFound("app_context".to_string()))?;
+        let sha256_hash = context
+            .data
+            .sha256_hash
+            .ok_or_else(|| WorkflowError::ContextValueNotFound("sha256_hash".to_string()))?;
+        let file_size_bytes = context
+            .data
+            .file_size_bytes
+            .ok_or_else(|| WorkflowError::ContextValueNotFound("file_size_bytes".to_string()))?;
+        let wasm_bytes = context
+            .data
+            .wasm_bytes
+            .as_ref()
+            .ok_or_else(|| WorkflowError::ContextValueNotFound("wasm_bytes".to_string()))?
+            .clone();
+
+        let descriptor = &context.data.config.descriptor;
+
+        debug!("Registering WASM module in manager: {}", descriptor.name);
+
+        // Get WASM module manager from app context
+        let wasm_manager =
+            app_context
+                .wasm_manager
+                .as_ref()
+                .ok_or_else(|| WorkflowError::StepFailed {
+                    step_id: StepId::new("register_module"),
+                    message: "WASM module manager not initialized".to_string(),
+                })?;
+
+        // Create module metadata
+        let now = std::time::SystemTime::now()
+            .duration_since(std::time::UNIX_EPOCH)
+            .unwrap_or_else(|_| Duration::from_nanos(0))
+            .as_nanos() as u64;
+
+        let module_uuid = Uuid::new_v4();
+
+        let module = WasmModule {
+            module_uuid,
+            module_meta: WasmModuleMeta {
+                name: descriptor.name.clone(),
+                file_path: descriptor.file_path.clone(),
+                sha256_hash,
+                size_bytes: file_size_bytes,
+                created_at: now,
+                last_accessed_at: now,
+                access_count: 0,
+                attach_points: descriptor.attach_points.clone(),
+                wasm_bytes,
+            },
+        };
+
+        // Clone name for logging before mutable borrow
+        let module_name = descriptor.name.clone();
+
+        // Register module in manager
+        wasm_manager
+            .register_module_internal(module)
+            .map_err(|e| WorkflowError::StepFailed {
+                step_id: StepId::new("register_module"),
+                message: format!("Failed to register module: {}", e),
+            })?;
+
+        // Store module UUID in typed data
+        context.data.module_uuid = Some(module_uuid);
+
+        info!(
+            "WASM module registered successfully: {} (UUID: {})",
+            module_name, module_uuid
+        );
+
+        Ok(StepResult::Success)
+    }
+
+    fn is_retryable(&self, _error: &WorkflowError) -> bool {
+        false // Registration is a simple operation, not retryable
+    }
+}
+
+/// Create WASM module registration workflow
+///
+/// This workflow handles the complete process of registering a WASM module:
+/// - Validates the module descriptor
+/// - Calculates SHA256 hash for deduplication
+/// - Checks for duplicates
+/// - Loads WASM bytes into memory
+/// - Validates WASM component format
+/// - Registers the module in the manager
+///
+/// Workflow configuration:
+/// - ValidateDescriptor: No retry, 5s timeout (fast validation)
+/// - CalculateHash: 3 retries, 60s timeout (I/O intensive, may need retry)
+/// - CheckDuplicate: No retry, 5s timeout (fast check)
+/// - LoadWasmBytes: 3 retries, 60s timeout (I/O intensive)
+/// - ValidateWasmComponent: No retry, 30s timeout (CPU intensive validation)
+/// - RegisterModule: No retry, 5s timeout (fast registration)
+pub fn create_wasm_module_registration_workflow() -> WorkflowDefinition<WasmRegistrationWorkflowData>
+{
+    WorkflowDefinition::new("wasm_module_registration", "WASM Module Registration")
+        .add_step(
+            StepDefinition::new(
+                "validate_descriptor",
+                "Validate Descriptor",
+                Arc::new(ValidateDescriptorStep),
+            )
+            .with_timeout(Duration::from_secs(5))
+            .with_failure_action(FailureAction::FailWorkflow),
+        )
+        .add_step(
+            StepDefinition::new(
+                "calculate_hash",
+                "Calculate SHA256 Hash",
+                Arc::new(CalculateHashStep),
+            )
+            .with_retry(RetryPolicy {
+                max_attempts: 3,
+                backoff: BackoffStrategy::Fixed(Duration::from_secs(1)),
+            })
+            .with_timeout(Duration::from_secs(60))
+            .with_failure_action(FailureAction::FailWorkflow)
+            .depends_on(&["validate_descriptor"]),
+        )
+        .add_step(
+            StepDefinition::new(
+                "check_duplicate",
+                "Check Duplicate Hash",
+                Arc::new(CheckDuplicateStep),
+            )
+            .with_timeout(Duration::from_secs(5))
+            .with_failure_action(FailureAction::FailWorkflow)
+            .depends_on(&["calculate_hash"]),
+        )
+        .add_step(
+            StepDefinition::new(
+                "load_wasm_bytes",
+                "Load WASM Bytes",
+                Arc::new(LoadWasmBytesStep),
+            )
+            .with_retry(RetryPolicy {
+                max_attempts: 3,
+                backoff: BackoffStrategy::Fixed(Duration::from_secs(1)),
+            })
+            .with_timeout(Duration::from_secs(60))
+            .with_failure_action(FailureAction::FailWorkflow)
+            .depends_on(&["check_duplicate"]),
+        )
+        .add_step(
+            StepDefinition::new(
+                "validate_wasm_component",
+                "Validate WASM Component",
+                Arc::new(ValidateWasmComponentStep),
+            )
+            .with_timeout(Duration::from_secs(30))
+            .with_failure_action(FailureAction::FailWorkflow)
+            .depends_on(&["load_wasm_bytes"]),
+        )
+        .add_step(
+            StepDefinition::new(
+                "register_module",
+                "Register Module",
+                Arc::new(RegisterModuleStep),
+            )
+            .with_timeout(Duration::from_secs(5))
+            .with_failure_action(FailureAction::FailWorkflow)
+            .depends_on(&["validate_wasm_component"]),
+        )
+}
+
+/// Helper to create initial workflow data for WASM module registration
+pub fn create_wasm_registration_workflow_data(
+    config: WasmModuleConfigRequest,
+    app_context: Arc<AppContext>,
+) -> WasmRegistrationWorkflowData {
+    WasmRegistrationWorkflowData {
+        config,
+        wasm_bytes: None,
+        sha256_hash: None,
+        file_size_bytes: None,
+        module_uuid: None,
+        app_context: Some(app_context),
+    }
+}

third_party/sglang/sgl-model-gateway/src/core/steps/wasm_module_removal.rs

@@ -0,0 +1,180 @@
+use std::{sync::Arc, time::Duration};
+
+use async_trait::async_trait;
+use tracing::{debug, info};
+use uuid::Uuid;
+use wfaas::{
+    FailureAction, StepDefinition, StepExecutor, StepId, StepResult, WorkflowContext,
+    WorkflowDefinition, WorkflowError, WorkflowResult,
+};
+
+use super::workflow_data::WasmRemovalWorkflowData;
+use crate::app_context::AppContext;
+
+/// WASM module removal request
+#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
+pub struct WasmModuleRemovalRequest {
+    /// Module UUID to remove
+    pub module_uuid: Uuid,
+    /// Cached UUID string for worker_url() method
+    pub(crate) uuid_string: String,
+}
+
+impl WasmModuleRemovalRequest {
+    pub fn new(module_uuid: Uuid) -> Self {
+        Self {
+            module_uuid,
+            uuid_string: module_uuid.to_string(),
+        }
+    }
+}
+
+/// Step 1: Find module to remove
+///
+/// Verifies that the module exists before attempting removal.
+pub struct FindModuleToRemoveStep;
+
+#[async_trait]
+impl StepExecutor<WasmRemovalWorkflowData> for FindModuleToRemoveStep {
+    async fn execute(
+        &self,
+        context: &mut WorkflowContext<WasmRemovalWorkflowData>,
+    ) -> WorkflowResult<StepResult> {
+        let removal_request = &context.data.config;
+        let app_context = context
+            .data
+            .app_context
+            .as_ref()
+            .ok_or_else(|| WorkflowError::ContextValueNotFound("app_context".to_string()))?;
+
+        debug!("Finding module to remove: {}", removal_request.module_uuid);
+
+        // Get WASM module manager from app context
+        let wasm_manager =
+            app_context
+                .wasm_manager
+                .as_ref()
+                .ok_or_else(|| WorkflowError::StepFailed {
+                    step_id: StepId::new("find_module_to_remove"),
+                    message: "WASM module manager not initialized".to_string(),
+                })?;
+
+        // Check if module exists
+        let module = wasm_manager
+            .get_module(removal_request.module_uuid)
+            .map_err(|e| WorkflowError::StepFailed {
+                step_id: StepId::new("find_module_to_remove"),
+                message: format!("Failed to get module: {}", e),
+            })?;
+
+        if module.is_none() {
+            return Err(WorkflowError::StepFailed {
+                step_id: StepId::new("find_module_to_remove"),
+                message: format!("Module with UUID {} not found", removal_request.module_uuid),
+            });
+        }
+
+        // Clone uuid for logging before mutable borrow
+        let module_uuid = removal_request.module_uuid;
+
+        // Store the module ID in typed data
+        context.data.module_id = Some(module_uuid.to_string());
+
+        info!("Module found for removal: {}", module_uuid);
+        Ok(StepResult::Success)
+    }
+
+    fn is_retryable(&self, _error: &WorkflowError) -> bool {
+        false // Module not found is not retryable
+    }
+}
+
+/// Step 2: Remove module from WasmModuleManager
+///
+/// Removes the module from the manager's module map.
+pub struct RemoveModuleStep;
+
+#[async_trait]
+impl StepExecutor<WasmRemovalWorkflowData> for RemoveModuleStep {
+    async fn execute(
+        &self,
+        context: &mut WorkflowContext<WasmRemovalWorkflowData>,
+    ) -> WorkflowResult<StepResult> {
+        let removal_request = &context.data.config;
+        let app_context = context
+            .data
+            .app_context
+            .as_ref()
+            .ok_or_else(|| WorkflowError::ContextValueNotFound("app_context".to_string()))?;
+
+        debug!("Removing WASM module: {}", removal_request.module_uuid);
+
+        // Get WASM module manager from app context
+        let wasm_manager =
+            app_context
+                .wasm_manager
+                .as_ref()
+                .ok_or_else(|| WorkflowError::StepFailed {
+                    step_id: StepId::new("remove_module"),
+                    message: "WASM module manager not initialized".to_string(),
+                })?;
+
+        // Remove module from manager
+        wasm_manager
+            .remove_module_internal(removal_request.module_uuid)
+            .map_err(|e| WorkflowError::StepFailed {
+                step_id: StepId::new("remove_module"),
+                message: format!("Failed to remove module: {}", e),
+            })?;
+
+        info!(
+            "WASM module removed successfully: {}",
+            removal_request.module_uuid
+        );
+        Ok(StepResult::Success)
+    }
+
+    fn is_retryable(&self, _error: &WorkflowError) -> bool {
+        false // Removal is not retryable
+    }
+}
+
+/// Create WASM module removal workflow
+///
+/// This workflow handles the process of removing a WASM module:
+/// - Finds the module to remove
+/// - Removes it from the manager
+///
+/// Workflow configuration:
+/// - FindModuleToRemove: No retry, 5s timeout (fast lookup)
+/// - RemoveModule: No retry, 5s timeout (fast removal)
+pub fn create_wasm_module_removal_workflow() -> WorkflowDefinition<WasmRemovalWorkflowData> {
+    WorkflowDefinition::new("wasm_module_removal", "WASM Module Removal")
+        .add_step(
+            StepDefinition::new(
+                "find_module_to_remove",
+                "Find Module to Remove",
+                Arc::new(FindModuleToRemoveStep),
+            )
+            .with_timeout(Duration::from_secs(5))
+            .with_failure_action(FailureAction::FailWorkflow),
+        )
+        .add_step(
+            StepDefinition::new("remove_module", "Remove Module", Arc::new(RemoveModuleStep))
+                .with_timeout(Duration::from_secs(5))
+                .with_failure_action(FailureAction::FailWorkflow)
+                .depends_on(&["find_module_to_remove"]),
+        )
+}
+
+/// Helper to create initial workflow data for WASM module removal
+pub fn create_wasm_removal_workflow_data(
+    config: WasmModuleRemovalRequest,
+    app_context: Arc<AppContext>,
+) -> WasmRemovalWorkflowData {
+    WasmRemovalWorkflowData {
+        config,
+        module_id: None,
+        app_context: Some(app_context),
+    }
+}

third_party/sglang/sgl-model-gateway/src/core/steps/worker/external/create_workers.rs

@@ -0,0 +1,169 @@
+//! External worker creation step.
+
+use std::{collections::HashMap, sync::Arc, time::Duration};
+
+use async_trait::async_trait;
+use tracing::{debug, info};
+use wfaas::{StepExecutor, StepResult, WorkflowContext, WorkflowError, WorkflowResult};
+
+use crate::core::{
+    circuit_breaker::CircuitBreakerConfig,
+    steps::workflow_data::{ExternalWorkerWorkflowData, WorkerList},
+    worker::{HealthConfig, RuntimeType, WorkerType},
+    BasicWorkerBuilder, ConnectionMode, Worker,
+};
+
+/// Normalize URL for external APIs (ensure https://).
+fn normalize_external_url(url: &str) -> String {
+    if url.starts_with("http://") || url.starts_with("https://") {
+        url.to_string()
+    } else {
+        format!("https://{}", url)
+    }
+}
+
+/// Step 2: Create worker objects for each discovered model.
+pub struct CreateExternalWorkersStep;
+
+#[async_trait]
+impl StepExecutor<ExternalWorkerWorkflowData> for CreateExternalWorkersStep {
+    async fn execute(
+        &self,
+        context: &mut WorkflowContext<ExternalWorkerWorkflowData>,
+    ) -> WorkflowResult<StepResult> {
+        let config = &context.data.config;
+        let app_context = context
+            .data
+            .app_context
+            .as_ref()
+            .ok_or_else(|| WorkflowError::ContextValueNotFound("app_context".to_string()))?;
+        let model_cards = &context.data.model_cards;
+
+        // Build configs from router settings
+        let circuit_breaker_config = {
+            let cfg = app_context.router_config.effective_circuit_breaker_config();
+            CircuitBreakerConfig {
+                failure_threshold: cfg.failure_threshold,
+                success_threshold: cfg.success_threshold,
+                timeout_duration: Duration::from_secs(cfg.timeout_duration_secs),
+                window_duration: Duration::from_secs(cfg.window_duration_secs),
+            }
+        };
+
+        let health_config = {
+            let cfg = &app_context.router_config.health_check;
+            HealthConfig {
+                timeout_secs: cfg.timeout_secs,
+                check_interval_secs: cfg.check_interval_secs,
+                endpoint: cfg.endpoint.clone(),
+                failure_threshold: cfg.failure_threshold,
+                success_threshold: cfg.success_threshold,
+                disable_health_check: cfg.disable_health_check || config.disable_health_check,
+            }
+        };
+
+        // Build labels from config
+        let mut labels: HashMap<String, String> = config.labels.clone();
+        if let Some(priority) = config.priority {
+            labels.insert("priority".to_string(), priority.to_string());
+        }
+        if let Some(cost) = config.cost {
+            labels.insert("cost".to_string(), cost.to_string());
+        }
+
+        // Normalize URL (ensure https:// for external APIs)
+        let normalized_url = normalize_external_url(&config.url);
+
+        let mut workers = Vec::new();
+
+        // Handle wildcard mode: create a single worker with empty models list
+        if model_cards.is_empty() {
+            debug!("Creating wildcard worker (no models) for {}", config.url);
+
+            let mut builder = BasicWorkerBuilder::new(normalized_url.clone())
+                .models(vec![]) // Empty models = accepts any model
+                .worker_type(WorkerType::Regular)
+                .connection_mode(ConnectionMode::Http)
+                .runtime_type(RuntimeType::External)
+                .circuit_breaker_config(circuit_breaker_config.clone())
+                .health_config(health_config.clone());
+
+            if let Some(ref api_key) = config.api_key {
+                builder = builder.api_key(api_key.clone());
+            }
+
+            if !labels.is_empty() {
+                builder = builder.labels(labels.clone());
+            }
+
+            let worker = Arc::new(builder.build()) as Arc<dyn Worker>;
+            if health_config.disable_health_check {
+                worker.set_healthy(true);
+            } else {
+                worker.set_healthy(false);
+            }
+
+            info!(
+                "Created wildcard worker at {} (accepts any model, user auth forwarded)",
+                normalized_url
+            );
+
+            workers.push(worker);
+        } else {
+            debug!(
+                "Creating {} external workers for {}",
+                model_cards.len(),
+                config.url
+            );
+
+            // Create a worker for each model
+            for model_card in model_cards.iter() {
+                let mut builder = BasicWorkerBuilder::new(normalized_url.clone())
+                    .model(model_card.clone())
+                    .worker_type(WorkerType::Regular)
+                    .connection_mode(ConnectionMode::Http)
+                    .runtime_type(RuntimeType::External)
+                    .circuit_breaker_config(circuit_breaker_config.clone())
+                    .health_config(health_config.clone());
+
+                if let Some(ref api_key) = config.api_key {
+                    builder = builder.api_key(api_key.clone());
+                }
+
+                if !labels.is_empty() {
+                    builder = builder.labels(labels.clone());
+                }
+
+                let worker = Arc::new(builder.build()) as Arc<dyn Worker>;
+                if health_config.disable_health_check {
+                    worker.set_healthy(true);
+                } else {
+                    worker.set_healthy(false);
+                }
+
+                debug!(
+                    "Created external worker for model {} at {}",
+                    model_card.id, normalized_url
+                );
+
+                workers.push(worker);
+            }
+
+            info!(
+                "Created {} external workers from {}",
+                workers.len(),
+                config.url
+            );
+        }
+
+        // Store results in workflow data
+        context.data.workers = Some(WorkerList::from_workers(&workers));
+        context.data.actual_workers = Some(workers);
+        context.data.labels = labels;
+        Ok(StepResult::Success)
+    }
+
+    fn is_retryable(&self, _error: &WorkflowError) -> bool {
+        false
+    }
+}

third_party/sglang/sgl-model-gateway/src/core/steps/worker/external/discover_models.rs

@@ -0,0 +1,274 @@
+//! Model discovery step for external API endpoints.
+
+use std::{collections::HashMap, time::Duration};
+
+use async_trait::async_trait;
+use once_cell::sync::Lazy;
+use regex::Regex;
+use reqwest::Client;
+use serde::Deserialize;
+use tracing::{debug, info};
+use wfaas::{StepExecutor, StepId, StepResult, WorkflowContext, WorkflowError, WorkflowResult};
+
+use crate::core::{
+    model_card::{ModelCard, ProviderType},
+    model_type::ModelType,
+    steps::workflow_data::ExternalWorkerWorkflowData,
+};
+
+// HTTP client for API calls
+static HTTP_CLIENT: Lazy<Client> = Lazy::new(|| {
+    Client::builder()
+        .timeout(Duration::from_secs(30))
+        .build()
+        .expect("Failed to create HTTP client")
+});
+
+// Regex to strip date suffix: -YYYY-MM-DD or -YYYY-MM
+static DATE_SUFFIX_PATTERN: Lazy<Regex> =
+    Lazy::new(|| Regex::new(r"-\d{4}-\d{2}(-\d{2})?$").expect("Invalid date regex"));
+
+/// OpenAI /v1/models response format.
+#[derive(Debug, Clone, Deserialize)]
+pub struct ModelsResponse {
+    pub data: Vec<ModelInfo>,
+    #[serde(default)]
+    pub object: String,
+}
+
+/// Individual model information from /v1/models.
+#[derive(Debug, Clone, Deserialize)]
+pub struct ModelInfo {
+    pub id: String,
+    #[serde(default)]
+    pub object: String,
+    #[serde(default)]
+    pub created: Option<u64>,
+    #[serde(default)]
+    pub owned_by: Option<String>,
+}
+
+/// Group models by base name (stripping date suffixes) and create ModelCards with aliases.
+///
+/// # Example
+/// Input:  `["gpt-4o", "gpt-4o-2024-05-13", "gpt-4o-2024-08-06", "gpt-4o-2024-11-20"]`
+/// Output: `ModelCard { id: "gpt-4o", aliases: ["gpt-4o-2024-05-13", "gpt-4o-2024-08-06", "gpt-4o-2024-11-20"] }`
+pub fn group_models_into_cards(models: Vec<ModelInfo>) -> Vec<ModelCard> {
+    // Group model IDs by base name (with date stripped)
+    let mut groups: HashMap<String, Vec<String>> = HashMap::new();
+    for model in &models {
+        let base = DATE_SUFFIX_PATTERN.replace(&model.id, "").to_string();
+        groups.entry(base).or_default().push(model.id.clone());
+    }
+
+    // Create ModelCard for each group
+    groups
+        .into_values()
+        .map(|mut variants| {
+            // Sort: shortest first (base name), then alphabetically
+            variants.sort_by(|a, b| a.len().cmp(&b.len()).then_with(|| a.cmp(b)));
+
+            let primary_id = variants.remove(0); // shortest = primary ID
+            let aliases = variants; // rest = aliases
+
+            let model_type = infer_model_type_from_id(&primary_id);
+            let provider = infer_provider_from_id(&primary_id);
+
+            let mut card = ModelCard::new(&primary_id)
+                .with_aliases(aliases)
+                .with_model_type(model_type);
+
+            if let Some(p) = provider {
+                card = card.with_provider(p);
+            }
+
+            card
+        })
+        .collect()
+}
+
+/// Infer ModelType from model ID string.
+pub fn infer_model_type_from_id(id: &str) -> ModelType {
+    let id_lower = id.to_lowercase();
+
+    // Embedding models
+    if id_lower.contains("embed") || id_lower.contains("ada-002") {
+        return ModelType::EMBED_MODEL;
+    }
+
+    // Rerank models
+    if id_lower.contains("rerank") {
+        return ModelType::RERANK_MODEL;
+    }
+
+    // Image generation models
+    if id_lower.starts_with("dall-e")
+        || id_lower.starts_with("sora")
+        || (id_lower.contains("image") && !id_lower.contains("vision"))
+    {
+        return ModelType::IMAGE_MODEL;
+    }
+
+    // Audio models
+    if id_lower.starts_with("tts")
+        || id_lower.starts_with("whisper")
+        || id_lower.contains("audio")
+        || id_lower.contains("realtime")
+        || id_lower.contains("transcribe")
+    {
+        return ModelType::AUDIO_MODEL;
+    }
+
+    // Moderation models
+    if id_lower.contains("moderation") {
+        return ModelType::MODERATION_MODEL;
+    }
+
+    // Vision LLM
+    if id_lower.contains("vision") || id_lower.contains("4o") {
+        return ModelType::VISION_LLM;
+    }
+
+    // Reasoning models
+    if id_lower.starts_with("o1") || id_lower.starts_with("o3") {
+        return ModelType::REASONING_LLM;
+    }
+
+    // Default to standard LLM
+    ModelType::LLM
+}
+
+/// Infer provider type from model ID string.
+fn infer_provider_from_id(id: &str) -> Option<ProviderType> {
+    let id_lower = id.to_lowercase();
+
+    // OpenAI models
+    if id_lower.starts_with("gpt")
+        || id_lower.starts_with("o1")
+        || id_lower.starts_with("o3")
+        || id_lower.starts_with("dall-e")
+        || id_lower.starts_with("whisper")
+        || id_lower.starts_with("tts")
+        || id_lower.starts_with("text-embedding")
+        || id_lower.starts_with("babbage")
+        || id_lower.starts_with("davinci")
+        || id_lower.contains("omni")
+    {
+        return Some(ProviderType::OpenAI);
+    }
+
+    // xAI/Grok models
+    if id_lower.starts_with("grok") {
+        return Some(ProviderType::XAI);
+    }
+
+    // Anthropic Claude models
+    if id_lower.starts_with("claude") {
+        return Some(ProviderType::Anthropic);
+    }
+
+    // Google Gemini models
+    if id_lower.starts_with("gemini") {
+        return Some(ProviderType::Gemini);
+    }
+
+    None
+}
+
+/// Fetch models from /v1/models endpoint.
+async fn fetch_models(url: &str, api_key: Option<&str>) -> Result<Vec<ModelCard>, String> {
+    let base_url = url.trim_end_matches('/');
+    let models_url = format!("{}/v1/models", base_url);
+
+    let mut req = HTTP_CLIENT.get(&models_url);
+    if let Some(key) = api_key {
+        req = req.bearer_auth(key);
+    }
+
+    let response = req
+        .send()
+        .await
+        .map_err(|e| format!("Failed to connect to {}: {}", models_url, e))?;
+
+    if !response.status().is_success() {
+        return Err(format!(
+            "Server returned status {} from {}",
+            response.status(),
+            models_url
+        ));
+    }
+
+    let models_response: ModelsResponse = response
+        .json()
+        .await
+        .map_err(|e| format!("Failed to parse models response: {}", e))?;
+
+    debug!(
+        "Fetched {} raw models from {}",
+        models_response.data.len(),
+        url
+    );
+
+    let model_cards = group_models_into_cards(models_response.data);
+
+    debug!(
+        "Grouped into {} model cards with aliases",
+        model_cards.len()
+    );
+
+    Ok(model_cards)
+}
+
+/// Step 1: Discover models from external /v1/models endpoint.
+pub struct DiscoverModelsStep;
+
+#[async_trait]
+impl StepExecutor<ExternalWorkerWorkflowData> for DiscoverModelsStep {
+    async fn execute(
+        &self,
+        context: &mut WorkflowContext<ExternalWorkerWorkflowData>,
+    ) -> WorkflowResult<StepResult> {
+        let config = &context.data.config;
+
+        // If no API key is provided, skip model discovery and use wildcard mode.
+        if config.api_key.as_ref().is_none_or(|k| k.is_empty()) {
+            info!(
+                "No API key provided for {} - using wildcard mode (accepts any model). \
+                 User's Authorization header will be forwarded to backend.",
+                config.url
+            );
+            // Leave model_cards empty for wildcard mode
+            return Ok(StepResult::Success);
+        }
+
+        debug!("Discovering models from external endpoint {}", config.url);
+
+        let model_cards = fetch_models(&config.url, config.api_key.as_deref())
+            .await
+            .map_err(|e| WorkflowError::StepFailed {
+                step_id: StepId::new("discover_models"),
+                message: format!("Failed to discover models from {}: {}", config.url, e),
+            })?;
+
+        if model_cards.is_empty() {
+            return Err(WorkflowError::StepFailed {
+                step_id: StepId::new("discover_models"),
+                message: format!("No models discovered from {}", config.url),
+            });
+        }
+
+        info!(
+            "Discovered {} models from {}: {:?}",
+            model_cards.len(),
+            config.url,
+            model_cards.iter().map(|c| &c.id).collect::<Vec<_>>()
+        );
+
+        context.data.model_cards = model_cards;
+        Ok(StepResult::Success)
+    }
+
+    fn is_retryable(&self, _error: &WorkflowError) -> bool {
+        true
+    }
+}

third_party/sglang/sgl-model-gateway/src/core/steps/worker/external/mod.rs

@@ -0,0 +1,121 @@
+//! External worker registration steps for OpenAI-compatible API endpoints.
+//!
+//! These steps handle the discovery and creation of workers that connect to
+//! external API providers (OpenAI, Anthropic, etc.) via HTTP.
+
+mod create_workers;
+mod discover_models;
+
+use std::{sync::Arc, time::Duration};
+
+pub use create_workers::CreateExternalWorkersStep;
+pub use discover_models::{
+    group_models_into_cards, infer_model_type_from_id, DiscoverModelsStep, ModelInfo,
+    ModelsResponse,
+};
+use wfaas::{BackoffStrategy, FailureAction, RetryPolicy, StepDefinition, WorkflowDefinition};
+
+use super::shared::{ActivateWorkersStep, RegisterWorkersStep, UpdatePoliciesStep};
+use crate::{
+    app_context::AppContext, core::steps::workflow_data::ExternalWorkerWorkflowData,
+    protocols::worker_spec::WorkerConfigRequest,
+};
+
+/// Create external worker registration workflow definition.
+///
+/// DAG structure with parallel execution opportunities:
+/// ```text
+///              discover_models
+///                    │
+///              create_workers
+///                    │
+///             register_workers
+///                    │
+///       ┌────────────┴────────────┐
+///       │                         │
+///  update_policies         activate_workers
+///       │                         │
+///       └────────────┴────────────┘
+/// ```
+pub fn create_external_worker_workflow() -> WorkflowDefinition<ExternalWorkerWorkflowData> {
+    WorkflowDefinition::new(
+        "external_worker_registration",
+        "External Worker Registration",
+    )
+    // Step 1: Discover models from /v1/models endpoint
+    .add_step(
+        StepDefinition::new(
+            "discover_models",
+            "Discover Models",
+            Arc::new(DiscoverModelsStep),
+        )
+        .with_retry(RetryPolicy {
+            max_attempts: 3,
+            backoff: BackoffStrategy::Exponential {
+                base: Duration::from_secs(1),
+                max: Duration::from_secs(10),
+            },
+        })
+        .with_timeout(Duration::from_secs(30))
+        .with_failure_action(FailureAction::FailWorkflow),
+    )
+    // Step 2: Create workers for each model
+    .add_step(
+        StepDefinition::new(
+            "create_workers",
+            "Create Workers",
+            Arc::new(CreateExternalWorkersStep),
+        )
+        .with_timeout(Duration::from_secs(5))
+        .with_failure_action(FailureAction::FailWorkflow)
+        .depends_on(&["discover_models"]),
+    )
+    // Step 3: Register workers (shared step)
+    .add_step(
+        StepDefinition::new(
+            "register_workers",
+            "Register Workers",
+            Arc::new(RegisterWorkersStep),
+        )
+        .with_timeout(Duration::from_secs(5))
+        .with_failure_action(FailureAction::FailWorkflow)
+        .depends_on(&["create_workers"]),
+    )
+    // Step 4a: Update policies (parallel with activation)
+    .add_step(
+        StepDefinition::new(
+            "update_policies",
+            "Update Policies",
+            Arc::new(UpdatePoliciesStep),
+        )
+        .with_timeout(Duration::from_secs(5))
+        .with_failure_action(FailureAction::ContinueNextStep)
+        .depends_on(&["register_workers"]),
+    )
+    // Step 4b: Activate workers (parallel with policy update)
+    .add_step(
+        StepDefinition::new(
+            "activate_workers",
+            "Activate Workers",
+            Arc::new(ActivateWorkersStep),
+        )
+        .with_timeout(Duration::from_secs(5))
+        .with_failure_action(FailureAction::FailWorkflow)
+        .depends_on(&["register_workers"]),
+    )
+}
+
+/// Helper to create initial workflow data for external worker registration
+pub fn create_external_worker_workflow_data(
+    config: WorkerConfigRequest,
+    app_context: Arc<AppContext>,
+) -> ExternalWorkerWorkflowData {
+    ExternalWorkerWorkflowData {
+        config,
+        model_cards: Vec::new(),
+        workers: None,
+        labels: std::collections::HashMap::new(),
+        app_context: Some(app_context),
+        actual_workers: None,
+    }
+}

third_party/sglang/sgl-model-gateway/src/core/steps/worker/local/create_worker.rs

@@ -0,0 +1,398 @@
+//! Local worker creation step.
+
+use std::{collections::HashMap, sync::Arc, time::Duration};
+
+use async_trait::async_trait;
+use tracing::debug;
+use wfaas::{StepExecutor, StepId, StepResult, WorkflowContext, WorkflowError, WorkflowResult};
+
+use crate::{
+    app_context::AppContext,
+    core::{
+        circuit_breaker::CircuitBreakerConfig,
+        model_card::ModelCard,
+        steps::workflow_data::LocalWorkerWorkflowData,
+        worker::{HealthConfig, RuntimeType, WorkerType},
+        BasicWorkerBuilder, ConnectionMode, DPAwareWorkerBuilder, Worker, UNKNOWN_MODEL_ID,
+    },
+    protocols::worker_spec::WorkerConfigRequest,
+};
+
+/// Step 3: Create worker object(s) with merged configuration + metadata.
+///
+/// This step:
+/// 1. Merges discovered labels with config labels
+/// 2. Determines the model ID from various sources
+/// 3. Creates ModelCard with metadata
+/// 4. Builds worker(s) - either single worker or multiple DP-aware workers
+/// 5. Outputs unified `workers: Vec<Arc<dyn Worker>>` for downstream steps
+pub struct CreateLocalWorkerStep;
+
+#[async_trait]
+impl StepExecutor<LocalWorkerWorkflowData> for CreateLocalWorkerStep {
+    async fn execute(
+        &self,
+        context: &mut WorkflowContext<LocalWorkerWorkflowData>,
+    ) -> WorkflowResult<StepResult> {
+        let config = &context.data.config;
+        let app_context = context
+            .data
+            .app_context
+            .as_ref()
+            .ok_or_else(|| WorkflowError::ContextValueNotFound("app_context".to_string()))?;
+        let connection_mode =
+            context.data.connection_mode.as_ref().ok_or_else(|| {
+                WorkflowError::ContextValueNotFound("connection_mode".to_string())
+            })?;
+        let discovered_labels = &context.data.discovered_labels;
+
+        // Check if worker already exists
+        if app_context
+            .worker_registry
+            .get_by_url(&config.url)
+            .is_some()
+        {
+            return Err(WorkflowError::StepFailed {
+                step_id: StepId::new("create_worker"),
+                message: format!("Worker {} already exists", config.url),
+            });
+        }
+
+        // Build labels from config
+        let mut config_labels = config.labels.clone();
+        if let Some(priority) = config.priority {
+            config_labels.insert("priority".to_string(), priority.to_string());
+        }
+        if let Some(cost) = config.cost {
+            config_labels.insert("cost".to_string(), cost.to_string());
+        }
+
+        // Merge: discovered labels first, then config labels (config takes precedence)
+        let mut final_labels = discovered_labels.clone();
+        for (key, value) in &config_labels {
+            final_labels.insert(key.clone(), value.clone());
+        }
+
+        // Determine model_id: config > served_model_name > model_path > UNKNOWN_MODEL_ID
+        let model_id = config
+            .model_id
+            .clone()
+            .or_else(|| final_labels.get("served_model_name").cloned())
+            .or_else(|| final_labels.get("model_path").cloned())
+            .unwrap_or_else(|| UNKNOWN_MODEL_ID.to_string());
+
+        if model_id != UNKNOWN_MODEL_ID {
+            debug!("Using model_id: {}", model_id);
+        }
+
+        // Create ModelCard
+        let model_card = build_model_card(&model_id, config, &final_labels);
+
+        debug!(
+            "Creating worker {} with {} discovered + {} config = {} final labels",
+            config.url,
+            discovered_labels.len(),
+            config_labels.len(),
+            final_labels.len()
+        );
+
+        // Parse worker type
+        let worker_type = parse_worker_type(config);
+
+        // Get runtime type (for gRPC workers)
+        let runtime_type = determine_runtime_type(connection_mode, &context.data, config);
+
+        // Build circuit breaker config
+        let circuit_breaker_config = build_circuit_breaker_config(app_context);
+
+        // Build health config
+        let health_config = build_health_config(app_context, config);
+
+        // Normalize URL
+        let normalized_url = normalize_url(&config.url, connection_mode);
+
+        if normalized_url != config.url {
+            debug!(
+                "Normalized worker URL: {} -> {} ({:?})",
+                config.url, normalized_url, connection_mode
+            );
+        }
+
+        // Create workers - always output as Vec for unified downstream handling
+        let workers = if config.dp_aware {
+            create_dp_aware_workers(
+                &context.data,
+                &normalized_url,
+                model_card,
+                worker_type,
+                connection_mode,
+                runtime_type,
+                circuit_breaker_config,
+                health_config,
+                config,
+                &final_labels,
+            )?
+        } else {
+            create_single_worker(
+                &normalized_url,
+                model_card,
+                worker_type,
+                connection_mode,
+                runtime_type,
+                circuit_breaker_config,
+                health_config,
+                config,
+                &final_labels,
+            )
+        };
+
+        // Update workflow data
+        context.data.actual_workers = Some(workers);
+        context.data.final_labels = final_labels;
+        Ok(StepResult::Success)
+    }
+
+    fn is_retryable(&self, _error: &WorkflowError) -> bool {
+        false
+    }
+}
+
+fn build_model_card(
+    model_id: &str,
+    config: &WorkerConfigRequest,
+    labels: &HashMap<String, String>,
+) -> ModelCard {
+    let mut card = ModelCard::new(model_id);
+
+    if let Some(ref tokenizer_path) = config.tokenizer_path {
+        card = card.with_tokenizer_path(tokenizer_path.clone());
+    }
+    if let Some(ref reasoning_parser) = config.reasoning_parser {
+        card = card.with_reasoning_parser(reasoning_parser.clone());
+    }
+    if let Some(ref tool_parser) = config.tool_parser {
+        card = card.with_tool_parser(tool_parser.clone());
+    }
+    if let Some(ref chat_template) = config.chat_template {
+        card = card.with_chat_template(chat_template.clone());
+    }
+    if let Some(model_type_str) = labels.get("model_type") {
+        card = card.with_hf_model_type(model_type_str.clone());
+    }
+    if let Some(architectures_json) = labels.get("architectures") {
+        if let Ok(architectures) = serde_json::from_str::<Vec<String>>(architectures_json) {
+            card = card.with_architectures(architectures);
+        }
+    }
+
+    // Parse classification model id2label mapping
+    // The proto field is id2label_json: JSON string like {"0": "negative", "1": "positive"}
+    if let Some(id2label_json) = labels.get("id2label_json") {
+        if !id2label_json.is_empty() {
+            // Parse JSON: keys are string indices, values are label names
+            if let Ok(string_map) = serde_json::from_str::<HashMap<String, String>>(id2label_json) {
+                // Convert string keys ("0", "1") to u32 keys (0, 1)
+                let id2label: HashMap<u32, String> = string_map
+                    .into_iter()
+                    .filter_map(|(k, v)| k.parse::<u32>().ok().map(|idx| (idx, v)))
+                    .collect();
+
+                if !id2label.is_empty() {
+                    card = card.with_id2label(id2label);
+                    debug!("Parsed id2label with {} classes", card.num_labels);
+                }
+            }
+        }
+    }
+    // Fallback: if num_labels is set but id2label wasn't parsed, create default labels
+    // Match logic in serving_classify.py::_get_id2label_mapping
+    else if let Some(num_labels_str) = labels.get("num_labels") {
+        if let Ok(num_labels) = num_labels_str.parse::<u32>() {
+            if num_labels > 0 {
+                // Create default mapping: {0: "LABEL_0", 1: "LABEL_1", ...}
+                let id2label: HashMap<u32, String> = (0..num_labels)
+                    .map(|i| (i, format!("LABEL_{}", i)))
+                    .collect();
+                card = card.with_id2label(id2label);
+                debug!("Created default id2label with {} classes", num_labels);
+            }
+        }
+    }
+
+    card
+}
+
+fn parse_worker_type(config: &WorkerConfigRequest) -> WorkerType {
+    config
+        .worker_type
+        .as_ref()
+        .map(|t| match t.as_str() {
+            "prefill" => WorkerType::Prefill {
+                bootstrap_port: config.bootstrap_port,
+            },
+            "decode" => WorkerType::Decode,
+            _ => WorkerType::Regular,
+        })
+        .unwrap_or(WorkerType::Regular)
+}
+
+fn determine_runtime_type(
+    connection_mode: &ConnectionMode,
+    data: &LocalWorkerWorkflowData,
+    config: &WorkerConfigRequest,
+) -> RuntimeType {
+    if !matches!(connection_mode, ConnectionMode::Grpc { .. }) {
+        return RuntimeType::Sglang;
+    }
+
+    if let Some(ref detected_runtime) = data.detected_runtime_type {
+        match detected_runtime.as_str() {
+            "vllm" => RuntimeType::Vllm,
+            _ => RuntimeType::Sglang,
+        }
+    } else if let Some(ref runtime) = config.runtime {
+        match runtime.as_str() {
+            "vllm" => RuntimeType::Vllm,
+            _ => RuntimeType::Sglang,
+        }
+    } else {
+        RuntimeType::Sglang
+    }
+}
+
+fn build_circuit_breaker_config(app_context: &AppContext) -> CircuitBreakerConfig {
+    let cfg = app_context.router_config.effective_circuit_breaker_config();
+    CircuitBreakerConfig {
+        failure_threshold: cfg.failure_threshold,
+        success_threshold: cfg.success_threshold,
+        timeout_duration: Duration::from_secs(cfg.timeout_duration_secs),
+        window_duration: Duration::from_secs(cfg.window_duration_secs),
+    }
+}
+
+fn build_health_config(app_context: &AppContext, config: &WorkerConfigRequest) -> HealthConfig {
+    let cfg = &app_context.router_config.health_check;
+    HealthConfig {
+        timeout_secs: cfg.timeout_secs,
+        check_interval_secs: cfg.check_interval_secs,
+        endpoint: cfg.endpoint.clone(),
+        failure_threshold: cfg.failure_threshold,
+        success_threshold: cfg.success_threshold,
+        disable_health_check: cfg.disable_health_check || config.disable_health_check,
+    }
+}
+
+fn normalize_url(url: &str, connection_mode: &ConnectionMode) -> String {
+    if url.starts_with("http://") || url.starts_with("https://") || url.starts_with("grpc://") {
+        url.to_string()
+    } else {
+        match connection_mode {
+            ConnectionMode::Http => format!("http://{}", url),
+            ConnectionMode::Grpc { .. } => format!("grpc://{}", url),
+        }
+    }
+}
+
+#[allow(clippy::too_many_arguments)]
+fn create_dp_aware_workers(
+    data: &LocalWorkerWorkflowData,
+    normalized_url: &str,
+    model_card: ModelCard,
+    worker_type: WorkerType,
+    connection_mode: &ConnectionMode,
+    runtime_type: RuntimeType,
+    circuit_breaker_config: CircuitBreakerConfig,
+    health_config: HealthConfig,
+    config: &WorkerConfigRequest,
+    final_labels: &HashMap<String, String>,
+) -> Result<Vec<Arc<dyn Worker>>, WorkflowError> {
+    let dp_info = data
+        .dp_info
+        .as_ref()
+        .ok_or_else(|| WorkflowError::ContextValueNotFound("dp_info".to_string()))?;
+
+    debug!(
+        "Creating {} DP-aware workers for {} (dp_size: {})",
+        dp_info.dp_size, normalized_url, dp_info.dp_size
+    );
+
+    let mut workers = Vec::with_capacity(dp_info.dp_size);
+    for rank in 0..dp_info.dp_size {
+        let mut builder =
+            DPAwareWorkerBuilder::new(normalized_url.to_string(), rank, dp_info.dp_size)
+                .model(model_card.clone())
+                .worker_type(worker_type.clone())
+                .connection_mode(connection_mode.clone())
+                .runtime_type(runtime_type.clone())
+                .circuit_breaker_config(circuit_breaker_config.clone())
+                .health_config(health_config.clone());
+
+        if let Some(ref api_key) = config.api_key {
+            builder = builder.api_key(api_key.clone());
+        }
+        if !final_labels.is_empty() {
+            builder = builder.labels(final_labels.clone());
+        }
+
+        let worker = Arc::new(builder.build()) as Arc<dyn Worker>;
+        if health_config.disable_health_check {
+            worker.set_healthy(true);
+        } else {
+            worker.set_healthy(false);
+        }
+        workers.push(worker);
+
+        debug!(
+            "Created DP-aware worker {}@{}/{} ({:?})",
+            normalized_url, rank, dp_info.dp_size, connection_mode
+        );
+    }
+
+    Ok(workers)
+}
+
+#[allow(clippy::too_many_arguments)]
+fn create_single_worker(
+    normalized_url: &str,
+    model_card: ModelCard,
+    worker_type: WorkerType,
+    connection_mode: &ConnectionMode,
+    runtime_type: RuntimeType,
+    circuit_breaker_config: CircuitBreakerConfig,
+    health_config: HealthConfig,
+    config: &WorkerConfigRequest,
+    final_labels: &HashMap<String, String>,
+) -> Vec<Arc<dyn Worker>> {
+    let health_check_disabled = health_config.disable_health_check;
+
+    let mut builder = BasicWorkerBuilder::new(normalized_url.to_string())
+        .model(model_card)
+        .worker_type(worker_type)
+        .connection_mode(connection_mode.clone())
+        .runtime_type(runtime_type)
+        .circuit_breaker_config(circuit_breaker_config)
+        .health_config(health_config);
+
+    if let Some(ref api_key) = config.api_key {
+        builder = builder.api_key(api_key.clone());
+    }
+    if !final_labels.is_empty() {
+        builder = builder.labels(final_labels.clone());
+    }
+
+    let worker = Arc::new(builder.build()) as Arc<dyn Worker>;
+    if health_check_disabled {
+        worker.set_healthy(true);
+    } else {
+        worker.set_healthy(false);
+    }
+
+    debug!(
+        "Created worker object for {} ({:?}) with {} labels",
+        normalized_url,
+        connection_mode,
+        final_labels.len()
+    );
+
+    vec![worker]
+}

third_party/sglang/sgl-model-gateway/src/core/steps/worker/local/detect_connection.rs

@@ -0,0 +1,144 @@
+//! Connection mode detection step.
+
+use std::time::Duration;
+
+use async_trait::async_trait;
+use reqwest::Client;
+use tracing::debug;
+use wfaas::{StepExecutor, StepId, StepResult, WorkflowContext, WorkflowError, WorkflowResult};
+
+use super::strip_protocol;
+use crate::{
+    core::{steps::workflow_data::LocalWorkerWorkflowData, ConnectionMode},
+    routers::grpc::client::GrpcClient,
+};
+
+/// Try HTTP health check.
+async fn try_http_health_check(
+    url: &str,
+    timeout_secs: u64,
+    client: &Client,
+) -> Result<(), String> {
+    let is_https = url.starts_with("https://");
+    let protocol = if is_https { "https" } else { "http" };
+    let clean_url = strip_protocol(url);
+    let health_url = format!("{}://{}/health", protocol, clean_url);
+
+    client
+        .get(&health_url)
+        .timeout(Duration::from_secs(timeout_secs))
+        .send()
+        .await
+        .and_then(reqwest::Response::error_for_status)
+        .map_err(|e| format!("Health check failed: {}", e))?;
+
+    Ok(())
+}
+
+/// Perform gRPC health check with runtime type.
+async fn do_grpc_health_check(
+    grpc_url: &str,
+    timeout_secs: u64,
+    runtime_type: &str,
+) -> Result<(), String> {
+    let connect_future = GrpcClient::connect(grpc_url, runtime_type);
+    let client = tokio::time::timeout(Duration::from_secs(timeout_secs), connect_future)
+        .await
+        .map_err(|_| "gRPC connection timeout".to_string())?
+        .map_err(|e| format!("gRPC connection failed: {}", e))?;
+
+    let health_future = client.health_check();
+    tokio::time::timeout(Duration::from_secs(timeout_secs), health_future)
+        .await
+        .map_err(|_| "gRPC health check timeout".to_string())?
+        .map_err(|e| format!("gRPC health check failed: {}", e))?;
+
+    Ok(())
+}
+
+/// Try gRPC health check (tries SGLang first, then vLLM if not specified).
+async fn try_grpc_health_check(
+    url: &str,
+    timeout_secs: u64,
+    runtime_type: Option<&str>,
+) -> Result<(), String> {
+    let grpc_url = if url.starts_with("grpc://") {
+        url.to_string()
+    } else {
+        format!("grpc://{}", strip_protocol(url))
+    };
+
+    match runtime_type {
+        Some(runtime) => do_grpc_health_check(&grpc_url, timeout_secs, runtime).await,
+        None => {
+            // Try SGLang first, then vLLM as fallback
+            if let Ok(()) = do_grpc_health_check(&grpc_url, timeout_secs, "sglang").await {
+                return Ok(());
+            }
+            do_grpc_health_check(&grpc_url, timeout_secs, "vllm")
+                .await
+                .map_err(|e| format!("gRPC failed (tried SGLang and vLLM): {}", e))
+        }
+    }
+}
+
+/// Step 1: Detect connection mode by probing HTTP and gRPC.
+pub struct DetectConnectionModeStep;
+
+#[async_trait]
+impl StepExecutor<LocalWorkerWorkflowData> for DetectConnectionModeStep {
+    async fn execute(
+        &self,
+        context: &mut WorkflowContext<LocalWorkerWorkflowData>,
+    ) -> WorkflowResult<StepResult> {
+        let config = &context.data.config;
+        let app_context = context
+            .data
+            .app_context
+            .as_ref()
+            .ok_or_else(|| WorkflowError::ContextValueNotFound("app_context".to_string()))?;
+
+        debug!(
+            "Detecting connection mode for {} (timeout: {}s, max_attempts: {})",
+            config.url, config.health_check_timeout_secs, config.max_connection_attempts
+        );
+
+        // Try both protocols in parallel
+        let url = config.url.clone();
+        let timeout = config.health_check_timeout_secs;
+        let client = &app_context.client;
+        let runtime_type = config.runtime.as_deref();
+
+        let (http_result, grpc_result) = tokio::join!(
+            try_http_health_check(&url, timeout, client),
+            try_grpc_health_check(&url, timeout, runtime_type)
+        );
+
+        let connection_mode = match (http_result, grpc_result) {
+            (Ok(_), _) => {
+                debug!("{} detected as HTTP", config.url);
+                ConnectionMode::Http
+            }
+            (_, Ok(_)) => {
+                debug!("{} detected as gRPC", config.url);
+                ConnectionMode::Grpc { port: None }
+            }
+            (Err(http_err), Err(grpc_err)) => {
+                return Err(WorkflowError::StepFailed {
+                    step_id: StepId::new("detect_connection_mode"),
+                    message: format!(
+                        "Both HTTP and gRPC health checks failed for {}: HTTP: {}, gRPC: {}",
+                        config.url, http_err, grpc_err
+                    ),
+                });
+            }
+        };
+
+        context.data.connection_mode = Some(connection_mode);
+        Ok(StepResult::Success)
+    }
+
+    fn is_retryable(&self, _error: &WorkflowError) -> bool {
+        true
+    }
+}

third_party/sglang/sgl-model-gateway/src/core/steps/worker/local/discover_dp.rs

@@ -0,0 +1,78 @@
+//! Data Parallel (DP) information discovery step.
+
+use async_trait::async_trait;
+use tracing::debug;
+use wfaas::{StepExecutor, StepId, StepResult, WorkflowContext, WorkflowError, WorkflowResult};
+
+use super::discover_metadata::get_server_info;
+use crate::core::{steps::workflow_data::LocalWorkerWorkflowData, UNKNOWN_MODEL_ID};
+
+/// DP (Data Parallel) information for a worker.
+#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
+pub struct DpInfo {
+    pub dp_size: usize,
+    pub model_id: String,
+}
+
+/// Get DP info for a worker URL.
+pub async fn get_dp_info(url: &str, api_key: Option<&str>) -> Result<DpInfo, String> {
+    let info = get_server_info(url, api_key).await?;
+
+    let dp_size = info
+        .dp_size
+        .ok_or_else(|| format!("No dp_size in response from {}", url))?;
+
+    let model_id = info
+        .model_id
+        .filter(|s| !s.is_empty())
+        .or(info.served_model_name.filter(|s| !s.is_empty()))
+        .or_else(|| {
+            info.model_path
+                .and_then(|path| path.split('/').next_back().map(|s| s.to_string()))
+        })
+        .unwrap_or_else(|| UNKNOWN_MODEL_ID.to_string());
+
+    Ok(DpInfo { dp_size, model_id })
+}
+
+/// Step 2b: Discover DP (Data Parallel) information (only for DP-aware workers).
+pub struct DiscoverDPInfoStep;
+
+#[async_trait]
+impl StepExecutor<LocalWorkerWorkflowData> for DiscoverDPInfoStep {
+    async fn execute(
+        &self,
+        context: &mut WorkflowContext<LocalWorkerWorkflowData>,
+    ) -> WorkflowResult<StepResult> {
+        let config = &context.data.config;
+
+        if !config.dp_aware {
+            debug!(
+                "Worker {} is not DP-aware, skipping DP discovery",
+                config.url
+            );
+            return Ok(StepResult::Success);
+        }
+
+        debug!("Discovering DP info for {} (DP-aware)", config.url);
+
+        let dp_info = get_dp_info(&config.url, config.api_key.as_deref())
+            .await
+            .map_err(|e| WorkflowError::StepFailed {
+                step_id: StepId::new("discover_dp_info"),
+                message: format!("Failed to get DP info: {}", e),
+            })?;
+
+        debug!(
+            "Discovered DP size {} for {} (model: {})",
+            dp_info.dp_size, config.url, dp_info.model_id
+        );
+
+        context.data.dp_info = Some(dp_info);
+        Ok(StepResult::Success)
+    }
+
+    fn is_retryable(&self, _error: &WorkflowError) -> bool {
+        true
+    }
+}

third_party/sglang/sgl-model-gateway/src/core/steps/worker/local/discover_metadata.rs

@@ -0,0 +1,320 @@
+//! Metadata discovery step for local workers.
+
+use std::{collections::HashMap, time::Duration};
+
+use async_trait::async_trait;
+use once_cell::sync::Lazy;
+use reqwest::Client;
+use serde::{Deserialize, Serialize};
+use serde_json::Value;
+use tracing::{debug, warn};
+use wfaas::{StepExecutor, StepResult, WorkflowContext, WorkflowError, WorkflowResult};
+
+use super::strip_protocol;
+use crate::{
+    core::{steps::workflow_data::LocalWorkerWorkflowData, ConnectionMode},
+    routers::grpc::client::GrpcClient,
+};
+
+// HTTP client for metadata fetching
+static HTTP_CLIENT: Lazy<Client> = Lazy::new(|| {
+    Client::builder()
+        .timeout(Duration::from_secs(10))
+        .build()
+        .expect("Failed to create HTTP client")
+});
+
+/// Server information returned from /server_info endpoint.
+#[derive(Debug, Clone, Deserialize, Serialize)]
+pub struct ServerInfo {
+    #[serde(alias = "model")]
+    pub model_id: Option<String>,
+    pub model_path: Option<String>,
+    pub served_model_name: Option<String>,
+    pub tp_size: Option<usize>,
+    pub dp_size: Option<usize>,
+    pub load_balance_method: Option<String>,
+    pub disaggregation_mode: Option<String>,
+    pub version: Option<String>,
+    pub max_batch_size: Option<usize>,
+    pub max_total_tokens: Option<usize>,
+    pub max_prefill_tokens: Option<usize>,
+    pub max_running_requests: Option<usize>,
+    pub max_num_reqs: Option<usize>,
+}
+
+/// Model information returned from /model_info endpoint.
+#[derive(Debug, Clone, Deserialize, Serialize)]
+pub struct ModelInfo {
+    pub model_path: Option<String>,
+    pub tokenizer_path: Option<String>,
+    pub is_generation: Option<bool>,
+    pub model_type: Option<String>,
+    pub architectures: Option<Vec<String>>,
+}
+
+/// Fallback function to GET JSON from old endpoint (with "get_" prefix) for backward compatibility.
+async fn get_json_fallback(
+    base_url: &str,
+    endpoint: &str,
+    api_key: Option<&str>,
+) -> Result<Value, String> {
+    // FIXME: This fallback logic should be removed together with /get_server_info
+    // and /get_model_info endpoints in http_server.py
+    warn!(
+        concat!(
+            "Endpoint '/{}' returned 404, falling back to '/get_{}' for backward compatibility. ",
+            "The '/get_{}' endpoint is deprecated and will be removed in a future version. ",
+            "Please use '/{}' instead."
+        ),
+        endpoint, endpoint, endpoint, endpoint
+    );
+
+    let old_url = format!("{}/get_{}", base_url, endpoint);
+    let mut req = HTTP_CLIENT.get(&old_url);
+    if let Some(key) = api_key {
+        req = req.bearer_auth(key);
+    }
+
+    let response = req
+        .send()
+        .await
+        .map_err(|e| format!("Failed to connect to {}: {}", old_url, e))?;
+
+    if !response.status().is_success() {
+        return Err(format!(
+            "Server returned status {} from {}",
+            response.status(),
+            old_url
+        ));
+    }
+
+    response
+        .json::<Value>()
+        .await
+        .map_err(|e| format!("Failed to parse response from {}: {}", old_url, e))
+}
+
+/// Get server info from /server_info endpoint.
+pub async fn get_server_info(url: &str, api_key: Option<&str>) -> Result<ServerInfo, String> {
+    let base_url = url.trim_end_matches('/');
+    let server_info_url = format!("{}/server_info", base_url);
+
+    let mut req = HTTP_CLIENT.get(&server_info_url);
+    if let Some(key) = api_key {
+        req = req.bearer_auth(key);
+    }
+
+    let response = req
+        .send()
+        .await
+        .map_err(|e| format!("Failed to connect to {}: {}", server_info_url, e))?;
+
+    // If /server_info returns 404, fallback to /get_server_info for backward compatibility
+    if response.status() == reqwest::StatusCode::NOT_FOUND {
+        let json = get_json_fallback(base_url, "server_info", api_key).await?;
+        return serde_json::from_value(json)
+            .map_err(|e| format!("Failed to parse server info: {}", e));
+    }
+
+    if !response.status().is_success() {
+        return Err(format!(
+            "Server returned status {} from {}",
+            response.status(),
+            server_info_url
+        ));
+    }
+
+    let json = response
+        .json::<Value>()
+        .await
+        .map_err(|e| format!("Failed to parse response from {}: {}", server_info_url, e))?;
+
+    serde_json::from_value(json).map_err(|e| format!("Failed to parse server info: {}", e))
+}
+
+/// Get model info from /model_info endpoint.
+pub async fn get_model_info(url: &str, api_key: Option<&str>) -> Result<ModelInfo, String> {
+    let base_url = url.trim_end_matches('/');
+    let model_info_url = format!("{}/model_info", base_url);
+
+    let mut req = HTTP_CLIENT.get(&model_info_url);
+    if let Some(key) = api_key {
+        req = req.bearer_auth(key);
+    }
+
+    let response = req
+        .send()
+        .await
+        .map_err(|e| format!("Failed to connect to {}: {}", model_info_url, e))?;
+
+    // If /model_info returns 404, fallback to /get_model_info for backward compatibility
+    if response.status() == reqwest::StatusCode::NOT_FOUND {
+        let json = get_json_fallback(base_url, "model_info", api_key).await?;
+        return serde_json::from_value(json)
+            .map_err(|e| format!("Failed to parse model info: {}", e));
+    }
+
+    if !response.status().is_success() {
+        return Err(format!(
+            "Server returned status {} from {}",
+            response.status(),
+            model_info_url
+        ));
+    }
+
+    response
+        .json::<ModelInfo>()
+        .await
+        .map_err(|e| format!("Failed to parse response from {}: {}", model_info_url, e))
+}
+
+/// Fetch gRPC metadata (returns labels and detected runtime type).
+async fn fetch_grpc_metadata(
+    url: &str,
+    runtime_type: Option<&str>,
+) -> Result<(HashMap<String, String>, String), String> {
+    let grpc_url = if url.starts_with("grpc://") {
+        url.to_string()
+    } else {
+        format!("grpc://{}", strip_protocol(url))
+    };
+
+    async fn do_fetch(
+        grpc_url: &str,
+        runtime_type: &str,
+    ) -> Result<HashMap<String, String>, String> {
+        let client = GrpcClient::connect(grpc_url, runtime_type)
+            .await
+            .map_err(|e| format!("Failed to connect to gRPC: {}", e))?;
+
+        let model_info = client
+            .get_model_info()
+            .await
+            .map_err(|e| format!("Failed to fetch gRPC metadata: {}", e))?;
+
+        Ok(model_info.to_labels())
+    }
+
+    match runtime_type {
+        Some(runtime) => {
+            let labels = do_fetch(&grpc_url, runtime).await?;
+            Ok((labels, runtime.to_string()))
+        }
+        None => {
+            // Try SGLang first, then vLLM as fallback
+            if let Ok(labels) = do_fetch(&grpc_url, "sglang").await {
+                return Ok((labels, "sglang".to_string()));
+            }
+            let labels = do_fetch(&grpc_url, "vllm")
+                .await
+                .map_err(|e| format!("gRPC metadata failed (tried SGLang and vLLM): {}", e))?;
+            Ok((labels, "vllm".to_string()))
+        }
+    }
+}
+
+/// Step 2a: Discover metadata from worker.
+pub struct DiscoverMetadataStep;
+
+#[async_trait]
+impl StepExecutor<LocalWorkerWorkflowData> for DiscoverMetadataStep {
+    async fn execute(
+        &self,
+        context: &mut WorkflowContext<LocalWorkerWorkflowData>,
+    ) -> WorkflowResult<StepResult> {
+        let config = &context.data.config;
+        let connection_mode =
+            context.data.connection_mode.as_ref().ok_or_else(|| {
+                WorkflowError::ContextValueNotFound("connection_mode".to_string())
+            })?;
+
+        debug!(
+            "Discovering metadata for {} ({:?})",
+            config.url, connection_mode
+        );
+
+        let (discovered_labels, detected_runtime) = match connection_mode {
+            ConnectionMode::Http => {
+                let mut labels = HashMap::new();
+
+                // Fetch from /server_info for server-related metadata
+                if let Ok(server_info) =
+                    get_server_info(&config.url, config.api_key.as_deref()).await
+                {
+                    if let Some(model_path) = server_info.model_path.filter(|s| !s.is_empty()) {
+                        labels.insert("model_path".to_string(), model_path);
+                    }
+                    if let Some(served_model_name) =
+                        server_info.served_model_name.filter(|s| !s.is_empty())
+                    {
+                        labels.insert("served_model_name".to_string(), served_model_name);
+                    }
+                    if let Some(tp_size) = server_info.tp_size {
+                        labels.insert("tp_size".to_string(), tp_size.to_string());
+                    }
+                    if let Some(dp_size) = server_info.dp_size {
+                        labels.insert("dp_size".to_string(), dp_size.to_string());
+                    }
+                    if let Some(load_balance_method) = server_info.load_balance_method {
+                        labels.insert("load_balance_method".to_string(), load_balance_method);
+                    }
+                    if let Some(disaggregation_mode) = server_info.disaggregation_mode {
+                        labels.insert("disaggregation_mode".to_string(), disaggregation_mode);
+                    }
+                }
+
+                // Fetch from /model_info for model-related metadata
+                if let Ok(model_info) = get_model_info(&config.url, config.api_key.as_deref()).await
+                {
+                    if let Some(tokenizer_path) =
+                        model_info.tokenizer_path.filter(|s| !s.is_empty())
+                    {
+                        labels.insert("tokenizer_path".to_string(), tokenizer_path);
+                    }
+                    if let Some(model_type) = model_info.model_type.filter(|s| !s.is_empty()) {
+                        labels.insert("model_type".to_string(), model_type);
+                    }
+                    if let Some(architectures) = model_info.architectures.filter(|a| !a.is_empty())
+                    {
+                        if let Ok(json_str) = serde_json::to_string(&architectures) {
+                            labels.insert("architectures".to_string(), json_str);
+                        }
+                    }
+                }
+
+                Ok((labels, None))
+            }
+            ConnectionMode::Grpc { .. } => {
+                let runtime_type = config.runtime.as_deref();
+                fetch_grpc_metadata(&config.url, runtime_type)
+                    .await
+                    .map(|(labels, runtime)| (labels, Some(runtime)))
+            }
+        }
+        .unwrap_or_else(|e| {
+            warn!("Failed to fetch metadata for {}: {}", config.url, e);
+            (HashMap::new(), None)
+        });
+
+        let url = config.url.clone();
+        debug!(
+            "Discovered {} metadata labels for {}",
+            discovered_labels.len(),
+            url
+        );
+
+        // Update workflow data
+        context.data.discovered_labels = discovered_labels;
+        if let Some(runtime) = detected_runtime {
+            debug!("Detected runtime type: {}", runtime);
+            context.data.detected_runtime_type = Some(runtime);
+        }
+
+        Ok(StepResult::Success)
+    }
+
+    fn is_retryable(&self, _error: &WorkflowError) -> bool {
+        true
+    }
+}

third_party/sglang/sgl-model-gateway/src/core/steps/worker/local/find_worker_to_update.rs

@@ -0,0 +1,59 @@
+//! Step to find a worker to update based on URL.
+
+use async_trait::async_trait;
+use tracing::debug;
+use wfaas::{StepExecutor, StepId, StepResult, WorkflowContext, WorkflowError, WorkflowResult};
+
+use super::find_workers_by_url;
+use crate::core::steps::workflow_data::WorkerUpdateWorkflowData;
+
+/// Step to find workers to update based on URL.
+///
+/// For DP-aware workers, finds all workers with matching URL prefix.
+/// For regular workers, finds the single worker with exact URL match.
+pub struct FindWorkerToUpdateStep;
+
+#[async_trait]
+impl StepExecutor<WorkerUpdateWorkflowData> for FindWorkerToUpdateStep {
+    async fn execute(
+        &self,
+        context: &mut WorkflowContext<WorkerUpdateWorkflowData>,
+    ) -> WorkflowResult<StepResult> {
+        let worker_url = &context.data.worker_url;
+        let dp_aware = context.data.dp_aware;
+        let app_context = context
+            .data
+            .app_context
+            .as_ref()
+            .ok_or_else(|| WorkflowError::ContextValueNotFound("app_context".to_string()))?;
+
+        let workers_to_update =
+            find_workers_by_url(&app_context.worker_registry, worker_url, dp_aware);
+
+        if workers_to_update.is_empty() {
+            let error_msg = if dp_aware {
+                format!("No workers found with prefix {}@", worker_url)
+            } else {
+                format!("Worker {} not found", worker_url)
+            };
+            return Err(WorkflowError::StepFailed {
+                step_id: StepId::new("find_worker_to_update"),
+                message: error_msg,
+            });
+        }
+
+        debug!(
+            "Found {} worker(s) to update for {}",
+            workers_to_update.len(),
+            worker_url
+        );
+
+        context.data.workers_to_update = Some(workers_to_update);
+
+        Ok(StepResult::Success)
+    }
+
+    fn is_retryable(&self, _error: &WorkflowError) -> bool {
+        false
+    }
+}

third_party/sglang/sgl-model-gateway/src/core/steps/worker/local/find_workers_to_remove.rs

@@ -0,0 +1,82 @@
+//! Step to find workers to remove based on URL.
+
+use std::collections::HashSet;
+
+use async_trait::async_trait;
+use tracing::debug;
+use wfaas::{StepExecutor, StepId, StepResult, WorkflowContext, WorkflowError, WorkflowResult};
+
+use super::find_workers_by_url;
+use crate::core::steps::workflow_data::{WorkerList, WorkerRemovalWorkflowData};
+
+/// Request structure for worker removal.
+#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
+pub struct WorkerRemovalRequest {
+    pub url: String,
+    pub dp_aware: bool,
+}
+
+/// Step to find workers to remove based on URL.
+///
+/// For DP-aware workers, finds all workers with matching URL prefix.
+/// For regular workers, finds the single worker with exact URL match.
+pub struct FindWorkersToRemoveStep;
+
+#[async_trait]
+impl StepExecutor<WorkerRemovalWorkflowData> for FindWorkersToRemoveStep {
+    async fn execute(
+        &self,
+        context: &mut WorkflowContext<WorkerRemovalWorkflowData>,
+    ) -> WorkflowResult<StepResult> {
+        let request = &context.data.config;
+        let app_context = context
+            .data
+            .app_context
+            .as_ref()
+            .ok_or_else(|| WorkflowError::ContextValueNotFound("app_context".to_string()))?;
+
+        let workers_to_remove =
+            find_workers_by_url(&app_context.worker_registry, &request.url, request.dp_aware);
+
+        if workers_to_remove.is_empty() {
+            let error_msg = if request.dp_aware {
+                format!("No workers found with prefix {}@", request.url)
+            } else {
+                format!("Worker {} not found", request.url)
+            };
+            return Err(WorkflowError::StepFailed {
+                step_id: StepId::new("find_workers_to_remove"),
+                message: error_msg,
+            });
+        }
+
+        debug!(
+            "Found {} worker(s) to remove for {}",
+            workers_to_remove.len(),
+            request.url
+        );
+
+        // Store workers and their model IDs for subsequent steps
+        let worker_urls: Vec<String> = workers_to_remove
+            .iter()
+            .map(|w| w.url().to_string())
+            .collect();
+
+        let affected_models: HashSet<String> = workers_to_remove
+            .iter()
+            .map(|w| w.model_id().to_string())
+            .collect();
+
+        // Update workflow data
+        context.data.workers_to_remove = Some(WorkerList::from_workers(&workers_to_remove));
+        context.data.actual_workers_to_remove = Some(workers_to_remove);
+        context.data.worker_urls = worker_urls;
+        context.data.affected_models = affected_models;
+
+        Ok(StepResult::Success)
+    }
+
+    fn is_retryable(&self, _error: &WorkflowError) -> bool {
+        false
+    }
+}

third_party/sglang/sgl-model-gateway/src/core/steps/worker/local/mod.rs

@@ -0,0 +1,363 @@
+mod create_worker;
+mod detect_connection;
+mod discover_dp;
+mod discover_metadata;
+mod find_worker_to_update;
+mod find_workers_to_remove;
+mod remove_from_policy_registry;
+mod remove_from_worker_registry;
+mod submit_tokenizer_job;
+mod update_policies_for_worker;
+mod update_remaining_policies;
+mod update_worker_properties;
+
+use std::{sync::Arc, time::Duration};
+
+/// Strip protocol prefix (http://, https://, grpc://) from URL.
+pub(crate) fn strip_protocol(url: &str) -> String {
+    url.trim_start_matches("http://")
+        .trim_start_matches("https://")
+        .trim_start_matches("grpc://")
+        .to_string()
+}
+
+pub use create_worker::CreateLocalWorkerStep;
+pub use detect_connection::DetectConnectionModeStep;
+pub use discover_dp::{get_dp_info, DiscoverDPInfoStep, DpInfo};
+pub use discover_metadata::DiscoverMetadataStep;
+pub use find_worker_to_update::FindWorkerToUpdateStep;
+pub use find_workers_to_remove::{FindWorkersToRemoveStep, WorkerRemovalRequest};
+pub use remove_from_policy_registry::RemoveFromPolicyRegistryStep;
+pub use remove_from_worker_registry::RemoveFromWorkerRegistryStep;
+pub use submit_tokenizer_job::SubmitTokenizerJobStep;
+pub use update_policies_for_worker::UpdatePoliciesForWorkerStep;
+pub use update_remaining_policies::UpdateRemainingPoliciesStep;
+pub use update_worker_properties::UpdateWorkerPropertiesStep;
+use wfaas::{BackoffStrategy, FailureAction, RetryPolicy, StepDefinition, WorkflowDefinition};
+
+use super::shared::{ActivateWorkersStep, RegisterWorkersStep, UpdatePoliciesStep};
+use crate::{
+    app_context::AppContext,
+    config::RouterConfig,
+    core::{
+        steps::workflow_data::{
+            LocalWorkerWorkflowData, WorkerRemovalWorkflowData, WorkerUpdateWorkflowData,
+        },
+        Worker, WorkerRegistry,
+    },
+    protocols::worker_spec::{WorkerConfigRequest, WorkerUpdateRequest},
+};
+
+/// Find workers by URL, supporting both DP-aware (prefix match) and regular (exact match) modes.
+///
+/// For DP-aware workers, finds all workers with URL prefix `{url}@`.
+/// For regular workers, finds the single worker with exact URL match.
+pub(crate) fn find_workers_by_url(
+    registry: &WorkerRegistry,
+    url: &str,
+    dp_aware: bool,
+) -> Vec<Arc<dyn Worker>> {
+    if dp_aware {
+        let worker_url_prefix = format!("{}@", url);
+        registry
+            .get_all()
+            .iter()
+            .filter(|worker| worker.url().starts_with(&worker_url_prefix))
+            .cloned()
+            .collect()
+    } else {
+        match registry.get_by_url(url) {
+            Some(worker) => vec![worker],
+            None => Vec::new(),
+        }
+    }
+}
+
+pub fn create_local_worker_workflow(
+    router_config: &RouterConfig,
+) -> WorkflowDefinition<LocalWorkerWorkflowData> {
+    let detect_timeout = Duration::from_secs(router_config.worker_startup_timeout_secs);
+
+    // Calculate max_attempts based on timeout
+    let timeout_secs = detect_timeout.as_secs() as f64;
+    let effective_timeout = timeout_secs * 0.9;
+    let max_attempts = if effective_timeout > 10.0 {
+        (5 + ((effective_timeout - 10.0) / 5.0).ceil() as u32).max(3)
+    } else {
+        3
+    };
+
+    WorkflowDefinition::new("local_worker_registration", "Local Worker Registration")
+        // Step 1: Detect connection mode (HTTP vs gRPC)
+        .add_step(
+            StepDefinition::new(
+                "detect_connection_mode",
+                "Detect Connection Mode",
+                Arc::new(DetectConnectionModeStep),
+            )
+            .with_retry(RetryPolicy {
+                max_attempts,
+                backoff: BackoffStrategy::Linear {
+                    increment: Duration::from_secs(1),
+                    max: Duration::from_secs(5),
+                },
+            })
+            .with_timeout(detect_timeout)
+            .with_failure_action(FailureAction::FailWorkflow),
+        )
+        // Step 2a: Discover metadata (parallel with DP discovery)
+        .add_step(
+            StepDefinition::new(
+                "discover_metadata",
+                "Discover Metadata",
+                Arc::new(DiscoverMetadataStep),
+            )
+            .with_retry(RetryPolicy {
+                max_attempts: 3,
+                backoff: BackoffStrategy::Fixed(Duration::from_secs(1)),
+            })
+            .with_timeout(Duration::from_secs(10))
+            .with_failure_action(FailureAction::ContinueNextStep)
+            .depends_on(&["detect_connection_mode"]),
+        )
+        // Step 2b: Discover DP info (after metadata to avoid concurrent /server_info calls)
+        .add_step(
+            StepDefinition::new(
+                "discover_dp_info",
+                "Discover DP Info",
+                Arc::new(DiscoverDPInfoStep),
+            )
+            .with_retry(RetryPolicy {
+                max_attempts: 3,
+                backoff: BackoffStrategy::Fixed(Duration::from_secs(1)),
+            })
+            .with_timeout(Duration::from_secs(10))
+            .with_failure_action(FailureAction::FailWorkflow)
+            .depends_on(&["discover_metadata"]),
+        )
+        // Step 3: Create worker(s)
+        .add_step(
+            StepDefinition::new(
+                "create_worker",
+                "Create Worker",
+                Arc::new(CreateLocalWorkerStep),
+            )
+            .with_timeout(Duration::from_secs(5))
+            .with_failure_action(FailureAction::FailWorkflow)
+            .depends_on(&["discover_dp_info"]),
+        )
+        // Step 4: Register workers (shared step)
+        .add_step(
+            StepDefinition::new(
+                "register_workers",
+                "Register Workers",
+                Arc::new(RegisterWorkersStep),
+            )
+            .with_timeout(Duration::from_secs(5))
+            .with_failure_action(FailureAction::FailWorkflow)
+            .depends_on(&["create_worker"]),
+        )
+        .add_step(
+            StepDefinition::new(
+                "submit_tokenizer_job",
+                "Submit Tokenizer Job",
+                Arc::new(SubmitTokenizerJobStep),
+            )
+            .with_timeout(Duration::from_secs(5))
+            .with_failure_action(FailureAction::ContinueNextStep)
+            .depends_on(&["register_workers"]),
+        )
+        // Step 5a: Update policies (parallel with activation)
+        .add_step(
+            StepDefinition::new(
+                "update_policies",
+                "Update Policies",
+                Arc::new(UpdatePoliciesStep),
+            )
+            .with_timeout(Duration::from_secs(5))
+            .with_failure_action(FailureAction::ContinueNextStep)
+            .depends_on(&["register_workers"]),
+        )
+        // Step 5b: Activate workers (parallel with policy update)
+        .add_step(
+            StepDefinition::new(
+                "activate_workers",
+                "Activate Workers",
+                Arc::new(ActivateWorkersStep),
+            )
+            .with_timeout(Duration::from_secs(5))
+            .with_failure_action(FailureAction::FailWorkflow)
+            .depends_on(&["register_workers"]),
+        )
+}
+
+/// Create a worker removal workflow definition.
+///
+/// DAG structure:
+/// ```text
+///     find_workers_to_remove
+///              │
+///     remove_from_policy_registry
+///              │
+///     remove_from_worker_registry
+///              │
+///     update_remaining_policies
+/// ```
+pub fn create_worker_removal_workflow() -> WorkflowDefinition<WorkerRemovalWorkflowData> {
+    WorkflowDefinition::new("worker_removal", "Remove worker from router")
+        .add_step(
+            StepDefinition::new(
+                "find_workers_to_remove",
+                "Find workers to remove",
+                Arc::new(FindWorkersToRemoveStep),
+            )
+            .with_timeout(Duration::from_secs(10))
+            .with_retry(RetryPolicy {
+                max_attempts: 1,
+                backoff: BackoffStrategy::Fixed(Duration::from_secs(0)),
+            }),
+        )
+        .add_step(
+            StepDefinition::new(
+                "remove_from_policy_registry",
+                "Remove workers from policy registry",
+                Arc::new(RemoveFromPolicyRegistryStep),
+            )
+            .with_timeout(Duration::from_secs(10))
+            .with_retry(RetryPolicy {
+                max_attempts: 1,
+                backoff: BackoffStrategy::Fixed(Duration::from_secs(0)),
+            })
+            .depends_on(&["find_workers_to_remove"]),
+        )
+        .add_step(
+            StepDefinition::new(
+                "remove_from_worker_registry",
+                "Remove workers from worker registry",
+                Arc::new(RemoveFromWorkerRegistryStep),
+            )
+            .with_timeout(Duration::from_secs(10))
+            .with_retry(RetryPolicy {
+                max_attempts: 1,
+                backoff: BackoffStrategy::Fixed(Duration::from_secs(0)),
+            })
+            .depends_on(&["remove_from_policy_registry"]),
+        )
+        .add_step(
+            StepDefinition::new(
+                "update_remaining_policies",
+                "Update cache-aware policies for remaining workers",
+                Arc::new(UpdateRemainingPoliciesStep),
+            )
+            .with_timeout(Duration::from_secs(10))
+            .with_retry(RetryPolicy {
+                max_attempts: 1,
+                backoff: BackoffStrategy::Fixed(Duration::from_secs(0)),
+            })
+            .depends_on(&["remove_from_worker_registry"]),
+        )
+}
+
+/// Create a worker update workflow definition.
+///
+/// DAG structure:
+/// ```text
+///     find_worker_to_update
+///              │
+///     update_worker_properties
+///              │
+///     update_policies_for_worker
+/// ```
+pub fn create_worker_update_workflow() -> WorkflowDefinition<WorkerUpdateWorkflowData> {
+    WorkflowDefinition::new("worker_update", "Update worker properties")
+        .add_step(
+            StepDefinition::new(
+                "find_worker_to_update",
+                "Find worker to update",
+                Arc::new(FindWorkerToUpdateStep),
+            )
+            .with_timeout(Duration::from_secs(10))
+            .with_retry(RetryPolicy {
+                max_attempts: 1,
+                backoff: BackoffStrategy::Fixed(Duration::from_secs(0)),
+            }),
+        )
+        .add_step(
+            StepDefinition::new(
+                "update_worker_properties",
+                "Update worker properties",
+                Arc::new(UpdateWorkerPropertiesStep),
+            )
+            .with_timeout(Duration::from_secs(10))
+            .with_retry(RetryPolicy {
+                max_attempts: 1,
+                backoff: BackoffStrategy::Fixed(Duration::from_secs(0)),
+            })
+            .depends_on(&["find_worker_to_update"]),
+        )
+        .add_step(
+            StepDefinition::new(
+                "update_policies_for_worker",
+                "Update policies for updated worker",
+                Arc::new(UpdatePoliciesForWorkerStep),
+            )
+            .with_timeout(Duration::from_secs(10))
+            .with_retry(RetryPolicy {
+                max_attempts: 1,
+                backoff: BackoffStrategy::Fixed(Duration::from_secs(0)),
+            })
+            .depends_on(&["update_worker_properties"]),
+        )
+}
+
+/// Helper to create initial workflow data for local worker registration
+pub fn create_local_worker_workflow_data(
+    config: WorkerConfigRequest,
+    app_context: Arc<AppContext>,
+) -> LocalWorkerWorkflowData {
+    LocalWorkerWorkflowData {
+        config,
+        connection_mode: None,
+        discovered_labels: std::collections::HashMap::new(),
+        dp_info: None,
+        workers: None,
+        final_labels: std::collections::HashMap::new(),
+        detected_runtime_type: None,
+        app_context: Some(app_context),
+        actual_workers: None,
+    }
+}
+
+/// Helper to create initial workflow data for worker removal
+pub fn create_worker_removal_workflow_data(
+    url: String,
+    dp_aware: bool,
+    app_context: Arc<AppContext>,
+) -> WorkerRemovalWorkflowData {
+    WorkerRemovalWorkflowData {
+        config: WorkerRemovalRequest { url, dp_aware },
+        workers_to_remove: None,
+        worker_urls: Vec::new(),
+        affected_models: std::collections::HashSet::new(),
+        app_context: Some(app_context),
+        actual_workers_to_remove: None,
+    }
+}
+
+/// Helper to create initial workflow data for worker update
+pub fn create_worker_update_workflow_data(
+    worker_url: String,
+    update_config: WorkerUpdateRequest,
+    app_context: Arc<AppContext>,
+) -> WorkerUpdateWorkflowData {
+    // Determine if this is a DP-aware update based on URL pattern
+    let dp_aware = worker_url.contains('@');
+    WorkerUpdateWorkflowData {
+        config: update_config,
+        worker_url,
+        dp_aware,
+        app_context: Some(app_context),
+        workers_to_update: None,
+        updated_workers: None,
+    }
+}

third_party/sglang/sgl-model-gateway/src/core/steps/worker/local/remove_from_policy_registry.rs

@@ -0,0 +1,61 @@
+//! Step to remove workers from policy registry.
+
+use async_trait::async_trait;
+use tracing::debug;
+use wfaas::{StepExecutor, StepResult, WorkflowContext, WorkflowError, WorkflowResult};
+
+use crate::core::steps::workflow_data::WorkerRemovalWorkflowData;
+
+/// Step to remove workers from the policy registry.
+///
+/// Removes each worker from cache-aware policies and notifies
+/// the policy registry of worker removal.
+pub struct RemoveFromPolicyRegistryStep;
+
+#[async_trait]
+impl StepExecutor<WorkerRemovalWorkflowData> for RemoveFromPolicyRegistryStep {
+    async fn execute(
+        &self,
+        context: &mut WorkflowContext<WorkerRemovalWorkflowData>,
+    ) -> WorkflowResult<StepResult> {
+        let app_context = context
+            .data
+            .app_context
+            .as_ref()
+            .ok_or_else(|| WorkflowError::ContextValueNotFound("app_context".to_string()))?;
+        let workers_to_remove = context
+            .data
+            .actual_workers_to_remove
+            .as_ref()
+            .ok_or_else(|| WorkflowError::ContextValueNotFound("workers_to_remove".to_string()))?;
+
+        debug!(
+            "Removing {} worker(s) from policy registry",
+            workers_to_remove.len()
+        );
+
+        for worker in workers_to_remove.iter() {
+            let model_id = worker.model_id().to_string();
+            let worker_url = worker.url();
+
+            // Remove from cache-aware policy
+            app_context
+                .policy_registry
+                .remove_worker_from_cache_aware(&model_id, worker_url);
+
+            // Notify policy registry
+            app_context.policy_registry.on_worker_removed(&model_id);
+        }
+
+        debug!(
+            "Removed {} worker(s) from policy registry",
+            workers_to_remove.len()
+        );
+
+        Ok(StepResult::Success)
+    }
+
+    fn is_retryable(&self, _error: &WorkflowError) -> bool {
+        false
+    }
+}

third_party/sglang/sgl-model-gateway/src/core/steps/worker/local/remove_from_worker_registry.rs

@@ -0,0 +1,103 @@
+//! Step to remove workers from worker registry.
+
+use std::collections::HashSet;
+
+use async_trait::async_trait;
+use tracing::{debug, warn};
+use wfaas::{StepExecutor, StepResult, WorkflowContext, WorkflowError, WorkflowResult};
+
+use crate::{
+    core::steps::workflow_data::WorkerRemovalWorkflowData, observability::metrics::Metrics,
+};
+
+/// Step to remove workers from the worker registry.
+///
+/// Removes each worker by URL from the central worker registry.
+pub struct RemoveFromWorkerRegistryStep;
+
+#[async_trait]
+impl StepExecutor<WorkerRemovalWorkflowData> for RemoveFromWorkerRegistryStep {
+    async fn execute(
+        &self,
+        context: &mut WorkflowContext<WorkerRemovalWorkflowData>,
+    ) -> WorkflowResult<StepResult> {
+        let app_context = context
+            .data
+            .app_context
+            .as_ref()
+            .ok_or_else(|| WorkflowError::ContextValueNotFound("app_context".to_string()))?;
+        let worker_urls = &context.data.worker_urls;
+
+        debug!(
+            "Removing {} worker(s) from worker registry",
+            worker_urls.len()
+        );
+
+        // Collect unique worker configurations before removal for pool size updates
+        let unique_configs: HashSet<_> = worker_urls
+            .iter()
+            .filter_map(|url| app_context.worker_registry.get_by_url(url))
+            .map(|w| {
+                let meta = w.metadata();
+                (
+                    meta.worker_type.clone(),
+                    meta.connection_mode.clone(),
+                    w.model_id().to_string(),
+                )
+            })
+            .collect();
+
+        let mut removed_count = 0;
+        for worker_url in worker_urls.iter() {
+            if app_context
+                .worker_registry
+                .remove_by_url(worker_url)
+                .is_some()
+            {
+                removed_count += 1;
+            }
+        }
+
+        // Log if some workers were already removed (e.g., by another process)
+        if removed_count != worker_urls.len() {
+            warn!(
+                "Removed {} of {} workers (some may have been removed by another process)",
+                removed_count,
+                worker_urls.len()
+            );
+        } else {
+            debug!("Removed {} worker(s) from registry", removed_count);
+        }
+
+        // Update Layer 3 worker pool size metrics for unique configurations
+        for (worker_type, connection_mode, model_id) in unique_configs {
+            // Get labels before moving values into get_workers_filtered
+            let worker_type_label = worker_type.as_metric_label();
+            let connection_mode_label = connection_mode.as_metric_label();
+
+            let pool_size = app_context
+                .worker_registry
+                .get_workers_filtered(
+                    Some(&model_id),
+                    Some(worker_type),
+                    Some(connection_mode),
+                    None,
+                    false,
+                )
+                .len();
+
+            Metrics::set_worker_pool_size(
+                worker_type_label,
+                connection_mode_label,
+                &model_id,
+                pool_size,
+            );
+        }
+
+        Ok(StepResult::Success)
+    }
+
+    fn is_retryable(&self, _error: &WorkflowError) -> bool {
+        false
+    }
+}

third_party/sglang/sgl-model-gateway/src/core/steps/worker/local/submit_tokenizer_job.rs

@@ -0,0 +1,135 @@
+//! Tokenizer registration step for local workers.
+//!
+//! This step submits a Job::AddTokenizer to the job queue, which triggers the
+//! tokenizer_registration workflow. The workflow handles validation, deduplication,
+//! and caching - this step just submits the job.
+
+use async_trait::async_trait;
+use tracing::{debug, info, warn};
+use wfaas::{StepExecutor, StepResult, WorkflowContext, WorkflowError, WorkflowResult};
+
+use crate::{
+    core::{
+        steps::{workflow_data::LocalWorkerWorkflowData, TokenizerConfigRequest},
+        Job,
+    },
+    tokenizer::TokenizerRegistry,
+};
+
+/// Step: Submit tokenizer registration job for the worker's model
+///
+/// This step submits a Job::AddTokenizer to the job queue rather than loading
+/// the tokenizer directly. This ensures tokenizer registration goes through
+/// the unified tokenizer_registration workflow.
+pub struct SubmitTokenizerJobStep;
+
+#[async_trait]
+impl StepExecutor<LocalWorkerWorkflowData> for SubmitTokenizerJobStep {
+    async fn execute(
+        &self,
+        context: &mut WorkflowContext<LocalWorkerWorkflowData>,
+    ) -> WorkflowResult<StepResult> {
+        let labels = &context.data.final_labels;
+        let app_context = context
+            .data
+            .app_context
+            .as_ref()
+            .ok_or_else(|| WorkflowError::ContextValueNotFound("app_context".to_string()))?;
+        let workers = context
+            .data
+            .actual_workers
+            .as_ref()
+            .ok_or_else(|| WorkflowError::ContextValueNotFound("workers".to_string()))?;
+
+        // Get job queue
+        let job_queue = match app_context.worker_job_queue.get() {
+            Some(queue) => queue,
+            None => {
+                warn!("Job queue not available, skipping tokenizer registration");
+                return Ok(StepResult::Success);
+            }
+        };
+
+        // Get chat_template: worker config > global router config
+        let chat_template = context
+            .data
+            .config
+            .chat_template
+            .clone()
+            .or_else(|| app_context.router_config.chat_template.clone());
+
+        // Get cache config from router config
+        let cache_config = app_context.router_config.tokenizer_cache.to_option();
+
+        for worker in workers.iter() {
+            let model_id = worker.model_id().to_string();
+
+            // Get tokenizer path with fallback chain:
+            // 1. Worker labels: tokenizer_path
+            // 2. Worker labels: model_path
+            // 3. Router config (CLI args): --tokenizer-path
+            // 4. Router config (CLI args): --model-path
+            let tokenizer_path: String = if let Some(path) = labels
+                .get("tokenizer_path")
+                .or_else(|| labels.get("model_path"))
+            {
+                path.clone()
+            } else if let Some(path) = app_context
+                .router_config
+                .tokenizer_path
+                .as_ref()
+                .or(app_context.router_config.model_path.as_ref())
+            {
+                debug!(
+                    "Using router config tokenizer path '{}' for model {}",
+                    path, model_id
+                );
+                path.clone()
+            } else {
+                warn!(
+                    "No tokenizer_path or model_path found for model {} (checked worker labels and router config)",
+                    model_id
+                );
+                continue;
+            };
+
+            // Note: We don't check if tokenizer already exists here.
+            // The registry.load() handles deduplication gracefully (returns AlreadyExists).
+            // This simplifies the code and ensures consistent behavior.
+
+            info!(
+                "Submitting tokenizer registration job for model {} from {}",
+                model_id, tokenizer_path
+            );
+
+            // Create tokenizer config request
+            let config = TokenizerConfigRequest {
+                id: TokenizerRegistry::generate_id(),
+                name: model_id.clone(),
+                source: tokenizer_path,
+                chat_template_path: chat_template.clone(),
+                cache_config: cache_config.clone(),
+                fail_on_duplicate: false,
+            };
+
+            // Submit job (fire-and-forget, don't wait for completion)
+            if let Err(e) = job_queue
+                .submit(Job::AddTokenizer {
+                    config: Box::new(config),
+                })
+                .await
+            {
+                warn!(
+                    "Failed to submit tokenizer job for model {}: {}",
+                    model_id, e
+                );
+            }
+        }
+
+        Ok(StepResult::Success)
+    }
+
+    fn is_retryable(&self, _error: &WorkflowError) -> bool {
+        false // Job submission failures are not retryable at this level
+    }
+}

third_party/sglang/sgl-model-gateway/src/core/steps/worker/local/update_policies_for_worker.rs

@@ -0,0 +1,72 @@
+//! Step to update policies for updated workers.
+
+use std::collections::HashSet;
+
+use async_trait::async_trait;
+use tracing::debug;
+use wfaas::{StepExecutor, StepResult, WorkflowContext, WorkflowError, WorkflowResult};
+
+use crate::core::steps::workflow_data::WorkerUpdateWorkflowData;
+
+/// Step to update policies for updated workers.
+///
+/// After workers are updated, this step re-initializes cache-aware policies
+/// for the affected models to reflect any priority or label changes.
+pub struct UpdatePoliciesForWorkerStep;
+
+#[async_trait]
+impl StepExecutor<WorkerUpdateWorkflowData> for UpdatePoliciesForWorkerStep {
+    async fn execute(
+        &self,
+        context: &mut WorkflowContext<WorkerUpdateWorkflowData>,
+    ) -> WorkflowResult<StepResult> {
+        let app_context = context
+            .data
+            .app_context
+            .as_ref()
+            .ok_or_else(|| WorkflowError::ContextValueNotFound("app_context".to_string()))?;
+        let updated_workers =
+            context.data.updated_workers.as_ref().ok_or_else(|| {
+                WorkflowError::ContextValueNotFound("updated_workers".to_string())
+            })?;
+
+        // Collect affected models
+        let affected_models: HashSet<String> = updated_workers
+            .iter()
+            .map(|w| w.model_id().to_string())
+            .collect();
+
+        debug!(
+            "Updating policies for {} affected model(s) after worker update",
+            affected_models.len()
+        );
+
+        for model_id in &affected_models {
+            let workers = app_context.worker_registry.get_by_model(model_id);
+
+            if let Some(policy) = app_context.policy_registry.get_policy(model_id) {
+                if policy.name() == "cache_aware" && !workers.is_empty() {
+                    // Re-initialize cache-aware policy with updated workers
+                    app_context
+                        .policy_registry
+                        .init_cache_aware_policy(model_id, &workers);
+
+                    debug!(
+                        "Updated cache-aware policy for model {} ({} workers)",
+                        model_id,
+                        workers.len()
+                    );
+                }
+            }
+
+            // Notify policy registry of the update
+            app_context.policy_registry.on_worker_added(model_id, None);
+        }
+
+        Ok(StepResult::Success)
+    }
+
+    fn is_retryable(&self, _error: &WorkflowError) -> bool {
+        false
+    }
+}

third_party/sglang/sgl-model-gateway/src/core/steps/worker/local/update_remaining_policies.rs

@@ -0,0 +1,69 @@
+//! Step to update cache-aware policies for remaining workers after removal.
+
+use async_trait::async_trait;
+use tracing::{debug, info};
+use wfaas::{StepExecutor, StepResult, WorkflowContext, WorkflowError, WorkflowResult};
+
+use crate::core::steps::workflow_data::WorkerRemovalWorkflowData;
+
+/// Step to update cache-aware policies for remaining workers.
+///
+/// After workers are removed, this step re-initializes cache-aware policies
+/// for the affected models using the remaining workers.
+pub struct UpdateRemainingPoliciesStep;
+
+#[async_trait]
+impl StepExecutor<WorkerRemovalWorkflowData> for UpdateRemainingPoliciesStep {
+    async fn execute(
+        &self,
+        context: &mut WorkflowContext<WorkerRemovalWorkflowData>,
+    ) -> WorkflowResult<StepResult> {
+        let app_context = context
+            .data
+            .app_context
+            .as_ref()
+            .ok_or_else(|| WorkflowError::ContextValueNotFound("app_context".to_string()))?;
+        let affected_models = &context.data.affected_models;
+        let worker_urls = &context.data.worker_urls;
+
+        debug!(
+            "Updating cache-aware policies for {} affected model(s)",
+            affected_models.len()
+        );
+
+        for model_id in affected_models.iter() {
+            let remaining_workers = app_context.worker_registry.get_by_model(model_id);
+
+            if let Some(policy) = app_context.policy_registry.get_policy(model_id) {
+                if policy.name() == "cache_aware" && !remaining_workers.is_empty() {
+                    app_context
+                        .policy_registry
+                        .init_cache_aware_policy(model_id, &remaining_workers);
+
+                    debug!(
+                        "Updated cache-aware policy for model {} ({} remaining workers)",
+                        model_id,
+                        remaining_workers.len()
+                    );
+                }
+            }
+        }
+
+        // Log final result at info level
+        if worker_urls.len() == 1 {
+            info!("Removed worker {}", worker_urls[0]);
+        } else {
+            info!(
+                "Removed {} DP-aware workers: {:?}",
+                worker_urls.len(),
+                worker_urls
+            );
+        }
+
+        Ok(StepResult::Success)
+    }
+
+    fn is_retryable(&self, _error: &WorkflowError) -> bool {
+        false
+    }
+}

third_party/sglang/sgl-model-gateway/src/core/steps/worker/local/update_worker_properties.rs

@@ -0,0 +1,152 @@
+//! Step to update worker properties.
+
+use std::sync::Arc;
+
+use async_trait::async_trait;
+use tracing::{debug, info};
+use wfaas::{StepExecutor, StepResult, WorkflowContext, WorkflowError, WorkflowResult};
+
+use crate::core::{
+    steps::workflow_data::WorkerUpdateWorkflowData, BasicWorkerBuilder, HealthConfig, Worker,
+};
+
+/// Step to update worker properties.
+///
+/// This step creates new worker instances with updated properties and
+/// re-registers them to replace the old workers in the registry.
+pub struct UpdateWorkerPropertiesStep;
+
+#[async_trait]
+impl StepExecutor<WorkerUpdateWorkflowData> for UpdateWorkerPropertiesStep {
+    async fn execute(
+        &self,
+        context: &mut WorkflowContext<WorkerUpdateWorkflowData>,
+    ) -> WorkflowResult<StepResult> {
+        let request = &context.data.config;
+        let app_context = context
+            .data
+            .app_context
+            .as_ref()
+            .ok_or_else(|| WorkflowError::ContextValueNotFound("app_context".to_string()))?
+            .clone();
+        let workers_to_update = context
+            .data
+            .workers_to_update
+            .as_ref()
+            .ok_or_else(|| WorkflowError::ContextValueNotFound("workers_to_update".to_string()))?
+            .clone();
+
+        debug!(
+            "Updating properties for {} worker(s)",
+            workers_to_update.len()
+        );
+
+        let mut updated_workers: Vec<Arc<dyn Worker>> = Vec::with_capacity(workers_to_update.len());
+
+        for worker in workers_to_update.iter() {
+            // Build updated labels - merge new labels into existing ones
+            let mut updated_labels = worker.metadata().labels.clone();
+            if let Some(ref new_labels) = request.labels {
+                for (key, value) in new_labels {
+                    updated_labels.insert(key.clone(), value.clone());
+                }
+            }
+
+            // Update priority if specified (stored in labels)
+            if let Some(priority) = request.priority {
+                updated_labels.insert("priority".to_string(), priority.to_string());
+            }
+
+            // Update cost if specified (stored in labels)
+            if let Some(cost) = request.cost {
+                updated_labels.insert("cost".to_string(), cost.to_string());
+            }
+
+            // Build updated health config
+            let existing_health = &worker.metadata().health_config;
+            let updated_health_config = HealthConfig {
+                timeout_secs: request
+                    .health_check_timeout_secs
+                    .unwrap_or(existing_health.timeout_secs),
+                check_interval_secs: request
+                    .health_check_interval_secs
+                    .unwrap_or(existing_health.check_interval_secs),
+                endpoint: existing_health.endpoint.clone(),
+                failure_threshold: request
+                    .health_failure_threshold
+                    .unwrap_or(existing_health.failure_threshold),
+                success_threshold: request
+                    .health_success_threshold
+                    .unwrap_or(existing_health.success_threshold),
+                disable_health_check: request
+                    .disable_health_check
+                    .unwrap_or(existing_health.disable_health_check),
+            };
+
+            // Determine API key: use new one if provided, otherwise keep existing
+            let updated_api_key = request
+                .api_key
+                .clone()
+                .or_else(|| worker.metadata().api_key.clone());
+
+            // Create a new worker with updated properties
+            let new_worker: Arc<dyn Worker> = if worker.is_dp_aware() {
+                // For DP-aware workers, extract DP info and rebuild
+                let dp_rank = worker.dp_rank().unwrap_or(0);
+                let dp_size = worker.dp_size().unwrap_or(1);
+                let base_url = worker.base_url().to_string();
+
+                let mut builder =
+                    crate::core::DPAwareWorkerBuilder::new(base_url, dp_rank, dp_size)
+                        .worker_type(worker.worker_type().clone())
+                        .connection_mode(worker.connection_mode().clone())
+                        .runtime_type(worker.metadata().runtime_type.clone())
+                        .labels(updated_labels)
+                        .health_config(updated_health_config.clone())
+                        .models(worker.metadata().models.clone());
+
+                if let Some(ref api_key) = updated_api_key {
+                    builder = builder.api_key(api_key.clone());
+                }
+
+                Arc::new(builder.build())
+            } else {
+                // For basic workers, rebuild with updated properties
+                let mut builder = BasicWorkerBuilder::new(worker.url())
+                    .worker_type(worker.worker_type().clone())
+                    .connection_mode(worker.connection_mode().clone())
+                    .runtime_type(worker.metadata().runtime_type.clone())
+                    .labels(updated_labels)
+                    .health_config(updated_health_config.clone())
+                    .models(worker.metadata().models.clone());
+
+                if let Some(ref api_key) = updated_api_key {
+                    builder = builder.api_key(api_key.clone());
+                }
+
+                Arc::new(builder.build())
+            };
+
+            // Re-register the worker (this replaces the old one)
+            app_context.worker_registry.register(new_worker.clone());
+
+            updated_workers.push(new_worker);
+        }
+
+        // Log result
+        if updated_workers.len() == 1 {
+            info!("Updated worker {}", updated_workers[0].url());
+        } else {
+            info!("Updated {} workers", updated_workers.len());
+        }
+
+        // Store updated workers for subsequent steps
+        context.data.updated_workers = Some(updated_workers);
+
+        Ok(StepResult::Success)
+    }
+
+    fn is_retryable(&self, _error: &WorkflowError) -> bool {
+        false
+    }
+}

third_party/sglang/sgl-model-gateway/src/core/steps/worker/mod.rs

@@ -0,0 +1,19 @@
+pub mod external;
+pub mod local;
+pub mod shared;
+
+pub use external::{
+    create_external_worker_workflow, create_external_worker_workflow_data, group_models_into_cards,
+    infer_model_type_from_id, CreateExternalWorkersStep, DiscoverModelsStep, ModelInfo,
+    ModelsResponse,
+};
+pub use local::{
+    create_local_worker_workflow, create_local_worker_workflow_data,
+    create_worker_removal_workflow, create_worker_removal_workflow_data,
+    create_worker_update_workflow, create_worker_update_workflow_data, CreateLocalWorkerStep,
+    DetectConnectionModeStep, DiscoverDPInfoStep, DiscoverMetadataStep, DpInfo,
+    FindWorkerToUpdateStep, FindWorkersToRemoveStep, RemoveFromPolicyRegistryStep,
+    RemoveFromWorkerRegistryStep, UpdatePoliciesForWorkerStep, UpdateRemainingPoliciesStep,
+    UpdateWorkerPropertiesStep, WorkerRemovalRequest,
+};
+pub use shared::{ActivateWorkersStep, RegisterWorkersStep, UpdatePoliciesStep, WorkerList};

third_party/sglang/sgl-model-gateway/src/core/steps/worker/shared/activate.rs

@@ -0,0 +1,37 @@
+//! Unified worker activation step.
+
+use async_trait::async_trait;
+use tracing::info;
+use wfaas::{
+    StepExecutor, StepResult, WorkflowContext, WorkflowData, WorkflowError, WorkflowResult,
+};
+
+use crate::core::steps::workflow_data::WorkerRegistrationData;
+
+/// Unified step to activate workers by marking them as healthy.
+///
+/// This is the final step in any worker registration workflow.
+/// Works with any workflow data type that implements `WorkerRegistrationData`.
+pub struct ActivateWorkersStep;
+
+#[async_trait]
+impl<D: WorkerRegistrationData + WorkflowData> StepExecutor<D> for ActivateWorkersStep {
+    async fn execute(&self, context: &mut WorkflowContext<D>) -> WorkflowResult<StepResult> {
+        let workers = context
+            .data
+            .get_actual_workers()
+            .ok_or_else(|| WorkflowError::ContextValueNotFound("workers".to_string()))?;
+
+        for worker in workers.iter() {
+            worker.set_healthy(true);
+        }
+
+        info!("Activated {} worker(s) (marked as healthy)", workers.len());
+
+        Ok(StepResult::Success)
+    }
+
+    fn is_retryable(&self, _error: &WorkflowError) -> bool {
+        false
+    }
+}

third_party/sglang/sgl-model-gateway/src/core/steps/worker/shared/mod.rs

@@ -0,0 +1,21 @@
+//! Shared worker registration steps used by both local and external workflows.
+//!
+//! These steps are designed to work with any worker type and can be composed
+//! into different workflows using the DAG-based workflow engine.
+
+mod activate;
+mod register;
+mod update_policies;
+
+use std::sync::Arc;
+
+pub use activate::ActivateWorkersStep;
+pub use register::RegisterWorkersStep;
+pub use update_policies::UpdatePoliciesStep;
+
+use crate::core::Worker;
+
+/// Type alias for a collection of workers in workflow context.
+/// Both local (single/DP-aware) and external (multi-model) workflows
+/// use this unified type for consistency.
+pub type WorkerList = Vec<Arc<dyn Worker>>;

third_party/sglang/sgl-model-gateway/src/core/steps/worker/shared/register.rs

@@ -0,0 +1,99 @@
+//! Unified worker registration step.
+
+use std::{collections::HashSet, sync::Arc};
+
+use async_trait::async_trait;
+use tracing::debug;
+use wfaas::{
+    StepExecutor, StepResult, WorkflowContext, WorkflowData, WorkflowError, WorkflowResult,
+};
+
+use crate::{core::steps::workflow_data::WorkerRegistrationData, observability::metrics::Metrics};
+
+/// Unified step to register workers in the registry.
+///
+/// Works with both single workers and batches. Always expects `workers` key
+/// in context containing `Vec<Arc<dyn Worker>>`.
+/// Works with any workflow data type that implements `WorkerRegistrationData`.
+pub struct RegisterWorkersStep;
+
+#[async_trait]
+impl<D: WorkerRegistrationData + WorkflowData> StepExecutor<D> for RegisterWorkersStep {
+    async fn execute(&self, context: &mut WorkflowContext<D>) -> WorkflowResult<StepResult> {
+        let app_context = context
+            .data
+            .get_app_context()
+            .ok_or_else(|| WorkflowError::ContextValueNotFound("app_context".to_string()))?
+            .clone();
+
+        let workers = context
+            .data
+            .get_actual_workers()
+            .ok_or_else(|| WorkflowError::ContextValueNotFound("workers".to_string()))?;
+
+        let mut worker_ids = Vec::with_capacity(workers.len());
+
+        for worker in workers.iter() {
+            let worker_id = app_context.worker_registry.register(Arc::clone(worker));
+            debug!(
+                "Registered worker {} (model: {}) with ID {:?}",
+                worker.url(),
+                worker.model_id(),
+                worker_id
+            );
+            worker_ids.push(worker_id);
+        }
+
+        // Collect unique worker configurations to avoid redundant metric updates
+        let unique_configs: HashSet<_> = workers
+            .iter()
+            .map(|w| {
+                let meta = w.metadata();
+                (
+                    meta.worker_type.clone(),
+                    meta.connection_mode.clone(),
+                    w.model_id().to_string(),
+                )
+            })
+            .collect();
+
+        // Update Layer 3 worker pool size metrics per unique type/connection/model
+        for (worker_type, connection_mode, model_id) in unique_configs {
+            // Get labels before moving values into get_workers_filtered
+            let worker_type_label = worker_type.as_metric_label();
+            let connection_mode_label = connection_mode.as_metric_label();
+
+            let pool_size = app_context
+                .worker_registry
+                .get_workers_filtered(
+                    Some(&model_id),
+                    Some(worker_type),
+                    Some(connection_mode),
+                    None,
+                    false,
+                )
+                .len();
+
+            Metrics::set_worker_pool_size(
+                worker_type_label,
+                connection_mode_label,
+                &model_id,
+                pool_size,
+            );
+        }
+
+        // Note: worker_ids are stored for potential future use but not persisted
+        // as they are internal registry identifiers
+        debug!(
+            "Registered {} workers with IDs: {:?}",
+            worker_ids.len(),
+            worker_ids
+        );
+
+        Ok(StepResult::Success)
+    }
+
+    fn is_retryable(&self, _error: &WorkflowError) -> bool {
+        false
+    }
+}

third_party/sglang/sgl-model-gateway/src/core/steps/worker/shared/update_policies.rs

@@ -0,0 +1,156 @@
+//! Unified policy update step.
+
+use std::sync::Arc;
+
+use async_trait::async_trait;
+use tracing::{debug, warn};
+use wfaas::{
+    StepExecutor, StepResult, WorkflowContext, WorkflowData, WorkflowError, WorkflowResult,
+};
+
+use crate::core::{steps::workflow_data::WorkerRegistrationData, Worker};
+
+/// Unified step to update policy registry for registered workers.
+///
+/// Handles both local workers (same model, possibly DP-aware) and
+/// external workers (different models per worker).
+pub struct UpdatePoliciesStep;
+
+impl UpdatePoliciesStep {
+    /// Check for conflicts between prefill and decode worker configurations for a model.
+    fn check_worker_conflicts(&self, model_id: &str, workers: &[Arc<dyn Worker>]) {
+        let prefill_workers: Vec<_> = workers
+            .iter()
+            .filter(|w| {
+                w.metadata()
+                    .labels
+                    .get("disaggregation_mode")
+                    .map(|s| s.as_str())
+                    == Some("prefill")
+            })
+            .collect();
+
+        let decode_workers: Vec<_> = workers
+            .iter()
+            .filter(|w| {
+                w.metadata()
+                    .labels
+                    .get("disaggregation_mode")
+                    .map(|s| s.as_str())
+                    == Some("decode")
+            })
+            .collect();
+
+        if prefill_workers.is_empty() || decode_workers.is_empty() {
+            return;
+        }
+
+        // Compare configurations of prefill vs decode workers
+        if let (Some(pw), Some(dw)) = (prefill_workers.first(), decode_workers.first()) {
+            let pl = &pw.metadata().labels;
+            let dl = &dw.metadata().labels;
+
+            // Define keys to check for equality
+            let keys_to_check = ["tp_size", "dp_size", "load_balance_method"];
+
+            for key in keys_to_check {
+                let p_val = pl.get(key);
+                let d_val = dl.get(key);
+                if p_val != d_val {
+                    warn!(
+                        "Model {} has conflicting {}: prefill={:?}, decode={:?}",
+                        model_id, key, p_val, d_val
+                    );
+                }
+            }
+
+            // Specific check for Data-Parallel consistency
+            if let Some(dp_size) = pl.get("dp_size").and_then(|s| s.parse::<usize>().ok()) {
+                if dp_size > 1 {
+                    let plb = pl.get("load_balance_method").map(|s| s.as_str());
+                    if plb != Some("follow_bootstrap_room") {
+                        warn!(
+                            "Model {} has dp_size > 1 but load_balance_method is not 'follow_bootstrap_room' on prefill workers. This may cause rank mismatch in disaggregated mode.",
+                            model_id
+                        );
+                    }
+                }
+            }
+        }
+    }
+}
+
+#[async_trait]
+impl<D: WorkerRegistrationData + WorkflowData> StepExecutor<D> for UpdatePoliciesStep {
+    async fn execute(&self, context: &mut WorkflowContext<D>) -> WorkflowResult<StepResult> {
+        let app_context = context
+            .data
+            .get_app_context()
+            .ok_or_else(|| WorkflowError::ContextValueNotFound("app_context".to_string()))?
+            .clone();
+
+        let workers = context
+            .data
+            .get_actual_workers()
+            .ok_or_else(|| WorkflowError::ContextValueNotFound("workers".to_string()))?;
+
+        let labels = context
+            .data
+            .get_labels()
+            .ok_or_else(|| WorkflowError::ContextValueNotFound("labels".to_string()))?;
+
+        let policy_hint = labels.get("policy").map(|s| s.as_str());
+
+        // Track unique model IDs we've updated policies for
+        let mut updated_models = Vec::new();
+
+        for worker in workers.iter() {
+            let model_id = worker.model_id().to_string();
+
+            // Notify policy registry
+            app_context
+                .policy_registry
+                .on_worker_added(&model_id, policy_hint);
+
+            // Initialize cache-aware policy if configured
+            let all_workers = app_context.worker_registry.get_by_model(&model_id);
+
+            // Check for configuration conflicts between prefill and decode
+            self.check_worker_conflicts(&model_id, &all_workers);
+            if let Some(policy) = app_context.policy_registry.get_policy(&model_id) {
+                if policy.name() == "cache_aware" {
+                    app_context
+                        .policy_registry
+                        .init_cache_aware_policy(&model_id, &all_workers);
+                }
+            }
+
+            if !updated_models.contains(&model_id) {
+                updated_models.push(model_id);
+            }
+        }
+
+        // Initialize bucket policies for prefill workers (local workers only)
+        let prefill_workers = app_context.worker_registry.get_prefill_workers();
+        if !prefill_workers.is_empty() {
+            let policy = app_context.policy_registry.get_prefill_policy();
+            if policy.name() == "bucket" {
+                app_context
+                    .policy_registry
+                    .init_pd_bucket_policies(&prefill_workers);
+            }
+        }
+
+        debug!(
+            "Updated policies for {} workers across {} models",
+            workers.len(),
+            updated_models.len()
+        );
+
+        Ok(StepResult::Success)
+    }
+
+    fn is_retryable(&self, _error: &WorkflowError) -> bool {
+        false
+    }
+}

third_party/sglang/sgl-model-gateway/src/core/steps/workflow_data.rs

@@ -0,0 +1,375 @@
+//! Typed workflow data structures
+//!
+//! This module defines the typed data structures for all workflows, enabling
+//! compile-time type safety and state persistence. Each workflow has its own
+//! strongly-typed data structure, and steps are typed to their specific workflow.
+//!
+//! # Shared Step Trait
+//!
+//! For steps that are shared between local and external worker workflows,
+//! we use the `WorkerRegistrationData` trait. This trait provides a common
+//! interface for accessing worker data while maintaining full type safety.
+
+use std::{collections::HashMap, sync::Arc};
+
+use serde::{Deserialize, Serialize};
+use wfaas::{WorkflowData, WorkflowError};
+
+use super::{
+    mcp_registration::McpServerConfigRequest, tokenizer_registration::TokenizerConfigRequest,
+    wasm_module_registration::WasmModuleConfigRequest,
+    wasm_module_removal::WasmModuleRemovalRequest, worker::local::WorkerRemovalRequest,
+};
+/// Re-export the protocol types for convenience
+pub use crate::protocols::worker_spec::{
+    WorkerConfigRequest, WorkerUpdateRequest as ProtocolUpdateRequest,
+};
+use crate::{
+    app_context::AppContext,
+    core::{model_card::ModelCard, Worker},
+    protocols::worker_spec::{
+        WorkerConfigRequest as ProtocolWorkerConfigRequest,
+        WorkerUpdateRequest as ProtocolWorkerUpdateRequest,
+    },
+};
+
+// ============================================================================
+// Shared trait for worker registration workflows
+// ============================================================================
+
+/// Trait for workflow data that supports worker registration operations.
+///
+/// This trait is implemented by both `LocalWorkerWorkflowData` and
+/// `ExternalWorkerWorkflowData`, allowing shared steps to work with either
+/// workflow type while maintaining full type safety.
+pub trait WorkerRegistrationData: WorkflowData {
+    /// Get the application context (transient, not serialized).
+    fn get_app_context(&self) -> Option<&Arc<AppContext>>;
+
+    /// Get the actual worker objects (transient, not serialized).
+    fn get_actual_workers(&self) -> Option<&Vec<Arc<dyn Worker>>>;
+
+    /// Get the labels for policy registration.
+    fn get_labels(&self) -> Option<&HashMap<String, String>>;
+}
+
+/// Wrapper for worker list that can be serialized
+#[derive(Debug, Clone, Default, Serialize, Deserialize)]
+pub struct WorkerList {
+    /// Worker URLs (we can't serialize Arc<dyn Worker>, so we store URLs)
+    pub worker_urls: Vec<String>,
+}
+
+impl WorkerList {
+    pub fn new() -> Self {
+        Self {
+            worker_urls: Vec::new(),
+        }
+    }
+
+    pub fn from_workers(workers: &[Arc<dyn Worker>]) -> Self {
+        Self {
+            worker_urls: workers.iter().map(|w| w.url().to_string()).collect(),
+        }
+    }
+}
+
+// ============================================================================
+// Workflow-specific data types
+// ============================================================================
+
+/// Data for tokenizer registration workflow
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct TokenizerWorkflowData {
+    pub config: TokenizerConfigRequest,
+    pub vocab_size: Option<usize>,
+    /// Application context (transient, must be re-initialized after deserialization)
+    #[serde(skip, default)]
+    pub app_context: Option<Arc<AppContext>>,
+}
+
+impl WorkflowData for TokenizerWorkflowData {
+    fn workflow_type() -> &'static str {
+        "tokenizer_registration"
+    }
+}
+
+impl TokenizerWorkflowData {
+    /// Validate that all transient fields are properly initialized.
+    ///
+    /// Call this after deserializing workflow state to ensure runtime fields
+    /// have been repopulated.
+    pub fn validate_initialized(&self) -> Result<(), WorkflowError> {
+        if self.app_context.is_none() {
+            return Err(WorkflowError::ContextValueNotFound(
+                "app_context not initialized after deserialization".into(),
+            ));
+        }
+        Ok(())
+    }
+}
+
+/// Data for local worker registration workflow
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct LocalWorkerWorkflowData {
+    pub config: ProtocolWorkerConfigRequest,
+    pub connection_mode: Option<crate::core::ConnectionMode>,
+    pub discovered_labels: HashMap<String, String>,
+    pub dp_info: Option<super::worker::local::DpInfo>,
+    pub workers: Option<WorkerList>,
+    pub final_labels: HashMap<String, String>,
+    /// Detected runtime type (for gRPC workers)
+    pub detected_runtime_type: Option<String>,
+    /// Application context (transient, must be re-initialized after deserialization)
+    #[serde(skip, default)]
+    pub app_context: Option<Arc<AppContext>>,
+    /// Actual worker objects (transient, not serialized)
+    #[serde(skip, default)]
+    pub actual_workers: Option<Vec<Arc<dyn Worker>>>,
+}
+
+impl WorkflowData for LocalWorkerWorkflowData {
+    fn workflow_type() -> &'static str {
+        "local_worker_registration"
+    }
+}
+
+impl LocalWorkerWorkflowData {
+    /// Validate that all transient fields are properly initialized.
+    pub fn validate_initialized(&self) -> Result<(), WorkflowError> {
+        if self.app_context.is_none() {
+            return Err(WorkflowError::ContextValueNotFound(
+                "app_context not initialized after deserialization".into(),
+            ));
+        }
+        Ok(())
+    }
+}
+
+impl WorkerRegistrationData for LocalWorkerWorkflowData {
+    fn get_app_context(&self) -> Option<&Arc<AppContext>> {
+        self.app_context.as_ref()
+    }
+
+    fn get_actual_workers(&self) -> Option<&Vec<Arc<dyn Worker>>> {
+        self.actual_workers.as_ref()
+    }
+
+    fn get_labels(&self) -> Option<&HashMap<String, String>> {
+        Some(&self.final_labels)
+    }
+}
+
+/// Data for external worker registration workflow
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct ExternalWorkerWorkflowData {
+    pub config: ProtocolWorkerConfigRequest,
+    /// Discovered model cards from /v1/models endpoint
+    pub model_cards: Vec<ModelCard>,
+    pub workers: Option<WorkerList>,
+    /// Labels for policies (derived from config)
+    pub labels: HashMap<String, String>,
+    /// Application context (transient, must be re-initialized after deserialization)
+    #[serde(skip, default)]
+    pub app_context: Option<Arc<AppContext>>,
+    /// Actual worker objects (transient, not serialized)
+    #[serde(skip, default)]
+    pub actual_workers: Option<Vec<Arc<dyn Worker>>>,
+}
+
+impl WorkflowData for ExternalWorkerWorkflowData {
+    fn workflow_type() -> &'static str {
+        "external_worker_registration"
+    }
+}
+
+impl ExternalWorkerWorkflowData {
+    /// Validate that all transient fields are properly initialized.
+    pub fn validate_initialized(&self) -> Result<(), WorkflowError> {
+        if self.app_context.is_none() {
+            return Err(WorkflowError::ContextValueNotFound(
+                "app_context not initialized after deserialization".into(),
+            ));
+        }
+        Ok(())
+    }
+}
+
+impl WorkerRegistrationData for ExternalWorkerWorkflowData {
+    fn get_app_context(&self) -> Option<&Arc<AppContext>> {
+        self.app_context.as_ref()
+    }
+
+    fn get_actual_workers(&self) -> Option<&Vec<Arc<dyn Worker>>> {
+        self.actual_workers.as_ref()
+    }
+
+    fn get_labels(&self) -> Option<&HashMap<String, String>> {
+        Some(&self.labels)
+    }
+}
+
+/// Data for worker removal workflow
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct WorkerRemovalWorkflowData {
+    pub config: WorkerRemovalRequest,
+    pub workers_to_remove: Option<WorkerList>,
+    /// URLs of workers being removed
+    pub worker_urls: Vec<String>,
+    /// Model IDs affected by the removal
+    pub affected_models: std::collections::HashSet<String>,
+    /// Application context (transient, must be re-initialized after deserialization)
+    #[serde(skip, default)]
+    pub app_context: Option<Arc<AppContext>>,
+    /// Actual worker objects to remove (transient, not serialized)
+    #[serde(skip, default)]
+    pub actual_workers_to_remove: Option<Vec<Arc<dyn Worker>>>,
+}
+
+impl WorkflowData for WorkerRemovalWorkflowData {
+    fn workflow_type() -> &'static str {
+        "worker_removal"
+    }
+}
+
+impl WorkerRemovalWorkflowData {
+    /// Validate that all transient fields are properly initialized.
+    pub fn validate_initialized(&self) -> Result<(), WorkflowError> {
+        if self.app_context.is_none() {
+            return Err(WorkflowError::ContextValueNotFound(
+                "app_context not initialized after deserialization".into(),
+            ));
+        }
+        Ok(())
+    }
+}
+
+/// Data for worker update workflow
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct WorkerUpdateWorkflowData {
+    pub config: ProtocolWorkerUpdateRequest,
+    /// URL of worker(s) to update
+    pub worker_url: String,
+    /// Whether to update all DP-aware workers with matching prefix
+    pub dp_aware: bool,
+    /// Application context (transient, must be re-initialized after deserialization)
+    #[serde(skip, default)]
+    pub app_context: Option<Arc<AppContext>>,
+    /// Workers to update (transient, not serialized)
+    #[serde(skip, default)]
+    pub workers_to_update: Option<Vec<Arc<dyn Worker>>>,
+    /// Updated worker objects (transient, not serialized)
+    #[serde(skip, default)]
+    pub updated_workers: Option<Vec<Arc<dyn Worker>>>,
+}
+
+impl WorkflowData for WorkerUpdateWorkflowData {
+    fn workflow_type() -> &'static str {
+        "worker_update"
+    }
+}
+
+impl WorkerUpdateWorkflowData {
+    /// Validate that all transient fields are properly initialized.
+    pub fn validate_initialized(&self) -> Result<(), WorkflowError> {
+        if self.app_context.is_none() {
+            return Err(WorkflowError::ContextValueNotFound(
+                "app_context not initialized after deserialization".into(),
+            ));
+        }
+        Ok(())
+    }
+}
+
+/// Data for MCP server registration workflow
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct McpWorkflowData {
+    pub config: McpServerConfigRequest,
+    pub validated: bool,
+    /// Application context (transient, must be re-initialized after deserialization)
+    #[serde(skip, default)]
+    pub app_context: Option<Arc<AppContext>>,
+    /// Connected MCP client (transient, not serialized)
+    #[serde(skip, default)]
+    pub mcp_client: Option<Arc<rmcp::service::RunningService<rmcp::RoleClient, ()>>>,
+}
+
+impl WorkflowData for McpWorkflowData {
+    fn workflow_type() -> &'static str {
+        "mcp_registration"
+    }
+}
+
+impl McpWorkflowData {
+    /// Validate that all transient fields are properly initialized.
+    pub fn validate_initialized(&self) -> Result<(), WorkflowError> {
+        if self.app_context.is_none() {
+            return Err(WorkflowError::ContextValueNotFound(
+                "app_context not initialized after deserialization".into(),
+            ));
+        }
+        Ok(())
+    }
+}
+
+/// Data for WASM module registration workflow
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct WasmRegistrationWorkflowData {
+    pub config: WasmModuleConfigRequest,
+    pub wasm_bytes: Option<Vec<u8>>,
+    /// SHA256 hash of the module file (32 bytes)
+    pub sha256_hash: Option<[u8; 32]>,
+    /// File size in bytes
+    pub file_size_bytes: Option<u64>,
+    /// UUID assigned to the registered module
+    pub module_uuid: Option<uuid::Uuid>,
+    /// Application context (transient, must be re-initialized after deserialization)
+    #[serde(skip, default)]
+    pub app_context: Option<Arc<AppContext>>,
+}
+
+impl WorkflowData for WasmRegistrationWorkflowData {
+    fn workflow_type() -> &'static str {
+        "wasm_module_registration"
+    }
+}
+
+impl WasmRegistrationWorkflowData {
+    /// Validate that all transient fields are properly initialized.
+    pub fn validate_initialized(&self) -> Result<(), WorkflowError> {
+        if self.app_context.is_none() {
+            return Err(WorkflowError::ContextValueNotFound(
+                "app_context not initialized after deserialization".into(),
+            ));
+        }
+        Ok(())
+    }
+}
+
+/// Data for WASM module removal workflow
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct WasmRemovalWorkflowData {
+    pub config: WasmModuleRemovalRequest,
+    pub module_id: Option<String>,
+    /// Application context (transient, must be re-initialized after deserialization)
+    #[serde(skip, default)]
+    pub app_context: Option<Arc<AppContext>>,
+}
+
+impl WorkflowData for WasmRemovalWorkflowData {
+    fn workflow_type() -> &'static str {
+        "wasm_module_removal"
+    }
+}
+
+impl WasmRemovalWorkflowData {
+    /// Validate that all transient fields are properly initialized.
+    pub fn validate_initialized(&self) -> Result<(), WorkflowError> {
+        if self.app_context.is_none() {
+            return Err(WorkflowError::ContextValueNotFound(
+                "app_context not initialized after deserialization".into(),
+            ));
+        }
+        Ok(())
+    }
+}

third_party/sglang/sgl-model-gateway/src/core/steps/workflow_engines.rs

@@ -0,0 +1,166 @@
+//! Typed workflow engines collection
+//!
+//! This module provides a collection of typed workflow engines for different workflow types.
+//! Each workflow type has its own engine with compile-time type safety.
+
+use std::sync::Arc;
+
+use wfaas::{EventSubscriber, InMemoryStore, WorkflowEngine};
+
+use super::{
+    create_external_worker_workflow, create_local_worker_workflow,
+    create_mcp_registration_workflow, create_tokenizer_registration_workflow,
+    create_wasm_module_registration_workflow, create_wasm_module_removal_workflow,
+    create_worker_removal_workflow, create_worker_update_workflow, ExternalWorkerWorkflowData,
+    LocalWorkerWorkflowData, McpWorkflowData, TokenizerWorkflowData, WasmRegistrationWorkflowData,
+    WasmRemovalWorkflowData, WorkerRemovalWorkflowData, WorkerUpdateWorkflowData,
+};
+use crate::config::RouterConfig;
+
+/// Type alias for local worker workflow engine
+pub type LocalWorkerEngine =
+    WorkflowEngine<LocalWorkerWorkflowData, InMemoryStore<LocalWorkerWorkflowData>>;
+
+/// Type alias for external worker workflow engine
+pub type ExternalWorkerEngine =
+    WorkflowEngine<ExternalWorkerWorkflowData, InMemoryStore<ExternalWorkerWorkflowData>>;
+
+/// Type alias for worker removal workflow engine
+pub type WorkerRemovalEngine =
+    WorkflowEngine<WorkerRemovalWorkflowData, InMemoryStore<WorkerRemovalWorkflowData>>;
+
+/// Type alias for worker update workflow engine
+pub type WorkerUpdateEngine =
+    WorkflowEngine<WorkerUpdateWorkflowData, InMemoryStore<WorkerUpdateWorkflowData>>;
+
+/// Type alias for MCP registration workflow engine
+pub type McpEngine = WorkflowEngine<McpWorkflowData, InMemoryStore<McpWorkflowData>>;
+
+/// Type alias for tokenizer registration workflow engine
+pub type TokenizerEngine =
+    WorkflowEngine<TokenizerWorkflowData, InMemoryStore<TokenizerWorkflowData>>;
+
+/// Type alias for WASM registration workflow engine
+pub type WasmRegistrationEngine =
+    WorkflowEngine<WasmRegistrationWorkflowData, InMemoryStore<WasmRegistrationWorkflowData>>;
+
+/// Type alias for WASM removal workflow engine
+pub type WasmRemovalEngine =
+    WorkflowEngine<WasmRemovalWorkflowData, InMemoryStore<WasmRemovalWorkflowData>>;
+
+/// Collection of typed workflow engines
+///
+/// Each workflow type has its own engine with compile-time type safety.
+/// This replaces the old `WorkflowEngine<AnyWorkflowData, ...>` approach.
+#[derive(Clone, Debug)]
+pub struct WorkflowEngines {
+    /// Engine for local worker registration workflows
+    pub local_worker: Arc<LocalWorkerEngine>,
+    /// Engine for external worker registration workflows
+    pub external_worker: Arc<ExternalWorkerEngine>,
+    /// Engine for worker removal workflows
+    pub worker_removal: Arc<WorkerRemovalEngine>,
+    /// Engine for worker update workflows
+    pub worker_update: Arc<WorkerUpdateEngine>,
+    /// Engine for MCP server registration workflows
+    pub mcp: Arc<McpEngine>,
+    /// Engine for tokenizer registration workflows
+    pub tokenizer: Arc<TokenizerEngine>,
+    /// Engine for WASM module registration workflows
+    pub wasm_registration: Arc<WasmRegistrationEngine>,
+    /// Engine for WASM module removal workflows
+    pub wasm_removal: Arc<WasmRemovalEngine>,
+}
+
+impl WorkflowEngines {
+    /// Create and initialize all workflow engines with their workflow definitions
+    pub fn new(router_config: &RouterConfig) -> Self {
+        // Create local worker engine
+        let local_worker = WorkflowEngine::new();
+        local_worker
+            .register_workflow(create_local_worker_workflow(router_config))
+            .expect("local_worker_registration workflow should be valid");
+
+        // Create external worker engine
+        let external_worker = WorkflowEngine::new();
+        external_worker
+            .register_workflow(create_external_worker_workflow())
+            .expect("external_worker_registration workflow should be valid");
+
+        // Create worker removal engine
+        let worker_removal = WorkflowEngine::new();
+        worker_removal
+            .register_workflow(create_worker_removal_workflow())
+            .expect("worker_removal workflow should be valid");
+
+        // Create worker update engine
+        let worker_update = WorkflowEngine::new();
+        worker_update
+            .register_workflow(create_worker_update_workflow())
+            .expect("worker_update workflow should be valid");
+
+        // Create MCP engine
+        let mcp = WorkflowEngine::new();
+        mcp.register_workflow(create_mcp_registration_workflow())
+            .expect("mcp_registration workflow should be valid");
+
+        // Create tokenizer engine
+        let tokenizer = WorkflowEngine::new();
+        tokenizer
+            .register_workflow(create_tokenizer_registration_workflow())
+            .expect("tokenizer_registration workflow should be valid");
+
+        // Create WASM registration engine
+        let wasm_registration = WorkflowEngine::new();
+        wasm_registration
+            .register_workflow(create_wasm_module_registration_workflow())
+            .expect("wasm_module_registration workflow should be valid");
+
+        // Create WASM removal engine
+        let wasm_removal = WorkflowEngine::new();
+        wasm_removal
+            .register_workflow(create_wasm_module_removal_workflow())
+            .expect("wasm_module_removal workflow should be valid");
+
+        Self {
+            local_worker: Arc::new(local_worker),
+            external_worker: Arc::new(external_worker),
+            worker_removal: Arc::new(worker_removal),
+            worker_update: Arc::new(worker_update),
+            mcp: Arc::new(mcp),
+            tokenizer: Arc::new(tokenizer),
+            wasm_registration: Arc::new(wasm_registration),
+            wasm_removal: Arc::new(wasm_removal),
+        }
+    }
+
+    /// Subscribe an event subscriber to all workflow engines
+    pub async fn subscribe_all<S: EventSubscriber + 'static>(&self, subscriber: Arc<S>) {
+        self.local_worker
+            .event_bus()
+            .subscribe(subscriber.clone())
+            .await;
+        self.external_worker
+            .event_bus()
+            .subscribe(subscriber.clone())
+            .await;
+        self.worker_removal
+            .event_bus()
+            .subscribe(subscriber.clone())
+            .await;
+        self.worker_update
+            .event_bus()
+            .subscribe(subscriber.clone())
+            .await;
+        self.mcp.event_bus().subscribe(subscriber.clone()).await;
+        self.tokenizer
+            .event_bus()
+            .subscribe(subscriber.clone())
+            .await;
+        self.wasm_registration
+            .event_bus()
+            .subscribe(subscriber.clone())
+            .await;
+        self.wasm_removal.event_bus().subscribe(subscriber).await;
+    }
+}

third_party/sglang/sgl-model-gateway/src/core/token_bucket.rs

@@ -0,0 +1,279 @@
+use std::{
+    sync::Arc,
+    time::{Duration, Instant},
+};
+
+use parking_lot::Mutex;
+use tokio::sync::Notify;
+use tracing::{debug, trace};
+
+/// Token bucket for rate limiting.
+///
+/// This implementation provides:
+/// - Smooth rate limiting with configurable refill rate
+/// - Burst capacity handling
+/// - Fair queuing for waiting requests via Notify
+/// - Sync token return for Drop handlers (via `return_tokens_sync`)
+///
+/// Uses `parking_lot::Mutex` for sync-compatible locking (no async required).
+#[derive(Clone)]
+pub struct TokenBucket {
+    inner: Arc<Mutex<TokenBucketInner>>,
+    notify: Arc<Notify>,
+    capacity: f64,
+    refill_rate: f64, // tokens per second
+}
+
+struct TokenBucketInner {
+    tokens: f64,
+    last_refill: Instant,
+}
+
+impl TokenBucket {
+    /// Create a new token bucket
+    ///
+    /// # Arguments
+    /// * `capacity` - Maximum number of tokens (burst capacity)
+    /// * `refill_rate` - Tokens added per second (0 for pure concurrency limiting)
+    pub fn new(capacity: usize, refill_rate: usize) -> Self {
+        let capacity = capacity as f64;
+        // Allow refill_rate=0 for pure concurrency limiting (semaphore behavior)
+        // When refill_rate=0, tokens are only returned via return_tokens()
+        let refill_rate = refill_rate as f64;
+
+        Self {
+            inner: Arc::new(Mutex::new(TokenBucketInner {
+                tokens: capacity,
+                last_refill: Instant::now(),
+            })),
+            notify: Arc::new(Notify::new()),
+            capacity,
+            refill_rate,
+        }
+    }
+
+    /// Try to acquire tokens immediately.
+    ///
+    /// Returns `Ok(())` if tokens were acquired, `Err(())` if insufficient tokens.
+    pub async fn try_acquire(&self, tokens: f64) -> Result<(), ()> {
+        self.try_acquire_sync(tokens)
+    }
+
+    /// Sync version of try_acquire (for internal use).
+    fn try_acquire_sync(&self, tokens: f64) -> Result<(), ()> {
+        let mut inner = self.inner.lock();
+
+        let now = Instant::now();
+        let elapsed = now.duration_since(inner.last_refill).as_secs_f64();
+        let refill_amount = elapsed * self.refill_rate;
+
+        inner.tokens = (inner.tokens + refill_amount).min(self.capacity);
+        inner.last_refill = now;
+
+        trace!(
+            "Token bucket: {} tokens available, requesting {}",
+            inner.tokens,
+            tokens
+        );
+
+        if inner.tokens >= tokens {
+            inner.tokens -= tokens;
+            debug!(
+                "Token bucket: acquired {} tokens, {} remaining",
+                tokens, inner.tokens
+            );
+            Ok(())
+        } else {
+            Err(())
+        }
+    }
+
+    /// Acquire tokens, waiting if necessary.
+    ///
+    /// When `refill_rate=0`, waits indefinitely for tokens to be returned via `return_tokens()`.
+    /// Use `acquire_timeout()` to set an appropriate timeout.
+    pub async fn acquire(&self, tokens: f64) -> Result<(), tokio::time::error::Elapsed> {
+        if self.try_acquire(tokens).await.is_ok() {
+            return Ok(());
+        }
+
+        // When refill_rate=0 (pure concurrency limiting), tokens only come back
+        // via return_tokens(), so we wait on notify signal only.
+        if self.refill_rate == 0.0 {
+            debug!(
+                "Token bucket: waiting indefinitely for {} tokens (refill_rate=0)",
+                tokens
+            );
+
+            loop {
+                // Wait for notify signal from return_tokens()
+                self.notify.notified().await;
+
+                if self.try_acquire(tokens).await.is_ok() {
+                    return Ok(());
+                }
+            }
+        }
+
+        let wait_time = {
+            let inner = self.inner.lock();
+            let tokens_needed = tokens - inner.tokens;
+            let wait_secs = (tokens_needed / self.refill_rate).max(0.0);
+            Duration::from_secs_f64(wait_secs)
+        };
+
+        debug!(
+            "Token bucket: waiting {:?} for {} tokens",
+            wait_time, tokens
+        );
+
+        tokio::time::timeout(wait_time, async {
+            loop {
+                if self.try_acquire(tokens).await.is_ok() {
+                    return;
+                }
+                tokio::select! {
+                    _ = self.notify.notified() => {},
+                    _ = tokio::time::sleep(Duration::from_millis(10)) => {},
+                }
+            }
+        })
+        .await?;
+
+        Ok(())
+    }
+
+    /// Acquire tokens with custom timeout.
+    pub async fn acquire_timeout(
+        &self,
+        tokens: f64,
+        timeout: Duration,
+    ) -> Result<(), tokio::time::error::Elapsed> {
+        tokio::time::timeout(timeout, self.acquire(tokens)).await?
+    }
+
+    /// Return tokens to the bucket (sync version).
+    ///
+    /// This is safe to call from sync contexts (e.g., Drop handlers).
+    /// Uses `parking_lot::Mutex` which never blocks indefinitely.
+    pub fn return_tokens_sync(&self, tokens: f64) {
+        {
+            let mut inner = self.inner.lock();
+            inner.tokens = (inner.tokens + tokens).min(self.capacity);
+            debug!(
+                "Token bucket: returned {} tokens, {} available",
+                tokens, inner.tokens
+            );
+        } // Release lock before notify
+        self.notify.notify_waiters();
+    }
+
+    /// Return tokens to the bucket (async version for API compatibility).
+    pub async fn return_tokens(&self, tokens: f64) {
+        self.return_tokens_sync(tokens);
+    }
+
+    /// Get current available tokens (for monitoring).
+    pub async fn available_tokens(&self) -> f64 {
+        let mut inner = self.inner.lock();
+
+        let now = Instant::now();
+        let elapsed = now.duration_since(inner.last_refill).as_secs_f64();
+        let refill_amount = elapsed * self.refill_rate;
+
+        inner.tokens = (inner.tokens + refill_amount).min(self.capacity);
+        inner.last_refill = now;
+
+        inner.tokens
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[tokio::test]
+    async fn test_token_bucket_basic() {
+        let bucket = TokenBucket::new(10, 5);
+
+        assert!(bucket.try_acquire(5.0).await.is_ok());
+        assert!(bucket.try_acquire(5.0).await.is_ok());
+
+        assert!(bucket.try_acquire(1.0).await.is_err());
+
+        tokio::time::sleep(Duration::from_millis(300)).await;
+
+        assert!(bucket.try_acquire(1.0).await.is_ok());
+    }
+
+    #[tokio::test]
+    async fn test_token_bucket_refill() {
+        let bucket = TokenBucket::new(10, 10);
+
+        assert!(bucket.try_acquire(10.0).await.is_ok());
+
+        tokio::time::sleep(Duration::from_millis(500)).await;
+
+        let available = bucket.available_tokens().await;
+        assert!((4.0..=6.0).contains(&available));
+    }
+
+    #[tokio::test]
+    async fn test_token_bucket_zero_refill_rate() {
+        // With refill_rate=0, tokens should only come back via return_tokens()
+        let bucket = TokenBucket::new(2, 0);
+
+        // Acquire both tokens
+        assert!(bucket.try_acquire(1.0).await.is_ok());
+        assert!(bucket.try_acquire(1.0).await.is_ok());
+
+        // No more tokens available
+        assert!(bucket.try_acquire(1.0).await.is_err());
+
+        // Wait - should NOT refill automatically
+        tokio::time::sleep(Duration::from_millis(500)).await;
+        assert!(bucket.try_acquire(1.0).await.is_err());
+
+        // Return a token - now we should be able to acquire
+        bucket.return_tokens(1.0).await;
+        assert!(bucket.try_acquire(1.0).await.is_ok());
+
+        // No more tokens again
+        assert!(bucket.try_acquire(1.0).await.is_err());
+    }
+
+    #[tokio::test]
+    async fn test_token_bucket_zero_refill_with_notify() {
+        // Test that acquire wakes up when tokens are returned
+        let bucket = Arc::new(TokenBucket::new(1, 0));
+
+        // Acquire the only token
+        assert!(bucket.try_acquire(1.0).await.is_ok());
+
+        let bucket_clone = bucket.clone();
+
+        // Spawn a task that will return the token after a delay
+        tokio::spawn(async move {
+            tokio::time::sleep(Duration::from_millis(100)).await;
+            bucket_clone.return_tokens(1.0).await;
+        });
+
+        // This should wait and then succeed when token is returned
+        let result = bucket.acquire_timeout(1.0, Duration::from_secs(1)).await;
+        assert!(result.is_ok());
+    }
+
+    #[tokio::test]
+    async fn test_return_tokens_sync() {
+        // Test that sync return works correctly
+        let bucket = TokenBucket::new(2, 0);
+
+        assert!(bucket.try_acquire(1.0).await.is_ok());
+        assert!(bucket.try_acquire(1.0).await.is_ok());
+        assert!(bucket.try_acquire(1.0).await.is_err());
+
+        // Use sync return
+        bucket.return_tokens_sync(1.0);
+        assert!(bucket.try_acquire(1.0).await.is_ok());
+    }
+}

third_party/sglang/sgl-model-gateway/src/core/worker_builder.rs

@@ -0,0 +1,553 @@
+use std::collections::HashMap;
+
+use super::{
+    circuit_breaker::{CircuitBreaker, CircuitBreakerConfig},
+    model_card::ModelCard,
+    model_type::ModelType,
+    worker::{
+        BasicWorker, ConnectionMode, DPAwareWorker, HealthConfig, RuntimeType, WorkerMetadata,
+        WorkerRoutingKeyLoad, WorkerType,
+    },
+};
+use crate::{observability::metrics::Metrics, routers::grpc::client::GrpcClient};
+
+/// Builder for creating BasicWorker instances with fluent API
+pub struct BasicWorkerBuilder {
+    url: String,
+    api_key: Option<String>,
+    worker_type: WorkerType,
+    connection_mode: ConnectionMode,
+    runtime_type: RuntimeType,
+    labels: HashMap<String, String>,
+    models: Vec<ModelCard>,
+    health_config: HealthConfig,
+    circuit_breaker_config: CircuitBreakerConfig,
+    grpc_client: Option<GrpcClient>,
+}
+
+impl BasicWorkerBuilder {
+    /// Create a new builder with only the URL
+    pub fn new(url: impl Into<String>) -> Self {
+        Self {
+            url: url.into(),
+            api_key: None,
+            worker_type: WorkerType::Regular,
+            connection_mode: ConnectionMode::Http,
+            runtime_type: RuntimeType::default(),
+            labels: HashMap::new(),
+            models: Vec::new(),
+            health_config: HealthConfig::default(),
+            circuit_breaker_config: CircuitBreakerConfig::default(),
+            grpc_client: None,
+        }
+    }
+
+    /// Create a new builder with URL and worker type (for backwards compatibility)
+    pub fn new_with_type(url: impl Into<String>, worker_type: WorkerType) -> Self {
+        Self {
+            url: url.into(),
+            api_key: None,
+            worker_type,
+            connection_mode: ConnectionMode::Http,
+            runtime_type: RuntimeType::default(),
+            labels: HashMap::new(),
+            models: Vec::new(),
+            health_config: HealthConfig::default(),
+            circuit_breaker_config: CircuitBreakerConfig::default(),
+            grpc_client: None,
+        }
+    }
+
+    /// Set the API key
+    pub fn api_key(mut self, api_key: impl Into<String>) -> Self {
+        self.api_key = Some(api_key.into());
+        self
+    }
+
+    /// Set the worker type (Regular, Prefill, or Decode)
+    pub fn worker_type(mut self, worker_type: WorkerType) -> Self {
+        self.worker_type = worker_type;
+        self
+    }
+
+    /// Set the connection mode (HTTP or gRPC)
+    pub fn connection_mode(mut self, mode: ConnectionMode) -> Self {
+        self.connection_mode = mode;
+        self
+    }
+
+    /// Set the runtime type (SGLang or vLLM)
+    pub fn runtime_type(mut self, runtime_type: RuntimeType) -> Self {
+        self.runtime_type = runtime_type;
+        self
+    }
+
+    /// Set labels for worker identification
+    pub fn labels(mut self, labels: HashMap<String, String>) -> Self {
+        self.labels = labels;
+        self
+    }
+
+    /// Add a single label
+    pub fn label(mut self, key: impl Into<String>, value: impl Into<String>) -> Self {
+        self.labels.insert(key.into(), value.into());
+        self
+    }
+
+    /// Set health check configuration
+    pub fn health_config(mut self, config: HealthConfig) -> Self {
+        self.health_config = config;
+        self
+    }
+
+    /// Set circuit breaker configuration
+    pub fn circuit_breaker_config(mut self, config: CircuitBreakerConfig) -> Self {
+        self.circuit_breaker_config = config;
+        self
+    }
+
+    /// Set gRPC client for gRPC workers
+    pub fn grpc_client(mut self, client: GrpcClient) -> Self {
+        self.grpc_client = Some(client);
+        self
+    }
+
+    /// Set models this worker can serve
+    pub fn models(mut self, models: Vec<ModelCard>) -> Self {
+        self.models = models;
+        self
+    }
+
+    /// Add a single model this worker can serve
+    pub fn model(mut self, model: ModelCard) -> Self {
+        self.models.push(model);
+        self
+    }
+
+    /// Build the BasicWorker instance
+    pub fn build(self) -> BasicWorker {
+        use std::sync::{
+            atomic::{AtomicBool, AtomicUsize},
+            Arc, RwLock as StdRwLock,
+        };
+
+        use tokio::sync::OnceCell;
+
+        let bootstrap_host = match url::Url::parse(&self.url) {
+            Ok(parsed) => parsed.host_str().unwrap_or("localhost").to_string(),
+            Err(_) if !self.url.contains("://") => {
+                match url::Url::parse(&format!("http://{}", self.url)) {
+                    Ok(parsed) => parsed.host_str().unwrap_or("localhost").to_string(),
+                    Err(_) => {
+                        tracing::warn!(
+                            "Failed to parse URL '{}', defaulting to localhost",
+                            self.url
+                        );
+                        "localhost".to_string()
+                    }
+                }
+            }
+            Err(_) => {
+                tracing::warn!(
+                    "Failed to parse URL '{}', defaulting to localhost",
+                    self.url
+                );
+                "localhost".to_string()
+            }
+        };
+
+        let bootstrap_port = match self.worker_type {
+            WorkerType::Prefill { bootstrap_port } => bootstrap_port,
+            _ => None,
+        };
+
+        let metadata = WorkerMetadata {
+            url: self.url.clone(),
+            api_key: self.api_key,
+            worker_type: self.worker_type,
+            connection_mode: self.connection_mode,
+            runtime_type: self.runtime_type,
+            labels: self.labels,
+            health_config: self.health_config,
+            bootstrap_host,
+            bootstrap_port,
+            models: self.models,                // Empty = accepts any model
+            default_provider: None,             // Native/passthrough
+            default_model_type: ModelType::LLM, // Standard LLM capabilities
+        };
+
+        // Use OnceCell for lock-free gRPC client access after initialization
+        let grpc_client = Arc::new(match self.grpc_client {
+            Some(client) => {
+                let cell = OnceCell::new();
+                // Pre-set the client if provided (blocking set is fine during construction)
+                cell.set(Arc::new(client)).ok();
+                cell
+            }
+            None => OnceCell::new(),
+        });
+
+        let healthy = true;
+        Metrics::set_worker_health(&self.url, healthy);
+
+        BasicWorker {
+            metadata,
+            load_counter: Arc::new(AtomicUsize::new(0)),
+            worker_routing_key_load: Arc::new(WorkerRoutingKeyLoad::new(&self.url)),
+            processed_counter: Arc::new(AtomicUsize::new(0)),
+            healthy: Arc::new(AtomicBool::new(healthy)),
+            consecutive_failures: Arc::new(AtomicUsize::new(0)),
+            consecutive_successes: Arc::new(AtomicUsize::new(0)),
+            circuit_breaker: CircuitBreaker::with_config_and_label(
+                self.circuit_breaker_config,
+                self.url.clone(),
+            ),
+            grpc_client,
+            models_override: Arc::new(StdRwLock::new(None)),
+        }
+    }
+}
+
+/// Builder for creating DPAwareWorker instances with fluent API
+pub struct DPAwareWorkerBuilder {
+    base_url: String,
+    api_key: Option<String>,
+    dp_rank: usize,
+    dp_size: usize,
+    worker_type: WorkerType,
+    connection_mode: ConnectionMode,
+    runtime_type: RuntimeType,
+    labels: HashMap<String, String>,
+    models: Vec<ModelCard>,
+    health_config: HealthConfig,
+    circuit_breaker_config: CircuitBreakerConfig,
+    grpc_client: Option<GrpcClient>,
+}
+
+impl DPAwareWorkerBuilder {
+    /// Create a new DP-aware worker builder
+    pub fn new(base_url: impl Into<String>, dp_rank: usize, dp_size: usize) -> Self {
+        Self {
+            base_url: base_url.into(),
+            api_key: None,
+            dp_rank,
+            dp_size,
+            worker_type: WorkerType::Regular,
+            connection_mode: ConnectionMode::Http,
+            runtime_type: RuntimeType::default(),
+            labels: HashMap::new(),
+            models: Vec::new(),
+            health_config: HealthConfig::default(),
+            circuit_breaker_config: CircuitBreakerConfig::default(),
+            grpc_client: None,
+        }
+    }
+
+    /// Create a new DP-aware worker builder with worker type (for backwards compatibility)
+    pub fn new_with_type(
+        base_url: impl Into<String>,
+        dp_rank: usize,
+        dp_size: usize,
+        worker_type: WorkerType,
+    ) -> Self {
+        Self {
+            base_url: base_url.into(),
+            api_key: None,
+            dp_rank,
+            dp_size,
+            worker_type,
+            connection_mode: ConnectionMode::Http,
+            runtime_type: RuntimeType::default(),
+            labels: HashMap::new(),
+            models: Vec::new(),
+            health_config: HealthConfig::default(),
+            circuit_breaker_config: CircuitBreakerConfig::default(),
+            grpc_client: None,
+        }
+    }
+
+    /// Set the API key
+    pub fn api_key(mut self, api_key: impl Into<String>) -> Self {
+        self.api_key = Some(api_key.into());
+        self
+    }
+
+    /// Set the worker type (Regular, Prefill, or Decode)
+    pub fn worker_type(mut self, worker_type: WorkerType) -> Self {
+        self.worker_type = worker_type;
+        self
+    }
+
+    /// Set the connection mode (HTTP or gRPC)
+    pub fn connection_mode(mut self, mode: ConnectionMode) -> Self {
+        self.connection_mode = mode;
+        self
+    }
+
+    /// Set the runtime type (SGLang or vLLM)
+    pub fn runtime_type(mut self, runtime_type: RuntimeType) -> Self {
+        self.runtime_type = runtime_type;
+        self
+    }
+
+    /// Set labels for worker identification
+    pub fn labels(mut self, labels: HashMap<String, String>) -> Self {
+        self.labels = labels;
+        self
+    }
+
+    /// Add a single label
+    pub fn label(mut self, key: impl Into<String>, value: impl Into<String>) -> Self {
+        self.labels.insert(key.into(), value.into());
+        self
+    }
+
+    /// Set health check configuration
+    pub fn health_config(mut self, config: HealthConfig) -> Self {
+        self.health_config = config;
+        self
+    }
+
+    /// Set circuit breaker configuration
+    pub fn circuit_breaker_config(mut self, config: CircuitBreakerConfig) -> Self {
+        self.circuit_breaker_config = config;
+        self
+    }
+
+    /// Set gRPC client for gRPC workers
+    pub fn grpc_client(mut self, client: GrpcClient) -> Self {
+        self.grpc_client = Some(client);
+        self
+    }
+
+    /// Set models this worker can serve
+    pub fn models(mut self, models: Vec<ModelCard>) -> Self {
+        self.models = models;
+        self
+    }
+
+    /// Add a single model this worker can serve
+    pub fn model(mut self, model: ModelCard) -> Self {
+        self.models.push(model);
+        self
+    }
+
+    /// Build the DPAwareWorker instance
+    pub fn build(self) -> DPAwareWorker {
+        let worker_url = format!("{}@{}", self.base_url, self.dp_rank);
+        let mut builder = BasicWorkerBuilder::new(worker_url)
+            .models(self.models)
+            .worker_type(self.worker_type)
+            .connection_mode(self.connection_mode)
+            .runtime_type(self.runtime_type)
+            .labels(self.labels)
+            .health_config(self.health_config)
+            .circuit_breaker_config(self.circuit_breaker_config);
+
+        if let Some(client) = self.grpc_client {
+            builder = builder.grpc_client(client);
+        }
+        if let Some(api_key) = self.api_key {
+            builder = builder.api_key(api_key);
+        }
+
+        let base_worker = builder.build();
+        DPAwareWorker::with_base_worker(base_worker, self.base_url, self.dp_rank, self.dp_size)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::time::Duration;
+
+    use super::*;
+    use crate::core::worker::Worker;
+
+    #[test]
+    fn test_basic_worker_builder_minimal() {
+        let worker = BasicWorkerBuilder::new("http://localhost:8080").build();
+
+        assert_eq!(worker.url(), "http://localhost:8080");
+        assert_eq!(worker.worker_type(), &WorkerType::Regular);
+        assert_eq!(worker.connection_mode(), &ConnectionMode::Http);
+        assert!(worker.is_healthy());
+    }
+
+    #[test]
+    fn test_basic_worker_builder_with_type() {
+        let worker = BasicWorkerBuilder::new("http://localhost:8080")
+            .worker_type(WorkerType::Decode)
+            .build();
+
+        assert_eq!(worker.url(), "http://localhost:8080");
+        assert_eq!(worker.worker_type(), &WorkerType::Decode);
+        assert_eq!(worker.connection_mode(), &ConnectionMode::Http);
+        assert!(worker.is_healthy());
+    }
+
+    #[test]
+    fn test_basic_worker_builder_full() {
+        let mut labels = HashMap::new();
+        labels.insert("env".to_string(), "prod".to_string());
+        labels.insert("region".to_string(), "us-east".to_string());
+
+        let health_config = HealthConfig {
+            endpoint: "/health".to_string(),
+            timeout_secs: 30,
+            check_interval_secs: 60,
+            failure_threshold: 3,
+            success_threshold: 2,
+            disable_health_check: false,
+        };
+
+        let cb_config = CircuitBreakerConfig {
+            failure_threshold: 10,
+            success_threshold: 5,
+            timeout_duration: Duration::from_millis(2000),
+            window_duration: Duration::from_millis(30000),
+        };
+
+        let worker = BasicWorkerBuilder::new("http://localhost:8080")
+            .worker_type(WorkerType::Prefill {
+                bootstrap_port: None,
+            })
+            .connection_mode(ConnectionMode::Grpc { port: Some(50051) })
+            .labels(labels.clone())
+            .health_config(health_config.clone())
+            .circuit_breaker_config(cb_config)
+            .build();
+
+        assert_eq!(worker.url(), "http://localhost:8080");
+        assert_eq!(
+            worker.worker_type(),
+            &WorkerType::Prefill {
+                bootstrap_port: None
+            }
+        );
+        assert_eq!(
+            worker.connection_mode(),
+            &ConnectionMode::Grpc { port: Some(50051) }
+        );
+        assert_eq!(worker.metadata().labels, labels);
+        assert_eq!(
+            worker.metadata().health_config.endpoint,
+            health_config.endpoint
+        );
+        assert_eq!(
+            worker.metadata().health_config.timeout_secs,
+            health_config.timeout_secs
+        );
+        assert_eq!(
+            worker.metadata().health_config.check_interval_secs,
+            health_config.check_interval_secs
+        );
+        assert_eq!(
+            worker.metadata().health_config.failure_threshold,
+            health_config.failure_threshold
+        );
+        assert_eq!(
+            worker.metadata().health_config.success_threshold,
+            health_config.success_threshold
+        );
+    }
+
+    #[test]
+    fn test_basic_worker_builder_with_single_label() {
+        let worker = BasicWorkerBuilder::new("http://localhost:8080")
+            .worker_type(WorkerType::Decode)
+            .label("env", "staging")
+            .label("version", "v1.2.3")
+            .build();
+
+        assert_eq!(
+            worker.metadata().labels.get("env"),
+            Some(&"staging".to_string())
+        );
+        assert_eq!(
+            worker.metadata().labels.get("version"),
+            Some(&"v1.2.3".to_string())
+        );
+    }
+
+    #[test]
+    fn test_dp_aware_worker_builder_minimal() {
+        let worker = DPAwareWorkerBuilder::new("http://localhost:8080", 2, 8).build();
+
+        assert_eq!(worker.url(), "http://localhost:8080@2");
+        assert_eq!(worker.dp_rank(), Some(2));
+        assert_eq!(worker.dp_size(), Some(8));
+        assert_eq!(worker.worker_type(), &WorkerType::Regular);
+    }
+
+    #[test]
+    fn test_dp_aware_worker_builder_full() {
+        let mut labels = HashMap::new();
+        labels.insert("cluster".to_string(), "main".to_string());
+
+        let health_config = HealthConfig {
+            endpoint: "/status".to_string(),
+            timeout_secs: 20,
+            check_interval_secs: 45,
+            failure_threshold: 5,
+            success_threshold: 3,
+            disable_health_check: false,
+        };
+
+        let worker = DPAwareWorkerBuilder::new("http://localhost:8080", 3, 16)
+            .worker_type(WorkerType::Prefill {
+                bootstrap_port: Some(9090),
+            })
+            .connection_mode(ConnectionMode::Http)
+            .labels(labels.clone())
+            .health_config(health_config.clone())
+            .api_key("test_api_key")
+            .build();
+
+        assert_eq!(worker.url(), "http://localhost:8080@3");
+        assert_eq!(worker.dp_rank(), Some(3));
+        assert_eq!(worker.dp_size(), Some(16));
+        assert_eq!(worker.metadata().labels, labels);
+        assert_eq!(
+            worker.metadata().health_config.endpoint,
+            health_config.endpoint
+        );
+        assert_eq!(
+            worker.metadata().health_config.timeout_secs,
+            health_config.timeout_secs
+        );
+        assert_eq!(
+            worker.metadata().health_config.check_interval_secs,
+            health_config.check_interval_secs
+        );
+        assert_eq!(
+            worker.metadata().health_config.failure_threshold,
+            health_config.failure_threshold
+        );
+        assert_eq!(
+            worker.metadata().health_config.success_threshold,
+            health_config.success_threshold
+        );
+    }
+
+    #[test]
+    fn test_dp_aware_worker_with_grpc() {
+        let worker = DPAwareWorkerBuilder::new("grpc://cluster.local", 1, 4)
+            .worker_type(WorkerType::Decode)
+            .connection_mode(ConnectionMode::Grpc { port: Some(50051) })
+            .label("transport", "grpc")
+            .build();
+
+        assert_eq!(worker.url(), "grpc://cluster.local@1");
+        assert_eq!(worker.dp_rank(), Some(1));
+        assert_eq!(worker.dp_size(), Some(4));
+        assert_eq!(worker.worker_type(), &WorkerType::Decode);
+        assert_eq!(
+            worker.connection_mode(),
+            &ConnectionMode::Grpc { port: Some(50051) }
+        );
+        assert_eq!(
+            worker.metadata().labels.get("transport"),
+            Some(&"grpc".to_string())
+        );
+    }
+}

third_party/sglang/sgl-model-gateway/src/core/worker_manager.rs

@@ -0,0 +1,394 @@
+//! Worker Management Module
+//!
+//! Provides worker lifecycle operations and fan-out request utilities.
+
+use std::{collections::HashMap, sync::Arc, time::Duration};
+
+use axum::response::{IntoResponse, Response};
+use futures::{
+    future,
+    stream::{self, StreamExt},
+};
+use http::StatusCode;
+use serde_json::Value;
+use tokio::{
+    sync::{watch, Mutex},
+    task::JoinHandle,
+};
+use tracing::{debug, info, warn};
+
+use crate::{
+    core::{metrics_aggregator::MetricPack, ConnectionMode, Worker, WorkerRegistry, WorkerType},
+    policies::PolicyRegistry,
+    protocols::worker_spec::{FlushCacheResult, WorkerLoadInfo, WorkerLoadsResult},
+};
+
+const REQUEST_TIMEOUT: Duration = Duration::from_secs(5);
+const MAX_CONCURRENT: usize = 32;
+
+/// Result of a fan-out request to a single worker
+struct WorkerResponse {
+    url: String,
+    result: Result<reqwest::Response, reqwest::Error>,
+}
+
+/// Fan out requests to workers in parallel
+async fn fan_out(
+    workers: &[Arc<dyn Worker>],
+    client: &reqwest::Client,
+    endpoint: &str,
+    method: reqwest::Method,
+) -> Vec<WorkerResponse> {
+    let futures: Vec<_> = workers
+        .iter()
+        .map(|worker| {
+            let client = client.clone();
+            let url = worker.url().to_string();
+            let full_url = format!("{}/{}", url, endpoint);
+            let api_key = worker.api_key().clone();
+            let method = method.clone();
+
+            async move {
+                let mut req = client.request(method, &full_url).timeout(REQUEST_TIMEOUT);
+                if let Some(key) = api_key {
+                    req = req.bearer_auth(key);
+                }
+                WorkerResponse {
+                    url,
+                    result: req.send().await,
+                }
+            }
+        })
+        .collect();
+
+    stream::iter(futures)
+        .buffer_unordered(MAX_CONCURRENT)
+        .collect()
+        .await
+}
+
+pub enum EngineMetricsResult {
+    Ok(String),
+    Err(String),
+}
+
+impl IntoResponse for EngineMetricsResult {
+    fn into_response(self) -> Response {
+        match self {
+            Self::Ok(text) => (StatusCode::OK, text).into_response(),
+            Self::Err(msg) => (StatusCode::INTERNAL_SERVER_ERROR, msg).into_response(),
+        }
+    }
+}
+
+pub struct WorkerManager;
+
+impl WorkerManager {
+    pub fn get_worker_urls(registry: &Arc<WorkerRegistry>) -> Vec<String> {
+        registry
+            .get_all()
+            .iter()
+            .map(|w| w.url().to_string())
+            .collect()
+    }
+
+    pub async fn flush_cache_all(
+        worker_registry: &WorkerRegistry,
+        client: &reqwest::Client,
+    ) -> FlushCacheResult {
+        let workers = worker_registry.get_all();
+        let total_workers = workers.len();
+
+        let http_workers: Vec<_> = workers
+            .into_iter()
+            .filter(|w| matches!(w.connection_mode(), ConnectionMode::Http))
+            .collect();
+
+        if http_workers.is_empty() {
+            return FlushCacheResult {
+                successful: vec![],
+                failed: vec![],
+                total_workers,
+                http_workers: 0,
+                message: "No HTTP workers available for cache flush".to_string(),
+            };
+        }
+
+        info!(
+            "Flushing cache on {} HTTP workers (out of {} total)",
+            http_workers.len(),
+            total_workers
+        );
+
+        let responses = fan_out(&http_workers, client, "flush_cache", reqwest::Method::POST).await;
+
+        let mut successful = Vec::new();
+        let mut failed = Vec::new();
+
+        for resp in responses {
+            match resp.result {
+                Ok(r) if r.status().is_success() => successful.push(resp.url),
+                Ok(r) => failed.push((resp.url, format!("HTTP {}", r.status()))),
+                Err(e) => failed.push((resp.url, e.to_string())),
+            }
+        }
+
+        let message = if failed.is_empty() {
+            format!(
+                "Successfully flushed cache on all {} HTTP workers",
+                successful.len()
+            )
+        } else {
+            format!(
+                "Cache flush: {} succeeded, {} failed",
+                successful.len(),
+                failed.len()
+            )
+        };
+
+        info!("{}", message);
+
+        FlushCacheResult {
+            successful,
+            failed,
+            total_workers,
+            http_workers: http_workers.len(),
+            message,
+        }
+    }
+
+    pub async fn get_all_worker_loads(
+        worker_registry: &WorkerRegistry,
+        client: &reqwest::Client,
+    ) -> WorkerLoadsResult {
+        let workers = worker_registry.get_all();
+        let total_workers = workers.len();
+
+        let futures: Vec<_> = workers
+            .iter()
+            .map(|worker| {
+                let url = worker.url().to_string();
+                let api_key = worker.api_key().clone();
+                let worker_type = match worker.worker_type() {
+                    WorkerType::Regular => None,
+                    WorkerType::Prefill { .. } => Some("prefill".to_string()),
+                    WorkerType::Decode => Some("decode".to_string()),
+                };
+                let is_http = matches!(worker.connection_mode(), ConnectionMode::Http);
+                let client = client.clone();
+
+                async move {
+                    let load = if is_http {
+                        Self::parse_load_response(&client, &url, api_key.as_deref()).await
+                    } else {
+                        -1
+                    };
+                    WorkerLoadInfo {
+                        worker: url,
+                        worker_type,
+                        load,
+                    }
+                }
+            })
+            .collect();
+
+        let loads = future::join_all(futures).await;
+        let successful = loads.iter().filter(|l| l.load >= 0).count();
+        let failed = loads.iter().filter(|l| l.load < 0).count();
+
+        WorkerLoadsResult {
+            loads,
+            total_workers,
+            successful,
+            failed,
+        }
+    }
+
+    async fn parse_load_response(
+        client: &reqwest::Client,
+        url: &str,
+        api_key: Option<&str>,
+    ) -> isize {
+        let load_url = format!("{}/get_load", url);
+        let mut req = client.get(&load_url).timeout(REQUEST_TIMEOUT);
+        if let Some(key) = api_key {
+            req = req.bearer_auth(key);
+        }
+
+        match req.send().await {
+            Ok(r) if r.status().is_success() => match r.json::<Value>().await {
+                Ok(json) if json.is_array() => json
+                    .as_array()
+                    .unwrap()
+                    .iter()
+                    .filter_map(|e| e.get("num_tokens").and_then(|v| v.as_i64()))
+                    .sum::<i64>() as isize,
+                _ => -1,
+            },
+            _ => -1,
+        }
+    }
+
+    pub async fn get_engine_metrics(
+        worker_registry: &WorkerRegistry,
+        client: &reqwest::Client,
+    ) -> EngineMetricsResult {
+        let workers = worker_registry.get_all();
+
+        if workers.is_empty() {
+            return EngineMetricsResult::Err("No available workers".to_string());
+        }
+
+        let responses = fan_out(&workers, client, "metrics", reqwest::Method::GET).await;
+
+        let mut metric_packs = Vec::new();
+        for resp in responses {
+            if let Ok(r) = resp.result {
+                if r.status().is_success() {
+                    if let Ok(text) = r.text().await {
+                        metric_packs.push(MetricPack {
+                            labels: vec![("worker_addr".into(), resp.url)],
+                            metrics_text: text,
+                        });
+                    }
+                }
+            }
+        }
+
+        if metric_packs.is_empty() {
+            return EngineMetricsResult::Err("All backend requests failed".to_string());
+        }
+
+        match crate::core::metrics_aggregator::aggregate_metrics(metric_packs) {
+            Ok(text) => EngineMetricsResult::Ok(text),
+            Err(e) => EngineMetricsResult::Err(format!("Failed to aggregate metrics: {}", e)),
+        }
+    }
+}
+
+/// Load monitoring service that periodically fetches worker loads
+pub struct LoadMonitor {
+    worker_registry: Arc<WorkerRegistry>,
+    policy_registry: Arc<PolicyRegistry>,
+    client: reqwest::Client,
+    interval: Duration,
+    tx: watch::Sender<HashMap<String, isize>>,
+    rx: watch::Receiver<HashMap<String, isize>>,
+    monitor_handle: Arc<Mutex<Option<JoinHandle<()>>>>,
+}
+
+impl LoadMonitor {
+    pub fn new(
+        worker_registry: Arc<WorkerRegistry>,
+        policy_registry: Arc<PolicyRegistry>,
+        client: reqwest::Client,
+        interval_secs: u64,
+    ) -> Self {
+        let (tx, rx) = watch::channel(HashMap::new());
+
+        Self {
+            worker_registry,
+            policy_registry,
+            client,
+            interval: Duration::from_secs(interval_secs),
+            tx,
+            rx,
+            monitor_handle: Arc::new(Mutex::new(None)),
+        }
+    }
+
+    pub async fn start(&self) {
+        let mut handle_guard = self.monitor_handle.lock().await;
+        if handle_guard.is_some() {
+            debug!("Load monitoring already running");
+            return;
+        }
+
+        info!(
+            "Starting load monitoring with interval: {:?}",
+            self.interval
+        );
+
+        let worker_registry = Arc::clone(&self.worker_registry);
+        let policy_registry = Arc::clone(&self.policy_registry);
+        let client = self.client.clone();
+        let interval = self.interval;
+        let tx = self.tx.clone();
+
+        let handle = tokio::spawn(async move {
+            Self::monitor_loop(worker_registry, policy_registry, client, interval, tx).await;
+        });
+
+        *handle_guard = Some(handle);
+    }
+
+    pub async fn stop(&self) {
+        let mut handle_guard = self.monitor_handle.lock().await;
+        if let Some(handle) = handle_guard.take() {
+            info!("Stopping load monitoring");
+            handle.abort();
+            let _ = handle.await; // Wait for task to finish
+        }
+    }
+
+    pub fn subscribe(&self) -> watch::Receiver<HashMap<String, isize>> {
+        self.rx.clone()
+    }
+
+    async fn monitor_loop(
+        worker_registry: Arc<WorkerRegistry>,
+        policy_registry: Arc<PolicyRegistry>,
+        client: reqwest::Client,
+        interval: Duration,
+        tx: watch::Sender<HashMap<String, isize>>,
+    ) {
+        let mut interval_timer = tokio::time::interval(interval);
+
+        loop {
+            interval_timer.tick().await;
+
+            let power_of_two_policies = policy_registry.get_all_power_of_two_policies();
+
+            if power_of_two_policies.is_empty() {
+                debug!("No PowerOfTwo policies found, skipping load fetch");
+                continue;
+            }
+
+            let result = WorkerManager::get_all_worker_loads(&worker_registry, &client).await;
+
+            let mut loads = HashMap::new();
+            for load_info in result.loads {
+                loads.insert(load_info.worker, load_info.load);
+            }
+
+            if !loads.is_empty() {
+                debug!(
+                    "Fetched loads from {} workers, updating {} PowerOfTwo policies",
+                    loads.len(),
+                    power_of_two_policies.len()
+                );
+                for policy in &power_of_two_policies {
+                    policy.update_loads(&loads);
+                }
+                let _ = tx.send(loads);
+            } else {
+                warn!("No loads fetched from workers");
+            }
+        }
+    }
+
+    pub async fn is_running(&self) -> bool {
+        let handle_guard = self.monitor_handle.lock().await;
+        handle_guard.is_some()
+    }
+}
+
+impl Drop for LoadMonitor {
+    fn drop(&mut self) {
+        if let Ok(mut handle_guard) = self.monitor_handle.try_lock() {
+            if let Some(handle) = handle_guard.take() {
+                handle.abort();
+            }
+        }
+    }
+}

third_party/sglang/sgl-model-gateway/src/core/worker_registry.rs

@@ -0,0 +1,835 @@
+//! Worker Registry for multi-router support
+//!
+//! Provides centralized registry for workers with model-based indexing
+//!
+//! # Performance Optimizations
+//! The model index uses immutable Arc snapshots instead of RwLock for lock-free reads.
+//! This is critical for high-concurrency scenarios where many requests query the same model.
+//!
+//! # Consistent Hash Ring
+//! The registry maintains a pre-computed hash ring per model for O(log n) consistent hashing.
+//! The ring is rebuilt only when workers are added/removed, not per-request.
+//! Uses virtual nodes (150 per worker) for even distribution and blake3 for stable hashing.
+
+use std::sync::{Arc, RwLock};
+
+use dashmap::DashMap;
+use smg_mesh::OptionalMeshSyncManager;
+use uuid::Uuid;
+
+use crate::{
+    core::{
+        circuit_breaker::CircuitState,
+        worker::{HealthChecker, RuntimeType, WorkerType},
+        ConnectionMode, Worker,
+    },
+    observability::metrics::Metrics,
+};
+
+/// Number of virtual nodes per physical worker for even distribution.
+/// 150 is a common choice that provides good balance between memory and distribution.
+const VIRTUAL_NODES_PER_WORKER: usize = 150;
+
+/// Consistent hash ring for O(log n) worker selection.
+///
+/// Each worker is placed at multiple positions (virtual nodes) on the ring
+/// based on hash(worker_url + vnode_index). This provides:
+/// - Even key distribution across workers
+/// - Minimal key redistribution when workers are added/removed (~1/N keys move)
+/// - O(log n) lookup via binary search
+///
+/// Uses blake3 for stable, fast hashing that's consistent across Rust versions.
+#[derive(Debug, Clone)]
+pub struct HashRing {
+    /// Sorted list of (ring_position, worker_url)
+    /// Multiple entries per worker (virtual nodes) for even distribution.
+    /// Uses Arc<str> to share URL across all virtual nodes (150 refs vs 150 copies).
+    entries: Arc<[(u64, Arc<str>)]>,
+}
+
+impl HashRing {
+    /// Build a hash ring from a list of workers.
+    /// Creates VIRTUAL_NODES_PER_WORKER entries per worker for even distribution.
+    pub fn new(workers: &[Arc<dyn Worker>]) -> Self {
+        let mut entries: Vec<(u64, Arc<str>)> =
+            Vec::with_capacity(workers.len() * VIRTUAL_NODES_PER_WORKER);
+
+        for worker in workers {
+            // Create Arc<str> once per worker, share across all virtual nodes
+            let url: Arc<str> = Arc::from(worker.url());
+            let url_bytes = url.as_bytes();
+
+            // Create multiple virtual nodes per worker
+            for vnode in 0..VIRTUAL_NODES_PER_WORKER {
+                let mut hasher = blake3::Hasher::new();
+                hasher.update(url_bytes);
+                hasher.update(b"#");
+                hasher.update(&(vnode as u64).to_le_bytes());
+                let hash = hasher.finalize();
+                let pos = u64::from_le_bytes(hash.as_bytes()[..8].try_into().unwrap());
+                entries.push((pos, Arc::clone(&url)));
+            }
+        }
+
+        // Sort by ring position for binary search
+        entries.sort_unstable_by_key(|(pos, _)| *pos);
+
+        Self {
+            entries: Arc::from(entries.into_boxed_slice()),
+        }
+    }
+
+    /// Hash a string to a ring position using blake3 (stable across versions).
+    #[inline]
+    fn hash_position(s: &str) -> u64 {
+        let hash = blake3::hash(s.as_bytes());
+        // Take first 8 bytes as u64
+        u64::from_le_bytes(hash.as_bytes()[..8].try_into().unwrap())
+    }
+
+    /// Find worker URL for a key using consistent hashing.
+    /// Returns the first healthy worker URL at or after the key's position (clockwise).
+    ///
+    /// - `key`: The routing key to hash
+    /// - `is_healthy`: Function to check if a worker URL is healthy
+    pub fn find_healthy_url<F>(&self, key: &str, is_healthy: F) -> Option<&str>
+    where
+        F: Fn(&str) -> bool,
+    {
+        if self.entries.is_empty() {
+            return None;
+        }
+
+        let key_pos = Self::hash_position(key);
+
+        // Binary search to find first entry at or after key_pos
+        let start = self.entries.partition_point(|(pos, _)| *pos < key_pos);
+
+        // Walk clockwise from start, wrapping around
+        // Track visited URLs to avoid checking same worker multiple times (virtual nodes)
+        let mut checked_urls =
+            std::collections::HashSet::with_capacity(self.worker_count().min(16));
+
+        for i in 0..self.entries.len() {
+            let (_, url) = &self.entries[(start + i) % self.entries.len()];
+            let url_str: &str = url;
+
+            // Skip if we already checked this worker (from another virtual node)
+            if !checked_urls.insert(url_str) {
+                continue;
+            }
+
+            if is_healthy(url_str) {
+                return Some(url_str);
+            }
+        }
+
+        None
+    }
+
+    /// Check if the ring is empty
+    pub fn is_empty(&self) -> bool {
+        self.entries.is_empty()
+    }
+
+    /// Get the number of entries in the ring (including virtual nodes)
+    pub fn len(&self) -> usize {
+        self.entries.len()
+    }
+
+    /// Get the number of unique workers in the ring
+    pub fn worker_count(&self) -> usize {
+        self.entries.len() / VIRTUAL_NODES_PER_WORKER.max(1)
+    }
+}
+
+/// Unique identifier for a worker
+#[derive(Debug, Clone, Hash, Eq, PartialEq)]
+pub struct WorkerId(String);
+
+impl WorkerId {
+    /// Create a new worker ID
+    pub fn new() -> Self {
+        Self(Uuid::new_v4().to_string())
+    }
+
+    /// Create a worker ID from a string
+    pub fn from_string(s: String) -> Self {
+        Self(s)
+    }
+
+    /// Get the ID as a string
+    pub fn as_str(&self) -> &str {
+        &self.0
+    }
+}
+
+impl Default for WorkerId {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+/// Model index using immutable snapshots for lock-free reads.
+/// Each model maps to an Arc'd slice of workers that can be read without locking.
+/// Updates create new snapshots (copy-on-write semantics).
+type ModelIndex = Arc<DashMap<String, Arc<[Arc<dyn Worker>]>>>;
+
+/// Worker registry with model-based indexing
+#[derive(Debug)]
+pub struct WorkerRegistry {
+    /// All workers indexed by ID
+    workers: Arc<DashMap<WorkerId, Arc<dyn Worker>>>,
+
+    /// Model index for O(1) lookups using immutable snapshots.
+    /// Uses Arc<[T]> instead of Arc<RwLock<Vec<T>>> for lock-free reads.
+    model_index: ModelIndex,
+
+    /// Consistent hash rings per model for O(log n) routing.
+    /// Rebuilt on worker add/remove (copy-on-write).
+    hash_rings: Arc<DashMap<String, Arc<HashRing>>>,
+
+    /// Workers indexed by worker type
+    type_workers: Arc<DashMap<WorkerType, Vec<WorkerId>>>,
+
+    /// Workers indexed by connection mode
+    connection_workers: Arc<DashMap<ConnectionMode, Vec<WorkerId>>>,
+
+    /// URL to worker ID mapping
+    url_to_id: Arc<DashMap<String, WorkerId>>,
+    /// Optional mesh sync manager for state synchronization
+    /// When None, the registry works independently without mesh synchronization
+    /// Uses RwLock for thread-safe access when setting mesh_sync after initialization
+    mesh_sync: Arc<RwLock<OptionalMeshSyncManager>>,
+}
+
+impl WorkerRegistry {
+    /// Create a new worker registry
+    pub fn new() -> Self {
+        Self {
+            workers: Arc::new(DashMap::new()),
+            model_index: Arc::new(DashMap::new()),
+            hash_rings: Arc::new(DashMap::new()),
+            type_workers: Arc::new(DashMap::new()),
+            connection_workers: Arc::new(DashMap::new()),
+            url_to_id: Arc::new(DashMap::new()),
+            mesh_sync: Arc::new(RwLock::new(None)),
+        }
+    }
+
+    /// Rebuild the hash ring for a model based on current workers in the model index
+    fn rebuild_hash_ring(&self, model_id: &str) {
+        if let Some(workers) = self.model_index.get(model_id) {
+            let ring = HashRing::new(&workers);
+            self.hash_rings.insert(model_id.to_string(), Arc::new(ring));
+        } else {
+            // No workers for this model, remove the ring
+            self.hash_rings.remove(model_id);
+        }
+    }
+
+    /// Get the hash ring for a model (O(1) lookup)
+    pub fn get_hash_ring(&self, model_id: &str) -> Option<Arc<HashRing>> {
+        self.hash_rings.get(model_id).map(|r| Arc::clone(&r))
+    }
+
+    /// Set mesh sync manager (thread-safe, can be called after initialization)
+    pub fn set_mesh_sync(&self, mesh_sync: OptionalMeshSyncManager) {
+        *self.mesh_sync.write().unwrap() = mesh_sync;
+    }
+
+    /// Register a new worker
+    pub fn register(&self, worker: Arc<dyn Worker>) -> WorkerId {
+        let worker_id = if let Some(existing_id) = self.url_to_id.get(worker.url()) {
+            // Worker with this URL already exists, update it
+            existing_id.clone()
+        } else {
+            WorkerId::new()
+        };
+
+        // Store worker
+        self.workers.insert(worker_id.clone(), worker.clone());
+
+        // Update URL mapping
+        self.url_to_id
+            .insert(worker.url().to_string(), worker_id.clone());
+
+        // Update model index for O(1) lookups using copy-on-write
+        // This creates a new immutable snapshot with the added worker
+        let model_id = worker.model_id().to_string();
+        self.model_index
+            .entry(model_id.clone())
+            .and_modify(|existing| {
+                // Create new snapshot with the additional worker
+                let mut new_workers: Vec<Arc<dyn Worker>> = existing.iter().cloned().collect();
+                new_workers.push(worker.clone());
+                *existing = Arc::from(new_workers.into_boxed_slice());
+            })
+            .or_insert_with(|| Arc::from(vec![worker.clone()].into_boxed_slice()));
+
+        // Rebuild hash ring for this model
+        self.rebuild_hash_ring(&model_id);
+
+        // Update type index (clone needed for DashMap key ownership)
+        self.type_workers
+            .entry(worker.worker_type().clone())
+            .or_default()
+            .push(worker_id.clone());
+
+        // Update connection mode index (clone needed for DashMap key ownership)
+        self.connection_workers
+            .entry(worker.connection_mode().clone())
+            .or_default()
+            .push(worker_id.clone());
+
+        // Sync to mesh if enabled (no-op if mesh is not enabled)
+        if let Some(ref mesh_sync) = *self.mesh_sync.read().unwrap() {
+            mesh_sync.sync_worker_state(
+                worker_id.as_str().to_string(),
+                worker.model_id().to_string(),
+                worker.url().to_string(),
+                worker.is_healthy(),
+                0.0, // TODO: Get actual load
+            );
+        }
+
+        worker_id
+    }
+
+    /// Reserve (or retrieve) a stable UUID for a worker URL.
+    /// Uses atomic entry API to avoid race conditions between check and insert.
+    pub fn reserve_id_for_url(&self, url: &str) -> WorkerId {
+        self.url_to_id.entry(url.to_string()).or_default().clone()
+    }
+
+    /// Best-effort lookup of the URL for a given worker ID.
+    pub fn get_url_by_id(&self, worker_id: &WorkerId) -> Option<String> {
+        if let Some(worker) = self.get(worker_id) {
+            return Some(worker.url().to_string());
+        }
+        self.url_to_id
+            .iter()
+            .find_map(|entry| (entry.value() == worker_id).then(|| entry.key().clone()))
+    }
+
+    /// Remove a worker by ID
+    pub fn remove(&self, worker_id: &WorkerId) -> Option<Arc<dyn Worker>> {
+        if let Some((_, worker)) = self.workers.remove(worker_id) {
+            // Remove from URL mapping
+            self.url_to_id.remove(worker.url());
+
+            // Remove from model index using copy-on-write
+            // Create new snapshot without the removed worker
+            let worker_url = worker.url();
+            let model_id = worker.model_id().to_string();
+            if let Some(mut entry) = self.model_index.get_mut(&model_id) {
+                let new_workers: Vec<Arc<dyn Worker>> = entry
+                    .iter()
+                    .filter(|w| w.url() != worker_url)
+                    .cloned()
+                    .collect();
+                *entry = Arc::from(new_workers.into_boxed_slice());
+            }
+
+            // Rebuild hash ring for this model
+            self.rebuild_hash_ring(&model_id);
+
+            // Remove from type index
+            if let Some(mut type_workers) = self.type_workers.get_mut(worker.worker_type()) {
+                type_workers.retain(|id| id != worker_id);
+            }
+
+            // Remove from connection mode index
+            if let Some(mut conn_workers) =
+                self.connection_workers.get_mut(worker.connection_mode())
+            {
+                conn_workers.retain(|id| id != worker_id);
+            }
+
+            worker.set_healthy(false);
+            Metrics::remove_worker_metrics(worker.url());
+
+            // Sync removal to mesh if enabled (no-op if mesh is not enabled)
+            if let Some(ref mesh_sync) = *self.mesh_sync.read().unwrap() {
+                mesh_sync.remove_worker_state(worker_id.as_str());
+            }
+
+            Some(worker)
+        } else {
+            None
+        }
+    }
+
+    /// Remove a worker by URL
+    pub fn remove_by_url(&self, url: &str) -> Option<Arc<dyn Worker>> {
+        if let Some((_, worker_id)) = self.url_to_id.remove(url) {
+            self.remove(&worker_id)
+        } else {
+            None
+        }
+    }
+
+    /// Get a worker by ID
+    pub fn get(&self, worker_id: &WorkerId) -> Option<Arc<dyn Worker>> {
+        self.workers.get(worker_id).map(|entry| entry.clone())
+    }
+
+    /// Get a worker by URL
+    pub fn get_by_url(&self, url: &str) -> Option<Arc<dyn Worker>> {
+        self.url_to_id.get(url).and_then(|id| self.get(&id))
+    }
+
+    /// Empty worker slice constant for returning when no workers found
+    const EMPTY_WORKERS: &'static [Arc<dyn Worker>] = &[];
+
+    /// Get all workers for a model (O(1) optimized, lock-free)
+    /// Returns an Arc to the immutable worker slice - just an atomic refcount bump.
+    /// This is the fastest possible read path with zero contention.
+    pub fn get_by_model(&self, model_id: &str) -> Arc<[Arc<dyn Worker>]> {
+        self.model_index
+            .get(model_id)
+            .map(|workers| Arc::clone(&workers))
+            .unwrap_or_else(|| Arc::from(Self::EMPTY_WORKERS))
+    }
+
+    /// Get all workers by worker type
+    pub fn get_by_type(&self, worker_type: &WorkerType) -> Vec<Arc<dyn Worker>> {
+        self.type_workers
+            .get(worker_type)
+            .map(|ids| ids.iter().filter_map(|id| self.get(id)).collect())
+            .unwrap_or_default()
+    }
+
+    /// Update worker health status and sync to mesh
+    pub fn update_worker_health(&self, worker_id: &WorkerId, is_healthy: bool) {
+        if let Some(worker) = self.workers.get(worker_id) {
+            // Update worker health (if Worker trait has a method for this)
+            // For now, we'll just sync to mesh
+
+            // Sync to mesh if enabled (no-op if mesh is not enabled)
+            if let Some(ref mesh_sync) = *self.mesh_sync.read().unwrap() {
+                mesh_sync.sync_worker_state(
+                    worker_id.as_str().to_string(),
+                    worker.model_id().to_string(),
+                    worker.url().to_string(),
+                    is_healthy,
+                    0.0, // TODO: Get actual load
+                );
+            }
+        }
+    }
+
+    /// Get all prefill workers (regardless of bootstrap_port)
+    pub fn get_prefill_workers(&self) -> Vec<Arc<dyn Worker>> {
+        self.workers
+            .iter()
+            .filter_map(|entry| {
+                let worker = entry.value();
+                match worker.worker_type() {
+                    WorkerType::Prefill { .. } => Some(worker.clone()),
+                    _ => None,
+                }
+            })
+            .collect()
+    }
+
+    /// Get all decode workers
+    pub fn get_decode_workers(&self) -> Vec<Arc<dyn Worker>> {
+        self.get_by_type(&WorkerType::Decode)
+    }
+
+    /// Get all workers by connection mode
+    pub fn get_by_connection(&self, connection_mode: &ConnectionMode) -> Vec<Arc<dyn Worker>> {
+        self.connection_workers
+            .get(connection_mode)
+            .map(|ids| ids.iter().filter_map(|id| self.get(id)).collect())
+            .unwrap_or_default()
+    }
+
+    /// Get the number of workers in the registry
+    pub fn len(&self) -> usize {
+        self.workers.len()
+    }
+
+    /// Check if the registry is empty
+    pub fn is_empty(&self) -> bool {
+        self.workers.is_empty()
+    }
+
+    /// Get all workers
+    pub fn get_all(&self) -> Vec<Arc<dyn Worker>> {
+        self.workers
+            .iter()
+            .map(|entry| entry.value().clone())
+            .collect()
+    }
+
+    /// Get all workers with their IDs
+    pub fn get_all_with_ids(&self) -> Vec<(WorkerId, Arc<dyn Worker>)> {
+        self.workers
+            .iter()
+            .map(|entry| (entry.key().clone(), entry.value().clone()))
+            .collect()
+    }
+
+    /// Get all worker URLs
+    pub fn get_all_urls(&self) -> Vec<String> {
+        self.workers
+            .iter()
+            .map(|entry| entry.value().url().to_string())
+            .collect()
+    }
+
+    pub fn get_all_urls_with_api_key(&self) -> Vec<(String, Option<String>)> {
+        self.workers
+            .iter()
+            .map(|entry| {
+                (
+                    entry.value().url().to_string(),
+                    entry.value().api_key().clone(),
+                )
+            })
+            .collect()
+    }
+
+    /// Get all model IDs with workers (lock-free)
+    pub fn get_models(&self) -> Vec<String> {
+        self.model_index
+            .iter()
+            .filter(|entry| !entry.value().is_empty())
+            .map(|entry| entry.key().clone())
+            .collect()
+    }
+
+    /// Get workers filtered by multiple criteria
+    ///
+    /// This method allows flexible filtering of workers based on:
+    /// - model_id: Filter by specific model
+    /// - worker_type: Filter by worker type (Regular, Prefill, Decode)
+    /// - connection_mode: Filter by connection mode (Http, Grpc)
+    /// - runtime_type: Filter by runtime type (Sglang, Vllm, External)
+    /// - healthy_only: Only return healthy workers
+    pub fn get_workers_filtered(
+        &self,
+        model_id: Option<&str>,
+        worker_type: Option<WorkerType>,
+        connection_mode: Option<ConnectionMode>,
+        runtime_type: Option<RuntimeType>,
+        healthy_only: bool,
+    ) -> Vec<Arc<dyn Worker>> {
+        // Start with the most efficient collection based on filters
+        // Use model index when possible as it's O(1) lookup
+        let workers: Vec<Arc<dyn Worker>> = if let Some(model) = model_id {
+            self.get_by_model(model).to_vec()
+        } else {
+            self.get_all()
+        };
+
+        // Apply remaining filters
+        workers
+            .into_iter()
+            .filter(|w| {
+                // Check worker_type if specified
+                if let Some(ref wtype) = worker_type {
+                    if *w.worker_type() != *wtype {
+                        return false;
+                    }
+                }
+
+                // Check connection_mode if specified (using matches for flexible gRPC matching)
+                if let Some(ref conn) = connection_mode {
+                    if !w.connection_mode().matches(conn) {
+                        return false;
+                    }
+                }
+
+                // Check runtime_type if specified
+                if let Some(ref rt) = runtime_type {
+                    if w.metadata().runtime_type != *rt {
+                        return false;
+                    }
+                }
+
+                // Check health if required
+                if healthy_only && !w.is_healthy() {
+                    return false;
+                }
+
+                true
+            })
+            .collect()
+    }
+
+    /// Get worker statistics (lock-free)
+    pub fn stats(&self) -> WorkerRegistryStats {
+        let total_workers = self.workers.len();
+        // Count models directly instead of allocating Vec via get_models() (lock-free)
+        let total_models = self
+            .model_index
+            .iter()
+            .filter(|entry| !entry.value().is_empty())
+            .count();
+
+        let mut healthy_count = 0;
+        let mut total_load = 0;
+        let mut regular_count = 0;
+        let mut prefill_count = 0;
+        let mut decode_count = 0;
+        let mut http_count = 0;
+        let mut grpc_count = 0;
+        let mut cb_open_count = 0;
+        let mut cb_half_open_count = 0;
+
+        // Iterate DashMap directly to avoid cloning all workers via get_all()
+        for entry in self.workers.iter() {
+            let worker = entry.value();
+            if worker.is_healthy() {
+                healthy_count += 1;
+            }
+            total_load += worker.load();
+
+            match worker.worker_type() {
+                WorkerType::Regular => regular_count += 1,
+                WorkerType::Prefill { .. } => prefill_count += 1,
+                WorkerType::Decode => decode_count += 1,
+            }
+
+            match worker.connection_mode() {
+                ConnectionMode::Http => http_count += 1,
+                ConnectionMode::Grpc { .. } => grpc_count += 1,
+            }
+
+            match worker.circuit_breaker().state() {
+                CircuitState::Open => cb_open_count += 1,
+                CircuitState::HalfOpen => cb_half_open_count += 1,
+                CircuitState::Closed => {}
+            }
+        }
+
+        WorkerRegistryStats {
+            total_workers,
+            total_models,
+            healthy_workers: healthy_count,
+            unhealthy_workers: total_workers.saturating_sub(healthy_count),
+            total_load,
+            regular_workers: regular_count,
+            prefill_workers: prefill_count,
+            decode_workers: decode_count,
+            http_workers: http_count,
+            grpc_workers: grpc_count,
+            circuit_breaker_open: cb_open_count,
+            circuit_breaker_half_open: cb_half_open_count,
+        }
+    }
+
+    /// Get counts of regular and PD workers efficiently (O(1))
+    /// This avoids the overhead of get_all() which allocates memory and iterates all workers
+    pub fn get_worker_distribution(&self) -> (usize, usize) {
+        // Use the existing type_workers index for O(1) lookup
+        let regular_count = self
+            .type_workers
+            .get(&WorkerType::Regular)
+            .map(|v| v.len())
+            .unwrap_or(0);
+
+        // Get total workers count efficiently from DashMap
+        let total_workers = self.workers.len();
+
+        // PD workers are any workers that are not Regular
+        let pd_count = total_workers.saturating_sub(regular_count);
+
+        (regular_count, pd_count)
+    }
+
+    /// Start a health checker for all workers in the registry
+    /// This should be called once after the registry is populated with workers
+    pub(crate) fn start_health_checker(&self, check_interval_secs: u64) -> HealthChecker {
+        use std::sync::{
+            atomic::{AtomicBool, Ordering},
+            Arc,
+        };
+
+        let shutdown = Arc::new(AtomicBool::new(false));
+        let shutdown_clone = shutdown.clone();
+        let workers_ref = self.workers.clone();
+
+        let handle = tokio::spawn(async move {
+            let mut interval =
+                tokio::time::interval(tokio::time::Duration::from_secs(check_interval_secs));
+
+            loop {
+                interval.tick().await;
+
+                // Check for shutdown signal
+                if shutdown_clone.load(Ordering::Acquire) {
+                    tracing::debug!("Registry health checker shutting down");
+                    break;
+                }
+
+                // Get all workers from registry
+                let workers: Vec<Arc<dyn Worker>> = workers_ref
+                    .iter()
+                    .map(|entry| entry.value().clone())
+                    .collect();
+
+                // Perform health checks in parallel for better performance
+                // This is especially important when there are many workers
+                let health_futures: Vec<_> = workers
+                    .iter()
+                    .filter(|worker| !worker.metadata().health_config.disable_health_check)
+                    .map(|worker| {
+                        let worker = worker.clone();
+                        async move {
+                            let _ = worker.check_health_async().await;
+                        }
+                    })
+                    .collect();
+                futures::future::join_all(health_futures).await;
+            }
+        });
+
+        HealthChecker::new(handle, shutdown)
+    }
+}
+
+impl Default for WorkerRegistry {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+/// Statistics for the worker registry
+#[derive(Debug, Clone)]
+pub struct WorkerRegistryStats {
+    /// Total number of registered workers
+    pub total_workers: usize,
+    /// Number of unique models served
+    pub total_models: usize,
+    /// Number of workers passing health checks
+    pub healthy_workers: usize,
+    /// Number of workers failing health checks
+    pub unhealthy_workers: usize,
+    /// Sum of current load across all workers
+    pub total_load: usize,
+    /// Number of regular (non-PD) workers
+    pub regular_workers: usize,
+    /// Number of prefill workers (PD mode)
+    pub prefill_workers: usize,
+    /// Number of decode workers (PD mode)
+    pub decode_workers: usize,
+    /// Number of HTTP-connected workers
+    pub http_workers: usize,
+    /// Number of gRPC-connected workers
+    pub grpc_workers: usize,
+    /// Number of workers with circuit breaker in Open state (not accepting requests)
+    pub circuit_breaker_open: usize,
+    /// Number of workers with circuit breaker in HalfOpen state (testing recovery)
+    pub circuit_breaker_half_open: usize,
+}
+
+#[cfg(test)]
+mod tests {
+    use std::collections::HashMap;
+
+    use super::*;
+    use crate::core::{circuit_breaker::CircuitBreakerConfig, BasicWorkerBuilder};
+
+    #[test]
+    fn test_worker_registry() {
+        let registry = WorkerRegistry::new();
+
+        // Create a worker with labels
+        let mut labels = HashMap::new();
+        labels.insert("model_id".to_string(), "llama-3-8b".to_string());
+        labels.insert("priority".to_string(), "50".to_string());
+        labels.insert("cost".to_string(), "0.8".to_string());
+
+        let worker: Box<dyn Worker> = Box::new(
+            BasicWorkerBuilder::new("http://worker1:8080")
+                .worker_type(WorkerType::Regular)
+                .labels(labels)
+                .circuit_breaker_config(CircuitBreakerConfig::default())
+                .api_key("test_api_key")
+                .build(),
+        );
+
+        // Register worker
+        let worker_id = registry.register(Arc::from(worker));
+
+        assert!(registry.get(&worker_id).is_some());
+        assert!(registry.get_by_url("http://worker1:8080").is_some());
+        assert_eq!(registry.get_by_model("llama-3-8b").len(), 1);
+        assert_eq!(registry.get_by_type(&WorkerType::Regular).len(), 1);
+        assert_eq!(registry.get_by_connection(&ConnectionMode::Http).len(), 1);
+
+        let stats = registry.stats();
+        assert_eq!(stats.total_workers, 1);
+        assert_eq!(stats.total_models, 1);
+
+        // Remove worker
+        registry.remove(&worker_id);
+        assert!(registry.get(&worker_id).is_none());
+    }
+
+    #[test]
+    fn test_model_index_fast_lookup() {
+        let registry = WorkerRegistry::new();
+
+        // Create workers for different models
+        let mut labels1 = HashMap::new();
+        labels1.insert("model_id".to_string(), "llama-3".to_string());
+        let worker1: Box<dyn Worker> = Box::new(
+            BasicWorkerBuilder::new("http://worker1:8080")
+                .worker_type(WorkerType::Regular)
+                .labels(labels1)
+                .circuit_breaker_config(CircuitBreakerConfig::default())
+                .api_key("test_api_key")
+                .build(),
+        );
+
+        let mut labels2 = HashMap::new();
+        labels2.insert("model_id".to_string(), "llama-3".to_string());
+        let worker2: Box<dyn Worker> = Box::new(
+            BasicWorkerBuilder::new("http://worker2:8080")
+                .worker_type(WorkerType::Regular)
+                .labels(labels2)
+                .circuit_breaker_config(CircuitBreakerConfig::default())
+                .api_key("test_api_key")
+                .build(),
+        );
+
+        let mut labels3 = HashMap::new();
+        labels3.insert("model_id".to_string(), "gpt-4".to_string());
+        let worker3: Box<dyn Worker> = Box::new(
+            BasicWorkerBuilder::new("http://worker3:8080")
+                .worker_type(WorkerType::Regular)
+                .labels(labels3)
+                .circuit_breaker_config(CircuitBreakerConfig::default())
+                .api_key("test_api_key")
+                .build(),
+        );
+
+        // Register workers
+        registry.register(Arc::from(worker1));
+        registry.register(Arc::from(worker2));
+        registry.register(Arc::from(worker3));
+
+        let llama_workers = registry.get_by_model("llama-3");
+        assert_eq!(llama_workers.len(), 2);
+        let urls: Vec<String> = llama_workers.iter().map(|w| w.url().to_string()).collect();
+        assert!(urls.contains(&"http://worker1:8080".to_string()));
+        assert!(urls.contains(&"http://worker2:8080".to_string()));
+
+        let gpt_workers = registry.get_by_model("gpt-4");
+        assert_eq!(gpt_workers.len(), 1);
+        assert_eq!(gpt_workers[0].url(), "http://worker3:8080");
+
+        let unknown_workers = registry.get_by_model("unknown-model");
+        assert_eq!(unknown_workers.len(), 0);
+
+        registry.remove_by_url("http://worker1:8080");
+        let llama_workers_after = registry.get_by_model("llama-3");
+        assert_eq!(llama_workers_after.len(), 1);
+        assert_eq!(llama_workers_after[0].url(), "http://worker2:8080");
+    }
+}

third_party/sglang/sgl-model-gateway/src/core/worker_service.rs

@@ -0,0 +1,366 @@
+//! Worker Service - Business logic layer for worker operations
+//!
+//! This module provides a clean separation between HTTP concerns (in routers)
+//! and business logic for worker management. The service orchestrates
+//! WorkerRegistry and JobQueue operations.
+
+use std::sync::Arc;
+
+use axum::{
+    http::StatusCode,
+    response::{IntoResponse, Response},
+    Json,
+};
+use serde_json::json;
+use tracing::warn;
+
+use crate::{
+    config::RouterConfig,
+    core::{worker::worker_to_info, worker_registry::WorkerId, Job, JobQueue, WorkerRegistry},
+    protocols::worker_spec::{
+        WorkerConfigRequest, WorkerErrorResponse, WorkerInfo, WorkerUpdateRequest,
+    },
+};
+
+/// Error types for worker service operations
+#[derive(Debug)]
+pub enum WorkerServiceError {
+    /// Worker with given ID was not found
+    NotFound { worker_id: String },
+    /// Invalid worker ID format (expected UUID)
+    InvalidId { raw: String, message: String },
+    /// Job queue not initialized
+    QueueNotInitialized,
+    /// Failed to submit job to queue
+    QueueSubmitFailed { message: String },
+}
+
+impl WorkerServiceError {
+    pub fn error_code(&self) -> &'static str {
+        match self {
+            Self::NotFound { .. } => "WORKER_NOT_FOUND",
+            Self::InvalidId { .. } => "BAD_REQUEST",
+            Self::QueueNotInitialized => "INTERNAL_SERVER_ERROR",
+            Self::QueueSubmitFailed { .. } => "INTERNAL_SERVER_ERROR",
+        }
+    }
+
+    pub fn status_code(&self) -> StatusCode {
+        match self {
+            Self::NotFound { .. } => StatusCode::NOT_FOUND,
+            Self::InvalidId { .. } => StatusCode::BAD_REQUEST,
+            Self::QueueNotInitialized => StatusCode::INTERNAL_SERVER_ERROR,
+            Self::QueueSubmitFailed { .. } => StatusCode::INTERNAL_SERVER_ERROR,
+        }
+    }
+}
+
+impl std::fmt::Display for WorkerServiceError {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            Self::NotFound { worker_id } => write!(f, "Worker {} not found", worker_id),
+            Self::InvalidId { raw, message } => {
+                write!(
+                    f,
+                    "Invalid worker_id '{}' (expected UUID). Error: {}",
+                    raw, message
+                )
+            }
+            Self::QueueNotInitialized => write!(f, "Job queue not initialized"),
+            Self::QueueSubmitFailed { message } => write!(f, "{}", message),
+        }
+    }
+}
+
+impl std::error::Error for WorkerServiceError {}
+
+impl IntoResponse for WorkerServiceError {
+    fn into_response(self) -> Response {
+        let error = WorkerErrorResponse {
+            error: self.to_string(),
+            code: self.error_code().to_string(),
+        };
+        (self.status_code(), Json(error)).into_response()
+    }
+}
+
+/// Result of creating a worker (async job submission)
+#[derive(Debug)]
+pub struct CreateWorkerResult {
+    pub worker_id: WorkerId,
+    pub url: String,
+    pub location: String,
+}
+
+impl IntoResponse for CreateWorkerResult {
+    fn into_response(self) -> Response {
+        let response = json!({
+            "status": "accepted",
+            "worker_id": self.worker_id.as_str(),
+            "url": self.url,
+            "location": self.location,
+            "message": "Worker addition queued for background processing"
+        });
+        (
+            StatusCode::ACCEPTED,
+            [(http::header::LOCATION, self.location)],
+            Json(response),
+        )
+            .into_response()
+    }
+}
+
+/// Result of deleting a worker (async job submission)
+#[derive(Debug)]
+pub struct DeleteWorkerResult {
+    pub worker_id: WorkerId,
+    pub url: String,
+}
+
+impl IntoResponse for DeleteWorkerResult {
+    fn into_response(self) -> Response {
+        let response = json!({
+            "status": "accepted",
+            "worker_id": self.worker_id.as_str(),
+            "message": "Worker removal queued for background processing"
+        });
+        (StatusCode::ACCEPTED, Json(response)).into_response()
+    }
+}
+
+/// Result of updating a worker (async job submission)
+#[derive(Debug)]
+pub struct UpdateWorkerResult {
+    pub worker_id: WorkerId,
+    pub url: String,
+}
+
+impl IntoResponse for UpdateWorkerResult {
+    fn into_response(self) -> Response {
+        let response = json!({
+            "status": "accepted",
+            "worker_id": self.worker_id.as_str(),
+            "message": "Worker update queued for background processing"
+        });
+        (StatusCode::ACCEPTED, Json(response)).into_response()
+    }
+}
+
+/// Result of listing workers
+#[derive(Debug)]
+pub struct ListWorkersResult {
+    pub workers: Vec<WorkerInfo>,
+    pub total: usize,
+    pub prefill_count: usize,
+    pub decode_count: usize,
+    pub regular_count: usize,
+}
+
+impl IntoResponse for ListWorkersResult {
+    fn into_response(self) -> Response {
+        let response = json!({
+            "workers": self.workers,
+            "total": self.total,
+            "stats": {
+                "prefill_count": self.prefill_count,
+                "decode_count": self.decode_count,
+                "regular_count": self.regular_count,
+            }
+        });
+        Json(response).into_response()
+    }
+}
+
+/// Wrapper for WorkerInfo to implement IntoResponse
+pub struct GetWorkerResponse(pub WorkerInfo);
+
+impl IntoResponse for GetWorkerResponse {
+    fn into_response(self) -> Response {
+        Json(self.0).into_response()
+    }
+}
+
+/// Worker Service - Orchestrates worker business logic
+///
+/// This service provides a clean API for worker operations, separating
+/// business logic from HTTP concerns. Handlers in server.rs become thin
+/// wrappers that translate between HTTP and this service.
+pub struct WorkerService {
+    worker_registry: Arc<WorkerRegistry>,
+    job_queue: Arc<std::sync::OnceLock<Arc<JobQueue>>>,
+    router_config: RouterConfig,
+}
+
+impl WorkerService {
+    /// Create a new WorkerService
+    pub fn new(
+        worker_registry: Arc<WorkerRegistry>,
+        job_queue: Arc<std::sync::OnceLock<Arc<JobQueue>>>,
+        router_config: RouterConfig,
+    ) -> Self {
+        Self {
+            worker_registry,
+            job_queue,
+            router_config,
+        }
+    }
+
+    /// Parse and validate a worker ID string
+    pub fn parse_worker_id(&self, raw: &str) -> Result<WorkerId, WorkerServiceError> {
+        uuid::Uuid::parse_str(raw)
+            .map(|_| WorkerId::from_string(raw.to_string()))
+            .map_err(|e| WorkerServiceError::InvalidId {
+                raw: raw.to_string(),
+                message: e.to_string(),
+            })
+    }
+
+    /// Get the job queue, returning an error if not initialized
+    fn get_job_queue(&self) -> Result<&Arc<JobQueue>, WorkerServiceError> {
+        self.job_queue
+            .get()
+            .ok_or(WorkerServiceError::QueueNotInitialized)
+    }
+
+    pub async fn create_worker(
+        &self,
+        mut config: WorkerConfigRequest,
+    ) -> Result<CreateWorkerResult, WorkerServiceError> {
+        if self.router_config.api_key.is_some() && config.api_key.is_none() {
+            warn!(
+                "Adding worker {} without API key while router has API key configured. \
+                Worker will be accessible without authentication. \
+                If the worker requires the same API key as the router, please specify it explicitly.",
+                config.url
+            );
+        }
+
+        config.dp_aware = self.router_config.dp_aware;
+
+        let worker_url = config.url.clone();
+        let worker_id = self.worker_registry.reserve_id_for_url(&worker_url);
+
+        let job = Job::AddWorker {
+            config: Box::new(config),
+        };
+
+        self.get_job_queue()?
+            .submit(job)
+            .await
+            .map_err(|e| WorkerServiceError::QueueSubmitFailed { message: e })?;
+
+        let location = format!("/workers/{}", worker_id.as_str());
+
+        Ok(CreateWorkerResult {
+            worker_id,
+            url: worker_url,
+            location,
+        })
+    }
+
+    /// List all workers with their info
+    pub fn list_workers(&self) -> ListWorkersResult {
+        let workers = self.worker_registry.get_all_with_ids();
+        let worker_infos: Vec<WorkerInfo> = workers
+            .iter()
+            .map(|(worker_id, worker)| {
+                let mut info = worker_to_info(worker);
+                info.id = worker_id.as_str().to_string();
+                info
+            })
+            .collect();
+
+        let stats = self.worker_registry.stats();
+
+        ListWorkersResult {
+            workers: worker_infos,
+            total: stats.total_workers,
+            prefill_count: stats.prefill_workers,
+            decode_count: stats.decode_workers,
+            regular_count: stats.regular_workers,
+        }
+    }
+
+    pub fn get_worker(&self, worker_id_raw: &str) -> Result<GetWorkerResponse, WorkerServiceError> {
+        let worker_id = self.parse_worker_id(worker_id_raw)?;
+        let job_queue = self.get_job_queue()?;
+
+        if let Some(worker) = self.worker_registry.get(&worker_id) {
+            let worker_url = worker.url().to_string();
+            let mut worker_info = worker_to_info(&worker);
+            worker_info.id = worker_id.as_str().to_string();
+            if let Some(status) = job_queue.get_status(&worker_url) {
+                worker_info.job_status = Some(status);
+            }
+            return Ok(GetWorkerResponse(worker_info));
+        }
+
+        if let Some(worker_url) = self.worker_registry.get_url_by_id(&worker_id) {
+            if let Some(status) = job_queue.get_status(&worker_url) {
+                return Ok(GetWorkerResponse(WorkerInfo::pending(
+                    worker_id.as_str(),
+                    worker_url,
+                    Some(status),
+                )));
+            }
+        }
+
+        Err(WorkerServiceError::NotFound {
+            worker_id: worker_id_raw.to_string(),
+        })
+    }
+
+    /// Delete a worker by ID (submits async job)
+    pub async fn delete_worker(
+        &self,
+        worker_id_raw: &str,
+    ) -> Result<DeleteWorkerResult, WorkerServiceError> {
+        let worker_id = self.parse_worker_id(worker_id_raw)?;
+
+        let url = self
+            .worker_registry
+            .get_url_by_id(&worker_id)
+            .ok_or_else(|| WorkerServiceError::NotFound {
+                worker_id: worker_id_raw.to_string(),
+            })?;
+
+        let job = Job::RemoveWorker { url: url.clone() };
+
+        let job_queue = self.get_job_queue()?;
+        job_queue
+            .submit(job)
+            .await
+            .map_err(|e| WorkerServiceError::QueueSubmitFailed { message: e })?;
+
+        Ok(DeleteWorkerResult { worker_id, url })
+    }
+
+    /// Update a worker by ID (submits async job)
+    pub async fn update_worker(
+        &self,
+        worker_id_raw: &str,
+        update: WorkerUpdateRequest,
+    ) -> Result<UpdateWorkerResult, WorkerServiceError> {
+        let worker_id = self.parse_worker_id(worker_id_raw)?;
+
+        let url = self
+            .worker_registry
+            .get_url_by_id(&worker_id)
+            .ok_or_else(|| WorkerServiceError::NotFound {
+                worker_id: worker_id_raw.to_string(),
+            })?;
+
+        let job = Job::UpdateWorker {
+            url: url.clone(),
+            update: Box::new(update),
+        };
+
+        let job_queue = self.get_job_queue()?;
+        job_queue
+            .submit(job)
+            .await
+            .map_err(|e| WorkerServiceError::QueueSubmitFailed { message: e })?;
+
+        Ok(UpdateWorkerResult { worker_id, url })
+    }
+}