chore: vendor sglang v0.5.10 snapshot

third_party/sglang/sgl-model-gateway/tests/reliability/circuit_breaker_test.rs

@@ -0,0 +1,243 @@
+//! Circuit breaker integration tests
+//!
+//! Tests for circuit breaker behavior: opening, half-open state, recovery, and isolation.
+
+use axum::{
+    body::Body,
+    extract::Request,
+    http::{header::CONTENT_TYPE, StatusCode},
+};
+use serde_json::json;
+use smg::config::{CircuitBreakerConfig, RetryConfig, RouterConfig};
+use tower::ServiceExt;
+
+use crate::common::{AppTestContext, TestRouterConfig, TestWorkerConfig};
+
+#[cfg(test)]
+mod circuit_breaker_tests {
+    use super::*;
+
+    /// Test that circuit breaker opens after consecutive failures
+    #[tokio::test]
+    async fn test_circuit_breaker_opens_after_failures() {
+        let config = TestRouterConfig::round_robin_with_circuit_breaker(
+            3200,
+            CircuitBreakerConfig {
+                failure_threshold: 3,
+                success_threshold: 2,
+                timeout_duration_secs: 2,
+                window_duration_secs: 10,
+            },
+        );
+
+        let ctx = AppTestContext::new_with_config(
+            config,
+            vec![TestWorkerConfig::flaky(19100, 1.0)], // Always fail
+        )
+        .await;
+
+        let app = ctx.create_app().await;
+
+        // Make requests until we see failures (500) and then circuit breaker opens (503)
+        let mut saw_500 = false;
+        let mut saw_503 = false;
+
+        for _ in 0..10 {
+            let payload = json!({
+                "text": "Test circuit breaker",
+                "stream": false
+            });
+
+            let req = Request::builder()
+                .method("POST")
+                .uri("/generate")
+                .header(CONTENT_TYPE, "application/json")
+                .body(Body::from(serde_json::to_string(&payload).unwrap()))
+                .unwrap();
+
+            let resp = app.clone().oneshot(req).await.unwrap();
+            match resp.status() {
+                StatusCode::INTERNAL_SERVER_ERROR => saw_500 = true,
+                StatusCode::SERVICE_UNAVAILABLE => saw_503 = true,
+                _ => {}
+            }
+        }
+
+        // Should see 500 (worker error) and eventually 503 (circuit breaker open)
+        assert!(
+            saw_500 || saw_503,
+            "Should see either 500 (worker error) or 503 (circuit breaker open)"
+        );
+
+        ctx.shutdown().await;
+    }
+
+    /// Test circuit breaker with disabled flag
+    #[tokio::test]
+    async fn test_circuit_breaker_disabled() {
+        let config = RouterConfig::builder()
+            .regular_mode(vec![])
+            .round_robin_policy()
+            .host("127.0.0.1")
+            .port(3201)
+            .max_payload_size(256 * 1024 * 1024)
+            .request_timeout_secs(600)
+            .worker_startup_timeout_secs(5)
+            .worker_startup_check_interval_secs(1)
+            .max_concurrent_requests(64)
+            .queue_timeout_secs(60)
+            .disable_circuit_breaker()
+            .disable_retries()
+            .build_unchecked();
+
+        let ctx = AppTestContext::new_with_config(
+            config,
+            vec![TestWorkerConfig::flaky(19101, 1.0)], // Always fail
+        )
+        .await;
+
+        let app = ctx.create_app().await;
+
+        // With circuit breaker disabled, should always see 500 (never 503)
+        for _ in 0..5 {
+            let payload = json!({
+                "text": "Test disabled CB",
+                "stream": false
+            });
+
+            let req = Request::builder()
+                .method("POST")
+                .uri("/generate")
+                .header(CONTENT_TYPE, "application/json")
+                .body(Body::from(serde_json::to_string(&payload).unwrap()))
+                .unwrap();
+
+            let resp = app.clone().oneshot(req).await.unwrap();
+            // With CB disabled, we expect 500 errors (not 503 from CB)
+            assert_eq!(
+                resp.status(),
+                StatusCode::INTERNAL_SERVER_ERROR,
+                "Should get 500 with CB disabled, not CB-related 503"
+            );
+        }
+        ctx.shutdown().await;
+    }
+
+    /// Test circuit breaker per-worker isolation
+    #[tokio::test]
+    async fn test_circuit_breaker_per_worker_isolation() {
+        let config = TestRouterConfig::round_robin_with_circuit_breaker(
+            3202,
+            CircuitBreakerConfig {
+                failure_threshold: 2,
+                success_threshold: 1,
+                timeout_duration_secs: 2,
+                window_duration_secs: 10,
+            },
+        );
+
+        let ctx = AppTestContext::new_with_config(
+            config,
+            vec![
+                TestWorkerConfig::flaky(19102, 1.0), // Always fail
+                TestWorkerConfig::healthy(19103),    // Always succeed
+            ],
+        )
+        .await;
+
+        let app = ctx.create_app().await;
+
+        // Send enough requests to trigger CB on the failing worker
+        // The healthy worker should continue to serve requests
+        let mut success_count = 0;
+        for _ in 0..20 {
+            let payload = json!({
+                "text": "Test isolation",
+                "stream": false
+            });
+
+            let req = Request::builder()
+                .method("POST")
+                .uri("/generate")
+                .header(CONTENT_TYPE, "application/json")
+                .body(Body::from(serde_json::to_string(&payload).unwrap()))
+                .unwrap();
+
+            let resp = app.clone().oneshot(req).await.unwrap();
+            if resp.status() == StatusCode::OK {
+                success_count += 1;
+            }
+        }
+
+        // After CB opens on failing worker, healthy worker should handle all requests
+        // We should see at least some successes
+        assert!(
+            success_count > 0,
+            "Should have some successful requests from healthy worker"
+        );
+
+        ctx.shutdown().await;
+    }
+
+    /// Test circuit breaker with retries enabled
+    #[tokio::test]
+    async fn test_circuit_breaker_with_retries() {
+        let config = RouterConfig::builder()
+            .regular_mode(vec![])
+            .round_robin_policy()
+            .host("127.0.0.1")
+            .port(3203)
+            .max_payload_size(256 * 1024 * 1024)
+            .request_timeout_secs(600)
+            .worker_startup_timeout_secs(5)
+            .worker_startup_check_interval_secs(1)
+            .max_concurrent_requests(64)
+            .queue_timeout_secs(60)
+            .retry_config(RetryConfig {
+                max_retries: 3,
+                initial_backoff_ms: 10,
+                max_backoff_ms: 50,
+                ..Default::default()
+            })
+            .circuit_breaker_config(CircuitBreakerConfig {
+                failure_threshold: 2,
+                success_threshold: 1,
+                timeout_duration_secs: 2,
+                window_duration_secs: 10,
+            })
+            .build_unchecked();
+
+        let ctx = AppTestContext::new_with_config(
+            config,
+            vec![
+                TestWorkerConfig::flaky(19104, 1.0), // Always fail
+                TestWorkerConfig::healthy(19105),    // Always succeed
+            ],
+        )
+        .await;
+
+        let app = ctx.create_app().await;
+
+        // With retries, requests should succeed by retrying on healthy worker
+        let payload = json!({
+            "text": "Test with retries",
+            "stream": false
+        });
+
+        let req = Request::builder()
+            .method("POST")
+            .uri("/generate")
+            .header(CONTENT_TYPE, "application/json")
+            .body(Body::from(serde_json::to_string(&payload).unwrap()))
+            .unwrap();
+
+        let resp = app.oneshot(req).await.unwrap();
+        assert_eq!(
+            resp.status(),
+            StatusCode::OK,
+            "Request should succeed via retry on healthy worker"
+        );
+
+        ctx.shutdown().await;
+    }
+}

third_party/sglang/sgl-model-gateway/tests/reliability/fault_tolerance_test.rs

@@ -0,0 +1,309 @@
+//! Fault tolerance integration tests
+//!
+//! Tests for system resilience: worker failures, network issues, and recovery.
+
+use std::sync::{
+    atomic::{AtomicUsize, Ordering},
+    Arc,
+};
+
+use axum::{
+    body::Body,
+    extract::Request,
+    http::{header::CONTENT_TYPE, StatusCode},
+};
+use serde_json::json;
+use smg::config::{CircuitBreakerConfig, RetryConfig};
+use tower::ServiceExt;
+
+use crate::common::{AppTestContext, TestRouterConfig, TestWorkerConfig};
+
+#[cfg(test)]
+mod fault_tolerance_tests {
+    use super::*;
+
+    /// Test that requests are rerouted when a worker fails
+    #[tokio::test]
+    async fn test_worker_failure_reroute() {
+        let config = TestRouterConfig::round_robin_with_reliability(
+            4100,
+            RetryConfig {
+                max_retries: 3,
+                initial_backoff_ms: 10,
+                max_backoff_ms: 100,
+                ..Default::default()
+            },
+            CircuitBreakerConfig {
+                failure_threshold: 2,
+                success_threshold: 1,
+                timeout_duration_secs: 2,
+                window_duration_secs: 10,
+            },
+        );
+
+        let ctx = AppTestContext::new_with_config(
+            config,
+            vec![
+                TestWorkerConfig::flaky(20100, 1.0), // Always fails
+                TestWorkerConfig::healthy(20101),    // Always succeeds
+            ],
+        )
+        .await;
+
+        let app = ctx.create_app().await;
+
+        // Requests should succeed via retry to healthy worker
+        for i in 0..10 {
+            let payload = json!({
+                "text": format!("Fault tolerance test {}", i),
+                "stream": false
+            });
+
+            let req = Request::builder()
+                .method("POST")
+                .uri("/generate")
+                .header(CONTENT_TYPE, "application/json")
+                .body(Body::from(serde_json::to_string(&payload).unwrap()))
+                .unwrap();
+
+            let resp = app.clone().oneshot(req).await.unwrap();
+            assert_eq!(
+                resp.status(),
+                StatusCode::OK,
+                "Request should succeed via reroute to healthy worker"
+            );
+        }
+
+        ctx.shutdown().await;
+    }
+
+    /// Test behavior when all workers are temporarily unavailable
+    #[tokio::test]
+    async fn test_all_workers_temporarily_failing() {
+        let config = TestRouterConfig::round_robin_with_retry(
+            4101,
+            RetryConfig {
+                max_retries: 2,
+                initial_backoff_ms: 10,
+                max_backoff_ms: 50,
+                ..Default::default()
+            },
+        );
+
+        let ctx = AppTestContext::new_with_config(
+            config,
+            vec![
+                TestWorkerConfig::flaky(20102, 1.0), // Always fails
+                TestWorkerConfig::flaky(20103, 1.0), // Always fails
+            ],
+        )
+        .await;
+
+        let app = ctx.create_app().await;
+
+        let payload = json!({
+            "text": "Test with all failing workers",
+            "stream": false
+        });
+
+        let req = Request::builder()
+            .method("POST")
+            .uri("/generate")
+            .header(CONTENT_TYPE, "application/json")
+            .body(Body::from(serde_json::to_string(&payload).unwrap()))
+            .unwrap();
+
+        let resp = app.oneshot(req).await.unwrap();
+        // Should fail when all workers are failing
+        assert!(
+            resp.status() == StatusCode::INTERNAL_SERVER_ERROR
+                || resp.status() == StatusCode::SERVICE_UNAVAILABLE,
+            "Request should fail when all workers are failing, got {}",
+            resp.status()
+        );
+
+        ctx.shutdown().await;
+    }
+
+    /// Test graceful handling of slow workers
+    #[tokio::test]
+    async fn test_slow_worker_handling() {
+        let config = TestRouterConfig::round_robin(4102);
+
+        let ctx = AppTestContext::new_with_config(
+            config,
+            vec![
+                TestWorkerConfig::slow(20104, 500), // Slow worker
+                TestWorkerConfig::healthy(20105),   // Fast worker
+            ],
+        )
+        .await;
+
+        let app = ctx.create_app().await;
+
+        // Send concurrent requests
+        let mut handles = Vec::new();
+        let success_count = Arc::new(AtomicUsize::new(0));
+
+        for i in 0..10 {
+            let app_clone = app.clone();
+            let success_clone = Arc::clone(&success_count);
+
+            let handle = tokio::spawn(async move {
+                let payload = json!({
+                    "text": format!("Slow worker test {}", i),
+                    "stream": false
+                });
+
+                let req = Request::builder()
+                    .method("POST")
+                    .uri("/generate")
+                    .header(CONTENT_TYPE, "application/json")
+                    .body(Body::from(serde_json::to_string(&payload).unwrap()))
+                    .unwrap();
+
+                let resp = app_clone.oneshot(req).await.unwrap();
+                if resp.status() == StatusCode::OK {
+                    success_clone.fetch_add(1, Ordering::SeqCst);
+                }
+            });
+
+            handles.push(handle);
+        }
+
+        for handle in handles {
+            handle.await.unwrap();
+        }
+
+        // All requests should eventually succeed
+        assert_eq!(
+            success_count.load(Ordering::SeqCst),
+            10,
+            "All requests should succeed despite slow worker"
+        );
+
+        ctx.shutdown().await;
+    }
+
+    /// Test circuit breaker prevents cascading failures
+    #[tokio::test]
+    async fn test_circuit_breaker_prevents_cascade() {
+        let config = TestRouterConfig::round_robin_with_reliability(
+            4103,
+            RetryConfig {
+                max_retries: 3,
+                initial_backoff_ms: 10,
+                max_backoff_ms: 50,
+                ..Default::default()
+            },
+            CircuitBreakerConfig {
+                failure_threshold: 2,
+                success_threshold: 1,
+                timeout_duration_secs: 5,
+                window_duration_secs: 10,
+            },
+        );
+
+        let ctx = AppTestContext::new_with_config(
+            config,
+            vec![
+                TestWorkerConfig::flaky(20106, 1.0), // Failing worker
+                TestWorkerConfig::healthy(20107),    // Healthy worker
+            ],
+        )
+        .await;
+
+        let app = ctx.create_app().await;
+        let mut success_count = 0;
+
+        // Send many requests - after CB opens, all should route to healthy worker
+        for i in 0..20 {
+            let payload = json!({
+                "text": format!("Circuit breaker cascade test {}", i),
+                "stream": false
+            });
+
+            let req = Request::builder()
+                .method("POST")
+                .uri("/generate")
+                .header(CONTENT_TYPE, "application/json")
+                .body(Body::from(serde_json::to_string(&payload).unwrap()))
+                .unwrap();
+
+            let resp = app.clone().oneshot(req).await.unwrap();
+            if resp.status() == StatusCode::OK {
+                success_count += 1;
+            }
+        }
+
+        // Most requests should succeed after CB opens on failing worker
+        assert!(
+            success_count >= 15,
+            "Most requests should succeed after circuit breaker opens, got {} successes",
+            success_count
+        );
+
+        ctx.shutdown().await;
+    }
+
+    /// Test recovery after worker comes back online (simulated via healthy worker)
+    #[tokio::test]
+    async fn test_system_stability_under_partial_failure() {
+        let config = TestRouterConfig::round_robin_with_reliability(
+            4104,
+            RetryConfig {
+                max_retries: 2,
+                initial_backoff_ms: 10,
+                max_backoff_ms: 50,
+                ..Default::default()
+            },
+            CircuitBreakerConfig {
+                failure_threshold: 3,
+                success_threshold: 1,
+                timeout_duration_secs: 2,
+                window_duration_secs: 10,
+            },
+        );
+
+        let ctx = AppTestContext::new_with_config(
+            config,
+            vec![
+                TestWorkerConfig::flaky(20108, 0.5), // 50% failure rate
+                TestWorkerConfig::healthy(20109),    // Always succeeds
+            ],
+        )
+        .await;
+
+        let app = ctx.create_app().await;
+        let mut success_count = 0;
+
+        // System should maintain stability with partial failures
+        for i in 0..30 {
+            let payload = json!({
+                "text": format!("Stability test {}", i),
+                "stream": false
+            });
+
+            let req = Request::builder()
+                .method("POST")
+                .uri("/generate")
+                .header(CONTENT_TYPE, "application/json")
+                .body(Body::from(serde_json::to_string(&payload).unwrap()))
+                .unwrap();
+
+            let resp = app.clone().oneshot(req).await.unwrap();
+            if resp.status() == StatusCode::OK {
+                success_count += 1;
+            }
+        }
+
+        // With retries and one healthy worker, most should succeed
+        assert!(
+            success_count >= 25,
+            "System should maintain stability under partial failure, got {} successes",
+            success_count
+        );
+
+        ctx.shutdown().await;
+    }
+}

third_party/sglang/sgl-model-gateway/tests/reliability/mod.rs

@@ -0,0 +1,6 @@
+//! Reliability integration tests
+
+pub mod circuit_breaker_test;
+pub mod fault_tolerance_test;
+pub mod rate_limiting_test;
+pub mod retries_test;

third_party/sglang/sgl-model-gateway/tests/reliability/rate_limiting_test.rs

@@ -0,0 +1,263 @@
+//! Rate limiting integration tests
+//!
+//! Tests for rate limiting and concurrency control.
+
+use std::sync::{
+    atomic::{AtomicUsize, Ordering},
+    Arc,
+};
+
+use axum::{
+    body::Body,
+    extract::Request,
+    http::{header::CONTENT_TYPE, StatusCode},
+};
+use serde_json::json;
+use smg::config::RouterConfig;
+use tower::ServiceExt;
+
+use crate::common::{AppTestContext, TestRouterConfig, TestWorkerConfig};
+
+#[cfg(test)]
+mod rate_limiting_tests {
+    use super::*;
+
+    /// Test that concurrent requests are handled within limits
+    #[tokio::test]
+    async fn test_concurrent_requests_within_limit() {
+        let config = TestRouterConfig::with_concurrency(3400, 10);
+
+        let ctx =
+            AppTestContext::new_with_config(config, vec![TestWorkerConfig::slow(19300, 50)]).await;
+
+        let app = ctx.create_app().await;
+
+        // Send 5 concurrent requests (within limit of 10)
+        let mut handles = Vec::new();
+        let success_count = Arc::new(AtomicUsize::new(0));
+
+        for i in 0..5 {
+            let app_clone = app.clone();
+            let success_clone = Arc::clone(&success_count);
+
+            let handle = tokio::spawn(async move {
+                let payload = json!({
+                    "text": format!("Concurrent request {}", i),
+                    "stream": false
+                });
+
+                let req = Request::builder()
+                    .method("POST")
+                    .uri("/generate")
+                    .header(CONTENT_TYPE, "application/json")
+                    .body(Body::from(serde_json::to_string(&payload).unwrap()))
+                    .unwrap();
+
+                let resp = app_clone.oneshot(req).await.unwrap();
+                if resp.status() == StatusCode::OK {
+                    success_clone.fetch_add(1, Ordering::SeqCst);
+                }
+            });
+
+            handles.push(handle);
+        }
+
+        // Wait for all requests to complete
+        for handle in handles {
+            handle.await.unwrap();
+        }
+
+        // All requests should succeed within the limit
+        assert_eq!(
+            success_count.load(Ordering::SeqCst),
+            5,
+            "All concurrent requests should succeed within limit"
+        );
+
+        ctx.shutdown().await;
+    }
+
+    /// Test rate limit tokens per second
+    #[tokio::test]
+    async fn test_rate_limit_tokens() {
+        let config = RouterConfig::builder()
+            .regular_mode(vec![])
+            .random_policy()
+            .host("127.0.0.1")
+            .port(3401)
+            .max_payload_size(256 * 1024 * 1024)
+            .request_timeout_secs(600)
+            .worker_startup_timeout_secs(5)
+            .worker_startup_check_interval_secs(1)
+            .max_concurrent_requests(100)
+            .rate_limit_tokens_per_second(50) // 50 tokens/sec
+            .queue_timeout_secs(60)
+            .build_unchecked();
+
+        let ctx =
+            AppTestContext::new_with_config(config, vec![TestWorkerConfig::healthy(19301)]).await;
+
+        let app = ctx.create_app().await;
+
+        // Send requests and verify they succeed
+        let mut success_count = 0;
+        for i in 0..10 {
+            let payload = json!({
+                "text": format!("Rate limited request {}", i),
+                "stream": false
+            });
+
+            let req = Request::builder()
+                .method("POST")
+                .uri("/generate")
+                .header(CONTENT_TYPE, "application/json")
+                .body(Body::from(serde_json::to_string(&payload).unwrap()))
+                .unwrap();
+
+            let resp = app.clone().oneshot(req).await.unwrap();
+            if resp.status() == StatusCode::OK {
+                success_count += 1;
+            }
+        }
+
+        // Most requests should succeed (some might be rate limited)
+        assert!(
+            success_count >= 5,
+            "At least half of requests should succeed, got {}",
+            success_count
+        );
+
+        ctx.shutdown().await;
+    }
+
+    /// Test unlimited concurrent requests when set to 0
+    #[tokio::test]
+    async fn test_unlimited_concurrent_requests() {
+        let config = TestRouterConfig::with_concurrency(3402, 0); // Unlimited
+
+        let ctx =
+            AppTestContext::new_with_config(config, vec![TestWorkerConfig::slow(19302, 10)]).await;
+
+        let app = ctx.create_app().await;
+
+        // Send many concurrent requests
+        let mut handles = Vec::new();
+        let success_count = Arc::new(AtomicUsize::new(0));
+
+        for i in 0..20 {
+            let app_clone = app.clone();
+            let success_clone = Arc::clone(&success_count);
+
+            let handle = tokio::spawn(async move {
+                let payload = json!({
+                    "text": format!("Unlimited request {}", i),
+                    "stream": false
+                });
+
+                let req = Request::builder()
+                    .method("POST")
+                    .uri("/generate")
+                    .header(CONTENT_TYPE, "application/json")
+                    .body(Body::from(serde_json::to_string(&payload).unwrap()))
+                    .unwrap();
+
+                let resp = app_clone.oneshot(req).await.unwrap();
+                if resp.status() == StatusCode::OK {
+                    success_clone.fetch_add(1, Ordering::SeqCst);
+                }
+            });
+
+            handles.push(handle);
+        }
+
+        for handle in handles {
+            handle.await.unwrap();
+        }
+
+        // All requests should succeed with unlimited concurrency
+        assert_eq!(
+            success_count.load(Ordering::SeqCst),
+            20,
+            "All requests should succeed with unlimited concurrency"
+        );
+
+        ctx.shutdown().await;
+    }
+
+    /// Test queue behavior when requests exceed capacity
+    #[tokio::test]
+    async fn test_queue_behavior() {
+        let config = RouterConfig::builder()
+            .regular_mode(vec![])
+            .random_policy()
+            .host("127.0.0.1")
+            .port(3403)
+            .max_payload_size(256 * 1024 * 1024)
+            .request_timeout_secs(600)
+            .worker_startup_timeout_secs(5)
+            .worker_startup_check_interval_secs(1)
+            .max_concurrent_requests(2) // Very low limit
+            .queue_size(5) // Small queue
+            .queue_timeout_secs(1) // Short timeout
+            .build_unchecked();
+
+        let ctx = AppTestContext::new_with_config(
+            config,
+            vec![TestWorkerConfig::slow(19303, 200)], // Longer delay to test queuing
+        )
+        .await;
+
+        let app = ctx.create_app().await;
+
+        // Send requests that will exceed capacity
+        let mut handles = Vec::new();
+        let success_count = Arc::new(AtomicUsize::new(0));
+        let error_count = Arc::new(AtomicUsize::new(0));
+
+        for i in 0..10 {
+            let app_clone = app.clone();
+            let success_clone = Arc::clone(&success_count);
+            let error_clone = Arc::clone(&error_count);
+
+            let handle = tokio::spawn(async move {
+                let payload = json!({
+                    "text": format!("Queue test {}", i),
+                    "stream": false
+                });
+
+                let req = Request::builder()
+                    .method("POST")
+                    .uri("/generate")
+                    .header(CONTENT_TYPE, "application/json")
+                    .body(Body::from(serde_json::to_string(&payload).unwrap()))
+                    .unwrap();
+
+                let resp = app_clone.oneshot(req).await.unwrap();
+                if resp.status() == StatusCode::OK {
+                    success_clone.fetch_add(1, Ordering::SeqCst);
+                } else {
+                    error_clone.fetch_add(1, Ordering::SeqCst);
+                }
+            });
+
+            handles.push(handle);
+        }
+
+        for handle in handles {
+            handle.await.unwrap();
+        }
+
+        // Some requests should succeed, some might timeout or be rejected
+        let successes = success_count.load(Ordering::SeqCst);
+        let errors = error_count.load(Ordering::SeqCst);
+
+        assert!(
+            successes > 0,
+            "At least some requests should succeed, got {} successes and {} errors",
+            successes,
+            errors
+        );
+
+        ctx.shutdown().await;
+    }
+}

third_party/sglang/sgl-model-gateway/tests/reliability/retries_test.rs

@@ -0,0 +1,214 @@
+//! Retry mechanism integration tests
+//!
+//! Tests for retry behavior: exponential backoff, max retries, and retry-on-failure scenarios.
+
+use axum::{
+    body::Body,
+    extract::Request,
+    http::{header::CONTENT_TYPE, StatusCode},
+};
+use serde_json::json;
+use smg::config::{RetryConfig, RouterConfig};
+use tower::ServiceExt;
+
+use crate::common::{AppTestContext, TestRouterConfig, TestWorkerConfig};
+
+#[cfg(test)]
+mod retry_tests {
+    use super::*;
+
+    /// Test that retries succeed when at least one worker is healthy
+    #[tokio::test]
+    async fn test_retry_succeeds_with_healthy_fallback() {
+        let retry_config = RetryConfig {
+            max_retries: 3,
+            initial_backoff_ms: 10,
+            max_backoff_ms: 100,
+            ..Default::default()
+        };
+        let config = TestRouterConfig::round_robin_with_retry(3300, retry_config);
+
+        let ctx = AppTestContext::new_with_config(
+            config,
+            vec![
+                TestWorkerConfig::flaky(19200, 1.0), // First worker always fails
+                TestWorkerConfig::healthy(19201),    // Second worker always succeeds
+            ],
+        )
+        .await;
+
+        let app = ctx.create_app().await;
+
+        // Request should succeed via retry to healthy worker
+        let payload = json!({
+            "text": "Test retry to healthy worker",
+            "stream": false
+        });
+
+        let req = Request::builder()
+            .method("POST")
+            .uri("/generate")
+            .header(CONTENT_TYPE, "application/json")
+            .body(Body::from(serde_json::to_string(&payload).unwrap()))
+            .unwrap();
+
+        let resp = app.oneshot(req).await.unwrap();
+        assert_eq!(
+            resp.status(),
+            StatusCode::OK,
+            "Request should succeed via retry"
+        );
+
+        ctx.shutdown().await;
+    }
+
+    /// Test that retries are disabled when configured
+    #[tokio::test]
+    async fn test_retries_disabled() {
+        let config = RouterConfig::builder()
+            .regular_mode(vec![])
+            .round_robin_policy()
+            .host("127.0.0.1")
+            .port(3301)
+            .max_payload_size(256 * 1024 * 1024)
+            .request_timeout_secs(600)
+            .worker_startup_timeout_secs(5)
+            .worker_startup_check_interval_secs(1)
+            .max_concurrent_requests(64)
+            .queue_timeout_secs(60)
+            .disable_retries()
+            .build_unchecked();
+
+        let ctx = AppTestContext::new_with_config(
+            config,
+            vec![TestWorkerConfig::flaky(19202, 1.0)], // Always fail
+        )
+        .await;
+
+        let app = ctx.create_app().await;
+
+        // With retries disabled, request should fail immediately
+        let payload = json!({
+            "text": "Test no retries",
+            "stream": false
+        });
+
+        let req = Request::builder()
+            .method("POST")
+            .uri("/generate")
+            .header(CONTENT_TYPE, "application/json")
+            .body(Body::from(serde_json::to_string(&payload).unwrap()))
+            .unwrap();
+
+        let resp = app.oneshot(req).await.unwrap();
+        assert_eq!(
+            resp.status(),
+            StatusCode::INTERNAL_SERVER_ERROR,
+            "Request should fail without retries"
+        );
+
+        ctx.shutdown().await;
+    }
+
+    /// Test max retries limit
+    #[tokio::test]
+    async fn test_max_retries_limit() {
+        let retry_config = RetryConfig {
+            max_retries: 2,
+            initial_backoff_ms: 10,
+            max_backoff_ms: 50,
+            ..Default::default()
+        };
+        let config = TestRouterConfig::round_robin_with_retry(3302, retry_config);
+
+        let ctx = AppTestContext::new_with_config(
+            config,
+            vec![TestWorkerConfig::flaky(19203, 1.0)], // Always fail
+        )
+        .await;
+
+        let app = ctx.create_app().await;
+
+        // All retries will fail, should return error after exhausting retries
+        let payload = json!({
+            "text": "Test max retries",
+            "stream": false
+        });
+
+        let start = std::time::Instant::now();
+        let req = Request::builder()
+            .method("POST")
+            .uri("/generate")
+            .header(CONTENT_TYPE, "application/json")
+            .body(Body::from(serde_json::to_string(&payload).unwrap()))
+            .unwrap();
+
+        let resp = app.oneshot(req).await.unwrap();
+        let elapsed = start.elapsed();
+
+        // Should eventually fail after retries
+        assert!(
+            resp.status() == StatusCode::INTERNAL_SERVER_ERROR
+                || resp.status() == StatusCode::SERVICE_UNAVAILABLE,
+            "Should fail after exhausting retries, got {}",
+            resp.status()
+        );
+
+        // Should take some time due to backoff (at least initial_backoff_ms)
+        // With 2 retries and 10ms initial backoff, should take at least 10ms
+        // But don't make this too strict as timing can vary
+        assert!(
+            elapsed.as_millis() >= 5,
+            "Should have some backoff delay, got {}ms",
+            elapsed.as_millis()
+        );
+
+        ctx.shutdown().await;
+    }
+
+    /// Test retry with multiple workers - should eventually find healthy one
+    #[tokio::test]
+    async fn test_retry_finds_healthy_worker() {
+        let retry_config = RetryConfig {
+            max_retries: 5,
+            initial_backoff_ms: 5,
+            max_backoff_ms: 50,
+            ..Default::default()
+        };
+        let config = TestRouterConfig::round_robin_with_retry(3303, retry_config);
+
+        let ctx = AppTestContext::new_with_config(
+            config,
+            vec![
+                TestWorkerConfig::flaky(19204, 1.0), // Fail
+                TestWorkerConfig::flaky(19205, 1.0), // Fail
+                TestWorkerConfig::healthy(19206),    // Succeed
+            ],
+        )
+        .await;
+
+        let app = ctx.create_app().await;
+
+        // With round robin and retries, should eventually hit the healthy worker
+        let payload = json!({
+            "text": "Test find healthy worker",
+            "stream": false
+        });
+
+        let req = Request::builder()
+            .method("POST")
+            .uri("/generate")
+            .header(CONTENT_TYPE, "application/json")
+            .body(Body::from(serde_json::to_string(&payload).unwrap()))
+            .unwrap();
+
+        let resp = app.oneshot(req).await.unwrap();
+        assert_eq!(
+            resp.status(),
+            StatusCode::OK,
+            "Should succeed by retrying until finding healthy worker"
+        );
+
+        ctx.shutdown().await;
+    }
+}