//! Prefill/Decode (PD) routing integration tests //! //! Tests for prefill-decode disaggregation routing mode. use axum::{ body::Body, extract::Request, http::{header::CONTENT_TYPE, StatusCode}, }; use serde_json::json; use smg::config::RouterConfig; use tower::ServiceExt; use crate::common::{ mock_worker::{HealthStatus, MockWorkerConfig, WorkerType}, AppTestContext, TestWorkerConfig, }; #[cfg(test)] mod pd_routing_tests { use super::*; /// Test basic PD mode routing with prefill and decode workers #[tokio::test] async fn test_pd_mode_basic_routing() { let config = RouterConfig::builder() .prefill_decode_mode( vec![ ("http://127.0.0.1:19800".to_string(), None), ("http://127.0.0.1:19801".to_string(), None), ], vec![ "http://127.0.0.1:19802".to_string(), "http://127.0.0.1:19803".to_string(), ], ) .power_of_two_policy(1) .host("127.0.0.1") .port(3800) .max_payload_size(256 * 1024 * 1024) .request_timeout_secs(600) .worker_startup_timeout_secs(5) .worker_startup_check_interval_secs(1) .max_concurrent_requests(64) .queue_timeout_secs(60) .build_unchecked(); // Note: For PD mode tests, we need to start prefill and decode workers separately // The test context will need to handle this specially let ctx = AppTestContext::new_with_config( config, vec![ // Prefill workers TestWorkerConfig::prefill(19800), TestWorkerConfig::prefill(19801), // Decode workers TestWorkerConfig::decode(19802), TestWorkerConfig::decode(19803), ], ) .await; let app = ctx.create_app().await; // Send requests and verify they succeed for i in 0..10 { let payload = json!({ "text": format!("PD mode request {}", i), "stream": false }); let req = Request::builder() .method("POST") .uri("/generate") .header(CONTENT_TYPE, "application/json") .body(Body::from(serde_json::to_string(&payload).unwrap())) .unwrap(); let resp = app.clone().oneshot(req).await.unwrap(); assert_eq!( resp.status(), StatusCode::OK, "PD mode request should succeed" ); } ctx.shutdown().await; } /// Test PD mode with round robin policy #[tokio::test] async fn test_pd_mode_round_robin() { let config = RouterConfig::builder() .prefill_decode_mode( vec![("http://127.0.0.1:19810".to_string(), None)], vec![ "http://127.0.0.1:19811".to_string(), "http://127.0.0.1:19812".to_string(), ], ) .round_robin_policy() .host("127.0.0.1") .port(3801) .max_payload_size(256 * 1024 * 1024) .request_timeout_secs(600) .worker_startup_timeout_secs(5) .worker_startup_check_interval_secs(1) .max_concurrent_requests(64) .queue_timeout_secs(60) .build_unchecked(); let ctx = AppTestContext::new_with_config( config, vec![ TestWorkerConfig::prefill(19810), TestWorkerConfig::decode(19811), TestWorkerConfig::decode(19812), ], ) .await; let app = ctx.create_app().await; let mut success_count = 0; for i in 0..20 { let payload = json!({ "text": format!("PD round robin {}", i), "stream": false }); let req = Request::builder() .method("POST") .uri("/generate") .header(CONTENT_TYPE, "application/json") .body(Body::from(serde_json::to_string(&payload).unwrap())) .unwrap(); let resp = app.clone().oneshot(req).await.unwrap(); if resp.status() == StatusCode::OK { success_count += 1; } } assert_eq!( success_count, 20, "All requests should succeed in PD mode with round robin" ); ctx.shutdown().await; } /// Test PD mode handles worker failures gracefully #[tokio::test] async fn test_pd_mode_with_failing_decode_worker() { use smg::config::RetryConfig; let config = RouterConfig::builder() .prefill_decode_mode( vec![("http://127.0.0.1:19820".to_string(), None)], vec![ "http://127.0.0.1:19821".to_string(), "http://127.0.0.1:19822".to_string(), ], ) .round_robin_policy() .host("127.0.0.1") .port(3802) .max_payload_size(256 * 1024 * 1024) .request_timeout_secs(600) .worker_startup_timeout_secs(5) .worker_startup_check_interval_secs(1) .max_concurrent_requests(64) .queue_timeout_secs(60) .retry_config(RetryConfig { max_retries: 3, initial_backoff_ms: 10, max_backoff_ms: 50, ..Default::default() }) .build_unchecked(); let ctx = AppTestContext::new_with_config( config, vec![ TestWorkerConfig::prefill(19820), MockWorkerConfig { port: 19821, worker_type: WorkerType::Decode, health_status: HealthStatus::Healthy, response_delay_ms: 0, fail_rate: 1.0, // Failing decode worker }, TestWorkerConfig::decode(19822), // Healthy decode worker ], ) .await; let app = ctx.create_app().await; // Request should succeed via retry to healthy decode worker let payload = json!({ "text": "Test with failing decode worker", "stream": false }); let req = Request::builder() .method("POST") .uri("/generate") .header(CONTENT_TYPE, "application/json") .body(Body::from(serde_json::to_string(&payload).unwrap())) .unwrap(); let resp = app.oneshot(req).await.unwrap(); assert_eq!( resp.status(), StatusCode::OK, "Request should succeed via retry to healthy decode worker" ); ctx.shutdown().await; } }