gahow/agentic-pd-hybrid
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

chore: vendor sglang v0.5.10 snapshot

Browse Source
This commit is contained in:
Gahow Wang
2026-04-24 12:29:36 +00:00
parent 78f0d15221
commit bded08301f
4308 changed files with 1200894 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

222
third_party/sglang/sgl-model-gateway/tests/routing/pd_routing_test.rs vendored Normal file
View File

@@ -0,0 +1,222 @@
//! Prefill/Decode (PD) routing integration tests
//!
//! Tests for prefill-decode disaggregation routing mode.
use axum::{
body::Body,
extract::Request,
http::{header::CONTENT_TYPE, StatusCode},
};
use serde_json::json;
use smg::config::RouterConfig;
use tower::ServiceExt;
use crate::common::{
mock_worker::{HealthStatus, MockWorkerConfig, WorkerType},
AppTestContext, TestWorkerConfig,
};
#[cfg(test)]
mod pd_routing_tests {
use super::*;
/// Test basic PD mode routing with prefill and decode workers
#[tokio::test]
async fn test_pd_mode_basic_routing() {
let config = RouterConfig::builder()
.prefill_decode_mode(
vec![
("http://127.0.0.1:19800".to_string(), None),
("http://127.0.0.1:19801".to_string(), None),
],
vec![
"http://127.0.0.1:19802".to_string(),
"http://127.0.0.1:19803".to_string(),
],
)
.power_of_two_policy(1)
.host("127.0.0.1")
.port(3800)
.max_payload_size(256 * 1024 * 1024)
.request_timeout_secs(600)
.worker_startup_timeout_secs(5)
.worker_startup_check_interval_secs(1)
.max_concurrent_requests(64)
.queue_timeout_secs(60)
.build_unchecked();
// Note: For PD mode tests, we need to start prefill and decode workers separately
// The test context will need to handle this specially
let ctx = AppTestContext::new_with_config(
config,
vec![
// Prefill workers
TestWorkerConfig::prefill(19800),
TestWorkerConfig::prefill(19801),
// Decode workers
TestWorkerConfig::decode(19802),
TestWorkerConfig::decode(19803),
],
)
.await;
let app = ctx.create_app().await;
// Send requests and verify they succeed
for i in 0..10 {
let payload = json!({
"text": format!("PD mode request {}", i),
"stream": false
});
let req = Request::builder()
.method("POST")
.uri("/generate")
.header(CONTENT_TYPE, "application/json")
.body(Body::from(serde_json::to_string(&payload).unwrap()))
.unwrap();
let resp = app.clone().oneshot(req).await.unwrap();
assert_eq!(
resp.status(),
StatusCode::OK,
"PD mode request should succeed"
);
}
ctx.shutdown().await;
}
/// Test PD mode with round robin policy
#[tokio::test]
async fn test_pd_mode_round_robin() {
let config = RouterConfig::builder()
.prefill_decode_mode(
vec![("http://127.0.0.1:19810".to_string(), None)],
vec![
"http://127.0.0.1:19811".to_string(),
"http://127.0.0.1:19812".to_string(),
],
)
.round_robin_policy()
.host("127.0.0.1")
.port(3801)
.max_payload_size(256 * 1024 * 1024)
.request_timeout_secs(600)
.worker_startup_timeout_secs(5)
.worker_startup_check_interval_secs(1)
.max_concurrent_requests(64)
.queue_timeout_secs(60)
.build_unchecked();
let ctx = AppTestContext::new_with_config(
config,
vec![
TestWorkerConfig::prefill(19810),
TestWorkerConfig::decode(19811),
TestWorkerConfig::decode(19812),
],
)
.await;
let app = ctx.create_app().await;
let mut success_count = 0;
for i in 0..20 {
let payload = json!({
"text": format!("PD round robin {}", i),
"stream": false
});
let req = Request::builder()
.method("POST")
.uri("/generate")
.header(CONTENT_TYPE, "application/json")
.body(Body::from(serde_json::to_string(&payload).unwrap()))
.unwrap();
let resp = app.clone().oneshot(req).await.unwrap();
if resp.status() == StatusCode::OK {
success_count += 1;
}
}
assert_eq!(
success_count, 20,
"All requests should succeed in PD mode with round robin"
);
ctx.shutdown().await;
}
/// Test PD mode handles worker failures gracefully
#[tokio::test]
async fn test_pd_mode_with_failing_decode_worker() {
use smg::config::RetryConfig;
let config = RouterConfig::builder()
.prefill_decode_mode(
vec![("http://127.0.0.1:19820".to_string(), None)],
vec![
"http://127.0.0.1:19821".to_string(),
"http://127.0.0.1:19822".to_string(),
],
)
.round_robin_policy()
.host("127.0.0.1")
.port(3802)
.max_payload_size(256 * 1024 * 1024)
.request_timeout_secs(600)
.worker_startup_timeout_secs(5)
.worker_startup_check_interval_secs(1)
.max_concurrent_requests(64)
.queue_timeout_secs(60)
.retry_config(RetryConfig {
max_retries: 3,
initial_backoff_ms: 10,
max_backoff_ms: 50,
..Default::default()
})
.build_unchecked();
let ctx = AppTestContext::new_with_config(
config,
vec![
TestWorkerConfig::prefill(19820),
MockWorkerConfig {
port: 19821,
worker_type: WorkerType::Decode,
health_status: HealthStatus::Healthy,
response_delay_ms: 0,
fail_rate: 1.0, // Failing decode worker
},
TestWorkerConfig::decode(19822), // Healthy decode worker
],
)
.await;
let app = ctx.create_app().await;
// Request should succeed via retry to healthy decode worker
let payload = json!({
"text": "Test with failing decode worker",
"stream": false
});
let req = Request::builder()
.method("POST")
.uri("/generate")
.header(CONTENT_TYPE, "application/json")
.body(Body::from(serde_json::to_string(&payload).unwrap()))
.unwrap();
let resp = app.oneshot(req).await.unwrap();
assert_eq!(
resp.status(),
StatusCode::OK,
"Request should succeed via retry to healthy decode worker"
);
ctx.shutdown().await;
}
}