gahow/agentic-pd-hybrid
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

chore: vendor sglang v0.5.10 snapshot

Browse Source
This commit is contained in:
Gahow Wang
2026-04-24 12:29:36 +00:00
parent 78f0d15221
commit bded08301f
4308 changed files with 1200894 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

36
third_party/sglang/sgl-model-gateway/benches/consistent_hash_bench.rs vendored Normal file
View File

@@ -0,0 +1,36 @@
use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criterion};
use smg_mesh::consistent_hash::ConsistentHashRing;
fn setup_ring(node_count: usize) -> ConsistentHashRing {
let mut ring = ConsistentHashRing::new();
for i in 0..node_count {
ring.add_node(&format!("node-{}", i));
}
ring
}
fn bench_consistent_hash(c: &mut Criterion) {
let mut group = c.benchmark_group("ConsistentHashRing");
for size in [10, 100, 500].iter() {
let ring = setup_ring(*size);
let key = "test-request-key-for-rate-limiting";
let node_name = "node-5";
group.bench_with_input(BenchmarkId::new("get_owners", size), size, |b, _| {
b.iter(|| {
black_box(ring.get_owners(key));
});
});
group.bench_with_input(BenchmarkId::new("is_owner", size), size, |b, _| {
b.iter(|| {
black_box(ring.is_owner(key, node_name));
});
});
}
group.finish();
}
criterion_group!(benches, bench_consistent_hash);
criterion_main!(benches);

260
third_party/sglang/sgl-model-gateway/benches/manual_policy_benchmark.rs vendored Normal file
View File

@@ -0,0 +1,260 @@
use std::sync::Arc;
use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criterion, Throughput};
use smg::{
core::{BasicWorkerBuilder, Worker, WorkerType},
policies::{LoadBalancingPolicy, ManualPolicy, SelectWorkerInfo},
};
use tokio::runtime::Runtime;
// ============================================================================
// Test Helpers
// ============================================================================
fn create_workers(count: usize) -> Vec<Arc<dyn Worker>> {
(0..count)
.map(|i| {
Arc::new(
BasicWorkerBuilder::new(format!("http://worker-{}:8000", i))
.worker_type(WorkerType::Regular)
.build(),
) as Arc<dyn Worker>
})
.collect()
}
fn select_with_key(
rt: &Runtime,
policy: &ManualPolicy,
workers: &[Arc<dyn Worker>],
key: &str,
) -> Option<usize> {
let mut headers = http::HeaderMap::new();
headers.insert("x-smg-routing-key", key.parse().unwrap());
let info = SelectWorkerInfo {
headers: Some(&headers),
..Default::default()
};
rt.block_on(policy.select_worker(workers, &info))
}
fn warmup_keys(rt: &Runtime, policy: &ManualPolicy, workers: &[Arc<dyn Worker>], keys: &[String]) {
for key in keys {
select_with_key(rt, policy, workers, key);
}
}
fn gen_keys(count: usize, prefix: &str) -> Vec<String> {
(0..count).map(|i| format!("{}{}", prefix, i)).collect()
}
// ============================================================================
// Benchmarks
// ============================================================================
fn bench_fast_path_hit(c: &mut Criterion) {
let rt = Runtime::new().unwrap();
let mut group = c.benchmark_group("manual_policy/fast_path");
for worker_count in [4, 16, 64, 256] {
let policy = ManualPolicy::new();
let workers = create_workers(worker_count);
let keys = gen_keys(1000, "user-");
warmup_keys(&rt, &policy, &workers, &keys);
group.throughput(Throughput::Elements(1));
group.bench_with_input(
BenchmarkId::new("workers", worker_count),
&worker_count,
|b, _| {
let mut idx = 0;
b.iter(|| {
let result = select_with_key(&rt, &policy, &workers, &keys[idx % keys.len()]);
idx += 1;
black_box(result)
});
},
);
}
group.finish();
}
fn bench_slow_path_vacant(c: &mut Criterion) {
let rt = Runtime::new().unwrap();
let mut group = c.benchmark_group("manual_policy/slow_path_vacant");
for worker_count in [4, 16, 64, 256] {
let workers = create_workers(worker_count);
group.throughput(Throughput::Elements(1));
group.bench_with_input(
BenchmarkId::new("workers", worker_count),
&worker_count,
|b, _| {
let policy = ManualPolicy::new();
let mut idx = 0;
b.iter(|| {
let key = format!("new-user-{}", idx);
let result = select_with_key(&rt, &policy, &workers, &key);
idx += 1;
black_box(result)
});
},
);
}
group.finish();
}
fn bench_no_routing_key(c: &mut Criterion) {
let rt = Runtime::new().unwrap();
let mut group = c.benchmark_group("manual_policy/no_routing_key");
for worker_count in [4, 16, 64, 256] {
let policy = ManualPolicy::new();
let workers = create_workers(worker_count);
group.throughput(Throughput::Elements(1));
group.bench_with_input(
BenchmarkId::new("workers", worker_count),
&worker_count,
|b, _| {
let info = SelectWorkerInfo::default();
b.iter(|| black_box(rt.block_on(policy.select_worker(&workers, &info))));
},
);
}
group.finish();
}
fn bench_failover(c: &mut Criterion) {
let rt = Runtime::new().unwrap();
let mut group = c.benchmark_group("manual_policy/failover");
group.sample_size(50);
for worker_count in [4, 16, 64] {
group.bench_with_input(
BenchmarkId::new("workers", worker_count),
&worker_count,
|b, &count| {
b.iter_with_setup(
|| {
let policy = ManualPolicy::new();
let workers = create_workers(count);
let idx = select_with_key(&rt, &policy, &workers, "failover-test").unwrap();
workers[idx].set_healthy(false);
(policy, workers)
},
|(policy, workers)| {
black_box(select_with_key(&rt, &policy, &workers, "failover-test"))
},
);
},
);
}
group.finish();
}
fn bench_concurrent(c: &mut Criterion) {
let rt = Arc::new(
tokio::runtime::Builder::new_multi_thread()
.worker_threads(4)
.build()
.unwrap(),
);
let mut group = c.benchmark_group("manual_policy/concurrent");
group.sample_size(50);
for num_threads in [2, 4, 8, 16] {
group.bench_with_input(
BenchmarkId::new("threads", num_threads),
&num_threads,
|b, &threads| {
b.iter(|| {
let policy = Arc::new(ManualPolicy::new());
let workers: Arc<Vec<Arc<dyn Worker>>> = Arc::new(create_workers(16));
rt.block_on(async {
let handles: Vec<_> = (0..threads)
.map(|t| {
let policy = Arc::clone(&policy);
let workers = Arc::clone(&workers);
tokio::spawn(async move {
for i in 0..500 {
let key = if i % 5 == 0 {
format!("thread{}_user{}", t, i)
} else {
format!("shared_user{}", i % 50)
};
let mut headers = http::HeaderMap::new();
headers.insert("x-smg-routing-key", key.parse().unwrap());
let info = SelectWorkerInfo {
headers: Some(&headers),
..Default::default()
};
let _ =
black_box(policy.select_worker(&workers, &info).await);
}
})
})
.collect();
for h in handles {
h.await.unwrap();
}
});
});
},
);
}
group.finish();
}
fn bench_cache_size_impact(c: &mut Criterion) {
let rt = Runtime::new().unwrap();
let mut group = c.benchmark_group("manual_policy/cache_size");
for cache_size in [100, 1000, 10000, 100000] {
let policy = ManualPolicy::new();
let workers = create_workers(16);
let keys = gen_keys(cache_size, "user-");
warmup_keys(&rt, &policy, &workers, &keys);
group.throughput(Throughput::Elements(1));
group.bench_with_input(BenchmarkId::new("keys", cache_size), &cache_size, |b, _| {
let mut idx = 0;
b.iter(|| {
let result = select_with_key(&rt, &policy, &workers, &keys[idx % keys.len()]);
idx += 1;
black_box(result)
});
});
}
group.finish();
}
fn bench_comparison_baseline(c: &mut Criterion) {
use rand::Rng;
let mut group = c.benchmark_group("manual_policy/vs_baseline");
let workers = create_workers(16);
// Baseline: raw random selection without any policy overhead
group.bench_function("raw_random", |b| {
let mut rng = rand::rng();
b.iter(|| black_box(rng.random_range(0..workers.len())));
});
group.finish();
}
criterion_group!(
benches,
bench_fast_path_hit,
bench_slow_path_vacant,
bench_no_routing_key,
bench_failover,
bench_concurrent,
bench_cache_size_impact,
bench_comparison_baseline,
);
criterion_main!(benches);

670
third_party/sglang/sgl-model-gateway/benches/request_processing.rs vendored Normal file
View File

@@ -0,0 +1,670 @@
use std::time::Instant;
use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criterion, Throughput};
use serde_json::{from_str, to_string, to_value, to_vec};
use smg::{
core::{BasicWorker, BasicWorkerBuilder, Worker, WorkerType},
protocols::{
chat::{ChatCompletionRequest, ChatMessage, MessageContent},
common::StringOrArray,
completion::CompletionRequest,
generate::GenerateRequest,
sampling_params::SamplingParams,
},
routers::http::pd_types::{generate_room_id, RequestWithBootstrap},
};
fn create_test_worker() -> BasicWorker {
BasicWorkerBuilder::new("http://test-server:8000")
.worker_type(WorkerType::Prefill {
bootstrap_port: Some(5678),
})
.build()
}
// Helper function to get bootstrap info from worker
fn get_bootstrap_info(worker: &BasicWorker) -> (String, Option<u16>) {
let hostname = worker.bootstrap_host().to_string();
let bootstrap_port = worker.bootstrap_port();
(hostname, bootstrap_port)
}
/// Create a default GenerateRequest for benchmarks with minimal fields set
fn default_generate_request() -> GenerateRequest {
GenerateRequest {
text: None,
model: None,
input_ids: None,
input_embeds: None,
image_data: None,
video_data: None,
audio_data: None,
sampling_params: None,
return_logprob: None,
logprob_start_len: None,
top_logprobs_num: None,
token_ids_logprob: None,
return_text_in_logprobs: false,
stream: false,
log_metrics: true,
return_hidden_states: false,
modalities: None,
session_params: None,
lora_path: None,
lora_id: None,
custom_logit_processor: None,
bootstrap_host: None,
bootstrap_port: None,
bootstrap_room: None,
bootstrap_pair_key: None,
data_parallel_rank: None,
background: false,
conversation_id: None,
priority: None,
extra_key: None,
no_logs: false,
custom_labels: None,
return_bytes: false,
return_entropy: false,
rid: None,
}
}
/// Create a default ChatCompletionRequest for benchmarks with minimal fields set
#[allow(deprecated)]
fn default_chat_completion_request() -> ChatCompletionRequest {
ChatCompletionRequest {
// Required fields in OpenAI order
messages: vec![],
model: String::new(),
// Use default for all other fields
..Default::default()
}
}
/// Create a default CompletionRequest for benchmarks with minimal fields set
fn default_completion_request() -> CompletionRequest {
CompletionRequest {
model: String::new(),
prompt: StringOrArray::String(String::new()),
suffix: None,
max_tokens: None,
temperature: None,
top_p: None,
n: None,
stream: false,
stream_options: None,
logprobs: None,
echo: false,
stop: None,
presence_penalty: None,
frequency_penalty: None,
best_of: None,
logit_bias: None,
user: None,
seed: None,
// SGLang Extensions
top_k: None,
min_p: None,
min_tokens: None,
repetition_penalty: None,
regex: None,
ebnf: None,
json_schema: None,
stop_token_ids: None,
no_stop_trim: false,
ignore_eos: false,
skip_special_tokens: true,
// SGLang Extensions
lora_path: None,
session_params: None,
return_hidden_states: false,
sampling_seed: None,
other: serde_json::Map::new(),
}
}
// Sample request data for benchmarks
fn create_sample_generate_request() -> GenerateRequest {
GenerateRequest {
text: Some("Write a story about artificial intelligence".to_string()),
sampling_params: Some(SamplingParams {
max_new_tokens: Some(100),
temperature: Some(0.8),
top_p: Some(0.9),
top_k: Some(50),
frequency_penalty: Some(0.0),
presence_penalty: Some(0.0),
repetition_penalty: Some(1.0),
..Default::default()
}),
..default_generate_request()
}
}
#[allow(deprecated)]
fn create_sample_chat_completion_request() -> ChatCompletionRequest {
ChatCompletionRequest {
model: "gpt-3.5-turbo".to_string(),
messages: vec![
ChatMessage::System {
content: MessageContent::Text("You are a helpful assistant".to_string()),
name: None,
},
ChatMessage::User {
content: MessageContent::Text(
"Explain quantum computing in simple terms".to_string(),
),
name: None,
},
],
max_tokens: Some(150),
max_completion_tokens: Some(150),
temperature: Some(0.7),
top_p: Some(1.0),
n: Some(1),
presence_penalty: Some(0.0),
frequency_penalty: Some(0.0),
parallel_tool_calls: Some(true),
..default_chat_completion_request()
}
}
fn create_sample_completion_request() -> CompletionRequest {
CompletionRequest {
model: "text-davinci-003".to_string(),
prompt: StringOrArray::String("Complete this sentence: The future of AI is".to_string()),
max_tokens: Some(50),
temperature: Some(0.8),
top_p: Some(1.0),
n: Some(1),
presence_penalty: Some(0.0),
frequency_penalty: Some(0.0),
best_of: Some(1),
..default_completion_request()
}
}
#[allow(deprecated)]
fn create_large_chat_completion_request() -> ChatCompletionRequest {
let mut messages = vec![ChatMessage::System {
content: MessageContent::Text(
"You are a helpful assistant with extensive knowledge.".to_string(),
),
name: None,
}];
// Add many user/assistant pairs to simulate a long conversation
for i in 0..50 {
messages.push(ChatMessage::User {
content: MessageContent::Text(format!("Question {}: What do you think about topic number {} which involves complex reasoning about multiple interconnected systems and their relationships?", i, i)),
name: None,
});
messages.push(ChatMessage::Assistant {
content: Some(MessageContent::Text(format!("Answer {}: This is a detailed response about topic {} that covers multiple aspects and provides comprehensive analysis of the interconnected systems you mentioned.", i, i))),
name: None,
tool_calls: None,
reasoning_content: None,
});
}
ChatCompletionRequest {
model: "gpt-4".to_string(),
messages,
max_tokens: Some(1000),
max_completion_tokens: Some(1000),
temperature: Some(0.7),
top_p: Some(0.95),
n: Some(1),
presence_penalty: Some(0.1),
frequency_penalty: Some(0.1),
top_logprobs: Some(5),
seed: Some(42),
parallel_tool_calls: Some(true),
..default_chat_completion_request()
}
}
// Benchmark JSON serialization
fn bench_json_serialization(c: &mut Criterion) {
let mut group = c.benchmark_group("json_serialization");
let generate_req = create_sample_generate_request();
let chat_req = create_sample_chat_completion_request();
let completion_req = create_sample_completion_request();
let large_chat_req = create_large_chat_completion_request();
group.bench_function("generate_request", |b| {
b.iter(|| {
let json = to_string(black_box(&generate_req)).unwrap();
black_box(json);
});
});
group.bench_function("chat_completion_request", |b| {
b.iter(|| {
let json = to_string(black_box(&chat_req)).unwrap();
black_box(json);
});
});
group.bench_function("completion_request", |b| {
b.iter(|| {
let json = to_string(black_box(&completion_req)).unwrap();
black_box(json);
});
});
group.bench_function("large_chat_completion_request", |b| {
b.iter(|| {
let json = to_string(black_box(&large_chat_req)).unwrap();
black_box(json);
});
});
group.bench_function("generate_request_to_bytes", |b| {
b.iter(|| {
let bytes = to_vec(black_box(&generate_req)).unwrap();
black_box(bytes);
});
});
group.finish();
}
// Benchmark JSON deserialization
fn bench_json_deserialization(c: &mut Criterion) {
let mut group = c.benchmark_group("json_deserialization");
let generate_json = to_string(&create_sample_generate_request()).unwrap();
let chat_json = to_string(&create_sample_chat_completion_request()).unwrap();
let completion_json = to_string(&create_sample_completion_request()).unwrap();
let large_chat_json = to_string(&create_large_chat_completion_request()).unwrap();
group.bench_function("generate_request", |b| {
b.iter(|| {
let req: GenerateRequest = from_str(black_box(&generate_json)).unwrap();
black_box(req);
});
});
group.bench_function("chat_completion_request", |b| {
b.iter(|| {
let req: ChatCompletionRequest = from_str(black_box(&chat_json)).unwrap();
black_box(req);
});
});
group.bench_function("completion_request", |b| {
b.iter(|| {
let req: CompletionRequest = from_str(black_box(&completion_json)).unwrap();
black_box(req);
});
});
group.bench_function("large_chat_completion_request", |b| {
b.iter(|| {
let req: ChatCompletionRequest = from_str(black_box(&large_chat_json)).unwrap();
black_box(req);
});
});
group.finish();
}
// Benchmark bootstrap injection (replaces request adaptation)
fn bench_bootstrap_injection(c: &mut Criterion) {
let mut group = c.benchmark_group("bootstrap_injection");
let generate_req = create_sample_generate_request();
let chat_req = create_sample_chat_completion_request();
let completion_req = create_sample_completion_request();
let large_chat_req = create_large_chat_completion_request();
let worker = create_test_worker();
let (hostname, bootstrap_port) = get_bootstrap_info(&worker);
group.bench_function("generate_bootstrap_injection", |b| {
b.iter(|| {
let request_with_bootstrap = RequestWithBootstrap {
original: &generate_req,
bootstrap_host: hostname.clone(),
bootstrap_port,
bootstrap_room: generate_room_id(),
};
let json = to_value(black_box(&request_with_bootstrap)).unwrap();
black_box(json);
});
});
group.bench_function("chat_completion_bootstrap_injection", |b| {
b.iter(|| {
let request_with_bootstrap = RequestWithBootstrap {
original: &chat_req,
bootstrap_host: hostname.clone(),
bootstrap_port,
bootstrap_room: generate_room_id(),
};
let json = to_value(black_box(&request_with_bootstrap)).unwrap();
black_box(json);
});
});
group.bench_function("completion_bootstrap_injection", |b| {
b.iter(|| {
let request_with_bootstrap = RequestWithBootstrap {
original: &completion_req,
bootstrap_host: hostname.clone(),
bootstrap_port,
bootstrap_room: generate_room_id(),
};
let json = to_value(black_box(&request_with_bootstrap)).unwrap();
black_box(json);
});
});
group.bench_function("large_chat_completion_bootstrap_injection", |b| {
b.iter(|| {
let request_with_bootstrap = RequestWithBootstrap {
original: &large_chat_req,
bootstrap_host: hostname.clone(),
bootstrap_port,
bootstrap_room: generate_room_id(),
};
let json = to_value(black_box(&request_with_bootstrap)).unwrap();
black_box(json);
});
});
group.finish();
}
// Benchmark direct JSON routing (replaces regular routing)
fn bench_direct_json_routing(c: &mut Criterion) {
let mut group = c.benchmark_group("direct_json_routing");
let generate_req = create_sample_generate_request();
let chat_req = create_sample_chat_completion_request();
let completion_req = create_sample_completion_request();
group.bench_function("generate_to_json", |b| {
b.iter(|| {
let json = to_value(black_box(&generate_req)).unwrap();
black_box(json);
});
});
group.bench_function("generate_to_json_string", |b| {
b.iter(|| {
let json = to_string(black_box(&generate_req)).unwrap();
black_box(json);
});
});
group.bench_function("generate_to_bytes", |b| {
b.iter(|| {
let bytes = to_vec(black_box(&generate_req)).unwrap();
black_box(bytes);
});
});
group.bench_function("chat_completion_to_json", |b| {
b.iter(|| {
let json = to_value(black_box(&chat_req)).unwrap();
black_box(json);
});
});
group.bench_function("chat_completion_to_json_string", |b| {
b.iter(|| {
let json = to_string(black_box(&chat_req)).unwrap();
black_box(json);
});
});
group.bench_function("completion_to_json", |b| {
b.iter(|| {
let json = to_value(black_box(&completion_req)).unwrap();
black_box(json);
});
});
group.finish();
}
// Benchmark throughput with different request sizes
fn bench_throughput_by_size(c: &mut Criterion) {
let mut group = c.benchmark_group("throughput_by_size");
// Create requests of different sizes
let small_generate = GenerateRequest {
text: Some("Hi".to_string()),
..default_generate_request()
};
let medium_generate = GenerateRequest {
text: Some("Write a medium length story about AI".repeat(10)),
..default_generate_request()
};
let large_generate = GenerateRequest {
text: Some("Write a very long and detailed story about artificial intelligence and its impact on society".repeat(100)),
..default_generate_request()
};
let worker = create_test_worker();
let (hostname, bootstrap_port) = get_bootstrap_info(&worker);
for (name, req) in [
("small", &small_generate),
("medium", &medium_generate),
("large", &large_generate),
] {
let json = to_string(req).unwrap();
let size_bytes = json.len();
let hostname_clone = hostname.clone();
group.throughput(Throughput::Bytes(size_bytes as u64));
group.bench_with_input(BenchmarkId::new("serialize", name), &req, |b, req| {
b.iter(|| {
let json = to_string(black_box(req)).unwrap();
black_box(json);
});
});
group.bench_with_input(
BenchmarkId::new("deserialize", name),
&json,
|b, json_str| {
b.iter(|| {
let req: GenerateRequest = black_box(from_str(json_str)).unwrap();
black_box(req);
});
},
);
group.bench_with_input(
BenchmarkId::new("bootstrap_inject", name),
&req,
move |b, req| {
let hostname = hostname_clone.clone();
b.iter(|| {
let request_with_bootstrap = RequestWithBootstrap {
original: req,
bootstrap_host: hostname.clone(),
bootstrap_port,
bootstrap_room: generate_room_id(),
};
let json = to_value(&request_with_bootstrap).unwrap();
black_box(json);
});
},
);
}
group.finish();
}
// Benchmark full round-trip: deserialize -> inject bootstrap -> serialize
fn bench_full_round_trip(c: &mut Criterion) {
let mut group = c.benchmark_group("full_round_trip");
let generate_json = to_string(&create_sample_generate_request()).unwrap();
let chat_json = to_string(&create_sample_chat_completion_request()).unwrap();
let completion_json = to_string(&create_sample_completion_request()).unwrap();
let worker = create_test_worker();
let (hostname, bootstrap_port) = get_bootstrap_info(&worker);
group.bench_function("generate_openai_to_pd_pipeline", |b| {
b.iter(|| {
// Deserialize OpenAI request
let req: GenerateRequest = from_str(black_box(&generate_json)).unwrap();
// Create wrapper with bootstrap fields
let request_with_bootstrap = RequestWithBootstrap {
original: &req,
bootstrap_host: hostname.clone(),
bootstrap_port,
bootstrap_room: generate_room_id(),
};
// Serialize final request
let pd_json = to_string(&request_with_bootstrap).unwrap();
black_box(pd_json);
});
});
group.bench_function("chat_completion_openai_to_pd_pipeline", |b| {
b.iter(|| {
let req: ChatCompletionRequest = from_str(black_box(&chat_json)).unwrap();
let request_with_bootstrap = RequestWithBootstrap {
original: &req,
bootstrap_host: hostname.clone(),
bootstrap_port,
bootstrap_room: generate_room_id(),
};
let pd_json = to_string(&request_with_bootstrap).unwrap();
black_box(pd_json);
});
});
group.bench_function("completion_openai_to_pd_pipeline", |b| {
b.iter(|| {
let req: CompletionRequest = from_str(black_box(&completion_json)).unwrap();
let request_with_bootstrap = RequestWithBootstrap {
original: &req,
bootstrap_host: hostname.clone(),
bootstrap_port,
bootstrap_room: generate_room_id(),
};
let pd_json = to_string(&request_with_bootstrap).unwrap();
black_box(pd_json);
});
});
group.bench_function("generate_direct_json_pipeline", |b| {
b.iter(|| {
// Deserialize OpenAI request
let req: GenerateRequest = from_str(black_box(&generate_json)).unwrap();
// Convert to JSON for direct routing (no bootstrap injection)
let routing_json = to_value(&req).unwrap();
let json_string = to_string(&routing_json).unwrap();
black_box(json_string);
});
});
group.finish();
}
fn benchmark_summary(c: &mut Criterion) {
let group = c.benchmark_group("benchmark_summary");
println!("\nSGLang Model Gateway Performance Benchmark Suite");
println!("=================================================");
// Quick performance overview
let generate_req = create_sample_generate_request();
let worker = create_test_worker();
println!("\nQuick Performance Overview:");
// Measure serialization
let start = Instant::now();
for _ in 0..1000 {
let _ = black_box(to_string(&generate_req).unwrap());
}
let serialize_time = start.elapsed().as_nanos() / 1000;
println!(" * Serialization (avg): {:>8} ns/req", serialize_time);
// Measure deserialization
let json = to_string(&generate_req).unwrap();
let start = Instant::now();
for _ in 0..1000 {
let _: GenerateRequest = black_box(from_str(&json).unwrap());
}
let deserialize_time = start.elapsed().as_nanos() / 1000;
println!(
" * Deserialization (avg): {:>8} ns/req",
deserialize_time
);
// Measure bootstrap injection (replaces adaptation)
let (hostname, bootstrap_port) = get_bootstrap_info(&worker);
let start = Instant::now();
for _ in 0..1000 {
let request_with_bootstrap = RequestWithBootstrap {
original: &generate_req,
bootstrap_host: hostname.clone(),
bootstrap_port,
bootstrap_room: generate_room_id(),
};
let _ = black_box(to_value(&request_with_bootstrap).unwrap());
}
let inject_time = start.elapsed().as_nanos() / 1000;
println!(" * Bootstrap Injection (avg): {:>6} ns/req", inject_time);
// Calculate ratios
let total_pipeline = serialize_time + deserialize_time + inject_time;
println!(" * Total Pipeline (avg): {:>8} ns/req", total_pipeline);
println!("\nPerformance Insights:");
if deserialize_time > serialize_time * 2 {
println!(" • Deserialization is significantly faster than serialization");
}
if inject_time < serialize_time / 10 {
println!(
" • Bootstrap injection overhead is negligible ({:.1}% of serialization)",
(inject_time as f64 / serialize_time as f64) * 100.0
);
}
if total_pipeline < 100_000 {
println!(" • Total pipeline latency is excellent (< 100μs)");
}
println!("\nSimplification Benefits:");
println!(" • Eliminated complex type conversion layer");
println!(" • Reduced memory allocations");
println!(" • Automatic field preservation (no manual mapping)");
println!(" • Direct JSON manipulation improves performance");
println!("\nRecommendations:");
if serialize_time > deserialize_time {
println!(" • Focus optimization efforts on serialization rather than deserialization");
}
println!(" • PD mode overhead is minimal - safe to use for latency-sensitive workloads");
println!(" • Consider batching small requests to improve overall throughput");
println!("\n{}", "=".repeat(50));
group.finish();
}
criterion_group!(
benches,
benchmark_summary,
bench_json_serialization,
bench_json_deserialization,
bench_bootstrap_injection,
bench_direct_json_routing,
bench_throughput_by_size,
bench_full_round_trip
);
criterion_main!(benches);

59
third_party/sglang/sgl-model-gateway/benches/router_registry_bench.rs vendored Normal file
View File

@@ -0,0 +1,59 @@
use std::{collections::HashMap, sync::Arc};
use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criterion};
use smg::core::{BasicWorkerBuilder, CircuitBreakerConfig, WorkerRegistry, WorkerType};
// Helper to populate registry
fn setup_registry(count: usize) -> Arc<WorkerRegistry> {
let registry = Arc::new(WorkerRegistry::new());
for i in 0..count {
let mut labels = HashMap::new();
labels.insert("model_id".to_string(), "benchmark-model".to_string());
let worker_type = if i % 2 == 0 {
WorkerType::Regular
} else {
WorkerType::Decode
};
let worker = BasicWorkerBuilder::new(format!("http://worker-{}:8000", i))
.worker_type(worker_type)
.labels(labels)
.circuit_breaker_config(CircuitBreakerConfig::default())
.build();
registry.register(Arc::from(worker));
}
registry
}
fn bench_optimizations(c: &mut Criterion) {
let mut group = c.benchmark_group("Registry Optimizations");
// We test with 5000 workers to simulate high load
let size = 5000;
let registry = setup_registry(size);
// The OLD method (Slow: Allocates vector + Clones ARCs)
group.bench_function(BenchmarkId::new("Old: get_all()", size), |b| {
b.iter(|| {
black_box(registry.get_all());
});
});
// The NEW method (Fast: O(1) Lookup, Zero Allocation)
group.bench_function(
BenchmarkId::new("New: get_worker_distribution()", size),
|b| {
b.iter(|| {
black_box(registry.get_worker_distribution());
});
},
);
group.finish();
}
criterion_group!(benches, bench_optimizations);
criterion_main!(benches);

1099
third_party/sglang/sgl-model-gateway/benches/tree_benchmark.rs vendored Normal file
View File

File diff suppressed because it is too large Load Diff

110
third_party/sglang/sgl-model-gateway/benches/wasm_middleware_latency.rs vendored Normal file
View File

@@ -0,0 +1,110 @@
use std::sync::Arc;
use axum::{
body::Body,
http::{HeaderMap, Request, Response, StatusCode},
middleware,
response::IntoResponse,
};
use criterion::{criterion_group, criterion_main, Criterion};
use http_body_util::BodyExt;
use smg::{
app_context::AppContext, config::RouterConfig, middleware::wasm_middleware,
protocols::chat::ChatCompletionRequest, routers::RouterTrait, server::AppState,
};
use tokio::runtime::Runtime;
use tower::{Layer, Service};
#[derive(Debug)]
struct MockRouter;
#[async_trait::async_trait]
impl RouterTrait for MockRouter {
fn as_any(&self) -> &dyn std::any::Any {
self
}
async fn route_chat(
&self,
_headers: Option<&HeaderMap>,
_body: &ChatCompletionRequest,
_model_id: Option<&str>,
) -> Response<Body> {
StatusCode::OK.into_response()
}
fn router_type(&self) -> &'static str {
"mock"
}
}
/// Mock service that simulates a streaming response with a 500ms delay.
async fn mock_next_streaming(_req: Request<Body>) -> Response<Body> {
let (tx, rx) = tokio::sync::mpsc::channel(16);
tokio::spawn(async move {
// Send first chunk immediately
let _ = tx
.send(Ok::<_, std::io::Error>(bytes::Bytes::from("chunk 1 ")))
.await;
// Simulate generation delay
tokio::time::sleep(tokio::time::Duration::from_millis(500)).await;
// Send final chunk
let _ = tx
.send(Ok::<_, std::io::Error>(bytes::Bytes::from("chunk 2")))
.await;
});
Response::new(Body::from_stream(
tokio_stream::wrappers::ReceiverStream::new(rx),
))
}
fn bench_wasm_middleware_buffering(c: &mut Criterion) {
let rt = Runtime::new().unwrap();
// Setup AppContext with WASM enabled
let config = RouterConfig::builder().enable_wasm(true).build_unchecked();
let context = rt.block_on(AppContext::from_config(config, 30)).unwrap();
let app_state = Arc::new(AppState {
router: Arc::new(MockRouter),
context: Arc::new(context),
concurrency_queue_tx: None,
router_manager: None,
mesh_handler: None,
mesh_sync_manager: None,
});
c.bench_function("wasm_middleware_pre_fix_latency", |b| {
b.iter(|| {
rt.block_on(async {
let req = Request::builder()
.uri("/v1/chat/completions")
.body(Body::empty())
.unwrap();
// Create the service by applying the middleware layer to the mock streamer
let mut service =
middleware::from_fn_with_state(app_state.clone(), wasm_middleware).layer(
tower::service_fn(|req: Request<Body>| async move {
Ok::<_, std::convert::Infallible>(mock_next_streaming(req).await)
}),
);
// Explicitly poll the service
let response: Response<Body> =
service.call(req).await.expect("Middleware service failed");
// Measure how long it takes to receive the FIRST frame
let mut body = response.into_body();
let _first_frame = body.frame().await;
});
});
});
}
criterion_group! {
name = benches;
config = Criterion::default().sample_size(10);
targets = bench_wasm_middleware_buffering
}
criterion_main!(benches);