gahow/agentic-pd-hybrid
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

chore: vendor sglang v0.5.10 snapshot

Browse Source
This commit is contained in:
Gahow Wang
2026-04-24 12:29:36 +00:00
parent 78f0d15221
commit bded08301f
4308 changed files with 1200894 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

465
third_party/sglang/sgl-model-gateway/bindings/golang/src/postprocessor.rs vendored Normal file
View File

@@ -0,0 +1,465 @@
//! Postprocessing FFI functions for gRPC stream chunks
//!
//! This module provides C-compatible functions for postprocessing gRPC stream chunks:
//! - Parse tool calls from model output
//! - Convert proto format to OpenAI format
//! - Handle reasoning content parsing
//!
//! These functions are designed to be called for each stream chunk, but can be optimized
//! with batching in the future.
use std::ffi::{CStr, CString};
use std::os::raw::{c_char, c_int};
use std::ptr;
use std::sync::Arc;
use serde_json::Value;
use smg_grpc_client::sglang_proto as proto;
use super::error::{SglErrorCode, set_error_message};
use super::grpc_converter::GrpcResponseConverterHandle;
use tokio::runtime::Runtime;
use once_cell::sync::Lazy;
/// Global tokio runtime for async operations
static RUNTIME: Lazy<Runtime> = Lazy::new(|| {
Runtime::new().expect("Failed to create tokio runtime for postprocessor FFI")
});
/// Postprocess a gRPC stream chunk to OpenAI format
///
/// This function:
/// 1. Parses the proto chunk from JSON
/// 2. Converts it to OpenAI format using the converter handle
/// 3. Returns the OpenAI format JSON
///
/// # Arguments
/// * `converter_handle` - Converter handle (created with sgl_grpc_response_converter_create)
/// * `proto_chunk_json` - JSON string of proto.GenerateResponse
/// * `openai_json_out` - Pointer to receive OpenAI format JSON (must be freed with sgl_free_string)
/// * `is_done_out` - Pointer to receive is_done flag (1 if stream is complete, 0 otherwise)
/// * `error_out` - Optional pointer to receive error message
///
/// # Returns
/// * SglErrorCode::Success on success, error code on failure
#[no_mangle]
pub unsafe extern "C" fn sgl_postprocess_stream_chunk(
converter_handle: *mut GrpcResponseConverterHandle,
proto_chunk_json: *const c_char,
openai_json_out: *mut *mut c_char,
is_done_out: *mut c_int,
error_out: *mut *mut c_char,
) -> SglErrorCode {
if converter_handle.is_null()
|| proto_chunk_json.is_null()
|| openai_json_out.is_null()
|| is_done_out.is_null()
{
set_error_message(error_out, "Invalid arguments: null pointer");
return SglErrorCode::InvalidArgument;
}
let proto_chunk_str = match CStr::from_ptr(proto_chunk_json).to_str() {
Ok(s) => s,
Err(_) => {
set_error_message(error_out, "Invalid UTF-8 in proto_chunk_json");
return SglErrorCode::InvalidArgument;
}
};
// Parse proto.GenerateResponse from JSON
let json_value: Value = match serde_json::from_str(proto_chunk_str) {
Ok(v) => v,
Err(e) => {
set_error_message(error_out, &format!("Failed to parse proto chunk JSON: {}", e));
return SglErrorCode::ParsingError;
}
};
// Build proto::GenerateResponse from JSON value
let mut proto_response = proto::GenerateResponse {
request_id: json_value
.get("request_id")
.and_then(|v| v.as_str())
.unwrap_or("")
.to_string(),
response: None,
};
// Parse the response oneof field
let is_done = if let Some(chunk_json) = json_value.get("chunk") {
let chunk = proto::GenerateStreamChunk {
token_ids: chunk_json
.get("token_ids")
.and_then(|v| v.as_array())
.map(|arr| {
arr.iter()
.filter_map(|v| v.as_u64().map(|n| n as u32))
.collect()
})
.unwrap_or_default(),
prompt_tokens: chunk_json
.get("prompt_tokens")
.and_then(|v| v.as_i64())
.map(|n| n as i32)
.unwrap_or(0),
completion_tokens: chunk_json
.get("completion_tokens")
.and_then(|v| v.as_i64())
.map(|n| n as i32)
.unwrap_or(0),
cached_tokens: chunk_json
.get("cached_tokens")
.and_then(|v| v.as_i64())
.map(|n| n as i32)
.unwrap_or(0),
output_logprobs: None,
hidden_states: vec![],
input_logprobs: None,
index: 0,
};
proto_response.response = Some(proto::generate_response::Response::Chunk(chunk));
false
} else if let Some(complete_json) = json_value.get("complete") {
let complete = proto::GenerateComplete {
output_ids: complete_json
.get("output_ids")
.and_then(|v| v.as_array())
.map(|arr| {
arr.iter()
.filter_map(|v| v.as_u64().map(|n| n as u32))
.collect()
})
.unwrap_or_default(),
finish_reason: complete_json
.get("finish_reason")
.and_then(|v| v.as_str())
.unwrap_or("")
.to_string(),
prompt_tokens: complete_json
.get("prompt_tokens")
.and_then(|v| v.as_i64())
.map(|n| n as i32)
.unwrap_or(0),
completion_tokens: complete_json
.get("completion_tokens")
.and_then(|v| v.as_i64())
.map(|n| n as i32)
.unwrap_or(0),
cached_tokens: complete_json
.get("cached_tokens")
.and_then(|v| v.as_i64())
.map(|n| n as i32)
.unwrap_or(0),
output_logprobs: None,
all_hidden_states: vec![],
input_logprobs: None,
matched_stop: None,
index: 0,
};
proto_response.response = Some(proto::generate_response::Response::Complete(complete));
true
} else if let Some(error_json) = json_value.get("error") {
let error = proto::GenerateError {
message: error_json
.get("message")
.and_then(|v| v.as_str())
.unwrap_or("")
.to_string(),
http_status_code: error_json
.get("http_status_code")
.and_then(|v| v.as_str())
.unwrap_or("500")
.to_string(),
details: error_json
.get("details")
.and_then(|v| v.as_str())
.unwrap_or("")
.to_string(),
};
proto_response.response = Some(proto::generate_response::Response::Error(error));
true
} else {
set_error_message(
error_out,
"Proto chunk JSON must contain 'chunk', 'complete', or 'error' field",
);
return SglErrorCode::ParsingError;
};
// Convert proto chunk to OpenAI format using the converter's convert_chunk function
// We'll use the existing converter API instead of calling the internal function directly
let proto_chunk_json_cstr = match CString::new(proto_chunk_str) {
Ok(s) => s,
Err(e) => {
set_error_message(error_out, &format!("Failed to create C string: {}", e));
return SglErrorCode::MemoryError;
}
};
// Use the existing converter API
let mut openai_json_ptr: *mut c_char = ptr::null_mut();
let result = super::grpc_converter::sgl_grpc_response_converter_convert_chunk(
converter_handle,
proto_chunk_json_cstr.as_ptr(),
&mut openai_json_ptr,
error_out,
);
if result == SglErrorCode::Success {
*openai_json_out = openai_json_ptr;
*is_done_out = if is_done { 1 } else { 0 };
SglErrorCode::Success
} else {
*openai_json_out = ptr::null_mut();
*is_done_out = if is_done { 1 } else { 0 };
result
}
}
/// Postprocess multiple gRPC stream chunks in batch (reduces FFI overhead)
///
/// This function processes multiple chunks in a single FFI call, significantly reducing
/// FFI overhead in streaming scenarios.
///
/// # Arguments
/// * `converter_handle` - Converter handle (created with sgl_grpc_response_converter_create)
/// * `proto_chunks_json_array` - JSON array string of proto.GenerateResponse chunks
/// * `max_chunks` - Maximum number of chunks to process (for safety)
/// * `openai_chunks_json_array_out` - Pointer to receive JSON array of OpenAI format chunks (must be freed with sgl_free_string)
/// * `chunks_count_out` - Pointer to receive number of processed chunks
/// * `error_out` - Optional pointer to receive error message
///
/// # Returns
/// * SglErrorCode::Success on success, error code on failure
#[no_mangle]
pub unsafe extern "C" fn sgl_postprocess_stream_chunks_batch(
converter_handle: *mut GrpcResponseConverterHandle,
proto_chunks_json_array: *const c_char,
max_chunks: c_int,
openai_chunks_json_array_out: *mut *mut c_char,
chunks_count_out: *mut c_int,
error_out: *mut *mut c_char,
) -> SglErrorCode {
if converter_handle.is_null()
|| proto_chunks_json_array.is_null()
|| openai_chunks_json_array_out.is_null()
|| chunks_count_out.is_null()
{
set_error_message(error_out, "Invalid arguments: null pointer");
return SglErrorCode::InvalidArgument;
}
let chunks_array_str = match CStr::from_ptr(proto_chunks_json_array).to_str() {
Ok(s) => s,
Err(_) => {
set_error_message(error_out, "Invalid UTF-8 in proto_chunks_json_array");
return SglErrorCode::InvalidArgument;
}
};
// Parse JSON array of chunks
let chunks_array: Vec<Value> = match serde_json::from_str(chunks_array_str) {
Ok(arr) => arr,
Err(e) => {
set_error_message(
error_out,
&format!("Failed to parse chunks JSON array: {}", e),
);
return SglErrorCode::ParsingError;
}
};
// Limit batch size for safety
let max_chunks_usize = max_chunks as usize;
let chunks_to_process = if chunks_array.len() > max_chunks_usize {
&chunks_array[..max_chunks_usize]
} else {
&chunks_array
};
let handle_ref = &mut *converter_handle;
let tokenizer = Arc::clone(&handle_ref.tokenizer);
let model = handle_ref.model.clone();
let request_id = handle_ref.request_id.clone();
let created = handle_ref.created;
let system_fingerprint = handle_ref.system_fingerprint.clone();
// Process chunks in batch
let mut results = Vec::new();
let mut has_error = false;
let mut error_msg = String::new();
for chunk_json in chunks_to_process {
// Parse proto.GenerateResponse from JSON
let mut proto_response = proto::GenerateResponse {
request_id: chunk_json
.get("request_id")
.and_then(|v| v.as_str())
.unwrap_or("")
.to_string(),
response: None,
};
// Parse the response oneof field (same logic as single chunk processing)
let _is_done = if let Some(chunk_json) = chunk_json.get("chunk") {
let chunk = proto::GenerateStreamChunk {
token_ids: chunk_json
.get("token_ids")
.and_then(|v| v.as_array())
.map(|arr| {
arr.iter()
.filter_map(|v| v.as_u64().map(|n| n as u32))
.collect()
})
.unwrap_or_default(),
prompt_tokens: chunk_json
.get("prompt_tokens")
.and_then(|v| v.as_i64())
.map(|n| n as i32)
.unwrap_or(0),
completion_tokens: chunk_json
.get("completion_tokens")
.and_then(|v| v.as_i64())
.map(|n| n as i32)
.unwrap_or(0),
cached_tokens: chunk_json
.get("cached_tokens")
.and_then(|v| v.as_i64())
.map(|n| n as i32)
.unwrap_or(0),
output_logprobs: None,
hidden_states: vec![],
input_logprobs: None,
index: 0,
};
proto_response.response = Some(proto::generate_response::Response::Chunk(chunk));
false
} else if let Some(complete_json) = chunk_json.get("complete") {
let complete = proto::GenerateComplete {
output_ids: complete_json
.get("output_ids")
.and_then(|v| v.as_array())
.map(|arr| {
arr.iter()
.filter_map(|v| v.as_u64().map(|n| n as u32))
.collect()
})
.unwrap_or_default(),
finish_reason: complete_json
.get("finish_reason")
.and_then(|v| v.as_str())
.unwrap_or("")
.to_string(),
prompt_tokens: complete_json
.get("prompt_tokens")
.and_then(|v| v.as_i64())
.map(|n| n as i32)
.unwrap_or(0),
completion_tokens: complete_json
.get("completion_tokens")
.and_then(|v| v.as_i64())
.map(|n| n as i32)
.unwrap_or(0),
cached_tokens: complete_json
.get("cached_tokens")
.and_then(|v| v.as_i64())
.map(|n| n as i32)
.unwrap_or(0),
output_logprobs: None,
all_hidden_states: vec![],
input_logprobs: None,
matched_stop: None,
index: 0,
};
proto_response.response = Some(proto::generate_response::Response::Complete(complete));
true
} else if let Some(error_json) = chunk_json.get("error") {
let error = proto::GenerateError {
message: error_json
.get("message")
.and_then(|v| v.as_str())
.unwrap_or("")
.to_string(),
http_status_code: error_json
.get("http_status_code")
.and_then(|v| v.as_str())
.unwrap_or("500")
.to_string(),
details: error_json
.get("details")
.and_then(|v| v.as_str())
.unwrap_or("")
.to_string(),
};
proto_response.response = Some(proto::generate_response::Response::Error(error));
true
} else {
error_msg = format!(
"Chunk JSON must contain 'chunk', 'complete', or 'error' field: {}",
chunk_json
);
has_error = true;
break;
};
// Convert proto chunk to OpenAI format
let result = RUNTIME.block_on(async {
super::grpc_converter::convert_proto_chunk_to_openai(
proto_response,
handle_ref,
&tokenizer,
&model,
&request_id,
created,
system_fingerprint.as_deref(),
)
.await
});
match result {
Ok(Some(openai_response)) => {
results.push(openai_response);
}
Ok(None) => {
// Empty response, skip
}
Err(e) => {
error_msg = format!("Postprocessing failed for chunk: {}", e);
has_error = true;
break;
}
}
}
if has_error {
set_error_message(error_out, &error_msg);
return SglErrorCode::ParsingError;
}
// Serialize results to JSON array
let results_json = match serde_json::to_string(&results) {
Ok(s) => s,
Err(e) => {
set_error_message(
error_out,
&format!("Failed to serialize results JSON array: {}", e),
);
return SglErrorCode::ParsingError;
}
};
let results_cstr = match CString::new(results_json) {
Ok(s) => s,
Err(e) => {
set_error_message(error_out, &format!("Failed to create C string: {}", e));
return SglErrorCode::MemoryError;
}
};
*openai_chunks_json_array_out = results_cstr.into_raw();
*chunks_count_out = results.len() as c_int;
SglErrorCode::Success
}