chore: vendor sglang v0.5.10 snapshot
This commit is contained in:
758
third_party/sglang/sgl-model-gateway/bindings/golang/src/grpc_converter.rs
vendored
Normal file
758
third_party/sglang/sgl-model-gateway/bindings/golang/src/grpc_converter.rs
vendored
Normal file
@@ -0,0 +1,758 @@
|
||||
//! gRPC response converter FFI functions
|
||||
|
||||
use std::ffi::{CStr, CString};
|
||||
use std::os::raw::{c_char, c_int};
|
||||
use std::ptr;
|
||||
use std::sync::Arc;
|
||||
use std::collections::HashMap;
|
||||
use serde_json::Value;
|
||||
use tokio::runtime::Runtime;
|
||||
use once_cell::sync::Lazy;
|
||||
|
||||
use smg::tokenizer::traits::Tokenizer;
|
||||
use smg::tokenizer::stream::DecodeStream;
|
||||
use smg::tool_parser::ToolParser;
|
||||
use smg::protocols::common::{Tool, ToolChoice, ToolChoiceValue, ToolCallDelta, FunctionCallDelta, Usage, StringOrArray};
|
||||
use smg::tokenizer::stop::StopSequenceDecoder;
|
||||
use smg_grpc_client::sglang_proto as proto;
|
||||
|
||||
use super::error::{SglErrorCode, set_error_message, clear_error_message};
|
||||
use super::tokenizer::TokenizerHandle;
|
||||
use super::utils::generate_tool_call_id;
|
||||
|
||||
/// Global parser factory (initialized once)
|
||||
// Use the re-exported ParserFactory from tool_parser module
|
||||
static PARSER_FACTORY: Lazy<smg::tool_parser::ParserFactory> = Lazy::new(|| {
|
||||
// ParserFactory is re-exported from tool_parser::factory, so we can use it directly
|
||||
smg::tool_parser::ParserFactory::default()
|
||||
});
|
||||
|
||||
/// Global tokio runtime for async operations
|
||||
static RUNTIME: Lazy<Runtime> = Lazy::new(|| {
|
||||
Runtime::new().expect("Failed to create tokio runtime for gRPC converter FFI")
|
||||
});
|
||||
|
||||
/// Handle for gRPC response converter (maintains state for streaming)
|
||||
#[repr(C)]
|
||||
pub struct GrpcResponseConverterHandle {
|
||||
pub(crate) tokenizer: Arc<dyn Tokenizer>,
|
||||
pub(crate) tool_parser: Option<Arc<tokio::sync::Mutex<Box<dyn ToolParser>>>>,
|
||||
pub(crate) stop_decoder: Option<Arc<tokio::sync::Mutex<StopSequenceDecoder>>>,
|
||||
pub(crate) model: String,
|
||||
pub(crate) request_id: String,
|
||||
pub(crate) created: u64,
|
||||
pub(crate) system_fingerprint: Option<String>,
|
||||
pub(crate) tools: Option<Vec<Tool>>,
|
||||
pub(crate) tool_choice: Option<ToolChoice>,
|
||||
pub(crate) history_tool_calls_count: usize,
|
||||
pub(crate) stream_buffers: HashMap<u32, String>, // Per-index text buffers
|
||||
pub(crate) decode_streams: HashMap<u32, DecodeStream>, // Per-index incremental decoders
|
||||
pub(crate) has_tool_calls: HashMap<u32, bool>, // Track if tool calls were emitted
|
||||
pub(crate) is_first_chunk: HashMap<u32, bool>, // Track first chunk per index
|
||||
pub(crate) prompt_tokens: HashMap<u32, i32>, // Track prompt tokens per index (from chunks)
|
||||
pub(crate) completion_tokens: HashMap<u32, i32>, // Track completion tokens per index (cumulative)
|
||||
pub(crate) initial_prompt_tokens: Option<i32>, // Initial prompt tokens from request (if available)
|
||||
pub(crate) skip_special_tokens: bool, // Whether to skip special tokens when decoding
|
||||
}
|
||||
|
||||
/// Create a gRPC response converter handle
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `tokenizer_handle` - Tokenizer handle (must be valid)
|
||||
/// * `model` - Model name
|
||||
/// * `request_id` - Request ID
|
||||
/// * `tools_json` - Optional JSON array of tools
|
||||
/// * `tool_choice_json` - Optional JSON object for tool_choice
|
||||
/// * `stop` - Optional stop sequences (JSON array)
|
||||
/// * `stop_token_ids` - Optional stop token IDs (JSON array)
|
||||
/// * `skip_special_tokens` - Whether to skip special tokens
|
||||
/// * `error_out` - Optional pointer to receive error message
|
||||
///
|
||||
/// # Returns
|
||||
/// * Pointer to GrpcResponseConverterHandle on success, null on failure
|
||||
#[no_mangle]
|
||||
pub unsafe extern "C" fn sgl_grpc_response_converter_create(
|
||||
tokenizer_handle: *mut TokenizerHandle,
|
||||
model: *const c_char,
|
||||
request_id: *const c_char,
|
||||
tools_json: *const c_char,
|
||||
tool_choice_json: *const c_char,
|
||||
stop: *const c_char,
|
||||
stop_token_ids: *const c_char,
|
||||
skip_special_tokens: c_int,
|
||||
error_out: *mut *mut c_char,
|
||||
) -> *mut GrpcResponseConverterHandle {
|
||||
if tokenizer_handle.is_null() || model.is_null() || request_id.is_null() {
|
||||
set_error_message(error_out, "Invalid arguments: null pointer");
|
||||
return ptr::null_mut();
|
||||
}
|
||||
|
||||
let model_str = match CStr::from_ptr(model).to_str() {
|
||||
Ok(s) => s,
|
||||
Err(_) => {
|
||||
set_error_message(error_out, "Invalid UTF-8 in model");
|
||||
return ptr::null_mut();
|
||||
}
|
||||
};
|
||||
|
||||
let request_id_str = match CStr::from_ptr(request_id).to_str() {
|
||||
Ok(s) => s,
|
||||
Err(_) => {
|
||||
set_error_message(error_out, "Invalid UTF-8 in request_id");
|
||||
return ptr::null_mut();
|
||||
}
|
||||
};
|
||||
|
||||
let handle_ref = &*tokenizer_handle;
|
||||
let tokenizer = Arc::clone(&handle_ref.tokenizer);
|
||||
|
||||
// Parse tools if provided
|
||||
let tools: Option<Vec<Tool>> = if !tools_json.is_null() {
|
||||
match CStr::from_ptr(tools_json).to_str() {
|
||||
Ok(s) => serde_json::from_str::<Vec<Tool>>(s).ok(),
|
||||
Err(_) => None,
|
||||
}
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
// Parse tool_choice if provided
|
||||
let tool_choice: Option<ToolChoice> = if !tool_choice_json.is_null() {
|
||||
match CStr::from_ptr(tool_choice_json).to_str() {
|
||||
Ok(s) => serde_json::from_str::<ToolChoice>(s).ok(),
|
||||
Err(_) => None,
|
||||
}
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
// Parse stop sequences
|
||||
let stop: Option<StringOrArray> = if !stop.is_null() {
|
||||
let stop_str = match CStr::from_ptr(stop).to_str() {
|
||||
Ok(s) => s,
|
||||
Err(_) => return ptr::null_mut(),
|
||||
};
|
||||
serde_json::from_str::<StringOrArray>(stop_str).ok()
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
// Parse stop token IDs
|
||||
let stop_token_ids: Option<Vec<u32>> = if !stop_token_ids.is_null() {
|
||||
let ids_str = match CStr::from_ptr(stop_token_ids).to_str() {
|
||||
Ok(s) => s,
|
||||
Err(_) => return ptr::null_mut(),
|
||||
};
|
||||
serde_json::from_str::<Vec<u32>>(ids_str).ok()
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
// Create stop decoder if needed
|
||||
let stop_decoder = if stop.is_some() || stop_token_ids.is_some() {
|
||||
Some(Arc::new(tokio::sync::Mutex::new(
|
||||
smg::routers::grpc::utils::create_stop_decoder(
|
||||
&tokenizer,
|
||||
stop.as_ref(),
|
||||
stop_token_ids.as_ref(),
|
||||
skip_special_tokens != 0,
|
||||
false, // no_stop_trim
|
||||
),
|
||||
)))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
// Create tool parser if tools are provided
|
||||
let tool_parser = if tools.is_some() {
|
||||
PARSER_FACTORY.registry().create_for_model(model_str)
|
||||
.map(|p| Arc::new(tokio::sync::Mutex::new(p)))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
// Get system fingerprint from model (simplified)
|
||||
let system_fingerprint = Some("fp_placeholder".to_string()); // TODO: Get actual fingerprint
|
||||
|
||||
Box::into_raw(Box::new(GrpcResponseConverterHandle {
|
||||
tokenizer,
|
||||
tool_parser,
|
||||
stop_decoder,
|
||||
model: model_str.to_string(),
|
||||
request_id: request_id_str.to_string(),
|
||||
created: std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_secs(),
|
||||
system_fingerprint,
|
||||
tools,
|
||||
tool_choice,
|
||||
history_tool_calls_count: 0,
|
||||
stream_buffers: HashMap::new(),
|
||||
decode_streams: HashMap::new(),
|
||||
has_tool_calls: HashMap::new(),
|
||||
is_first_chunk: HashMap::new(),
|
||||
prompt_tokens: HashMap::new(),
|
||||
completion_tokens: HashMap::new(),
|
||||
initial_prompt_tokens: None, // Will be set from stream handle
|
||||
skip_special_tokens: skip_special_tokens != 0,
|
||||
}))
|
||||
}
|
||||
|
||||
/// Convert a gRPC GenerateResponse chunk to OpenAI format
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `handle` - Converter handle
|
||||
/// * `response_json` - JSON string of proto.GenerateResponse
|
||||
/// * `result_json_out` - Pointer to receive OpenAI format JSON (must be freed with sgl_free_string)
|
||||
/// * `error_out` - Optional pointer to receive error message
|
||||
///
|
||||
/// # Returns
|
||||
/// * SglErrorCode::Success on success, error code on failure
|
||||
#[no_mangle]
|
||||
pub unsafe extern "C" fn sgl_grpc_response_converter_convert_chunk(
|
||||
handle: *mut GrpcResponseConverterHandle,
|
||||
response_json: *const c_char,
|
||||
result_json_out: *mut *mut c_char,
|
||||
error_out: *mut *mut c_char,
|
||||
) -> SglErrorCode {
|
||||
if handle.is_null() || response_json.is_null() || result_json_out.is_null() {
|
||||
set_error_message(error_out, "Invalid arguments: null pointer");
|
||||
return SglErrorCode::InvalidArgument;
|
||||
}
|
||||
|
||||
let response_str = match CStr::from_ptr(response_json).to_str() {
|
||||
Ok(s) => s,
|
||||
Err(_) => {
|
||||
set_error_message(error_out, "Invalid UTF-8 in response_json");
|
||||
return SglErrorCode::InvalidArgument;
|
||||
}
|
||||
};
|
||||
|
||||
// Parse proto.GenerateResponse from JSON
|
||||
let json_value: Value = match serde_json::from_str(response_str) {
|
||||
Ok(v) => v,
|
||||
Err(e) => {
|
||||
set_error_message(error_out, &format!("Failed to parse response JSON: {}", e));
|
||||
return SglErrorCode::ParsingError;
|
||||
}
|
||||
};
|
||||
|
||||
// Build proto::GenerateResponse from JSON value
|
||||
let mut proto_response = proto::GenerateResponse {
|
||||
request_id: json_value.get("request_id")
|
||||
.and_then(|v| v.as_str())
|
||||
.unwrap_or("")
|
||||
.to_string(),
|
||||
response: None,
|
||||
};
|
||||
|
||||
// Parse the response oneof field
|
||||
if let Some(chunk_json) = json_value.get("chunk") {
|
||||
let chunk = proto::GenerateStreamChunk {
|
||||
token_ids: chunk_json.get("token_ids")
|
||||
.and_then(|v| v.as_array())
|
||||
.map(|arr| arr.iter().filter_map(|v| v.as_u64().map(|n| n as u32)).collect())
|
||||
.unwrap_or_default(),
|
||||
prompt_tokens: chunk_json.get("prompt_tokens")
|
||||
.and_then(|v| v.as_i64())
|
||||
.map(|n| n as i32)
|
||||
.unwrap_or(0),
|
||||
completion_tokens: chunk_json.get("completion_tokens")
|
||||
.and_then(|v| v.as_i64())
|
||||
.map(|n| n as i32)
|
||||
.unwrap_or(0),
|
||||
cached_tokens: chunk_json.get("cached_tokens")
|
||||
.and_then(|v| v.as_i64())
|
||||
.map(|n| n as i32)
|
||||
.unwrap_or(0),
|
||||
output_logprobs: None,
|
||||
hidden_states: vec![],
|
||||
input_logprobs: None,
|
||||
index: 0,
|
||||
};
|
||||
proto_response.response = Some(proto::generate_response::Response::Chunk(chunk));
|
||||
} else if let Some(complete_json) = json_value.get("complete") {
|
||||
let complete = proto::GenerateComplete {
|
||||
output_ids: complete_json.get("output_ids")
|
||||
.and_then(|v| v.as_array())
|
||||
.map(|arr| arr.iter().filter_map(|v| v.as_u64().map(|n| n as u32)).collect())
|
||||
.unwrap_or_default(),
|
||||
finish_reason: complete_json.get("finish_reason")
|
||||
.and_then(|v| v.as_str())
|
||||
.unwrap_or("")
|
||||
.to_string(),
|
||||
prompt_tokens: complete_json.get("prompt_tokens")
|
||||
.and_then(|v| v.as_i64())
|
||||
.map(|n| n as i32)
|
||||
.unwrap_or(0),
|
||||
completion_tokens: complete_json.get("completion_tokens")
|
||||
.and_then(|v| v.as_i64())
|
||||
.map(|n| n as i32)
|
||||
.unwrap_or(0),
|
||||
cached_tokens: complete_json.get("cached_tokens")
|
||||
.and_then(|v| v.as_i64())
|
||||
.map(|n| n as i32)
|
||||
.unwrap_or(0),
|
||||
output_logprobs: None,
|
||||
all_hidden_states: vec![],
|
||||
input_logprobs: None,
|
||||
matched_stop: None,
|
||||
index: 0,
|
||||
};
|
||||
proto_response.response = Some(proto::generate_response::Response::Complete(complete));
|
||||
} else if let Some(error_json) = json_value.get("error") {
|
||||
let error = proto::GenerateError {
|
||||
message: error_json.get("message")
|
||||
.and_then(|v| v.as_str())
|
||||
.unwrap_or("")
|
||||
.to_string(),
|
||||
http_status_code: error_json.get("http_status_code")
|
||||
.and_then(|v| v.as_str())
|
||||
.unwrap_or("500")
|
||||
.to_string(),
|
||||
details: error_json.get("details")
|
||||
.and_then(|v| v.as_str())
|
||||
.unwrap_or("")
|
||||
.to_string(),
|
||||
};
|
||||
proto_response.response = Some(proto::generate_response::Response::Error(error));
|
||||
} else {
|
||||
set_error_message(error_out, "Response JSON must contain 'chunk', 'complete', or 'error' field");
|
||||
return SglErrorCode::ParsingError;
|
||||
}
|
||||
|
||||
let handle_ref = &mut *handle;
|
||||
let tokenizer = Arc::clone(&handle_ref.tokenizer);
|
||||
let model = handle_ref.model.clone();
|
||||
let request_id = handle_ref.request_id.clone();
|
||||
let created = handle_ref.created;
|
||||
let system_fingerprint = handle_ref.system_fingerprint.clone();
|
||||
|
||||
// Use tokio runtime to run async code
|
||||
let result = RUNTIME.block_on(async {
|
||||
convert_proto_chunk_to_openai(
|
||||
proto_response,
|
||||
handle_ref,
|
||||
&tokenizer,
|
||||
&model,
|
||||
&request_id,
|
||||
created,
|
||||
system_fingerprint.as_deref(),
|
||||
)
|
||||
.await
|
||||
});
|
||||
|
||||
match result {
|
||||
Ok(Some(openai_response)) => {
|
||||
// Serialize to JSON
|
||||
let result_str = match serde_json::to_string(&openai_response) {
|
||||
Ok(s) => s,
|
||||
Err(e) => {
|
||||
set_error_message(error_out, &format!("Failed to serialize response: {}", e));
|
||||
return SglErrorCode::ParsingError;
|
||||
}
|
||||
};
|
||||
|
||||
let result_cstr = match CString::new(result_str) {
|
||||
Ok(s) => s,
|
||||
Err(e) => {
|
||||
set_error_message(error_out, &format!("Failed to create result string: {}", e));
|
||||
return SglErrorCode::MemoryError;
|
||||
}
|
||||
};
|
||||
|
||||
*result_json_out = result_cstr.into_raw();
|
||||
clear_error_message(error_out);
|
||||
SglErrorCode::Success
|
||||
}
|
||||
Ok(None) => {
|
||||
// No response to send (e.g., empty chunk)
|
||||
let empty = CString::new("").unwrap();
|
||||
*result_json_out = empty.into_raw();
|
||||
clear_error_message(error_out);
|
||||
SglErrorCode::Success
|
||||
}
|
||||
Err(e) => {
|
||||
set_error_message(error_out, &format!("Conversion error: {}", e));
|
||||
SglErrorCode::ParsingError
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Helper function to convert proto chunk to OpenAI format
|
||||
pub(crate) async fn convert_proto_chunk_to_openai(
|
||||
proto_response: proto::GenerateResponse,
|
||||
handle: &mut GrpcResponseConverterHandle,
|
||||
tokenizer: &Arc<dyn Tokenizer>,
|
||||
model: &str,
|
||||
request_id: &str,
|
||||
created: u64,
|
||||
system_fingerprint: Option<&str>,
|
||||
) -> Result<Option<smg::protocols::chat::ChatCompletionStreamResponse>, String> {
|
||||
use smg_grpc_client::sglang_proto::generate_response::Response::*;
|
||||
use smg::protocols::chat::{ChatCompletionStreamResponse, ChatMessageDelta, ChatStreamChoice};
|
||||
|
||||
match proto_response.response {
|
||||
Some(Chunk(chunk)) => {
|
||||
let index = chunk.index;
|
||||
|
||||
// Mark as not first chunk if we've seen this index before
|
||||
let is_first = handle.is_first_chunk.entry(index).or_insert(true);
|
||||
let first_chunk = *is_first;
|
||||
*is_first = false;
|
||||
|
||||
// Track token counts from chunks (cumulative values from proto)
|
||||
// These are cumulative values, so we always use the latest value
|
||||
// For prompt_tokens, if chunk value is 0, preserve existing value or use initial_prompt_tokens
|
||||
// This prevents overwriting valid prompt_tokens with 0
|
||||
if chunk.prompt_tokens > 0 {
|
||||
handle.prompt_tokens.insert(index, chunk.prompt_tokens);
|
||||
} else {
|
||||
// If chunk.prompt_tokens is 0, try to preserve existing value or use initial_prompt_tokens
|
||||
if !handle.prompt_tokens.contains_key(&index) {
|
||||
// No existing value, try to use initial_prompt_tokens
|
||||
if let Some(initial_prompt) = handle.initial_prompt_tokens {
|
||||
handle.prompt_tokens.insert(index, initial_prompt);
|
||||
}
|
||||
}
|
||||
// If existing value exists, keep it (don't overwrite with 0)
|
||||
}
|
||||
// For completion_tokens, always update (even if 0) as it's cumulative
|
||||
handle.completion_tokens.insert(index, chunk.completion_tokens);
|
||||
|
||||
// Process tokens through stop decoder if available, otherwise use incremental decoder
|
||||
let chunk_text = if let Some(ref stop_decoder) = handle.stop_decoder {
|
||||
let mut decoder_guard = stop_decoder.lock().await;
|
||||
let mut text = String::new();
|
||||
for &token_id in &chunk.token_ids {
|
||||
match decoder_guard.process_token(token_id).unwrap_or_else(|_| {
|
||||
smg::tokenizer::stop::SequenceDecoderOutput::Held
|
||||
}) {
|
||||
smg::tokenizer::stop::SequenceDecoderOutput::Text(t) => {
|
||||
text.push_str(&t);
|
||||
}
|
||||
smg::tokenizer::stop::SequenceDecoderOutput::StoppedWithText(t) => {
|
||||
text.push_str(&t);
|
||||
break;
|
||||
}
|
||||
smg::tokenizer::stop::SequenceDecoderOutput::Stopped => {
|
||||
break;
|
||||
}
|
||||
smg::tokenizer::stop::SequenceDecoderOutput::Held => {}
|
||||
}
|
||||
}
|
||||
text
|
||||
} else {
|
||||
// Use incremental decoder to handle multi-byte character boundaries
|
||||
let decode_stream = handle.decode_streams.entry(index).or_insert_with(|| {
|
||||
DecodeStream::new(
|
||||
Arc::clone(&tokenizer),
|
||||
&[], // No prompt tokens for completion
|
||||
handle.skip_special_tokens,
|
||||
)
|
||||
});
|
||||
|
||||
// Process tokens incrementally
|
||||
let mut text_parts = Vec::new();
|
||||
for &token_id in &chunk.token_ids {
|
||||
if let Ok(Some(text)) = decode_stream.step(token_id) {
|
||||
text_parts.push(text);
|
||||
}
|
||||
}
|
||||
text_parts.join("")
|
||||
};
|
||||
|
||||
if chunk_text.is_empty() {
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
// Send first chunk with role
|
||||
if first_chunk {
|
||||
let first_response = ChatCompletionStreamResponse {
|
||||
id: request_id.to_string(),
|
||||
object: "chat.completion.chunk".to_string(),
|
||||
created,
|
||||
model: model.to_string(),
|
||||
system_fingerprint: system_fingerprint.map(|s| s.to_string()),
|
||||
choices: vec![ChatStreamChoice {
|
||||
index,
|
||||
delta: ChatMessageDelta {
|
||||
role: Some("assistant".to_string()),
|
||||
content: None,
|
||||
tool_calls: None,
|
||||
reasoning_content: None,
|
||||
},
|
||||
logprobs: None,
|
||||
finish_reason: None,
|
||||
matched_stop: None,
|
||||
}],
|
||||
usage: None,
|
||||
};
|
||||
return Ok(Some(first_response));
|
||||
}
|
||||
|
||||
// Update stream buffer
|
||||
let stream_buffer = handle.stream_buffers.entry(index).or_default();
|
||||
stream_buffer.push_str(&chunk_text);
|
||||
|
||||
// Handle tool calls if tools are provided
|
||||
if let (Some(ref tools), Some(ref tool_parser)) = (handle.tools.as_ref(), handle.tool_parser.as_ref()) {
|
||||
let tool_choice_enabled = !matches!(
|
||||
handle.tool_choice,
|
||||
Some(ToolChoice::Value(ToolChoiceValue::None))
|
||||
);
|
||||
|
||||
if tool_choice_enabled {
|
||||
let mut parser_guard = tool_parser.lock().await;
|
||||
match parser_guard.parse_incremental(&chunk_text, tools).await {
|
||||
Ok(streaming_result) => {
|
||||
if !streaming_result.calls.is_empty() {
|
||||
handle.has_tool_calls.insert(index, true);
|
||||
// Convert tool call items to OpenAI format
|
||||
let tool_call_deltas: Vec<_> = streaming_result
|
||||
.calls
|
||||
.into_iter()
|
||||
.map(|item| {
|
||||
let id = if let Some(ref name) = item.name {
|
||||
generate_tool_call_id(
|
||||
model,
|
||||
name,
|
||||
item.tool_index,
|
||||
handle.history_tool_calls_count,
|
||||
)
|
||||
} else {
|
||||
format!("call_{}", item.tool_index)
|
||||
};
|
||||
|
||||
ToolCallDelta {
|
||||
index: item.tool_index as u32,
|
||||
id: Some(id),
|
||||
tool_type: if item.name.is_some() {
|
||||
Some("function".to_string())
|
||||
} else {
|
||||
None
|
||||
},
|
||||
function: Some(FunctionCallDelta {
|
||||
name: item.name,
|
||||
arguments: if !item.parameters.is_empty() {
|
||||
Some(item.parameters)
|
||||
} else {
|
||||
None
|
||||
},
|
||||
}),
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
|
||||
let tool_response = ChatCompletionStreamResponse {
|
||||
id: request_id.to_string(),
|
||||
object: "chat.completion.chunk".to_string(),
|
||||
created,
|
||||
model: model.to_string(),
|
||||
system_fingerprint: system_fingerprint.map(|s| s.to_string()),
|
||||
choices: vec![ChatStreamChoice {
|
||||
index,
|
||||
delta: ChatMessageDelta {
|
||||
role: Some("assistant".to_string()),
|
||||
content: None,
|
||||
tool_calls: Some(tool_call_deltas),
|
||||
reasoning_content: None,
|
||||
},
|
||||
logprobs: None,
|
||||
finish_reason: None,
|
||||
matched_stop: None,
|
||||
}],
|
||||
usage: None,
|
||||
};
|
||||
return Ok(Some(tool_response));
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
// Log error but continue with regular content
|
||||
tracing::warn!("Tool parser error: {}", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Regular content emission
|
||||
let content_response = ChatCompletionStreamResponse {
|
||||
id: request_id.to_string(),
|
||||
object: "chat.completion.chunk".to_string(),
|
||||
created,
|
||||
model: model.to_string(),
|
||||
system_fingerprint: system_fingerprint.map(|s| s.to_string()),
|
||||
choices: vec![ChatStreamChoice {
|
||||
index,
|
||||
delta: ChatMessageDelta {
|
||||
role: Some("assistant".to_string()),
|
||||
content: Some(chunk_text),
|
||||
tool_calls: None,
|
||||
reasoning_content: None,
|
||||
},
|
||||
logprobs: None,
|
||||
finish_reason: None,
|
||||
matched_stop: None,
|
||||
}],
|
||||
usage: None,
|
||||
};
|
||||
|
||||
Ok(Some(content_response))
|
||||
}
|
||||
Some(Complete(complete)) => {
|
||||
let index = complete.index;
|
||||
|
||||
// Flush any remaining text
|
||||
// Flush any remaining text from decode stream
|
||||
let mut final_text = handle.stream_buffers.remove(&index).unwrap_or_default();
|
||||
if let Some(ref mut decode_stream) = handle.decode_streams.get_mut(&index) {
|
||||
if let Ok(Some(remaining)) = decode_stream.flush() {
|
||||
final_text.push_str(&remaining);
|
||||
}
|
||||
}
|
||||
handle.decode_streams.remove(&index);
|
||||
|
||||
// Determine finish reason - ensure it's never empty
|
||||
// If finish_reason is empty, try to infer from other fields or use default
|
||||
let finish_reason = if handle.has_tool_calls.get(&index).copied().unwrap_or(false)
|
||||
&& (complete.finish_reason == "stop" || complete.finish_reason.is_empty())
|
||||
{
|
||||
"tool_calls".to_string()
|
||||
} else if complete.finish_reason.is_empty() || complete.finish_reason.trim().is_empty() {
|
||||
// If finish_reason is empty, try to infer from completion_tokens or use default
|
||||
if complete.completion_tokens > 0 {
|
||||
// If we have completion tokens, likely stopped normally
|
||||
"stop".to_string()
|
||||
} else if !complete.output_ids.is_empty() {
|
||||
// If we have output_ids, likely stopped normally
|
||||
"stop".to_string()
|
||||
} else {
|
||||
// Default fallback - always ensure we have a value
|
||||
"stop".to_string()
|
||||
}
|
||||
} else {
|
||||
complete.finish_reason.clone()
|
||||
};
|
||||
|
||||
// Ensure finish_reason is never empty (defensive check)
|
||||
let finish_reason = if finish_reason.is_empty() || finish_reason.trim().is_empty() {
|
||||
"stop".to_string()
|
||||
} else {
|
||||
finish_reason
|
||||
};
|
||||
|
||||
// Extract matched_stop
|
||||
let matched_stop = match &complete.matched_stop {
|
||||
Some(proto::generate_complete::MatchedStop::MatchedTokenId(token_id)) => {
|
||||
Some(Value::Number(serde_json::Number::from(*token_id)))
|
||||
}
|
||||
Some(proto::generate_complete::MatchedStop::MatchedStopStr(stop_str)) => {
|
||||
Some(Value::String(stop_str.clone()))
|
||||
}
|
||||
None => None,
|
||||
};
|
||||
|
||||
// Build usage - prefer values from complete message, but fallback to accumulated values from chunks
|
||||
// Complete message should have the final values, but sometimes they might be 0 or missing
|
||||
// Always use the latest cumulative value from chunks if available, otherwise use complete message value
|
||||
let mut prompt_tokens = handle.prompt_tokens.get(&index)
|
||||
.copied()
|
||||
.filter(|&v| v > 0)
|
||||
.unwrap_or(complete.prompt_tokens);
|
||||
let mut completion_tokens = handle.completion_tokens.get(&index)
|
||||
.copied()
|
||||
.filter(|&v| v > 0)
|
||||
.unwrap_or(complete.completion_tokens);
|
||||
|
||||
// Always try to use initial_prompt_tokens if prompt_tokens is 0 or missing
|
||||
// This is the most reliable source for prompt tokens since we calculate it from the request
|
||||
if prompt_tokens == 0 {
|
||||
if let Some(initial_prompt) = handle.initial_prompt_tokens {
|
||||
prompt_tokens = initial_prompt;
|
||||
}
|
||||
}
|
||||
|
||||
// If completion_tokens is 0, try to infer from output_ids or accumulated chunks
|
||||
if completion_tokens == 0 {
|
||||
// Try to use completion_tokens from complete message even if 0
|
||||
// Or calculate from output_ids
|
||||
if complete.completion_tokens > 0 {
|
||||
completion_tokens = complete.completion_tokens;
|
||||
} else if !complete.output_ids.is_empty() {
|
||||
completion_tokens = complete.output_ids.len() as i32;
|
||||
} else if let Some(&last_completion) = handle.completion_tokens.get(&index) {
|
||||
completion_tokens = last_completion;
|
||||
}
|
||||
}
|
||||
|
||||
// Final fallback: if both are still 0, try to use initial_prompt_tokens for prompt
|
||||
// and calculate completion from output_ids
|
||||
if prompt_tokens == 0 && completion_tokens == 0 {
|
||||
// Try to infer from output_ids if available
|
||||
let output_ids_len = complete.output_ids.len() as i32;
|
||||
if output_ids_len > 0 {
|
||||
completion_tokens = output_ids_len;
|
||||
// Always try to use initial_prompt_tokens for prompt
|
||||
if let Some(initial_prompt) = handle.initial_prompt_tokens {
|
||||
prompt_tokens = initial_prompt;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Final defensive check: ensure prompt_tokens is set if we have initial_prompt_tokens
|
||||
if prompt_tokens == 0 {
|
||||
if let Some(initial_prompt) = handle.initial_prompt_tokens {
|
||||
prompt_tokens = initial_prompt;
|
||||
}
|
||||
}
|
||||
|
||||
// Always create usage, even if values are 0 (defensive)
|
||||
let usage = Some(Usage {
|
||||
prompt_tokens: prompt_tokens.max(0) as u32,
|
||||
completion_tokens: completion_tokens.max(0) as u32,
|
||||
total_tokens: (prompt_tokens.max(0) + completion_tokens.max(0)) as u32,
|
||||
completion_tokens_details: None,
|
||||
});
|
||||
|
||||
let finish_response = ChatCompletionStreamResponse {
|
||||
id: request_id.to_string(),
|
||||
object: "chat.completion.chunk".to_string(),
|
||||
created,
|
||||
model: model.to_string(),
|
||||
system_fingerprint: system_fingerprint.map(|s| s.to_string()),
|
||||
choices: vec![ChatStreamChoice {
|
||||
index,
|
||||
delta: ChatMessageDelta {
|
||||
role: Some("assistant".to_string()),
|
||||
content: if !final_text.is_empty() {
|
||||
Some(final_text)
|
||||
} else {
|
||||
None
|
||||
},
|
||||
tool_calls: None,
|
||||
reasoning_content: None,
|
||||
},
|
||||
logprobs: None,
|
||||
finish_reason: Some(finish_reason),
|
||||
matched_stop,
|
||||
}],
|
||||
usage,
|
||||
};
|
||||
|
||||
Ok(Some(finish_response))
|
||||
}
|
||||
Some(Error(error)) => {
|
||||
Err(format!("Server error: {} (status: {})", error.message, error.http_status_code))
|
||||
}
|
||||
None => Ok(None),
|
||||
}
|
||||
}
|
||||
|
||||
/// Free a gRPC response converter handle
|
||||
#[no_mangle]
|
||||
pub unsafe extern "C" fn sgl_grpc_response_converter_free(handle: *mut GrpcResponseConverterHandle) {
|
||||
if !handle.is_null() {
|
||||
let _ = Box::from_raw(handle);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user