chore: vendor sglang v0.5.10 snapshot

third_party/sglang/sgl-model-gateway/bindings/golang/src/client.rs

@@ -0,0 +1,279 @@
+//! Client SDK FFI functions
+
+use std::ffi::{CStr, CString};
+use std::os::raw::{c_char};
+use std::ptr;
+use std::sync::Arc;
+use tokio::runtime::Runtime;
+use once_cell::sync::Lazy;
+use uuid::Uuid;
+
+use smg::tokenizer::create_tokenizer_from_file;
+use smg::tokenizer::traits::Tokenizer;
+use smg_grpc_client::sglang_scheduler::SglangSchedulerClient;
+use smg::protocols::chat::ChatCompletionRequest;
+use smg::routers::grpc::utils::{process_chat_messages, generate_tool_constraints};
+
+use super::error::{SglErrorCode, set_error_message};
+use super::grpc_converter::sgl_grpc_response_converter_create;
+use super::tokenizer::TokenizerHandle;
+use super::stream::SglangStreamHandle;
+
+/// Global tokio runtime for async operations
+static RUNTIME: Lazy<Runtime> = Lazy::new(|| {
+    Runtime::new().expect("Failed to create tokio runtime for client FFI")
+});
+
+/// Handle for complete client SDK (gRPC client + tokenizer)
+/// This handle manages the connection to sglang and provides a complete SDK interface
+pub struct SglangClientHandle {
+    pub(crate) client: Arc<SglangSchedulerClient>,
+    pub(crate) tokenizer: Arc<dyn Tokenizer>,
+}
+
+/// Handle for streaming request (includes prompt token count)
+#[allow(dead_code)]
+pub struct StreamRequestState {
+    pub(crate) prompt_tokens: i32, // Number of prompt tokens for this request
+}
+
+/// Create a new SGLang client handle
+///
+/// # Arguments
+/// * `endpoint` - gRPC endpoint (e.g., "grpc://localhost:20000")
+/// * `tokenizer_path` - Path to tokenizer directory
+/// * `error_out` - Optional pointer to receive error message
+///
+/// # Returns
+/// * Pointer to SglangClientHandle on success, null on failure
+#[no_mangle]
+pub unsafe extern "C" fn sgl_client_create(
+    endpoint: *const c_char,
+    tokenizer_path: *const c_char,
+    error_out: *mut *mut c_char,
+) -> *mut SglangClientHandle {
+    if endpoint.is_null() || tokenizer_path.is_null() {
+        set_error_message(error_out, "Invalid arguments: null pointer");
+        return ptr::null_mut();
+    }
+
+    let endpoint_str = match CStr::from_ptr(endpoint).to_str() {
+        Ok(s) => s,
+        Err(_) => {
+            set_error_message(error_out, "Invalid UTF-8 in endpoint");
+            return ptr::null_mut();
+        }
+    };
+
+    let tokenizer_path_str = match CStr::from_ptr(tokenizer_path).to_str() {
+        Ok(s) => s,
+        Err(_) => {
+            set_error_message(error_out, "Invalid UTF-8 in tokenizer_path");
+            return ptr::null_mut();
+        }
+    };
+
+    // Create tokenizer
+    let tokenizer = match create_tokenizer_from_file(tokenizer_path_str) {
+        Ok(t) => t,
+        Err(e) => {
+            set_error_message(error_out, &format!("Failed to create tokenizer: {}", e));
+            return ptr::null_mut();
+        }
+    };
+
+    // Create gRPC client
+    let client = match RUNTIME.block_on(async {
+        SglangSchedulerClient::connect(endpoint_str).await
+    }) {
+        Ok(c) => Arc::new(c),
+        Err(e) => {
+            set_error_message(error_out, &format!("Failed to connect to endpoint: {}", e));
+            return ptr::null_mut();
+        }
+    };
+
+    Box::into_raw(Box::new(SglangClientHandle {
+        client,
+        tokenizer,
+    }))
+}
+
+/// Free a client handle
+#[no_mangle]
+pub unsafe extern "C" fn sgl_client_free(handle: *mut SglangClientHandle) {
+    if !handle.is_null() {
+        let _ = Box::from_raw(handle);
+    }
+}
+
+/// Send a chat completion request and start streaming
+///
+/// # Arguments
+/// * `client_handle` - Client handle
+/// * `request_json` - OpenAI ChatCompletionRequest as JSON string
+/// * `stream_handle_out` - Pointer to receive stream handle
+/// * `error_out` - Optional pointer to receive error message
+///
+/// # Returns
+/// * SglErrorCode::Success on success, error code on failure
+#[no_mangle]
+pub unsafe extern "C" fn sgl_client_chat_completion_stream(
+    client_handle: *mut SglangClientHandle,
+    request_json: *const c_char,
+    stream_handle_out: *mut *mut SglangStreamHandle,
+    error_out: *mut *mut c_char,
+) -> SglErrorCode {
+    if client_handle.is_null() || request_json.is_null() || stream_handle_out.is_null() {
+        set_error_message(error_out, "Invalid arguments: null pointer");
+        return SglErrorCode::InvalidArgument;
+    }
+
+    let request_str = match CStr::from_ptr(request_json).to_str() {
+        Ok(s) => s,
+        Err(_) => {
+            set_error_message(error_out, "Invalid UTF-8 in request_json");
+            return SglErrorCode::InvalidArgument;
+        }
+    };
+
+    let client_ref = &*client_handle;
+    let client = Arc::clone(&client_ref.client);
+    let tokenizer = Arc::clone(&client_ref.tokenizer);
+
+    // Parse OpenAI ChatCompletionRequest
+    let chat_request: ChatCompletionRequest = match serde_json::from_str(request_str) {
+        Ok(req) => req,
+        Err(e) => {
+            set_error_message(error_out, &format!("Failed to parse request JSON: {}", e));
+            return SglErrorCode::ParsingError;
+        }
+    };
+
+    // Process messages and apply chat template
+    let processed_messages = match process_chat_messages(&chat_request, tokenizer.as_ref()) {
+        Ok(msgs) => msgs,
+        Err(e) => {
+            set_error_message(error_out, &format!("Failed to process messages: {}", e));
+            return SglErrorCode::TokenizationError;
+        }
+    };
+
+    // Tokenize
+    let token_ids = match tokenizer.encode(&processed_messages.text, false) {
+        Ok(encoding) => encoding.token_ids().to_vec(),
+        Err(e) => {
+            set_error_message(error_out, &format!("Failed to tokenize: {}", e));
+            return SglErrorCode::TokenizationError;
+        }
+    };
+    let prompt_tokens = token_ids.len() as i32; // Save prompt token count
+
+    // Generate tool constraints if needed
+    let tool_constraint = if let Some(tools) = chat_request.tools.as_ref() {
+        match generate_tool_constraints(tools, &chat_request.tool_choice, &chat_request.model) {
+            Ok(Some((constraint_type, constraint_value))) => Some((constraint_type, constraint_value)),
+            Ok(None) => None,
+            Err(e) => {
+                set_error_message(error_out, &format!("Failed to generate tool constraints: {}", e));
+                return SglErrorCode::ParsingError;
+            }
+        }
+    } else {
+        None
+    };
+
+    // Build GenerateRequest
+    let request_id = format!("chatcmpl-{}", Uuid::new_v4());
+    let proto_request = match client.build_generate_request_from_chat(
+        request_id.clone(),
+        &chat_request,
+        processed_messages.text,
+        token_ids,
+        processed_messages.multimodal_inputs,
+        tool_constraint,
+    ) {
+        Ok(req) => req,
+        Err(e) => {
+            set_error_message(error_out, &format!("Failed to build generate request: {}", e));
+            return SglErrorCode::ParsingError;
+        }
+    };
+
+    // Send request and get stream
+    let stream = match RUNTIME.block_on(async {
+        client.generate(proto_request).await
+    }) {
+        Ok(s) => s,
+        Err(e) => {
+            set_error_message(error_out, &format!("Failed to send request: {}", e));
+            return SglErrorCode::UnknownError;
+        }
+    };
+
+    // Create response converter
+    let tools_json = chat_request.tools.as_ref()
+        .and_then(|t| serde_json::to_string(t).ok())
+        .map(|s| CString::new(s).unwrap().into_raw());
+    let tool_choice_json = chat_request.tool_choice.as_ref()
+        .and_then(|tc| serde_json::to_string(tc).ok())
+        .map(|s| CString::new(s).unwrap().into_raw());
+    let stop_json = chat_request.stop.as_ref()
+        .and_then(|s| serde_json::to_string(s).ok())
+        .map(|s| CString::new(s).unwrap().into_raw());
+    let stop_token_ids_json = chat_request.stop_token_ids.as_ref()
+        .and_then(|ids| serde_json::to_string(ids).ok())
+        .map(|s| CString::new(s).unwrap().into_raw());
+
+    // Create tokenizer handle for converter (we'll create a temporary one)
+    let tokenizer_handle = Box::into_raw(Box::new(TokenizerHandle {
+        tokenizer: Arc::clone(&tokenizer),
+    }));
+
+    let converter = sgl_grpc_response_converter_create(
+        tokenizer_handle,
+        CString::new(chat_request.model.clone()).unwrap().as_ptr(),
+        CString::new(request_id.clone()).unwrap().as_ptr(),
+        tools_json.unwrap_or(ptr::null_mut()),
+        tool_choice_json.unwrap_or(ptr::null_mut()),
+        stop_json.unwrap_or(ptr::null_mut()),
+        stop_token_ids_json.unwrap_or(ptr::null_mut()),
+        if chat_request.skip_special_tokens { 1 } else { 0 },
+        error_out,
+    );
+
+    // Free temporary tokenizer handle (converter now owns the tokenizer)
+    let _ = Box::from_raw(tokenizer_handle);
+
+    if converter.is_null() {
+        return SglErrorCode::MemoryError;
+    }
+
+    // Clean up temporary CStrings
+    if let Some(ptr) = tools_json {
+        let _ = CString::from_raw(ptr);
+    }
+    if let Some(ptr) = tool_choice_json {
+        let _ = CString::from_raw(ptr);
+    }
+    if let Some(ptr) = stop_json {
+        let _ = CString::from_raw(ptr);
+    }
+    if let Some(ptr) = stop_token_ids_json {
+        let _ = CString::from_raw(ptr);
+    }
+
+    // Create converter handle and set initial_prompt_tokens immediately
+    let mut converter_handle = *Box::from_raw(converter);
+    converter_handle.initial_prompt_tokens = Some(prompt_tokens);
+
+    // Create stream handle with prompt_tokens
+    *stream_handle_out = Box::into_raw(Box::new(SglangStreamHandle {
+        stream: Arc::new(tokio::sync::Mutex::new(stream)),
+        converter: Arc::new(tokio::sync::Mutex::new(converter_handle)),
+        client: Arc::clone(&client),
+        prompt_tokens,
+    }));
+
+    SglErrorCode::Success
+}

third_party/sglang/sgl-model-gateway/bindings/golang/src/error.rs

@@ -0,0 +1,50 @@
+//! Error handling for FFI functions
+
+use std::ffi::CString;
+use std::os::raw::c_char;
+use std::ptr;
+
+/// Error codes returned by FFI functions
+#[repr(C)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum SglErrorCode {
+    Success = 0,
+    InvalidArgument = 1,
+    TokenizationError = 2,
+    ParsingError = 3,
+    MemoryError = 4,
+    UnknownError = 99,
+}
+
+/// Helper to set error message in FFI output parameter
+pub fn set_error_message(error_out: *mut *mut c_char, message: &str) {
+    unsafe {
+        if !error_out.is_null() {
+            if let Ok(cstr) = CString::new(message) {
+                *error_out = cstr.into_raw();
+            } else {
+                *error_out = ptr::null_mut();
+            }
+        }
+    }
+}
+
+/// Helper to set error message from format string
+pub fn set_error_message_fmt(error_out: *mut *mut c_char, fmt: std::fmt::Arguments) {
+    if !error_out.is_null() {
+        let msg = format!("{}", fmt);
+        set_error_message(error_out, &msg);
+    }
+}
+
+/// Helper to clear error message
+pub fn clear_error_message(error_out: *mut *mut c_char) {
+    unsafe {
+        if !error_out.is_null() {
+            *error_out = ptr::null_mut();
+        }
+    }
+}
+
+// Helper functions for error handling
+// Note: Some helper functions are kept for potential future use

third_party/sglang/sgl-model-gateway/bindings/golang/src/grpc_converter.rs

@@ -0,0 +1,758 @@
+//! gRPC response converter FFI functions
+
+use std::ffi::{CStr, CString};
+use std::os::raw::{c_char, c_int};
+use std::ptr;
+use std::sync::Arc;
+use std::collections::HashMap;
+use serde_json::Value;
+use tokio::runtime::Runtime;
+use once_cell::sync::Lazy;
+
+use smg::tokenizer::traits::Tokenizer;
+use smg::tokenizer::stream::DecodeStream;
+use smg::tool_parser::ToolParser;
+use smg::protocols::common::{Tool, ToolChoice, ToolChoiceValue, ToolCallDelta, FunctionCallDelta, Usage, StringOrArray};
+use smg::tokenizer::stop::StopSequenceDecoder;
+use smg_grpc_client::sglang_proto as proto;
+
+use super::error::{SglErrorCode, set_error_message, clear_error_message};
+use super::tokenizer::TokenizerHandle;
+use super::utils::generate_tool_call_id;
+
+/// Global parser factory (initialized once)
+// Use the re-exported ParserFactory from tool_parser module
+static PARSER_FACTORY: Lazy<smg::tool_parser::ParserFactory> = Lazy::new(|| {
+    // ParserFactory is re-exported from tool_parser::factory, so we can use it directly
+    smg::tool_parser::ParserFactory::default()
+});
+
+/// Global tokio runtime for async operations
+static RUNTIME: Lazy<Runtime> = Lazy::new(|| {
+    Runtime::new().expect("Failed to create tokio runtime for gRPC converter FFI")
+});
+
+/// Handle for gRPC response converter (maintains state for streaming)
+#[repr(C)]
+pub struct GrpcResponseConverterHandle {
+    pub(crate) tokenizer: Arc<dyn Tokenizer>,
+    pub(crate) tool_parser: Option<Arc<tokio::sync::Mutex<Box<dyn ToolParser>>>>,
+    pub(crate) stop_decoder: Option<Arc<tokio::sync::Mutex<StopSequenceDecoder>>>,
+    pub(crate) model: String,
+    pub(crate) request_id: String,
+    pub(crate) created: u64,
+    pub(crate) system_fingerprint: Option<String>,
+    pub(crate) tools: Option<Vec<Tool>>,
+    pub(crate) tool_choice: Option<ToolChoice>,
+    pub(crate) history_tool_calls_count: usize,
+    pub(crate) stream_buffers: HashMap<u32, String>, // Per-index text buffers
+    pub(crate) decode_streams: HashMap<u32, DecodeStream>, // Per-index incremental decoders
+    pub(crate) has_tool_calls: HashMap<u32, bool>, // Track if tool calls were emitted
+    pub(crate) is_first_chunk: HashMap<u32, bool>, // Track first chunk per index
+    pub(crate) prompt_tokens: HashMap<u32, i32>, // Track prompt tokens per index (from chunks)
+    pub(crate) completion_tokens: HashMap<u32, i32>, // Track completion tokens per index (cumulative)
+    pub(crate) initial_prompt_tokens: Option<i32>, // Initial prompt tokens from request (if available)
+    pub(crate) skip_special_tokens: bool, // Whether to skip special tokens when decoding
+}
+
+/// Create a gRPC response converter handle
+///
+/// # Arguments
+/// * `tokenizer_handle` - Tokenizer handle (must be valid)
+/// * `model` - Model name
+/// * `request_id` - Request ID
+/// * `tools_json` - Optional JSON array of tools
+/// * `tool_choice_json` - Optional JSON object for tool_choice
+/// * `stop` - Optional stop sequences (JSON array)
+/// * `stop_token_ids` - Optional stop token IDs (JSON array)
+/// * `skip_special_tokens` - Whether to skip special tokens
+/// * `error_out` - Optional pointer to receive error message
+///
+/// # Returns
+/// * Pointer to GrpcResponseConverterHandle on success, null on failure
+#[no_mangle]
+pub unsafe extern "C" fn sgl_grpc_response_converter_create(
+    tokenizer_handle: *mut TokenizerHandle,
+    model: *const c_char,
+    request_id: *const c_char,
+    tools_json: *const c_char,
+    tool_choice_json: *const c_char,
+    stop: *const c_char,
+    stop_token_ids: *const c_char,
+    skip_special_tokens: c_int,
+    error_out: *mut *mut c_char,
+) -> *mut GrpcResponseConverterHandle {
+    if tokenizer_handle.is_null() || model.is_null() || request_id.is_null() {
+        set_error_message(error_out, "Invalid arguments: null pointer");
+        return ptr::null_mut();
+    }
+
+    let model_str = match CStr::from_ptr(model).to_str() {
+        Ok(s) => s,
+        Err(_) => {
+            set_error_message(error_out, "Invalid UTF-8 in model");
+            return ptr::null_mut();
+        }
+    };
+
+    let request_id_str = match CStr::from_ptr(request_id).to_str() {
+        Ok(s) => s,
+        Err(_) => {
+            set_error_message(error_out, "Invalid UTF-8 in request_id");
+            return ptr::null_mut();
+        }
+    };
+
+    let handle_ref = &*tokenizer_handle;
+    let tokenizer = Arc::clone(&handle_ref.tokenizer);
+
+    // Parse tools if provided
+    let tools: Option<Vec<Tool>> = if !tools_json.is_null() {
+        match CStr::from_ptr(tools_json).to_str() {
+            Ok(s) => serde_json::from_str::<Vec<Tool>>(s).ok(),
+            Err(_) => None,
+        }
+    } else {
+        None
+    };
+
+    // Parse tool_choice if provided
+    let tool_choice: Option<ToolChoice> = if !tool_choice_json.is_null() {
+        match CStr::from_ptr(tool_choice_json).to_str() {
+            Ok(s) => serde_json::from_str::<ToolChoice>(s).ok(),
+            Err(_) => None,
+        }
+    } else {
+        None
+    };
+
+    // Parse stop sequences
+    let stop: Option<StringOrArray> = if !stop.is_null() {
+        let stop_str = match CStr::from_ptr(stop).to_str() {
+            Ok(s) => s,
+            Err(_) => return ptr::null_mut(),
+        };
+        serde_json::from_str::<StringOrArray>(stop_str).ok()
+    } else {
+        None
+    };
+
+    // Parse stop token IDs
+    let stop_token_ids: Option<Vec<u32>> = if !stop_token_ids.is_null() {
+        let ids_str = match CStr::from_ptr(stop_token_ids).to_str() {
+            Ok(s) => s,
+            Err(_) => return ptr::null_mut(),
+        };
+        serde_json::from_str::<Vec<u32>>(ids_str).ok()
+    } else {
+        None
+    };
+
+    // Create stop decoder if needed
+    let stop_decoder = if stop.is_some() || stop_token_ids.is_some() {
+        Some(Arc::new(tokio::sync::Mutex::new(
+            smg::routers::grpc::utils::create_stop_decoder(
+                &tokenizer,
+                stop.as_ref(),
+                stop_token_ids.as_ref(),
+                skip_special_tokens != 0,
+                false, // no_stop_trim
+            ),
+        )))
+    } else {
+        None
+    };
+
+    // Create tool parser if tools are provided
+    let tool_parser = if tools.is_some() {
+        PARSER_FACTORY.registry().create_for_model(model_str)
+            .map(|p| Arc::new(tokio::sync::Mutex::new(p)))
+    } else {
+        None
+    };
+
+    // Get system fingerprint from model (simplified)
+    let system_fingerprint = Some("fp_placeholder".to_string()); // TODO: Get actual fingerprint
+
+    Box::into_raw(Box::new(GrpcResponseConverterHandle {
+        tokenizer,
+        tool_parser,
+        stop_decoder,
+        model: model_str.to_string(),
+        request_id: request_id_str.to_string(),
+        created: std::time::SystemTime::now()
+            .duration_since(std::time::UNIX_EPOCH)
+            .unwrap()
+            .as_secs(),
+        system_fingerprint,
+        tools,
+        tool_choice,
+        history_tool_calls_count: 0,
+        stream_buffers: HashMap::new(),
+        decode_streams: HashMap::new(),
+        has_tool_calls: HashMap::new(),
+        is_first_chunk: HashMap::new(),
+        prompt_tokens: HashMap::new(),
+        completion_tokens: HashMap::new(),
+        initial_prompt_tokens: None, // Will be set from stream handle
+        skip_special_tokens: skip_special_tokens != 0,
+    }))
+}
+
+/// Convert a gRPC GenerateResponse chunk to OpenAI format
+///
+/// # Arguments
+/// * `handle` - Converter handle
+/// * `response_json` - JSON string of proto.GenerateResponse
+/// * `result_json_out` - Pointer to receive OpenAI format JSON (must be freed with sgl_free_string)
+/// * `error_out` - Optional pointer to receive error message
+///
+/// # Returns
+/// * SglErrorCode::Success on success, error code on failure
+#[no_mangle]
+pub unsafe extern "C" fn sgl_grpc_response_converter_convert_chunk(
+    handle: *mut GrpcResponseConverterHandle,
+    response_json: *const c_char,
+    result_json_out: *mut *mut c_char,
+    error_out: *mut *mut c_char,
+) -> SglErrorCode {
+    if handle.is_null() || response_json.is_null() || result_json_out.is_null() {
+        set_error_message(error_out, "Invalid arguments: null pointer");
+        return SglErrorCode::InvalidArgument;
+    }
+
+    let response_str = match CStr::from_ptr(response_json).to_str() {
+        Ok(s) => s,
+        Err(_) => {
+            set_error_message(error_out, "Invalid UTF-8 in response_json");
+            return SglErrorCode::InvalidArgument;
+        }
+    };
+
+    // Parse proto.GenerateResponse from JSON
+    let json_value: Value = match serde_json::from_str(response_str) {
+        Ok(v) => v,
+        Err(e) => {
+            set_error_message(error_out, &format!("Failed to parse response JSON: {}", e));
+            return SglErrorCode::ParsingError;
+        }
+    };
+
+    // Build proto::GenerateResponse from JSON value
+    let mut proto_response = proto::GenerateResponse {
+        request_id: json_value.get("request_id")
+            .and_then(|v| v.as_str())
+            .unwrap_or("")
+            .to_string(),
+        response: None,
+    };
+
+    // Parse the response oneof field
+    if let Some(chunk_json) = json_value.get("chunk") {
+        let chunk = proto::GenerateStreamChunk {
+            token_ids: chunk_json.get("token_ids")
+                .and_then(|v| v.as_array())
+                .map(|arr| arr.iter().filter_map(|v| v.as_u64().map(|n| n as u32)).collect())
+                .unwrap_or_default(),
+            prompt_tokens: chunk_json.get("prompt_tokens")
+                .and_then(|v| v.as_i64())
+                .map(|n| n as i32)
+                .unwrap_or(0),
+            completion_tokens: chunk_json.get("completion_tokens")
+                .and_then(|v| v.as_i64())
+                .map(|n| n as i32)
+                .unwrap_or(0),
+            cached_tokens: chunk_json.get("cached_tokens")
+                .and_then(|v| v.as_i64())
+                .map(|n| n as i32)
+                .unwrap_or(0),
+            output_logprobs: None,
+            hidden_states: vec![],
+            input_logprobs: None,
+            index: 0,
+        };
+        proto_response.response = Some(proto::generate_response::Response::Chunk(chunk));
+    } else if let Some(complete_json) = json_value.get("complete") {
+        let complete = proto::GenerateComplete {
+            output_ids: complete_json.get("output_ids")
+                .and_then(|v| v.as_array())
+                .map(|arr| arr.iter().filter_map(|v| v.as_u64().map(|n| n as u32)).collect())
+                .unwrap_or_default(),
+            finish_reason: complete_json.get("finish_reason")
+                .and_then(|v| v.as_str())
+                .unwrap_or("")
+                .to_string(),
+            prompt_tokens: complete_json.get("prompt_tokens")
+                .and_then(|v| v.as_i64())
+                .map(|n| n as i32)
+                .unwrap_or(0),
+            completion_tokens: complete_json.get("completion_tokens")
+                .and_then(|v| v.as_i64())
+                .map(|n| n as i32)
+                .unwrap_or(0),
+            cached_tokens: complete_json.get("cached_tokens")
+                .and_then(|v| v.as_i64())
+                .map(|n| n as i32)
+                .unwrap_or(0),
+            output_logprobs: None,
+            all_hidden_states: vec![],
+            input_logprobs: None,
+            matched_stop: None,
+            index: 0,
+        };
+        proto_response.response = Some(proto::generate_response::Response::Complete(complete));
+    } else if let Some(error_json) = json_value.get("error") {
+        let error = proto::GenerateError {
+            message: error_json.get("message")
+                .and_then(|v| v.as_str())
+                .unwrap_or("")
+                .to_string(),
+            http_status_code: error_json.get("http_status_code")
+                .and_then(|v| v.as_str())
+                .unwrap_or("500")
+                .to_string(),
+            details: error_json.get("details")
+                .and_then(|v| v.as_str())
+                .unwrap_or("")
+                .to_string(),
+        };
+        proto_response.response = Some(proto::generate_response::Response::Error(error));
+    } else {
+        set_error_message(error_out, "Response JSON must contain 'chunk', 'complete', or 'error' field");
+        return SglErrorCode::ParsingError;
+    }
+
+    let handle_ref = &mut *handle;
+    let tokenizer = Arc::clone(&handle_ref.tokenizer);
+    let model = handle_ref.model.clone();
+    let request_id = handle_ref.request_id.clone();
+    let created = handle_ref.created;
+    let system_fingerprint = handle_ref.system_fingerprint.clone();
+
+    // Use tokio runtime to run async code
+    let result = RUNTIME.block_on(async {
+        convert_proto_chunk_to_openai(
+            proto_response,
+            handle_ref,
+            &tokenizer,
+            &model,
+            &request_id,
+            created,
+            system_fingerprint.as_deref(),
+        )
+        .await
+    });
+
+    match result {
+        Ok(Some(openai_response)) => {
+            // Serialize to JSON
+            let result_str = match serde_json::to_string(&openai_response) {
+                Ok(s) => s,
+                Err(e) => {
+                    set_error_message(error_out, &format!("Failed to serialize response: {}", e));
+                    return SglErrorCode::ParsingError;
+                }
+            };
+
+            let result_cstr = match CString::new(result_str) {
+                Ok(s) => s,
+                Err(e) => {
+                    set_error_message(error_out, &format!("Failed to create result string: {}", e));
+                    return SglErrorCode::MemoryError;
+                }
+            };
+
+            *result_json_out = result_cstr.into_raw();
+            clear_error_message(error_out);
+            SglErrorCode::Success
+        }
+        Ok(None) => {
+            // No response to send (e.g., empty chunk)
+            let empty = CString::new("").unwrap();
+            *result_json_out = empty.into_raw();
+            clear_error_message(error_out);
+            SglErrorCode::Success
+        }
+        Err(e) => {
+            set_error_message(error_out, &format!("Conversion error: {}", e));
+            SglErrorCode::ParsingError
+        }
+    }
+}
+
+/// Helper function to convert proto chunk to OpenAI format
+pub(crate) async fn convert_proto_chunk_to_openai(
+    proto_response: proto::GenerateResponse,
+    handle: &mut GrpcResponseConverterHandle,
+    tokenizer: &Arc<dyn Tokenizer>,
+    model: &str,
+    request_id: &str,
+    created: u64,
+    system_fingerprint: Option<&str>,
+) -> Result<Option<smg::protocols::chat::ChatCompletionStreamResponse>, String> {
+    use smg_grpc_client::sglang_proto::generate_response::Response::*;
+    use smg::protocols::chat::{ChatCompletionStreamResponse, ChatMessageDelta, ChatStreamChoice};
+
+    match proto_response.response {
+        Some(Chunk(chunk)) => {
+            let index = chunk.index;
+
+            // Mark as not first chunk if we've seen this index before
+            let is_first = handle.is_first_chunk.entry(index).or_insert(true);
+            let first_chunk = *is_first;
+            *is_first = false;
+
+            // Track token counts from chunks (cumulative values from proto)
+            // These are cumulative values, so we always use the latest value
+            // For prompt_tokens, if chunk value is 0, preserve existing value or use initial_prompt_tokens
+            // This prevents overwriting valid prompt_tokens with 0
+            if chunk.prompt_tokens > 0 {
+                handle.prompt_tokens.insert(index, chunk.prompt_tokens);
+            } else {
+                // If chunk.prompt_tokens is 0, try to preserve existing value or use initial_prompt_tokens
+                if !handle.prompt_tokens.contains_key(&index) {
+                    // No existing value, try to use initial_prompt_tokens
+                    if let Some(initial_prompt) = handle.initial_prompt_tokens {
+                        handle.prompt_tokens.insert(index, initial_prompt);
+                    }
+                }
+                // If existing value exists, keep it (don't overwrite with 0)
+            }
+            // For completion_tokens, always update (even if 0) as it's cumulative
+            handle.completion_tokens.insert(index, chunk.completion_tokens);
+
+            // Process tokens through stop decoder if available, otherwise use incremental decoder
+            let chunk_text = if let Some(ref stop_decoder) = handle.stop_decoder {
+                let mut decoder_guard = stop_decoder.lock().await;
+                let mut text = String::new();
+                for &token_id in &chunk.token_ids {
+                    match decoder_guard.process_token(token_id).unwrap_or_else(|_| {
+                        smg::tokenizer::stop::SequenceDecoderOutput::Held
+                    }) {
+                        smg::tokenizer::stop::SequenceDecoderOutput::Text(t) => {
+                            text.push_str(&t);
+                        }
+                        smg::tokenizer::stop::SequenceDecoderOutput::StoppedWithText(t) => {
+                            text.push_str(&t);
+                            break;
+                        }
+                        smg::tokenizer::stop::SequenceDecoderOutput::Stopped => {
+                            break;
+                        }
+                        smg::tokenizer::stop::SequenceDecoderOutput::Held => {}
+                    }
+                }
+                text
+            } else {
+                // Use incremental decoder to handle multi-byte character boundaries
+                let decode_stream = handle.decode_streams.entry(index).or_insert_with(|| {
+                    DecodeStream::new(
+                        Arc::clone(&tokenizer),
+                        &[], // No prompt tokens for completion
+                        handle.skip_special_tokens,
+                    )
+                });
+
+                // Process tokens incrementally
+                let mut text_parts = Vec::new();
+                for &token_id in &chunk.token_ids {
+                    if let Ok(Some(text)) = decode_stream.step(token_id) {
+                        text_parts.push(text);
+                    }
+                }
+                text_parts.join("")
+            };
+
+            if chunk_text.is_empty() {
+                return Ok(None);
+            }
+
+            // Send first chunk with role
+            if first_chunk {
+                let first_response = ChatCompletionStreamResponse {
+                    id: request_id.to_string(),
+                    object: "chat.completion.chunk".to_string(),
+                    created,
+                    model: model.to_string(),
+                    system_fingerprint: system_fingerprint.map(|s| s.to_string()),
+                    choices: vec![ChatStreamChoice {
+                        index,
+                        delta: ChatMessageDelta {
+                            role: Some("assistant".to_string()),
+                            content: None,
+                            tool_calls: None,
+                            reasoning_content: None,
+                        },
+                        logprobs: None,
+                        finish_reason: None,
+                        matched_stop: None,
+                    }],
+                    usage: None,
+                };
+                return Ok(Some(first_response));
+            }
+
+            // Update stream buffer
+            let stream_buffer = handle.stream_buffers.entry(index).or_default();
+            stream_buffer.push_str(&chunk_text);
+
+            // Handle tool calls if tools are provided
+            if let (Some(ref tools), Some(ref tool_parser)) = (handle.tools.as_ref(), handle.tool_parser.as_ref()) {
+                let tool_choice_enabled = !matches!(
+                    handle.tool_choice,
+                    Some(ToolChoice::Value(ToolChoiceValue::None))
+                );
+
+                if tool_choice_enabled {
+                    let mut parser_guard = tool_parser.lock().await;
+                    match parser_guard.parse_incremental(&chunk_text, tools).await {
+                        Ok(streaming_result) => {
+                            if !streaming_result.calls.is_empty() {
+                                handle.has_tool_calls.insert(index, true);
+                                // Convert tool call items to OpenAI format
+                                let tool_call_deltas: Vec<_> = streaming_result
+                                    .calls
+                                    .into_iter()
+                                    .map(|item| {
+                                        let id = if let Some(ref name) = item.name {
+                                            generate_tool_call_id(
+                                                model,
+                                                name,
+                                                item.tool_index,
+                                                handle.history_tool_calls_count,
+                                            )
+                                        } else {
+                                            format!("call_{}", item.tool_index)
+                                        };
+
+                                        ToolCallDelta {
+                                            index: item.tool_index as u32,
+                                            id: Some(id),
+                                            tool_type: if item.name.is_some() {
+                                                Some("function".to_string())
+                                            } else {
+                                                None
+                                            },
+                                            function: Some(FunctionCallDelta {
+                                                name: item.name,
+                                                arguments: if !item.parameters.is_empty() {
+                                                    Some(item.parameters)
+                                                } else {
+                                                    None
+                                                },
+                                            }),
+                                        }
+                                    })
+                                    .collect();
+
+                                let tool_response = ChatCompletionStreamResponse {
+                                    id: request_id.to_string(),
+                                    object: "chat.completion.chunk".to_string(),
+                                    created,
+                                    model: model.to_string(),
+                                    system_fingerprint: system_fingerprint.map(|s| s.to_string()),
+                                    choices: vec![ChatStreamChoice {
+                                        index,
+                                        delta: ChatMessageDelta {
+                                            role: Some("assistant".to_string()),
+                                            content: None,
+                                            tool_calls: Some(tool_call_deltas),
+                                            reasoning_content: None,
+                                        },
+                                        logprobs: None,
+                                        finish_reason: None,
+                                        matched_stop: None,
+                                    }],
+                                    usage: None,
+                                };
+                                return Ok(Some(tool_response));
+                            }
+                        }
+                        Err(e) => {
+                            // Log error but continue with regular content
+                            tracing::warn!("Tool parser error: {}", e);
+                        }
+                    }
+                }
+            }
+
+            // Regular content emission
+            let content_response = ChatCompletionStreamResponse {
+                id: request_id.to_string(),
+                object: "chat.completion.chunk".to_string(),
+                created,
+                model: model.to_string(),
+                system_fingerprint: system_fingerprint.map(|s| s.to_string()),
+                choices: vec![ChatStreamChoice {
+                    index,
+                    delta: ChatMessageDelta {
+                        role: Some("assistant".to_string()),
+                        content: Some(chunk_text),
+                        tool_calls: None,
+                        reasoning_content: None,
+                    },
+                    logprobs: None,
+                    finish_reason: None,
+                    matched_stop: None,
+                }],
+                usage: None,
+            };
+
+            Ok(Some(content_response))
+        }
+        Some(Complete(complete)) => {
+            let index = complete.index;
+
+            // Flush any remaining text
+            // Flush any remaining text from decode stream
+            let mut final_text = handle.stream_buffers.remove(&index).unwrap_or_default();
+            if let Some(ref mut decode_stream) = handle.decode_streams.get_mut(&index) {
+                if let Ok(Some(remaining)) = decode_stream.flush() {
+                    final_text.push_str(&remaining);
+                }
+            }
+            handle.decode_streams.remove(&index);
+
+            // Determine finish reason - ensure it's never empty
+            // If finish_reason is empty, try to infer from other fields or use default
+            let finish_reason = if handle.has_tool_calls.get(&index).copied().unwrap_or(false)
+                && (complete.finish_reason == "stop" || complete.finish_reason.is_empty())
+            {
+                "tool_calls".to_string()
+            } else if complete.finish_reason.is_empty() || complete.finish_reason.trim().is_empty() {
+                // If finish_reason is empty, try to infer from completion_tokens or use default
+                if complete.completion_tokens > 0 {
+                    // If we have completion tokens, likely stopped normally
+                    "stop".to_string()
+                } else if !complete.output_ids.is_empty() {
+                    // If we have output_ids, likely stopped normally
+                    "stop".to_string()
+                } else {
+                    // Default fallback - always ensure we have a value
+                    "stop".to_string()
+                }
+            } else {
+                complete.finish_reason.clone()
+            };
+
+            // Ensure finish_reason is never empty (defensive check)
+            let finish_reason = if finish_reason.is_empty() || finish_reason.trim().is_empty() {
+                "stop".to_string()
+            } else {
+                finish_reason
+            };
+
+            // Extract matched_stop
+            let matched_stop = match &complete.matched_stop {
+                Some(proto::generate_complete::MatchedStop::MatchedTokenId(token_id)) => {
+                    Some(Value::Number(serde_json::Number::from(*token_id)))
+                }
+                Some(proto::generate_complete::MatchedStop::MatchedStopStr(stop_str)) => {
+                    Some(Value::String(stop_str.clone()))
+                }
+                None => None,
+            };
+
+            // Build usage - prefer values from complete message, but fallback to accumulated values from chunks
+            // Complete message should have the final values, but sometimes they might be 0 or missing
+            // Always use the latest cumulative value from chunks if available, otherwise use complete message value
+            let mut prompt_tokens = handle.prompt_tokens.get(&index)
+                .copied()
+                .filter(|&v| v > 0)
+                .unwrap_or(complete.prompt_tokens);
+            let mut completion_tokens = handle.completion_tokens.get(&index)
+                .copied()
+                .filter(|&v| v > 0)
+                .unwrap_or(complete.completion_tokens);
+
+            // Always try to use initial_prompt_tokens if prompt_tokens is 0 or missing
+            // This is the most reliable source for prompt tokens since we calculate it from the request
+            if prompt_tokens == 0 {
+                if let Some(initial_prompt) = handle.initial_prompt_tokens {
+                    prompt_tokens = initial_prompt;
+                }
+            }
+
+            // If completion_tokens is 0, try to infer from output_ids or accumulated chunks
+            if completion_tokens == 0 {
+                // Try to use completion_tokens from complete message even if 0
+                // Or calculate from output_ids
+                if complete.completion_tokens > 0 {
+                    completion_tokens = complete.completion_tokens;
+                } else if !complete.output_ids.is_empty() {
+                    completion_tokens = complete.output_ids.len() as i32;
+                } else if let Some(&last_completion) = handle.completion_tokens.get(&index) {
+                    completion_tokens = last_completion;
+                }
+            }
+
+            // Final fallback: if both are still 0, try to use initial_prompt_tokens for prompt
+            // and calculate completion from output_ids
+            if prompt_tokens == 0 && completion_tokens == 0 {
+                // Try to infer from output_ids if available
+                let output_ids_len = complete.output_ids.len() as i32;
+                if output_ids_len > 0 {
+                    completion_tokens = output_ids_len;
+                    // Always try to use initial_prompt_tokens for prompt
+                    if let Some(initial_prompt) = handle.initial_prompt_tokens {
+                        prompt_tokens = initial_prompt;
+                    }
+                }
+            }
+
+            // Final defensive check: ensure prompt_tokens is set if we have initial_prompt_tokens
+            if prompt_tokens == 0 {
+                if let Some(initial_prompt) = handle.initial_prompt_tokens {
+                    prompt_tokens = initial_prompt;
+                }
+            }
+
+            // Always create usage, even if values are 0 (defensive)
+            let usage = Some(Usage {
+                prompt_tokens: prompt_tokens.max(0) as u32,
+                completion_tokens: completion_tokens.max(0) as u32,
+                total_tokens: (prompt_tokens.max(0) + completion_tokens.max(0)) as u32,
+                completion_tokens_details: None,
+            });
+
+            let finish_response = ChatCompletionStreamResponse {
+                id: request_id.to_string(),
+                object: "chat.completion.chunk".to_string(),
+                created,
+                model: model.to_string(),
+                system_fingerprint: system_fingerprint.map(|s| s.to_string()),
+                choices: vec![ChatStreamChoice {
+                    index,
+                    delta: ChatMessageDelta {
+                        role: Some("assistant".to_string()),
+                        content: if !final_text.is_empty() {
+                            Some(final_text)
+                        } else {
+                            None
+                        },
+                        tool_calls: None,
+                        reasoning_content: None,
+                    },
+                    logprobs: None,
+                    finish_reason: Some(finish_reason),
+                    matched_stop,
+                }],
+                usage,
+            };
+
+            Ok(Some(finish_response))
+        }
+        Some(Error(error)) => {
+            Err(format!("Server error: {} (status: {})", error.message, error.http_status_code))
+        }
+        None => Ok(None),
+    }
+}
+
+/// Free a gRPC response converter handle
+#[no_mangle]
+pub unsafe extern "C" fn sgl_grpc_response_converter_free(handle: *mut GrpcResponseConverterHandle) {
+    if !handle.is_null() {
+        let _ = Box::from_raw(handle);
+    }
+}

third_party/sglang/sgl-model-gateway/bindings/golang/src/lib.rs

@@ -0,0 +1,103 @@
+//! FFI module for exposing sgl-model-gateway preprocessing and postprocessing functions
+//! to C-compatible languages (e.g., Golang via cgo)
+//!
+//! This module provides C-compatible function signatures for:
+//! - Tokenizer operations (encode, decode, chat template)
+//! - Tool parser operations (parse tool calls)
+//! - Tool constraint generation
+//! - gRPC client SDK (complete request-response flow)
+//!
+//! # Safety
+//! All functions marked with `#[no_mangle]` and `extern "C"` must be called
+//! with valid pointers and follow the documented memory management rules.
+
+// Re-export error types
+pub use error::{SglErrorCode, set_error_message, set_error_message_fmt, clear_error_message};
+
+// Re-export memory management functions
+pub use memory::{sgl_free_string, sgl_free_token_ids};
+
+// Re-export tokenizer functions
+pub use tokenizer::{
+    TokenizerHandle,
+    sgl_tokenizer_create_from_file,
+    sgl_tokenizer_encode,
+    sgl_tokenizer_apply_chat_template,
+    sgl_tokenizer_apply_chat_template_with_tools,
+    sgl_tokenizer_decode,
+    sgl_tokenizer_free,
+};
+
+// Re-export tool parser functions
+pub use tool_parser::{
+    ToolParserHandle,
+    sgl_tool_parser_create,
+    sgl_tool_parser_parse_complete,
+    sgl_tool_parser_parse_incremental,
+    sgl_tool_parser_reset,
+    sgl_tool_parser_free,
+};
+
+// Re-export gRPC converter functions
+pub use grpc_converter::{
+    GrpcResponseConverterHandle,
+    sgl_grpc_response_converter_create,
+    sgl_grpc_response_converter_convert_chunk,
+    sgl_grpc_response_converter_free,
+};
+
+// Re-export client SDK functions
+pub use client::{
+    SglangClientHandle,
+    sgl_client_create,
+    sgl_client_free,
+};
+
+// Re-export stream functions
+pub use stream::{
+    SglangStreamHandle,
+    sgl_stream_read_next,
+    sgl_stream_free,
+};
+
+// Re-export client stream function (defined in client.rs but used by stream)
+pub use client::sgl_client_chat_completion_stream;
+
+// Re-export preprocessor functions
+pub use preprocessor::{
+    sgl_preprocess_chat_request,
+    sgl_preprocess_chat_request_with_tokenizer,
+    sgl_preprocessed_request_free,
+};
+
+// Re-export postprocessor functions
+pub use postprocessor::{
+    sgl_postprocess_stream_chunk,
+    sgl_postprocess_stream_chunks_batch,
+};
+
+// Re-export utility functions
+pub use utils::sgl_generate_tool_constraints;
+
+// Sub-modules
+mod error;
+mod memory;
+mod tokenizer;
+mod tool_parser;
+mod grpc_converter;
+mod client;
+mod stream;
+mod utils;
+mod preprocessor;
+mod postprocessor;
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_error_codes() {
+        assert_eq!(SglErrorCode::Success as i32, 0);
+        assert_eq!(SglErrorCode::InvalidArgument as i32, 1);
+    }
+}

third_party/sglang/sgl-model-gateway/bindings/golang/src/memory.rs

@@ -0,0 +1,28 @@
+//! Memory management for FFI functions
+
+use std::ffi::CString;
+use std::os::raw::c_char;
+
+/// Free a C string allocated by Rust
+///
+/// # Safety
+/// This function must only be called with pointers returned by other FFI functions.
+/// Calling with arbitrary pointers or multiple times on the same pointer is undefined behavior.
+#[no_mangle]
+pub unsafe extern "C" fn sgl_free_string(s: *mut c_char) {
+    if !s.is_null() {
+        let _ = CString::from_raw(s);
+    }
+}
+
+/// Free token IDs array allocated by Rust
+///
+/// # Safety
+/// This function must only be called with pointers returned by `sgl_tokenizer_encode`.
+/// The `count` parameter must match the length of the array.
+#[no_mangle]
+pub unsafe extern "C" fn sgl_free_token_ids(ptr: *mut u32, count: usize) {
+    if !ptr.is_null() && count > 0 {
+        let _ = Vec::from_raw_parts(ptr, count, count);
+    }
+}

third_party/sglang/sgl-model-gateway/bindings/golang/src/postprocessor.rs

@@ -0,0 +1,465 @@
+//! Postprocessing FFI functions for gRPC stream chunks
+//!
+//! This module provides C-compatible functions for postprocessing gRPC stream chunks:
+//! - Parse tool calls from model output
+//! - Convert proto format to OpenAI format
+//! - Handle reasoning content parsing
+//!
+//! These functions are designed to be called for each stream chunk, but can be optimized
+//! with batching in the future.
+
+use std::ffi::{CStr, CString};
+use std::os::raw::{c_char, c_int};
+use std::ptr;
+use std::sync::Arc;
+use serde_json::Value;
+
+use smg_grpc_client::sglang_proto as proto;
+
+use super::error::{SglErrorCode, set_error_message};
+use super::grpc_converter::GrpcResponseConverterHandle;
+
+use tokio::runtime::Runtime;
+use once_cell::sync::Lazy;
+
+/// Global tokio runtime for async operations
+static RUNTIME: Lazy<Runtime> = Lazy::new(|| {
+    Runtime::new().expect("Failed to create tokio runtime for postprocessor FFI")
+});
+
+/// Postprocess a gRPC stream chunk to OpenAI format
+///
+/// This function:
+/// 1. Parses the proto chunk from JSON
+/// 2. Converts it to OpenAI format using the converter handle
+/// 3. Returns the OpenAI format JSON
+///
+/// # Arguments
+/// * `converter_handle` - Converter handle (created with sgl_grpc_response_converter_create)
+/// * `proto_chunk_json` - JSON string of proto.GenerateResponse
+/// * `openai_json_out` - Pointer to receive OpenAI format JSON (must be freed with sgl_free_string)
+/// * `is_done_out` - Pointer to receive is_done flag (1 if stream is complete, 0 otherwise)
+/// * `error_out` - Optional pointer to receive error message
+///
+/// # Returns
+/// * SglErrorCode::Success on success, error code on failure
+#[no_mangle]
+pub unsafe extern "C" fn sgl_postprocess_stream_chunk(
+    converter_handle: *mut GrpcResponseConverterHandle,
+    proto_chunk_json: *const c_char,
+    openai_json_out: *mut *mut c_char,
+    is_done_out: *mut c_int,
+    error_out: *mut *mut c_char,
+) -> SglErrorCode {
+    if converter_handle.is_null()
+        || proto_chunk_json.is_null()
+        || openai_json_out.is_null()
+        || is_done_out.is_null()
+    {
+        set_error_message(error_out, "Invalid arguments: null pointer");
+        return SglErrorCode::InvalidArgument;
+    }
+
+    let proto_chunk_str = match CStr::from_ptr(proto_chunk_json).to_str() {
+        Ok(s) => s,
+        Err(_) => {
+            set_error_message(error_out, "Invalid UTF-8 in proto_chunk_json");
+            return SglErrorCode::InvalidArgument;
+        }
+    };
+
+    // Parse proto.GenerateResponse from JSON
+    let json_value: Value = match serde_json::from_str(proto_chunk_str) {
+        Ok(v) => v,
+        Err(e) => {
+            set_error_message(error_out, &format!("Failed to parse proto chunk JSON: {}", e));
+            return SglErrorCode::ParsingError;
+        }
+    };
+
+    // Build proto::GenerateResponse from JSON value
+    let mut proto_response = proto::GenerateResponse {
+        request_id: json_value
+            .get("request_id")
+            .and_then(|v| v.as_str())
+            .unwrap_or("")
+            .to_string(),
+        response: None,
+    };
+
+    // Parse the response oneof field
+    let is_done = if let Some(chunk_json) = json_value.get("chunk") {
+        let chunk = proto::GenerateStreamChunk {
+            token_ids: chunk_json
+                .get("token_ids")
+                .and_then(|v| v.as_array())
+                .map(|arr| {
+                    arr.iter()
+                        .filter_map(|v| v.as_u64().map(|n| n as u32))
+                        .collect()
+                })
+                .unwrap_or_default(),
+            prompt_tokens: chunk_json
+                .get("prompt_tokens")
+                .and_then(|v| v.as_i64())
+                .map(|n| n as i32)
+                .unwrap_or(0),
+            completion_tokens: chunk_json
+                .get("completion_tokens")
+                .and_then(|v| v.as_i64())
+                .map(|n| n as i32)
+                .unwrap_or(0),
+            cached_tokens: chunk_json
+                .get("cached_tokens")
+                .and_then(|v| v.as_i64())
+                .map(|n| n as i32)
+                .unwrap_or(0),
+            output_logprobs: None,
+            hidden_states: vec![],
+            input_logprobs: None,
+            index: 0,
+        };
+        proto_response.response = Some(proto::generate_response::Response::Chunk(chunk));
+        false
+    } else if let Some(complete_json) = json_value.get("complete") {
+        let complete = proto::GenerateComplete {
+            output_ids: complete_json
+                .get("output_ids")
+                .and_then(|v| v.as_array())
+                .map(|arr| {
+                    arr.iter()
+                        .filter_map(|v| v.as_u64().map(|n| n as u32))
+                        .collect()
+                })
+                .unwrap_or_default(),
+            finish_reason: complete_json
+                .get("finish_reason")
+                .and_then(|v| v.as_str())
+                .unwrap_or("")
+                .to_string(),
+            prompt_tokens: complete_json
+                .get("prompt_tokens")
+                .and_then(|v| v.as_i64())
+                .map(|n| n as i32)
+                .unwrap_or(0),
+            completion_tokens: complete_json
+                .get("completion_tokens")
+                .and_then(|v| v.as_i64())
+                .map(|n| n as i32)
+                .unwrap_or(0),
+            cached_tokens: complete_json
+                .get("cached_tokens")
+                .and_then(|v| v.as_i64())
+                .map(|n| n as i32)
+                .unwrap_or(0),
+            output_logprobs: None,
+            all_hidden_states: vec![],
+            input_logprobs: None,
+            matched_stop: None,
+            index: 0,
+        };
+        proto_response.response = Some(proto::generate_response::Response::Complete(complete));
+        true
+    } else if let Some(error_json) = json_value.get("error") {
+        let error = proto::GenerateError {
+            message: error_json
+                .get("message")
+                .and_then(|v| v.as_str())
+                .unwrap_or("")
+                .to_string(),
+            http_status_code: error_json
+                .get("http_status_code")
+                .and_then(|v| v.as_str())
+                .unwrap_or("500")
+                .to_string(),
+            details: error_json
+                .get("details")
+                .and_then(|v| v.as_str())
+                .unwrap_or("")
+                .to_string(),
+        };
+        proto_response.response = Some(proto::generate_response::Response::Error(error));
+        true
+    } else {
+        set_error_message(
+            error_out,
+            "Proto chunk JSON must contain 'chunk', 'complete', or 'error' field",
+        );
+        return SglErrorCode::ParsingError;
+    };
+
+    // Convert proto chunk to OpenAI format using the converter's convert_chunk function
+    // We'll use the existing converter API instead of calling the internal function directly
+    let proto_chunk_json_cstr = match CString::new(proto_chunk_str) {
+        Ok(s) => s,
+        Err(e) => {
+            set_error_message(error_out, &format!("Failed to create C string: {}", e));
+            return SglErrorCode::MemoryError;
+        }
+    };
+
+    // Use the existing converter API
+    let mut openai_json_ptr: *mut c_char = ptr::null_mut();
+    let result = super::grpc_converter::sgl_grpc_response_converter_convert_chunk(
+        converter_handle,
+        proto_chunk_json_cstr.as_ptr(),
+        &mut openai_json_ptr,
+        error_out,
+    );
+
+    if result == SglErrorCode::Success {
+        *openai_json_out = openai_json_ptr;
+        *is_done_out = if is_done { 1 } else { 0 };
+        SglErrorCode::Success
+    } else {
+        *openai_json_out = ptr::null_mut();
+        *is_done_out = if is_done { 1 } else { 0 };
+        result
+    }
+}
+
+/// Postprocess multiple gRPC stream chunks in batch (reduces FFI overhead)
+///
+/// This function processes multiple chunks in a single FFI call, significantly reducing
+/// FFI overhead in streaming scenarios.
+///
+/// # Arguments
+/// * `converter_handle` - Converter handle (created with sgl_grpc_response_converter_create)
+/// * `proto_chunks_json_array` - JSON array string of proto.GenerateResponse chunks
+/// * `max_chunks` - Maximum number of chunks to process (for safety)
+/// * `openai_chunks_json_array_out` - Pointer to receive JSON array of OpenAI format chunks (must be freed with sgl_free_string)
+/// * `chunks_count_out` - Pointer to receive number of processed chunks
+/// * `error_out` - Optional pointer to receive error message
+///
+/// # Returns
+/// * SglErrorCode::Success on success, error code on failure
+#[no_mangle]
+pub unsafe extern "C" fn sgl_postprocess_stream_chunks_batch(
+    converter_handle: *mut GrpcResponseConverterHandle,
+    proto_chunks_json_array: *const c_char,
+    max_chunks: c_int,
+    openai_chunks_json_array_out: *mut *mut c_char,
+    chunks_count_out: *mut c_int,
+    error_out: *mut *mut c_char,
+) -> SglErrorCode {
+    if converter_handle.is_null()
+        || proto_chunks_json_array.is_null()
+        || openai_chunks_json_array_out.is_null()
+        || chunks_count_out.is_null()
+    {
+        set_error_message(error_out, "Invalid arguments: null pointer");
+        return SglErrorCode::InvalidArgument;
+    }
+
+    let chunks_array_str = match CStr::from_ptr(proto_chunks_json_array).to_str() {
+        Ok(s) => s,
+        Err(_) => {
+            set_error_message(error_out, "Invalid UTF-8 in proto_chunks_json_array");
+            return SglErrorCode::InvalidArgument;
+        }
+    };
+
+    // Parse JSON array of chunks
+    let chunks_array: Vec<Value> = match serde_json::from_str(chunks_array_str) {
+        Ok(arr) => arr,
+        Err(e) => {
+            set_error_message(
+                error_out,
+                &format!("Failed to parse chunks JSON array: {}", e),
+            );
+            return SglErrorCode::ParsingError;
+        }
+    };
+
+    // Limit batch size for safety
+    let max_chunks_usize = max_chunks as usize;
+    let chunks_to_process = if chunks_array.len() > max_chunks_usize {
+        &chunks_array[..max_chunks_usize]
+    } else {
+        &chunks_array
+    };
+
+    let handle_ref = &mut *converter_handle;
+    let tokenizer = Arc::clone(&handle_ref.tokenizer);
+    let model = handle_ref.model.clone();
+    let request_id = handle_ref.request_id.clone();
+    let created = handle_ref.created;
+    let system_fingerprint = handle_ref.system_fingerprint.clone();
+
+    // Process chunks in batch
+    let mut results = Vec::new();
+    let mut has_error = false;
+    let mut error_msg = String::new();
+
+    for chunk_json in chunks_to_process {
+        // Parse proto.GenerateResponse from JSON
+        let mut proto_response = proto::GenerateResponse {
+            request_id: chunk_json
+                .get("request_id")
+                .and_then(|v| v.as_str())
+                .unwrap_or("")
+                .to_string(),
+            response: None,
+        };
+
+        // Parse the response oneof field (same logic as single chunk processing)
+        let _is_done = if let Some(chunk_json) = chunk_json.get("chunk") {
+            let chunk = proto::GenerateStreamChunk {
+                token_ids: chunk_json
+                    .get("token_ids")
+                    .and_then(|v| v.as_array())
+                    .map(|arr| {
+                        arr.iter()
+                            .filter_map(|v| v.as_u64().map(|n| n as u32))
+                            .collect()
+                    })
+                    .unwrap_or_default(),
+                prompt_tokens: chunk_json
+                    .get("prompt_tokens")
+                    .and_then(|v| v.as_i64())
+                    .map(|n| n as i32)
+                    .unwrap_or(0),
+                completion_tokens: chunk_json
+                    .get("completion_tokens")
+                    .and_then(|v| v.as_i64())
+                    .map(|n| n as i32)
+                    .unwrap_or(0),
+                cached_tokens: chunk_json
+                    .get("cached_tokens")
+                    .and_then(|v| v.as_i64())
+                    .map(|n| n as i32)
+                    .unwrap_or(0),
+                output_logprobs: None,
+                hidden_states: vec![],
+                input_logprobs: None,
+                index: 0,
+            };
+            proto_response.response = Some(proto::generate_response::Response::Chunk(chunk));
+            false
+        } else if let Some(complete_json) = chunk_json.get("complete") {
+            let complete = proto::GenerateComplete {
+                output_ids: complete_json
+                    .get("output_ids")
+                    .and_then(|v| v.as_array())
+                    .map(|arr| {
+                        arr.iter()
+                            .filter_map(|v| v.as_u64().map(|n| n as u32))
+                            .collect()
+                    })
+                    .unwrap_or_default(),
+                finish_reason: complete_json
+                    .get("finish_reason")
+                    .and_then(|v| v.as_str())
+                    .unwrap_or("")
+                    .to_string(),
+                prompt_tokens: complete_json
+                    .get("prompt_tokens")
+                    .and_then(|v| v.as_i64())
+                    .map(|n| n as i32)
+                    .unwrap_or(0),
+                completion_tokens: complete_json
+                    .get("completion_tokens")
+                    .and_then(|v| v.as_i64())
+                    .map(|n| n as i32)
+                    .unwrap_or(0),
+                cached_tokens: complete_json
+                    .get("cached_tokens")
+                    .and_then(|v| v.as_i64())
+                    .map(|n| n as i32)
+                    .unwrap_or(0),
+                output_logprobs: None,
+                all_hidden_states: vec![],
+                input_logprobs: None,
+                matched_stop: None,
+                index: 0,
+            };
+            proto_response.response = Some(proto::generate_response::Response::Complete(complete));
+            true
+        } else if let Some(error_json) = chunk_json.get("error") {
+            let error = proto::GenerateError {
+                message: error_json
+                    .get("message")
+                    .and_then(|v| v.as_str())
+                    .unwrap_or("")
+                    .to_string(),
+                http_status_code: error_json
+                    .get("http_status_code")
+                    .and_then(|v| v.as_str())
+                    .unwrap_or("500")
+                    .to_string(),
+                details: error_json
+                    .get("details")
+                    .and_then(|v| v.as_str())
+                    .unwrap_or("")
+                    .to_string(),
+            };
+            proto_response.response = Some(proto::generate_response::Response::Error(error));
+            true
+        } else {
+            error_msg = format!(
+                "Chunk JSON must contain 'chunk', 'complete', or 'error' field: {}",
+                chunk_json
+            );
+            has_error = true;
+            break;
+        };
+
+        // Convert proto chunk to OpenAI format
+        let result = RUNTIME.block_on(async {
+            super::grpc_converter::convert_proto_chunk_to_openai(
+                proto_response,
+                handle_ref,
+                &tokenizer,
+                &model,
+                &request_id,
+                created,
+                system_fingerprint.as_deref(),
+            )
+            .await
+        });
+
+        match result {
+            Ok(Some(openai_response)) => {
+                results.push(openai_response);
+            }
+            Ok(None) => {
+                // Empty response, skip
+            }
+            Err(e) => {
+                error_msg = format!("Postprocessing failed for chunk: {}", e);
+                has_error = true;
+                break;
+            }
+        }
+    }
+
+    if has_error {
+        set_error_message(error_out, &error_msg);
+        return SglErrorCode::ParsingError;
+    }
+
+    // Serialize results to JSON array
+    let results_json = match serde_json::to_string(&results) {
+        Ok(s) => s,
+        Err(e) => {
+            set_error_message(
+                error_out,
+                &format!("Failed to serialize results JSON array: {}", e),
+            );
+            return SglErrorCode::ParsingError;
+        }
+    };
+
+    let results_cstr = match CString::new(results_json) {
+        Ok(s) => s,
+        Err(e) => {
+            set_error_message(error_out, &format!("Failed to create C string: {}", e));
+            return SglErrorCode::MemoryError;
+        }
+    };
+
+    *openai_chunks_json_array_out = results_cstr.into_raw();
+    *chunks_count_out = results.len() as c_int;
+
+    SglErrorCode::Success
+}

third_party/sglang/sgl-model-gateway/bindings/golang/src/preprocessor.rs

@@ -0,0 +1,372 @@
+//! Preprocessing FFI functions for chat requests
+//!
+//! This module provides C-compatible functions for preprocessing chat completion requests:
+//! - Apply chat_template to messages
+//! - Tokenize the processed text
+//! - Generate tool constraints
+//!
+//! These functions are designed to be called once per request, reducing FFI overhead.
+
+use std::ffi::{CStr, CString};
+use std::os::raw::{c_char, c_int};
+use std::ptr;
+use std::os::raw::c_uint;
+
+use smg::tokenizer::create_tokenizer_from_file;
+use smg::protocols::chat::ChatCompletionRequest;
+use smg::routers::grpc::utils::{process_chat_messages, generate_tool_constraints};
+
+use super::error::{SglErrorCode, set_error_message};
+use super::memory::{sgl_free_string, sgl_free_token_ids};
+use super::tokenizer::TokenizerHandle;
+
+/// Handle for preprocessed request
+#[repr(C)]
+pub struct PreprocessedRequestHandle {
+    pub(crate) prompt_text: CString,
+    pub(crate) token_ids: Vec<i32>,
+    pub(crate) tool_constraints_json: Option<CString>,
+    pub(crate) prompt_tokens: i32,
+}
+
+/// Preprocess a chat completion request
+///
+/// This function:
+/// 1. Applies chat_template to messages
+/// 2. Tokenizes the processed text
+/// 3. Generates tool constraints (if tools are present)
+///
+/// # Arguments
+/// * `request_json` - OpenAI ChatCompletionRequest as JSON string
+/// * `tokenizer_path` - Path to tokenizer directory
+/// * `prompt_text_out` - Pointer to receive prompt text (C string, must be freed with sgl_free_string)
+/// * `token_ids_out` - Pointer to receive token IDs array (must be freed with sgl_free_token_ids)
+/// * `token_ids_len_out` - Pointer to receive token IDs array length
+/// * `tool_constraints_json_out` - Optional pointer to receive tool constraints JSON (must be freed with sgl_free_string)
+/// * `prompt_tokens_out` - Pointer to receive prompt token count
+/// * `error_out` - Optional pointer to receive error message
+///
+/// # Returns
+/// * SglErrorCode::Success on success, error code on failure
+#[no_mangle]
+pub unsafe extern "C" fn sgl_preprocess_chat_request(
+    request_json: *const c_char,
+    tokenizer_path: *const c_char,
+    prompt_text_out: *mut *mut c_char,
+    token_ids_out: *mut *mut c_uint,
+    token_ids_len_out: *mut usize,
+    tool_constraints_json_out: *mut *mut c_char,
+    prompt_tokens_out: *mut c_int,
+    error_out: *mut *mut c_char,
+) -> SglErrorCode {
+    if request_json.is_null()
+        || tokenizer_path.is_null()
+        || prompt_text_out.is_null()
+        || token_ids_out.is_null()
+        || token_ids_len_out.is_null()
+        || prompt_tokens_out.is_null()
+    {
+        set_error_message(error_out, "Invalid arguments: null pointer");
+        return SglErrorCode::InvalidArgument;
+    }
+
+    // Parse input strings
+    let request_str = match CStr::from_ptr(request_json).to_str() {
+        Ok(s) => s,
+        Err(_) => {
+            set_error_message(error_out, "Invalid UTF-8 in request_json");
+            return SglErrorCode::InvalidArgument;
+        }
+    };
+
+    let tokenizer_path_str = match CStr::from_ptr(tokenizer_path).to_str() {
+        Ok(s) => s,
+        Err(_) => {
+            set_error_message(error_out, "Invalid UTF-8 in tokenizer_path");
+            return SglErrorCode::InvalidArgument;
+        }
+    };
+
+    // Parse ChatCompletionRequest
+    let chat_request: ChatCompletionRequest = match serde_json::from_str(request_str) {
+        Ok(req) => req,
+        Err(e) => {
+            set_error_message(error_out, &format!("Failed to parse request JSON: {}", e));
+            return SglErrorCode::ParsingError;
+        }
+    };
+
+    // Create tokenizer
+    let tokenizer = match create_tokenizer_from_file(tokenizer_path_str) {
+        Ok(t) => t,
+        Err(e) => {
+            set_error_message(error_out, &format!("Failed to create tokenizer: {}", e));
+            return SglErrorCode::TokenizationError;
+        }
+    };
+
+    // Process chat messages (apply chat_template)
+    let processed_messages = match process_chat_messages(&chat_request, tokenizer.as_ref()) {
+        Ok(msgs) => msgs,
+        Err(e) => {
+            set_error_message(error_out, &format!("Failed to process chat messages: {}", e));
+            return SglErrorCode::ParsingError;
+        }
+    };
+
+    // Tokenize the processed text
+    let encoding = match tokenizer.encode(&processed_messages.text, false) {
+        Ok(enc) => enc,
+        Err(e) => {
+            set_error_message(error_out, &format!("Tokenization failed: {}", e));
+            return SglErrorCode::TokenizationError;
+        }
+    };
+
+    let token_ids_vec: Vec<i32> = encoding
+        .token_ids()
+        .iter()
+        .map(|&id| id as i32)
+        .collect();
+
+    let prompt_tokens = token_ids_vec.len() as i32;
+
+    // Generate tool constraints if tools are present
+    let tool_constraints_json = if let Some(tools) = chat_request.tools.as_ref() {
+        match generate_tool_constraints(tools, &chat_request.tool_choice, &chat_request.model) {
+            Ok(Some(constraints)) => {
+                match serde_json::to_string(&constraints) {
+                    Ok(json_str) => Some(CString::new(json_str).unwrap()),
+                    Err(e) => {
+                        set_error_message(
+                            error_out,
+                            &format!("Failed to serialize tool constraints: {}", e),
+                        );
+                        return SglErrorCode::ParsingError;
+                    }
+                }
+            }
+            Ok(None) => None,
+            Err(e) => {
+                set_error_message(
+                    error_out,
+                    &format!("Failed to generate tool constraints: {}", e),
+                );
+                return SglErrorCode::ParsingError;
+            }
+        }
+    } else {
+        None
+    };
+
+    // Allocate memory for outputs
+    let prompt_text_cstr = match CString::new(processed_messages.text) {
+        Ok(s) => s,
+        Err(e) => {
+            set_error_message(error_out, &format!("Failed to create C string: {}", e));
+            return SglErrorCode::MemoryError;
+        }
+    };
+
+    let token_ids_len = token_ids_vec.len();
+    // Convert i32 to u32 for token IDs (as expected by the memory management functions)
+    let token_ids_u32: Vec<u32> = token_ids_vec.iter().map(|&id| id as u32).collect();
+    let token_ids_ptr = if token_ids_u32.is_empty() {
+        ptr::null_mut()
+    } else {
+        let boxed = token_ids_u32.into_boxed_slice();
+        Box::into_raw(boxed) as *mut c_uint
+    };
+
+    // Set output values
+    *prompt_text_out = prompt_text_cstr.into_raw();
+    *token_ids_out = token_ids_ptr;
+    *token_ids_len_out = token_ids_len;
+    *prompt_tokens_out = prompt_tokens;
+
+    if !tool_constraints_json_out.is_null() {
+        if let Some(constraints) = tool_constraints_json {
+            *tool_constraints_json_out = constraints.into_raw();
+        } else {
+            *tool_constraints_json_out = ptr::null_mut();
+        }
+    }
+
+    SglErrorCode::Success
+}
+
+/// Preprocess a chat completion request using an existing tokenizer handle
+///
+/// This function is similar to sgl_preprocess_chat_request, but accepts a TokenizerHandle
+/// instead of creating a new tokenizer. This allows reusing a cached tokenizer instance,
+/// significantly reducing initialization overhead in concurrent scenarios.
+///
+/// # Arguments
+/// * `request_json` - OpenAI ChatCompletionRequest as JSON string
+/// * `tokenizer_handle` - Existing tokenizer handle (must be valid)
+/// * `prompt_text_out` - Pointer to receive prompt text (C string, must be freed with sgl_free_string)
+/// * `token_ids_out` - Pointer to receive token IDs array (must be freed with sgl_free_token_ids)
+/// * `token_ids_len_out` - Pointer to receive token IDs array length
+/// * `tool_constraints_json_out` - Optional pointer to receive tool constraints JSON (must be freed with sgl_free_string)
+/// * `prompt_tokens_out` - Pointer to receive prompt token count
+/// * `error_out` - Optional pointer to receive error message
+///
+/// # Returns
+/// * SglErrorCode::Success on success, error code on failure
+#[no_mangle]
+pub unsafe extern "C" fn sgl_preprocess_chat_request_with_tokenizer(
+    request_json: *const c_char,
+    tokenizer_handle: *mut TokenizerHandle,
+    prompt_text_out: *mut *mut c_char,
+    token_ids_out: *mut *mut c_uint,
+    token_ids_len_out: *mut usize,
+    tool_constraints_json_out: *mut *mut c_char,
+    prompt_tokens_out: *mut c_int,
+    error_out: *mut *mut c_char,
+) -> SglErrorCode {
+    if request_json.is_null()
+        || tokenizer_handle.is_null()
+        || prompt_text_out.is_null()
+        || token_ids_out.is_null()
+        || token_ids_len_out.is_null()
+        || prompt_tokens_out.is_null()
+    {
+        set_error_message(error_out, "Invalid arguments: null pointer");
+        return SglErrorCode::InvalidArgument;
+    }
+
+    // Parse input string
+    let request_str = match CStr::from_ptr(request_json).to_str() {
+        Ok(s) => s,
+        Err(_) => {
+            set_error_message(error_out, "Invalid UTF-8 in request_json");
+            return SglErrorCode::InvalidArgument;
+        }
+    };
+
+    // Parse ChatCompletionRequest
+    let chat_request: ChatCompletionRequest = match serde_json::from_str(request_str) {
+        Ok(req) => req,
+        Err(e) => {
+            set_error_message(error_out, &format!("Failed to parse request JSON: {}", e));
+            return SglErrorCode::ParsingError;
+        }
+    };
+
+    // Use existing tokenizer from handle (no need to create new one!)
+    let handle_ref = &*tokenizer_handle;
+    let tokenizer = &handle_ref.tokenizer;
+
+    // Process chat messages (apply chat_template)
+    let processed_messages = match process_chat_messages(&chat_request, tokenizer.as_ref()) {
+        Ok(msgs) => msgs,
+        Err(e) => {
+            set_error_message(error_out, &format!("Failed to process chat messages: {}", e));
+            return SglErrorCode::ParsingError;
+        }
+    };
+
+    // Tokenize the processed text
+    let encoding = match tokenizer.encode(&processed_messages.text, false) {
+        Ok(enc) => enc,
+        Err(e) => {
+            set_error_message(error_out, &format!("Tokenization failed: {}", e));
+            return SglErrorCode::TokenizationError;
+        }
+    };
+
+    let token_ids_vec: Vec<i32> = encoding
+        .token_ids()
+        .iter()
+        .map(|&id| id as i32)
+        .collect();
+
+    let prompt_tokens = token_ids_vec.len() as i32;
+
+    // Generate tool constraints if tools are present
+    let tool_constraints_json = if let Some(tools) = chat_request.tools.as_ref() {
+        match generate_tool_constraints(tools, &chat_request.tool_choice, &chat_request.model) {
+            Ok(Some(constraints)) => {
+                match serde_json::to_string(&constraints) {
+                    Ok(json_str) => Some(CString::new(json_str).unwrap()),
+                    Err(e) => {
+                        set_error_message(
+                            error_out,
+                            &format!("Failed to serialize tool constraints: {}", e),
+                        );
+                        return SglErrorCode::ParsingError;
+                    }
+                }
+            }
+            Ok(None) => None,
+            Err(e) => {
+                set_error_message(
+                    error_out,
+                    &format!("Failed to generate tool constraints: {}", e),
+                );
+                return SglErrorCode::ParsingError;
+            }
+        }
+    } else {
+        None
+    };
+
+    // Allocate memory for outputs
+    let prompt_text_cstr = match CString::new(processed_messages.text) {
+        Ok(s) => s,
+        Err(e) => {
+            set_error_message(error_out, &format!("Failed to create C string: {}", e));
+            return SglErrorCode::MemoryError;
+        }
+    };
+
+    let token_ids_len = token_ids_vec.len();
+    // Convert i32 to u32 for token IDs (as expected by the memory management functions)
+    let token_ids_u32: Vec<u32> = token_ids_vec.iter().map(|&id| id as u32).collect();
+    let token_ids_ptr = if token_ids_u32.is_empty() {
+        ptr::null_mut()
+    } else {
+        let boxed = token_ids_u32.into_boxed_slice();
+        Box::into_raw(boxed) as *mut c_uint
+    };
+
+    // Set output values
+    *prompt_text_out = prompt_text_cstr.into_raw();
+    *token_ids_out = token_ids_ptr;
+    *token_ids_len_out = token_ids_len;
+    *prompt_tokens_out = prompt_tokens;
+
+    if !tool_constraints_json_out.is_null() {
+        if let Some(constraints) = tool_constraints_json {
+            *tool_constraints_json_out = constraints.into_raw();
+        } else {
+            *tool_constraints_json_out = ptr::null_mut();
+        }
+    }
+
+    SglErrorCode::Success
+}
+
+/// Free a preprocessed request handle (cleanup function)
+///
+/// This function frees the memory allocated by sgl_preprocess_chat_request.
+/// It should be called after the preprocessed data is no longer needed.
+#[no_mangle]
+pub unsafe extern "C" fn sgl_preprocessed_request_free(
+    prompt_text: *mut c_char,
+    token_ids: *mut c_uint,
+    token_ids_len: usize,
+    tool_constraints_json: *mut c_char,
+) {
+    if !prompt_text.is_null() {
+        sgl_free_string(prompt_text);
+    }
+
+    if !token_ids.is_null() && token_ids_len > 0 {
+        sgl_free_token_ids(token_ids, token_ids_len);
+    }
+
+    if !tool_constraints_json.is_null() {
+        sgl_free_string(tool_constraints_json);
+    }
+}

third_party/sglang/sgl-model-gateway/bindings/golang/src/stream.rs

@@ -0,0 +1,288 @@
+//! Stream handling FFI functions
+//!
+//! This module provides FFI (Foreign Function Interface) functions for managing
+//! streaming responses from the SGLang gRPC API. It handles:
+//!
+//! - Creating and managing stream handles
+//! - Reading chunks from streams and converting them to OpenAI format
+//! - Managing automatic abort on stream drop (via AbortOnDropStream)
+//! - Thread-safe access to streams and response converters
+//!
+//! # Safety
+//!
+//! All FFI functions are marked `unsafe` as per Rust FFI conventions. Callers must:
+//! - Pass valid pointers
+//! - Ensure proper pointer lifetime management
+//! - Call corresponding free functions for cleanup
+
+use std::ffi::CString;
+use std::os::raw::{c_char, c_int};
+use std::ptr;
+use std::sync::Arc;
+use tokio::runtime::Runtime;
+use once_cell::sync::Lazy;
+use futures_util::StreamExt;
+
+use smg_grpc_client::{sglang_proto as proto, sglang_scheduler::{SglangSchedulerClient, AbortOnDropStream}};
+
+use super::error::{SglErrorCode, set_error_message};
+use super::grpc_converter::{GrpcResponseConverterHandle, convert_proto_chunk_to_openai};
+
+/// Global tokio runtime for async operations
+static RUNTIME: Lazy<Runtime> = Lazy::new(|| {
+    Runtime::new().expect("Failed to create tokio runtime for stream FFI")
+});
+
+/// Handle for an active streaming request.
+///
+/// This struct manages the stream and response converter for a single request.
+/// It is wrapped in Arc and Mutex for thread-safe concurrent access.
+///
+/// # Fields
+///
+/// * `stream` - The gRPC stream wrapped in AbortOnDropStream for automatic cleanup
+/// * `converter` - Response converter that transforms proto messages to OpenAI format
+/// * `client` - The underlying gRPC client connection
+/// * `prompt_tokens` - Number of prompt tokens from the original request
+pub struct SglangStreamHandle {
+    pub(crate) stream: Arc<tokio::sync::Mutex<AbortOnDropStream>>,
+    pub(crate) converter: Arc<tokio::sync::Mutex<GrpcResponseConverterHandle>>,
+    #[allow(dead_code)]
+    pub(crate) client: Arc<SglangSchedulerClient>,
+    #[allow(dead_code)]
+    pub(crate) prompt_tokens: i32, // Number of prompt tokens for this request
+}
+
+/// Read next chunk from stream and convert to OpenAI format.
+///
+/// This function reads the next chunk from the gRPC stream, converts it from the
+/// internal protocol format to OpenAI-compatible JSON format, and returns it via
+/// the output parameters.
+///
+/// # Arguments
+///
+/// * `stream_handle` - Mutable pointer to the stream handle
+/// * `response_json_out` - Pointer to receive OpenAI format JSON string
+///   - Caller must free this with `sgl_free_string`
+///   - May be NULL if no data available
+/// * `is_done_out` - Pointer to receive completion status
+///   - 0 = stream has more data
+///   - 1 = stream is complete
+/// * `error_out` - Optional pointer to receive error message
+///   - Only set if function returns an error code
+///   - Must be freed with `sgl_free_string` if not NULL
+///
+/// # Returns
+///
+/// * `SglErrorCode::Success` - Successfully read a chunk or reached end of stream
+/// * Other error codes - See `SglErrorCode` for details
+///
+/// # Safety
+///
+/// - All pointers must be valid and properly aligned
+/// - `stream_handle` must point to a valid `SglangStreamHandle`
+/// - Output pointers must be writable
+///
+/// # Notes
+///
+/// - Complete messages are identified by the presence of `proto::GenerateResponse::Complete`
+/// - When is_done=1, this may be the last readable chunk or the stream may be ending
+/// - Subsequent calls after is_done=1 will mark the stream as complete internally
+#[no_mangle]
+pub unsafe extern "C" fn sgl_stream_read_next(
+    stream_handle: *mut SglangStreamHandle,
+    response_json_out: *mut *mut c_char,
+    is_done_out: *mut c_int,
+    error_out: *mut *mut c_char,
+) -> SglErrorCode {
+    if stream_handle.is_null() || response_json_out.is_null() || is_done_out.is_null() {
+        set_error_message(error_out, "Invalid arguments: null pointer");
+        return SglErrorCode::InvalidArgument;
+    }
+
+    let handle_ref = &*stream_handle;
+    let stream = Arc::clone(&handle_ref.stream);
+    let converter = Arc::clone(&handle_ref.converter);
+
+    // Read next chunk from stream
+    let chunk_result = RUNTIME.block_on(async {
+        let mut stream_guard = stream.lock().await;
+        stream_guard.next().await
+    });
+
+    match chunk_result {
+        Some(Ok(proto_response)) => {
+            // Convert proto response to OpenAI format
+            // We need to get the converter lock first
+            let conversion_result = RUNTIME.block_on(async {
+                let mut converter_guard = converter.lock().await;
+
+                // Clone necessary fields for conversion
+                let tokenizer = Arc::clone(&converter_guard.tokenizer);
+                let model = converter_guard.model.clone();
+                let request_id = converter_guard.request_id.clone();
+                let created = converter_guard.created;
+                let system_fingerprint = converter_guard.system_fingerprint.clone();
+
+                // Call the conversion function
+                convert_proto_chunk_to_openai(
+                    proto_response.clone(),
+                    &mut *converter_guard,
+                    &tokenizer,
+                    &model,
+                    &request_id,
+                    created,
+                    system_fingerprint.as_deref(),
+                )
+                .await
+            });
+
+            match conversion_result {
+                Ok(Some(openai_response)) => {
+                    // Serialize to JSON
+                    let result_str = match serde_json::to_string(&openai_response) {
+                        Ok(s) => s,
+                        Err(e) => {
+                            set_error_message(error_out, &format!("Failed to serialize response: {}", e));
+                            return SglErrorCode::ParsingError;
+                        }
+                    };
+
+                    let result_cstr = match CString::new(result_str) {
+                        Ok(s) => s,
+                        Err(e) => {
+                            set_error_message(error_out, &format!("Failed to create result string: {}", e));
+                            return SglErrorCode::MemoryError;
+                        }
+                    };
+
+                    // Check if this is a complete response (stream done)
+                    let is_complete = matches!(proto_response.response, Some(proto::generate_response::Response::Complete(_)) | Some(proto::generate_response::Response::Error(_)));
+
+                    *response_json_out = result_cstr.into_raw();
+                    *is_done_out = if is_complete { 1 } else { 0 };
+
+                    if is_complete {
+                        // Mark stream as completed
+                        // Ensure mark_completed() completes and is visible before returning
+                        // Use yield_now to ensure Release ordering is fully propagated
+                        RUNTIME.block_on(async {
+                            let stream_guard = stream.lock().await;
+                            stream_guard.mark_completed();
+                            // Keep the guard until mark_completed() is fully executed
+                            drop(stream_guard);
+                            // Yield to ensure Release ordering is propagated before returning
+                            // This prevents race condition where Free() is called immediately
+                            // and Drop might not see the mark_completed() write
+                            tokio::task::yield_now().await;
+                        });
+                    }
+
+                    SglErrorCode::Success
+                }
+                Ok(None) => {
+                    // No response to send (e.g., empty chunk)
+                    // Don't mark as completed - stream might continue
+                    // Just return null and let caller read more
+                    *response_json_out = ptr::null_mut();
+                    *is_done_out = 0; // Keep stream open, not done yet
+                    SglErrorCode::Success
+                }
+                Err(e) => {
+                    // Conversion error - don't mark as completed
+                    // Let the stream end naturally or return error without stopping stream
+                    set_error_message(error_out, &format!("Conversion error: {}", e));
+                    *response_json_out = ptr::null_mut();
+                    *is_done_out = 0; // Don't mark as done - let caller decide
+                    SglErrorCode::ParsingError
+                }
+            }
+        }
+        Some(Err(e)) => {
+            // Stream error - mark as completed to prevent abort
+            RUNTIME.block_on(async {
+                let stream_guard = stream.lock().await;
+                stream_guard.mark_completed();
+                drop(stream_guard);
+                // Yield to ensure Release ordering is propagated
+                tokio::task::yield_now().await;
+            });
+
+            set_error_message(error_out, &format!("Stream error: {}", e));
+            *is_done_out = 1;
+            SglErrorCode::UnknownError
+        }
+        None => {
+            // Stream ended naturally (no more chunks)
+            // Mark stream as completed before returning to prevent abort
+            RUNTIME.block_on(async {
+                let stream_guard = stream.lock().await;
+                stream_guard.mark_completed();
+                drop(stream_guard);
+                // Yield to ensure Release ordering is propagated
+                tokio::task::yield_now().await;
+            });
+
+            *response_json_out = ptr::null_mut();
+            *is_done_out = 1;
+            SglErrorCode::Success
+        }
+    }
+}
+
+/// Free a stream handle and release all associated resources.
+///
+/// This function must be called exactly once for each stream handle returned by
+/// `sgl_client_chat_completion_stream`. It marks the stream as completed internally
+/// to prevent abort signals from being sent when resources are cleaned up.
+///
+/// # Arguments
+///
+/// * `handle` - Mutable pointer to the stream handle to free
+///   - If NULL, this function does nothing
+///
+/// # Safety
+///
+/// - Must be called only once per handle
+/// - Handle must not be used after calling this function
+/// - After this call, the stream is no longer valid
+///
+/// # Notes
+///
+/// - This function internally calls `mark_completed()` before freeing to ensure
+///   the stream cleanup doesn't trigger an abort RPC to the server
+/// - Memory fences are used to ensure visibility across threads
+#[no_mangle]
+pub unsafe extern "C" fn sgl_stream_free(handle: *mut SglangStreamHandle) {
+    if !handle.is_null() {
+        let handle_ref = Box::from_raw(handle);
+
+        // Mark stream as completed to prevent abort on drop
+        // By this point, the stream should already be completed by ReadNext()
+        // but we call it again to be safe
+        RUNTIME.block_on(async {
+            let stream_guard = handle_ref.stream.lock().await;
+            stream_guard.mark_completed();
+            // Keep guard alive to ensure mark_completed() write completes
+            drop(stream_guard);
+            // Yield to ensure the atomic write is visible
+            tokio::task::yield_now().await;
+        });
+
+        // Use a strong memory fence to ensure mark_completed()'s Release write
+        // is visible before we drop the last Arc reference
+        std::sync::atomic::fence(std::sync::atomic::Ordering::SeqCst);
+
+        // Now drop all references - if mark_completed() was called successfully,
+        // the drop won't send an abort
+        drop(handle_ref.stream);
+
+        // Free converter
+        let converter = Arc::try_unwrap(handle_ref.converter)
+            .ok()
+            .map(|m| m.into_inner());
+        if let Some(conv) = converter {
+            super::grpc_converter::sgl_grpc_response_converter_free(Box::into_raw(Box::new(conv)));
+        }
+    }
+}

third_party/sglang/sgl-model-gateway/bindings/golang/src/tokenizer.rs

@@ -0,0 +1,388 @@
+//! Tokenizer FFI functions
+
+use std::ffi::{CStr, CString};
+use std::os::raw::{c_char, c_int};
+use std::ptr;
+use std::sync::Arc;
+use serde_json::Value;
+
+use smg::tokenizer::{
+    create_tokenizer_from_file,
+    traits::Tokenizer as TokenizerTrait,
+    chat_template::ChatTemplateParams,
+    huggingface::HuggingFaceTokenizer,
+};
+
+use super::error::{SglErrorCode, set_error_message, clear_error_message};
+
+#[cfg(target_os = "macos")]
+type BooleanT = libc::boolean_t;
+#[cfg(not(target_os = "macos"))]
+type BooleanT = libc::c_int;
+
+/// Opaque handle for a tokenizer instance
+#[repr(C)]
+pub struct TokenizerHandle {
+    pub(crate) tokenizer: Arc<dyn TokenizerTrait>,
+}
+
+/// Create a tokenizer from a file path
+///
+/// # Arguments
+/// * `path` - Path to tokenizer.json file (null-terminated C string)
+/// * `error_out` - Optional pointer to receive error message (must be freed with sgl_free_string)
+///
+/// # Returns
+/// * Pointer to TokenizerHandle on success, null on failure
+///
+/// # Safety
+/// The returned handle must be freed with `sgl_tokenizer_free`.
+#[no_mangle]
+pub unsafe extern "C" fn sgl_tokenizer_create_from_file(
+    path: *const c_char,
+    error_out: *mut *mut c_char,
+) -> *mut TokenizerHandle {
+    if path.is_null() {
+        set_error_message(error_out, "path cannot be null");
+        return ptr::null_mut();
+    }
+
+    let path_str = match CStr::from_ptr(path).to_str() {
+        Ok(s) => s,
+        Err(e) => {
+            set_error_message(error_out, &format!("Invalid UTF-8 in path: {}", e));
+            return ptr::null_mut();
+        }
+    };
+
+    match create_tokenizer_from_file(path_str) {
+        Ok(tokenizer) => {
+            clear_error_message(error_out);
+            Box::into_raw(Box::new(TokenizerHandle {
+                tokenizer,
+            }))
+        }
+        Err(e) => {
+            set_error_message(error_out, &e.to_string());
+            ptr::null_mut()
+        }
+    }
+}
+
+/// Encode text to token IDs
+///
+/// # Arguments
+/// * `handle` - Tokenizer handle (must not be null)
+/// * `text` - Input text (null-terminated C string)
+/// * `add_special_tokens` - Whether to add special tokens
+/// * `token_ids_out` - Pointer to receive array of token IDs (must be freed with sgl_free_token_ids)
+/// * `token_count_out` - Pointer to receive token count
+/// * `error_out` - Optional pointer to receive error message
+///
+/// # Returns
+/// * SglErrorCode::Success on success, error code on failure
+///
+/// # Safety
+/// The token_ids_out array must be freed with sgl_free_token_ids() after use.
+#[no_mangle]
+pub unsafe extern "C" fn sgl_tokenizer_encode(
+    handle: *mut TokenizerHandle,
+    text: *const c_char,
+    add_special_tokens: BooleanT,
+    token_ids_out: *mut *mut u32,
+    token_count_out: *mut usize,
+    error_out: *mut *mut c_char,
+) -> SglErrorCode {
+    if handle.is_null() || text.is_null() || token_ids_out.is_null() || token_count_out.is_null() {
+        set_error_message(error_out, "Invalid arguments: null pointer");
+        return SglErrorCode::InvalidArgument;
+    }
+
+    let text_str = match CStr::from_ptr(text).to_str() {
+        Ok(s) => s,
+        Err(_) => {
+            set_error_message(error_out, "Invalid UTF-8 in text");
+            return SglErrorCode::InvalidArgument;
+        }
+    };
+
+    let add_special_tokens_bool = add_special_tokens != 0;
+
+    let tokenizer = &(*handle).tokenizer;
+    match tokenizer.encode(text_str, add_special_tokens_bool) {
+        Ok(encoding) => {
+            let token_ids = encoding.token_ids();
+            let count = token_ids.len();
+
+            // Allocate memory for token IDs using Vec, then leak to give ownership to C
+            let vec = token_ids.to_vec();
+            let ptr = vec.as_ptr() as *mut u32;
+            let _ = std::mem::ManuallyDrop::new(vec);
+
+            *token_ids_out = ptr;
+            *token_count_out = count;
+            clear_error_message(error_out);
+            SglErrorCode::Success
+        }
+        Err(e) => {
+            set_error_message(error_out, &e.to_string());
+            SglErrorCode::TokenizationError
+        }
+    }
+}
+
+/// Apply chat template to messages with tools support
+///
+/// # Arguments
+/// * `handle` - Tokenizer handle
+/// * `messages_json` - JSON string of messages array
+/// * `tools_json` - Optional JSON string of tools array (null or empty string for no tools)
+/// * `result_out` - Pointer to receive result string (must be freed with sgl_free_string)
+/// * `error_out` - Optional pointer to receive error message
+///
+/// # Returns
+/// * SglErrorCode::Success on success, error code on failure
+#[no_mangle]
+pub unsafe extern "C" fn sgl_tokenizer_apply_chat_template_with_tools(
+    handle: *mut TokenizerHandle,
+    messages_json: *const c_char,
+    tools_json: *const c_char,
+    result_out: *mut *mut c_char,
+    error_out: *mut *mut c_char,
+) -> SglErrorCode {
+    if handle.is_null() || messages_json.is_null() || result_out.is_null() {
+        set_error_message(error_out, "Invalid arguments: null pointer");
+        return SglErrorCode::InvalidArgument;
+    }
+
+    let messages_str = match CStr::from_ptr(messages_json).to_str() {
+        Ok(s) => s,
+        Err(_) => {
+            set_error_message(error_out, "Invalid UTF-8 in messages_json");
+            return SglErrorCode::InvalidArgument;
+        }
+    };
+
+    // Parse JSON messages
+    let messages: Vec<Value> = match serde_json::from_str(messages_str) {
+        Ok(msgs) => msgs,
+        Err(e) => {
+            set_error_message(error_out, &format!("Failed to parse messages JSON: {}", e));
+            return SglErrorCode::InvalidArgument;
+        }
+    };
+
+    // Parse tools JSON if provided
+    let tools: Option<Vec<Value>> = if tools_json.is_null() {
+        None
+    } else {
+        let tools_str = match CStr::from_ptr(tools_json).to_str() {
+            Ok(s) => {
+                if s.is_empty() {
+                    None
+                } else {
+                    match serde_json::from_str::<Vec<Value>>(s) {
+                        Ok(t) => Some(t),
+                        Err(e) => {
+                            set_error_message(error_out, &format!("Failed to parse tools JSON: {}", e));
+                            return SglErrorCode::InvalidArgument;
+                        }
+                    }
+                }
+            }
+            Err(_) => {
+                set_error_message(error_out, "Invalid UTF-8 in tools_json");
+                return SglErrorCode::InvalidArgument;
+            }
+        };
+        tools_str
+    };
+
+    // Get the tokenizer from handle
+    let handle_ref = &*handle;
+    let tokenizer = &handle_ref.tokenizer;
+
+    // Try to downcast to HuggingFaceTokenizer
+    if let Some(hf_tokenizer) = tokenizer.as_any().downcast_ref::<HuggingFaceTokenizer>() {
+        // Apply chat template with tools
+        let empty_docs: [Value; 0] = [];
+        let tools_slice = tools.as_ref().map(|t| t.as_slice());
+        let params = ChatTemplateParams {
+            add_generation_prompt: true,
+            tools: tools_slice,
+            documents: Some(&empty_docs),
+            template_kwargs: None,
+        };
+
+        match hf_tokenizer.apply_chat_template(&messages, params) {
+            Ok(result) => {
+                let result_cstr = match CString::new(result) {
+                    Ok(s) => s,
+                    Err(e) => {
+                        set_error_message(error_out, &format!("Failed to create result string: {}", e));
+                        return SglErrorCode::MemoryError;
+                    }
+                };
+                *result_out = result_cstr.into_raw();
+                clear_error_message(error_out);
+                SglErrorCode::Success
+            }
+            Err(e) => {
+                set_error_message(error_out, &format!("Failed to apply chat template: {}", e));
+                SglErrorCode::TokenizationError
+            }
+        }
+    } else {
+        set_error_message(error_out, "Chat template is only supported for HuggingFace tokenizers");
+        SglErrorCode::TokenizationError
+    }
+}
+
+/// Apply chat template to messages
+///
+/// # Arguments
+/// * `handle` - Tokenizer handle
+/// * `messages_json` - JSON string of messages array
+/// * `result_out` - Pointer to receive result string (must be freed with sgl_free_string)
+/// * `error_out` - Optional pointer to receive error message
+///
+/// # Returns
+/// * SglErrorCode::Success on success, error code on failure
+#[no_mangle]
+pub unsafe extern "C" fn sgl_tokenizer_apply_chat_template(
+    handle: *mut TokenizerHandle,
+    messages_json: *const c_char,
+    result_out: *mut *mut c_char,
+    error_out: *mut *mut c_char,
+) -> SglErrorCode {
+    if handle.is_null() || messages_json.is_null() || result_out.is_null() {
+        set_error_message(error_out, "Invalid arguments: null pointer");
+        return SglErrorCode::InvalidArgument;
+    }
+
+    let messages_str = match CStr::from_ptr(messages_json).to_str() {
+        Ok(s) => s,
+        Err(_) => {
+            set_error_message(error_out, "Invalid UTF-8 in messages_json");
+            return SglErrorCode::InvalidArgument;
+        }
+    };
+
+    // Parse JSON messages
+    let messages: Vec<Value> = match serde_json::from_str(messages_str) {
+        Ok(msgs) => msgs,
+        Err(e) => {
+            set_error_message(error_out, &format!("Failed to parse messages JSON: {}", e));
+            return SglErrorCode::InvalidArgument;
+        }
+    };
+
+    // Get the tokenizer from handle
+    let handle_ref = &*handle;
+    let tokenizer = &handle_ref.tokenizer;
+
+    // Try to downcast to HuggingFaceTokenizer
+    if let Some(hf_tokenizer) = tokenizer.as_any().downcast_ref::<HuggingFaceTokenizer>() {
+        // Apply chat template with default parameters
+        // Use empty arrays instead of None to avoid template errors
+        // Set add_generation_prompt to true so the model knows to start generating
+        let empty_tools: [Value; 0] = [];
+        let empty_docs: [Value; 0] = [];
+        let params = ChatTemplateParams {
+            add_generation_prompt: true,  // Important: tells the model to start generating
+            tools: Some(&empty_tools),
+            documents: Some(&empty_docs),
+            template_kwargs: None,
+        };
+
+        match hf_tokenizer.apply_chat_template(&messages, params) {
+            Ok(result) => {
+                let result_cstr = match CString::new(result) {
+                    Ok(s) => s,
+                    Err(e) => {
+                        set_error_message(error_out, &format!("Failed to create result string: {}", e));
+                        return SglErrorCode::MemoryError;
+                    }
+                };
+                *result_out = result_cstr.into_raw();
+                clear_error_message(error_out);
+                SglErrorCode::Success
+            }
+            Err(e) => {
+                set_error_message(error_out, &format!("Failed to apply chat template: {}", e));
+                SglErrorCode::TokenizationError
+            }
+        }
+    } else {
+        set_error_message(error_out, "Chat template is only supported for HuggingFace tokenizers");
+        SglErrorCode::TokenizationError
+    }
+}
+
+/// Decode token IDs to text
+///
+/// # Arguments
+/// * `handle` - Tokenizer handle
+/// * `token_ids` - Array of token IDs
+/// * `token_count` - Number of tokens
+/// * `skip_special_tokens` - Whether to skip special tokens
+/// * `result_out` - Pointer to receive result string (must be freed with sgl_free_string)
+/// * `error_out` - Optional pointer to receive error message
+///
+/// # Returns
+/// * SglErrorCode::Success on success, error code on failure
+#[no_mangle]
+pub unsafe extern "C" fn sgl_tokenizer_decode(
+    handle: *mut TokenizerHandle,
+    token_ids: *const u32,
+    token_count: usize,
+    skip_special_tokens: c_int,
+    result_out: *mut *mut c_char,
+    error_out: *mut *mut c_char,
+) -> SglErrorCode {
+    if handle.is_null() || token_ids.is_null() || result_out.is_null() {
+        set_error_message(error_out, "Invalid arguments: null pointer");
+        return SglErrorCode::InvalidArgument;
+    }
+
+    if token_count == 0 {
+        let empty = CString::new("").unwrap();
+        *result_out = empty.into_raw();
+        clear_error_message(error_out);
+        return SglErrorCode::Success;
+    }
+
+    // Convert C array to Rust slice
+    let token_slice = std::slice::from_raw_parts(token_ids, token_count);
+
+    let tokenizer = &(*handle).tokenizer;
+    match tokenizer.decode(token_slice, skip_special_tokens != 0) {
+        Ok(text) => {
+            let result_cstr = match CString::new(text) {
+                Ok(s) => s,
+                Err(e) => {
+                    set_error_message(error_out, &format!("Failed to create result string: {}", e));
+                    return SglErrorCode::MemoryError;
+                }
+            };
+            *result_out = result_cstr.into_raw();
+            clear_error_message(error_out);
+            SglErrorCode::Success
+        }
+        Err(e) => {
+            set_error_message(error_out, &e.to_string());
+            SglErrorCode::TokenizationError
+        }
+    }
+}
+
+/// Free a tokenizer handle
+///
+/// # Safety
+/// This function must only be called once per handle, and the handle must not be used after calling.
+#[no_mangle]
+pub unsafe extern "C" fn sgl_tokenizer_free(handle: *mut TokenizerHandle) {
+    if !handle.is_null() {
+        let _ = Box::from_raw(handle);
+    }
+}

third_party/sglang/sgl-model-gateway/bindings/golang/src/tool_parser.rs

@@ -0,0 +1,329 @@
+//! Tool parser FFI functions
+
+use std::ffi::{CStr, CString};
+use std::os::raw::{c_char};
+use std::ptr;
+use std::sync::Arc;
+use std::collections::HashMap;
+use serde_json::{json, Value};
+use tokio::runtime::Runtime;
+use once_cell::sync::Lazy;
+
+use smg::tool_parser::{ParserFactory, ToolParser};
+use smg::protocols::common::Tool;
+
+use super::error::{SglErrorCode, set_error_message, clear_error_message};
+use super::utils::generate_tool_call_id;
+
+/// Global parser factory (initialized once)
+static PARSER_FACTORY: Lazy<ParserFactory> = Lazy::new(|| ParserFactory::new());
+
+/// Global tokio runtime for async operations
+static RUNTIME: Lazy<Runtime> = Lazy::new(|| {
+    Runtime::new().expect("Failed to create tokio runtime for tool parser FFI")
+});
+
+/// Opaque handle for a tool parser instance
+/// Note: For streaming, we need mutable access, so we use Arc<Mutex<>> internally
+/// Note: This is an opaque handle, C code doesn't access fields directly
+pub struct ToolParserHandle {
+    parser: Arc<tokio::sync::Mutex<Box<dyn ToolParser>>>,
+    model: String, // Store model name for ID generation
+    history_tool_calls_count: usize, // Track tool call count for ID generation
+    tool_index_to_id: HashMap<usize, String>, // Map tool_index to ID for incremental updates
+}
+
+/// Create a tool parser
+///
+/// # Arguments
+/// * `parser_type` - Parser type name (e.g., "json", "llama", "mistral") or model name (e.g., "gpt-4")
+/// * `error_out` - Optional pointer to receive error message
+///
+/// # Returns
+/// * Pointer to ToolParserHandle on success, null on failure
+#[no_mangle]
+pub unsafe extern "C" fn sgl_tool_parser_create(
+    parser_type: *const c_char,
+    error_out: *mut *mut c_char,
+) -> *mut ToolParserHandle {
+    if parser_type.is_null() {
+        set_error_message(error_out, "parser_type cannot be null");
+        return ptr::null_mut();
+    }
+
+    let type_str = match CStr::from_ptr(parser_type).to_str() {
+        Ok(s) => s,
+        Err(_) => {
+            set_error_message(error_out, "Invalid UTF-8 in parser_type");
+            return ptr::null_mut();
+        }
+    };
+
+    // Create parser using factory
+    // The factory will determine the parser type based on model name or use the provided type
+    let parser = if let Some(parser_box) = PARSER_FACTORY.registry().create_for_model(type_str) {
+        parser_box
+    } else if let Some(parser_box) = PARSER_FACTORY.registry().create_parser(type_str) {
+        parser_box
+    } else {
+        set_error_message(error_out, &format!("Unknown parser type: {}", type_str));
+        return ptr::null_mut();
+    };
+
+    Box::into_raw(Box::new(ToolParserHandle {
+        parser: Arc::new(tokio::sync::Mutex::new(parser)),
+        model: type_str.to_string(),
+        history_tool_calls_count: 0,
+        tool_index_to_id: HashMap::new(),
+    }))
+}
+
+/// Parse complete tool calls from text
+///
+/// # Arguments
+/// * `handle` - Tool parser handle
+/// * `text` - Input text to parse
+/// * `result_json_out` - Pointer to receive JSON result (must be freed with sgl_free_string)
+/// * `error_out` - Optional pointer to receive error message
+///
+/// # Returns
+/// * SglErrorCode::Success on success, error code on failure
+#[no_mangle]
+pub unsafe extern "C" fn sgl_tool_parser_parse_complete(
+    handle: *mut ToolParserHandle,
+    text: *const c_char,
+    result_json_out: *mut *mut c_char,
+    error_out: *mut *mut c_char,
+) -> SglErrorCode {
+    if handle.is_null() || text.is_null() || result_json_out.is_null() {
+        set_error_message(error_out, "Invalid arguments: null pointer");
+        return SglErrorCode::InvalidArgument;
+    }
+
+    let text_str = match CStr::from_ptr(text).to_str() {
+        Ok(s) => s,
+        Err(_) => {
+            set_error_message(error_out, "Invalid UTF-8 in text");
+            return SglErrorCode::InvalidArgument;
+        }
+    };
+
+    let handle_ref = &*handle;
+    let parser = Arc::clone(&handle_ref.parser);
+    let model = handle_ref.model.clone();
+    let history_count = handle_ref.history_tool_calls_count;
+
+    // Use tokio runtime to run async code
+    let result = RUNTIME.block_on(async {
+        let parser_guard = parser.lock().await;
+        parser_guard.parse_complete(text_str).await
+    });
+
+    match result {
+        Ok((normal_text, tool_calls)) => {
+            // Convert Rust ToolCall to OpenAI format
+            let openai_tool_calls: Vec<Value> = tool_calls
+                .into_iter()
+                .enumerate()
+                .map(|(index, tc)| {
+                    // Generate ID for this tool call
+                    let id = generate_tool_call_id(&model, &tc.function.name, index, history_count);
+                    json!({
+                        "id": id,
+                        "type": "function",
+                        "function": {
+                            "name": tc.function.name,
+                            "arguments": tc.function.arguments
+                        }
+                    })
+                })
+                .collect();
+
+            // Build result JSON
+            let result_json = json!({
+                "normal_text": normal_text,
+                "tool_calls": openai_tool_calls
+            });
+
+            let result_str = match serde_json::to_string(&result_json) {
+                Ok(s) => s,
+                Err(e) => {
+                    set_error_message(error_out, &format!("Failed to serialize JSON: {}", e));
+                    return SglErrorCode::ParsingError;
+                }
+            };
+
+            let result_cstr = match CString::new(result_str) {
+                Ok(s) => s,
+                Err(e) => {
+                    set_error_message(error_out, &format!("Failed to create result string: {}", e));
+                    return SglErrorCode::MemoryError;
+                }
+            };
+
+            *result_json_out = result_cstr.into_raw();
+            clear_error_message(error_out);
+            SglErrorCode::Success
+        }
+        Err(e) => {
+            set_error_message(error_out, &format!("Parse error: {}", e));
+            SglErrorCode::ParsingError
+        }
+    }
+}
+
+/// Parse tool calls incrementally from streaming chunks
+///
+/// # Arguments
+/// * `handle` - Tool parser handle
+/// * `chunk` - New text chunk from stream
+/// * `tools_json` - JSON array of available tools (for validation, can be null/empty)
+/// * `result_json_out` - Pointer to receive JSON result (must be freed with sgl_free_string)
+/// * `error_out` - Optional pointer to receive error message
+///
+/// # Returns
+/// * SglErrorCode::Success on success, error code on failure
+#[no_mangle]
+pub unsafe extern "C" fn sgl_tool_parser_parse_incremental(
+    handle: *mut ToolParserHandle,
+    chunk: *const c_char,
+    tools_json: *const c_char,
+    result_json_out: *mut *mut c_char,
+    error_out: *mut *mut c_char,
+) -> SglErrorCode {
+    if handle.is_null() || chunk.is_null() || result_json_out.is_null() {
+        set_error_message(error_out, "Invalid arguments: null pointer");
+        return SglErrorCode::InvalidArgument;
+    }
+
+    let chunk_str = match CStr::from_ptr(chunk).to_str() {
+        Ok(s) => s,
+        Err(_) => {
+            set_error_message(error_out, "Invalid UTF-8 in chunk");
+            return SglErrorCode::InvalidArgument;
+        }
+    };
+
+    // Parse tools JSON if provided
+    let tools: Vec<Tool> = if !tools_json.is_null() {
+        let tools_str = match CStr::from_ptr(tools_json).to_str() {
+            Ok(s) => s,
+            Err(_) => {
+                set_error_message(error_out, "Invalid UTF-8 in tools_json");
+                return SglErrorCode::InvalidArgument;
+            }
+        };
+        match serde_json::from_str::<Vec<Tool>>(tools_str) {
+            Ok(t) => t,
+            Err(_) => vec![], // If parsing fails, use empty tools
+        }
+    } else {
+        vec![]
+    };
+
+    let handle_ref = &*handle;
+    let parser = Arc::clone(&handle_ref.parser);
+    let model = handle_ref.model.clone();
+    let history_count = handle_ref.history_tool_calls_count;
+
+    // Use tokio runtime to run async code
+    let result = RUNTIME.block_on(async {
+        let mut parser_guard = parser.lock().await;
+        parser_guard.parse_incremental(chunk_str, &tools).await
+    });
+
+    match result {
+        Ok(streaming_result) => {
+            // Convert StreamingParseResult to OpenAI format
+            let handle_mut = &mut *handle;
+            let openai_tool_calls: Vec<Value> = streaming_result
+                .calls
+                .into_iter()
+                .map(|item| {
+                    // For incremental parsing, we may not have complete tool calls yet
+                    // Generate or reuse ID based on tool_index
+                    let id = if let Some(ref name) = item.name {
+                        // New tool call with name - generate ID and store it
+                        let id = generate_tool_call_id(&model, name, item.tool_index, history_count);
+                        handle_mut.tool_index_to_id.insert(item.tool_index, id.clone());
+                        id
+                    } else {
+                        // Parameter update - reuse existing ID for this tool_index
+                        handle_mut.tool_index_to_id
+                            .get(&item.tool_index)
+                            .cloned()
+                            .unwrap_or_else(|| format!("call_{}", item.tool_index))
+                    };
+
+                    json!({
+                        "id": id,
+                        "type": "function",
+                        "function": {
+                            "name": item.name.unwrap_or_default(),
+                            "arguments": item.parameters
+                        }
+                    })
+                })
+                .collect();
+
+            // Build result JSON
+            let result_json = json!({
+                "normal_text": streaming_result.normal_text,
+                "tool_calls": openai_tool_calls
+            });
+
+            let result_str = match serde_json::to_string(&result_json) {
+                Ok(s) => s,
+                Err(e) => {
+                    set_error_message(error_out, &format!("Failed to serialize JSON: {}", e));
+                    return SglErrorCode::ParsingError;
+                }
+            };
+
+            let result_cstr = match CString::new(result_str) {
+                Ok(s) => s,
+                Err(e) => {
+                    set_error_message(error_out, &format!("Failed to create result string: {}", e));
+                    return SglErrorCode::MemoryError;
+                }
+            };
+
+            *result_json_out = result_cstr.into_raw();
+            clear_error_message(error_out);
+            SglErrorCode::Success
+        }
+        Err(e) => {
+            set_error_message(error_out, &format!("Parse incremental error: {}", e));
+            SglErrorCode::ParsingError
+        }
+    }
+}
+
+/// Reset the parser state for reuse
+#[no_mangle]
+pub unsafe extern "C" fn sgl_tool_parser_reset(handle: *mut ToolParserHandle) {
+    if handle.is_null() {
+        return;
+    }
+
+    let handle_ref = &mut *handle;
+    let parser = Arc::clone(&handle_ref.parser);
+
+    // Reset parser state
+    RUNTIME.block_on(async {
+        let mut parser_guard = parser.lock().await;
+        parser_guard.reset();
+    });
+
+    // Reset history count and tool index mapping
+    handle_ref.history_tool_calls_count = 0;
+    handle_ref.tool_index_to_id.clear();
+}
+
+/// Free a tool parser handle
+#[no_mangle]
+pub unsafe extern "C" fn sgl_tool_parser_free(handle: *mut ToolParserHandle) {
+    if !handle.is_null() {
+        let _ = Box::from_raw(handle);
+    }
+}

third_party/sglang/sgl-model-gateway/bindings/golang/src/utils.rs

@@ -0,0 +1,44 @@
+//! Utility functions for FFI
+
+use uuid::Uuid;
+
+/// Helper function to generate tool call ID (matches router implementation)
+pub fn generate_tool_call_id(
+    model: &str,
+    function_name: &str,
+    index: usize,
+    history_tool_calls_count: usize,
+) -> String {
+    if model.to_lowercase().contains("kimi") {
+        // KimiK2 format: functions.{name}:{global_index}
+        format!("functions.{}:{}", function_name, history_tool_calls_count + index)
+    } else {
+        // Standard OpenAI format: call_{24-char-uuid}
+        format!("call_{}", &Uuid::new_v4().simple().to_string()[..24])
+    }
+}
+
+/// Generate tool constraints (placeholder implementation)
+///
+/// # Arguments
+/// * `tools_json` - JSON array of tools
+/// * `tool_choice_json` - JSON object representing tool_choice
+/// * `constraint_type_out` - Pointer to receive constraint type (e.g., "json_schema")
+/// * `constraint_schema_out` - Pointer to receive constraint schema JSON
+/// * `error_out` - Optional pointer to receive error message
+///
+/// # Returns
+/// * SglErrorCode::Success on success, error code on failure
+#[no_mangle]
+pub unsafe extern "C" fn sgl_generate_tool_constraints(
+    _tools_json: *const std::os::raw::c_char,
+    _tool_choice_json: *const std::os::raw::c_char,
+    _constraint_type_out: *mut *mut std::os::raw::c_char,
+    _constraint_schema_out: *mut *mut std::os::raw::c_char,
+    error_out: *mut *mut std::os::raw::c_char,
+) -> super::error::SglErrorCode {
+    // Implementation would parse JSON and call generate_tool_constraints
+    // This is a placeholder
+    super::error::set_error_message(error_out, "Tool constraint generation not yet implemented in FFI");
+    super::error::SglErrorCode::UnknownError
+}