chore: vendor sglang v0.5.10 snapshot

third_party/sglang/sgl-model-gateway/bindings/golang/internal/ffi/batch_postprocessor.go

@@ -0,0 +1,126 @@
+// Package ffi provides Go bindings for SGLang's Rust FFI (Foreign Function Interface).
+package ffi
+
+import (
+	"encoding/json"
+	"fmt"
+	"strings"
+	"time"
+)
+
+// BatchPostprocessor handles batch postprocessing of stream chunks to reduce FFI overhead
+type BatchPostprocessor struct {
+	converter     *GrpcResponseConverterHandle
+	buffer        []string
+	batchSize     int
+	flushInterval time.Duration
+	lastFlush     time.Time
+	timer         *time.Timer
+}
+
+// NewBatchPostprocessor creates a new batch postprocessor
+func NewBatchPostprocessor(converter *GrpcResponseConverterHandle, batchSize int, flushInterval time.Duration) *BatchPostprocessor {
+	if batchSize <= 0 {
+		batchSize = 1
+	}
+	if flushInterval < 0 {
+		flushInterval = 0
+	}
+
+	return &BatchPostprocessor{
+		converter:     converter,
+		buffer:        make([]string, 0, batchSize),
+		batchSize:     batchSize,
+		flushInterval: flushInterval,
+		lastFlush:     time.Now(),
+	}
+}
+
+// AddChunk adds a chunk to the buffer and processes if batch is full
+func (b *BatchPostprocessor) AddChunk(chunkJSON string) (results []string, shouldFlush bool, err error) {
+	if b.batchSize == 1 {
+		openaiJSON, _, err := PostprocessStreamChunk(b.converter, chunkJSON)
+		if err != nil {
+			return nil, false, err
+		}
+		return []string{openaiJSON}, false, nil
+	}
+
+	b.buffer = append(b.buffer, chunkJSON)
+	shouldProcess := len(b.buffer) >= b.batchSize
+	shouldFlushTimeout := b.flushInterval > 0 && time.Since(b.lastFlush) >= b.flushInterval
+
+	if shouldProcess || shouldFlushTimeout {
+		return b.processBatch()
+	}
+
+	return nil, false, nil
+}
+
+// Flush processes any remaining chunks in the buffer
+func (b *BatchPostprocessor) Flush() (results []string, err error) {
+	if len(b.buffer) == 0 {
+		return nil, nil
+	}
+
+	res, _, err := b.processBatch()
+	return res, err
+}
+
+// processBatch processes the current buffer and returns results
+func (b *BatchPostprocessor) processBatch() (results []string, shouldFlush bool, err error) {
+	if len(b.buffer) == 0 {
+		return nil, false, nil
+	}
+
+	var sb strings.Builder
+	sb.Grow(len(b.buffer) * 200)
+	sb.WriteString(`[`)
+	for i, chunkJSONStr := range b.buffer {
+		if i > 0 {
+			sb.WriteString(`,`)
+		}
+		sb.WriteString(chunkJSONStr)
+	}
+	sb.WriteString(`]`)
+	bufferJSON := sb.String()
+
+	resultJSON, _, err := PostprocessStreamChunksBatch(
+		b.converter,
+		bufferJSON,
+		b.batchSize*2,
+	)
+	if err != nil {
+		return nil, false, fmt.Errorf("batch postprocessing failed: %w", err)
+	}
+
+	var resultArray []json.RawMessage
+	if err := json.Unmarshal([]byte(resultJSON), &resultArray); err != nil {
+		return nil, false, fmt.Errorf("failed to unmarshal results array: %w", err)
+	}
+
+	resultStrings := make([]string, 0, len(resultArray))
+	for _, rawMsg := range resultArray {
+		resultStrings = append(resultStrings, string(rawMsg))
+	}
+
+	b.buffer = b.buffer[:0]
+	b.lastFlush = time.Now()
+
+	if b.timer != nil {
+		b.timer.Stop()
+		b.timer = nil
+	}
+
+	return resultStrings, false, nil
+}
+
+// Reset clears the buffer and resets the postprocessor state
+func (b *BatchPostprocessor) Reset() {
+	b.buffer = b.buffer[:0]
+	b.lastFlush = time.Now()
+	if b.timer != nil {
+		b.timer.Stop()
+		b.timer = nil
+	}
+}

third_party/sglang/sgl-model-gateway/bindings/golang/internal/ffi/client.go

@@ -0,0 +1,228 @@
+// Package ffi provides Go bindings for SGLang's Rust FFI (Foreign Function Interface).
+//
+// This package wraps the Rust FFI layer of SGLang, providing low-level access to:
+// - Client creation and connection management
+// - Chat completion streaming
+// - Stream reading and response conversion
+// - Memory management for C strings
+//
+// Internal use only: This package is intended for internal use by the sglang package.
+// End users should use the public sglang package instead.
+package ffi
+
+/*
+#cgo LDFLAGS: -lsgl_model_gateway_go -ldl
+#include <stdlib.h>
+#include <stdint.h>
+
+// Error codes
+typedef enum {
+    SGL_ERROR_SUCCESS = 0,
+    SGL_ERROR_INVALID_ARGUMENT = 1,
+    SGL_ERROR_TOKENIZATION_ERROR = 2,
+    SGL_ERROR_PARSING_ERROR = 3,
+    SGL_ERROR_MEMORY_ERROR = 4,
+    SGL_ERROR_UNKNOWN = 99
+} SglErrorCode;
+
+// Opaque handles
+typedef void* SglangClientHandle;
+typedef void* SglangStreamHandle;
+
+// Client SDK functions
+SglangClientHandle* sgl_client_create(const char* endpoint, const char* tokenizer_path, char** error_out);
+void sgl_client_free(SglangClientHandle* handle);
+SglErrorCode sgl_client_chat_completion_stream(SglangClientHandle* client_handle, const char* request_json, SglangStreamHandle** stream_handle_out, char** error_out);
+SglErrorCode sgl_stream_read_next(SglangStreamHandle* stream_handle, char** response_json_out, int* is_done_out, char** error_out);
+void sgl_stream_free(SglangStreamHandle* handle);
+void sgl_free_string(char* s);
+*/
+import "C"
+
+import (
+	"fmt"
+	"unsafe"
+)
+
+// ErrorCode represents FFI error codes returned by Rust functions.
+//
+// These codes indicate the result of FFI operations. Use Error() to get a human-readable
+// error message.
+type ErrorCode int
+
+const (
+	// ErrorSuccess indicates the operation completed successfully
+	ErrorSuccess ErrorCode = 0
+	// ErrorInvalidArgument indicates invalid arguments were passed to the FFI function
+	ErrorInvalidArgument ErrorCode = 1
+	// ErrorTokenizationError indicates an error during tokenization
+	ErrorTokenizationError ErrorCode = 2
+	// ErrorParsingError indicates an error parsing the response or request
+	ErrorParsingError ErrorCode = 3
+	// ErrorMemoryError indicates a memory allocation error
+	ErrorMemoryError ErrorCode = 4
+	// ErrorUnknown indicates an unclassified error
+	ErrorUnknown ErrorCode = 99
+)
+
+// Error implements the error interface for ErrorCode.
+func (e ErrorCode) Error() string {
+	switch e {
+	case ErrorSuccess:
+		return "success"
+	case ErrorInvalidArgument:
+		return "invalid argument"
+	case ErrorTokenizationError:
+		return "tokenization error"
+	case ErrorParsingError:
+		return "parsing error"
+	case ErrorMemoryError:
+		return "memory error"
+	case ErrorUnknown:
+		return "unknown error"
+	default:
+		return fmt.Sprintf("unknown error code: %d", e)
+	}
+}
+
+// SglangClientHandle wraps the Rust client SDK FFI handle.
+//
+// This struct maintains a connection to the SGLang gRPC server and is used
+// to create streams and manage the underlying Rust client resources.
+type SglangClientHandle struct {
+	handle *C.SglangClientHandle
+}
+
+// NewClient creates a new SGLang client handle via FFI.
+//
+// This function initializes the Rust client with the given endpoint and tokenizer path.
+//
+// Parameters:
+// - endpoint: gRPC endpoint URL (e.g., "grpc://localhost:20000")
+// - tokenizerPath: Path to tokenizer directory
+//
+// Returns:
+// - *SglangClientHandle: A new client handle
+// - error: An error if client creation failed
+func NewClient(endpoint, tokenizerPath string) (*SglangClientHandle, error) {
+	cEndpoint := C.CString(endpoint)
+	defer C.free(unsafe.Pointer(cEndpoint))
+
+	cTokenizerPath := C.CString(tokenizerPath)
+	defer C.free(unsafe.Pointer(cTokenizerPath))
+
+	var errorPtr *C.char
+	handle := C.sgl_client_create(cEndpoint, cTokenizerPath, &errorPtr)
+
+	if handle == nil {
+		errorMsg := ""
+		if errorPtr != nil {
+			errorMsg = C.GoString(errorPtr)
+			C.sgl_free_string(errorPtr)
+		}
+		if errorMsg == "" {
+			errorMsg = "failed to create client"
+		}
+		return nil, fmt.Errorf("%s", errorMsg)
+	}
+
+	return &SglangClientHandle{handle: handle}, nil
+}
+
+// Free releases the client handle
+func (h *SglangClientHandle) Free() {
+	if h.handle != nil {
+		C.sgl_client_free(h.handle)
+		h.handle = nil
+	}
+}
+
+// ChatCompletionStream creates a streaming chat completion request
+func (h *SglangClientHandle) ChatCompletionStream(requestJSON string) (*SglangStreamHandle, error) {
+	if h.handle == nil {
+		return nil, fmt.Errorf("client handle is nil")
+	}
+
+	cRequestJSON := C.CString(requestJSON)
+	defer C.free(unsafe.Pointer(cRequestJSON))
+
+	var streamHandle *C.SglangStreamHandle
+	var errorPtr *C.char
+
+	result := C.sgl_client_chat_completion_stream(
+		h.handle,
+		cRequestJSON,
+		&streamHandle,
+		&errorPtr,
+	)
+
+	if ErrorCode(result) != ErrorSuccess {
+		errorMsg := ""
+		if errorPtr != nil {
+			errorMsg = C.GoString(errorPtr)
+			C.sgl_free_string(errorPtr)
+		}
+		if errorMsg == "" {
+			errorMsg = fmt.Sprintf("error code %d", result)
+		}
+		return nil, fmt.Errorf("%s", errorMsg)
+	}
+
+	if streamHandle == nil {
+		return nil, fmt.Errorf("stream handle is nil")
+	}
+
+	return &SglangStreamHandle{handle: streamHandle}, nil
+}
+
+// SglangStreamHandle wraps the Rust stream FFI handle
+type SglangStreamHandle struct {
+	handle *C.SglangStreamHandle
+}
+
+// ReadNext reads the next chunk from the stream
+// Returns: (responseJSON, isDone, error)
+func (h *SglangStreamHandle) ReadNext() (string, bool, error) {
+	if h.handle == nil {
+		return "", true, fmt.Errorf("stream handle is nil")
+	}
+
+	var responseJSON *C.char
+	var isDone C.int
+	var errorPtr *C.char
+
+	result := C.sgl_stream_read_next(
+		h.handle,
+		&responseJSON,
+		&isDone,
+		&errorPtr,
+	)
+
+	if ErrorCode(result) != ErrorSuccess {
+		errorMsg := ""
+		if errorPtr != nil {
+			errorMsg = C.GoString(errorPtr)
+			C.sgl_free_string(errorPtr)
+		}
+		if errorMsg == "" {
+			errorMsg = fmt.Sprintf("error code %d", result)
+		}
+		return "", isDone == 1, fmt.Errorf("%s", errorMsg)
+	}
+
+	responseStr := ""
+	if responseJSON != nil {
+		responseStr = C.GoString(responseJSON)
+		C.sgl_free_string(responseJSON)
+	}
+
+	return responseStr, isDone == 1, nil
+}
+
+// Free releases the stream handle
+func (h *SglangStreamHandle) Free() {
+	if h.handle != nil {
+		C.sgl_stream_free(h.handle)
+		h.handle = nil
+	}
+}

third_party/sglang/sgl-model-gateway/bindings/golang/internal/ffi/grpc_converter.go

@@ -0,0 +1,275 @@
+package ffi
+
+/*
+#cgo LDFLAGS: -lsgl_model_gateway_go -ldl
+#include <stdlib.h>
+#include <stdint.h>
+
+// Error codes (must match client.go)
+typedef enum {
+    SGL_ERROR_SUCCESS = 0,
+    SGL_ERROR_INVALID_ARGUMENT = 1,
+    SGL_ERROR_TOKENIZATION_ERROR = 2,
+    SGL_ERROR_PARSING_ERROR = 3,
+    SGL_ERROR_MEMORY_ERROR = 4,
+    SGL_ERROR_UNKNOWN = 99
+} SglErrorCode;
+
+// Opaque handles
+typedef void* TokenizerHandle;
+typedef void* GrpcResponseConverterHandle;
+
+// Converter functions
+GrpcResponseConverterHandle* sgl_grpc_response_converter_create(
+    TokenizerHandle* tokenizer_handle,
+    const char* model,
+    const char* request_id,
+    const char* tools_json,
+    const char* tool_choice_json,
+    const char* stop,
+    const char* stop_token_ids,
+    int skip_special_tokens,
+    int initial_prompt_tokens,
+    char** error_out
+);
+
+void sgl_grpc_response_converter_free(GrpcResponseConverterHandle* handle);
+
+// Tokenizer functions
+TokenizerHandle* sgl_tokenizer_create_from_file(const char* tokenizer_path, char** error_out);
+void sgl_tokenizer_free(TokenizerHandle* handle);
+
+// Memory management
+void sgl_free_string(char* s);
+*/
+import "C"
+
+import (
+	"fmt"
+	"unsafe"
+)
+
+// CreateGrpcResponseConverter creates a gRPC response converter handle
+// This function creates a new tokenizer handle each time (for backward compatibility)
+// For better performance, use CreateGrpcResponseConverterWithTokenizer with a cached tokenizer
+func CreateGrpcResponseConverter(
+	tokenizerPath string,
+	model string,
+	requestID string,
+	toolsJSON string,
+	toolChoiceJSON string,
+	stopJSON string,
+	stopTokenIDs []uint32,
+	skipSpecialTokens bool,
+	initialPromptTokens int32,
+) (*GrpcResponseConverterHandle, error) {
+	// Create tokenizer handle
+	tokenizerHandle, err := createTokenizerHandle(tokenizerPath)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create tokenizer handle: %w", err)
+	}
+	defer C.sgl_tokenizer_free(tokenizerHandle)
+
+	return createGrpcResponseConverterWithTokenizerHandle(
+		tokenizerHandle,
+		model,
+		requestID,
+		toolsJSON,
+		toolChoiceJSON,
+		stopJSON,
+		stopTokenIDs,
+		skipSpecialTokens,
+		initialPromptTokens,
+	)
+}
+
+// CreateGrpcResponseConverterWithTokenizer creates a gRPC response converter handle using a cached tokenizer
+// This is more efficient as it reuses the tokenizer instead of creating a new one each time
+func CreateGrpcResponseConverterWithTokenizer(
+	tokenizerHandle *TokenizerHandle,
+	model string,
+	requestID string,
+	toolsJSON string,
+	toolChoiceJSON string,
+	stopJSON string,
+	stopTokenIDs []uint32,
+	skipSpecialTokens bool,
+	initialPromptTokens int32,
+) (*GrpcResponseConverterHandle, error) {
+	if tokenizerHandle == nil || tokenizerHandle.handle == nil {
+		return nil, fmt.Errorf("invalid tokenizer handle")
+	}
+
+	return createGrpcResponseConverterWithTokenizerHandle(
+		tokenizerHandle.handle,
+		model,
+		requestID,
+		toolsJSON,
+		toolChoiceJSON,
+		stopJSON,
+		stopTokenIDs,
+		skipSpecialTokens,
+		initialPromptTokens,
+	)
+}
+
+// createGrpcResponseConverterWithTokenizerHandle is the internal implementation
+func createGrpcResponseConverterWithTokenizerHandle(
+	tokenizerHandle *C.TokenizerHandle,
+	model string,
+	requestID string,
+	toolsJSON string,
+	toolChoiceJSON string,
+	stopJSON string,
+	stopTokenIDs []uint32,
+	skipSpecialTokens bool,
+	initialPromptTokens int32,
+) (*GrpcResponseConverterHandle, error) {
+
+	// Convert strings to C strings
+	modelC := C.CString(model)
+	defer C.free(unsafe.Pointer(modelC))
+
+	requestIDC := C.CString(requestID)
+	defer C.free(unsafe.Pointer(requestIDC))
+
+	var toolsJSONC *C.char
+	if toolsJSON != "" {
+		toolsJSONC = C.CString(toolsJSON)
+		defer C.free(unsafe.Pointer(toolsJSONC))
+	}
+
+	var toolChoiceJSONC *C.char
+	if toolChoiceJSON != "" {
+		toolChoiceJSONC = C.CString(toolChoiceJSON)
+		defer C.free(unsafe.Pointer(toolChoiceJSONC))
+	}
+
+	var stopJSONC *C.char
+	if stopJSON != "" {
+		stopJSONC = C.CString(stopJSON)
+		defer C.free(unsafe.Pointer(stopJSONC))
+	}
+
+	// Convert stop_token_ids to JSON string
+	stopTokenIDsJSON := ""
+	if len(stopTokenIDs) > 0 {
+		stopTokenIDsJSON = fmt.Sprintf("[%d", stopTokenIDs[0])
+		for i := 1; i < len(stopTokenIDs); i++ {
+			stopTokenIDsJSON += fmt.Sprintf(",%d", stopTokenIDs[i])
+		}
+		stopTokenIDsJSON += "]"
+	}
+
+	var stopTokenIDsJSONC *C.char
+	if stopTokenIDsJSON != "" {
+		stopTokenIDsJSONC = C.CString(stopTokenIDsJSON)
+		defer C.free(unsafe.Pointer(stopTokenIDsJSONC))
+	}
+
+	var errorOut *C.char
+	skipSpecialTokensC := C.int(0)
+	if skipSpecialTokens {
+		skipSpecialTokensC = C.int(1)
+	}
+
+	initialPromptTokensC := C.int(initialPromptTokens)
+
+	converterHandle := C.sgl_grpc_response_converter_create(
+		tokenizerHandle,
+		modelC,
+		requestIDC,
+		toolsJSONC,
+		toolChoiceJSONC,
+		stopJSONC,
+		stopTokenIDsJSONC,
+		skipSpecialTokensC,
+		initialPromptTokensC,
+		&errorOut,
+	)
+
+	if converterHandle == nil {
+		errorMsg := ""
+		if errorOut != nil {
+			errorMsg = C.GoString(errorOut)
+			C.sgl_free_string(errorOut)
+		}
+		if errorMsg == "" {
+			errorMsg = "failed to create converter handle"
+		}
+		return nil, fmt.Errorf("%s", errorMsg)
+	}
+
+	return &GrpcResponseConverterHandle{
+		handle: converterHandle,
+	}, nil
+}
+
+// FreeGrpcResponseConverter frees a gRPC response converter handle
+func FreeGrpcResponseConverter(handle *GrpcResponseConverterHandle) {
+	if handle != nil && handle.handle != nil {
+		C.sgl_grpc_response_converter_free(handle.handle)
+		handle.handle = nil
+	}
+}
+
+// TokenizerHandle wraps the Rust tokenizer FFI handle
+type TokenizerHandle struct {
+	handle *C.TokenizerHandle
+}
+
+// CreateTokenizerHandle creates a tokenizer handle (exported for caching)
+func CreateTokenizerHandle(tokenizerPath string) (*TokenizerHandle, error) {
+	tokenizerPathC := C.CString(tokenizerPath)
+	defer C.free(unsafe.Pointer(tokenizerPathC))
+
+	var errorOut *C.char
+	tokenizerHandle := C.sgl_tokenizer_create_from_file(tokenizerPathC, &errorOut)
+
+	if tokenizerHandle == nil {
+		errorMsg := ""
+		if errorOut != nil {
+			errorMsg = C.GoString(errorOut)
+			C.sgl_free_string(errorOut)
+		}
+		if errorMsg == "" {
+			errorMsg = "failed to create tokenizer handle"
+		}
+		return nil, fmt.Errorf("%s", errorMsg)
+	}
+
+	return &TokenizerHandle{
+		handle: tokenizerHandle,
+	}, nil
+}
+
+// FreeTokenizerHandle frees a tokenizer handle
+func FreeTokenizerHandle(handle *TokenizerHandle) {
+	if handle != nil && handle.handle != nil {
+		C.sgl_tokenizer_free(handle.handle)
+		handle.handle = nil
+	}
+}
+
+// createTokenizerHandle creates a tokenizer handle (helper function, internal use)
+func createTokenizerHandle(tokenizerPath string) (*C.TokenizerHandle, error) {
+	tokenizerPathC := C.CString(tokenizerPath)
+	defer C.free(unsafe.Pointer(tokenizerPathC))
+
+	var errorOut *C.char
+	tokenizerHandle := C.sgl_tokenizer_create_from_file(tokenizerPathC, &errorOut)
+
+	if tokenizerHandle == nil {
+		errorMsg := ""
+		if errorOut != nil {
+			errorMsg = C.GoString(errorOut)
+			C.sgl_free_string(errorOut)
+		}
+		if errorMsg == "" {
+			errorMsg = "failed to create tokenizer handle"
+		}
+		return nil, fmt.Errorf("%s", errorMsg)
+	}
+
+	return tokenizerHandle, nil
+}

third_party/sglang/sgl-model-gateway/bindings/golang/internal/ffi/postprocessor.go

@@ -0,0 +1,156 @@
+// Package ffi provides Go bindings for SGLang's Rust FFI (Foreign Function Interface).
+package ffi
+
+/*
+#cgo LDFLAGS: -lsgl_model_gateway_go -ldl
+#include <stdlib.h>
+#include <stdint.h>
+
+// Error codes (must match client.go)
+typedef enum {
+    SGL_ERROR_SUCCESS = 0,
+    SGL_ERROR_INVALID_ARGUMENT = 1,
+    SGL_ERROR_TOKENIZATION_ERROR = 2,
+    SGL_ERROR_PARSING_ERROR = 3,
+    SGL_ERROR_MEMORY_ERROR = 4,
+    SGL_ERROR_UNKNOWN = 99
+} SglErrorCode;
+
+// Opaque handle (must match grpc_converter.go)
+typedef void* GrpcResponseConverterHandle;
+
+// Postprocessor functions
+SglErrorCode sgl_postprocess_stream_chunk(
+    GrpcResponseConverterHandle* converter_handle,
+    const char* proto_chunk_json,
+    char** openai_json_out,
+    int* is_done_out,
+    char** error_out
+);
+
+SglErrorCode sgl_postprocess_stream_chunks_batch(
+    GrpcResponseConverterHandle* converter_handle,
+    const char* proto_chunks_json_array,
+    int max_chunks,
+    char** openai_chunks_json_array_out,
+    int* chunks_count_out,
+    char** error_out
+);
+
+// Memory management
+void sgl_free_string(char* s);
+*/
+import "C"
+
+import (
+	"fmt"
+	"unsafe"
+)
+
+// GrpcResponseConverterHandle wraps the Rust gRPC response converter FFI handle
+type GrpcResponseConverterHandle struct {
+	handle *C.GrpcResponseConverterHandle
+}
+
+// PostprocessStreamChunk postprocesses a gRPC stream chunk to OpenAI format
+//
+// This function:
+// 1. Parses the proto chunk from JSON
+// 2. Converts it to OpenAI format using the converter handle
+// 3. Returns the OpenAI format JSON
+//
+// Returns the OpenAI format JSON, is_done flag, and any error.
+func PostprocessStreamChunk(converterHandle *GrpcResponseConverterHandle, protoChunkJSON string) (openaiJSON string, isDone bool, err error) {
+	if converterHandle == nil || converterHandle.handle == nil {
+		return "", false, fmt.Errorf("invalid converter handle")
+	}
+
+	protoChunkJSONC := C.CString(protoChunkJSON)
+	defer C.free(unsafe.Pointer(protoChunkJSONC))
+
+	var openaiJSONOut *C.char
+	var isDoneOut C.int
+	var errorOut *C.char
+
+	errorCode := C.sgl_postprocess_stream_chunk(
+		converterHandle.handle,
+		protoChunkJSONC,
+		&openaiJSONOut,
+		&isDoneOut,
+		&errorOut,
+	)
+
+	if errorCode != C.SGL_ERROR_SUCCESS {
+		errorMsg := ""
+		if errorOut != nil {
+			errorMsg = C.GoString(errorOut)
+			C.sgl_free_string(errorOut)
+		}
+		return "", false, fmt.Errorf("postprocessing failed: %s", errorMsg)
+	}
+
+	openaiJSON = C.GoString(openaiJSONOut)
+	isDone = isDoneOut != 0
+
+	// Free the C string allocated by Rust
+	if openaiJSONOut != nil {
+		C.sgl_free_string(openaiJSONOut)
+	}
+
+	return openaiJSON, isDone, nil
+}
+
+// PostprocessStreamChunksBatch postprocesses multiple gRPC stream chunks in batch
+//
+// This function processes multiple chunks in a single FFI call, significantly reducing
+// FFI overhead in streaming scenarios.
+//
+// Arguments:
+// - converterHandle: Converter handle
+// - protoChunksJSONArray: JSON array string of proto chunks
+// - maxChunks: Maximum number of chunks to process (for safety, typically 10-20)
+//
+// Returns:
+// - openaiChunksJSONArray: JSON array of OpenAI format chunks
+// - chunksCount: Number of processed chunks
+// - error: Any error that occurred
+func PostprocessStreamChunksBatch(converterHandle *GrpcResponseConverterHandle, protoChunksJSONArray string, maxChunks int) (openaiChunksJSONArray string, chunksCount int, err error) {
+	if converterHandle == nil || converterHandle.handle == nil {
+		return "", 0, fmt.Errorf("invalid converter handle")
+	}
+
+	protoChunksJSONArrayC := C.CString(protoChunksJSONArray)
+	defer C.free(unsafe.Pointer(protoChunksJSONArrayC))
+
+	var openaiChunksJSONArrayOut *C.char
+	var chunksCountOut C.int
+	var errorOut *C.char
+
+	errorCode := C.sgl_postprocess_stream_chunks_batch(
+		converterHandle.handle,
+		protoChunksJSONArrayC,
+		C.int(maxChunks),
+		&openaiChunksJSONArrayOut,
+		&chunksCountOut,
+		&errorOut,
+	)
+
+	if errorCode != C.SGL_ERROR_SUCCESS {
+		errorMsg := ""
+		if errorOut != nil {
+			errorMsg = C.GoString(errorOut)
+			C.sgl_free_string(errorOut)
+		}
+		return "", 0, fmt.Errorf("batch postprocessing failed: %s", errorMsg)
+	}
+
+	openaiChunksJSONArray = C.GoString(openaiChunksJSONArrayOut)
+	chunksCount = int(chunksCountOut)
+
+	// Free the C string allocated by Rust
+	if openaiChunksJSONArrayOut != nil {
+		C.sgl_free_string(openaiChunksJSONArrayOut)
+	}
+
+	return openaiChunksJSONArray, chunksCount, nil
+}

third_party/sglang/sgl-model-gateway/bindings/golang/internal/ffi/preprocessor.go

@@ -0,0 +1,246 @@
+// Package ffi provides Go bindings for SGLang's Rust FFI (Foreign Function Interface).
+package ffi
+
+/*
+#cgo LDFLAGS: -lsgl_model_gateway_go -ldl
+#include <stdlib.h>
+#include <stdint.h>
+
+// Error codes (must match client.go)
+typedef enum {
+    SGL_ERROR_SUCCESS = 0,
+    SGL_ERROR_INVALID_ARGUMENT = 1,
+    SGL_ERROR_TOKENIZATION_ERROR = 2,
+    SGL_ERROR_PARSING_ERROR = 3,
+    SGL_ERROR_MEMORY_ERROR = 4,
+    SGL_ERROR_UNKNOWN = 99
+} SglErrorCode;
+
+// Preprocessor functions
+SglErrorCode sgl_preprocess_chat_request(
+    const char* request_json,
+    const char* tokenizer_path,
+    char** prompt_text_out,
+    uint32_t** token_ids_out,
+    size_t* token_ids_len_out,
+    char** tool_constraints_json_out,
+    int32_t* prompt_tokens_out,
+    char** error_out
+);
+
+// Opaque handle (must match grpc_converter.go)
+typedef void* TokenizerHandle;
+
+SglErrorCode sgl_preprocess_chat_request_with_tokenizer(
+    const char* request_json,
+    void* tokenizer_handle,
+    char** prompt_text_out,
+    uint32_t** token_ids_out,
+    size_t* token_ids_len_out,
+    char** tool_constraints_json_out,
+    int32_t* prompt_tokens_out,
+    char** error_out
+);
+
+void sgl_preprocessed_request_free(
+    char* prompt_text,
+    uint32_t* token_ids,
+    size_t token_ids_len,
+    char* tool_constraints_json
+);
+
+// Memory management
+void sgl_free_string(char* s);
+void sgl_free_token_ids(uint32_t* ptr, size_t count);
+*/
+import "C"
+
+import (
+	"fmt"
+	"unsafe"
+)
+
+// PreprocessedRequest represents a preprocessed chat request
+type PreprocessedRequest struct {
+	PromptText          string
+	TokenIDs            []uint32
+	ToolConstraintsJSON string
+	PromptTokens        int32
+	// Internal pointers for memory management
+	promptTextPtr          *C.char
+	tokenIDsPtr            *C.uint32_t
+	tokenIDsLen            uintptr
+	toolConstraintsJSONPtr *C.char
+}
+
+// PreprocessChatRequest preprocesses a chat completion request
+//
+// This function:
+// 1. Applies chat_template to messages
+// 2. Tokenizes the processed text
+// 3. Generates tool constraints (if tools are present)
+//
+// Returns the preprocessed request data and any error.
+func PreprocessChatRequest(requestJSON, tokenizerPath string) (*PreprocessedRequest, error) {
+	requestJSONC := C.CString(requestJSON)
+	defer C.free(unsafe.Pointer(requestJSONC))
+
+	tokenizerPathC := C.CString(tokenizerPath)
+	defer C.free(unsafe.Pointer(tokenizerPathC))
+
+	var promptTextOut *C.char
+	var tokenIDsOut *C.uint32_t
+	var tokenIDsLenOut C.size_t
+	var toolConstraintsJSONOut *C.char
+	var promptTokensOut C.int32_t
+	var errorOut *C.char
+
+	errorCode := C.sgl_preprocess_chat_request(
+		requestJSONC,
+		tokenizerPathC,
+		&promptTextOut,
+		&tokenIDsOut,
+		&tokenIDsLenOut,
+		&toolConstraintsJSONOut,
+		&promptTokensOut,
+		&errorOut,
+	)
+
+	if errorCode != C.SGL_ERROR_SUCCESS {
+		errorMsg := ""
+		if errorOut != nil {
+			errorMsg = C.GoString(errorOut)
+			C.sgl_free_string(errorOut)
+		}
+		return nil, fmt.Errorf("preprocessing failed: %s", errorMsg)
+	}
+
+	result := &PreprocessedRequest{
+		PromptText:          C.GoString(promptTextOut),
+		TokenIDs:            make([]uint32, tokenIDsLenOut),
+		ToolConstraintsJSON: "",
+		PromptTokens:        int32(promptTokensOut),
+	}
+
+	// Copy token IDs
+	if tokenIDsOut != nil && tokenIDsLenOut > 0 {
+		tokenIDsSlice := (*[1 << 30]C.uint32_t)(unsafe.Pointer(tokenIDsOut))[:tokenIDsLenOut:tokenIDsLenOut]
+		for i := range result.TokenIDs {
+			result.TokenIDs[i] = uint32(tokenIDsSlice[i])
+		}
+	}
+
+	// Copy tool constraints JSON if present
+	if toolConstraintsJSONOut != nil {
+		result.ToolConstraintsJSON = C.GoString(toolConstraintsJSONOut)
+	}
+
+	// Store pointers for later cleanup
+	result.promptTextPtr = promptTextOut
+	result.tokenIDsPtr = tokenIDsOut
+	result.tokenIDsLen = uintptr(tokenIDsLenOut)
+	result.toolConstraintsJSONPtr = toolConstraintsJSONOut
+
+	return result, nil
+}
+
+// PreprocessChatRequestWithTokenizer preprocesses a chat completion request using an existing tokenizer handle
+//
+// This function is similar to PreprocessChatRequest, but accepts a TokenizerHandle
+// instead of creating a new tokenizer. This allows reusing a cached tokenizer instance,
+// significantly reducing initialization overhead in concurrent scenarios.
+//
+// Returns the preprocessed request data and any error.
+func PreprocessChatRequestWithTokenizer(requestJSON string, tokenizerHandle *TokenizerHandle) (*PreprocessedRequest, error) {
+	requestJSONC := C.CString(requestJSON)
+	defer C.free(unsafe.Pointer(requestJSONC))
+
+	if tokenizerHandle == nil || tokenizerHandle.handle == nil {
+		return nil, fmt.Errorf("invalid tokenizer handle")
+	}
+
+	var promptTextOut *C.char
+	var tokenIDsOut *C.uint32_t
+	var tokenIDsLenOut C.size_t
+	var toolConstraintsJSONOut *C.char
+	var promptTokensOut C.int32_t
+	var errorOut *C.char
+
+	errorCode := C.sgl_preprocess_chat_request_with_tokenizer(
+		requestJSONC,
+		unsafe.Pointer(tokenizerHandle.handle), // Convert *C.TokenizerHandle to void*
+		&promptTextOut,
+		&tokenIDsOut,
+		&tokenIDsLenOut,
+		&toolConstraintsJSONOut,
+		&promptTokensOut,
+		&errorOut,
+	)
+
+	if errorCode != C.SGL_ERROR_SUCCESS {
+		errorMsg := ""
+		if errorOut != nil {
+			errorMsg = C.GoString(errorOut)
+			C.sgl_free_string(errorOut)
+		}
+		return nil, fmt.Errorf("preprocessing failed: %s", errorMsg)
+	}
+
+	result := &PreprocessedRequest{
+		PromptText:          C.GoString(promptTextOut),
+		TokenIDs:            make([]uint32, tokenIDsLenOut),
+		ToolConstraintsJSON: "",
+		PromptTokens:        int32(promptTokensOut),
+	}
+
+	// Copy token IDs
+	if tokenIDsOut != nil && tokenIDsLenOut > 0 {
+		tokenIDsSlice := (*[1 << 30]C.uint32_t)(unsafe.Pointer(tokenIDsOut))[:tokenIDsLenOut:tokenIDsLenOut]
+		for i := range result.TokenIDs {
+			result.TokenIDs[i] = uint32(tokenIDsSlice[i])
+		}
+	}
+
+	// Copy tool constraints JSON if present
+	if toolConstraintsJSONOut != nil {
+		result.ToolConstraintsJSON = C.GoString(toolConstraintsJSONOut)
+	}
+
+	// Store pointers for later cleanup
+	result.promptTextPtr = promptTextOut
+	result.tokenIDsPtr = tokenIDsOut
+	result.tokenIDsLen = uintptr(tokenIDsLenOut)
+	result.toolConstraintsJSONPtr = toolConstraintsJSONOut
+
+	return result, nil
+}
+
+// Free frees the memory allocated for a preprocessed request
+func (p *PreprocessedRequest) Free() {
+	if p.promptTextPtr != nil || p.tokenIDsPtr != nil || p.toolConstraintsJSONPtr != nil {
+		C.sgl_preprocessed_request_free(
+			p.promptTextPtr,
+			p.tokenIDsPtr,
+			C.size_t(p.tokenIDsLen),
+			p.toolConstraintsJSONPtr,
+		)
+		// Clear pointers to prevent double-free
+		p.promptTextPtr = nil
+		p.tokenIDsPtr = nil
+		p.tokenIDsLen = 0
+		p.toolConstraintsJSONPtr = nil
+	}
+}
+
+// FreePreprocessedRequest frees the memory allocated for a preprocessed request
+// This is a convenience function for direct pointer management
+func FreePreprocessedRequest(promptTextPtr *C.char, tokenIDsPtr *C.uint32_t, tokenIDsLen uintptr, toolConstraintsJSONPtr *C.char) {
+	if promptTextPtr != nil || tokenIDsPtr != nil || toolConstraintsJSONPtr != nil {
+		C.sgl_preprocessed_request_free(
+			promptTextPtr,
+			tokenIDsPtr,
+			C.size_t(tokenIDsLen),
+			toolConstraintsJSONPtr,
+		)
+	}
+}

third_party/sglang/sgl-model-gateway/bindings/golang/internal/grpc/client_grpc.go

@@ -0,0 +1,684 @@
+// Package grpc provides gRPC client implementation for SGLang
+package grpc
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"strconv"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/grpc/keepalive"
+	"google.golang.org/protobuf/types/known/timestamppb"
+
+	"github.com/sglang/sglang-go-grpc-sdk/internal/ffi"
+	"github.com/sglang/sglang-go-grpc-sdk/internal/proto"
+)
+
+type grpcClientStream interface {
+	Recv() (*proto.GenerateResponse, error)
+	CloseSend() error
+}
+
+// recvResult holds the result of a Recv() call
+type recvResult struct {
+	resp *proto.GenerateResponse
+	err  error
+}
+
+type GrpcClient struct {
+	conn            *grpc.ClientConn
+	client          proto.SglangSchedulerClient
+	tokenizerPath   string
+	tokenizerHandle *ffi.TokenizerHandle
+	bufferSizes     ChannelBufferSizes
+	timeouts        Timeouts
+	requestCounter  uint64 // Atomic counter to ensure unique request IDs
+}
+
+type ChannelBufferSizes struct {
+	ResultJSONChan int
+	ErrChan        int
+	RecvChan       int
+}
+
+type Timeouts struct {
+	KeepaliveTime    time.Duration
+	KeepaliveTimeout time.Duration
+	CloseTimeout     time.Duration
+}
+
+func NewGrpcClient(endpoint, tokenizerPath string, bufferSizes ChannelBufferSizes, timeouts Timeouts) (*GrpcClient, error) {
+	endpoint = strings.TrimPrefix(endpoint, "grpc://")
+	if !strings.Contains(endpoint, ":") {
+		return nil, fmt.Errorf("invalid endpoint format: %s (expected grpc://host:port)", endpoint)
+	}
+
+	keepaliveParams := keepalive.ClientParameters{
+		Time:                timeouts.KeepaliveTime,
+		Timeout:             timeouts.KeepaliveTimeout,
+		PermitWithoutStream: false,
+	}
+
+	opts := []grpc.DialOption{
+		grpc.WithTransportCredentials(insecure.NewCredentials()),
+		grpc.WithKeepaliveParams(keepaliveParams),
+	}
+
+	conn, err := grpc.NewClient(endpoint, opts...)
+	if err != nil {
+		return nil, fmt.Errorf("failed to connect to gRPC server: %w", err)
+	}
+
+	client := proto.NewSglangSchedulerClient(conn)
+
+	tokenizerHandle, err := ffi.CreateTokenizerHandle(tokenizerPath)
+	if err != nil {
+		conn.Close()
+		return nil, fmt.Errorf("failed to create tokenizer handle: %w", err)
+	}
+
+	return &GrpcClient{
+		conn:            conn,
+		client:          client,
+		tokenizerPath:   tokenizerPath,
+		tokenizerHandle: tokenizerHandle,
+		bufferSizes:     bufferSizes,
+		timeouts:        timeouts,
+	}, nil
+}
+
+func (c *GrpcClient) Close() error {
+	if c.tokenizerHandle != nil {
+		ffi.FreeTokenizerHandle(c.tokenizerHandle)
+		c.tokenizerHandle = nil
+	}
+
+	if c.conn != nil {
+		return c.conn.Close()
+	}
+	return nil
+}
+
+func (c *GrpcClient) CreateChatCompletionStream(ctx context.Context, reqJSON string) (*GrpcChatCompletionStream, error) {
+	if c.tokenizerHandle == nil {
+		return nil, fmt.Errorf("tokenizer handle is nil (should be created at startup)")
+	}
+
+	preprocessed, err := ffi.PreprocessChatRequestWithTokenizer(reqJSON, c.tokenizerHandle)
+	if err != nil {
+		return nil, fmt.Errorf("preprocessing failed: %w", err)
+	}
+	defer func() {
+		if preprocessed != nil {
+			preprocessed.Free()
+		}
+	}()
+
+	// Parse request JSON to get parameters
+	var reqMap map[string]interface{}
+	if err := json.Unmarshal([]byte(reqJSON), &reqMap); err != nil {
+		return nil, fmt.Errorf("failed to parse request JSON: %w", err)
+	}
+
+	model, _ := reqMap["model"].(string)
+	if model == "" {
+		model = "default"
+	}
+
+	// Build GenerateRequest
+	// Generate unique request ID using timestamp + atomic counter to avoid collisions
+	// This matches Rust version's UUID-based approach for uniqueness
+	counter := atomic.AddUint64(&c.requestCounter, 1)
+	requestID := fmt.Sprintf("chatcmpl-%d-%d", time.Now().UnixNano(), counter)
+	generateReq := &proto.GenerateRequest{
+		RequestId: requestID,
+		Tokenized: &proto.TokenizedInput{
+			OriginalText: preprocessed.PromptText,
+			InputIds:     preprocessed.TokenIDs,
+		},
+		Stream: true,
+	}
+
+	// Set sampling parameters
+	samplingParams := &proto.SamplingParams{
+		Temperature:       1.0,
+		TopP:              1.0,
+		TopK:              -1,
+		SkipSpecialTokens: true,
+	}
+
+	if temp, ok := reqMap["temperature"].(float64); ok {
+		samplingParams.Temperature = float32(temp)
+	}
+	if topP, ok := reqMap["top_p"].(float64); ok {
+		samplingParams.TopP = float32(topP)
+	}
+	if topK, ok := reqMap["top_k"].(float64); ok {
+		samplingParams.TopK = int32(topK)
+	}
+	var maxTokensInt *int32
+	if maxCompletionTokens, ok := reqMap["max_completion_tokens"].(float64); ok {
+		tokens := int32(maxCompletionTokens)
+		maxTokensInt = &tokens
+	} else if maxTokens, ok := reqMap["max_tokens"].(float64); ok {
+		tokens := int32(maxTokens)
+		maxTokensInt = &tokens
+	}
+	if maxTokensInt != nil {
+		samplingParams.MaxNewTokens = maxTokensInt
+	}
+
+	// Parse tool constraints if available
+	if preprocessed.ToolConstraintsJSON != "" {
+		var toolConstraints map[string]interface{}
+		if err := json.Unmarshal([]byte(preprocessed.ToolConstraintsJSON), &toolConstraints); err == nil {
+			if regex, ok := toolConstraints["regex"].(string); ok {
+				samplingParams.Constraint = &proto.SamplingParams_Regex{Regex: regex}
+			} else if jsonSchema, ok := toolConstraints["json_schema"].(string); ok {
+				samplingParams.Constraint = &proto.SamplingParams_JsonSchema{JsonSchema: jsonSchema}
+			}
+		}
+	}
+
+	generateReq.SamplingParams = samplingParams
+	generateReq.Timestamp = timestamppb.Now()
+
+	stream, err := c.client.Generate(ctx, generateReq)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create gRPC stream: %w", err)
+	}
+	toolsJSON := ""
+	if tools, ok := reqMap["tools"].([]interface{}); ok && len(tools) > 0 {
+		toolsBytes, _ := json.Marshal(tools)
+		toolsJSON = string(toolsBytes)
+	}
+
+	toolChoiceJSON := ""
+	if toolChoice, ok := reqMap["tool_choice"]; ok {
+		toolChoiceBytes, _ := json.Marshal(toolChoice)
+		toolChoiceJSON = string(toolChoiceBytes)
+	}
+
+	stopJSON := ""
+	if stop, ok := reqMap["stop"]; ok {
+		stopBytes, _ := json.Marshal(stop)
+		stopJSON = string(stopBytes)
+	}
+
+	stopTokenIDs := []uint32{}
+	if stopTokenIDsVal, ok := reqMap["stop_token_ids"].([]interface{}); ok {
+		for _, id := range stopTokenIDsVal {
+			if idFloat, ok := id.(float64); ok {
+				stopTokenIDs = append(stopTokenIDs, uint32(idFloat))
+			}
+		}
+	}
+
+	skipSpecialTokens := true
+	if skipSpecialTokensVal, ok := reqMap["skip_special_tokens"].(bool); ok {
+		skipSpecialTokens = skipSpecialTokensVal
+	}
+
+	if c.tokenizerHandle == nil {
+		stream.CloseSend()
+		return nil, fmt.Errorf("tokenizer handle is nil (should be created at startup)")
+	}
+
+	converterHandle, err := ffi.CreateGrpcResponseConverterWithTokenizer(
+		c.tokenizerHandle,
+		model,
+		generateReq.RequestId,
+		toolsJSON,
+		toolChoiceJSON,
+		stopJSON,
+		stopTokenIDs,
+		skipSpecialTokens,
+		preprocessed.PromptTokens, // Pass initial prompt tokens from preprocessing
+	)
+	if err != nil {
+		stream.CloseSend()
+		return nil, fmt.Errorf("failed to create converter handle: %w", err)
+	}
+
+	batchSize := 1
+	batchPostprocessor := ffi.NewBatchPostprocessor(converterHandle, batchSize, 0)
+
+	streamCtx, cancel := context.WithCancel(ctx)
+	grpcStream := &GrpcChatCompletionStream{
+		stream:             stream,
+		converterHandle:    converterHandle,
+		batchPostprocessor: batchPostprocessor,
+		batchSize:          batchSize,
+		ctx:                streamCtx,
+		cancel:             cancel,
+		resultJSONChan:     make(chan string, c.bufferSizes.ResultJSONChan),
+		errChan:            make(chan error, c.bufferSizes.ErrChan),
+		readLoopDone:       make(chan struct{}),
+		requestID:          generateReq.RequestId,
+		model:              model,
+		processWg:          sync.WaitGroup{},
+		closeTimeout:       c.timeouts.CloseTimeout,
+		bufferSizes:        c.bufferSizes,
+	}
+
+	go grpcStream.readLoop()
+
+	return grpcStream, nil
+}
+
+// GrpcChatCompletionStream represents a streaming chat completion via gRPC
+type GrpcChatCompletionStream struct {
+	stream             grpcClientStream
+	converterHandle    *ffi.GrpcResponseConverterHandle
+	batchPostprocessor *ffi.BatchPostprocessor
+	batchSize          int
+	ctx                context.Context
+	cancel             context.CancelFunc
+	closed             int32
+	resultJSONChan     chan string
+	errChan            chan error
+	readLoopDone       chan struct{}
+	requestID          string
+	model              string
+	processWg          sync.WaitGroup
+	closeTimeout       time.Duration
+	bufferSizes        ChannelBufferSizes
+	clientDisconnected int32 // Atomic flag: 1 if client disconnected, 0 otherwise
+}
+
+func (s *GrpcChatCompletionStream) readLoop() {
+	defer func() {
+		atomic.StoreInt32(&s.closed, 1)
+		s.processWg.Wait()
+		close(s.resultJSONChan)
+		close(s.errChan)
+		close(s.readLoopDone)
+		// Cancel context after channels are closed to ensure errors are read first
+		if s.cancel != nil {
+			s.cancel()
+		}
+	}()
+
+	recvChan := make(chan recvResult, s.bufferSizes.RecvChan)
+	const firstRecvTimeout = 60 * time.Second
+
+	go func() {
+		defer close(recvChan)
+		recvCount := 0
+		for {
+			select {
+			case <-s.ctx.Done():
+				// Skip CloseSend() if client disconnected
+				if atomic.LoadInt32(&s.clientDisconnected) == 0 {
+					_ = s.stream.CloseSend()
+				}
+				return
+			default:
+			}
+
+			recvCount++
+			var protoResp *proto.GenerateResponse
+			var err error
+
+			// First Recv() with timeout
+			if recvCount == 1 {
+				recvDone := make(chan recvResult, 1)
+				go func() {
+					resp, recvErr := s.stream.Recv()
+					recvDone <- recvResult{resp: resp, err: recvErr}
+				}()
+
+				select {
+				case result := <-recvDone:
+					protoResp = result.resp
+					err = result.err
+				case <-time.After(firstRecvTimeout):
+					timeoutErr := fmt.Errorf("stream.Recv() timeout after %v: backend may not be responding (request_id=%s)", firstRecvTimeout, s.requestID)
+					select {
+					case recvChan <- recvResult{resp: nil, err: timeoutErr}:
+					case <-s.ctx.Done():
+					}
+					return
+				case <-s.ctx.Done():
+					return
+				}
+			} else {
+				// Normal Recv()
+				protoResp, err = s.stream.Recv()
+			}
+
+			if err != nil {
+				select {
+				case recvChan <- recvResult{resp: nil, err: err}:
+				case <-s.ctx.Done():
+					return
+				}
+				return
+			}
+
+			select {
+			case <-s.ctx.Done():
+				// Skip CloseSend() if client disconnected
+				if atomic.LoadInt32(&s.clientDisconnected) == 0 {
+					_ = s.stream.CloseSend()
+				}
+				return
+			case recvChan <- recvResult{resp: protoResp, err: nil}:
+			}
+		}
+	}()
+
+	for {
+		select {
+		case <-s.ctx.Done():
+			// Skip CloseSend() if client disconnected
+			if atomic.LoadInt32(&s.clientDisconnected) == 0 {
+				_ = s.stream.CloseSend()
+			}
+			return
+		case result, ok := <-recvChan:
+			if !ok {
+				return
+			}
+			if result.err != nil {
+				if result.err == io.EOF {
+					results, flushErr := s.flushBatch()
+					if flushErr != nil {
+						select {
+						case s.errChan <- fmt.Errorf("failed to flush batch: %w", flushErr):
+						case <-s.ctx.Done():
+						}
+						return
+					}
+					for _, resultJSON := range results {
+						select {
+						case s.resultJSONChan <- resultJSON:
+						case <-s.ctx.Done():
+							return
+						}
+					}
+					return
+				}
+				select {
+				case s.errChan <- result.err:
+				case <-s.ctx.Done():
+				}
+				return
+			}
+
+			if result.resp != nil {
+				s.processWg.Add(1)
+				go func(resp *proto.GenerateResponse) {
+					defer s.processWg.Done()
+					s.processAndSendResponse(resp)
+				}(result.resp)
+			}
+		}
+	}
+}
+
+func (s *GrpcChatCompletionStream) processAndSendResponse(protoResp *proto.GenerateResponse) {
+	select {
+	case <-s.ctx.Done():
+		return
+	default:
+	}
+
+	if protoResp == nil {
+		return
+	}
+
+	protoJSON, err := protoToJSON(protoResp)
+	if err != nil {
+		select {
+		case s.errChan <- fmt.Errorf("failed to convert proto to JSON: %w", err):
+		case <-s.ctx.Done():
+		}
+		return
+	}
+
+	if s.batchPostprocessor == nil {
+		select {
+		case s.errChan <- fmt.Errorf("batch postprocessor is nil"):
+		case <-s.ctx.Done():
+		}
+		return
+	}
+
+	results, _, err := s.batchPostprocessor.AddChunk(protoJSON)
+	if err != nil {
+		select {
+		case s.errChan <- fmt.Errorf("batch postprocessing failed: %w", err):
+		case <-s.ctx.Done():
+		}
+		return
+	}
+
+	for _, resultJSON := range results {
+		select {
+		case s.resultJSONChan <- resultJSON:
+		case <-s.ctx.Done():
+			return
+		}
+	}
+}
+
+func (s *GrpcChatCompletionStream) RecvJSON() (string, error) {
+	// Use a loop instead of recursion to avoid stack overflow if there are many empty strings
+	for {
+		// Check errChan first to prioritize actual errors over context cancellation
+		select {
+		case err, ok := <-s.errChan:
+			if !ok {
+				return "", io.EOF
+			}
+			return "", err
+		default:
+		}
+
+		select {
+		case resultJSON, ok := <-s.resultJSONChan:
+			if !ok {
+				return "", io.EOF
+			}
+			// Skip empty strings and continue loop instead of recursing
+			if resultJSON != "" {
+				return resultJSON, nil
+			}
+			// Empty string, continue loop to get next result
+			continue
+		case err, ok := <-s.errChan:
+			if !ok {
+				return "", io.EOF
+			}
+			return "", err
+		case <-s.ctx.Done():
+			return "", s.ctx.Err()
+		}
+	}
+}
+
+// SetClientDisconnected marks that the client has disconnected.
+// When Close() is called, it will not call CloseSend() to avoid aborting the request on server side.
+func (s *GrpcChatCompletionStream) SetClientDisconnected() {
+	atomic.StoreInt32(&s.clientDisconnected, 1)
+}
+
+func (s *GrpcChatCompletionStream) Close() error {
+	if !atomic.CompareAndSwapInt32(&s.closed, 0, 1) {
+		return nil
+	}
+
+	if s.cancel != nil {
+		s.cancel()
+	}
+
+	clientDisconnected := atomic.LoadInt32(&s.clientDisconnected) == 1
+
+	select {
+	case <-s.readLoopDone:
+		// readLoop completed
+	default:
+		if !clientDisconnected {
+			// Call CloseSend() if client didn't disconnect
+			_ = s.stream.CloseSend()
+		}
+		select {
+		case <-s.readLoopDone:
+		case <-time.After(s.closeTimeout):
+		}
+	}
+
+	_, _ = s.flushBatch()
+
+	if s.converterHandle != nil {
+		ffi.FreeGrpcResponseConverter(s.converterHandle)
+	}
+
+	return nil
+}
+
+func (s *GrpcChatCompletionStream) flushBatch() ([]string, error) {
+	if s.batchPostprocessor != nil {
+		results, err := s.batchPostprocessor.Flush()
+		if err != nil {
+			return nil, fmt.Errorf("batch flush failed: %w", err)
+		}
+		return results, nil
+	}
+	return nil, nil
+}
+
+func protoToJSON(resp *proto.GenerateResponse) (string, error) {
+	var sb strings.Builder
+	sb.Grow(500)
+
+	sb.WriteString(`{"request_id":`)
+	if resp.RequestId == "" {
+		sb.WriteString(`""`)
+	} else {
+		requestIDJSON, err := json.Marshal(resp.RequestId)
+		if err != nil {
+			return "", err
+		}
+		sb.Write(requestIDJSON)
+	}
+
+	switch r := resp.Response.(type) {
+	case *proto.GenerateResponse_Chunk:
+		sb.WriteString(`,"chunk":{`)
+		sb.WriteString(`"token_ids":`)
+		tokenIDsJSON, err := json.Marshal(r.Chunk.TokenIds)
+		if err != nil {
+			return "", err
+		}
+		sb.Write(tokenIDsJSON)
+		sb.WriteString(`,"prompt_tokens":`)
+		sb.WriteString(strconv.FormatInt(int64(r.Chunk.PromptTokens), 10))
+		sb.WriteString(`,"completion_tokens":`)
+		sb.WriteString(strconv.FormatInt(int64(r.Chunk.CompletionTokens), 10))
+		sb.WriteString(`,"cached_tokens":`)
+		sb.WriteString(strconv.FormatInt(int64(r.Chunk.CachedTokens), 10))
+		sb.WriteString(`,"index":`)
+		sb.WriteString(strconv.FormatInt(int64(r.Chunk.Index), 10))
+		sb.WriteString(`}`)
+	case *proto.GenerateResponse_Complete:
+		sb.WriteString(`,"complete":{`)
+		sb.WriteString(`"output_ids":`)
+		outputIDsJSON, err := json.Marshal(r.Complete.OutputIds)
+		if err != nil {
+			return "", err
+		}
+		sb.Write(outputIDsJSON)
+		sb.WriteString(`,"finish_reason":`)
+		finishReasonJSON, err := json.Marshal(r.Complete.FinishReason)
+		if err != nil {
+			return "", err
+		}
+		sb.Write(finishReasonJSON)
+		sb.WriteString(`,"prompt_tokens":`)
+		sb.WriteString(strconv.FormatInt(int64(r.Complete.PromptTokens), 10))
+		sb.WriteString(`,"completion_tokens":`)
+		sb.WriteString(strconv.FormatInt(int64(r.Complete.CompletionTokens), 10))
+		sb.WriteString(`,"cached_tokens":`)
+		sb.WriteString(strconv.FormatInt(int64(r.Complete.CachedTokens), 10))
+		sb.WriteString(`}`)
+	case *proto.GenerateResponse_Error:
+		sb.WriteString(`,"error":{`)
+		sb.WriteString(`"message":`)
+		messageJSON, err := json.Marshal(r.Error.Message)
+		if err != nil {
+			return "", err
+		}
+		sb.Write(messageJSON)
+		sb.WriteString(`,"http_status_code":`)
+		httpStatusCodeJSON, err := json.Marshal(r.Error.HttpStatusCode)
+		if err != nil {
+			return "", err
+		}
+		sb.Write(httpStatusCodeJSON)
+		if r.Error.Details != "" {
+			sb.WriteString(`,"details":`)
+			detailsJSON, err := json.Marshal(r.Error.Details)
+			if err != nil {
+				return "", err
+			}
+			sb.Write(detailsJSON)
+		}
+		sb.WriteString(`}`)
+	}
+
+	sb.WriteString(`}`)
+	return sb.String(), nil
+}
+
+type ChatCompletionStreamResponse struct {
+	ID                string         `json:"id"`
+	Object            string         `json:"object"`
+	Created           int64          `json:"created"`
+	Model             string         `json:"model"`
+	SystemFingerprint string         `json:"system_fingerprint,omitempty"`
+	Choices           []StreamChoice `json:"choices"`
+	Usage             *Usage         `json:"usage,omitempty"`
+}
+
+// StreamChoice represents a choice in a streaming response
+type StreamChoice struct {
+	Index        int          `json:"index"`
+	Delta        MessageDelta `json:"delta"`
+	FinishReason string       `json:"finish_reason,omitempty"`
+}
+
+// MessageDelta represents incremental message updates
+type MessageDelta struct {
+	Role      string     `json:"role,omitempty"`
+	Content   string     `json:"content,omitempty"`
+	ToolCalls []ToolCall `json:"tool_calls,omitempty"`
+}
+
+// ToolCall represents a tool call in the response
+type ToolCall struct {
+	ID       string       `json:"id"`
+	Type     string       `json:"type"`
+	Function FunctionCall `json:"function"`
+}
+
+// FunctionCall represents a function call
+type FunctionCall struct {
+	Name      string `json:"name"`
+	Arguments string `json:"arguments"`
+}
+
+// Usage represents token usage information
+type Usage struct {
+	PromptTokens     int `json:"prompt_tokens"`
+	CompletionTokens int `json:"completion_tokens"`
+	TotalTokens      int `json:"total_tokens"`
+}

third_party/sglang/sgl-model-gateway/bindings/golang/internal/proto/sglang_scheduler_grpc.pb.go

@@ -0,0 +1,333 @@
+// Code generated by protoc-gen-go-grpc. DO NOT EDIT.
+// versions:
+// - protoc-gen-go-grpc v1.5.1
+// - protoc             v3.21.12
+// source: sglang_scheduler.proto
+
+package proto
+
+import (
+	context "context"
+	grpc "google.golang.org/grpc"
+	codes "google.golang.org/grpc/codes"
+	status "google.golang.org/grpc/status"
+)
+
+// This is a compile-time assertion to ensure that this generated file
+// is compatible with the grpc package it is being compiled against.
+// Requires gRPC-Go v1.64.0 or later.
+const _ = grpc.SupportPackageIsVersion9
+
+const (
+	SglangScheduler_Generate_FullMethodName      = "/sglang.grpc.scheduler.SglangScheduler/Generate"
+	SglangScheduler_Embed_FullMethodName         = "/sglang.grpc.scheduler.SglangScheduler/Embed"
+	SglangScheduler_HealthCheck_FullMethodName   = "/sglang.grpc.scheduler.SglangScheduler/HealthCheck"
+	SglangScheduler_Abort_FullMethodName         = "/sglang.grpc.scheduler.SglangScheduler/Abort"
+	SglangScheduler_GetModelInfo_FullMethodName  = "/sglang.grpc.scheduler.SglangScheduler/GetModelInfo"
+	SglangScheduler_GetServerInfo_FullMethodName = "/sglang.grpc.scheduler.SglangScheduler/GetServerInfo"
+)
+
+// SglangSchedulerClient is the client API for SglangScheduler service.
+//
+// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream.
+//
+// Service definition for SGLang scheduler communication
+// This protocol bridges the Rust router and Python scheduler
+type SglangSchedulerClient interface {
+	// Submit a generation request (supports streaming)
+	Generate(ctx context.Context, in *GenerateRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[GenerateResponse], error)
+	// Submit an embedding request
+	Embed(ctx context.Context, in *EmbedRequest, opts ...grpc.CallOption) (*EmbedResponse, error)
+	// Health check and metrics
+	HealthCheck(ctx context.Context, in *HealthCheckRequest, opts ...grpc.CallOption) (*HealthCheckResponse, error)
+	// Abort a running request
+	Abort(ctx context.Context, in *AbortRequest, opts ...grpc.CallOption) (*AbortResponse, error)
+	// Get model information
+	GetModelInfo(ctx context.Context, in *GetModelInfoRequest, opts ...grpc.CallOption) (*GetModelInfoResponse, error)
+	// Get server information
+	GetServerInfo(ctx context.Context, in *GetServerInfoRequest, opts ...grpc.CallOption) (*GetServerInfoResponse, error)
+}
+
+type sglangSchedulerClient struct {
+	cc grpc.ClientConnInterface
+}
+
+func NewSglangSchedulerClient(cc grpc.ClientConnInterface) SglangSchedulerClient {
+	return &sglangSchedulerClient{cc}
+}
+
+func (c *sglangSchedulerClient) Generate(ctx context.Context, in *GenerateRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[GenerateResponse], error) {
+	cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
+	stream, err := c.cc.NewStream(ctx, &SglangScheduler_ServiceDesc.Streams[0], SglangScheduler_Generate_FullMethodName, cOpts...)
+	if err != nil {
+		return nil, err
+	}
+	x := &grpc.GenericClientStream[GenerateRequest, GenerateResponse]{ClientStream: stream}
+	if err := x.ClientStream.SendMsg(in); err != nil {
+		return nil, err
+	}
+	if err := x.ClientStream.CloseSend(); err != nil {
+		return nil, err
+	}
+	return x, nil
+}
+
+// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name.
+type SglangScheduler_GenerateClient = grpc.ServerStreamingClient[GenerateResponse]
+
+func (c *sglangSchedulerClient) Embed(ctx context.Context, in *EmbedRequest, opts ...grpc.CallOption) (*EmbedResponse, error) {
+	cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
+	out := new(EmbedResponse)
+	err := c.cc.Invoke(ctx, SglangScheduler_Embed_FullMethodName, in, out, cOpts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *sglangSchedulerClient) HealthCheck(ctx context.Context, in *HealthCheckRequest, opts ...grpc.CallOption) (*HealthCheckResponse, error) {
+	cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
+	out := new(HealthCheckResponse)
+	err := c.cc.Invoke(ctx, SglangScheduler_HealthCheck_FullMethodName, in, out, cOpts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *sglangSchedulerClient) Abort(ctx context.Context, in *AbortRequest, opts ...grpc.CallOption) (*AbortResponse, error) {
+	cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
+	out := new(AbortResponse)
+	err := c.cc.Invoke(ctx, SglangScheduler_Abort_FullMethodName, in, out, cOpts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *sglangSchedulerClient) GetModelInfo(ctx context.Context, in *GetModelInfoRequest, opts ...grpc.CallOption) (*GetModelInfoResponse, error) {
+	cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
+	out := new(GetModelInfoResponse)
+	err := c.cc.Invoke(ctx, SglangScheduler_GetModelInfo_FullMethodName, in, out, cOpts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *sglangSchedulerClient) GetServerInfo(ctx context.Context, in *GetServerInfoRequest, opts ...grpc.CallOption) (*GetServerInfoResponse, error) {
+	cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
+	out := new(GetServerInfoResponse)
+	err := c.cc.Invoke(ctx, SglangScheduler_GetServerInfo_FullMethodName, in, out, cOpts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+// SglangSchedulerServer is the server API for SglangScheduler service.
+// All implementations must embed UnimplementedSglangSchedulerServer
+// for forward compatibility.
+//
+// Service definition for SGLang scheduler communication
+// This protocol bridges the Rust router and Python scheduler
+type SglangSchedulerServer interface {
+	// Submit a generation request (supports streaming)
+	Generate(*GenerateRequest, grpc.ServerStreamingServer[GenerateResponse]) error
+	// Submit an embedding request
+	Embed(context.Context, *EmbedRequest) (*EmbedResponse, error)
+	// Health check and metrics
+	HealthCheck(context.Context, *HealthCheckRequest) (*HealthCheckResponse, error)
+	// Abort a running request
+	Abort(context.Context, *AbortRequest) (*AbortResponse, error)
+	// Get model information
+	GetModelInfo(context.Context, *GetModelInfoRequest) (*GetModelInfoResponse, error)
+	// Get server information
+	GetServerInfo(context.Context, *GetServerInfoRequest) (*GetServerInfoResponse, error)
+	mustEmbedUnimplementedSglangSchedulerServer()
+}
+
+// UnimplementedSglangSchedulerServer must be embedded to have
+// forward compatible implementations.
+//
+// NOTE: this should be embedded by value instead of pointer to avoid a nil
+// pointer dereference when methods are called.
+type UnimplementedSglangSchedulerServer struct{}
+
+func (UnimplementedSglangSchedulerServer) Generate(*GenerateRequest, grpc.ServerStreamingServer[GenerateResponse]) error {
+	return status.Errorf(codes.Unimplemented, "method Generate not implemented")
+}
+func (UnimplementedSglangSchedulerServer) Embed(context.Context, *EmbedRequest) (*EmbedResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method Embed not implemented")
+}
+func (UnimplementedSglangSchedulerServer) HealthCheck(context.Context, *HealthCheckRequest) (*HealthCheckResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method HealthCheck not implemented")
+}
+func (UnimplementedSglangSchedulerServer) Abort(context.Context, *AbortRequest) (*AbortResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method Abort not implemented")
+}
+func (UnimplementedSglangSchedulerServer) GetModelInfo(context.Context, *GetModelInfoRequest) (*GetModelInfoResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method GetModelInfo not implemented")
+}
+func (UnimplementedSglangSchedulerServer) GetServerInfo(context.Context, *GetServerInfoRequest) (*GetServerInfoResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method GetServerInfo not implemented")
+}
+func (UnimplementedSglangSchedulerServer) mustEmbedUnimplementedSglangSchedulerServer() {}
+func (UnimplementedSglangSchedulerServer) testEmbeddedByValue()                         {}
+
+// UnsafeSglangSchedulerServer may be embedded to opt out of forward compatibility for this service.
+// Use of this interface is not recommended, as added methods to SglangSchedulerServer will
+// result in compilation errors.
+type UnsafeSglangSchedulerServer interface {
+	mustEmbedUnimplementedSglangSchedulerServer()
+}
+
+func RegisterSglangSchedulerServer(s grpc.ServiceRegistrar, srv SglangSchedulerServer) {
+	// If the following call pancis, it indicates UnimplementedSglangSchedulerServer was
+	// embedded by pointer and is nil.  This will cause panics if an
+	// unimplemented method is ever invoked, so we test this at initialization
+	// time to prevent it from happening at runtime later due to I/O.
+	if t, ok := srv.(interface{ testEmbeddedByValue() }); ok {
+		t.testEmbeddedByValue()
+	}
+	s.RegisterService(&SglangScheduler_ServiceDesc, srv)
+}
+
+func _SglangScheduler_Generate_Handler(srv interface{}, stream grpc.ServerStream) error {
+	m := new(GenerateRequest)
+	if err := stream.RecvMsg(m); err != nil {
+		return err
+	}
+	return srv.(SglangSchedulerServer).Generate(m, &grpc.GenericServerStream[GenerateRequest, GenerateResponse]{ServerStream: stream})
+}
+
+// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name.
+type SglangScheduler_GenerateServer = grpc.ServerStreamingServer[GenerateResponse]
+
+func _SglangScheduler_Embed_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(EmbedRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(SglangSchedulerServer).Embed(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: SglangScheduler_Embed_FullMethodName,
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(SglangSchedulerServer).Embed(ctx, req.(*EmbedRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+func _SglangScheduler_HealthCheck_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(HealthCheckRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(SglangSchedulerServer).HealthCheck(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: SglangScheduler_HealthCheck_FullMethodName,
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(SglangSchedulerServer).HealthCheck(ctx, req.(*HealthCheckRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+func _SglangScheduler_Abort_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(AbortRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(SglangSchedulerServer).Abort(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: SglangScheduler_Abort_FullMethodName,
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(SglangSchedulerServer).Abort(ctx, req.(*AbortRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+func _SglangScheduler_GetModelInfo_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(GetModelInfoRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(SglangSchedulerServer).GetModelInfo(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: SglangScheduler_GetModelInfo_FullMethodName,
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(SglangSchedulerServer).GetModelInfo(ctx, req.(*GetModelInfoRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+func _SglangScheduler_GetServerInfo_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(GetServerInfoRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(SglangSchedulerServer).GetServerInfo(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: SglangScheduler_GetServerInfo_FullMethodName,
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(SglangSchedulerServer).GetServerInfo(ctx, req.(*GetServerInfoRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+// SglangScheduler_ServiceDesc is the grpc.ServiceDesc for SglangScheduler service.
+// It's only intended for direct use with grpc.RegisterService,
+// and not to be introspected or modified (even as a copy)
+var SglangScheduler_ServiceDesc = grpc.ServiceDesc{
+	ServiceName: "sglang.grpc.scheduler.SglangScheduler",
+	HandlerType: (*SglangSchedulerServer)(nil),
+	Methods: []grpc.MethodDesc{
+		{
+			MethodName: "Embed",
+			Handler:    _SglangScheduler_Embed_Handler,
+		},
+		{
+			MethodName: "HealthCheck",
+			Handler:    _SglangScheduler_HealthCheck_Handler,
+		},
+		{
+			MethodName: "Abort",
+			Handler:    _SglangScheduler_Abort_Handler,
+		},
+		{
+			MethodName: "GetModelInfo",
+			Handler:    _SglangScheduler_GetModelInfo_Handler,
+		},
+		{
+			MethodName: "GetServerInfo",
+			Handler:    _SglangScheduler_GetServerInfo_Handler,
+		},
+	},
+	Streams: []grpc.StreamDesc{
+		{
+			StreamName:    "Generate",
+			Handler:       _SglangScheduler_Generate_Handler,
+			ServerStreams: true,
+		},
+	},
+	Metadata: "sglang_scheduler.proto",
+}